/*
 * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
 * Copyright (c) 2000-2009 by Hewlett-Packard Development Company.
 * All rights reserved.
 * Copyright (c) 2009-2018 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/* This file could be used for the following purposes:          */
/* - get the complete GC as a single link object file (module); */
/* - enable more compiler optimizations.                        */

/* Tip: to get the highest level of compiler optimizations, the typical */
/* compiler options (GCC) to use are:                                   */
/* -O3 -fno-strict-aliasing -march=native -Wall -fprofile-generate/use  */

/* Warning: GCC for Linux (for C++ clients only): Use -fexceptions both */
/* for GC and the client otherwise GC_thread_exit_proc() is not         */
/* guaranteed to be invoked (see the comments in pthread_start.c).      */

#ifndef __cplusplus
  /* static is desirable here for more efficient linkage.               */
  /* TODO: Enable this in case of the compilation as C++ code.          */
# define GC_INNER STATIC
# define GC_EXTERN GC_INNER
                /* STATIC is defined in gcconfig.h. */
#endif

/* Small files go first... */
/*
 * Copyright (c) 2001 by Hewlett-Packard Company. All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1995 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1997 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999 by Hewlett-Packard Company.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/*
 * This is mostly an internal header file.  Typical clients should
 * not use it.  Clients that define their own object kinds with
 * debugging allocators will probably want to include this, however.
 * No attempt is made to keep the namespace clean.  This should not be
 * included from header files that are frequently included by clients.
 */

#ifndef GC_DBG_MLC_H
#define GC_DBG_MLC_H

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

#ifndef GC_PRIVATE_H
#define GC_PRIVATE_H

#ifdef HAVE_CONFIG_H
/* include/config.h.  Generated from config.h.in by configure.  */
/* include/config.h.in.  Generated from configure.ac by autoheader.  */

/* Define to recognise all pointers to the interior of objects. */
#define ALL_INTERIOR_POINTERS 1

/* AO load, store and/or test-and-set primitives are implemented in
   libatomic_ops using locks. */
/* #undef BASE_ATOMIC_OPS_EMULATED */

/* Erroneously cleared dirty bits checking. Use only for debugging of the
   incremental collector. */
/* #undef CHECKSUMS */

/* Define to discover thread stack bounds on Darwin without trying to walk the
   frames on the stack. */
/* #undef DARWIN_DONT_PARSE_STACK */

/* Define to force debug headers on all objects. */
#define DBG_HDRS_ALL 1

/* Do not use user32.dll import library (Win32). */
/* #undef DONT_USE_USER32_DLL */

/* Define to support pointer mask/shift set at runtime. */
/* #undef DYNAMIC_POINTER_MASK */

/* Define to enable eCos target support. */
/* #undef ECOS */

/* Wine getenv may not return NULL for missing entry. */
/* #undef EMPTY_GETENV_RESULTS */

/* Define to enable alternative finalization interface. */
#define ENABLE_DISCLAIM 1

/* Define to enable internal debug assertions. */
/* #undef GC_ASSERTIONS */

/* Define to enable atomic uncollectible allocation. */
#define GC_ATOMIC_UNCOLLECTABLE 1

/* Use GCC atomic intrinsics instead of libatomic_ops primitives. */
#define GC_BUILTIN_ATOMIC 1

/* Define to build dynamic libraries with only API symbols exposed. */
/* #undef GC_DLL */

/* Skip the initial guess of data root sets. */
/* #undef GC_DONT_REGISTER_MAIN_STATIC_DATA */

/* Define to turn on GC_suspend_thread support (Linux only). */
#define GC_ENABLE_SUSPEND_THREAD 1

/* Define to include support for gcj. */
#define GC_GCJ_SUPPORT 1

/* Define if backtrace information is supported. */
/* #undef GC_HAVE_BUILTIN_BACKTRACE */

/* Define to use 'pthread_sigmask' function if needed. */
#define GC_HAVE_PTHREAD_SIGMASK 1

/* Enable Win32 DllMain-based approach of threads registering. */
/* #undef GC_INSIDE_DLL */

/* Missing execinfo.h header. */
/* #undef GC_MISSING_EXECINFO_H */

/* Missing sigsetjmp function. */
/* #undef GC_NO_SIGSETJMP */

/* Disable threads discovery in GC. */
/* #undef GC_NO_THREADS_DISCOVERY */

/* Read environment variables from the GC 'env' file. */
/* #undef GC_READ_ENV_FILE */

/* Define and export GC_wcsdup function. */
#define GC_REQUIRE_WCSDUP 1

/* Define to support platform-specific threads. */
#define GC_THREADS 1

/* Force the GC to use signals based on SIGRTMIN+k. */
#define GC_USESIGRT_SIGNALS 1

/* Define to cause the collector to redefine malloc and intercepted pthread
   routines with their real names while using dlsym to refer to the original
   routines. */
/* #undef GC_USE_DLOPEN_WRAP */

/* The major version number of this GC release. */
#define GC_VERSION_MAJOR 8

/* The micro version number of this GC release. */
#define GC_VERSION_MICRO 0

/* The minor version number of this GC release. */
#define GC_VERSION_MINOR 3

/* Define to support pthreads-win32 or winpthreads. */
/* #undef GC_WIN32_PTHREADS */

/* Define to install pthread_atfork() handlers by default. */
#define HANDLE_FORK 1

/* Define to use 'dladdr' function. */
#define HAVE_DLADDR 1

/* Define to 1 if you have the <dlfcn.h> header file. */
#define HAVE_DLFCN_H 1

/* Define to 1 if you have the 'dl_iterate_phdr' function. */
#define HAVE_DL_ITERATE_PHDR 1

/* Define to 1 if you have the <inttypes.h> header file. */
#define HAVE_INTTYPES_H 1

/* libatomic_ops AO_or primitive implementation is lock-free. */
/* #undef HAVE_LOCKFREE_AO_OR */

/* Define to use 'pthread_setname_np(const char*)' function. */
/* #undef HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID */

/* Define to use 'pthread_setname_np(pthread_t, const char*)' function. */
#define HAVE_PTHREAD_SETNAME_NP_WITH_TID 1

/* Define to use 'pthread_setname_np(pthread_t, const char*, void *)'
   function. */
/* #undef HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG */

/* Define to use 'pthread_set_name_np(pthread_t, const char*)' function. */
/* #undef HAVE_PTHREAD_SET_NAME_NP */

/* Define to 1 if you have the <stdint.h> header file. */
#define HAVE_STDINT_H 1

/* Define to 1 if you have the <stdio.h> header file. */
#define HAVE_STDIO_H 1

/* Define to 1 if you have the <stdlib.h> header file. */
#define HAVE_STDLIB_H 1

/* Define to 1 if you have the <strings.h> header file. */
#define HAVE_STRINGS_H 1

/* Define to 1 if you have the <string.h> header file. */
#define HAVE_STRING_H 1

/* Define to 1 if you have the <sys/stat.h> header file. */
#define HAVE_SYS_STAT_H 1

/* Define to 1 if you have the <sys/types.h> header file. */
#define HAVE_SYS_TYPES_H 1

/* Define to 1 if you have the <unistd.h> header file. */
#define HAVE_UNISTD_H 1

/* Do not define DYNAMIC_LOADING even if supported (i.e., build the collector
   with disabled tracing of dynamic library data roots). */
/* #undef IGNORE_DYNAMIC_LOADING */

/* Define to make it somewhat safer by default to finalize objects out of
   order by specifying a nonstandard finalization mark procedure. */
#define JAVA_FINALIZATION 1

/* Define to save back-pointers in debugging headers. */
#define KEEP_BACK_PTRS 1

/* Define to optimize for large heaps or root sets. */
/* #undef LARGE_CONFIG */

/* Define to the sub-directory where libtool stores uninstalled libraries. */
#define LT_OBJDIR ".libs/"

/* Define to build the collector with the support of the functionality to
   print max length of chain through unreachable objects ending in a reachable
   one. */
/* #undef MAKE_BACK_GRAPH */

/* Number of sequential garbage collections during those a candidate block for
   unmapping should be marked as free. */
#define MUNMAP_THRESHOLD 7

/* Define to not use system clock (cross compiling). */
/* #undef NO_CLOCK */

/* Disable debugging, like GC_dump and its callees. */
/* #undef NO_DEBUGGING */

/* Define to make the collector not allocate executable memory by default. */
#define NO_EXECUTE_PERMISSION 1

/* Missing getcontext function. */
/* #undef NO_GETCONTEXT */

/* Prohibit installation of pthread_atfork() handlers. */
/* #undef NO_HANDLE_FORK */

/* Name of package */
#define PACKAGE "gc"

/* Define to the address where bug reports for this package should be sent. */
#define PACKAGE_BUGREPORT "https://github.com/ivmai/bdwgc/issues"

/* Define to the full name of this package. */
#define PACKAGE_NAME "gc"

/* Define to the full name and version of this package. */
#define PACKAGE_STRING "gc 8.3.0"

/* Define to the one symbol short name of this package. */
#define PACKAGE_TARNAME "gc"

/* Define to the home page for this package. */
#define PACKAGE_URL ""

/* Define to the version of this package. */
#define PACKAGE_VERSION "8.3.0"

/* Define to enable parallel marking. */
#define PARALLEL_MARK 1

/* If defined, redirect free to this function. */
/* #undef REDIRECT_FREE */

/* If defined, redirect malloc to this function. */
/* #undef REDIRECT_MALLOC */

/* If defined, redirect GC_realloc to this function. */
/* #undef REDIRECT_REALLOC */

/* The number of caller frames saved when allocating with the debugging API.
   */
/* #undef SAVE_CALL_COUNT */

/* Shorten the headers to minimize object size at the expense of checking for
   writes past the end. */
/* #undef SHORT_DBG_HDRS */

/* Define to tune the collector for small heap sizes. */
/* #undef SMALL_CONFIG */

/* See the comment in gcconfig.h. */
/* #undef SOLARIS25_PROC_VDB_BUG_FIXED */

/* Define to 1 if all of the C89 standard headers exist (not just the ones
   required in a freestanding environment). This macro is provided for
   backward compatibility; new code need not use it. */
#define STDC_HEADERS 1

/* Define to work around a Solaris 5.3 bug (see dyn_load.c). */
/* #undef SUNOS53_SHARED_LIB */

/* Define to enable thread-local allocation optimization. */
#define THREAD_LOCAL_ALLOC 1

/* Use Unicode (W) variant of Win32 API instead of ASCII (A) one. */
/* #undef UNICODE */

/* Define to use of compiler-support for thread-local variables. */
/* #undef USE_COMPILER_TLS */

/* Define to use mmap instead of sbrk to expand the heap. */
#define USE_MMAP 1

/* Define to return memory to OS with munmap calls. */
#define USE_MUNMAP 1

/* Use rwlock for the allocator lock instead of mutex. */
#define USE_RWLOCK 1

/* Define to use Win32 VirtualAlloc (instead of sbrk or mmap) to expand the
   heap. */
/* #undef USE_WINALLOC */

/* Define to support tracking GC_malloc and friends for heap profiling tools.
   */
/* #undef VALGRIND_TRACKING */

/* Version number of package */
#define VERSION "8.3.0"

/* The POSIX feature macro. */
/* #undef _POSIX_C_SOURCE */

/* Indicates the use of pthreads (NetBSD). */
/* #undef _PTHREADS */

/* Required define if using POSIX threads. */
#define _REENTRANT 1

/* Define to '__inline__' or '__inline' if that's what the C compiler
   calls it, or to nothing if 'inline' is not supported under any name.  */
#ifndef __cplusplus
/* #undef inline */
#endif

#endif

#if !defined(GC_BUILD) && !defined(NOT_GCBUILD)
# define GC_BUILD
#endif

#if (defined(__linux__) || defined(__GLIBC__) || defined(__GNU__) \
     || defined(__CYGWIN__) || defined(HAVE_DLADDR) \
     || defined(GC_HAVE_PTHREAD_SIGMASK) \
     || defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID) \
     || defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG) \
     || defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID)) && !defined(_GNU_SOURCE)
  /* Can't test LINUX, since this must be defined before other includes. */
# define _GNU_SOURCE 1
#endif

#if defined(__INTERIX) && !defined(_ALL_SOURCE)
# define _ALL_SOURCE 1
#endif

#if (defined(DGUX) && defined(GC_THREADS) || defined(DGUX386_THREADS) \
     || defined(GC_DGUX386_THREADS)) && !defined(_USING_POSIX4A_DRAFT10)
# define _USING_POSIX4A_DRAFT10 1
#endif

#if defined(__MINGW32__) && !defined(__MINGW_EXCPT_DEFINE_PSDK) \
    && defined(__i386__) && defined(GC_EXTERN) /* defined in gc.c */
  /* See the description in mark.c.     */
# define __MINGW_EXCPT_DEFINE_PSDK 1
#endif

# if defined(NO_DEBUGGING) && !defined(GC_ASSERTIONS) && !defined(NDEBUG)
    /* To turn off assertion checking (in atomic_ops.h). */
#   define NDEBUG 1
# endif

#ifndef GC_H
# include "gc/gc.h"
#endif

#include <stdlib.h>
#if !defined(sony_news)
# include <stddef.h>
#endif

#ifdef DGUX
# include <sys/time.h>
# include <sys/resource.h>
#endif /* DGUX */

#ifdef BSD_TIME
# include <sys/time.h>
# include <sys/resource.h>
#endif /* BSD_TIME */

#ifdef PARALLEL_MARK
# define AO_REQUIRE_CAS
# if !defined(__GNUC__) && !defined(AO_ASSUME_WINDOWS98)
#   define AO_ASSUME_WINDOWS98
# endif
#endif

#include "gc/gc_tiny_fl.h"
#include "gc/gc_mark.h"

typedef GC_word word;
typedef GC_signed_word signed_word;

typedef int GC_bool;
#define TRUE 1
#define FALSE 0

#ifndef PTR_T_DEFINED
  typedef char * ptr_t; /* A generic pointer to which we can add        */
                        /* byte displacements and which can be used     */
                        /* for address comparisons.                     */
# define PTR_T_DEFINED
#endif

#ifndef SIZE_MAX
# include <limits.h>
#endif
#if defined(SIZE_MAX) && !defined(CPPCHECK)
# define GC_SIZE_MAX ((size_t)SIZE_MAX)
            /* Extra cast to workaround some buggy SIZE_MAX definitions. */
#else
# define GC_SIZE_MAX (~(size_t)0)
#endif

#if (GC_GNUC_PREREQ(3, 0) || defined(__clang__)) && !defined(LINT2)
# define EXPECT(expr, outcome) __builtin_expect(expr,outcome)
  /* Equivalent to (expr), but predict that usually (expr)==outcome. */
#else
# define EXPECT(expr, outcome) (expr)
#endif /* __GNUC__ */

/* Saturated addition of size_t values.  Used to avoid value wrap       */
/* around on overflow.  The arguments should have no side effects.      */
#define SIZET_SAT_ADD(a, b) \
            (EXPECT((a) < GC_SIZE_MAX - (b), TRUE) ? (a) + (b) : GC_SIZE_MAX)

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 2000-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2009-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/*
 * This header is private to the gc.  It is almost always included from
 * gc_priv.h.  However it is possible to include it by itself if just the
 * configuration macros are needed.  In that
 * case, a few declarations relying on types declared in gc_priv.h will be
 * omitted.
 */

#ifndef GCCONFIG_H
#define GCCONFIG_H

#ifndef GC_H
# ifdef HAVE_CONFIG_H
# endif


#endif

#ifdef CPPCHECK
# undef CLOCKS_PER_SEC
# undef FIXUP_POINTER
# undef POINTER_MASK
# undef POINTER_SHIFT
# undef REDIRECT_REALLOC
# undef _MAX_PATH
#endif

#ifndef PTR_T_DEFINED
  typedef char * ptr_t;
# define PTR_T_DEFINED
#endif

#if !defined(sony_news)
# include <stddef.h> /* For size_t, etc. */
#endif

/* Note: Only wrap our own declarations, and not the included headers.  */
/* In this case, wrap our entire file, but temporarily unwrap/rewrap    */
/* around #includes.  Types and macros do not need such wrapping, only  */
/* the declared global data and functions.                              */
#ifdef __cplusplus
# define EXTERN_C_BEGIN extern "C" {
# define EXTERN_C_END } /* extern "C" */
#else
# define EXTERN_C_BEGIN /* empty */
# define EXTERN_C_END /* empty */
#endif

EXTERN_C_BEGIN

/* Convenient internal macro to test version of Clang.  */
#if defined(__clang__) && defined(__clang_major__)
# define GC_CLANG_PREREQ(major, minor) \
    ((__clang_major__ << 8) + __clang_minor__ >= ((major) << 8) + (minor))
# define GC_CLANG_PREREQ_FULL(major, minor, patchlevel) \
            (GC_CLANG_PREREQ(major, (minor) + 1) \
                || (__clang_major__ == (major) && __clang_minor__ == (minor) \
                    && __clang_patchlevel__ >= (patchlevel)))
#else
# define GC_CLANG_PREREQ(major, minor) 0 /* FALSE */
# define GC_CLANG_PREREQ_FULL(major, minor, patchlevel) 0
#endif

/* Machine dependent parameters.  Some tuning parameters can be found   */
/* near the top of gc_priv.h.                                           */

/* Machine specific parts contributed by various people.  See README file. */

#if defined(__ANDROID__) && !defined(HOST_ANDROID)
  /* __ANDROID__ macro is defined by Android NDK gcc.   */
# define HOST_ANDROID 1
#endif

#if defined(TIZEN) && !defined(HOST_TIZEN)
# define HOST_TIZEN 1
#endif

#if defined(__SYMBIAN32__) && !defined(SYMBIAN)
# define SYMBIAN
# ifdef __WINS__
#   pragma data_seg(".data2")
# endif
#endif

/* First a unified test for Linux: */
# if (defined(linux) || defined(__linux__) || defined(HOST_ANDROID)) \
     && !defined(LINUX) && !defined(__native_client__)
#   define LINUX
# endif

/* And one for NetBSD: */
# if defined(__NetBSD__)
#   define NETBSD
# endif

/* And one for OpenBSD: */
# if defined(__OpenBSD__)
#   define OPENBSD
# endif

/* And one for FreeBSD: */
# if (defined(__FreeBSD__) || defined(__DragonFly__) \
      || defined(__FreeBSD_kernel__)) && !defined(FREEBSD) \
     && !defined(GC_NO_FREEBSD) /* Orbis compiler defines __FreeBSD__ */
#   define FREEBSD
# endif

#if defined(FREEBSD) || defined(NETBSD) || defined(OPENBSD)
# define ANY_BSD
#endif

# if defined(__EMBOX__)
#   define EMBOX
# endif

# if defined(__KOS__)
#   define KOS
# endif

# if defined(__QNX__) && !defined(QNX)
#   define QNX
# endif

/* And one for Darwin: */
# if defined(macosx) || (defined(__APPLE__) && defined(__MACH__))
#   define DARWIN
    EXTERN_C_END
#   include <TargetConditionals.h>
    EXTERN_C_BEGIN
# endif

/* Determine the machine type: */
# if defined(__native_client__)
#   define NACL
#   if !defined(__portable_native_client__) && !defined(__arm__)
#     define I386
#     define mach_type_known
#   else
      /* Here we will rely upon arch-specific defines. */
#   endif
# endif
# if defined(__aarch64__) && !defined(ANY_BSD) && !defined(DARWIN) \
     && !defined(LINUX) && !defined(KOS) && !defined(QNX) \
     && !defined(NN_BUILD_TARGET_PLATFORM_NX) && !defined(_WIN32)
#   define AARCH64
#   define NOSYS
#   define mach_type_known
# endif
# if defined(__arm) || defined(__arm__) || defined(__thumb__)
#   define ARM32
#   if defined(NACL) || defined(SYMBIAN)
#     define mach_type_known
#   elif !defined(ANY_BSD) && !defined(DARWIN) && !defined(LINUX) \
         && !defined(QNX) && !defined(NN_PLATFORM_CTR) \
         && !defined(SN_TARGET_PSP2) && !defined(_WIN32) \
         && !defined(__CEGCC__) && !defined(GC_NO_NOSYS)
#     define NOSYS
#     define mach_type_known
#   endif
# endif
# if defined(sun) && defined(mc68000) && !defined(CPPCHECK)
#   error SUNOS4 no longer supported
# endif
# if defined(hp9000s300) && !defined(CPPCHECK)
#   error M68K based HP machines no longer supported
# endif
# if defined(vax) || defined(__vax__)
#   define VAX
#   ifdef ultrix
#     define ULTRIX
#   else
#     define BSD
#   endif
#   define mach_type_known
# endif
# if defined(NETBSD) && defined(__vax__)
#   define VAX
#   define mach_type_known
# endif
# if (defined(mips) || defined(__mips) || defined(_mips)) \
     && !defined(__TANDEM) && !defined(ANY_BSD) && !defined(LINUX)
#   define MIPS
#   if defined(nec_ews) || defined(_nec_ews)
#     define EWS4800
#     define mach_type_known
#   elif defined(ultrix) || defined(__ultrix)
#     define ULTRIX
#     define mach_type_known
#   elif !defined(_WIN32_WCE) && !defined(__CEGCC__) && !defined(__MINGW32CE__)
#     define IRIX5 /* or IRIX 6.X */
#     define mach_type_known
#   endif /* !MSWINCE */
# endif
# if defined(DGUX) && (defined(i386) || defined(__i386__))
#   define I386
#   ifndef _USING_DGUX
#     define _USING_DGUX
#   endif
#   define mach_type_known
# endif
# if defined(sequent) && (defined(i386) || defined(__i386__))
#   define I386
#   define SEQUENT
#   define mach_type_known
# endif
# if (defined(sun) || defined(__sun)) && (defined(i386) || defined(__i386__))
#   define I386
#   define SOLARIS
#   define mach_type_known
# endif
# if (defined(sun) || defined(__sun)) && defined(__amd64)
#   define X86_64
#   define SOLARIS
#   define mach_type_known
# endif
# if (defined(__OS2__) || defined(__EMX__)) && defined(__32BIT__)
#   define I386
#   define OS2
#   define mach_type_known
# endif
# if defined(ibm032) && !defined(CPPCHECK)
#   error IBM PC/RT no longer supported
# endif
# if (defined(sun) || defined(__sun)) && (defined(sparc) || defined(__sparc))
            /* Test for SunOS 5.x */
    EXTERN_C_END
#   include <errno.h>
    EXTERN_C_BEGIN
#   define SPARC
#   define SOLARIS
#   define mach_type_known
# elif defined(sparc) && defined(unix) && !defined(sun) && !defined(linux) \
       && !defined(ANY_BSD)
#   define SPARC
#   define DRSNX
#   define mach_type_known
# endif
# if defined(_IBMR2)
#   define POWERPC
#   define AIX
#   define mach_type_known
# endif
# if defined(_M_XENIX) && defined(_M_SYSV) && defined(_M_I386)
        /* TODO: The above test may need refinement. */
#   define I386
#   if defined(_SCO_ELF)
#     define SCO_ELF
#   else
#     define SCO
#   endif
#   define mach_type_known
# endif
# if defined(_AUX_SOURCE) && !defined(CPPCHECK)
#   error A/UX no longer supported
# endif
# if defined(_PA_RISC1_0) || defined(_PA_RISC1_1) || defined(_PA_RISC2_0) \
     || defined(hppa) || defined(__hppa__)
#   define HP_PA
#   if !defined(LINUX) && !defined(HPUX) && !defined(OPENBSD)
#     define HPUX
#   endif
#   define mach_type_known
# endif
# if defined(__ia64) && (defined(_HPUX_SOURCE) || defined(__HP_aCC))
#   define IA64
#   ifndef HPUX
#     define HPUX
#   endif
#   define mach_type_known
# endif
# if (defined(__BEOS__) || defined(__HAIKU__)) && defined(_X86_)
#   define I386
#   define HAIKU
#   define mach_type_known
# endif
# if defined(__HAIKU__) && (defined(__amd64__) || defined(__x86_64__))
#   define X86_64
#   define HAIKU
#   define mach_type_known
# endif
# if defined(__alpha) || defined(__alpha__)
#   define ALPHA
#   if !defined(ANY_BSD) && !defined(LINUX)
#     define OSF1       /* a.k.a Digital Unix */
#   endif
#   define mach_type_known
# endif
# if defined(_AMIGA) && !defined(AMIGA)
#   define AMIGA
# endif
# ifdef AMIGA
#   define M68K
#   define mach_type_known
# endif
# if defined(THINK_C) \
     || (defined(__MWERKS__) && !defined(__powerc) && !defined(SYMBIAN))
#   define M68K
#   define MACOS
#   define mach_type_known
# endif
# if defined(__MWERKS__) && defined(__powerc) && !defined(__MACH__) \
     && !defined(SYMBIAN)
#   define POWERPC
#   define MACOS
#   define mach_type_known
# endif
# if defined(__rtems__) && (defined(i386) || defined(__i386__))
#   define I386
#   define RTEMS
#   define mach_type_known
# endif
# if defined(NeXT) && defined(mc68000)
#   define M68K
#   define NEXT
#   define mach_type_known
# endif
# if defined(NeXT) && (defined(i386) || defined(__i386__))
#   define I386
#   define NEXT
#   define mach_type_known
# endif
# if defined(bsdi) && (defined(i386) || defined(__i386__))
#   define I386
#   define BSDI
#   define mach_type_known
# endif
# if defined(__386BSD__) && !defined(mach_type_known)
#   define I386
#   define THREE86BSD
#   define mach_type_known
# endif
# if defined(_CX_UX) && defined(_M88K)
#   define M88K
#   define CX_UX
#   define mach_type_known
# endif
# if defined(DGUX) && defined(m88k)
#   define M88K
    /* DGUX defined */
#   define mach_type_known
# endif
# if defined(_WIN32_WCE) || defined(__CEGCC__) || defined(__MINGW32CE__)
    /* SH3, SH4, MIPS already defined for corresponding architectures */
#   if defined(SH3) || defined(SH4)
#     define SH
#   endif
#   if defined(x86) || defined(__i386__)
#     define I386
#   endif
#   if defined(_M_ARM) || defined(ARM) || defined(_ARM_)
#     define ARM32
#   endif
#   define MSWINCE
#   define mach_type_known
# else
#   if ((defined(_MSDOS) || defined(_MSC_VER)) && (_M_IX86 >= 300)) \
       || (defined(_WIN32) && !defined(__CYGWIN32__) && !defined(__CYGWIN__) \
           && !defined(__INTERIX) && !defined(SYMBIAN)) \
       || defined(__MINGW32__)
#     if defined(__LP64__) || defined(_M_X64)
#       define X86_64
#     elif defined(_M_ARM)
#       define ARM32
#     elif defined(_M_ARM64)
#       define AARCH64
#     else /* _M_IX86 */
#       define I386
#     endif
#     ifdef _XBOX_ONE
#       define MSWIN_XBOX1
#     else
#       ifndef MSWIN32
#         define MSWIN32 /* or Win64 */
#       endif
#       if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP)
#         define MSWINRT_FLAVOR
#       endif
#     endif
#     define mach_type_known
#   endif
#   if defined(_MSC_VER) && defined(_M_IA64)
#     define IA64
#     define MSWIN32    /* Really Win64, but we do not treat 64-bit     */
                        /* variants as a different platform.            */
#   endif
# endif /* !_WIN32_WCE && !__CEGCC__ && !__MINGW32CE__ */
# if defined(__DJGPP__)
#   define I386
#   ifndef DJGPP
#     define DJGPP  /* MSDOS running the DJGPP port of GCC */
#   endif
#   define mach_type_known
# endif
# if defined(__CYGWIN32__) || defined(__CYGWIN__)
#   if defined(__LP64__)
#     define X86_64
#   else
#     define I386
#   endif
#   define CYGWIN32
#   define mach_type_known
# endif /* __CYGWIN__ */
# if defined(__INTERIX)
#   define I386
#   define INTERIX
#   define mach_type_known
# endif
# if defined(_UTS) && !defined(mach_type_known)
#   define S370
#   define UTS4
#   define mach_type_known
# endif
# if defined(__pj__) && !defined(CPPCHECK)
#   error PicoJava no longer supported
    /* The implementation had problems, and I haven't heard of users    */
    /* in ages.  If you want it resurrected, let me know.               */
# endif
# if defined(__embedded__) && defined(PPC)
#   define POWERPC
#   define NOSYS
#   define mach_type_known
# endif
# if defined(__WATCOMC__) && defined(__386__)
#   define I386
#   if !defined(OS2) && !defined(MSWIN32) && !defined(DOS4GW)
#     if defined(__OS2__)
#       define OS2
#     elif defined(__WINDOWS_386__) || defined(__NT__)
#       define MSWIN32
#     else
#       define DOS4GW
#     endif
#   endif
#   define mach_type_known
# endif /* __WATCOMC__ && __386__ */
# if defined(__GNU__) && defined(__i386__)
    /* The Debian Hurd running on generic PC */
#   define HURD
#   define I386
#   define mach_type_known
# endif
# if defined(__GNU__) && defined(__x86_64__)
#   define HURD
#   define X86_64
#   define mach_type_known
# endif
# if defined(__TANDEM)
    /* Nonstop S-series */
    /* FIXME: Should recognize Integrity series? */
#   define MIPS
#   define NONSTOP
#   define mach_type_known
# endif
# if defined(__tile__) && defined(LINUX)
#   ifdef __tilegx__
#     define TILEGX
#   else
#     define TILEPRO
#   endif
#   define mach_type_known
# endif /* __tile__ */
# if defined(NN_BUILD_TARGET_PLATFORM_NX)
#   define AARCH64
#   define NINTENDO_SWITCH
#   define mach_type_known
# endif
# if defined(__EMSCRIPTEN__) || defined(EMSCRIPTEN)
#   define WEBASSEMBLY
#   ifndef EMSCRIPTEN
#     define EMSCRIPTEN
#   endif
#   define mach_type_known
# endif
# if defined(__wasi__)
#   define WEBASSEMBLY
#   define WASI
#   define mach_type_known
# endif

# if defined(__aarch64__) \
       && (defined(ANY_BSD) || defined(DARWIN) || defined(LINUX) \
           || defined(KOS) || defined(QNX))
#   define AARCH64
#   define mach_type_known
# elif defined(__arc__) && defined(LINUX)
#   define ARC
#   define mach_type_known
# elif (defined(__arm) || defined(__arm__) || defined(__arm32__) \
        || defined(__ARM__)) \
       && (defined(ANY_BSD) || defined(DARWIN) || defined(LINUX) \
           || defined(QNX) || defined(NN_PLATFORM_CTR) \
           || defined(SN_TARGET_PSP2))
#   define ARM32
#   define mach_type_known
# elif defined(__avr32__) && defined(LINUX)
#   define AVR32
#   define mach_type_known
# elif defined(__cris__) && defined(LINUX)
#   ifndef CRIS
#     define CRIS
#   endif
#   define mach_type_known
# elif defined(__e2k__) && defined(LINUX)
#   define E2K
#   define mach_type_known
# elif defined(__hexagon__) && defined(LINUX)
#   define HEXAGON
#   define mach_type_known
# elif (defined(__i386__) || defined(i386) || defined(__X86__)) \
       && (defined(ANY_BSD) || defined(DARWIN) || defined(EMBOX) \
           || defined(LINUX) || defined(QNX))
#   define I386
#   define mach_type_known
# elif (defined(__ia64) || defined(__ia64__)) && defined(LINUX)
#   define IA64
#   define mach_type_known
# elif defined(__loongarch__) && defined(LINUX)
#   define LOONGARCH
#   define mach_type_known
# elif defined(__m32r__) && defined(LINUX)
#   define M32R
#   define mach_type_known
# elif ((defined(__m68k__) || defined(m68k)) \
        && (defined(NETBSD) || defined(OPENBSD))) \
       || (defined(__mc68000__) && defined(LINUX))
#   define M68K
#   define mach_type_known
# elif (defined(__mips) || defined(_mips) || defined(mips)) \
       && (defined(ANY_BSD) || defined(LINUX))
#   define MIPS
#   define mach_type_known
# elif (defined(__NIOS2__) || defined(__NIOS2) || defined(__nios2__)) \
       && defined(LINUX)
#   define NIOS2 /* Altera NIOS2 */
#   define mach_type_known
# elif defined(__or1k__) && defined(LINUX)
#   define OR1K /* OpenRISC (or1k) */
#   define mach_type_known
# elif (defined(__powerpc__) || defined(__powerpc64__) || defined(__ppc__) \
            || defined(__ppc64__) || defined(powerpc) || defined(powerpc64)) \
       && (defined(ANY_BSD) || defined(DARWIN) || defined(LINUX))
#   define POWERPC
#   define mach_type_known
# elif defined(__riscv) && (defined(ANY_BSD) || defined(LINUX))
#   define RISCV
#   define mach_type_known
# elif defined(__s390__) && defined(LINUX)
#   define S390
#   define mach_type_known
# elif defined(__sh__) \
       && (defined(LINUX) || defined(NETBSD) || defined(OPENBSD))
#   define SH
#   define mach_type_known
# elif (defined(__sparc__) || defined(sparc)) \
       && (defined(ANY_BSD) || defined(LINUX))
#   define SPARC
#   define mach_type_known
# elif defined(__sw_64__) && defined(LINUX)
#   define SW_64
#   define mach_type_known
# elif (defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) \
        || defined(__X86_64__)) \
       && (defined(ANY_BSD) || defined(DARWIN) || defined(LINUX) \
           || defined(QNX))
#   define X86_64
#   define mach_type_known
# endif

/* Feel free to add more clauses here.  Or manually define the machine  */
/* type here.  A machine type is characterized by the architecture.     */
/* Some machine types are further subdivided by OS.  Macros such as     */
/* LINUX, FREEBSD, etc. distinguish them.  The distinction in these     */
/* cases is usually the stack starting address.                         */

# if !defined(mach_type_known) && !defined(CPPCHECK)
#   error The collector has not been ported to this machine/OS combination
# endif

                    /* Mapping is: M68K       ==> Motorola 680X0        */
                    /*             (NEXT, and SYSV (A/UX),              */
                    /*             MACOS and AMIGA variants)            */
                    /*             I386       ==> Intel 386             */
                    /*              (SEQUENT, OS2, SCO, LINUX, NETBSD,  */
                    /*               FREEBSD, THREE86BSD, MSWIN32,      */
                    /*               BSDI, SOLARIS, NEXT and others)    */
                    /*             NS32K      ==> Encore Multimax       */
                    /*             MIPS       ==> R2000 through R14K    */
                    /*                  (many variants)                 */
                    /*             VAX        ==> DEC VAX               */
                    /*                  (BSD, ULTRIX variants)          */
                    /*             HP_PA      ==> HP9000/700 & /800     */
                    /*                            HP/UX, LINUX          */
                    /*             SPARC      ==> SPARC v7/v8/v9        */
                    /*                 (SOLARIS, LINUX, DRSNX variants) */
                    /*             ALPHA      ==> DEC Alpha             */
                    /*                  (OSF1 and LINUX variants)       */
                    /*             LOONGARCH  ==> Loongson LoongArch    */
                    /*                  (LINUX 32- and 64-bit variants) */
                    /*             M88K       ==> Motorola 88XX0        */
                    /*                  (CX_UX and DGUX)                */
                    /*             S370       ==> 370-like machine      */
                    /*                  running Amdahl UTS4             */
                    /*             S390       ==> 390-like machine      */
                    /*                  running LINUX                   */
                    /*             AARCH64    ==> ARM AArch64           */
                    /*                  (LP64 and ILP32 variants)       */
                    /*             E2K        ==> Elbrus 2000           */
                    /*                  running LINUX                   */
                    /*             ARM32      ==> Intel StrongARM       */
                    /*                  (many variants)                 */
                    /*             IA64       ==> Intel IPF             */
                    /*                            (e.g. Itanium)        */
                    /*                  (LINUX and HPUX)                */
                    /*             SH         ==> Hitachi SuperH        */
                    /*                  (LINUX & MSWINCE)               */
                    /*             SW_64      ==> Sunway (Shenwei)      */
                    /*                  running LINUX                   */
                    /*             X86_64     ==> AMD x86-64            */
                    /*             POWERPC    ==> IBM/Apple PowerPC     */
                    /*                  (MACOS(<=9),DARWIN(incl.MACOSX),*/
                    /*                   LINUX, NETBSD, AIX, NOSYS      */
                    /*                   variants)                      */
                    /*                  Handles 32 and 64-bit variants. */
                    /*             ARC        ==> Synopsys ARC          */
                    /*             AVR32      ==> Atmel RISC 32-bit     */
                    /*             CRIS       ==> Axis Etrax            */
                    /*             M32R       ==> Renesas M32R          */
                    /*             NIOS2      ==> Altera NIOS2          */
                    /*             HEXAGON    ==> Qualcomm Hexagon      */
                    /*             OR1K       ==> OpenRISC/or1k         */
                    /*             RISCV      ==> RISC-V 32/64-bit      */
                    /*             TILEPRO    ==> Tilera TILEPro        */
                    /*             TILEGX     ==> Tilera TILE-Gx        */

/*
 * For each architecture and OS, the following need to be defined:
 *
 * CPP_WORDSZ is a simple integer constant representing the word size
 * in bits.  We assume byte addressability, where a byte has 8 bits.
 * We also assume CPP_WORDSZ is either 32 or 64.
 * (We care about the length of a pointer address, not hardware
 * bus widths.  Thus a 64-bit processor with a C compiler that uses
 * 32-bit pointers should use CPP_WORDSZ of 32, not 64.)
 *
 * CPP_PTRSZ is the pointer size in bits.  For most of the supported
 * targets, it is equal to CPP_WORDSZ.
 *
 * MACH_TYPE is a string representation of the machine type.
 * OS_TYPE is analogous for the OS.
 *
 * ALIGNMENT is the largest N, such that
 * all pointer are guaranteed to be aligned on N byte boundaries.
 * defining it to be 1 will always work, but perform poorly.
 *
 * DATASTART is the beginning of the data segment.
 * On some platforms SEARCH_FOR_DATA_START is defined.
 * The latter will cause GC_data_start to
 * be set to an address determined by accessing data backwards from _end
 * until an unmapped page is found.  DATASTART will be defined to be
 * GC_data_start.
 * On UNIX-like systems, the collector will scan the area between DATASTART
 * and DATAEND for root pointers.
 *
 * DATAEND, if not "end", where "end" is defined as "extern int end[]".
 * RTH suggests gaining access to linker script synth'd values with
 * this idiom instead of "&end", where "end" is defined as "extern int end".
 * Otherwise, "GCC will assume these are in .sdata/.sbss" and it will, e.g.,
 * cause failures on alpha*-*-* with -msmall-data or -fpic or mips-*-*
 * without any special options.
 *
 * STACKBOTTOM is the cold end of the stack, which is usually the
 * highest address in the stack.
 * Under PCR or OS/2, we have other ways of finding thread stacks.
 * For each machine, the following should:
 * 1) define STACK_GROWS_UP if the stack grows toward higher addresses, and
 * 2) define exactly one of
 *      STACKBOTTOM (should be defined to be an expression)
 *      LINUX_STACKBOTTOM
 *      HEURISTIC1
 *      HEURISTIC2
 * If STACKBOTTOM is defined, then its value will be used directly (as the
 * stack bottom).  If LINUX_STACKBOTTOM is defined, then it will be determined
 * with a method appropriate for most Linux systems.  Currently we look
 * first for __libc_stack_end (currently only if USE_LIBC_PRIVATES is
 * defined), and if that fails read it from /proc.  (If USE_LIBC_PRIVATES
 * is not defined and NO_PROC_STAT is defined, we revert to HEURISTIC2.)
 * If either of the last two macros are defined, then STACKBOTTOM is computed
 * during collector startup using one of the following two heuristics:
 * HEURISTIC1:  Take an address inside GC_init's frame, and round it up to
 *              the next multiple of STACK_GRAN.
 * HEURISTIC2:  Take an address inside GC_init's frame, increment it repeatedly
 *              in small steps (decrement if STACK_GROWS_UP), and read the value
 *              at each location.  Remember the value when the first
 *              Segmentation violation or Bus error is signaled.  Round that
 *              to the nearest plausible page boundary, and use that instead
 *              of STACKBOTTOM.
 *
 * Gustavo Rodriguez-Rivera points out that on most (all?) Unix machines,
 * the value of environ is a pointer that can serve as STACKBOTTOM.
 * I expect that HEURISTIC2 can be replaced by this approach, which
 * interferes far less with debugging.  However it has the disadvantage
 * that it's confused by a putenv call before the collector is initialized.
 * This could be dealt with by intercepting putenv ...
 *
 * If no expression for STACKBOTTOM can be found, and neither of the above
 * heuristics are usable, the collector can still be used with all of the above
 * undefined, provided one of the following is done:
 * 1) GC_mark_roots can be changed to somehow mark from the correct stack(s)
 *    without reference to STACKBOTTOM.  This is appropriate for use in
 *    conjunction with thread packages, since there will be multiple stacks.
 *    (Allocating thread stacks in the heap, and treating them as ordinary
 *    heap data objects is also possible as a last resort.  However, this is
 *    likely to introduce significant amounts of excess storage retention
 *    unless the dead parts of the thread stacks are periodically cleared.)
 * 2) Client code may set GC_stackbottom before calling any GC_ routines.
 *    If the author of the client code controls the main program, this is
 *    easily accomplished by introducing a new main program, setting
 *    GC_stackbottom to the address of a local variable, and then calling
 *    the original main program.  The new main program would read something
 *    like (provided real_main() is not inlined by the compiler):
 *
 *              #include "gc/gc.h"
 *
 *              main(argc, argv, envp)
 *              int argc;
 *              char **argv, **envp;
 *              {
 *                  volatile int dummy;
 *
 *                  GC_stackbottom = (ptr_t)(&dummy);
 *                  return real_main(argc, argv, envp);
 *              }
 *
 *
 * Each architecture may also define the style of virtual dirty bit
 * implementation to be used:
 *   GWW_VDB: Use Win32 GetWriteWatch primitive.
 *   MPROTECT_VDB: Write protect the heap and catch faults.
 *   PROC_VDB: Use the SVR4 /proc primitives to read dirty bits.
 *   SOFT_VDB: Use the Linux /proc primitives to track dirty bits.
 *
 * The first and second one may be combined, in which case a runtime
 * selection will be made, based on GetWriteWatch availability.
 *
 * An architecture may define DYNAMIC_LOADING if dyn_load.c
 * defined GC_register_dynamic_libraries() for the architecture.
 *
 * An architecture may define PREFETCH(x) to preload the cache with *x.
 * This defaults to GCC built-in operation (or a no-op for other compilers).
 *
 * GC_PREFETCH_FOR_WRITE(x) is used if *x is about to be written.
 *
 * An architecture may also define CLEAR_DOUBLE(x) to be a fast way to
 * clear 2 pointers at GC_malloc-aligned address x.  The default
 * implementation is just to store two NULL pointers.
 *
 * HEAP_START may be defined as the initial address hint for mmap-based
 * allocation.
 */

# ifdef LINUX /* TODO: FreeBSD too? */
    EXTERN_C_END
#   include <features.h> /* for __GLIBC__ and __GLIBC_MINOR__, at least */
    EXTERN_C_BEGIN
# endif

/* Convenient internal macro to test glibc version (if compiled against). */
#if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
# define GC_GLIBC_PREREQ(major, minor) \
            ((__GLIBC__ << 8) + __GLIBC_MINOR__ >= ((major) << 8) + (minor))
#else
# define GC_GLIBC_PREREQ(major, minor) 0 /* FALSE */
#endif

/* Align a ptr_t pointer down/up to a given boundary.               */
#define PTR_ALIGN_DOWN(p, b) ((ptr_t)((word)(p) & ~(word)((b)-1)))
#define PTR_ALIGN_UP(p, b) \
                ((ptr_t)(((word)(p) + (word)((b)-1)) & ~(word)((b)-1)))

/* If available, we can use __builtin_unwind_init() to push the     */
/* relevant registers onto the stack.                               */
# if GC_GNUC_PREREQ(2, 8) \
     && !GC_GNUC_PREREQ(11, 0) /* broken at least in 11.2.0 on cygwin64 */ \
     && !defined(__INTEL_COMPILER) && !defined(__PATHCC__) \
     && !defined(__FUJITSU) /* for FX10 system */ \
     && !(defined(POWERPC) && defined(DARWIN)) /* for MacOS X 10.3.9 */ \
     && !defined(E2K) && !defined(RTEMS) \
     && !defined(__ARMCC_VERSION) /* does not exist in armcc gnu emu */ \
     && (!defined(__clang__) \
         || GC_CLANG_PREREQ(8, 0) /* was no-op in clang-3 at least */)
#   define HAVE_BUILTIN_UNWIND_INIT
# endif

#if (defined(__CC_ARM) || defined(CX_UX) || defined(DJGPP) || defined(EMBOX) \
     || defined(EWS4800) || defined(LINUX) || defined(OS2) \
     || defined(RTEMS) || defined(UTS4) || defined(MSWIN32) \
     || defined(MSWINCE)) && !defined(NO_UNDERSCORE_SETJMP)
# define NO_UNDERSCORE_SETJMP
#endif

#define STACK_GRAN 0x1000000

/* The common OS-specific definitions (should be applicable to  */
/* all (or most, at least) supported architectures).            */

# ifdef CYGWIN32
#   define OS_TYPE "CYGWIN32"
#   define RETRY_GET_THREAD_CONTEXT
#   ifdef USE_WINALLOC
#     define GWW_VDB
#   elif defined(USE_MMAP)
#     define USE_MMAP_ANON
#   endif
# endif /* CYGWIN32 */

# ifdef DARWIN
#   define OS_TYPE "DARWIN"
#   define DYNAMIC_LOADING
    /* TODO: see get_end(3), get_etext() and get_end() should not be used. */
    /* These aren't used when dyld support is enabled (it is by default).  */
#   define DATASTART ((ptr_t)get_etext())
#   define DATAEND   ((ptr_t)get_end())
#   define USE_MMAP_ANON
    /* There seems to be some issues with trylock hanging on darwin.    */
    /* TODO: This should be looked into some more.                      */
#   define NO_PTHREAD_TRYLOCK
#   ifndef TARGET_OS_XR
#     define TARGET_OS_XR 0
#   endif
#   ifndef TARGET_OS_VISION
#     define TARGET_OS_VISION 0
#   endif
# endif /* DARWIN */

# ifdef EMBOX
#   define OS_TYPE "EMBOX"
    extern int _modules_data_start[];
    extern int _apps_bss_end[];
#   define DATASTART ((ptr_t)_modules_data_start)
#   define DATAEND ((ptr_t)_apps_bss_end)
    /* Note: the designated area might be quite large (several  */
    /* dozens of MBs) as it includes data and bss of all apps   */
    /* and modules of the built binary image.                   */
# endif /* EMBOX */

# ifdef FREEBSD
#   define OS_TYPE "FREEBSD"
#   define FREEBSD_STACKBOTTOM
#   ifdef __ELF__
#     define DYNAMIC_LOADING
#   endif
#   if !defined(ALPHA) && !defined(SPARC)
      extern char etext[];
#     define DATASTART GC_FreeBSDGetDataStart(0x1000, (ptr_t)etext)
#     define DATASTART_USES_BSDGETDATASTART
#     ifndef GC_FREEBSD_THREADS
#       define MPROTECT_VDB
#     endif
#   endif
# endif /* FREEBSD */

# ifdef HAIKU
#   define OS_TYPE "HAIKU"
#   define DYNAMIC_LOADING
#   define MPROTECT_VDB
    EXTERN_C_END
#   include <OS.h>
    EXTERN_C_BEGIN
#   define GETPAGESIZE() (unsigned)B_PAGE_SIZE
# endif /* HAIKU */

# ifdef HPUX
#   define OS_TYPE "HPUX"
    extern int __data_start[];
#   define DATASTART ((ptr_t)(__data_start))
#   ifdef USE_MMAP
#     define USE_MMAP_ANON
#   endif
#   define DYNAMIC_LOADING
#   define GETPAGESIZE() (unsigned)sysconf(_SC_PAGE_SIZE)
# endif /* HPUX */

# ifdef HURD
#   define OS_TYPE "HURD"
#   define HEURISTIC2
#   define SEARCH_FOR_DATA_START
    extern int _end[];
#   define DATAEND ((ptr_t)(_end))
    /* TODO: MPROTECT_VDB is not quite working yet? */
#   define DYNAMIC_LOADING
#   define USE_MMAP_ANON
# endif /* HURD */

# ifdef LINUX
#   define OS_TYPE "LINUX"
#   if defined(FORCE_MPROTECT_BEFORE_MADVISE) \
       || defined(PREFER_MMAP_PROT_NONE)
#     define COUNT_UNMAPPED_REGIONS
#   endif
#   define RETRY_TKILL_ON_EAGAIN
#   if !defined(MIPS) && !defined(POWERPC)
#     define LINUX_STACKBOTTOM
#   endif
#   if defined(__ELF__) && !defined(IA64)
#     define DYNAMIC_LOADING
#   endif
#   if defined(__ELF__) && !defined(ARC) && !defined(RISCV) \
       && !defined(S390) && !defined(TILEGX) && !defined(TILEPRO)
      extern int _end[];
#     define DATAEND ((ptr_t)(_end))
#   endif
#   if !defined(REDIRECT_MALLOC) && !defined(E2K)
      /* Requires Linux 2.3.47 or later. */
#     define MPROTECT_VDB
#   else
      /* We seem to get random errors in the incremental mode,  */
      /* possibly because the Linux threads implementation      */
      /* itself is a malloc client and cannot deal with the     */
      /* signals.  fread() uses malloc too.                     */
      /* In case of e2k, unless -fsemi-spec-ld (or -O0) option  */
      /* is passed to gcc (both when compiling libgc and the    */
      /* client), a semi-speculative optimization may lead to   */
      /* SIGILL (with ILL_ILLOPN si_code) instead of SIGSEGV.   */
#   endif
# endif /* LINUX */

# ifdef KOS
#   define OS_TYPE "KOS"
#   define HEURISTIC1 /* relies on pthread_attr_getstack actually */
#   ifndef USE_GET_STACKBASE_FOR_MAIN
      /* Note: this requires -lpthread option. */
#     define USE_GET_STACKBASE_FOR_MAIN
#   endif
    extern int __data_start[];
#   define DATASTART ((ptr_t)(__data_start))
# endif /* KOS */

# ifdef MACOS
#   define OS_TYPE "MACOS"
#   ifndef __LOWMEM__
      EXTERN_C_END
#     include <LowMem.h>
      EXTERN_C_BEGIN
#   endif
    /* See os_dep.c for details of global data segments.        */
#   define STACKBOTTOM ((ptr_t)LMGetCurStackBase())
#   define DATAEND /* not needed */
# endif /* MACOS */

# ifdef MSWIN32
#   define OS_TYPE "MSWIN32"
    /* STACKBOTTOM and DATASTART are handled specially in os_dep.c.     */
#   if !defined(CPPCHECK)
#     define DATAEND /* not needed */
#   endif
#   define GWW_VDB
# endif

# ifdef MSWINCE
#   define OS_TYPE "MSWINCE"
#   if !defined(CPPCHECK)
#     define DATAEND /* not needed */
#   endif
# endif

# ifdef NACL
#   define OS_TYPE "NACL"
#   if defined(__GLIBC__)
#     define DYNAMIC_LOADING
#   endif
#   define DATASTART ((ptr_t)0x10020000)
    extern int _end[];
#   define DATAEND ((ptr_t)_end)
#   undef STACK_GRAN
#   define STACK_GRAN 0x10000
#   define HEURISTIC1
#   define NO_PTHREAD_GETATTR_NP
#   define USE_MMAP_ANON
#   define GETPAGESIZE() 65536 /* FIXME: Not real page size */
#   define MAX_NACL_GC_THREADS 1024
# endif /* NACL */

# ifdef NETBSD
#   define OS_TYPE "NETBSD"
#   define HEURISTIC2
#   ifdef __ELF__
#     define SEARCH_FOR_DATA_START
#     define DYNAMIC_LOADING
#   elif !defined(MIPS) /* TODO: probably do not exclude it */
      extern char etext[];
#     define DATASTART ((ptr_t)(etext))
#   endif
# endif /* NETBSD */

# ifdef NEXT
#   define OS_TYPE "NEXT"
#   define DATASTART ((ptr_t)get_etext())
#   define DATASTART_IS_FUNC
#   define DATAEND /* not needed */
# endif

# ifdef OPENBSD
#   define OS_TYPE "OPENBSD"
#   ifndef GC_OPENBSD_THREADS
#     define HEURISTIC2
#   endif
#   ifdef __ELF__
      extern int __data_start[];
#     define DATASTART ((ptr_t)__data_start)
      extern int _end[];
#     define DATAEND ((ptr_t)(&_end))
#     define DYNAMIC_LOADING
#   else
      extern char etext[];
#     define DATASTART ((ptr_t)(etext))
#   endif
#   define MPROTECT_VDB
# endif /* OPENBSD */

# ifdef QNX
#   define OS_TYPE "QNX"
#   define SA_RESTART 0
#   ifndef QNX_STACKBOTTOM /* TODO: not the default one for now */
#     define HEURISTIC1
#   endif
    extern char etext[];
#   define DATASTART ((ptr_t)etext)
    extern int _end[];
#   define DATAEND ((ptr_t)_end)
# endif /* QNX */

# ifdef SOLARIS
#   define OS_TYPE "SOLARIS"
    extern int _etext[], _end[];
    ptr_t GC_SysVGetDataStart(size_t, ptr_t);
#   define DATASTART_IS_FUNC
#   define DATAEND ((ptr_t)(_end))
#   if !defined(USE_MMAP) && defined(REDIRECT_MALLOC)
#     define USE_MMAP 1
        /* Otherwise we now use calloc.  Mmap may result in the     */
        /* heap interleaved with thread stacks, which can result in */
        /* excessive blacklisting.  Sbrk is unusable since it       */
        /* doesn't interact correctly with the system malloc.       */
#   endif
#   ifdef USE_MMAP
#     define HEAP_START (ptr_t)0x40000000
#   else
#     define HEAP_START DATAEND
#   endif
#   ifndef GC_THREADS
#     define MPROTECT_VDB
#   endif
#   define DYNAMIC_LOADING
    /* Define STACKBOTTOM as (ptr_t)_start worked through 2.7,      */
    /* but reportedly breaks under 2.8.  It appears that the stack  */
    /* base is a property of the executable, so this should not     */
    /* break old executables.                                       */
    /* HEURISTIC1 reportedly no longer works under Solaris 2.7.     */
    /* HEURISTIC2 probably works, but this appears to be preferable.*/
    /* Apparently USRSTACK is defined to be USERLIMIT, but in some  */
    /* installations that's undefined.  We work around this with a  */
    /* gross hack:                                                  */
    EXTERN_C_END
#   include <sys/vmparam.h>
    EXTERN_C_BEGIN
#   ifdef USERLIMIT
      /* This should work everywhere, but doesn't.  */
#     define STACKBOTTOM ((ptr_t)USRSTACK)
#   else
#     define HEURISTIC2
#   endif
# endif /* SOLARIS */

# ifdef SYMBIAN
#   define OS_TYPE "SYMBIAN"
#   define DATASTART (ptr_t)ALIGNMENT /* cannot be null */
#   define DATAEND (ptr_t)ALIGNMENT
#   ifndef USE_MMAP
      /* sbrk() is not available. */
#     define USE_MMAP 1
#   endif
# endif /* SYMBIAN */

# ifdef M68K
#   define MACH_TYPE "M68K"
#   define CPP_WORDSZ 32
#   define ALIGNMENT 2
#   ifdef OPENBSD
      /* Nothing specific. */
#   endif
#   ifdef NETBSD
      /* Nothing specific. */
#   endif
#   ifdef LINUX
#       ifdef __ELF__
#         if GC_GLIBC_PREREQ(2, 0)
#           define SEARCH_FOR_DATA_START
#         else
            extern char **__environ;
#           define DATASTART ((ptr_t)(&__environ))
                             /* hideous kludge: __environ is the first */
                             /* word in crt0.o, and delimits the start */
                             /* of the data segment, no matter which   */
                             /* ld options were passed through.        */
                             /* We could use _etext instead, but that  */
                             /* would include .rodata, which may       */
                             /* contain large read-only data tables    */
                             /* that we'd rather not scan.             */
#         endif
#       else
          extern int etext[];
#         define DATASTART PTR_ALIGN_UP(etext, 0x1000)
#       endif
#   endif
#   ifdef AMIGA
#       define OS_TYPE "AMIGA"
                /* STACKBOTTOM and DATASTART handled specially  */
                /* in os_dep.c                                  */
#       define DATAEND  /* not needed */
#       define GETPAGESIZE() 4096
#   endif
#   ifdef MACOS
#     define GETPAGESIZE() 4096
#   endif
#   ifdef NEXT
#     define STACKBOTTOM ((ptr_t)0x4000000)
#   endif
# endif

# ifdef POWERPC
#   define MACH_TYPE "POWERPC"
#   ifdef MACOS
#     define CPP_WORDSZ 32
#     define ALIGNMENT 2  /* Still necessary?  Could it be 4?   */
#   endif
#   ifdef LINUX
#     if defined(__powerpc64__)
#       define CPP_WORDSZ 64
#       ifndef HBLKSIZE
#         define HBLKSIZE 4096
#       endif
#     else
#       define CPP_WORDSZ 32
#       define ALIGNMENT 4
#     endif
      /* HEURISTIC1 has been reliably reported to fail for a 32-bit     */
      /* executable on a 64-bit kernel.                                 */
#     if defined(__bg__)
        /* The Linux Compute Node Kernel (used on BlueGene systems)     */
        /* does not support LINUX_STACKBOTTOM way.                      */
#       define HEURISTIC2
#       define NO_PTHREAD_GETATTR_NP
#     else
#       define LINUX_STACKBOTTOM
#     endif
#     define SEARCH_FOR_DATA_START
#     ifndef SOFT_VDB
#       define SOFT_VDB
#     endif
#   endif
#   ifdef DARWIN
#     if defined(__ppc64__)
#       define CPP_WORDSZ 64
#       define STACKBOTTOM ((ptr_t)0x7fff5fc00000)
#       define CACHE_LINE_SIZE 64
#       ifndef HBLKSIZE
#         define HBLKSIZE 4096
#       endif
#     else
#       define CPP_WORDSZ 32
#       define ALIGNMENT 4
#       define STACKBOTTOM ((ptr_t)0xc0000000)
#     endif
#     define MPROTECT_VDB
#     if defined(USE_PPC_PREFETCH) && defined(__GNUC__)
        /* The performance impact of prefetches is untested */
#       define PREFETCH(x) \
          __asm__ __volatile__ ("dcbt 0,%0" : : "r" ((const void *) (x)))
#       define GC_PREFETCH_FOR_WRITE(x) \
          __asm__ __volatile__ ("dcbtst 0,%0" : : "r" ((const void *) (x)))
#     endif
#   endif
#   ifdef OPENBSD
#     if defined(__powerpc64__)
#       define CPP_WORDSZ 64
#     else
#       define CPP_WORDSZ 32
#       define ALIGNMENT 4
#     endif
#   endif
#   ifdef FREEBSD
#     if defined(__powerpc64__)
#       define CPP_WORDSZ 64
#       ifndef HBLKSIZE
#         define HBLKSIZE 4096
#       endif
#     else
#       define CPP_WORDSZ 32
#       define ALIGNMENT 4
#     endif
#   endif
#   ifdef NETBSD
#     define CPP_WORDSZ 32
#     define ALIGNMENT 4
#   endif
#   ifdef SN_TARGET_PS3
#     define OS_TYPE "SN_TARGET_PS3"
#     define CPP_WORDSZ 32
#     define NO_GETENV
      extern int _end[];
      extern int __bss_start;
#     define DATASTART ((ptr_t)(__bss_start))
#     define DATAEND ((ptr_t)(_end))
#     define STACKBOTTOM ((ptr_t)ps3_get_stack_bottom())
#     define NO_PTHREAD_TRYLOCK
                /* Current LOCK() implementation for PS3 explicitly     */
                /* use pthread_mutex_lock for some reason.              */
#   endif
#   ifdef AIX
#     define OS_TYPE "AIX"
#     undef ALIGNMENT /* in case it's defined   */
#     undef IA64
      /* DOB: some AIX installs stupidly define IA64 in */
      /* /usr/include/sys/systemcfg.h                   */
#     ifdef __64BIT__
#       define CPP_WORDSZ 64
#       define STACKBOTTOM ((ptr_t)0x1000000000000000)
#     else
#       define CPP_WORDSZ 32
#       define STACKBOTTOM ((ptr_t)((ulong)&errno))
#     endif
#     define USE_MMAP_ANON
        /* From AIX linker man page:
        _text Specifies the first location of the program.
        _etext Specifies the first location after the program.
        _data Specifies the first location of the data.
        _edata Specifies the first location after the initialized data
        _end or end Specifies the first location after all data.
        */
      extern int _data[], _end[];
#     define DATASTART ((ptr_t)((ulong)_data))
#     define DATAEND ((ptr_t)((ulong)_end))
      extern int errno;
#     define MPROTECT_VDB
#     define DYNAMIC_LOADING
        /* For really old versions of AIX, this may have to be removed. */
#   endif
#   ifdef NOSYS
#     define OS_TYPE "NOSYS"
#     define CPP_WORDSZ 32
#     define ALIGNMENT 4
      extern void __end[], __dso_handle[];
#     define DATASTART ((ptr_t)__dso_handle) /* OK, that's ugly.    */
#     define DATAEND ((ptr_t)(__end))
        /* Stack starts at 0xE0000000 for the simulator.  */
#     undef STACK_GRAN
#     define STACK_GRAN 0x10000000
#     define HEURISTIC1
#   endif
# endif /* POWERPC */

# ifdef VAX
#   define MACH_TYPE "VAX"
#   define CPP_WORDSZ 32
#   define ALIGNMENT 4  /* Pointers are longword aligned by 4.2 C compiler */
    extern char etext[];
#   define DATASTART ((ptr_t)(etext))
#   ifdef BSD
#       define OS_TYPE "BSD"
#       define HEURISTIC1
                        /* HEURISTIC2 may be OK, but it's hard to test. */
#   endif
#   ifdef ULTRIX
#       define OS_TYPE "ULTRIX"
#       define STACKBOTTOM ((ptr_t)0x7fffc800)
#   endif
# endif /* VAX */

# ifdef SPARC
#   define MACH_TYPE "SPARC"
#   if defined(__arch64__) || defined(__sparcv9)
#     define CPP_WORDSZ 64
#     define ELF_CLASS ELFCLASS64
#   else
#     define CPP_WORDSZ 32
#     define ALIGNMENT 4        /* Required by hardware */
#   endif
    /* Don't define USE_ASM_PUSH_REGS.  We do use an asm helper, but    */
    /* not to push the registers on the mark stack.                     */
#   ifdef SOLARIS
#       define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext)
#       define PROC_VDB
#       define GETPAGESIZE() (unsigned)sysconf(_SC_PAGESIZE)
                /* getpagesize() appeared to be missing from at least   */
                /* one Solaris 5.4 installation.  Weird.                */
#   endif
#   ifdef DRSNX
#       define OS_TYPE "DRSNX"
        extern int etext[];
        ptr_t GC_SysVGetDataStart(size_t, ptr_t);
#       define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)etext)
#       define DATASTART_IS_FUNC
#       define MPROTECT_VDB
#       define STACKBOTTOM ((ptr_t)0xdfff0000)
#       define DYNAMIC_LOADING
#   endif
#   ifdef LINUX
#     if !defined(__ELF__) && !defined(CPPCHECK)
#       error Linux SPARC a.out not supported
#     endif
#     define SVR4
      extern int _etext[];
      ptr_t GC_SysVGetDataStart(size_t, ptr_t);
#     ifdef __arch64__
#       define DATASTART GC_SysVGetDataStart(0x100000, (ptr_t)_etext)
#     else
#       define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext)
#     endif
#     define DATASTART_IS_FUNC
#   endif
#   ifdef OPENBSD
      /* Nothing specific. */
#   endif
#   ifdef NETBSD
      /* Nothing specific. */
#   endif
#   ifdef FREEBSD
        extern char etext[];
        extern char edata[];
#       if !defined(CPPCHECK)
          extern char end[];
#       endif
#       define NEED_FIND_LIMIT
#       define DATASTART ((ptr_t)(&etext))
        void * GC_find_limit(void *, int);
#       define DATAEND (ptr_t)GC_find_limit(DATASTART, TRUE)
#       define DATAEND_IS_FUNC
#       define GC_HAVE_DATAREGION2
#       define DATASTART2 ((ptr_t)(&edata))
#       define DATAEND2 ((ptr_t)(&end))
#   endif
# endif /* SPARC */

# ifdef I386
#   define MACH_TYPE "I386"
#   if (defined(__LP64__) || defined(_WIN64)) && !defined(CPPCHECK)
#     error This should be handled as X86_64
#   else
#     define CPP_WORDSZ 32
#     define ALIGNMENT 4
                        /* Appears to hold for all "32 bit" compilers   */
                        /* except Borland.  The -a4 option fixes        */
                        /* Borland.  For Watcom the option is -zp4.     */
#   endif
#   ifdef SEQUENT
#       define OS_TYPE "SEQUENT"
        extern int etext[];
#       define DATASTART PTR_ALIGN_UP(etext, 0x1000)
#       define STACKBOTTOM ((ptr_t)0x3ffff000)
#   endif
#   ifdef HAIKU
      extern int etext[];
#     define DATASTART PTR_ALIGN_UP(etext, 0x1000)
#   endif
#   ifdef HURD
      /* Nothing specific. */
#   endif
#   ifdef EMBOX
      /* Nothing specific. */
#   endif
#   ifdef NACL
      /* Nothing specific. */
#   endif
#   ifdef QNX
      /* Nothing specific. */
#   endif
#   ifdef SOLARIS
#       define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)_etext)
        /* At least in Solaris 2.5, PROC_VDB gives wrong values for     */
        /* dirty bits.  It appears to be fixed in 2.8 and 2.9.          */
#       ifdef SOLARIS25_PROC_VDB_BUG_FIXED
#         define PROC_VDB
#       endif
#   endif
#   ifdef SCO
#       define OS_TYPE "SCO"
        extern int etext[];
#       define DATASTART (PTR_ALIGN_UP(etext, 0x400000) \
                            + ((word)(etext) & 0xfff))
#       define STACKBOTTOM ((ptr_t)0x7ffffffc)
#   endif
#   ifdef SCO_ELF
#       define OS_TYPE "SCO_ELF"
        extern int etext[];
#       define DATASTART ((ptr_t)(etext))
#       define STACKBOTTOM ((ptr_t)0x08048000)
#       define DYNAMIC_LOADING
#       define ELF_CLASS ELFCLASS32
#   endif
#   ifdef DGUX
#       define OS_TYPE "DGUX"
        extern int _etext, _end;
        ptr_t GC_SysVGetDataStart(size_t, ptr_t);
#       define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)(&_etext))
#       define DATASTART_IS_FUNC
#       define DATAEND ((ptr_t)(&_end))
#       define HEURISTIC2
#       define DYNAMIC_LOADING
#       ifndef USE_MMAP
#         define USE_MMAP 1
#       endif
#       define MAP_FAILED (void *)((word)-1)
#       define HEAP_START (ptr_t)0x40000000
#   endif /* DGUX */
#   ifdef LINUX
#       define HEAP_START (ptr_t)0x1000
                /* This encourages mmap to give us low addresses,       */
                /* thus allowing the heap to grow to ~3 GB.             */
#       ifdef __ELF__
#            if GC_GLIBC_PREREQ(2, 0) || defined(HOST_ANDROID)
#                define SEARCH_FOR_DATA_START
#            else
                 extern char **__environ;
#                define DATASTART ((ptr_t)(&__environ))
                              /* hideous kludge: __environ is the first */
                              /* word in crt0.o, and delimits the start */
                              /* of the data segment, no matter which   */
                              /* ld options were passed through.        */
                              /* We could use _etext instead, but that  */
                              /* would include .rodata, which may       */
                              /* contain large read-only data tables    */
                              /* that we'd rather not scan.             */
#            endif
#            if !defined(GC_NO_SIGSETJMP) && (defined(HOST_TIZEN) \
                    || (defined(HOST_ANDROID) \
                        && !(GC_GNUC_PREREQ(4, 8) || GC_CLANG_PREREQ(3, 2) \
                             || __ANDROID_API__ >= 18)))
               /* Older Android NDK releases lack sigsetjmp in x86 libc */
               /* (setjmp is used instead to find data_start).  The bug */
               /* is fixed in Android NDK r8e (so, ok to use sigsetjmp  */
               /* if gcc4.8+, clang3.2+ or Android API level 18+).      */
#              define GC_NO_SIGSETJMP 1
#            endif
#       else
             extern int etext[];
#            define DATASTART PTR_ALIGN_UP(etext, 0x1000)
#       endif
#       ifdef USE_I686_PREFETCH
#         define PREFETCH(x) \
            __asm__ __volatile__ ("prefetchnta %0" : : "m"(*(char *)(x)))
            /* Empirically prefetcht0 is much more effective at reducing     */
            /* cache miss stalls for the targeted load instructions.  But it */
            /* seems to interfere enough with other cache traffic that the   */
            /* net result is worse than prefetchnta.                         */
#         ifdef FORCE_WRITE_PREFETCH
            /* Using prefetches for write seems to have a slight negative    */
            /* impact on performance, at least for a PIII/500.               */
#           define GC_PREFETCH_FOR_WRITE(x) \
              __asm__ __volatile__ ("prefetcht0 %0" : : "m"(*(char *)(x)))
#         else
#           define GC_NO_PREFETCH_FOR_WRITE
#         endif
#       elif defined(USE_3DNOW_PREFETCH)
#         define PREFETCH(x) \
            __asm__ __volatile__ ("prefetch %0" : : "m"(*(char *)(x)))
#         define GC_PREFETCH_FOR_WRITE(x) \
            __asm__ __volatile__ ("prefetchw %0" : : "m"(*(char *)(x)))
#       endif
#       if defined(__GLIBC__) && !defined(__UCLIBC__) \
           && !defined(GLIBC_TSX_BUG_FIXED)
          /* Workaround lock elision implementation for some glibc.     */
#         define GLIBC_2_19_TSX_BUG
          EXTERN_C_END
#         include <gnu/libc-version.h> /* for gnu_get_libc_version() */
          EXTERN_C_BEGIN
#       endif
#       ifndef SOFT_VDB
#         define SOFT_VDB
#       endif
#   endif
#   ifdef CYGWIN32
#       define WOW64_THREAD_CONTEXT_WORKAROUND
#       define DATASTART ((ptr_t)GC_DATASTART)  /* From gc.h */
#       define DATAEND   ((ptr_t)GC_DATAEND)
#       ifndef USE_WINALLOC
#         /* MPROTECT_VDB does not work, it leads to a spurious exit.   */
#       endif
#   endif
#   ifdef INTERIX
#     define OS_TYPE "INTERIX"
      extern int _data_start__[];
      extern int _bss_end__[];
#     define DATASTART ((ptr_t)_data_start__)
#     define DATAEND   ((ptr_t)_bss_end__)
#     define STACKBOTTOM ({ ptr_t rv; \
                            __asm__ __volatile__ ("movl %%fs:4, %%eax" \
                                                  : "=a" (rv)); \
                            rv; })
#     define USE_MMAP_ANON
#   endif
#   ifdef OS2
#       define OS_TYPE "OS2"
                /* STACKBOTTOM and DATASTART are handled specially in   */
                /* os_dep.c. OS2 actually has the right                 */
                /* system call!                                         */
#       define DATAEND  /* not needed */
#       define GETPAGESIZE() os2_getpagesize()
#   endif
#   ifdef MSWIN32
#       define WOW64_THREAD_CONTEXT_WORKAROUND
#       define RETRY_GET_THREAD_CONTEXT
#       define MPROTECT_VDB
#   endif
#   ifdef MSWINCE
        /* Nothing specific. */
#   endif
#   ifdef DJGPP
#       define OS_TYPE "DJGPP"
        EXTERN_C_END
#       include "stubinfo.h"
        EXTERN_C_BEGIN
        extern int etext[];
        extern int _stklen;
        extern int __djgpp_stack_limit;
#       define DATASTART PTR_ALIGN_UP(etext, 0x200)
/* #define STACKBOTTOM ((ptr_t)((word)_stubinfo+_stubinfo->size+_stklen)) */
#       define STACKBOTTOM ((ptr_t)((word)__djgpp_stack_limit + _stklen))
                /* This may not be right.  */
#   endif
#   ifdef OPENBSD
      /* Nothing specific. */
#   endif
#   ifdef FREEBSD
#       if defined(__GLIBC__)
            extern int _end[];
#           define DATAEND ((ptr_t)(_end))
#       endif
#   endif
#   ifdef NETBSD
      /* Nothing specific. */
#   endif
#   ifdef THREE86BSD
#       define OS_TYPE "THREE86BSD"
#       define HEURISTIC2
        extern char etext[];
#       define DATASTART ((ptr_t)(etext))
#   endif
#   ifdef BSDI
#       define OS_TYPE "BSDI"
#       define HEURISTIC2
        extern char etext[];
#       define DATASTART ((ptr_t)(etext))
#   endif
#   ifdef NEXT
#       define STACKBOTTOM ((ptr_t)0xc0000000)
#   endif
#   ifdef RTEMS
#       define OS_TYPE "RTEMS"
        EXTERN_C_END
#       include <sys/unistd.h>
        EXTERN_C_BEGIN
        extern int etext[];
        void *rtems_get_stack_bottom(void);
#       define InitStackBottom rtems_get_stack_bottom()
#       define DATASTART ((ptr_t)etext)
#       define STACKBOTTOM ((ptr_t)InitStackBottom)
#   endif
#   ifdef DOS4GW
#     define OS_TYPE "DOS4GW"
      extern long __nullarea;
      extern char _end;
      extern char *_STACKTOP;
      /* Depending on calling conventions Watcom C either precedes      */
      /* or does not precedes with underscore names of C-variables.     */
      /* Make sure startup code variables always have the same names.   */
#     pragma aux __nullarea "*";
#     pragma aux _end "*";
#     define STACKBOTTOM ((ptr_t)_STACKTOP)
                         /* confused? me too. */
#     define DATASTART ((ptr_t)(&__nullarea))
#     define DATAEND ((ptr_t)(&_end))
#     define GETPAGESIZE() 4096
#   endif
#   ifdef DARWIN
#     define DARWIN_DONT_PARSE_STACK 1
#     define STACKBOTTOM ((ptr_t)0xc0000000)
#     define MPROTECT_VDB
#   endif
# endif /* I386 */

# ifdef NS32K
#   define MACH_TYPE "NS32K"
#   define CPP_WORDSZ 32
#   define ALIGNMENT 4
    extern char **environ;
#   define DATASTART ((ptr_t)(&environ))
                              /* hideous kludge: environ is the first   */
                              /* word in crt0.o, and delimits the start */
                              /* of the data segment, no matter which   */
                              /* ld options were passed through.        */
#   define STACKBOTTOM ((ptr_t)0xfffff000) /* for Encore */
# endif /* NS32K */

# ifdef LOONGARCH
#   define MACH_TYPE "LoongArch"
#   define CPP_WORDSZ _LOONGARCH_SZPTR
#   ifdef LINUX
#     pragma weak __data_start
      extern int __data_start[];
#     define DATASTART ((ptr_t)(__data_start))
#   endif
# endif /* LOONGARCH */

# ifdef SW_64
#   define MACH_TYPE "SW_64"
#   define CPP_WORDSZ 64
#   ifdef LINUX
      /* Nothing specific. */
#   endif
# endif /* SW_64 */

# ifdef MIPS
#   define MACH_TYPE "MIPS"
#   ifdef LINUX
#     ifdef _MIPS_SZPTR
#       define CPP_WORDSZ _MIPS_SZPTR
#     else
#       define CPP_WORDSZ 32
#       define ALIGNMENT 4
#     endif
#     pragma weak __data_start
      extern int __data_start[];
#     define DATASTART ((ptr_t)(__data_start))
#     ifndef HBLKSIZE
#       define HBLKSIZE 4096
#     endif
#     if GC_GLIBC_PREREQ(2, 2)
#       define LINUX_STACKBOTTOM
#     else
#       define STACKBOTTOM ((ptr_t)0x7fff8000)
#     endif
#   endif
#   ifdef EWS4800
#     define OS_TYPE "EWS4800"
#     define HEURISTIC2
#     if defined(_MIPS_SZPTR) && (_MIPS_SZPTR == 64)
#       define CPP_WORDSZ _MIPS_SZPTR
        extern int _fdata[], _end[];
#       define DATASTART ((ptr_t)_fdata)
#       define DATAEND ((ptr_t)_end)
#     else
#       define CPP_WORDSZ 32
#       define ALIGNMENT 4
        extern int etext[], edata[];
#       if !defined(CPPCHECK)
          extern int end[];
#       endif
        extern int _DYNAMIC_LINKING[], _gp[];
#       define DATASTART (PTR_ALIGN_UP(etext, 0x40000) \
                            + ((word)(etext) & 0xffff))
#       define DATAEND ((ptr_t)(edata))
#       define GC_HAVE_DATAREGION2
#       define DATASTART2 (_DYNAMIC_LINKING \
                            ? PTR_ALIGN_UP((ptr_t)_gp + 0x8000, 0x40000) \
                            : (ptr_t)edata)
#       define DATAEND2 ((ptr_t)(end))
#     endif
#   endif
#   ifdef ULTRIX
#     define OS_TYPE "ULTRIX"
#     define CPP_WORDSZ 32
#     define ALIGNMENT 4
#     define HEURISTIC2
#     define DATASTART ((ptr_t)0x10000000)
                              /* Could probably be slightly higher since */
                              /* startup code allocates lots of stuff.   */
#   endif
#   ifdef IRIX5
#     define OS_TYPE "IRIX5"
#     ifdef _MIPS_SZPTR
#       define CPP_WORDSZ _MIPS_SZPTR
#     else
#       define CPP_WORDSZ 32
#       define ALIGNMENT 4
#     endif
#     define HEURISTIC2
      extern int _fdata[];
#     define DATASTART ((ptr_t)(_fdata))
#     ifdef USE_MMAP
#       define HEAP_START (ptr_t)0x30000000
#     else
#       define HEAP_START DATASTART
#     endif
                              /* Lowest plausible heap address.         */
                              /* In the MMAP case, we map there.        */
                              /* In either case it is used to identify  */
                              /* heap sections so they're not           */
                              /* considered as roots.                   */
/*#       define MPROTECT_VDB DOB: this should work, but there is evidence */
/*              of recent breakage.                                        */
#     define DYNAMIC_LOADING
#   endif
#   ifdef MSWINCE
#     define CPP_WORDSZ 32
#     define ALIGNMENT 4
#   endif
#   ifdef NETBSD
#     define CPP_WORDSZ 32
#     define ALIGNMENT 4
#     ifndef __ELF__
#       define DATASTART ((ptr_t)0x10000000)
#       define STACKBOTTOM ((ptr_t)0x7ffff000)
#     endif
#   endif
#   ifdef OPENBSD
#     define CPP_WORDSZ 64 /* all OpenBSD/mips platforms are 64-bit */
#   endif
#   ifdef FREEBSD
#     define CPP_WORDSZ 32
#     define ALIGNMENT 4
#   endif
#   ifdef NONSTOP
#     define OS_TYPE "NONSTOP"
#     define CPP_WORDSZ 32
#     define DATASTART ((ptr_t)0x08000000)
      extern char **environ;
#     define DATAEND ((ptr_t)(environ - 0x10))
#     define STACKBOTTOM ((ptr_t)0x4fffffff)
#   endif
# endif /* MIPS */

# ifdef NIOS2
#   define MACH_TYPE "NIOS2"
#   define CPP_WORDSZ 32
#   ifndef HBLKSIZE
#     define HBLKSIZE 4096
#   endif
#   ifdef LINUX
      extern int __data_start[];
#     define DATASTART ((ptr_t)(__data_start))
#   endif
# endif /* NIOS2 */

# ifdef OR1K
#   define MACH_TYPE "OR1K"
#   define CPP_WORDSZ 32
#   ifndef HBLKSIZE
#     define HBLKSIZE 4096
#   endif
#   ifdef LINUX
      extern int __data_start[];
#     define DATASTART ((ptr_t)(__data_start))
#   endif
# endif /* OR1K */

# ifdef HP_PA
#   define MACH_TYPE "HP_PA"
#   ifdef __LP64__
#     define CPP_WORDSZ 64
#   else
#     define CPP_WORDSZ 32
#   endif
#   define STACK_GROWS_UP
#   ifdef HPUX
#     ifndef GC_THREADS
#       define MPROTECT_VDB
#     endif
#     ifdef USE_HPUX_FIXED_STACKBOTTOM
        /* The following appears to work for 7xx systems running HP/UX  */
        /* 9.xx.  Furthermore, it might result in much faster           */
        /* collections than HEURISTIC2, which may involve scanning      */
        /* segments that directly precede the stack.  It is not the     */
        /* default, since it may not work on older machine/OS           */
        /* combinations. (Thanks to Raymond X.T. Nijssen for uncovering */
        /* this.)                                                       */
        /* This technique also doesn't work with HP/UX 11.xx.  The      */
        /* stack size is settable using the kernel maxssiz variable,    */
        /* and in 11.23 and latter, the size can be set dynamically.    */
        /* It also doesn't handle SHMEM_MAGIC binaries which have       */
        /* stack and data in the first quadrant.                        */
#       define STACKBOTTOM ((ptr_t)0x7b033000) /* from /etc/conf/h/param.h */
#     elif defined(USE_ENVIRON_POINTER)
        /* Gustavo Rodriguez-Rivera suggested changing HEURISTIC2       */
        /* to this.  Note that the GC must be initialized before the    */
        /* first putenv call.  Unfortunately, some clients do not obey. */
        extern char ** environ;
#       define STACKBOTTOM ((ptr_t)environ)
#     elif !defined(HEURISTIC2)
        /* This uses pst_vm_status support. */
#       define HPUX_MAIN_STACKBOTTOM
#     endif
#     ifndef __GNUC__
#       define PREFETCH(x)  do { \
                              register long addr = (long)(x); \
                              (void) _asm ("LDW", 0, 0, addr, 0); \
                            } while (0)
#     endif
#   endif /* HPUX */
#   ifdef LINUX
#     define SEARCH_FOR_DATA_START
#   endif
#   ifdef OPENBSD
      /* Nothing specific. */
#   endif
# endif /* HP_PA */

# ifdef ALPHA
#   define MACH_TYPE "ALPHA"
#   define CPP_WORDSZ 64
#   ifdef NETBSD
#       define ELFCLASS32 32
#       define ELFCLASS64 64
#       define ELF_CLASS ELFCLASS64
#   endif
#   ifdef OPENBSD
      /* Nothing specific. */
#   endif
#   ifdef FREEBSD
        /* MPROTECT_VDB is not yet supported at all on FreeBSD/alpha. */
/* Handle unmapped hole alpha*-*-freebsd[45]* puts between etext and edata. */
        extern char etext[];
        extern char edata[];
#       if !defined(CPPCHECK)
          extern char end[];
#       endif
#       define NEED_FIND_LIMIT
#       define DATASTART ((ptr_t)(&etext))
        void * GC_find_limit(void *, int);
#       define DATAEND (ptr_t)GC_find_limit(DATASTART, TRUE)
#       define DATAEND_IS_FUNC
#       define GC_HAVE_DATAREGION2
#       define DATASTART2 ((ptr_t)(&edata))
#       define DATAEND2 ((ptr_t)(&end))
#   endif
#   ifdef OSF1
#       define OS_TYPE "OSF1"
#       define DATASTART ((ptr_t)0x140000000)
        extern int _end[];
#       define DATAEND ((ptr_t)(&_end))
        extern char ** environ;
        /* round up from the value of environ to the nearest page boundary */
        /* Probably breaks if putenv is called before collector            */
        /* initialization.                                                 */
#       define STACKBOTTOM ((ptr_t)(((word)(environ) | (getpagesize()-1))+1))
/* #define HEURISTIC2 */
        /* Normally HEURISTIC2 is too conservative, since               */
        /* the text segment immediately follows the stack.              */
        /* Hence we give an upper pound.                                */
        /* This is currently unused, since we disabled HEURISTIC2       */
        extern int __start[];
#       define HEURISTIC2_LIMIT ((ptr_t)((word)(__start) \
                                         & ~(word)(getpagesize()-1)))
#       ifndef GC_OSF1_THREADS
          /* Unresolved signal issues with threads.     */
#         define MPROTECT_VDB
#       endif
#       define DYNAMIC_LOADING
#   endif
#   ifdef LINUX
#       ifdef __ELF__
#         define SEARCH_FOR_DATA_START
#       else
#         define DATASTART ((ptr_t)0x140000000)
          extern int _end[];
#         define DATAEND ((ptr_t)(_end))
#       endif
#   endif
# endif /* ALPHA */

# ifdef IA64
#   define MACH_TYPE "IA64"
#   ifdef HPUX
#       ifdef _ILP32
#         define CPP_WORDSZ 32
            /* Requires 8 byte alignment for malloc */
#         define ALIGNMENT 4
#       else
#         if !defined(_LP64) && !defined(CPPCHECK)
#           error Unknown ABI
#         endif
#         define CPP_WORDSZ 64
            /* Requires 16 byte alignment for malloc */
#         define ALIGNMENT 8
#       endif
        /* Note that the GC must be initialized before the 1st putenv call. */
        extern char ** environ;
#       define STACKBOTTOM ((ptr_t)environ)
#       define HPUX_STACKBOTTOM
        /* The following was empirically determined, and is probably    */
        /* not very robust.                                             */
        /* Note that the backing store base seems to be at a nice       */
        /* address minus one page.                                      */
#       define BACKING_STORE_DISPLACEMENT 0x1000000
#       define BACKING_STORE_ALIGNMENT 0x1000
        /* Known to be wrong for recent HP/UX versions!!!       */
#   endif
#   ifdef LINUX
#       define CPP_WORDSZ 64
        /* The following works on NUE and older kernels:        */
        /* define STACKBOTTOM ((ptr_t)0xa000000000000000l)      */
        /* TODO: LINUX_STACKBOTTOM does not work on NUE.        */
        /* We also need the base address of the register stack  */
        /* backing store.                                       */
#       define SEARCH_FOR_DATA_START
#       ifdef __GNUC__
#         define DYNAMIC_LOADING
#       else
          /* In the Intel compiler environment, we seem to end up with  */
          /* statically linked executables and an undefined reference   */
          /* to _DYNAMIC                                                */
#       endif
#       ifdef __GNUC__
#         ifndef __INTEL_COMPILER
#           define PREFETCH(x) \
              __asm__ ("        lfetch  [%0]": : "r"(x))
#           define GC_PREFETCH_FOR_WRITE(x) \
              __asm__ ("        lfetch.excl     [%0]": : "r"(x))
#           define CLEAR_DOUBLE(x) \
              __asm__ ("        stf.spill       [%0]=f0": : "r"((void *)(x)))
#         else
            EXTERN_C_END
#           include <ia64intrin.h>
            EXTERN_C_BEGIN
#           define PREFETCH(x) __lfetch(__lfhint_none, (x))
#           define GC_PREFETCH_FOR_WRITE(x) __lfetch(__lfhint_nta, (x))
#           define CLEAR_DOUBLE(x) __stf_spill((void *)(x), 0)
#         endif /* __INTEL_COMPILER */
#       endif
#   endif
#   ifdef MSWIN32
      /* FIXME: This is a very partial guess.  There is no port, yet.   */
#     if defined(_WIN64)
#       define CPP_WORDSZ 64
#     else
#       define CPP_WORDSZ 32   /* Is this possible?     */
#       define ALIGNMENT 8
#     endif
#   endif
# endif /* IA64 */

# ifdef E2K
#   define MACH_TYPE "E2K"
#   ifdef __LP64__
#     define CPP_WORDSZ 64
#   else
#     define CPP_WORDSZ 32
#   endif
#   ifndef HBLKSIZE
#     define HBLKSIZE 4096
#   endif
#   ifdef LINUX
      extern int __dso_handle[];
#     define DATASTART ((ptr_t)__dso_handle)
#     ifdef REDIRECT_MALLOC
#       define NO_PROC_FOR_LIBRARIES
#     endif
#   endif
# endif /* E2K */

# ifdef M88K
#   define MACH_TYPE "M88K"
#   define CPP_WORDSZ 32
#   define ALIGNMENT 4
#   define STACKBOTTOM ((char*)0xf0000000) /* determined empirically */
    extern int etext[];
#   ifdef CX_UX
#       define OS_TYPE "CX_UX"
#       define DATASTART (PTR_ALIGN_UP(etext, 0x400000) + 0x10000)
#   endif
#   ifdef DGUX
#       define OS_TYPE "DGUX"
        ptr_t GC_SysVGetDataStart(size_t, ptr_t);
#       define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)etext)
#       define DATASTART_IS_FUNC
#   endif
# endif /* M88K */

# ifdef S370
    /* If this still works, and if anyone cares, this should probably   */
    /* be moved to the S390 category.                                   */
#   define MACH_TYPE "S370"
#   define CPP_WORDSZ 32
#   define ALIGNMENT 4  /* Required by hardware */
#   ifdef UTS4
#       define OS_TYPE "UTS4"
        extern int _etext[];
        extern int _end[];
        ptr_t GC_SysVGetDataStart(size_t, ptr_t);
#       define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext)
#       define DATASTART_IS_FUNC
#       define DATAEND ((ptr_t)(_end))
#       define HEURISTIC2
#   endif
# endif /* S370 */

# ifdef S390
#   define MACH_TYPE "S390"
#   ifndef __s390x__
#     define CPP_WORDSZ 32
#   else
#     define CPP_WORDSZ 64
#     ifndef HBLKSIZE
#       define HBLKSIZE 4096
#     endif
#   endif
#   ifdef LINUX
        extern int __data_start[] __attribute__((__weak__));
#       define DATASTART ((ptr_t)(__data_start))
        extern int _end[] __attribute__((__weak__));
#       define DATAEND ((ptr_t)(_end))
#       define CACHE_LINE_SIZE 256
#       define GETPAGESIZE() 4096
#       ifndef SOFT_VDB
#         define SOFT_VDB
#       endif
#   endif
# endif /* S390 */

# ifdef AARCH64
#   define MACH_TYPE "AARCH64"
#   ifdef __ILP32__
#     define CPP_WORDSZ 32
#   else
#     define CPP_WORDSZ 64
#   endif
#   ifndef HBLKSIZE
#     define HBLKSIZE 4096
#   endif
#   ifdef LINUX
#     if defined(HOST_ANDROID)
#       define SEARCH_FOR_DATA_START
#     else
        extern int __data_start[] __attribute__((__weak__));
#       define DATASTART ((ptr_t)__data_start)
#     endif
#   endif
#   ifdef DARWIN
      /* OS X, iOS, visionOS */
#     define DARWIN_DONT_PARSE_STACK 1
#     define STACKBOTTOM ((ptr_t)0x16fdfffff)
#     if (TARGET_OS_IPHONE || TARGET_OS_XR || TARGET_OS_VISION)
        /* MPROTECT_VDB causes use of non-public API like exc_server,   */
        /* this could be a reason for blocking the client application   */
        /* in the store.                                                */
#     elif TARGET_OS_OSX
#       define MPROTECT_VDB
#     endif
#   endif
#   ifdef FREEBSD
      /* Nothing specific. */
#   endif
#   ifdef NETBSD
#     define ELF_CLASS ELFCLASS64
#   endif
#   ifdef OPENBSD
      /* Nothing specific. */
#   endif
#   ifdef NINTENDO_SWITCH
#     define OS_TYPE "NINTENDO_SWITCH"
      extern int __bss_end[];
#     define NO_HANDLE_FORK 1
#     define DATASTART (ptr_t)ALIGNMENT /* cannot be null */
#     define DATAEND (ptr_t)(&__bss_end)
      void *switch_get_stack_bottom(void);
#     define STACKBOTTOM ((ptr_t)switch_get_stack_bottom())
#     ifndef HAVE_CLOCK_GETTIME
#       define HAVE_CLOCK_GETTIME 1
#     endif
#   endif
#   ifdef KOS
      /* Nothing specific. */
#   endif
#   ifdef QNX
      /* Nothing specific. */
#   endif
#   ifdef MSWIN32   /* UWP */
      /* TODO: Enable MPROTECT_VDB */
#   endif
#   ifdef NOSYS
#     define OS_TYPE "NOSYS"
      /* __data_start is usually defined in the target linker script.   */
      extern int __data_start[];
#     define DATASTART ((ptr_t)__data_start)
      extern void *__stack_base__;
#     define STACKBOTTOM ((ptr_t)__stack_base__)
#   endif
# endif /* AARCH64 */

# ifdef ARM32
#   if defined(NACL)
#     define MACH_TYPE "NACL"
#   else
#     define MACH_TYPE "ARM32"
#   endif
#   define CPP_WORDSZ 32
#   ifdef LINUX
#       if GC_GLIBC_PREREQ(2, 0) || defined(HOST_ANDROID)
#           define SEARCH_FOR_DATA_START
#       else
            extern char **__environ;
#           define DATASTART ((ptr_t)(&__environ))
                              /* hideous kludge: __environ is the first */
                              /* word in crt0.o, and delimits the start */
                              /* of the data segment, no matter which   */
                              /* ld options were passed through.        */
                              /* We could use _etext instead, but that  */
                              /* would include .rodata, which may       */
                              /* contain large read-only data tables    */
                              /* that we'd rather not scan.             */
#       endif
#   endif
#   ifdef MSWINCE
      /* Nothing specific. */
#   endif
#   ifdef FREEBSD
      /* Nothing specific. */
#   endif
#   ifdef DARWIN
      /* iOS */
#     define DARWIN_DONT_PARSE_STACK 1
#     define STACKBOTTOM ((ptr_t)0x30000000)
      /* MPROTECT_VDB causes use of non-public API.     */
#   endif
#   ifdef NACL
      /* Nothing specific. */
#   endif
#   ifdef NETBSD
      /* Nothing specific. */
#   endif
#   ifdef OPENBSD
      /* Nothing specific. */
#   endif
#   ifdef QNX
      /* Nothing specific. */
#   endif
#   ifdef SN_TARGET_PSP2
#     define OS_TYPE "SN_TARGET_PSP2"
#     define NO_HANDLE_FORK 1
#     ifndef HBLKSIZE
#       define HBLKSIZE 65536 /* page size is 64 KB */
#     endif
#     define DATASTART (ptr_t)ALIGNMENT
#     define DATAEND (ptr_t)ALIGNMENT
      void *psp2_get_stack_bottom(void);
#     define STACKBOTTOM ((ptr_t)psp2_get_stack_bottom())
#   endif
#   ifdef NN_PLATFORM_CTR
#     define OS_TYPE "NN_PLATFORM_CTR"
      extern unsigned char Image$$ZI$$ZI$$Base[];
#     define DATASTART (ptr_t)(Image$$ZI$$ZI$$Base)
      extern unsigned char Image$$ZI$$ZI$$Limit[];
#     define DATAEND (ptr_t)(Image$$ZI$$ZI$$Limit)
      void *n3ds_get_stack_bottom(void);
#     define STACKBOTTOM ((ptr_t)n3ds_get_stack_bottom())
#   endif
#   ifdef MSWIN32   /* UWP */
      /* TODO: Enable MPROTECT_VDB */
#   endif
#   ifdef NOSYS
#     define OS_TYPE "NOSYS"
      /* __data_start is usually defined in the target linker script.  */
      extern int __data_start[];
#     define DATASTART ((ptr_t)(__data_start))
      /* __stack_base__ is set in newlib/libc/sys/arm/crt0.S  */
      extern void *__stack_base__;
#     define STACKBOTTOM ((ptr_t)(__stack_base__))
#   endif
#   ifdef SYMBIAN
      /* Nothing specific. */
#   endif
#endif /* ARM32 */

# ifdef CRIS
#   define MACH_TYPE "CRIS"
#   define CPP_WORDSZ 32
#   define ALIGNMENT 1
#   ifdef LINUX
#     define SEARCH_FOR_DATA_START
#   endif
# endif /* CRIS */

# if defined(SH) && !defined(SH4)
#   define MACH_TYPE "SH"
#   define CPP_WORDSZ 32
#   define ALIGNMENT 4
#   ifdef LINUX
#     define SEARCH_FOR_DATA_START
#   endif
#   ifdef NETBSD
      /* Nothing specific. */
#   endif
#   ifdef OPENBSD
      /* Nothing specific. */
#   endif
#   ifdef MSWINCE
      /* Nothing specific. */
#   endif
# endif

# ifdef SH4
#   define MACH_TYPE "SH4"
#   define CPP_WORDSZ 32
#   define ALIGNMENT 4
#   ifdef MSWINCE
      /* Nothing specific. */
#   endif
# endif /* SH4 */

# ifdef AVR32
#   define MACH_TYPE "AVR32"
#   define CPP_WORDSZ 32
#   ifdef LINUX
#     define SEARCH_FOR_DATA_START
#   endif
# endif /* AVR32 */

# ifdef M32R
#   define MACH_TYPE "M32R"
#   define CPP_WORDSZ 32
#   ifdef LINUX
#     define SEARCH_FOR_DATA_START
#   endif
# endif /* M32R */

# ifdef X86_64
#   define MACH_TYPE "X86_64"
#   ifdef __ILP32__
#     define CPP_WORDSZ 32
#   else
#     define CPP_WORDSZ 64
#   endif
#   ifndef HBLKSIZE
#     define HBLKSIZE 4096
#   endif
#   ifndef CACHE_LINE_SIZE
#     define CACHE_LINE_SIZE 64
#   endif
#   ifdef PLATFORM_GETMEM
#     define OS_TYPE "PLATFORM_GETMEM"
#     define DATASTART (ptr_t)ALIGNMENT
#     define DATAEND (ptr_t)ALIGNMENT
      EXTERN_C_END
#     include <pthread.h>
      EXTERN_C_BEGIN
      void *platform_get_stack_bottom(void);
#     define STACKBOTTOM ((ptr_t)platform_get_stack_bottom())
#   endif
#   ifdef LINUX
#       define SEARCH_FOR_DATA_START
#       if defined(__GLIBC__) && !defined(__UCLIBC__)
          /* A workaround for GCF (Google Cloud Function) which does    */
          /* not support mmap() for "/dev/zero".  Should not cause any  */
          /* harm to other targets.                                     */
#         define USE_MMAP_ANON
#       endif
#       if defined(__GLIBC__) && !defined(__UCLIBC__) \
           && !defined(GETCONTEXT_FPU_BUG_FIXED)
          /* At present, there's a bug in glibc getcontext() on         */
          /* Linux/x86_64 (it clears FPU exception mask).  We define    */
          /* this macro to workaround it.                               */
          /* TODO: This seems to be fixed in glibc 2.14.                */
#         define GETCONTEXT_FPU_EXCMASK_BUG
#       endif
#       if defined(__GLIBC__) && !defined(__UCLIBC__) \
           && !defined(GLIBC_TSX_BUG_FIXED)
          /* Workaround lock elision implementation for some glibc.     */
#         define GLIBC_2_19_TSX_BUG
          EXTERN_C_END
#         include <gnu/libc-version.h> /* for gnu_get_libc_version() */
          EXTERN_C_BEGIN
#       endif
#       ifndef SOFT_VDB
#         define SOFT_VDB
#       endif
#   endif
#   ifdef DARWIN
#     define DARWIN_DONT_PARSE_STACK 1
#     define STACKBOTTOM ((ptr_t)0x7fff5fc00000)
#     define MPROTECT_VDB
#   endif
#   ifdef FREEBSD
#       if defined(__GLIBC__)
            extern int _end[];
#           define DATAEND ((ptr_t)(_end))
#       endif
#       if defined(__DragonFly__)
            /* DragonFly BSD still has vm.max_proc_mmap, according to   */
            /* its mmap(2) man page.                                    */
#           define COUNT_UNMAPPED_REGIONS
#       endif
#   endif
#   ifdef NETBSD
      /* Nothing specific. */
#   endif
#   ifdef OPENBSD
      /* Nothing specific. */
#   endif
#   ifdef HAIKU
#     define HEURISTIC2
#     define SEARCH_FOR_DATA_START
#   endif
#   ifdef HURD
      /* Nothing specific. */
#   endif
#   ifdef QNX
      /* Nothing specific. */
#   endif
#   ifdef SOLARIS
#     define ELF_CLASS ELFCLASS64
#     define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)_etext)
#     ifdef SOLARIS25_PROC_VDB_BUG_FIXED
#       define PROC_VDB
#     endif
#   endif
#   ifdef CYGWIN32
#     ifndef USE_WINALLOC
#       if defined(THREAD_LOCAL_ALLOC)
            /* TODO: For an unknown reason, thread-local allocations    */
            /* lead to spurious process exit after the fault handler is */
            /* once invoked.                                            */
#       else
#         define MPROTECT_VDB
#       endif
#     endif
#   endif
#   ifdef MSWIN_XBOX1
#     define OS_TYPE "MSWIN_XBOX1"
#     define NO_GETENV
#     define DATASTART (ptr_t)ALIGNMENT
#     define DATAEND (ptr_t)ALIGNMENT
      LONG64 durango_get_stack_bottom(void);
#     define STACKBOTTOM ((ptr_t)durango_get_stack_bottom())
#     define GETPAGESIZE() 4096
#     ifndef USE_MMAP
#       define USE_MMAP 1
#     endif
      /* The following is from sys/mman.h:  */
#     define PROT_NONE  0
#     define PROT_READ  1
#     define PROT_WRITE 2
#     define PROT_EXEC  4
#     define MAP_PRIVATE 2
#     define MAP_FIXED  0x10
#     define MAP_FAILED ((void *)-1)
#   endif
#   ifdef MSWIN32
#       define RETRY_GET_THREAD_CONTEXT
#       if !defined(__GNUC__) || defined(__INTEL_COMPILER) \
           || (GC_GNUC_PREREQ(4, 7) && !defined(__MINGW64__))
          /* Older GCC and Mingw-w64 (both GCC and Clang) do not    */
          /* support SetUnhandledExceptionFilter() properly on x64. */
#         define MPROTECT_VDB
#       endif
#   endif
# endif /* X86_64 */

# ifdef ARC
#   define MACH_TYPE "ARC"
#   define CPP_WORDSZ 32
#   define CACHE_LINE_SIZE 64
#   ifdef LINUX
      extern int __data_start[] __attribute__((__weak__));
#     define DATASTART ((ptr_t)__data_start)
#   endif
# endif /* ARC */

# ifdef HEXAGON
#   define MACH_TYPE "HEXAGON"
#   define CPP_WORDSZ 32
#   ifdef LINUX
#     if defined(__GLIBC__)
#       define SEARCH_FOR_DATA_START
#     elif !defined(CPPCHECK)
#       error Unknown Hexagon libc configuration
#     endif
#   endif
# endif /* HEXAGON */

# ifdef TILEPRO
#   define MACH_TYPE "TILEPro"
#   define CPP_WORDSZ 32
#   define PREFETCH(x) __insn_prefetch(x)
#   define CACHE_LINE_SIZE 64
#   ifdef LINUX
      extern int __data_start[];
#     define DATASTART ((ptr_t)__data_start)
#   endif
# endif /* TILEPRO */

# ifdef TILEGX
#   define MACH_TYPE "TILE-Gx"
#   define CPP_WORDSZ (__SIZEOF_PTRDIFF_T__ * 8)
#   if CPP_WORDSZ == 32
#     define CLEAR_DOUBLE(x) (void)(*(long long *)(x) = 0)
#   endif
#   define PREFETCH(x) __insn_prefetch_l1(x)
#   define CACHE_LINE_SIZE 64
#   ifdef LINUX
      extern int __data_start[];
#     define DATASTART ((ptr_t)__data_start)
#   endif
# endif /* TILEGX */

# ifdef RISCV
#   define MACH_TYPE "RISC-V"
#   define CPP_WORDSZ __riscv_xlen /* 32 or 64 */
#   ifdef FREEBSD
      /* Nothing specific. */
#   endif
#   ifdef LINUX
      extern int __data_start[] __attribute__((__weak__));
#     define DATASTART ((ptr_t)__data_start)
#   endif
#   ifdef NETBSD
      /* Nothing specific. */
#   endif
#   ifdef OPENBSD
      /* Nothing specific. */
#   endif
# endif /* RISCV */

# ifdef WEBASSEMBLY
#   define MACH_TYPE "WebAssembly"
#   if defined(__wasm64__) && !defined(CPPCHECK)
#     error 64-bit WebAssembly is not yet supported
#   endif
#   define CPP_WORDSZ 32
#   define ALIGNMENT 4
#   ifdef EMSCRIPTEN
#     define OS_TYPE "EMSCRIPTEN"
#     define DATASTART (ptr_t)ALIGNMENT
#     define DATAEND (ptr_t)ALIGNMENT
      /* Emscripten does emulate mmap and munmap, but those should  */
      /* not be used in the collector, since WebAssembly lacks the  */
      /* native support of memory mapping.  Use sbrk() instead.     */
#     undef USE_MMAP
#     undef USE_MUNMAP
#     if defined(GC_THREADS) && !defined(CPPCHECK)
#       error No threads support yet
#     endif
#   endif
#   ifdef WASI
#     define OS_TYPE "WASI"
      extern char __global_base, __heap_base;
#     define STACKBOTTOM ((ptr_t)&__global_base)
#     define DATASTART   ((ptr_t)&__global_base)
#     define DATAEND     ((ptr_t)&__heap_base)
#     ifndef GC_NO_SIGSETJMP
#       define GC_NO_SIGSETJMP 1 /* no support of signals */
#     endif
#     ifndef NO_CLOCK
#       define NO_CLOCK 1 /* no support of clock */
#     endif
#     undef USE_MMAP /* similar to Emscripten */
#     undef USE_MUNMAP
#     if defined(GC_THREADS) && !defined(CPPCHECK)
#       error No threads support yet
#     endif
#   endif
# endif /* WEBASSEMBLY */

#if defined(__GLIBC__) && !defined(DONT_USE_LIBC_PRIVATES)
  /* Use glibc's stack-end marker. */
# define USE_LIBC_PRIVATES
#endif

#ifdef NO_RETRY_GET_THREAD_CONTEXT
# undef RETRY_GET_THREAD_CONTEXT
#endif

#if defined(LINUX_STACKBOTTOM) && defined(NO_PROC_STAT) \
    && !defined(USE_LIBC_PRIVATES)
    /* This combination will fail, since we have no way to get  */
    /* the stack bottom.  Use HEURISTIC2 instead.               */
#   undef LINUX_STACKBOTTOM
#   define HEURISTIC2
    /* This may still fail on some architectures like IA64.     */
    /* We tried ...                                             */
#endif

#if defined(USE_MMAP_ANON) && !defined(USE_MMAP)
#   define USE_MMAP 1
#elif (defined(LINUX) || defined(OPENBSD)) && defined(USE_MMAP)
    /* The kernel may do a somewhat better job merging mappings etc.    */
    /* with anonymous mappings.                                         */
#   define USE_MMAP_ANON
#endif

#if defined(GC_LINUX_THREADS) && defined(REDIRECT_MALLOC) \
    && !defined(USE_PROC_FOR_LIBRARIES) && !defined(NO_PROC_FOR_LIBRARIES)
    /* Nptl allocates thread stacks with mmap, which is fine.  But it   */
    /* keeps a cache of thread stacks.  Thread stacks contain the       */
    /* thread control blocks.  These in turn contain a pointer to       */
    /* (sizeof(void*) from the beginning of) the dtv for thread-local   */
    /* storage, which is calloc allocated.  If we don't scan the cached */
    /* thread stacks, we appear to lose the dtv.  This tends to         */
    /* result in something that looks like a bogus dtv count, which     */
    /* tends to result in a memset call on a block that is way too      */
    /* large.  Sometimes we're lucky and the process just dies ...      */
    /* There seems to be a similar issue with some other memory         */
    /* allocated by the dynamic loader.                                 */
    /* This should be avoidable by either:                              */
    /* - Defining USE_PROC_FOR_LIBRARIES here.                          */
    /*   That performs very poorly, precisely because we end up         */
    /*   scanning cached stacks.                                        */
    /* - Have calloc look at its callers.                               */
    /*   In spite of the fact that it is gross and disgusting.          */
    /* In fact neither seems to suffice, probably in part because       */
    /* even with USE_PROC_FOR_LIBRARIES, we don't scan parts of stack   */
    /* segments that appear to be out of bounds.  Thus we actually      */
    /* do both, which seems to yield the best results.                  */
#   define USE_PROC_FOR_LIBRARIES
#endif

#ifndef OS_TYPE
# define OS_TYPE ""
#endif

#ifndef DATAEND
# if !defined(CPPCHECK)
    extern int end[];
# endif
# define DATAEND ((ptr_t)(end))
#endif

/* Workaround for Android NDK clang 3.5+ (as of NDK r10e) which does    */
/* not provide correct _end symbol.  Unfortunately, alternate __end__   */
/* symbol is provided only by NDK "bfd" linker.                         */
#if defined(HOST_ANDROID) && defined(__clang__) \
    && !defined(BROKEN_UUENDUU_SYM)
# undef DATAEND
# pragma weak __end__
  extern int __end__[];
# define DATAEND (__end__ != 0 ? (ptr_t)__end__ : (ptr_t)_end)
#endif

#if defined(SOLARIS) || defined(DRSNX) || defined(UTS4)
        /* OS has SVR4 generic features.        */
        /* Probably others also qualify.        */
# define SVR4
#endif

#if defined(CYGWIN32) || defined(MSWIN32) || defined(MSWINCE)
# define ANY_MSWIN
#endif

#if defined(HAVE_SYS_TYPES_H) \
    || !(defined(AMIGA) || defined(MACOS) || defined(MSWINCE) \
         || defined(OS2) || defined(PCR) || defined(SN_TARGET_ORBIS) \
         || defined(SN_TARGET_PSP2) || defined(__CC_ARM))
  EXTERN_C_END
# include <sys/types.h>
  EXTERN_C_BEGIN
#endif /* HAVE_SYS_TYPES_H */

#if defined(HAVE_UNISTD_H) \
    || !(defined(AMIGA) || defined(MACOS) \
         || defined(MSWIN32) || defined(MSWINCE) || defined(MSWIN_XBOX1) \
         || defined(NINTENDO_SWITCH) || defined(NN_PLATFORM_CTR) \
         || defined(OS2) || defined(PCR) || defined(SN_TARGET_ORBIS) \
         || defined(SN_TARGET_PSP2) || defined(__CC_ARM))
  EXTERN_C_END
# include <unistd.h>
  EXTERN_C_BEGIN
#endif /* HAVE_UNISTD_H */

#if !defined(ANY_MSWIN) && !defined(GETPAGESIZE)
# if defined(DGUX) || defined(HOST_ANDROID) || defined(HOST_TIZEN) \
     || defined(KOS) || (defined(LINUX) && defined(SPARC))
#   define GETPAGESIZE() (unsigned)sysconf(_SC_PAGESIZE)
# else
#   define GETPAGESIZE() (unsigned)getpagesize()
# endif
#endif /* !ANY_MSWIN && !GETPAGESIZE */

#if defined(HOST_ANDROID) && !(__ANDROID_API__ >= 23) \
    && ((defined(MIPS) && (CPP_WORDSZ == 32)) \
        || defined(ARM32) || defined(I386) /* but not x32 */)
  /* tkill() exists only on arm32/mips(32)/x86. */
  /* NDK r11+ deprecates tkill() but keeps it for Mono clients. */
# define USE_TKILL_ON_ANDROID
#endif

#if defined(MPROTECT_VDB) && defined(__GLIBC__) && !GC_GLIBC_PREREQ(2, 2)
# error glibc too old?
#endif

#if defined(SOLARIS) || defined(DRSNX)
        /* OS has SOLARIS style semi-undocumented interface     */
        /* to dynamic loader.                                   */
# define SOLARISDL
        /* OS has SOLARIS style signal handlers.        */
# define SUNOS5SIGS
#endif

#if defined(HPUX)
# define SUNOS5SIGS
#endif

#if defined(FREEBSD) && (defined(__DragonFly__) || __FreeBSD__ >= 4 \
                         || __FreeBSD_kernel__ >= 4 || defined(__GLIBC__))
# define SUNOS5SIGS
#endif

#if defined(ANY_BSD) || defined(HAIKU) || defined(HURD) || defined(IRIX5) \
    || defined(OSF1) || defined(SUNOS5SIGS)
# define USE_SEGV_SIGACT
# if defined(IRIX5) && defined(_sigargs) /* Irix 5.x, not 6.x */ \
     || (defined(FREEBSD) && defined(SUNOS5SIGS)) \
     || defined(HPUX) || defined(HURD) || defined(NETBSD)
    /* We may get SIGBUS.       */
#   define USE_BUS_SIGACT
# endif
#endif

#if !defined(GC_EXPLICIT_SIGNALS_UNBLOCK) && defined(SUNOS5SIGS) \
    && !defined(GC_NO_PTHREAD_SIGMASK)
# define GC_EXPLICIT_SIGNALS_UNBLOCK
#endif

#if !defined(NO_SIGNALS_UNBLOCK_IN_MAIN) && defined(GC_NO_PTHREAD_SIGMASK)
# define NO_SIGNALS_UNBLOCK_IN_MAIN
#endif

#ifndef PARALLEL_MARK
# undef GC_PTHREADS_PARAMARK /* just in case it is defined by client */
#elif defined(GC_PTHREADS) && !defined(GC_PTHREADS_PARAMARK) \
      && !defined(__MINGW32__)
  /* Use pthread-based parallel mark implementation.    */
  /* Except for MinGW 32/64 to workaround a deadlock in */
  /* winpthreads-3.0b internals.                        */
# define GC_PTHREADS_PARAMARK
#endif

#if !defined(NO_MARKER_SPECIAL_SIGMASK) \
    && (defined(NACL) || defined(GC_WIN32_PTHREADS) \
        || (defined(GC_PTHREADS_PARAMARK) && defined(GC_WIN32_THREADS)) \
        || defined(GC_NO_PTHREAD_SIGMASK))
  /* Either there is no pthread_sigmask(), or GC marker thread cannot   */
  /* steal and drop user signal calls.                                  */
# define NO_MARKER_SPECIAL_SIGMASK
#endif

#ifdef GC_NETBSD_THREADS
# define SIGRTMIN 33
# define SIGRTMAX 63
  /* It seems to be necessary to wait until threads have restarted.     */
  /* But it is unclear why that is the case.                            */
# define GC_NETBSD_THREADS_WORKAROUND
#endif

#ifdef GC_OPENBSD_THREADS
  EXTERN_C_END
# include <sys/param.h>
  EXTERN_C_BEGIN
#endif /* GC_OPENBSD_THREADS */

#if defined(AIX) || defined(ANY_BSD) || defined(BSD) || defined(DARWIN) \
    || defined(DGUX) || defined(HAIKU) || defined(HPUX) || defined(HURD) \
    || defined(IRIX5) || defined(LINUX) || defined(OSF1) || defined(QNX) \
    || defined(SVR4)
# define UNIX_LIKE      /* Basic Unix-like system calls work.   */
#endif

#if defined(CPPCHECK)
# undef CPP_WORDSZ
# define CPP_WORDSZ (__SIZEOF_PTRDIFF_T__ * 8)
#elif CPP_WORDSZ != 32 && CPP_WORDSZ != 64
#   error Bad word size
#endif

#ifndef CPP_PTRSZ
# define CPP_PTRSZ CPP_WORDSZ
#endif

#ifndef CPPCHECK
# if GC_SIZEOF_PTR * 8 != CPP_PTRSZ
#   error Bad pointer size
# endif
#endif /* !CPPCHECK */

#ifndef ALIGNMENT
# if !defined(CPP_PTRSZ) && !defined(CPPCHECK)
#   error Undefined both ALIGNMENT and CPP_PTRSZ
# endif
# define ALIGNMENT (CPP_PTRSZ >> 3)
#endif /* !ALIGNMENT */

#ifdef PCR
# undef DYNAMIC_LOADING
# undef STACKBOTTOM
# undef HEURISTIC1
# undef HEURISTIC2
# undef PROC_VDB
# undef MPROTECT_VDB
# define PCR_VDB
#endif

#if !defined(STACKBOTTOM) && (defined(ECOS) || defined(NOSYS)) \
    && !defined(CPPCHECK)
# error Undefined STACKBOTTOM
#endif

#ifdef IGNORE_DYNAMIC_LOADING
# undef DYNAMIC_LOADING
#endif

#if defined(SMALL_CONFIG) && !defined(GC_DISABLE_INCREMENTAL)
  /* Presumably not worth the space it takes.   */
# define GC_DISABLE_INCREMENTAL
#endif

#if (defined(MSWIN32) || defined(MSWINCE)) && !defined(USE_WINALLOC)
  /* USE_WINALLOC is only an option for Cygwin. */
# define USE_WINALLOC 1
#endif

#ifdef USE_WINALLOC
# undef USE_MMAP
#endif

#if defined(ANY_BSD) || defined(DARWIN) || defined(HAIKU) \
    || defined(IRIX5) || defined(LINUX) || defined(SOLARIS) \
    || ((defined(CYGWIN32) || defined(USE_MMAP) || defined(USE_MUNMAP)) \
        && !defined(USE_WINALLOC))
  /* Try both sbrk and mmap, in that order.     */
# define MMAP_SUPPORTED
#endif

/* Xbox One (DURANGO) may not need to be this aggressive, but the       */
/* default is likely too lax under heavy allocation pressure.           */
/* The platform does not have a virtual paging system, so it does not   */
/* have a large virtual address space that a standard x64 platform has. */
#if defined(USE_MUNMAP) && !defined(MUNMAP_THRESHOLD) \
    && (defined(SN_TARGET_PS3) \
        || defined(SN_TARGET_PSP2) || defined(MSWIN_XBOX1))
# define MUNMAP_THRESHOLD 3
#endif

#if defined(GC_DISABLE_INCREMENTAL) || defined(DEFAULT_VDB)
# undef GWW_VDB
# undef MPROTECT_VDB
# undef PCR_VDB
# undef PROC_VDB
# undef SOFT_VDB
#endif

#ifdef NO_GWW_VDB
# undef GWW_VDB
#endif

#ifdef NO_MPROTECT_VDB
# undef MPROTECT_VDB
#endif

#ifdef NO_SOFT_VDB
# undef SOFT_VDB
#endif

#if defined(SOFT_VDB) && defined(SOFT_VDB_LINUX_VER_STATIC_CHECK)
  EXTERN_C_END
# include <linux/version.h> /* for LINUX_VERSION[_CODE] */
  EXTERN_C_BEGIN
# if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0)
    /* Not reliable in kernels prior to v3.18.  */
#   undef SOFT_VDB
# endif
#endif /* SOFT_VDB */

#ifdef GC_DISABLE_INCREMENTAL
# undef CHECKSUMS
#endif

#ifdef USE_GLOBAL_ALLOC
  /* Cannot pass MEM_WRITE_WATCH to GlobalAlloc().      */
# undef GWW_VDB
#endif

#if defined(BASE_ATOMIC_OPS_EMULATED)
  /* GC_write_fault_handler() cannot use lock-based atomic primitives   */
  /* as this could lead to a deadlock.                                  */
# undef MPROTECT_VDB
#endif

#if defined(USE_PROC_FOR_LIBRARIES) && defined(GC_LINUX_THREADS)
  /* Incremental GC based on mprotect is incompatible with /proc roots. */
# undef MPROTECT_VDB
#endif

#if defined(MPROTECT_VDB) && defined(GC_PREFER_MPROTECT_VDB)
  /* Choose MPROTECT_VDB manually (if multiple strategies available).   */
# undef PCR_VDB
# undef PROC_VDB
  /* GWW_VDB, SOFT_VDB are handled in os_dep.c. */
#endif

#ifdef PROC_VDB
  /* Mutually exclusive VDB implementations (for now).  */
# undef MPROTECT_VDB
  /* For a test purpose only.   */
# undef SOFT_VDB
#endif

#if defined(MPROTECT_VDB) && !defined(MSWIN32) && !defined(MSWINCE)
  EXTERN_C_END
# include <signal.h> /* for SA_SIGINFO, SIGBUS */
  EXTERN_C_BEGIN
#endif

#if defined(SIGBUS) && !defined(HAVE_SIGBUS) && !defined(CPPCHECK)
# define HAVE_SIGBUS
#endif

#ifndef SA_SIGINFO
# define NO_SA_SIGACTION
#endif

#if (defined(NO_SA_SIGACTION) || defined(GC_NO_SIGSETJMP)) \
    && defined(MPROTECT_VDB) && !defined(DARWIN) \
    && !defined(MSWIN32) && !defined(MSWINCE)
# undef MPROTECT_VDB
#endif

#if !defined(PCR_VDB) && !defined(PROC_VDB) && !defined(MPROTECT_VDB) \
    && !defined(GWW_VDB) && !defined(SOFT_VDB) && !defined(DEFAULT_VDB) \
    && !defined(GC_DISABLE_INCREMENTAL)
# define DEFAULT_VDB
#endif

#if defined(CHECK_SOFT_VDB) && !defined(CPPCHECK) \
    && (defined(GC_PREFER_MPROTECT_VDB) \
        || !defined(SOFT_VDB) || !defined(MPROTECT_VDB))
# error Invalid config for CHECK_SOFT_VDB
#endif

#if (defined(GC_DISABLE_INCREMENTAL) || defined(BASE_ATOMIC_OPS_EMULATED) \
     || defined(REDIRECT_MALLOC) || defined(SMALL_CONFIG) \
     || defined(REDIRECT_MALLOC_IN_HEADER) || defined(CHECKSUMS)) \
    && !defined(NO_MANUAL_VDB)
  /* TODO: Implement CHECKSUMS for manual VDB. */
# define NO_MANUAL_VDB
#endif

#if !defined(PROC_VDB) && !defined(SOFT_VDB) \
    && !defined(NO_VDB_FOR_STATIC_ROOTS)
  /* Cannot determine whether a static root page is dirty?      */
# define NO_VDB_FOR_STATIC_ROOTS
#endif

#if defined(MPROTECT_VDB) && !defined(DONT_COUNT_PROTECTED_REGIONS) \
    && !defined(COUNT_PROTECTED_REGIONS) \
    && (defined(LINUX) || defined(__DragonFly__))
# define COUNT_PROTECTED_REGIONS
#endif

#if (defined(COUNT_PROTECTED_REGIONS) || defined(COUNT_UNMAPPED_REGIONS)) \
    && !defined(GC_UNMAPPED_REGIONS_SOFT_LIMIT)
  /* The default limit of vm.max_map_count on Linux is ~65530.      */
  /* There is approximately one mapped region to every protected or */
  /* unmapped region.  Therefore if we aim to use up to half of     */
  /* vm.max_map_count for the GC (leaving half for the rest of the  */
  /* process) then the number of such regions should be one quarter */
  /* of vm.max_map_count.                                           */
# if defined(__DragonFly__)
#   define GC_UNMAPPED_REGIONS_SOFT_LIMIT (1000000 / 4)
# else
#   define GC_UNMAPPED_REGIONS_SOFT_LIMIT 16384
# endif
#endif

#if ((defined(UNIX_LIKE) && (defined(DARWIN) || defined(HAIKU) \
                             || defined(HURD) || defined(OPENBSD) \
                             || defined(QNX) || defined(ARM32) \
                             || defined(AVR32) || defined(MIPS) \
                             || defined(NIOS2) || defined(OR1K))) \
     || (defined(LINUX) && !defined(__gnu_linux__)) \
     || (defined(RTEMS) && defined(I386)) || defined(HOST_ANDROID)) \
    && !defined(NO_GETCONTEXT)
# define NO_GETCONTEXT 1
#endif

#if defined(MSWIN32) && !defined(CONSOLE_LOG) && defined(_MSC_VER) \
    && defined(_DEBUG) && !defined(NO_CRT)
  /* This should be included before intrin.h to workaround some bug */
  /* in Windows Kit (as of 10.0.17763) headers causing redefinition */
  /* of _malloca macro.                                             */
  EXTERN_C_END
# include <crtdbg.h> /* for _CrtDbgReport */
  EXTERN_C_BEGIN
#endif

#ifndef PREFETCH
# if (GC_GNUC_PREREQ(3, 0) || defined(__clang__)) && !defined(NO_PREFETCH)
#   define PREFETCH(x) __builtin_prefetch((x), 0, 0)
# elif defined(_MSC_VER) && !defined(NO_PREFETCH) \
       && (defined(_M_IX86) || defined(_M_X64)) && !defined(_CHPE_ONLY_) \
       && (_MSC_VER >= 1900) /* VS 2015+ */
    EXTERN_C_END
#   include <intrin.h>
    EXTERN_C_BEGIN
#   define PREFETCH(x) _mm_prefetch((const char *)(x), _MM_HINT_T0)
    /* TODO: Support also _M_ARM and _M_ARM64 (__prefetch).     */
# else
#   define PREFETCH(x) (void)0
# endif
#endif /* !PREFETCH */

#ifndef GC_PREFETCH_FOR_WRITE
  /* The default GC_PREFETCH_FOR_WRITE(x) is defined in gc_inline.h,    */
  /* the later one is included from gc_priv.h.                          */
#endif

#ifndef CACHE_LINE_SIZE
# define CACHE_LINE_SIZE 32     /* Wild guess   */
#endif

#ifndef STATIC
# ifdef GC_ASSERTIONS
#   define STATIC /* ignore to aid debugging (or profiling) */
# else
#   define STATIC static
# endif
#endif

#if defined(AMIGA) || defined(DOS4GW) || defined(EMBOX) || defined(KOS) \
    || defined(MACOS) || defined(NINTENDO_SWITCH) || defined(NONSTOP) \
    || defined(OS2) || defined(PCR) || defined(RTEMS) \
    || defined(SN_TARGET_ORBIS) || defined(SN_TARGET_PS3) \
    || defined(SN_TARGET_PSP2) || defined(USE_WINALLOC) || defined(__CC_ARM)
# define NO_UNIX_GET_MEM
#endif

/* Do we need the GC_find_limit machinery to find the end of    */
/* a data segment (or the backing store base)?                  */
#if defined(HEURISTIC2) || defined(SEARCH_FOR_DATA_START) \
    || defined(HPUX_MAIN_STACKBOTTOM) || defined(IA64) \
    || (defined(CYGWIN32) && defined(I386) && defined(USE_MMAP) \
        && !defined(USE_WINALLOC)) \
    || (defined(NETBSD) && defined(__ELF__)) || defined(OPENBSD) \
    || ((defined(SVR4) || defined(AIX) || defined(DGUX) \
         || defined(DATASTART_USES_BSDGETDATASTART)) && !defined(PCR))
# define NEED_FIND_LIMIT
#endif

#if defined(LINUX) && (defined(USE_PROC_FOR_LIBRARIES) || defined(IA64) \
                       || !defined(SMALL_CONFIG))
# define NEED_PROC_MAPS
#endif

#if defined(LINUX) || defined(HURD) || defined(__GLIBC__)
# define REGISTER_LIBRARIES_EARLY
  /* We sometimes use dl_iterate_phdr, which may acquire an internal    */
  /* lock.  This isn't safe after the world has stopped.  So we must    */
  /* call GC_register_dynamic_libraries before stopping the world.      */
  /* For performance reasons, this may be beneficial on other           */
  /* platforms as well, though it should be avoided on Windows.         */
#endif /* LINUX */

#if defined(SEARCH_FOR_DATA_START)
  extern ptr_t GC_data_start;
# define DATASTART GC_data_start
#endif

#ifndef HEAP_START
# define HEAP_START ((ptr_t)0)
#endif

#ifndef CLEAR_DOUBLE
# define CLEAR_DOUBLE(x) \
                (void)(((ptr_t *)(x))[0] = NULL, ((ptr_t *)(x))[1] = NULL)
#endif

/* Some libc implementations like bionic, musl and glibc 2.34   */
/* do not have libpthread.so because the pthreads-related code  */
/* is located in libc.so, thus potential calloc calls from such */
/* code are forwarded to real (libc) calloc without any special */
/* handling on the libgc side.  Checking glibc version at       */
/* compile time for the purpose seems to be fine.               */
#if defined(GC_LINUX_THREADS) && defined(REDIRECT_MALLOC) \
    && defined(__GLIBC__) && !GC_GLIBC_PREREQ(2, 34) \
    && !defined(HAVE_LIBPTHREAD_SO)
# define HAVE_LIBPTHREAD_SO
#endif

#if defined(GC_LINUX_THREADS) && defined(REDIRECT_MALLOC) \
    && !defined(INCLUDE_LINUX_THREAD_DESCR)
  /* Will not work, since libc and the dynamic loader use thread        */
  /* locals, sometimes as the only reference.                           */
# define INCLUDE_LINUX_THREAD_DESCR
#endif

#if !defined(CPPCHECK)
# if defined(GC_IRIX_THREADS) && !defined(IRIX5)
#   error Inconsistent configuration
# endif
# if defined(GC_LINUX_THREADS) && !defined(LINUX) && !defined(NACL)
#   error Inconsistent configuration
# endif
# if defined(GC_NETBSD_THREADS) && !defined(NETBSD)
#   error Inconsistent configuration
# endif
# if defined(GC_FREEBSD_THREADS) && !defined(FREEBSD)
#   error Inconsistent configuration
# endif
# if defined(GC_SOLARIS_THREADS) && !defined(SOLARIS)
#   error Inconsistent configuration
# endif
# if defined(GC_HPUX_THREADS) && !defined(HPUX)
#   error Inconsistent configuration
# endif
# if defined(GC_AIX_THREADS) && !defined(_AIX)
#   error Inconsistent configuration
# endif
# if defined(GC_WIN32_THREADS) && !defined(ANY_MSWIN) && !defined(MSWIN_XBOX1)
#   error Inconsistent configuration
# endif
# if defined(GC_WIN32_PTHREADS) && defined(CYGWIN32)
#   error Inconsistent configuration
# endif
#endif /* !CPPCHECK */

#if defined(PCR) || defined(GC_WIN32_THREADS) || defined(GC_PTHREADS) \
    || ((defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH) \
         || defined(SN_TARGET_PS3) \
         || defined(SN_TARGET_PSP2)) && defined(GC_THREADS))
# define THREADS
#endif

#if defined(PARALLEL_MARK) && !defined(THREADS) && !defined(CPPCHECK)
# error Invalid config: PARALLEL_MARK requires GC_THREADS
#endif

#if defined(GWW_VDB) && !defined(USE_WINALLOC) && !defined(CPPCHECK)
# error Invalid config: GWW_VDB requires USE_WINALLOC
#endif

/* Whether GC_page_size is to be set to a value other than page size.   */
#if defined(CYGWIN32) && (defined(MPROTECT_VDB) || defined(USE_MUNMAP)) \
    || (!defined(ANY_MSWIN) && !defined(WASI) && !defined(USE_MMAP) \
        && (defined(GC_DISABLE_INCREMENTAL) || defined(DEFAULT_VDB)))
  /* Cygwin: use the allocation granularity instead.  Other than WASI   */
  /* or Windows: use HBLKSIZE instead (unless mmap() is used).          */
# define ALT_PAGESIZE_USED
# ifndef GC_NO_VALLOC
    /* Nonetheless, we need the real page size is some extra functions. */
#   define REAL_PAGESIZE_NEEDED
# endif
#endif

#if defined(GC_PTHREADS) && !defined(GC_DARWIN_THREADS) \
    && !defined(GC_WIN32_THREADS) && !defined(PLATFORM_STOP_WORLD) \
    && !defined(SN_TARGET_PSP2)
# define PTHREAD_STOP_WORLD_IMPL
#endif

#if defined(PTHREAD_STOP_WORLD_IMPL) && !defined(NACL)
# define SIGNAL_BASED_STOP_WORLD
#endif

#if (defined(E2K) || defined(HP_PA) || defined(IA64) || defined(M68K) \
     || defined(NO_SA_SIGACTION)) && defined(SIGNAL_BASED_STOP_WORLD)
# define SUSPEND_HANDLER_NO_CONTEXT
#endif

#if (defined(MSWIN32) || defined(MSWINCE) \
        || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS))) \
    && !defined(NO_CRT) && !defined(NO_WRAP_MARK_SOME)
  /* Under rare conditions, we may end up marking from nonexistent      */
  /* memory.  Hence we need to be prepared to recover by running        */
  /* GC_mark_some with a suitable handler in place.                     */
  /* TODO: Should we also define it for Cygwin?                         */
# define WRAP_MARK_SOME
#endif

#if !defined(MSWIN32) && !defined(MSWINCE) || defined(__GNUC__) \
    || defined(NO_CRT)
# define NO_SEH_AVAILABLE
#endif

#ifdef GC_WIN32_THREADS
  /* The number of copied registers in copy_ptr_regs.   */
# if defined(I386)
#   ifdef WOW64_THREAD_CONTEXT_WORKAROUND
#     define PUSHED_REGS_COUNT 9
#   else
#     define PUSHED_REGS_COUNT 7
#   endif
# elif defined(X86_64)
#   ifdef XMM_CANT_STORE_PTRS
      /* If pointers can't be located in Xmm registers. */
#     define PUSHED_REGS_COUNT 15
#   else
      /* gcc-13 may store pointers into SIMD registers when     */
      /* certain compiler optimizations are enabled.            */
#     define PUSHED_REGS_COUNT (15+32)
#   endif
# elif defined(SHx)
#   define PUSHED_REGS_COUNT 15
# elif defined(ARM32)
#   define PUSHED_REGS_COUNT 13
# elif defined(AARCH64)
#   define PUSHED_REGS_COUNT 30
# elif defined(MIPS) || defined(ALPHA)
#   define PUSHED_REGS_COUNT 28
# elif defined(PPC)
#   define PUSHED_REGS_COUNT 29
# endif
#endif /* GC_WIN32_THREADS */

#if !defined(GC_PTHREADS) && !defined(GC_PTHREADS_PARAMARK)
# undef HAVE_PTHREAD_SETNAME_NP_WITH_TID
# undef HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG
# undef HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID
# undef HAVE_PTHREAD_SET_NAME_NP
#endif

#ifdef USE_RWLOCK
  /* At least in the Linux threads implementation, rwlock primitives    */
  /* are not atomic in respect to signals, and suspending externally    */
  /* a thread which is running inside pthread_rwlock_rdlock() may lead  */
  /* to a deadlock.                                                     */
  /* TODO: As a workaround GC_suspend_thread() API is disabled.         */
# undef GC_ENABLE_SUSPEND_THREAD
#endif

#ifndef GC_NO_THREADS_DISCOVERY
# ifdef GC_DARWIN_THREADS
    /* Task-based thread registration requires stack-frame-walking code. */
#   if defined(DARWIN_DONT_PARSE_STACK)
#     define GC_NO_THREADS_DISCOVERY
#   endif
# elif defined(GC_WIN32_THREADS)
    /* DllMain-based thread registration is currently incompatible      */
    /* with thread-local allocation, pthreads and WinCE.                */
#   if (!defined(GC_DLL) && !defined(GC_INSIDE_DLL)) || defined(GC_PTHREADS) \
       || defined(MSWINCE) || defined(NO_CRT) || defined(THREAD_LOCAL_ALLOC)
#     define GC_NO_THREADS_DISCOVERY
#   endif
# else
#   define GC_NO_THREADS_DISCOVERY
# endif
#endif /* !GC_NO_THREADS_DISCOVERY */

#if defined(GC_DISCOVER_TASK_THREADS) && defined(GC_NO_THREADS_DISCOVERY) \
    && !defined(CPPCHECK)
# error Defined both GC_DISCOVER_TASK_THREADS and GC_NO_THREADS_DISCOVERY
#endif

#if defined(PARALLEL_MARK) && !defined(DEFAULT_STACK_MAYBE_SMALL) \
    && (defined(HPUX) || defined(GC_DGUX386_THREADS) \
        || defined(NO_GETCONTEXT) /* e.g. musl */)
    /* TODO: Test default stack size in configure. */
# define DEFAULT_STACK_MAYBE_SMALL
#endif

#ifdef PARALLEL_MARK
  /* The minimum stack size for a marker thread. */
# define MIN_STACK_SIZE (8 * HBLKSIZE * sizeof(ptr_t))
#endif

#if defined(HOST_ANDROID) && !defined(THREADS) \
    && !defined(USE_GET_STACKBASE_FOR_MAIN)
  /* Always use pthread_attr_getstack on Android ("-lpthread" option is  */
  /* not needed to be specified manually) since GC_linux_main_stack_base */
  /* causes app crash if invoked inside Dalvik VM.                       */
# define USE_GET_STACKBASE_FOR_MAIN
#endif

/* Outline pthread primitives to use in GC_get_[main_]stack_base.       */
#if ((defined(FREEBSD) && defined(__GLIBC__)) /* kFreeBSD */ \
     || defined(LINUX) || defined(KOS) || defined(NETBSD)) \
    && !defined(NO_PTHREAD_GETATTR_NP)
# define HAVE_PTHREAD_GETATTR_NP 1
#elif defined(FREEBSD) && !defined(__GLIBC__) \
      && !defined(NO_PTHREAD_ATTR_GET_NP)
# define HAVE_PTHREAD_NP_H 1 /* requires include pthread_np.h */
# define HAVE_PTHREAD_ATTR_GET_NP 1
#endif

#if !defined(HAVE_CLOCK_GETTIME) && defined(_POSIX_TIMERS) \
    && (defined(CYGWIN32) || (defined(LINUX) && defined(__USE_POSIX199309)))
# define HAVE_CLOCK_GETTIME 1
#endif

#if defined(GC_PTHREADS) && !defined(E2K) && !defined(IA64) \
    && (!defined(DARWIN) || defined(DARWIN_DONT_PARSE_STACK)) \
    && !defined(SN_TARGET_PSP2) && !defined(REDIRECT_MALLOC)
  /* Note: unimplemented in case of redirection of malloc() because     */
  /* the client-provided function might call some pthreads primitive    */
  /* which, in turn, may use malloc() internally.                       */
# define STACKPTR_CORRECTOR_AVAILABLE
#endif

#if defined(UNIX_LIKE) && defined(THREADS) && !defined(NO_CANCEL_SAFE) \
    && !defined(HOST_ANDROID)
  /* Make the code cancellation-safe.  This basically means that we     */
  /* ensure that cancellation requests are ignored while we are in      */
  /* the collector.  This applies only to Posix deferred cancellation;  */
  /* we don't handle Posix asynchronous cancellation.                   */
  /* Note that this only works if pthread_setcancelstate is             */
  /* async-signal-safe, at least in the absence of asynchronous         */
  /* cancellation.  This appears to be true for the glibc version,      */
  /* though it is not documented.  Without that assumption, there       */
  /* seems to be no way to safely wait in a signal handler, which       */
  /* we need to do for thread suspension.                               */
  /* Also note that little other code appears to be cancellation-safe.  */
  /* Hence it may make sense to turn this off for performance.          */
# define CANCEL_SAFE
#endif

#ifdef CANCEL_SAFE
# define IF_CANCEL(x) x
#else
# define IF_CANCEL(x) /* empty */
#endif

#if !defined(CAN_HANDLE_FORK) && !defined(NO_HANDLE_FORK) \
    && !defined(HAVE_NO_FORK) \
    && ((defined(GC_PTHREADS) && !defined(NACL) \
         && !defined(GC_WIN32_PTHREADS) && !defined(USE_WINALLOC)) \
        || (defined(DARWIN) && defined(MPROTECT_VDB) /* && !THREADS */) \
        || (defined(HANDLE_FORK) && defined(GC_PTHREADS)))
  /* Attempts (where supported and requested) to make GC_malloc work in */
  /* a child process fork'ed from a multi-threaded parent.              */
# define CAN_HANDLE_FORK
#endif

/* Workaround "failed to create new win32 semaphore" Cygwin fatal error */
/* during semaphores fixup-after-fork.                                  */
#if defined(CYGWIN32) && defined(GC_WIN32_THREADS) \
    && defined(CAN_HANDLE_FORK) && !defined(EMULATE_PTHREAD_SEMAPHORE) \
    && !defined(CYGWIN_SEM_FIXUP_AFTER_FORK_BUG_FIXED)
# define EMULATE_PTHREAD_SEMAPHORE
#endif

#if defined(CAN_HANDLE_FORK) && !defined(CAN_CALL_ATFORK) \
    && !defined(GC_NO_CAN_CALL_ATFORK) && !defined(HOST_TIZEN) \
    && !defined(HURD) && (!defined(HOST_ANDROID) || __ANDROID_API__ >= 21)
  /* Have working pthread_atfork().     */
# define CAN_CALL_ATFORK
#endif

#if !defined(CAN_HANDLE_FORK) && !defined(HAVE_NO_FORK) \
    && !(defined(CYGWIN32) || defined(SOLARIS) || defined(UNIX_LIKE))
# define HAVE_NO_FORK
#endif

#if !defined(USE_MARK_BITS) && !defined(USE_MARK_BYTES) \
    && defined(PARALLEL_MARK)
  /* Minimize compare-and-swap usage.   */
# define USE_MARK_BYTES
#endif

#if (defined(MSWINCE) && !defined(__CEGCC__) || defined(MSWINRT_FLAVOR)) \
    && !defined(NO_GETENV)
# define NO_GETENV
#endif

#if (defined(NO_GETENV) || defined(MSWINCE)) && !defined(NO_GETENV_WIN32)
# define NO_GETENV_WIN32
#endif

#if !defined(MSGBOX_ON_ERROR) && !defined(NO_MSGBOX_ON_ERROR) \
    && !defined(SMALL_CONFIG) && defined(MSWIN32) \
    && !defined(MSWINRT_FLAVOR) && !defined(MSWIN_XBOX1)
  /* Show a Windows message box with "OK" button on a GC fatal error.   */
  /* Client application is terminated once the user clicks the button.  */
# define MSGBOX_ON_ERROR
#endif

#ifndef STRTOULL
# if defined(_WIN64) && !defined(__GNUC__)
#   define STRTOULL _strtoui64
# elif defined(_LLP64) || defined(__LLP64__) || defined(_WIN64)
#   define STRTOULL strtoull
# else
    /* strtoul() fits since sizeof(long) >= sizeof(word).       */
#   define STRTOULL strtoul
# endif
#endif /* !STRTOULL */

#ifndef GC_WORD_C
# if defined(_WIN64) && !defined(__GNUC__)
#   define GC_WORD_C(val) val##ui64
# elif defined(_LLP64) || defined(__LLP64__) || defined(_WIN64)
#   define GC_WORD_C(val) val##ULL
# else
#   define GC_WORD_C(val) ((word)val##UL)
# endif
#endif /* !GC_WORD_C */

#if defined(__has_feature)
  /* __has_feature() is supported.      */
# if __has_feature(address_sanitizer)
#   define ADDRESS_SANITIZER
# endif
# if __has_feature(memory_sanitizer)
#   define MEMORY_SANITIZER
# endif
# if __has_feature(thread_sanitizer) && defined(THREADS)
#   define THREAD_SANITIZER
# endif
#else
# ifdef __SANITIZE_ADDRESS__
    /* GCC v4.8+ */
#   define ADDRESS_SANITIZER
# endif
# if defined(__SANITIZE_THREAD__) && defined(THREADS)
    /* GCC v7.1+ */
#   define THREAD_SANITIZER
# endif
#endif /* !__has_feature */

#if defined(SPARC)
# define ASM_CLEAR_CODE /* Stack clearing is crucial, and we    */
                        /* include assembly code to do it well. */
#endif

/* Can we save call chain in objects for debugging?  Set NFRAMES        */
/* (number of saved frames) and NARGS (number of arguments for each     */
/* frame) to reasonable values for the platform.                        */
/* Define SAVE_CALL_CHAIN if we can.  SAVE_CALL_COUNT can be specified  */
/* at build time, though we feel free to adjust it slightly.            */
/* Define NEED_CALLINFO if we either save the call stack or             */
/* GC_ADD_CALLER is defined.  Note: GC_CAN_SAVE_CALL_STACKS is defined  */
/* (for certain platforms) in gc_config_macros.h file.                  */
#if defined(SPARC) \
    || ((defined(I386) || defined(X86_64)) \
        && (defined(LINUX) || defined(__GLIBC__)))
  /* Linux/x86: SAVE_CALL_CHAIN is supported if the code is compiled to */
  /* save frame pointers by default, i.e. no -fomit-frame-pointer flag. */
# define CAN_SAVE_CALL_ARGS
#endif

#if defined(SAVE_CALL_COUNT) && !defined(GC_ADD_CALLER) \
    && defined(GC_CAN_SAVE_CALL_STACKS)
# define SAVE_CALL_CHAIN
#endif

#ifdef SAVE_CALL_CHAIN
# if defined(SAVE_CALL_NARGS) && defined(CAN_SAVE_CALL_ARGS)
#   define NARGS SAVE_CALL_NARGS
# else
#   define NARGS 0      /* Number of arguments to save for each call.   */
# endif
# if !defined(SAVE_CALL_COUNT) || defined(CPPCHECK)
#   define NFRAMES 6    /* Number of frames to save. Even for   */
                        /* alignment reasons.                   */
# else
#   define NFRAMES ((SAVE_CALL_COUNT + 1) & ~1)
# endif
# define NEED_CALLINFO
#elif defined(GC_ADD_CALLER)
# define NFRAMES 1
# define NARGS 0
# define NEED_CALLINFO
#endif

#if (defined(FREEBSD) || (defined(DARWIN) && !defined(_POSIX_C_SOURCE)) \
        || (defined(SOLARIS) && (!defined(_XOPEN_SOURCE) \
                                 || defined(__EXTENSIONS__))) \
        || defined(LINUX)) && !defined(HAVE_DLADDR)
# define HAVE_DLADDR 1
#endif

#if defined(MAKE_BACK_GRAPH) && !defined(DBG_HDRS_ALL)
# define DBG_HDRS_ALL 1
#endif

#if defined(POINTER_MASK) && !defined(POINTER_SHIFT)
# define POINTER_SHIFT 0
#elif !defined(POINTER_MASK) && defined(POINTER_SHIFT)
# define POINTER_MASK GC_WORD_MAX
#endif

#if defined(FIXUP_POINTER)
  /* Custom FIXUP_POINTER(p).   */
# define NEED_FIXUP_POINTER
#elif defined(DYNAMIC_POINTER_MASK)
# define FIXUP_POINTER(p) \
            (p = (ptr_t)(((word)(p) & GC_pointer_mask) << GC_pointer_shift))
# undef POINTER_MASK
# undef POINTER_SHIFT
# define NEED_FIXUP_POINTER
#elif defined(POINTER_MASK)
# define FIXUP_POINTER(p) \
            (p = (ptr_t)(((word)(p) & (POINTER_MASK)) << (POINTER_SHIFT)))
                            /* Extra parentheses around custom-defined  */
                            /* POINTER_MASK/SHIFT are intentional.      */
# define NEED_FIXUP_POINTER
#else
# define FIXUP_POINTER(p) (void)(p)
#endif

#ifdef LINT2
  /* A macro (based on a tricky expression) to prevent false warnings   */
  /* like "Array compared to 0", "Comparison of identical expressions", */
  /* "Untrusted loop bound" output by some static code analysis tools.  */
  /* The argument should not be a literal value.  The result is         */
  /* converted to word type.  (Actually, GC_word is used instead of     */
  /* word type as the latter might be undefined at the place of use.)   */
# define COVERT_DATAFLOW(w) (~(GC_word)(w)^(~(GC_word)0))
#else
# define COVERT_DATAFLOW(w) ((GC_word)(w))
#endif

#if CPP_PTRSZ > CPP_WORDSZ
  /* TODO: Cannot use tricky operations on a pointer.   */
# define COVERT_DATAFLOW_P(p) ((ptr_t)(p))
#else
# define COVERT_DATAFLOW_P(p) ((ptr_t)COVERT_DATAFLOW(p))
#endif

#if defined(REDIRECT_MALLOC) && defined(THREADS) && !defined(LINUX) \
    && !defined(REDIRECT_MALLOC_IN_HEADER)
    /* May work on other platforms (e.g. Darwin) provided the client    */
    /* ensures all the client threads are registered with the GC,       */
    /* e.g. by using the preprocessor-based interception of the thread  */
    /* primitives (i.e., define GC_THREADS and include gc.h from all    */
    /* the client files those are using pthread_create and friends).    */
#endif

#ifdef GC_PRIVATE_H
        /* This relies on some type definitions from gc_priv.h, from    */
        /* where it's normally included.                                */
        /*                                                              */
        /* How to get heap memory from the OS:                          */
        /* Note that sbrk()-like allocation is preferred, since it      */
        /* usually makes it possible to merge consecutively allocated   */
        /* chunks.  It also avoids unintended recursion with            */
        /* REDIRECT_MALLOC macro defined.                               */
        /* GET_MEM() argument should be of size_t type and have         */
        /* no side-effect.  GET_MEM() returns HBLKSIZE-aligned chunk;   */
        /* 0 is taken to mean failure.                                  */
        /* In case of MMAP_SUPPORTED, the argument must also be         */
        /* a multiple of a physical page size.                          */
        /* GET_MEM is currently not assumed to retrieve 0 filled space, */
        /* though we should perhaps take advantage of the case in which */
        /* does.                                                        */
# define hblk GC_hblk_s
  struct hblk;  /* See gc_priv.h. */
# if defined(PCR)
    char * real_malloc(size_t bytes);
#   define GET_MEM(bytes) HBLKPTR(real_malloc(SIZET_SAT_ADD(bytes, \
                                                            GC_page_size)) \
                                          + GC_page_size-1)
# elif defined(OS2)
    void * os2_alloc(size_t bytes);
#   define GET_MEM(bytes) HBLKPTR((ptr_t)os2_alloc( \
                                            SIZET_SAT_ADD(bytes, \
                                                          GC_page_size)) \
                                  + GC_page_size-1)
# elif defined(NEXT) || defined(DOS4GW) || defined(NONSTOP) \
        || (defined(AMIGA) && !defined(GC_AMIGA_FASTALLOC)) \
        || (defined(SOLARIS) && !defined(USE_MMAP)) || defined(RTEMS) \
        || defined(EMBOX) || defined(KOS) || defined(__CC_ARM)
    /* TODO: Use page_alloc() directly on Embox.    */
#   if defined(REDIRECT_MALLOC) && !defined(CPPCHECK)
#     error Malloc redirection is unsupported
#   endif
#   define GET_MEM(bytes) HBLKPTR((size_t)calloc(1, \
                                            SIZET_SAT_ADD(bytes, \
                                                          GC_page_size)) \
                                  + GC_page_size - 1)
# elif defined(MSWIN_XBOX1)
    ptr_t GC_durango_get_mem(size_t bytes);
#   define GET_MEM(bytes) (struct hblk *)GC_durango_get_mem(bytes)
# elif defined(MSWIN32) || defined(CYGWIN32)
    ptr_t GC_win32_get_mem(size_t bytes);
#   define GET_MEM(bytes) (struct hblk *)GC_win32_get_mem(bytes)
# elif defined(MACOS)
#   if defined(USE_TEMPORARY_MEMORY)
      Ptr GC_MacTemporaryNewPtr(size_t size, Boolean clearMemory);
#     define GET_MEM(bytes) HBLKPTR(GC_MacTemporaryNewPtr( \
                                        SIZET_SAT_ADD(bytes, \
                                                      GC_page_size), true) \
                        + GC_page_size-1)
#   else
#     define GET_MEM(bytes) HBLKPTR(NewPtrClear(SIZET_SAT_ADD(bytes, \
                                                              GC_page_size)) \
                                    + GC_page_size-1)
#   endif
# elif defined(MSWINCE)
    ptr_t GC_wince_get_mem(size_t bytes);
#   define GET_MEM(bytes) (struct hblk *)GC_wince_get_mem(bytes)
# elif defined(AMIGA) && defined(GC_AMIGA_FASTALLOC)
    void *GC_amiga_get_mem(size_t bytes);
#   define GET_MEM(bytes) HBLKPTR((size_t)GC_amiga_get_mem( \
                                            SIZET_SAT_ADD(bytes, \
                                                          GC_page_size)) \
                          + GC_page_size-1)
# elif defined(PLATFORM_GETMEM)
    void *platform_get_mem(size_t bytes);
#   define GET_MEM(bytes) (struct hblk*)platform_get_mem(bytes)
# elif defined(SN_TARGET_PS3)
    void *ps3_get_mem(size_t bytes);
#   define GET_MEM(bytes) (struct hblk*)ps3_get_mem(bytes)
# elif defined(SN_TARGET_PSP2)
    void *psp2_get_mem(size_t bytes);
#   define GET_MEM(bytes) (struct hblk*)psp2_get_mem(bytes)
# elif defined(NINTENDO_SWITCH)
    void *switch_get_mem(size_t bytes);
#   define GET_MEM(bytes) (struct hblk*)switch_get_mem(bytes)
# elif defined(HAIKU)
    ptr_t GC_haiku_get_mem(size_t bytes);
#   define GET_MEM(bytes) (struct hblk*)GC_haiku_get_mem(bytes)
# elif defined(EMSCRIPTEN_TINY)
    void *emmalloc_memalign(size_t alignment, size_t size);
#   define GET_MEM(bytes) (struct hblk*)emmalloc_memalign(GC_page_size, bytes)
# else
    ptr_t GC_unix_get_mem(size_t bytes);
#   define GET_MEM(bytes) (struct hblk *)GC_unix_get_mem(bytes)
# endif
#endif /* GC_PRIVATE_H */

EXTERN_C_END

#endif /* GCCONFIG_H */


#if !defined(GC_ATOMIC_UNCOLLECTABLE) && defined(ATOMIC_UNCOLLECTABLE)
  /* For compatibility with old-style naming. */
# define GC_ATOMIC_UNCOLLECTABLE
#endif

#ifndef GC_INNER
  /* This tagging macro must be used at the start of every variable     */
  /* definition which is declared with GC_EXTERN.  Should be also used  */
  /* for the GC-scope function definitions and prototypes.  Must not be */
  /* used in gcconfig.h.  Shouldn't be used for the debugging-only      */
  /* functions.  Currently, not used for the functions declared in or   */
  /* called from the "dated" source files (located in "extra" folder).  */
# if defined(GC_DLL) && defined(__GNUC__) && !defined(ANY_MSWIN)
#   if GC_GNUC_PREREQ(4, 0) && !defined(GC_NO_VISIBILITY)
      /* See the corresponding GC_API definition. */
#     define GC_INNER __attribute__((__visibility__("hidden")))
#   else
      /* The attribute is unsupported. */
#     define GC_INNER /* empty */
#   endif
# else
#   define GC_INNER /* empty */
# endif

# define GC_EXTERN extern GC_INNER
  /* Used only for the GC-scope variables (prefixed with "GC_")         */
  /* declared in the header files.  Must not be used for thread-local   */
  /* variables.  Must not be used in gcconfig.h.  Shouldn't be used for */
  /* the debugging-only or profiling-only variables.  Currently, not    */
  /* used for the variables accessed from the "dated" source files      */
  /* (specific.c/h, and in the "extra" folder).                         */
  /* The corresponding variable definition must start with GC_INNER.    */
#endif /* !GC_INNER */

#ifdef __cplusplus
  /* Register storage specifier is deprecated in C++11. */
# define REGISTER /* empty */
#else
  /* Used only for several local variables in the performance-critical  */
  /* functions.  Should not be used for new code.                       */
# define REGISTER register
#endif

#if defined(CPPCHECK)
# define MACRO_BLKSTMT_BEGIN {
# define MACRO_BLKSTMT_END   }
# define LOCAL_VAR_INIT_OK =0 /* to avoid "uninit var" false positive */
#else
# define MACRO_BLKSTMT_BEGIN do {
# define MACRO_BLKSTMT_END   } while (0)
# define LOCAL_VAR_INIT_OK /* empty */
#endif

#if defined(M68K) && defined(__GNUC__)
  /* By default, __alignof__(void*) is 2 on m68k.  Use this         */
  /* attribute to have the proper pointer alignment (i.e. 4-byte    */
  /* one on the given 32-bit architecture).                         */
# define GC_ATTR_PTRT_ALIGNED __attribute__((__aligned__(sizeof(ptr_t))))
#else
# define GC_ATTR_PTRT_ALIGNED /* empty */
#endif

  typedef GC_uintptr_t GC_funcptr_uint;
# define FUNCPTR_IS_DATAPTR

typedef unsigned int unsigned32;

typedef struct hblkhdr hdr;

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

#ifndef GC_HEADERS_H
#define GC_HEADERS_H

#if !defined(GC_PRIVATE_H) && !defined(CPPCHECK)
# error gc_hdrs.h should be included from gc_priv.h
#endif

#if CPP_WORDSZ != 32 && CPP_WORDSZ < 36 && !defined(CPPCHECK)
# error Get a real machine
#endif

EXTERN_C_BEGIN

/*
 * The 2 level tree data structure that is used to find block headers.
 * If there are more than 32 bits in a pointer, the top level is a hash
 * table.
 *
 * This defines HDR, GET_HDR, and SET_HDR, the main macros used to
 * retrieve and set object headers.
 *
 * We take advantage of a header lookup
 * cache.  This is a locally declared direct mapped cache, used inside
 * the marker.  The HC_GET_HDR macro uses and maintains this
 * cache.  Assuming we get reasonable hit rates, this shaves a few
 * memory references from each pointer validation.
 */

#if CPP_WORDSZ > 32
# define HASH_TL
#endif

/* Define appropriate out-degrees for each of the two tree levels       */
#if defined(LARGE_CONFIG) || !defined(SMALL_CONFIG)
# define LOG_BOTTOM_SZ 10
#else
# define LOG_BOTTOM_SZ 11
        /* Keep top index size reasonable with smaller blocks.  */
#endif
#define BOTTOM_SZ (1 << LOG_BOTTOM_SZ)

#ifndef HASH_TL
# define LOG_TOP_SZ (CPP_WORDSZ - LOG_BOTTOM_SZ - LOG_HBLKSIZE)
#else
# define LOG_TOP_SZ 11
#endif
#define TOP_SZ (1 << LOG_TOP_SZ)

/* #define COUNT_HDR_CACHE_HITS  */

#ifdef COUNT_HDR_CACHE_HITS
  extern word GC_hdr_cache_hits; /* used for debugging/profiling */
  extern word GC_hdr_cache_misses;
# define HC_HIT() (void)(++GC_hdr_cache_hits)
# define HC_MISS() (void)(++GC_hdr_cache_misses)
#else
# define HC_HIT() /* empty */
# define HC_MISS() /* empty */
#endif

typedef struct hce {
  word block_addr;    /* right shifted by LOG_HBLKSIZE */
  hdr * hce_hdr;
} hdr_cache_entry;

#define HDR_CACHE_SIZE 8  /* power of 2 */

#define DECLARE_HDR_CACHE \
        hdr_cache_entry hdr_cache[HDR_CACHE_SIZE]

#define INIT_HDR_CACHE BZERO(hdr_cache, sizeof(hdr_cache))

#define HCE(h) (hdr_cache + ((ADDR(h) >> LOG_HBLKSIZE) & (HDR_CACHE_SIZE-1)))

#define HCE_VALID_FOR(hce, h) \
                ((hce) -> block_addr == (ADDR(h) >> LOG_HBLKSIZE))

#define HCE_HDR(h) ((hce) -> hce_hdr)

#ifdef PRINT_BLACK_LIST
  GC_INNER hdr * GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce,
                                      ptr_t source);
# define HEADER_CACHE_MISS(p, hce, source) \
          GC_header_cache_miss(p, hce, source)
#else
  GC_INNER hdr * GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce);
# define HEADER_CACHE_MISS(p, hce, source) GC_header_cache_miss(p, hce)
#endif

/* Set hhdr to the header for p.  Analogous to GET_HDR below,           */
/* except that in the case of large objects, it gets the header for     */
/* the object beginning if GC_all_interior_pointers is set.             */
/* Returns zero if p points to somewhere other than the first page      */
/* of an object, and it is not a valid pointer to the object.           */
#define HC_GET_HDR(p, hhdr, source) \
        { /* cannot use do-while(0) here */ \
          hdr_cache_entry * hce = HCE(p); \
          if (EXPECT(HCE_VALID_FOR(hce, p), TRUE)) { \
            HC_HIT(); \
            hhdr = hce -> hce_hdr; \
          } else { \
            hhdr = HEADER_CACHE_MISS(p, hce, source); \
            if (NULL == hhdr) break; /* go to the enclosing loop end */ \
          } \
        }

typedef struct bi {
    hdr * index[BOTTOM_SZ];
        /*
         * The bottom level index contains one of three kinds of values:
         * 0 means we're not responsible for this block,
         *   or this is a block other than the first one in a free block.
         * 1 < (long)X <= MAX_JUMP means the block starts at least
         *        X * HBLKSIZE bytes before the current address.
         * A valid pointer points to a hdr structure. (The above can't be
         * valid pointers due to the GET_MEM return convention.)
         */
    struct bi * asc_link;       /* All indices are linked in    */
                                /* ascending order...           */
    struct bi * desc_link;      /* ... and in descending order. */
    word key;                   /* High-order address bits.     */
# ifdef HASH_TL
    struct bi * hash_link;      /* Hash chain link.             */
# endif
} bottom_index;

/* bottom_index GC_all_nils; - really part of GC_arrays */

/* extern bottom_index * GC_top_index []; - really part of GC_arrays */
                                /* Each entry points to a bottom_index. */
                                /* On a 32 bit machine, it points to    */
                                /* the index for a set of high-order    */
                                /* bits equal to the index.  For longer */
                                /* addresses, we hash the high-order    */
                                /* bits to compute the index in         */
                                /* GC_top_index, and each entry points  */
                                /* to a hash chain.                     */
                                /* The last entry in each chain is      */
                                /* GC_all_nils.                         */


#define MAX_JUMP (HBLKSIZE-1)

#define HDR_FROM_BI(bi, p) \
                (bi)->index[(ADDR(p) >> LOG_HBLKSIZE) & (BOTTOM_SZ-1)]
#ifndef HASH_TL
# define BI(p) GC_top_index[ADDR(p) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE)]
# define HDR_INNER(p) HDR_FROM_BI(BI(p),p)
# ifdef SMALL_CONFIG
#     define HDR(p) GC_find_header(p)
# else
#     define HDR(p) HDR_INNER(p)
# endif
# define GET_BI(p, bottom_indx) (void)((bottom_indx) = BI(p))
# define GET_HDR(p, hhdr) (void)((hhdr) = HDR(p))
# define SET_HDR(p, hhdr) (void)(HDR_INNER(p) = (hhdr))
# define GET_HDR_ADDR(p, ha) (void)((ha) = &HDR_INNER(p))
#else /* hash */
  /* Hash function for tree top level */
# define TL_HASH(hi) ((hi) & (TOP_SZ-1))
  /* Set bottom_indx to point to the bottom index for address p */
# define GET_BI(p, bottom_indx) \
        do { \
          REGISTER word hi = ADDR(p) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE); \
          REGISTER bottom_index * _bi = GC_top_index[TL_HASH(hi)]; \
          while (_bi -> key != hi && _bi != GC_all_nils) \
              _bi = _bi -> hash_link; \
          (bottom_indx) = _bi; \
        } while (0)
# define GET_HDR_ADDR(p, ha) \
        do { \
          REGISTER bottom_index * bi; \
          GET_BI(p, bi); \
          (ha) = &HDR_FROM_BI(bi, p); \
        } while (0)
# define GET_HDR(p, hhdr) \
        do { \
          REGISTER hdr ** _ha; \
          GET_HDR_ADDR(p, _ha); \
          (hhdr) = *_ha; \
        } while (0)
# define SET_HDR(p, hhdr) \
        do { \
          REGISTER bottom_index * bi; \
          GET_BI(p, bi); \
          GC_ASSERT(bi != GC_all_nils); \
          HDR_FROM_BI(bi, p) = (hhdr); \
        } while (0)
# define HDR(p) GC_find_header(p)
#endif

/* Is the result a forwarding address to someplace closer to the        */
/* beginning of the block or NULL?                                      */
#define IS_FORWARDING_ADDR_OR_NIL(hhdr) ((size_t)ADDR(hhdr) <= MAX_JUMP)

/* Get an HBLKSIZE-aligned address closer to the beginning of the block */
/* h.  Assumes hhdr == HDR(h), IS_FORWARDING_ADDR(hhdr) and hhdr is not */
/* NULL.  HDR(result) is expected to be non-NULL.                       */
#define FORWARDED_ADDR(h, hhdr) ((struct hblk *)(h) - (size_t)(hhdr))

EXTERN_C_END

#endif /* GC_HEADERS_H */


#ifndef GC_ATTR_NO_SANITIZE_ADDR
# ifndef ADDRESS_SANITIZER
#   define GC_ATTR_NO_SANITIZE_ADDR /* empty */
# elif GC_CLANG_PREREQ(3, 8)
#   define GC_ATTR_NO_SANITIZE_ADDR __attribute__((no_sanitize("address")))
# else
#   define GC_ATTR_NO_SANITIZE_ADDR __attribute__((no_sanitize_address))
# endif
#endif /* !GC_ATTR_NO_SANITIZE_ADDR */

#ifndef GC_ATTR_NO_SANITIZE_MEMORY
# ifndef MEMORY_SANITIZER
#   define GC_ATTR_NO_SANITIZE_MEMORY /* empty */
# elif GC_CLANG_PREREQ(3, 8)
#   define GC_ATTR_NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory")))
# else
#   define GC_ATTR_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory))
# endif
#endif /* !GC_ATTR_NO_SANITIZE_MEMORY */

#ifndef GC_ATTR_NO_SANITIZE_THREAD
# ifndef THREAD_SANITIZER
#   define GC_ATTR_NO_SANITIZE_THREAD /* empty */
# elif GC_CLANG_PREREQ(3, 8)
#   define GC_ATTR_NO_SANITIZE_THREAD __attribute__((no_sanitize("thread")))
# else
    /* It seems that no_sanitize_thread attribute has no effect if the  */
    /* function is inlined (as of gcc 11.1.0, at least).                */
#   define GC_ATTR_NO_SANITIZE_THREAD \
                GC_ATTR_NOINLINE __attribute__((no_sanitize_thread))
# endif
#endif /* !GC_ATTR_NO_SANITIZE_THREAD */

#ifndef UNUSED_ARG
# define UNUSED_ARG(arg) ((void)(arg))
#endif

#ifdef HAVE_CONFIG_H
  /* The "inline" keyword is determined by Autoconf AC_C_INLINE.    */
# define GC_INLINE static inline
#elif defined(_MSC_VER) || defined(__INTEL_COMPILER) || defined(__DMC__) \
        || (GC_GNUC_PREREQ(3, 0) && defined(__STRICT_ANSI__)) \
        || defined(__BORLANDC__) || defined(__WATCOMC__)
# define GC_INLINE static __inline
#elif GC_GNUC_PREREQ(3, 0) || defined(__sun)
# define GC_INLINE static inline
#else
# define GC_INLINE static
#endif

#ifndef GC_ATTR_NOINLINE
# if GC_GNUC_PREREQ(4, 0)
#   define GC_ATTR_NOINLINE __attribute__((__noinline__))
# elif _MSC_VER >= 1400
#   define GC_ATTR_NOINLINE __declspec(noinline)
# else
#   define GC_ATTR_NOINLINE /* empty */
# endif
#endif

#ifndef GC_API_OSCALL
  /* This is used to identify GC routines called by name from OS.       */
# if defined(__GNUC__)
#   if GC_GNUC_PREREQ(4, 0) && !defined(GC_NO_VISIBILITY)
      /* Same as GC_API if GC_DLL.      */
#     define GC_API_OSCALL extern __attribute__((__visibility__("default")))
#   else
      /* The attribute is unsupported.  */
#     define GC_API_OSCALL extern
#   endif
# else
#   define GC_API_OSCALL GC_API
# endif
#endif

#ifndef GC_API_PRIV
# define GC_API_PRIV GC_API
#endif

#if defined(THREADS) && !defined(NN_PLATFORM_CTR)
/*
 * Copyright (c) 2017 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/* This is a private GC header which provides an implementation of      */
/* libatomic_ops subset primitives sufficient for GC assuming that GCC  */
/* atomic intrinsics are available (and have correct implementation).   */
/* This is enabled by defining GC_BUILTIN_ATOMIC macro.  Otherwise,     */
/* libatomic_ops library is used to define the primitives.              */

#ifndef GC_ATOMIC_OPS_H
#define GC_ATOMIC_OPS_H

#ifdef GC_BUILTIN_ATOMIC




# ifdef __cplusplus
    extern "C" {
# endif

  typedef size_t AO_t;

# ifdef GC_PRIVATE_H /* have GC_INLINE */
#   define AO_INLINE GC_INLINE
# else
#   define AO_INLINE static __inline
# endif

# if !defined(THREAD_SANITIZER) && !defined(GC_PRIVATE_H)
    /* Similar to that in gcconfig.h.   */
#   if defined(__has_feature)
#     if __has_feature(thread_sanitizer)
#       define THREAD_SANITIZER
#     endif
#   elif defined(__SANITIZE_THREAD__)
#     define THREAD_SANITIZER
#   endif
# endif /* !THREAD_SANITIZER && !GC_PRIVATE_H */

  typedef unsigned char AO_TS_t;
# define AO_TS_CLEAR 0
# define AO_TS_INITIALIZER (AO_TS_t)AO_TS_CLEAR
# if defined(__GCC_ATOMIC_TEST_AND_SET_TRUEVAL) && !defined(CPPCHECK)
#   define AO_TS_SET __GCC_ATOMIC_TEST_AND_SET_TRUEVAL
# else
#   define AO_TS_SET (AO_TS_t)1 /* true */
# endif
# define AO_CLEAR(p) __atomic_clear(p, __ATOMIC_RELEASE)
# define AO_test_and_set_acquire(p) \
        (__atomic_test_and_set(p, __ATOMIC_ACQUIRE) ? AO_TS_SET : AO_TS_CLEAR)
# define AO_HAVE_test_and_set_acquire

# define AO_compiler_barrier() __atomic_signal_fence(__ATOMIC_SEQ_CST)

# if defined(THREAD_SANITIZER) && !defined(AO_USE_ATOMIC_THREAD_FENCE)
    /* Workaround a compiler warning (reported by gcc-11, at least)     */
    /* that atomic_thread_fence is unsupported with thread sanitizer.   */
    AO_INLINE void
    AO_nop_full(void)
    {
      volatile AO_TS_t dummy = AO_TS_INITIALIZER;
      (void)__atomic_test_and_set(&dummy, __ATOMIC_SEQ_CST);
    }
# else
#   define AO_nop_full() __atomic_thread_fence(__ATOMIC_SEQ_CST)
# endif
# define AO_HAVE_nop_full

# define AO_fetch_and_add(p, v) __atomic_fetch_add(p, v, __ATOMIC_RELAXED)
# define AO_HAVE_fetch_and_add
# define AO_fetch_and_add1(p) AO_fetch_and_add(p, 1)
# define AO_HAVE_fetch_and_add1
# define AO_fetch_and_sub1(p) AO_fetch_and_add(p, ~(AO_t)0 /* -1 */)
# define AO_HAVE_fetch_and_sub1

# define AO_or(p, v) (void)__atomic_or_fetch(p, v, __ATOMIC_RELAXED)
# define AO_HAVE_or

# define AO_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
# define AO_HAVE_load
# define AO_load_acquire(p) __atomic_load_n(p, __ATOMIC_ACQUIRE)
# define AO_HAVE_load_acquire
/* AO_load_acquire_read(p) is not defined as it is unused, but we   */
/* need its AO_HAVE_x macro defined.                                */
# define AO_HAVE_load_acquire_read

# define AO_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
# define AO_HAVE_store
# define AO_store_release(p, v) __atomic_store_n(p, v, __ATOMIC_RELEASE)
# define AO_HAVE_store_release
# define AO_store_release_write(p, v) AO_store_release(p, v)
# define AO_HAVE_store_release_write

# define AO_char_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
# define AO_HAVE_char_load
# define AO_char_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
# define AO_HAVE_char_store

# ifdef AO_REQUIRE_CAS
    AO_INLINE int
    AO_compare_and_swap_release(volatile AO_t *p, AO_t ov, AO_t nv)
    {
      return (int)__atomic_compare_exchange_n(p, &ov, nv, 0,
                        __ATOMIC_RELEASE, __ATOMIC_RELAXED /* on fail */);
    }
#   define AO_HAVE_compare_and_swap_release
# endif

# ifdef __cplusplus
    } /* extern "C" */
# endif

# ifndef NO_LOCKFREE_AO_OR
    /* __atomic_or_fetch is assumed to be lock-free.    */
#   define HAVE_LOCKFREE_AO_OR 1
# endif

#else
  /* Fallback to libatomic_ops. */
# include "atomic_ops.h"

  /* AO_compiler_barrier, AO_load and AO_store should be defined for    */
  /* all targets; the rest of the primitives are guaranteed to exist    */
  /* only if AO_REQUIRE_CAS is defined (or if the corresponding         */
  /* AO_HAVE_x macro is defined).  i686 and x86_64 targets have         */
  /* AO_nop_full, AO_load_acquire, AO_store_release, at least.          */
# if (!defined(AO_HAVE_load) || !defined(AO_HAVE_store)) && !defined(CPPCHECK)
#   error AO_load or AO_store is missing; probably old version of atomic_ops
# endif

#endif /* !GC_BUILTIN_ATOMIC */

#if defined(GC_BUILTIN_ATOMIC) || CPP_PTRSZ > CPP_WORDSZ
  /* Assume that GCC atomic intrinsics are available (and have correct  */
  /* implementation).  p should be of a pointer to ptr_t (char*) value. */
# define GC_cptr_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
# define GC_cptr_load_acquire(p) __atomic_load_n(p, __ATOMIC_ACQUIRE)
# define GC_cptr_load_acquire_read(p) GC_cptr_load_acquire(p)
# define GC_cptr_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
# define GC_cptr_store_release(p, v) __atomic_store_n(p, v, __ATOMIC_RELEASE)
# define GC_cptr_store_release_write(p, v) GC_cptr_store_release(p, v)
# ifdef AO_REQUIRE_CAS
    AO_INLINE int
    GC_cptr_compare_and_swap(char *volatile *p, char *ov, char *nv)
    {
      return (int)__atomic_compare_exchange_n(p, &ov, nv, 0,
                        __ATOMIC_RELAXED, __ATOMIC_RELAXED);
    }
# endif
#else
  /* Redirect to the AO_ primitives.  Assume the size of AO_t matches   */
  /* that of a pointer.                                                 */
# define GC_cptr_load(p) (char *)AO_load((volatile AO_t *)(p))
# define GC_cptr_load_acquire(p) (char *)AO_load_acquire((volatile AO_t *)(p))
# define GC_cptr_load_acquire_read(p) \
            (char *)AO_load_acquire_read((volatile AO_t *)(p))
# define GC_cptr_store(p, v) AO_store((volatile AO_t *)(p), (AO_t)(v))
# define GC_cptr_store_release(p, v) \
            AO_store_release((volatile AO_t *)(p), (AO_t)(v))
# define GC_cptr_store_release_write(p, v) \
            AO_store_release_write((volatile AO_t *)(p), (AO_t)(v))
# ifdef AO_REQUIRE_CAS
#   define GC_cptr_compare_and_swap(p, ov, nv) \
            AO_compare_and_swap((volatile AO_t *)(p), (AO_t)(ov), (AO_t)(nv))
# endif
#endif /* !GC_BUILTIN_ATOMIC && CPP_PTRSZ == CPP_WORDSZ */

#endif /* GC_ATOMIC_OPS_H */

# ifndef AO_HAVE_compiler_barrier
#   define AO_HAVE_compiler_barrier 1
# endif
#endif

#ifdef ANY_MSWIN
# ifndef WIN32_LEAN_AND_MEAN
#   define WIN32_LEAN_AND_MEAN 1
# endif
# define NOSERVICE
# include <windows.h>
# include <winbase.h>
#endif /* ANY_MSWIN */

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

#ifndef GC_LOCKS_H
#define GC_LOCKS_H

#if !defined(GC_PRIVATE_H) && !defined(CPPCHECK)
# error gc_locks.h should be included from gc_priv.h
#endif

/* Mutual exclusion between allocator/collector routines.  Needed if    */
/* there is more than one allocator thread.  Note that I_HOLD_LOCK,     */
/* I_DONT_HOLD_LOCK and I_HOLD_READER_LOCK are used only positively in  */
/* assertions, and may return TRUE in the "don't know" case.            */

#ifdef THREADS

# ifdef PCR
#   include <base/PCR_Base.h>
#   include <th/PCR_Th.h>
# endif

  EXTERN_C_BEGIN

# ifdef PCR
    GC_EXTERN PCR_Th_ML GC_allocate_ml;
#   define UNCOND_LOCK() PCR_Th_ML_Acquire(&GC_allocate_ml)
#   define UNCOND_UNLOCK() PCR_Th_ML_Release(&GC_allocate_ml)
# elif defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH)
    extern void GC_lock(void);
    extern void GC_unlock(void);
#   define UNCOND_LOCK() GC_lock()
#   define UNCOND_UNLOCK() GC_unlock()
# endif

# if (!defined(AO_HAVE_test_and_set_acquire) || defined(GC_RTEMS_PTHREADS) \
       || defined(SN_TARGET_PS3) \
       || defined(GC_WIN32_THREADS) || defined(BASE_ATOMIC_OPS_EMULATED) \
       || defined(LINT2) || defined(USE_RWLOCK)) && defined(GC_PTHREADS)
#   define USE_PTHREAD_LOCKS
#   undef USE_SPIN_LOCK
#   if (defined(LINT2) || defined(GC_WIN32_THREADS) || defined(USE_RWLOCK)) \
       && !defined(NO_PTHREAD_TRYLOCK)
      /* pthread_mutex_trylock may not win in GC_lock on Win32, */
      /* due to builtin support for spinning first?             */
#     define NO_PTHREAD_TRYLOCK
#   endif
# endif

# if defined(GC_WIN32_THREADS) && !defined(USE_PTHREAD_LOCKS) \
     || defined(GC_PTHREADS)
#   define NO_THREAD ((unsigned long)(-1L))
                /* != NUMERIC_THREAD_ID(pthread_self()) for any thread */
#   ifdef GC_ASSERTIONS
      GC_EXTERN unsigned long GC_lock_holder;
#     define UNSET_LOCK_HOLDER() (void)(GC_lock_holder = NO_THREAD)
#   endif
# endif /* GC_WIN32_THREADS || GC_PTHREADS */

# if defined(GC_WIN32_THREADS) && !defined(USE_PTHREAD_LOCKS)
#   ifdef USE_RWLOCK
      GC_EXTERN SRWLOCK GC_allocate_ml;
#   else
      GC_EXTERN CRITICAL_SECTION GC_allocate_ml;
#   endif
#   ifdef GC_ASSERTIONS
#     define SET_LOCK_HOLDER() (void)(GC_lock_holder = GetCurrentThreadId())
#     define I_HOLD_LOCK() (!GC_need_to_lock \
                            || GC_lock_holder == GetCurrentThreadId())
#     ifdef THREAD_SANITIZER
#       define I_DONT_HOLD_LOCK() TRUE /* Conservatively say yes */
#     else
#       define I_DONT_HOLD_LOCK() (!GC_need_to_lock \
                            || GC_lock_holder != GetCurrentThreadId())
#     endif
#     ifdef USE_RWLOCK
#       define UNCOND_READER_LOCK() \
                { GC_ASSERT(I_DONT_HOLD_LOCK()); \
                  AcquireSRWLockShared(&GC_allocate_ml); }
#       define UNCOND_READER_UNLOCK() \
                { GC_ASSERT(I_DONT_HOLD_LOCK()); \
                  ReleaseSRWLockShared(&GC_allocate_ml); }
#       define UNCOND_LOCK() \
                { GC_ASSERT(I_DONT_HOLD_LOCK()); \
                  AcquireSRWLockExclusive(&GC_allocate_ml); \
                  SET_LOCK_HOLDER(); }
#       define UNCOND_UNLOCK() \
                { GC_ASSERT(I_HOLD_LOCK()); \
                  UNSET_LOCK_HOLDER(); \
                  ReleaseSRWLockExclusive(&GC_allocate_ml); }
#     else
#       define UNCOND_LOCK() \
                { GC_ASSERT(I_DONT_HOLD_LOCK()); \
                  EnterCriticalSection(&GC_allocate_ml); \
                  SET_LOCK_HOLDER(); }
#       define UNCOND_UNLOCK() \
                { GC_ASSERT(I_HOLD_LOCK()); UNSET_LOCK_HOLDER(); \
                  LeaveCriticalSection(&GC_allocate_ml); }
#     endif
#   else
#     ifdef USE_RWLOCK
#       define UNCOND_READER_LOCK() AcquireSRWLockShared(&GC_allocate_ml)
#       define UNCOND_READER_UNLOCK() ReleaseSRWLockShared(&GC_allocate_ml)
#       define UNCOND_LOCK() AcquireSRWLockExclusive(&GC_allocate_ml)
#       define UNCOND_UNLOCK() ReleaseSRWLockExclusive(&GC_allocate_ml)
#     else
#       define UNCOND_LOCK() EnterCriticalSection(&GC_allocate_ml)
#       define UNCOND_UNLOCK() LeaveCriticalSection(&GC_allocate_ml)
#     endif
#   endif /* !GC_ASSERTIONS */
# elif defined(GC_PTHREADS)
    EXTERN_C_END
#   include <pthread.h>
    EXTERN_C_BEGIN
    /* Posix allows pthread_t to be a struct, though it rarely is.  */
    /* Unfortunately, we need to use a pthread_t to index a data    */
    /* structure.  It also helps if comparisons don't involve a     */
    /* function call.  Hence we introduce platform-dependent macros */
    /* to compare pthread_t ids and to map them to integers.        */
    /* The mapping to integers does not need to result in different */
    /* integers for each thread, though that should be true as much */
    /* as possible.                                                 */
    /* Refine to exclude platforms on which pthread_t is struct.    */
#   if !defined(GC_WIN32_PTHREADS)
#     define NUMERIC_THREAD_ID(id) ((unsigned long)(id))
#     define THREAD_EQUAL(id1, id2) ((id1) == (id2))
#     define NUMERIC_THREAD_ID_UNIQUE
#   elif defined(__WINPTHREADS_VERSION_MAJOR) /* winpthreads */
#     define NUMERIC_THREAD_ID(id) ((unsigned long)(id))
#     define THREAD_EQUAL(id1, id2) ((id1) == (id2))
#     ifndef _WIN64
        /* NUMERIC_THREAD_ID is 32-bit and not unique on Win64. */
#       define NUMERIC_THREAD_ID_UNIQUE
#     endif
#   else /* pthreads-win32 */
#     define NUMERIC_THREAD_ID(id) ((unsigned long)(word)(id.p))
      /* Using documented internal details of pthreads-win32 library.   */
      /* Faster than pthread_equal().  Should not change with           */
      /* future versions of pthreads-win32 library.                     */
#     define THREAD_EQUAL(id1, id2) ((id1.p == id2.p) && (id1.x == id2.x))
#     undef NUMERIC_THREAD_ID_UNIQUE
      /* Generic definitions based on pthread_equal() always work but   */
      /* will result in poor performance (as NUMERIC_THREAD_ID is       */
      /* defined to just a constant) and weak assertion checking.       */
#   endif

#   ifdef SN_TARGET_PSP2
      EXTERN_C_END
#     include "psp2-support.h"
      EXTERN_C_BEGIN
      GC_EXTERN WapiMutex GC_allocate_ml_PSP2;
#     define UNCOND_LOCK() { int res; GC_ASSERT(I_DONT_HOLD_LOCK()); \
                              res = PSP2_MutexLock(&GC_allocate_ml_PSP2); \
                              GC_ASSERT(0 == res); (void)res; \
                              SET_LOCK_HOLDER(); }
#     define UNCOND_UNLOCK() { int res; GC_ASSERT(I_HOLD_LOCK()); \
                              UNSET_LOCK_HOLDER(); \
                              res = PSP2_MutexUnlock(&GC_allocate_ml_PSP2); \
                              GC_ASSERT(0 == res); (void)res; }

#   elif (!defined(THREAD_LOCAL_ALLOC) || defined(USE_SPIN_LOCK)) \
         && !defined(USE_PTHREAD_LOCKS) && !defined(THREAD_SANITIZER) \
         && !defined(USE_RWLOCK)
      /* In the THREAD_LOCAL_ALLOC case, the allocator lock tends to    */
      /* be held for long periods, if it is held at all.  Thus spinning */
      /* and sleeping for fixed periods are likely to result in         */
      /* significant wasted time.  We thus rely mostly on queued locks. */
#     undef USE_SPIN_LOCK
#     define USE_SPIN_LOCK
      GC_EXTERN volatile AO_TS_t GC_allocate_lock;
      GC_INNER void GC_lock(void);
#     ifdef GC_ASSERTIONS
#       define UNCOND_LOCK() \
              { GC_ASSERT(I_DONT_HOLD_LOCK()); \
                if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_SET) \
                  GC_lock(); \
                SET_LOCK_HOLDER(); }
#       define UNCOND_UNLOCK() \
              { GC_ASSERT(I_HOLD_LOCK()); UNSET_LOCK_HOLDER(); \
                AO_CLEAR(&GC_allocate_lock); }
#     else
#       define UNCOND_LOCK() \
              { if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_SET) \
                  GC_lock(); }
#       define UNCOND_UNLOCK() AO_CLEAR(&GC_allocate_lock)
#     endif /* !GC_ASSERTIONS */
#   else /* THREAD_LOCAL_ALLOC || USE_PTHREAD_LOCKS */
#     ifndef USE_PTHREAD_LOCKS
#       define USE_PTHREAD_LOCKS
#     endif
#   endif /* THREAD_LOCAL_ALLOC || USE_PTHREAD_LOCKS */
#   ifdef USE_PTHREAD_LOCKS
      EXTERN_C_END
#     include <pthread.h>
      EXTERN_C_BEGIN
#     ifdef GC_ASSERTIONS
        GC_INNER void GC_lock(void);
#       define UNCOND_LOCK() { GC_ASSERT(I_DONT_HOLD_LOCK()); \
                                GC_lock(); SET_LOCK_HOLDER(); }
#     endif
#     ifdef USE_RWLOCK
        GC_EXTERN pthread_rwlock_t GC_allocate_ml;
#       ifdef GC_ASSERTIONS
#         define UNCOND_READER_LOCK() \
                        { GC_ASSERT(I_DONT_HOLD_LOCK()); \
                          (void)pthread_rwlock_rdlock(&GC_allocate_ml); }
#         define UNCOND_READER_UNLOCK() \
                        { GC_ASSERT(I_DONT_HOLD_LOCK()); \
                          (void)pthread_rwlock_unlock(&GC_allocate_ml); }
#         define UNCOND_UNLOCK() \
                        { GC_ASSERT(I_HOLD_LOCK()); UNSET_LOCK_HOLDER(); \
                          (void)pthread_rwlock_unlock(&GC_allocate_ml); }
#       else
#         define UNCOND_READER_LOCK() \
                                (void)pthread_rwlock_rdlock(&GC_allocate_ml)
#         define UNCOND_READER_UNLOCK() UNCOND_UNLOCK()
#         define UNCOND_LOCK() (void)pthread_rwlock_wrlock(&GC_allocate_ml)
#         define UNCOND_UNLOCK() (void)pthread_rwlock_unlock(&GC_allocate_ml)
#       endif /* !GC_ASSERTIONS */
#     else
        GC_EXTERN pthread_mutex_t GC_allocate_ml;
#       ifdef GC_ASSERTIONS
#         define UNCOND_UNLOCK() \
                        { GC_ASSERT(I_HOLD_LOCK()); UNSET_LOCK_HOLDER(); \
                          pthread_mutex_unlock(&GC_allocate_ml); }
#       else
#         if defined(NO_PTHREAD_TRYLOCK)
#           define UNCOND_LOCK() pthread_mutex_lock(&GC_allocate_ml)
#         else
            GC_INNER void GC_lock(void);
#           define UNCOND_LOCK() \
                { if (0 != pthread_mutex_trylock(&GC_allocate_ml)) \
                    GC_lock(); }
#         endif
#         define UNCOND_UNLOCK() pthread_mutex_unlock(&GC_allocate_ml)
#       endif /* !GC_ASSERTIONS */
#     endif /* !USE_RWLOCK */
#   endif /* USE_PTHREAD_LOCKS */
#   ifdef GC_ASSERTIONS
      /* The allocator lock holder.     */
#     define SET_LOCK_HOLDER() \
                (void)(GC_lock_holder = NUMERIC_THREAD_ID(pthread_self()))
#     define I_HOLD_LOCK() \
                (!GC_need_to_lock \
                 || GC_lock_holder == NUMERIC_THREAD_ID(pthread_self()))
#     if !defined(NUMERIC_THREAD_ID_UNIQUE) || defined(THREAD_SANITIZER)
#       define I_DONT_HOLD_LOCK() TRUE /* Conservatively say yes */
#     else
#       define I_DONT_HOLD_LOCK() \
                (!GC_need_to_lock \
                 || GC_lock_holder != NUMERIC_THREAD_ID(pthread_self()))
#     endif
#   endif /* GC_ASSERTIONS */
#   ifndef GC_WIN32_THREADS
      GC_EXTERN volatile unsigned char GC_collecting;
#     ifdef AO_HAVE_char_store
#       define ENTER_GC() AO_char_store(&GC_collecting, TRUE)
#       define EXIT_GC() AO_char_store(&GC_collecting, FALSE)
#     else
#       define ENTER_GC() (void)(GC_collecting = TRUE)
#       define EXIT_GC() (void)(GC_collecting = FALSE)
#     endif
#   endif /* !GC_WIN32_THREADS */
# endif /* GC_PTHREADS */
# if defined(GC_ALWAYS_MULTITHREADED) \
      && (defined(USE_PTHREAD_LOCKS) || defined(USE_SPIN_LOCK))
#   define GC_need_to_lock TRUE
#   define set_need_to_lock() (void)0
# else
#   if defined(GC_ALWAYS_MULTITHREADED) && !defined(CPPCHECK)
#     error Runtime initialization of the allocator lock is needed!
#   endif
#   undef GC_ALWAYS_MULTITHREADED
    GC_EXTERN GC_bool GC_need_to_lock;
#   ifdef THREAD_SANITIZER
        /* To workaround TSan false positive (e.g., when                */
        /* GC_pthread_create is called from multiple threads in         */
        /* parallel), do not set GC_need_to_lock if it is already set.  */
#       define set_need_to_lock() \
                (void)(*(GC_bool volatile *)&GC_need_to_lock \
                        ? FALSE \
                        : (GC_need_to_lock = TRUE))
#   else
#       define set_need_to_lock() (void)(GC_need_to_lock = TRUE)
                                        /* We are multi-threaded now.   */
#   endif
# endif

  EXTERN_C_END

#else /* !THREADS */
#   define LOCK() (void)0
#   define UNLOCK() (void)0
#   ifdef GC_ASSERTIONS
#     define I_HOLD_LOCK() TRUE
#     define I_DONT_HOLD_LOCK() TRUE
                /* Used only in positive assertions or to test whether  */
                /* we still need to acquire the allocator lock.         */
                /* TRUE works in either case.                           */
#   endif
#endif /* !THREADS */

#if defined(UNCOND_LOCK) && !defined(LOCK)
# if (defined(LINT2) && defined(USE_PTHREAD_LOCKS)) \
     || defined(GC_ALWAYS_MULTITHREADED)
    /* Instruct code analysis tools not to care about GC_need_to_lock   */
    /* influence to LOCK/UNLOCK semantic.                               */
#   define LOCK() UNCOND_LOCK()
#   define UNLOCK() UNCOND_UNLOCK()
#   ifdef UNCOND_READER_LOCK
#     define READER_LOCK() UNCOND_READER_LOCK()
#     define READER_UNLOCK() UNCOND_READER_UNLOCK()
#   endif
# else
                /* At least two thread running; need to lock.   */
#   define LOCK() do { if (GC_need_to_lock) UNCOND_LOCK(); } while (0)
#   define UNLOCK() do { if (GC_need_to_lock) UNCOND_UNLOCK(); } while (0)
#   ifdef UNCOND_READER_LOCK
#     define READER_LOCK() \
                do { if (GC_need_to_lock) UNCOND_READER_LOCK(); } while (0)
#     define READER_UNLOCK() \
                do { if (GC_need_to_lock) UNCOND_READER_UNLOCK(); } while (0)
#   endif
# endif
#endif /* UNCOND_LOCK && !LOCK */

#ifdef READER_LOCK
# define HAS_REAL_READER_LOCK
# define I_HOLD_READER_LOCK() TRUE /* TODO: implement */
#else
# define READER_LOCK() LOCK()
# define READER_UNLOCK() UNLOCK()
# ifdef GC_ASSERTIONS
    /* A macro to check that the allocator lock is held at least in the */
    /* reader mode.                                                     */
#   define I_HOLD_READER_LOCK() I_HOLD_LOCK()
# endif
#endif /* !READER_LOCK */

/* A variant of READER_UNLOCK() which ensures that data written before  */
/* the unlock will be visible to the thread which acquires the          */
/* allocator lock in the exclusive mode.  But according to some rwlock  */
/* documentation: writers synchronize with prior writers and readers.   */
#define READER_UNLOCK_RELEASE() READER_UNLOCK()

# ifndef ENTER_GC
#   define ENTER_GC()
#   define EXIT_GC()
# endif

#endif /* GC_LOCKS_H */


#ifdef GC_ASSERTIONS
# define GC_ASSERT(expr) \
        do { \
          if (EXPECT(!(expr), FALSE)) { \
            GC_err_printf("Assertion failure: %s:%d\n", __FILE__, __LINE__); \
            ABORT("assertion failure"); \
          } \
        } while (0)
#else
# define GC_ASSERT(expr)
#endif

#include "gc/gc_inline.h"

/*********************************/
/*                               */
/* Definitions for conservative  */
/* collector.                    */
/*                               */
/*********************************/

/* Prevent certain compiler warnings by making a cast through       */
/* a pointer-sized numeric type.                                    */
#define CAST_THRU_UINTPTR(t, x) ((t)(GC_uintptr_t)(x))

#define CAST_AWAY_VOLATILE_PVOID(p) \
                        CAST_THRU_UINTPTR(/* no volatile */ void *, p)

/* Create a ptr_t pointer from a number (of word type). */
#define MAKE_CPTR(w) ((ptr_t)(word)(w))

#define GC_WORD_MAX (~(word)0)

/* Convert given pointer to its address.  Result is of word type.   */
#define ADDR(p) ((word)(p))

#define ADDR_LT(p,q) GC_ADDR_LT(p,q)
#define ADDR_GE(p,q) (!ADDR_LT(p,q))

/* Check whether pointer p is in range [s, e_p1).  p should not     */
/* have side effects.                                               */
#define ADDR_INSIDE(p, s, e_p1) (ADDR_GE(p, s) && ADDR_LT(p, e_p1))

/* Handy definitions to compare and adjust pointers in a stack.     */
#ifdef STACK_GROWS_UP
# define HOTTER_THAN(p,q) ADDR_LT(q, p) /* inverse */
# define MAKE_COOLER(p,d) \
            (void)((p) -= ADDR(p) > (word)((d) * sizeof(*(p))) ? (d) : 0)
# define MAKE_HOTTER(p,d) (void)((p) += (d))
#else
# define HOTTER_THAN(p,q) ADDR_LT(p, q)
# define MAKE_COOLER(p,d) \
            (void)((p) += ADDR(p) <= (word)(GC_WORD_MAX \
                                            - (d) * sizeof(*(p))) ? (d) : 0)
# define MAKE_HOTTER(p,d) (void)((p) -= (d))
#endif /* !STACK_GROWS_UP */

/* Clear/set flags (given by a mask) in a pointer.  */
#define CPTR_CLEAR_FLAGS(p, mask) (ptr_t)((word)(p) & ~(word)(mask))
#define CPTR_SET_FLAGS(p, mask) (ptr_t)((word)(p) | (word)(mask))

#if defined(AMIGA) && defined(__SASC)
#   define GC_FAR __far
#else
#   define GC_FAR
#endif

/*********************************/
/*                               */
/* Easily changeable parameters  */
/*                               */
/*********************************/

/* #define ALL_INTERIOR_POINTERS */
                    /* Forces all pointers into the interior of an      */
                    /* object to be considered valid.  Also causes the  */
                    /* sizes of all objects to be inflated by at least  */
                    /* one byte.  This should suffice to guarantee      */
                    /* that in the presence of a compiler that does     */
                    /* not perform garbage-collector-unsafe             */
                    /* optimizations, all portable, strictly ANSI       */
                    /* conforming C programs should be safely usable    */
                    /* with malloc replaced by GC_malloc and free       */
                    /* calls removed.  There are several disadvantages: */
                    /* 1. There are probably no interesting, portable,  */
                    /*    strictly ANSI conforming C programs.          */
                    /* 2. This option makes it hard for the collector   */
                    /*    to allocate space that is not "pointed to"    */
                    /*    by integers, etc.  Under SunOS 4.X with a     */
                    /*    statically linked libc, we empirically        */
                    /*    observed that it would be difficult to        */
                    /*    allocate individual objects > 100 KB.         */
                    /*    Even if only smaller objects are allocated,   */
                    /*    more swap space is likely to be needed.       */
                    /*    Fortunately, much of this will never be       */
                    /*    touched.                                      */
                    /* If you can easily avoid using this option, do.   */
                    /* If not, try to keep individual objects small.    */
                    /* This is now really controlled at startup,        */
                    /* through GC_all_interior_pointers.                */

EXTERN_C_BEGIN

#ifndef GC_NO_FINALIZATION
# define GC_INVOKE_FINALIZERS() GC_notify_or_invoke_finalizers()
  GC_INNER void GC_notify_or_invoke_finalizers(void);
                        /* If GC_finalize_on_demand is not set, invoke  */
                        /* eligible finalizers. Otherwise:              */
                        /* Call *GC_finalizer_notifier if there are     */
                        /* finalizers to be run, and we haven't called  */
                        /* this procedure yet this GC cycle.            */

  GC_INNER void GC_finalize(void);
                        /* Perform all indicated finalization actions   */
                        /* on unmarked objects.                         */
                        /* Unreachable finalizable objects are enqueued */
                        /* for processing by GC_invoke_finalizers.      */
                        /* Invoked with the allocator lock.             */

# ifndef GC_TOGGLE_REFS_NOT_NEEDED
    GC_INNER void GC_process_togglerefs(void);
                        /* Process the toggle-refs before GC starts.    */
# endif
# ifndef SMALL_CONFIG
    GC_INNER void GC_print_finalization_stats(void);
# endif
#else
# define GC_INVOKE_FINALIZERS() (void)0
#endif /* GC_NO_FINALIZATION */

#if !defined(DONT_ADD_BYTE_AT_END)
# ifdef LINT2
    /* Explicitly instruct the code analysis tool that                  */
    /* GC_all_interior_pointers is assumed to have only 0 or 1 value.   */
#   define EXTRA_BYTES ((size_t)(GC_all_interior_pointers? 1 : 0))
# else
#   define EXTRA_BYTES (size_t)GC_all_interior_pointers
# endif
# define MAX_EXTRA_BYTES 1
#else
# define EXTRA_BYTES 0
# define MAX_EXTRA_BYTES 0
#endif

# ifdef LARGE_CONFIG
#   define MINHINCR 64
#   define MAXHINCR 4096
# else
#   define MINHINCR 16   /* Minimum heap increment, in blocks of HBLKSIZE.  */
                         /* Note: must be multiple of largest page size.    */
#   define MAXHINCR 2048 /* Maximum heap increment, in blocks.  */
# endif /* !LARGE_CONFIG */

# define BL_LIMIT GC_black_list_spacing
                           /* If we need a block of N bytes, and we have */
                           /* a block of N + BL_LIMIT bytes available,   */
                           /* and N > BL_LIMIT,                          */
                           /* but all possible positions in it are       */
                           /* blacklisted, we just use it anyway (and    */
                           /* print a warning, if warnings are enabled). */
                           /* This risks subsequently leaking the block  */
                           /* due to a false reference.  But not using   */
                           /* the block risks unreasonable immediate     */
                           /* heap growth.                               */

/*********************************/
/*                               */
/* Stack saving for debugging    */
/*                               */
/*********************************/

#ifdef NEED_CALLINFO
  struct callinfo {
    GC_return_addr_t ci_pc; /* pc of caller, not callee */
#   if NARGS > 0
      GC_hidden_pointer ci_arg[NARGS]; /* hide to avoid retention */
#   endif
#   if (NFRAMES * (NARGS + 1)) % 2 == 1
      /* Likely alignment problem. */
      ptr_t ci_dummy;
#   endif
  };

# ifdef SAVE_CALL_CHAIN
    /* Fill in the pc and argument information for up to NFRAMES of my  */
    /* callers.  Ignore my frame and my callers frame.                  */
    GC_INNER void GC_save_callers(struct callinfo info[NFRAMES]);
# endif

  GC_INNER void GC_print_callers(struct callinfo info[NFRAMES]);
#endif /* NEED_CALLINFO */

EXTERN_C_END

/*********************************/
/*                               */
/* OS interface routines         */
/*                               */
/*********************************/

#ifndef NO_CLOCK
#ifdef BSD_TIME
# undef CLOCK_TYPE
# undef GET_TIME
# undef MS_TIME_DIFF
# define CLOCK_TYPE struct timeval
# define CLOCK_TYPE_INITIALIZER { 0, 0 }
# define GET_TIME(x) \
                do { \
                  struct rusage rusage; \
                  getrusage(RUSAGE_SELF, &rusage); \
                  x = rusage.ru_utime; \
                } while (0)
# define MS_TIME_DIFF(a,b) ((unsigned long)((long)(a.tv_sec-b.tv_sec) * 1000 \
                + (long)(a.tv_usec - b.tv_usec) / 1000 \
                - (a.tv_usec < b.tv_usec \
                   && (long)(a.tv_usec - b.tv_usec) % 1000 != 0 ? 1 : 0)))
                            /* "a" time is expected to be not earlier than  */
                            /* "b" one; the result has unsigned long type.  */
# define NS_FRAC_TIME_DIFF(a, b) ((unsigned long) \
                ((a.tv_usec < b.tv_usec \
                  && (long)(a.tv_usec - b.tv_usec) % 1000 != 0 ? 1000L : 0) \
                 + (long)(a.tv_usec - b.tv_usec) % 1000) * 1000)
                        /* The total time difference could be computed as   */
                        /* MS_TIME_DIFF(a,b)*1000000+NS_FRAC_TIME_DIFF(a,b).*/

#elif defined(MSWIN32) || defined(MSWINCE) || defined(WINXP_USE_PERF_COUNTER)
# if defined(MSWINRT_FLAVOR) || defined(WINXP_USE_PERF_COUNTER)
#   define CLOCK_TYPE ULONGLONG
#   define GET_TIME(x) \
                do { \
                  LARGE_INTEGER freq, tc; \
                  if (!QueryPerformanceFrequency(&freq)) \
                    ABORT("QueryPerformanceFrequency requires WinXP+"); \
                  /* Note: two standalone if statements are needed to   */ \
                  /* avoid MS VC false warning about potentially        */ \
                  /* uninitialized tc variable.                         */ \
                  if (!QueryPerformanceCounter(&tc)) \
                    ABORT("QueryPerformanceCounter failed"); \
                  x = (CLOCK_TYPE)((double)tc.QuadPart/freq.QuadPart * 1e9); \
                } while (0)
                /* TODO: Call QueryPerformanceFrequency once at GC init. */
#   define MS_TIME_DIFF(a, b) ((unsigned long)(((a) - (b)) / 1000000UL))
#   define NS_FRAC_TIME_DIFF(a, b) ((unsigned long)(((a) - (b)) % 1000000UL))
# else
#   define CLOCK_TYPE DWORD
#   define GET_TIME(x) (void)(x = GetTickCount())
#   define MS_TIME_DIFF(a, b) ((unsigned long)((a) - (b)))
#   define NS_FRAC_TIME_DIFF(a, b) 0UL
# endif /* !WINXP_USE_PERF_COUNTER */

#elif defined(NN_PLATFORM_CTR)
# define CLOCK_TYPE long long
  EXTERN_C_BEGIN
  CLOCK_TYPE n3ds_get_system_tick(void);
  CLOCK_TYPE n3ds_convert_tick_to_ms(CLOCK_TYPE tick);
  EXTERN_C_END
# define GET_TIME(x) (void)(x = n3ds_get_system_tick())
# define MS_TIME_DIFF(a,b) ((unsigned long)n3ds_convert_tick_to_ms((a)-(b)))
# define NS_FRAC_TIME_DIFF(a, b) 0UL /* TODO: implement it */

#elif defined(HAVE_CLOCK_GETTIME)
# include <time.h>
# define CLOCK_TYPE struct timespec
# define CLOCK_TYPE_INITIALIZER { 0, 0 }
# if defined(_POSIX_MONOTONIC_CLOCK) && !defined(NINTENDO_SWITCH)
#   define GET_TIME(x) \
                do { \
                  if (clock_gettime(CLOCK_MONOTONIC, &x) == -1) \
                    ABORT("clock_gettime failed"); \
                } while (0)
# else
#   define GET_TIME(x) \
                do { \
                  if (clock_gettime(CLOCK_REALTIME, &x) == -1) \
                    ABORT("clock_gettime failed"); \
                } while (0)
# endif
# define MS_TIME_DIFF(a, b) \
    /* a.tv_nsec - b.tv_nsec is in range -1e9 to 1e9 exclusively */ \
    ((unsigned long)((a).tv_nsec + (1000000L*1000 - (b).tv_nsec)) / 1000000UL \
     + ((unsigned long)((a).tv_sec - (b).tv_sec) * 1000UL) - 1000UL)
# define NS_FRAC_TIME_DIFF(a, b) \
    ((unsigned long)((a).tv_nsec + (1000000L*1000 - (b).tv_nsec)) % 1000000UL)

#else /* !BSD_TIME && !LINUX && !NN_PLATFORM_CTR && !MSWIN32 */
# include <time.h>
# if defined(FREEBSD) && !defined(CLOCKS_PER_SEC)
#   include <machine/limits.h>
#   define CLOCKS_PER_SEC CLK_TCK
# endif
# if !defined(CLOCKS_PER_SEC)
#   define CLOCKS_PER_SEC 1000000
    /* This is technically a bug in the implementation.                 */
    /* ANSI requires that CLOCKS_PER_SEC be defined.  But at least      */
    /* under SunOS 4.1.1, it isn't.  Also note that the combination of  */
    /* ANSI C and POSIX is incredibly gross here.  The type clock_t     */
    /* is used by both clock() and times().  But on some machines       */
    /* these use different notions of a clock tick, CLOCKS_PER_SEC      */
    /* seems to apply only to clock.  Hence we use it here.  On many    */
    /* machines, including SunOS, clock actually uses units of          */
    /* microseconds (which are not really clock ticks).                 */
# endif
# define CLOCK_TYPE clock_t
# define GET_TIME(x) (void)(x = clock())
# define MS_TIME_DIFF(a,b) (CLOCKS_PER_SEC % 1000 == 0 ? \
        (unsigned long)((a) - (b)) / (unsigned long)(CLOCKS_PER_SEC / 1000) \
        : ((unsigned long)((a) - (b)) * 1000) / (unsigned long)CLOCKS_PER_SEC)
  /* Avoid using double type since some targets (like ARM) might        */
  /* require -lm option for double-to-long conversion.                  */
# define NS_FRAC_TIME_DIFF(a, b) (CLOCKS_PER_SEC <= 1000 ? 0UL \
    : (unsigned long)(CLOCKS_PER_SEC <= (clock_t)1000000UL \
        ? (((a) - (b)) * ((clock_t)1000000UL / CLOCKS_PER_SEC) % 1000) * 1000 \
        : (CLOCKS_PER_SEC <= (clock_t)1000000UL * 1000 \
            ? ((a) - (b)) * ((clock_t)1000000UL * 1000 / CLOCKS_PER_SEC) \
            : (((a) - (b)) * (clock_t)1000000UL * 1000) / CLOCKS_PER_SEC) \
          % (clock_t)1000000UL))
#endif /* !BSD_TIME && !MSWIN32 */
# ifndef CLOCK_TYPE_INITIALIZER
    /* This is used to initialize CLOCK_TYPE variables (to some value)  */
    /* to avoid "variable might be uninitialized" compiler warnings.    */
#   define CLOCK_TYPE_INITIALIZER 0
# endif
#endif /* !NO_CLOCK */

/* We use bzero and bcopy internally.  They may not be available.       */
# if defined(SPARC) && defined(SUNOS4) \
     || (defined(M68K) && defined(NEXT)) || defined(VAX)
#   define BCOPY_EXISTS
# elif defined(AMIGA) || defined(DARWIN)
#   include <string.h>
#   define BCOPY_EXISTS
# elif defined(MACOS) && defined(POWERPC)
#   include <MacMemory.h>
#   define bcopy(x,y,n) BlockMoveData(x, y, n)
#   define bzero(x,n) BlockZero(x, n)
#   define BCOPY_EXISTS
# endif

# if !defined(BCOPY_EXISTS) || defined(CPPCHECK)
#   include <string.h>
#   define BCOPY(x,y,n) memcpy(y, x, (size_t)(n))
#   define BZERO(x,n)  memset(x, 0, (size_t)(n))
# else
#   define BCOPY(x,y,n) bcopy((void *)(x),(void *)(y),(size_t)(n))
#   define BZERO(x,n) bzero((void *)(x),(size_t)(n))
# endif

#ifdef PCR
# include "th/PCR_ThCtl.h"
#endif

EXTERN_C_BEGIN

#if defined(CPPCHECK) && defined(ANY_MSWIN)
# undef TEXT
# ifdef UNICODE
#   define TEXT(s) L##s
# else
#   define TEXT(s) s
# endif
#endif /* CPPCHECK && ANY_MSWIN */

/*
 * Stop and restart mutator threads.
 */
# ifdef PCR
#     define STOP_WORLD() \
        PCR_ThCtl_SetExclusiveMode(PCR_ThCtl_ExclusiveMode_stopNormal, \
                                   PCR_allSigsBlocked, \
                                   PCR_waitForever)
#     define START_WORLD() \
        PCR_ThCtl_SetExclusiveMode(PCR_ThCtl_ExclusiveMode_null, \
                                   PCR_allSigsBlocked, \
                                   PCR_waitForever)
# else
#   if defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH) \
       || defined(GC_WIN32_THREADS) || defined(GC_PTHREADS)
      GC_INNER void GC_stop_world(void);
      GC_INNER void GC_start_world(void);
#     define STOP_WORLD() GC_stop_world()
#     define START_WORLD() GC_start_world()
#   else
        /* Just do a sanity check: we are not inside GC_do_blocking().  */
#     define STOP_WORLD() GC_ASSERT(GC_blocked_sp == NULL)
#     define START_WORLD()
#   endif
# endif

/* Abandon ship */
# if defined(SMALL_CONFIG) || defined(PCR)
#   define GC_on_abort(msg) (void)0 /* be silent on abort */
# else
    GC_API_PRIV GC_abort_func GC_on_abort;
# endif
# if defined(CPPCHECK)
#   define ABORT(msg) { GC_on_abort(msg); abort(); }
# elif defined(PCR)
#   define ABORT(s) PCR_Base_Panic(s)
# else
#   if defined(MSWIN_XBOX1) && !defined(DebugBreak)
#     define DebugBreak() __debugbreak()
#   elif defined(MSWINCE) && !defined(DebugBreak) \
       && (!defined(UNDER_CE) || (defined(__MINGW32CE__) && !defined(ARM32)))
      /* This simplifies linking for WinCE (and, probably, doesn't      */
      /* hurt debugging much); use -DDebugBreak=DebugBreak to override  */
      /* this behavior if really needed.  This is also a workaround for */
      /* x86mingw32ce toolchain (if it is still declaring DebugBreak()  */
      /* instead of defining it as a macro).                            */
#     define DebugBreak() _exit(-1) /* there is no abort() in WinCE */
#   endif
#   if defined(MSWIN32) && (defined(NO_DEBUGGING) || defined(LINT2))
      /* A more user-friendly abort after showing fatal message.        */
#     define ABORT(msg) (GC_on_abort(msg), _exit(-1))
                /* Exit on error without running "at-exit" callbacks.   */
#   elif defined(MSWINCE) && defined(NO_DEBUGGING)
#     define ABORT(msg) (GC_on_abort(msg), ExitProcess(-1))
#   elif defined(MSWIN32) || defined(MSWINCE)
#     if defined(_CrtDbgBreak) && defined(_DEBUG) && defined(_MSC_VER)
#       define ABORT(msg) { GC_on_abort(msg); \
                            _CrtDbgBreak() /* __debugbreak() */; }
#     else
#       define ABORT(msg) { GC_on_abort(msg); DebugBreak(); }
                /* Note that: on a WinCE box, this could be silently    */
                /* ignored (i.e., the program is not aborted);          */
                /* DebugBreak is a statement in some toolchains.        */
#     endif
#   else
#     define ABORT(msg) (GC_on_abort(msg), abort())
#   endif /* !MSWIN32 */
# endif /* !PCR */

/* For abort message with 1-3 arguments.  C_msg and C_fmt should be     */
/* literals.  C_msg should not contain format specifiers.  Arguments    */
/* should match their format specifiers.                                */
#define ABORT_ARG1(C_msg, C_fmt, arg1) \
                MACRO_BLKSTMT_BEGIN \
                  GC_ERRINFO_PRINTF(C_msg /* + */ C_fmt "\n", arg1); \
                  ABORT(C_msg); \
                MACRO_BLKSTMT_END
#define ABORT_ARG2(C_msg, C_fmt, arg1, arg2) \
                MACRO_BLKSTMT_BEGIN \
                  GC_ERRINFO_PRINTF(C_msg /* + */ C_fmt "\n", arg1, arg2); \
                  ABORT(C_msg); \
                MACRO_BLKSTMT_END
#define ABORT_ARG3(C_msg, C_fmt, arg1, arg2, arg3) \
                MACRO_BLKSTMT_BEGIN \
                  GC_ERRINFO_PRINTF(C_msg /* + */ C_fmt "\n", \
                                    arg1, arg2, arg3); \
                  ABORT(C_msg); \
                MACRO_BLKSTMT_END

/* Same as ABORT but does not have 'no-return' attribute.       */
/* ABORT on a dummy condition (which is always true).           */
#define ABORT_RET(msg) \
    if ((GC_funcptr_uint)GC_current_warn_proc == ~(GC_funcptr_uint)0) {} \
    else ABORT(msg)

/* Exit abnormally, but without making a mess (e.g. out of memory) */
# ifdef PCR
#   define EXIT() PCR_Base_Exit(1,PCR_waitForever)
# else
#   define EXIT() (GC_on_abort(NULL), exit(1 /* EXIT_FAILURE */))
# endif

/* Print warning message, e.g. almost out of memory.    */
/* The argument (if any) format specifier should be:    */
/* "%s", "%p", "%"WARN_PRIdPTR or "%"WARN_PRIuPTR.      */
#define WARN(msg, arg) \
                GC_current_warn_proc("GC Warning: " msg, (GC_uintptr_t)(arg))
GC_EXTERN GC_warn_proc GC_current_warn_proc;

/* Print format type macro for decimal signed_word value passed WARN(). */
/* This could be redefined for Win64 or LLP64, but typically should     */
/* not be done as the WARN format string is, possibly, processed on the */
/* client side, so non-standard print type modifiers (like MS "I64d")   */
/* should be avoided here if possible.                                  */
/* TODO: Assuming sizeof(void*) == sizeof(long) or a little-endian machine. */
#ifndef WARN_PRIdPTR
# define WARN_PRIdPTR "ld"
# define WARN_PRIuPTR "lu"
#endif

/* A tagging macro (for a code static analyzer) to indicate that the    */
/* string obtained from an untrusted source (e.g., argv[], getenv) is   */
/* safe to use in a vulnerable operation (e.g., open, exec).            */
#define TRUSTED_STRING(s) COVERT_DATAFLOW_P(s)

/* Get environment entry */
#ifdef GC_READ_ENV_FILE
  GC_INNER char * GC_envfile_getenv(const char *name);
# define GETENV(name) GC_envfile_getenv(name)
#elif defined(NO_GETENV) && !defined(CPPCHECK)
# define GETENV(name) NULL
#elif defined(EMPTY_GETENV_RESULTS)
  /* Workaround for a reputed Wine bug.   */
  GC_INLINE char * fixed_getenv(const char *name)
  {
    char *value = getenv(name);
    return value != NULL && *value != '\0' ? value : NULL;
  }
# define GETENV(name) fixed_getenv(name)
#else
# define GETENV(name) getenv(name)
#endif

EXTERN_C_END

#if defined(DARWIN)
# include <mach/thread_status.h>
# ifndef MAC_OS_X_VERSION_MAX_ALLOWED
#   include <AvailabilityMacros.h>
                /* Include this header just to import the above macro.  */
# endif
# if defined(POWERPC)
#   if CPP_WORDSZ == 32
#     define GC_THREAD_STATE_T          ppc_thread_state_t
#   else
#     define GC_THREAD_STATE_T          ppc_thread_state64_t
#     define GC_MACH_THREAD_STATE       PPC_THREAD_STATE64
#     define GC_MACH_THREAD_STATE_COUNT PPC_THREAD_STATE64_COUNT
#   endif
# elif defined(I386) || defined(X86_64)
#   if CPP_WORDSZ == 32
#     if defined(i386_THREAD_STATE_COUNT) && !defined(x86_THREAD_STATE32_COUNT)
        /* Use old naming convention for i686.  */
#       define GC_THREAD_STATE_T                i386_thread_state_t
#       define GC_MACH_THREAD_STATE             i386_THREAD_STATE
#       define GC_MACH_THREAD_STATE_COUNT       i386_THREAD_STATE_COUNT
#     else
#       define GC_THREAD_STATE_T                x86_thread_state32_t
#       define GC_MACH_THREAD_STATE             x86_THREAD_STATE32
#       define GC_MACH_THREAD_STATE_COUNT       x86_THREAD_STATE32_COUNT
#     endif
#   else
#     define GC_THREAD_STATE_T          x86_thread_state64_t
#     define GC_MACH_THREAD_STATE       x86_THREAD_STATE64
#     define GC_MACH_THREAD_STATE_COUNT x86_THREAD_STATE64_COUNT
#   endif
# elif defined(ARM32) && defined(ARM_UNIFIED_THREAD_STATE) \
       && !defined(CPPCHECK)
#   define GC_THREAD_STATE_T            arm_unified_thread_state_t
#   define GC_MACH_THREAD_STATE         ARM_UNIFIED_THREAD_STATE
#   define GC_MACH_THREAD_STATE_COUNT   ARM_UNIFIED_THREAD_STATE_COUNT
# elif defined(ARM32)
#   define GC_THREAD_STATE_T            arm_thread_state_t
#   ifdef ARM_MACHINE_THREAD_STATE_COUNT
#     define GC_MACH_THREAD_STATE       ARM_MACHINE_THREAD_STATE
#     define GC_MACH_THREAD_STATE_COUNT ARM_MACHINE_THREAD_STATE_COUNT
#   endif
# elif defined(AARCH64)
#   define GC_THREAD_STATE_T            arm_thread_state64_t
#   define GC_MACH_THREAD_STATE         ARM_THREAD_STATE64
#   define GC_MACH_THREAD_STATE_COUNT   ARM_THREAD_STATE64_COUNT
# elif !defined(CPPCHECK)
#   error define GC_THREAD_STATE_T
# endif
# ifndef GC_MACH_THREAD_STATE
#   define GC_MACH_THREAD_STATE         MACHINE_THREAD_STATE
#   define GC_MACH_THREAD_STATE_COUNT   MACHINE_THREAD_STATE_COUNT
# endif

  /* Try to work out the right way to access thread state structure     */
  /* members.  The structure has changed its definition in different    */
  /* Darwin versions.  This now defaults to the (older) names           */
  /* without __, thus hopefully, not breaking any existing              */
  /* Makefile.direct builds.                                            */
# if __DARWIN_UNIX03
#   define THREAD_FLD_NAME(x) __ ## x
# else
#   define THREAD_FLD_NAME(x) x
# endif
# if defined(ARM32) && defined(ARM_UNIFIED_THREAD_STATE)
#   define THREAD_FLD(x) ts_32.THREAD_FLD_NAME(x)
# else
#   define THREAD_FLD(x) THREAD_FLD_NAME(x)
# endif
#endif /* DARWIN */

#ifndef WASI
# include <setjmp.h>
#endif

#include <stdio.h>

#if __STDC_VERSION__ >= 201112L
# include <assert.h> /* for static_assert */
#endif

EXTERN_C_BEGIN

/*********************************/
/*                               */
/* Word-size-dependent defines   */
/*                               */
/*********************************/

#define modWORDSZ(n) ((n) & (CPP_WORDSZ-1)) /* n mod size of word */
#define divWORDSZ(n) ((n) / CPP_WORDSZ)

#define SIGNB ((word)1 << (CPP_WORDSZ-1))
#define SIZET_SIGNB (GC_SIZE_MAX ^ (GC_SIZE_MAX >> 1))

#if CPP_PTRSZ / 8 != ALIGNMENT
# define UNALIGNED_PTRS
#endif

#define BYTES_TO_GRANULES(lb) ((lb) / GC_GRANULE_BYTES)
#define GRANULES_TO_BYTES(lg) ((lg) * GC_GRANULE_BYTES)
#define BYTES_TO_PTRS(lb) ((lb) / sizeof(ptr_t))
#define PTRS_TO_BYTES(lpw) ((lpw) * sizeof(ptr_t))
#define GRANULES_TO_PTRS(lg) ((lg) * GC_GRANULE_PTRS)

/* Convert size in bytes to that in pointers rounding up (but   */
/* not adding extra byte at end).                               */
#define BYTES_TO_PTRS_ROUNDUP(lb) BYTES_TO_PTRS((lb) + sizeof(ptr_t) - 1)

/*********************/
/*                   */
/*  Size Parameters  */
/*                   */
/*********************/

/* Heap block size, bytes. Should be power of 2.                */
/* Incremental GC with MPROTECT_VDB currently requires the      */
/* page size to be a multiple of HBLKSIZE.  Since most modern   */
/* architectures support variable page sizes down to 4 KB, and  */
/* i686 and x86_64 are generally 4 KB, we now default to 4 KB,  */
/* except for:                                                  */
/* - Alpha: seems to be used with 8 KB pages;                   */
/* - SMALL_CONFIG: want less block-level fragmentation.         */
#ifndef HBLKSIZE
# if defined(SMALL_CONFIG) && !defined(LARGE_CONFIG)
#   define CPP_LOG_HBLKSIZE 10
# elif defined(ALPHA)
#   define CPP_LOG_HBLKSIZE 13
# else
#   define CPP_LOG_HBLKSIZE 12
# endif
#else
# if HBLKSIZE == 512
#   define CPP_LOG_HBLKSIZE 9
# elif HBLKSIZE == 1024
#   define CPP_LOG_HBLKSIZE 10
# elif HBLKSIZE == 2048
#   define CPP_LOG_HBLKSIZE 11
# elif HBLKSIZE == 4096
#   define CPP_LOG_HBLKSIZE 12
# elif HBLKSIZE == 8192
#   define CPP_LOG_HBLKSIZE 13
# elif HBLKSIZE == 16384
#   define CPP_LOG_HBLKSIZE 14
# elif HBLKSIZE == 32768
#   define CPP_LOG_HBLKSIZE 15
# elif HBLKSIZE == 65536
#   define CPP_LOG_HBLKSIZE 16
# elif !defined(CPPCHECK)
#   error Bad HBLKSIZE value
# endif
# undef HBLKSIZE
#endif

#define LOG_HBLKSIZE ((size_t)CPP_LOG_HBLKSIZE)
#define HBLKSIZE ((size_t)1 << CPP_LOG_HBLKSIZE)

#define GC_SQRT_SIZE_MAX ((((size_t)1) << (sizeof(size_t) * 8 / 2)) - 1)

/*  Max size objects supported by free list (larger objects are */
/*  allocated directly with allchblk(), by rounding to the next */
/*  multiple of HBLKSIZE).                                      */
#define MAXOBJBYTES (HBLKSIZE >> 1)
#define MAXOBJGRANULES BYTES_TO_GRANULES(MAXOBJBYTES)

#define divHBLKSZ(n) ((n) >> LOG_HBLKSIZE)

#define HBLK_PTR_DIFF(p,q) divHBLKSZ((ptr_t)p - (ptr_t)q)
        /* Equivalent to subtracting 2 hblk pointers.   */
        /* We do it this way because a compiler should  */
        /* find it hard to use an integer division      */
        /* instead of a shift.  The bundled SunOS 4.1   */
        /* otherwise sometimes pessimizes the           */
        /* subtraction to involve a call to .div.       */

#define modHBLKSZ(n) ((n) & (HBLKSIZE-1))

#define HBLKPTR(objptr) ((struct hblk *)PTR_ALIGN_DOWN(objptr, HBLKSIZE))
#define HBLKDISPL(objptr) modHBLKSZ((size_t)(objptr))

/* Same as HBLKPTR() but points to the first block in the page.     */
#define HBLK_PAGE_ALIGNED(objptr) \
                ((struct hblk *)PTR_ALIGN_DOWN(objptr, GC_page_size))

/* Round up allocation size (in bytes) to a multiple of a granule.      */
#define ROUNDUP_GRANULE_SIZE(lb) /* lb should have no side-effect */ \
        (SIZET_SAT_ADD(lb, GC_GRANULE_BYTES-1) \
         & ~(size_t)(GC_GRANULE_BYTES-1))

/* Round up byte allocation request (after adding EXTRA_BYTES) to   */
/* a multiple of a granule, then convert it to granules.            */
#define ALLOC_REQUEST_GRANS(lb) /* lb should have no side-effect */ \
        BYTES_TO_GRANULES(SIZET_SAT_ADD(lb, GC_GRANULE_BYTES-1 + EXTRA_BYTES))

#if MAX_EXTRA_BYTES == 0
# define ADD_EXTRA_BYTES(lb) (lb)
# define SMALL_OBJ(lb) EXPECT((lb) <= MAXOBJBYTES, TRUE)
#else
# define ADD_EXTRA_BYTES(lb) /* lb should have no side-effect */ \
            SIZET_SAT_ADD(lb, EXTRA_BYTES)
# define SMALL_OBJ(lb) /* lb should have no side-effect */ \
            (EXPECT((lb) <= MAXOBJBYTES - MAX_EXTRA_BYTES, TRUE) \
             || (lb) <= MAXOBJBYTES - EXTRA_BYTES)
        /* This really just tests lb <= MAXOBJBYTES - EXTRA_BYTES, but  */
        /* we try to avoid looking up EXTRA_BYTES.                      */
#endif

/* Hash table representation of sets of pages.  Implements a map from   */
/* aligned HBLKSIZE chunks of the address space to one bit each.        */
/* This assumes it is OK to spuriously set bits, e.g. because multiple  */
/* addresses are represented by a single location.  Used by             */
/* black-listing code, and perhaps by dirty bit maintenance code.       */
#ifndef LOG_PHT_ENTRIES
# ifdef LARGE_CONFIG
#   if CPP_WORDSZ == 32
#     define LOG_PHT_ENTRIES 20 /* Collisions are impossible (because   */
                                /* of a 4 GB space limit).  Each table  */
                                /* takes 128 KB, some of which may      */
                                /* never be touched.                    */
#   else
#     define LOG_PHT_ENTRIES 21 /* Collisions likely at 2M blocks,      */
                                /* which is >= 8 GB.  Each table takes  */
                                /* 256 KB, some of which may never be   */
                                /* touched.                             */
#   endif
# elif !defined(SMALL_CONFIG)
#   define LOG_PHT_ENTRIES  18   /* Collisions are likely if heap grows */
                                 /* to more than 256K hblks >= 1 GB.    */
                                 /* Each hash table occupies 32 KB.     */
                                 /* Even for somewhat smaller heaps,    */
                                 /* say half that, collisions may be an */
                                 /* issue because we blacklist          */
                                 /* addresses outside the heap.         */
# else
#   define LOG_PHT_ENTRIES  15   /* Collisions are likely if heap grows */
                                 /* to more than 32K hblks (128 MB).    */
                                 /* Each hash table occupies 4 KB.      */
# endif
#endif /* !LOG_PHT_ENTRIES */

#define PHT_ENTRIES (1 << LOG_PHT_ENTRIES)
#define PHT_SIZE (PHT_ENTRIES > CPP_WORDSZ ? PHT_ENTRIES / CPP_WORDSZ : 1)
typedef word page_hash_table[PHT_SIZE];

#define PHT_HASH(p) ((size_t)((ADDR(p) >> LOG_HBLKSIZE) & (PHT_ENTRIES-1)))

#define get_pht_entry_from_index(bl, index) \
                (((bl)[divWORDSZ(index)] >> modWORDSZ(index)) & 1)
#define set_pht_entry_from_index(bl, index) \
                (void)((bl)[divWORDSZ(index)] |= (word)1 << modWORDSZ(index))

#if defined(THREADS) && defined(AO_HAVE_or)
  /* And, one more version for GC_add_to_black_list_normal/stack        */
  /* (invoked indirectly by GC_do_local_mark) and                       */
  /* async_set_pht_entry_from_index (invoked by GC_dirty or the write   */
  /* fault handler).                                                    */
# define set_pht_entry_from_index_concurrent(bl, index) \
                AO_or((volatile AO_t *)&(bl)[divWORDSZ(index)], \
                      (AO_t)1 << modWORDSZ(index))
# ifdef MPROTECT_VDB
#   define set_pht_entry_from_index_concurrent_volatile(bl, index) \
                set_pht_entry_from_index_concurrent(bl, index)
# endif
#else
# define set_pht_entry_from_index_concurrent(bl, index) \
                set_pht_entry_from_index(bl, index)
# ifdef MPROTECT_VDB
    /* Same as set_pht_entry_from_index() but avoiding the compound */
    /* assignment for a volatile array.                             */
#   define set_pht_entry_from_index_concurrent_volatile(bl, index) \
                (void)((bl)[divWORDSZ(index)] \
                    = (bl)[divWORDSZ(index)] | ((word)1 << modWORDSZ(index)))
# endif
#endif

/********************************************/
/*                                          */
/*    H e a p   B l o c k s                 */
/*                                          */
/********************************************/

#define MARK_BITS_PER_HBLK (HBLKSIZE/GC_GRANULE_BYTES)
                /* The upper bound.  We allocate 1 bit per allocation   */
                /* granule.  If MARK_BIT_PER_OBJ is not defined, we use */
                /* every n-th bit, where n is the number of allocation  */
                /* granules per object.  Otherwise, we only use the     */
                /* initial group of mark bits, and it is safe to        */
                /* allocate smaller header for large objects.           */

/* We maintain layout maps for heap blocks containing objects of a given */
/* size.  Each entry in this map describes a byte offset and has the     */
/* following type.                                                       */
struct hblkhdr {
    struct hblk * hb_next;      /* Link field for hblk free list         */
                                /* and for lists of chunks waiting to be */
                                /* reclaimed.                            */
    struct hblk * hb_prev;      /* Backwards link for free list.        */
    struct hblk * hb_block;     /* The corresponding block.             */
    unsigned char hb_obj_kind;
                         /* Kind of objects in the block.  Each kind    */
                         /* identifies a mark procedure and a set of    */
                         /* list headers.  Sometimes called regions.    */
    unsigned char hb_flags;
#       define IGNORE_OFF_PAGE  1       /* Ignore pointers that do not  */
                                        /* point to the first hblk of   */
                                        /* this object.                 */
#       define WAS_UNMAPPED 2   /* This is a free block, which has      */
                                /* been unmapped from the address       */
                                /* space.                               */
                                /* GC_remap must be invoked on it       */
                                /* before it can be reallocated.        */
                                /* Only set with USE_MUNMAP.            */
#       define FREE_BLK 4       /* Block is free, i.e. not in use.      */
#       ifdef ENABLE_DISCLAIM
#         define HAS_DISCLAIM 8
                                /* This kind has a callback on reclaim. */
#         define MARK_UNCONDITIONALLY 0x10
                                /* Mark from all objects, marked or     */
                                /* not.  Used to mark objects needed by */
                                /* reclaim notifier.                    */
#       endif
#       ifndef MARK_BIT_PER_OBJ
#         define LARGE_BLOCK 0x20
#       endif
    unsigned short hb_last_reclaimed;
                                /* Value of GC_gc_no when block was     */
                                /* last allocated or swept. May wrap.   */
                                /* For a free block, this is maintained */
                                /* only for USE_MUNMAP, and indicates   */
                                /* when the header was allocated, or    */
                                /* when the size of the block last      */
                                /* changed.                             */
#   ifdef MARK_BIT_PER_OBJ
      unsigned32 hb_inv_sz;     /* A good upper bound for 2**32/hb_sz.  */
                                /* For large objects, we use            */
                                /* LARGE_INV_SZ.                        */
#     define LARGE_INV_SZ ((unsigned32)1 << 16)
#   endif
    size_t hb_sz;
                /* If in use, size in bytes, of objects in the block.   */
                /* Otherwise, the size of the whole free block.         */
                /* We assume that this is convertible to signed_word    */
                /* without generating a negative result.  We avoid      */
                /* generating free blocks larger than that.             */
    word hb_descr;              /* Object descriptor for marking.       */
                                /* See gc_mark.h.                       */
#   ifndef MARK_BIT_PER_OBJ
      unsigned short * hb_map;  /* Essentially a table of remainders    */
                                /* mod BYTES_TO_GRANULES(hb_sz), except */
                                /* for large blocks.  See GC_obj_map.   */
#   endif
#   ifdef PARALLEL_MARK
      volatile AO_t hb_n_marks; /* Number of set mark bits, excluding   */
                                /* the one always set at the end.       */
                                /* Currently it is concurrently         */
                                /* updated and hence only approximate.  */
                                /* But a zero value does guarantee that */
                                /* the block contains no marked         */
                                /* objects.                             */
                                /* Ensuring this property means that we */
                                /* never decrement it to zero during a  */
                                /* collection, and hence the count may  */
                                /* be one too high.  Due to concurrent  */
                                /* updates, an arbitrary number of      */
                                /* increments, but not all of them (!)  */
                                /* may be lost, hence it may in theory  */
                                /* be much too low.                     */
                                /* The count may also be too high if    */
                                /* multiple mark threads mark the       */
                                /* same object due to a race.           */
#   else
      size_t hb_n_marks;        /* Without parallel marking, the count  */
#   endif                       /* is accurate.                         */
#   ifdef USE_MARK_BYTES
#     define HB_MARKS_SZ (MARK_BITS_PER_HBLK + 1)
        /* Unlike the other case, this is in units of bytes.  Since we  */
        /* force certain alignment, we need at most one mark bit per    */
        /* a granule.  But we do allocate and set one extra mark bit to */
        /* avoid an explicit check for the partial object at the end of */
        /* each block.                                                  */
      union {
        char _hb_marks[HB_MARKS_SZ];
                            /* The i'th byte is 1 if the object         */
                            /* starting at granule i or object i is     */
                            /* marked, 0 otherwise.                     */
                            /* The mark bit for the "one past the end"  */
                            /* object is always set to avoid a special  */
                            /* case test in the marker.                 */
        word dummy;     /* Force word alignment of mark bytes. */
      } _mark_byte_union;
#     define hb_marks _mark_byte_union._hb_marks
#   else
#     define HB_MARKS_SZ (MARK_BITS_PER_HBLK / CPP_WORDSZ + 1)
#     if defined(PARALLEL_MARK) \
         || (defined(THREAD_SANITIZER) && defined(THREADS))
        volatile AO_t hb_marks[HB_MARKS_SZ];
#     else
        word hb_marks[HB_MARKS_SZ];
#     endif
#   endif /* !USE_MARK_BYTES */
};

# define ANY_INDEX 23   /* "Random" mark bit index for assertions */

/*  heap block body */

# define HBLK_WORDS (HBLKSIZE/sizeof(word))
# define HBLK_GRANULES (HBLKSIZE/GC_GRANULE_BYTES)

/* The number of objects in a block dedicated to a certain size.        */
/* may erroneously yield zero (instead of one) for large objects.       */
# define HBLK_OBJS(sz_in_bytes) (HBLKSIZE/(sz_in_bytes))

struct hblk {
    char hb_body[HBLKSIZE];
};

# define HBLK_IS_FREE(hhdr) (((hhdr) -> hb_flags & FREE_BLK) != 0)

# define OBJ_SZ_TO_BLOCKS(lb) divHBLKSZ((lb) + HBLKSIZE-1)
# define OBJ_SZ_TO_BLOCKS_CHECKED(lb) /* lb should have no side-effect */ \
                                divHBLKSZ(SIZET_SAT_ADD(lb, HBLKSIZE-1))
    /* Size of block (in units of HBLKSIZE) needed to hold objects of   */
    /* given lb (in bytes).  The checked variant prevents wrap around.  */

/* The object free-list link.   */
# define obj_link(p) (*(void **)(p))

# define LOG_MAX_MARK_PROCS 6
# define MAX_MARK_PROCS (1 << LOG_MAX_MARK_PROCS)

/* Root sets.  Logically private to mark_rts.c.  But we don't want the  */
/* tables scanned, so we put them here.                                 */
/* MAX_ROOT_SETS is the maximum number of ranges that can be    */
/* registered as static roots.                                  */
# ifdef LARGE_CONFIG
#   define MAX_ROOT_SETS 8192
# elif !defined(SMALL_CONFIG)
#   define MAX_ROOT_SETS 2048
# else
#   define MAX_ROOT_SETS 512
# endif

# define MAX_EXCLUSIONS (MAX_ROOT_SETS/4)
/* Maximum number of segments that can be excluded from root sets.      */

/*
 * Data structure for excluded static roots.
 */
struct exclusion {
    ptr_t e_start;
    ptr_t e_end;
};

/* Data structure for list of root sets.                                */
/* We keep a hash table, so that we can filter out duplicate additions. */
/* Under Win32, we need to do a better job of filtering overlaps, so    */
/* we resort to sequential search, and pay the price.                   */
struct roots {
        ptr_t r_start;/* multiple of pointer size */
        ptr_t r_end;  /* multiple of pointer size and greater than r_start */
#       ifndef ANY_MSWIN
          struct roots * r_next;
#       endif
        GC_bool r_tmp;
                /* Delete before registering new dynamic libraries if set. */
};

#ifndef ANY_MSWIN
    /* Size of hash table index to roots.       */
#   define LOG_RT_SIZE 6
#   define RT_SIZE (1 << LOG_RT_SIZE) /* Power of 2, may be != MAX_ROOT_SETS */
#endif /* !ANY_MSWIN */

#if (!defined(MAX_HEAP_SECTS) || defined(CPPCHECK)) \
    && (defined(ANY_MSWIN) || defined(USE_PROC_FOR_LIBRARIES))
# ifdef LARGE_CONFIG
#   if CPP_WORDSZ > 32
#     define MAX_HEAP_SECTS 81920
#   else
#     define MAX_HEAP_SECTS 7680
#   endif
# elif defined(SMALL_CONFIG) && !defined(USE_PROC_FOR_LIBRARIES)
#   if defined(PARALLEL_MARK) && (defined(MSWIN32) || defined(CYGWIN32))
#     define MAX_HEAP_SECTS 384
#   else
#     define MAX_HEAP_SECTS 128         /* Roughly 256 MB (128*2048*1024) */
#   endif
# elif CPP_WORDSZ > 32
#   define MAX_HEAP_SECTS 1024          /* Roughly 8 GB */
# else
#   define MAX_HEAP_SECTS 512           /* Roughly 4 GB */
# endif
#endif /* !MAX_HEAP_SECTS */

typedef struct GC_ms_entry {
  ptr_t mse_start;      /* Beginning of object, pointer-aligned one.    */
# ifdef PARALLEL_MARK
    volatile AO_t mse_descr;
# else
    word mse_descr;     /* Descriptor; low order two bits are tags,     */
# endif                 /* as described in gc_mark.h.                   */
} mse;

typedef int mark_state_t;   /* Current state of marking.                */
                            /* Used to remember where we are during     */
                            /* concurrent marking.                      */

struct disappearing_link;
struct finalizable_object;

struct dl_hashtbl_s {
    struct disappearing_link **head;
    size_t entries;
    unsigned log_size;
};

struct fnlz_roots_s {
  struct finalizable_object **fo_head;
  /* List of objects that should be finalized now: */
  struct finalizable_object *finalize_now;
};

union toggle_ref_u {
  /* The least significant bit is used to distinguish between choices.  */
  void *strong_ref;
  GC_hidden_pointer weak_ref;
};

/* Extended descriptors.  GC_typed_mark_proc understands these. */
/* These are used for simple objects that are larger than what  */
/* can be described by a BITMAP_BITS-sized bitmap.              */
typedef struct {
    word ed_bitmap; /* the least significant bit corresponds to first word. */
    GC_bool ed_continued;       /* next entry is continuation.  */
} typed_ext_descr_t;

struct HeapSect {
    ptr_t hs_start;
    size_t hs_bytes;
};

/* Lists of all heap blocks and free lists      */
/* as well as other random data structures      */
/* that should not be scanned by the            */
/* collector.                                   */
/* These are grouped together in a struct       */
/* so that they can be easily skipped by the    */
/* GC_mark routine.                             */
/* The ordering is weird to make GC_malloc      */
/* faster by keeping the important fields       */
/* sufficiently close together that a           */
/* single load of a base register will do.      */
/* Scalars that could easily appear to          */
/* be pointers are also put here.               */
/* The main fields should precede any           */
/* conditionally included fields, so that       */
/* gc_inline.h will work even if a different    */
/* set of macros is defined when the client is  */
/* compiled.                                    */

struct _GC_arrays {
  word _heapsize;       /* Heap size in bytes (value never goes down).  */
  word _requested_heapsize;     /* Heap size due to explicit expansion. */
# define GC_heapsize_on_gc_disable GC_arrays._heapsize_on_gc_disable
  word _heapsize_on_gc_disable;
  word _last_heap_addr;
  word _large_free_bytes;
        /* Total bytes contained in blocks on large object free */
        /* list.                                                */
  word _large_allocd_bytes;
        /* Total number of bytes in allocated large objects blocks.     */
        /* For the purposes of this counter and the next one only, a    */
        /* large object is one that occupies a block of at least        */
        /* 2*HBLKSIZE.                                                  */
  word _max_large_allocd_bytes;
        /* Maximum number of bytes that were ever allocated in          */
        /* large object blocks.  This is used to help decide when it    */
        /* is safe to split up a large block.                           */
  word _bytes_allocd_before_gc;
                /* Number of bytes allocated before this        */
                /* collection cycle.                            */
# define GC_our_mem_bytes GC_arrays._our_mem_bytes
  word _our_mem_bytes;
# ifndef SEPARATE_GLOBALS
#   define GC_bytes_allocd GC_arrays._bytes_allocd
    word _bytes_allocd;
        /* Number of bytes allocated during this collection cycle.      */
# endif
  word _bytes_dropped;
        /* Number of black-listed bytes dropped during GC cycle */
        /* as a result of repeated scanning during allocation   */
        /* attempts.  These are treated largely as allocated,   */
        /* even though they are not useful to the client.       */
  word _bytes_finalized;
        /* Approximate number of bytes in objects (and headers) */
        /* that became ready for finalization in the last       */
        /* collection.                                          */
  word _bytes_freed;
        /* Number of explicitly deallocated bytes of memory     */
        /* since last collection.                               */
  word _finalizer_bytes_freed;
        /* Bytes of memory explicitly deallocated while         */
        /* finalizers were running.  Used to approximate memory */
        /* explicitly deallocated by finalizers.                */
  bottom_index *_all_bottom_indices;
        /* Pointer to the first (lowest address) bottom_index;  */
        /* assumes the allocator lock is held.                  */
  bottom_index *_all_bottom_indices_end;
        /* Pointer to the last (highest address) bottom_index;  */
        /* assumes the allocator lock is held.                  */
  ptr_t _scratch_free_ptr;
  hdr *_hdr_free_list;
# define GC_scratch_end_addr GC_arrays._scratch_end_addr
  word _scratch_end_addr; /* the end point of the current scratch area */
# if defined(IRIX5) || (defined(USE_PROC_FOR_LIBRARIES) && !defined(LINUX))
#   define USE_SCRATCH_LAST_END_PTR
#   define GC_scratch_last_end_addr GC_arrays._scratch_last_end_addr
    word _scratch_last_end_addr;
        /* The address of the end point of the last obtained scratch    */
        /* area.  Used by GC_register_dynamic_libraries().              */
# endif
# if defined(GC_ASSERTIONS) || defined(MAKE_BACK_GRAPH) \
     || defined(INCLUDE_LINUX_THREAD_DESCR) \
     || (defined(KEEP_BACK_PTRS) && ALIGNMENT == 1)
#   define SET_REAL_HEAP_BOUNDS
#   define GC_least_real_heap_addr GC_arrays._least_real_heap_addr
#   define GC_greatest_real_heap_addr GC_arrays._greatest_real_heap_addr
    word _least_real_heap_addr;
    word _greatest_real_heap_addr;
        /* Similar to GC_least/greatest_plausible_heap_addr but */
        /* do not include future (potential) heap expansion.    */
        /* Both variables are zero initially.                   */
# endif
  mse *_mark_stack;
        /* Limits of stack for GC_mark routine.  All ranges     */
        /* between GC_mark_stack (incl.) and GC_mark_stack_top  */
        /* (incl.) still need to be marked from.                */
  mse *_mark_stack_limit;
# ifdef PARALLEL_MARK
    mse *volatile _mark_stack_top;
        /* Updated only with the mark lock held, but read asynchronously. */
# else
    mse *_mark_stack_top;
# endif
# ifdef DYNAMIC_POINTER_MASK
#   define GC_pointer_mask GC_arrays._pointer_mask
#   define GC_pointer_shift GC_arrays._pointer_shift
    word _pointer_mask; /* Both mask and shift are zeros by default;    */
                        /* if mask is zero then correct it to ~0 at GC  */
                        /* initialization.                              */
    unsigned char _pointer_shift;
# endif
# define GC_mark_stack_too_small GC_arrays._mark_stack_too_small
  GC_bool _mark_stack_too_small;
                /* We need a larger mark stack.  May be set by          */
                /* client-supplied mark routines.                       */
# define GC_objects_are_marked GC_arrays._objects_are_marked
  GC_bool _objects_are_marked;
                /* Are there collectible marked objects in the heap?    */
# ifdef THREADS
#   define GC_roots_were_cleared GC_arrays._roots_were_cleared
    GC_bool _roots_were_cleared;
# endif
# define GC_explicit_typing_initialized GC_arrays._explicit_typing_initialized
# ifdef AO_HAVE_load_acquire
    volatile AO_t _explicit_typing_initialized;
# else
    GC_bool _explicit_typing_initialized;
# endif
  word _composite_in_use;
                /* Number of bytes in the accessible composite objects. */
  word _atomic_in_use;
                /* Number of bytes in the accessible atomic objects.    */
# define GC_last_heap_growth_gc_no GC_arrays._last_heap_growth_gc_no
  word _last_heap_growth_gc_no;
                /* GC number of latest successful GC_expand_hp_inner call. */
# ifdef USE_MUNMAP
#   define GC_unmapped_bytes GC_arrays._unmapped_bytes
    word _unmapped_bytes;
# else
#   define GC_unmapped_bytes 0
# endif
# if defined(COUNT_UNMAPPED_REGIONS) && defined(USE_MUNMAP)
#   define GC_num_unmapped_regions GC_arrays._num_unmapped_regions
    signed_word _num_unmapped_regions;
# else
#   define GC_num_unmapped_regions 0
# endif
  bottom_index * _all_nils;
# define GC_scan_ptr GC_arrays._scan_ptr
  struct hblk * _scan_ptr;
# ifdef PARALLEL_MARK
#   define GC_main_local_mark_stack GC_arrays._main_local_mark_stack
    mse *_main_local_mark_stack;
#   define GC_first_nonempty GC_arrays._first_nonempty
    volatile ptr_t _first_nonempty;
                        /* Lowest entry on mark stack that may be       */
                        /* nonempty. Updated only by initiating thread. */
# endif
# ifdef ENABLE_TRACE
#   define GC_trace_ptr GC_arrays._trace_ptr
    ptr_t _trace_ptr;
# endif
# if CPP_PTRSZ > CPP_WORDSZ
#   define GC_noop_sink_ptr GC_arrays._noop_sink_ptr
    volatile ptr_t _noop_sink_ptr;
# endif
# define GC_noop_sink GC_arrays._noop_sink
# if defined(AO_HAVE_store) && defined(THREAD_SANITIZER)
    volatile AO_t _noop_sink;
# else
    volatile word _noop_sink;
# endif
# define GC_mark_stack_size GC_arrays._mark_stack_size
  size_t _mark_stack_size;
# define GC_mark_state GC_arrays._mark_state
  mark_state_t _mark_state; /* Initialized to MS_NONE (0). */
# define GC_capacity_heap_sects GC_arrays._capacity_heap_sects
  size_t _capacity_heap_sects;
# define GC_n_heap_sects GC_arrays._n_heap_sects
  size_t _n_heap_sects; /* number of separately added heap sections */
# ifdef ANY_MSWIN
#   define GC_n_heap_bases GC_arrays._n_heap_bases
    size_t _n_heap_bases; /* see GC_heap_bases[] */
# endif
# ifdef USE_PROC_FOR_LIBRARIES
#   define GC_n_memory GC_arrays._n_memory
    word _n_memory;     /* Number of GET_MEM allocated memory sections. */
# endif
# ifdef GC_GCJ_SUPPORT
#   define GC_gcjobjfreelist GC_arrays._gcjobjfreelist
    ptr_t *_gcjobjfreelist;
# endif
# define GC_fo_entries GC_arrays._fo_entries
  size_t _fo_entries;
# ifndef GC_NO_FINALIZATION
#   define GC_dl_hashtbl GC_arrays._dl_hashtbl
#   define GC_fnlz_roots GC_arrays._fnlz_roots
#   define GC_log_fo_table_size GC_arrays._log_fo_table_size
#   ifndef GC_LONG_REFS_NOT_NEEDED
#     define GC_ll_hashtbl GC_arrays._ll_hashtbl
      struct dl_hashtbl_s _ll_hashtbl;
#   endif
    struct dl_hashtbl_s _dl_hashtbl;
    struct fnlz_roots_s _fnlz_roots;
    unsigned _log_fo_table_size;
#   ifndef GC_TOGGLE_REFS_NOT_NEEDED
#     define GC_toggleref_arr GC_arrays._toggleref_arr
#     define GC_toggleref_array_size GC_arrays._toggleref_array_size
#     define GC_toggleref_array_capacity GC_arrays._toggleref_array_capacity
      union toggle_ref_u *_toggleref_arr;
      size_t _toggleref_array_size;
      size_t _toggleref_array_capacity;
#   endif
# endif
# ifdef TRACE_BUF
#   define GC_trace_buf_pos GC_arrays._trace_buf_pos
    size_t _trace_buf_pos; /* an index in the circular buffer */
# endif
# ifdef ENABLE_DISCLAIM
#   define GC_finalized_kind GC_arrays._finalized_kind
    unsigned _finalized_kind;
# endif
# define n_root_sets GC_arrays._n_root_sets
# define GC_excl_table_entries GC_arrays._excl_table_entries
  size_t _n_root_sets;  /* GC_static_roots[0..n_root_sets) contains the */
                        /* valid root sets.                             */
  size_t _excl_table_entries;   /* Number of entries in use.    */
# define GC_ed_size GC_arrays._ed_size
# define GC_avail_descr GC_arrays._avail_descr
# define GC_ext_descriptors GC_arrays._ext_descriptors
  size_t _ed_size;      /* Current size of above arrays.        */
  size_t _avail_descr;  /* Next available slot.                 */
  typed_ext_descr_t *_ext_descriptors;  /* Points to array of extended  */
                                        /* descriptors.                 */
  GC_mark_proc _mark_procs[MAX_MARK_PROCS];
        /* Table of user-defined mark procedures.  There is     */
        /* a small number of these, which can be referenced     */
        /* by DS_PROC mark descriptors.  See gc_mark.h.         */
  char _modws_valid_offsets[sizeof(ptr_t)];
                        /* GC_valid_offsets[i] ==>                  */
                        /* GC_modws_valid_offsets[i%sizeof(ptr_t)]  */
# ifndef ANY_MSWIN
#   define GC_root_index GC_arrays._root_index
    struct roots * _root_index[RT_SIZE];
# endif
# if defined(SAVE_CALL_CHAIN) && !defined(DONT_SAVE_TO_LAST_STACK) \
     && (!defined(REDIRECT_MALLOC) || !defined(GC_HAVE_BUILTIN_BACKTRACE))
    struct callinfo _last_stack[NFRAMES];
                /* Stack at last garbage collection.  Useful for        */
                /* debugging mysterious object disappearances.  In the  */
                /* multi-threaded case, we currently only save the      */
                /* calling stack.  Not supported in case of malloc      */
                /* redirection because backtrace() may call malloc().   */
#   define SAVE_CALLERS_TO_LAST_STACK() GC_save_callers(GC_arrays._last_stack)
# else
#   define SAVE_CALLERS_TO_LAST_STACK() (void)0
# endif
# ifndef SEPARATE_GLOBALS
#   define GC_objfreelist GC_arrays._objfreelist
    void *_objfreelist[MAXOBJGRANULES+1];
                          /* free list for objects */
#   define GC_aobjfreelist GC_arrays._aobjfreelist
    void *_aobjfreelist[MAXOBJGRANULES+1];
                          /* free list for atomic objects       */
# endif
  void *_uobjfreelist[MAXOBJGRANULES+1];
                          /* Uncollectible but traced objects.  */
                          /* Objects on this and _auobjfreelist */
                          /* are always marked, except during   */
                          /* garbage collections.               */
# ifdef GC_ATOMIC_UNCOLLECTABLE
#   define GC_auobjfreelist GC_arrays._auobjfreelist
    void *_auobjfreelist[MAXOBJGRANULES+1];
                /* Atomic uncollectible but traced objects.     */
# endif
  size_t _size_map[MAXOBJBYTES+1];
        /* Number of granules to allocate when asked for a certain      */
        /* number of bytes (plus EXTRA_BYTES).  Should be accessed with */
        /* the allocator lock held.                                     */
# ifndef MARK_BIT_PER_OBJ
#   define GC_obj_map GC_arrays._obj_map
    unsigned short * _obj_map[MAXOBJGRANULES+1];
                       /* If not NULL, then a pointer to a map of valid */
                       /* object addresses.  GC_obj_map[lg][i] is       */
                       /* i % lg.  This is now used purely to replace   */
                       /* a division in the marker by a table lookup.   */
                       /* _obj_map[0] is used for large objects and     */
                       /* contains all nonzero entries.  This gets us   */
                       /* out of the marker fast path without an extra  */
                       /* test.                                         */
#   define OBJ_MAP_LEN  BYTES_TO_GRANULES(HBLKSIZE)
# endif
# define VALID_OFFSET_SZ HBLKSIZE
  char _valid_offsets[VALID_OFFSET_SZ];
                                /* GC_valid_offsets[i] == TRUE ==> i    */
                                /* is registered as a displacement.     */
# ifndef GC_DISABLE_INCREMENTAL
#   define GC_grungy_pages GC_arrays._grungy_pages
    page_hash_table _grungy_pages; /* Pages that were dirty at last     */
                                   /* GC_read_dirty.                    */
#   define GC_dirty_pages GC_arrays._dirty_pages
#   ifdef MPROTECT_VDB
      volatile
#   endif
      page_hash_table _dirty_pages;
                        /* Pages dirtied since last GC_read_dirty. */
# endif
# if (defined(CHECKSUMS) && (defined(GWW_VDB) || defined(SOFT_VDB))) \
     || defined(PROC_VDB)
#   define GC_written_pages GC_arrays._written_pages
    page_hash_table _written_pages;     /* Pages ever dirtied   */
# endif
# define GC_heap_sects GC_arrays._heap_sects
  struct HeapSect *_heap_sects;         /* Heap segments potentially    */
                                        /* client objects.              */
# if defined(USE_PROC_FOR_LIBRARIES)
#   define GC_our_memory GC_arrays._our_memory
    struct HeapSect _our_memory[MAX_HEAP_SECTS];
                                        /* All GET_MEM allocated        */
                                        /* memory.  Includes block      */
                                        /* headers and the like.        */
# endif
# ifdef ANY_MSWIN
#   define GC_heap_bases GC_arrays._heap_bases
    ptr_t _heap_bases[MAX_HEAP_SECTS];
                /* Start address of memory regions obtained from kernel. */
# endif
# ifdef MSWINCE
#   define GC_heap_lengths GC_arrays._heap_lengths
    word _heap_lengths[MAX_HEAP_SECTS];
                /* Committed lengths of memory regions obtained from kernel. */
# endif
  struct roots _static_roots[MAX_ROOT_SETS];
  struct exclusion _excl_table[MAX_EXCLUSIONS];
  /* Block header index; see gc_headers.h */
  bottom_index * _top_index[TOP_SZ];
};

GC_API_PRIV GC_FAR struct _GC_arrays GC_arrays;

#define GC_all_nils GC_arrays._all_nils
#define GC_atomic_in_use GC_arrays._atomic_in_use
#define GC_bytes_allocd_before_gc GC_arrays._bytes_allocd_before_gc
#define GC_bytes_dropped GC_arrays._bytes_dropped
#define GC_bytes_finalized GC_arrays._bytes_finalized
#define GC_bytes_freed GC_arrays._bytes_freed
#define GC_composite_in_use GC_arrays._composite_in_use
#define GC_excl_table GC_arrays._excl_table
#define GC_finalizer_bytes_freed GC_arrays._finalizer_bytes_freed
#define GC_heapsize GC_arrays._heapsize
#define GC_large_allocd_bytes GC_arrays._large_allocd_bytes
#define GC_large_free_bytes GC_arrays._large_free_bytes
#define GC_last_heap_addr GC_arrays._last_heap_addr
#define GC_mark_stack GC_arrays._mark_stack
#define GC_mark_stack_limit GC_arrays._mark_stack_limit
#define GC_mark_stack_top GC_arrays._mark_stack_top
#define GC_mark_procs GC_arrays._mark_procs
#define GC_max_large_allocd_bytes GC_arrays._max_large_allocd_bytes
#define GC_modws_valid_offsets GC_arrays._modws_valid_offsets
#define GC_requested_heapsize GC_arrays._requested_heapsize
#define GC_all_bottom_indices GC_arrays._all_bottom_indices
#define GC_all_bottom_indices_end GC_arrays._all_bottom_indices_end
#define GC_scratch_free_ptr GC_arrays._scratch_free_ptr
#define GC_hdr_free_list GC_arrays._hdr_free_list
#define GC_size_map GC_arrays._size_map
#define GC_static_roots GC_arrays._static_roots
#define GC_top_index GC_arrays._top_index
#define GC_uobjfreelist GC_arrays._uobjfreelist
#define GC_valid_offsets GC_arrays._valid_offsets

#define beginGC_arrays ((ptr_t)(&GC_arrays))
#define endGC_arrays (beginGC_arrays + sizeof(GC_arrays))

/* Object kinds: */
#ifndef MAXOBJKINDS
# ifdef SMALL_CONFIG
#   define MAXOBJKINDS 16
# else
#   define MAXOBJKINDS 24
# endif
#endif /* !MAXOBJKINDS */
GC_EXTERN struct obj_kind {
  void **ok_freelist;   /* Array of free-list headers for this kind of  */
                        /* object.  Point either to GC_arrays or to     */
                        /* storage allocated with GC_scratch_alloc.     */
  struct hblk **ok_reclaim_list;
                        /* List headers for lists of blocks waiting to  */
                        /* be swept.  Indexed by object size in         */
                        /* granules.                                    */
  word ok_descriptor;   /* Descriptor template for objects in this      */
                        /* block.                                       */
  GC_bool ok_relocate_descr;
                        /* Add object size in bytes to descriptor       */
                        /* template to obtain descriptor.  Otherwise    */
                        /* template is used as is.                      */
  GC_bool ok_init;
                /* Clear objects before putting them on the free list.  */
# ifdef ENABLE_DISCLAIM
    GC_bool ok_mark_unconditionally;
                        /* Mark from all, including unmarked, objects   */
                        /* in block.  Used to protect objects reachable */
                        /* from reclaim notifiers.                      */
    int (GC_CALLBACK *ok_disclaim_proc)(void * /*obj*/);
                        /* The disclaim procedure is called before obj  */
                        /* is reclaimed, but must also tolerate being   */
                        /* called with object from free list.  Non-zero */
                        /* exit prevents object from being reclaimed.   */
#   define OK_DISCLAIM_INITZ /* comma */, FALSE, 0
# else
#   define OK_DISCLAIM_INITZ /* empty */
# endif /* !ENABLE_DISCLAIM */
} GC_obj_kinds[MAXOBJKINDS];

#define beginGC_obj_kinds ((ptr_t)(&GC_obj_kinds[0]))
#define endGC_obj_kinds (beginGC_obj_kinds + sizeof(GC_obj_kinds))

/* Variables that used to be in GC_arrays, but need to be accessed by   */
/* inline allocation code.  If they were in GC_arrays, the inlined      */
/* allocation code would include GC_arrays offsets (as it did), which   */
/* introduce maintenance problems.                                      */

#ifdef SEPARATE_GLOBALS
  extern word GC_bytes_allocd;
        /* Number of bytes allocated during this collection cycle.      */
  extern ptr_t GC_objfreelist[MAXOBJGRANULES+1];
                          /* free list for NORMAL objects */
# define beginGC_objfreelist ((ptr_t)(&GC_objfreelist[0]))
# define endGC_objfreelist (beginGC_objfreelist + sizeof(GC_objfreelist))

  extern ptr_t GC_aobjfreelist[MAXOBJGRANULES+1];
                          /* free list for atomic (PTRFREE) objects     */
# define beginGC_aobjfreelist ((ptr_t)(&GC_aobjfreelist[0]))
# define endGC_aobjfreelist (beginGC_aobjfreelist + sizeof(GC_aobjfreelist))
#endif /* SEPARATE_GLOBALS */

/* Predefined kinds: */
#define PTRFREE GC_I_PTRFREE
#define NORMAL  GC_I_NORMAL
#define UNCOLLECTABLE 2
#ifdef GC_ATOMIC_UNCOLLECTABLE
# define AUNCOLLECTABLE 3
# define IS_UNCOLLECTABLE(k) (((k) & ~1) == UNCOLLECTABLE)
# define GC_N_KINDS_INITIAL_VALUE 4
#else
# define IS_UNCOLLECTABLE(k) ((k) == UNCOLLECTABLE)
# define GC_N_KINDS_INITIAL_VALUE 3
#endif

GC_EXTERN unsigned GC_n_kinds;

GC_EXTERN size_t GC_page_size;
                /* May mean the allocation granularity size, not page size. */

#ifdef REAL_PAGESIZE_NEEDED
  GC_EXTERN size_t GC_real_page_size;
#else
# define GC_real_page_size GC_page_size
#endif

/* Round up allocation size to a multiple of a page size.       */
/* GC_setpagesize() is assumed to be already invoked.           */
#define ROUNDUP_PAGESIZE(lb) /* lb should have no side-effect */ \
            (SIZET_SAT_ADD(lb, GC_page_size-1) & ~(GC_page_size-1))

/* Same as above but used to make GET_MEM() argument safe.      */
#ifdef MMAP_SUPPORTED
# define ROUNDUP_PAGESIZE_IF_MMAP(lb) ROUNDUP_PAGESIZE(lb)
#else
# define ROUNDUP_PAGESIZE_IF_MMAP(lb) (lb)
#endif

#ifdef ANY_MSWIN
  GC_EXTERN SYSTEM_INFO GC_sysinfo;
  GC_INNER GC_bool GC_is_heap_base(const void *p);
#endif

GC_EXTERN word GC_black_list_spacing;
                        /* Average number of bytes between blacklisted  */
                        /* blocks. Approximate.                         */
                        /* Counts only blocks that are                  */
                        /* "stack-blacklisted", i.e. that are           */
                        /* problematic in the interior of an object.    */

#ifdef GC_GCJ_SUPPORT
  extern struct hblk * GC_hblkfreelist[];
  extern word GC_free_bytes[];  /* Both remain visible to GNU GCJ.      */
#endif

GC_EXTERN word GC_root_size; /* Total size of registered root sections. */

GC_EXTERN GC_bool GC_debugging_started;
                                /* GC_debug_malloc has been called.     */

/* This is used by GC_do_blocking[_inner]().            */
struct blocking_data {
    GC_fn_type fn;
    void * client_data; /* and result */
};

/* This is used by GC_call_with_gc_active(), GC_push_all_stack_sections(). */
struct GC_traced_stack_sect_s {
  ptr_t saved_stack_ptr;
# ifdef IA64
    ptr_t saved_backing_store_ptr;
    ptr_t backing_store_end;
# endif
  struct GC_traced_stack_sect_s *prev;
};

#ifdef THREADS
  /* Process all "traced stack sections" - scan entire stack except for */
  /* frames belonging to the user functions invoked by GC_do_blocking.  */
  GC_INNER void GC_push_all_stack_sections(ptr_t lo, ptr_t hi,
                        struct GC_traced_stack_sect_s *traced_stack_sect);
  GC_EXTERN word GC_total_stacksize; /* updated on every push_all_stacks */
#else
  GC_EXTERN ptr_t GC_blocked_sp;
  GC_EXTERN struct GC_traced_stack_sect_s *GC_traced_stack_sect;
                        /* Points to the "frame" data held in stack by  */
                        /* the innermost GC_call_with_gc_active().      */
                        /* NULL if no such "frame" active.              */
#endif /* !THREADS */

#if defined(E2K) && defined(THREADS) || defined(IA64)
  /* The bottom of the register stack of the primordial thread. */
  /* E2K: holds the offset (ps_ofs) instead of a pointer.       */
  GC_EXTERN ptr_t GC_register_stackbottom;
#endif

#ifdef IA64
  /* Similar to GC_push_all_stack_sections() but for IA-64 registers store. */
  GC_INNER void GC_push_all_register_sections(ptr_t bs_lo, ptr_t bs_hi,
                        GC_bool eager,
                        struct GC_traced_stack_sect_s *traced_stack_sect);
#endif /* IA64 */

/* Marks are in a reserved area in each heap block.  Each object or */
/* granule has one mark bit associated with it.  Only those         */
/* corresponding to the beginning of an object are used.            */

/* Mark bit operations. */

/* Retrieve, set, clear the n-th mark bit in a given heap block.    */
/* (Recall that bit n corresponds to n-th object or allocation      */
/* granule relative to the beginning of the block, including unused */
/* space.)                                                          */

#ifdef USE_MARK_BYTES
# define mark_bit_from_hdr(hhdr,n) ((hhdr) -> hb_marks[n])
# define set_mark_bit_from_hdr(hhdr,n) (void)((hhdr) -> hb_marks[n] = 1)
# define clear_mark_bit_from_hdr(hhdr,n) (void)((hhdr) -> hb_marks[n] = 0)
#else
/* Set mark bit correctly, even if mark bits may be concurrently        */
/* accessed.                                                            */
# if defined(PARALLEL_MARK) || (defined(THREAD_SANITIZER) && defined(THREADS))
    /* Workaround TSan false positive: there is no race between         */
    /* mark_bit_from_hdr and set_mark_bit_from_hdr when n is different  */
    /* (alternatively, USE_MARK_BYTES could be used).  If TSan is off,  */
    /* AO_or() is used only if we set USE_MARK_BITS explicitly.         */
#   define OR_WORD(addr, bits) AO_or(addr, bits)
# else
#   define OR_WORD(addr, bits) (void)(*(addr) |= (bits))
# endif
# define mark_bit_from_hdr(hhdr,n) \
            (((hhdr) -> hb_marks[divWORDSZ(n)] >> modWORDSZ(n)) & (word)1)
# define set_mark_bit_from_hdr(hhdr,n) \
            OR_WORD((hhdr) -> hb_marks + divWORDSZ(n), (word)1 << modWORDSZ(n))
# define clear_mark_bit_from_hdr(hhdr,n) \
            (void)(((word *)CAST_AWAY_VOLATILE_PVOID( \
                                    (hhdr) -> hb_marks))[divWORDSZ(n)] \
                    &= ~((word)1 << modWORDSZ(n)))
#endif /* !USE_MARK_BYTES */

#ifdef MARK_BIT_PER_OBJ
# define MARK_BIT_NO(offset, sz) ((offset) / (sz))
        /* Get the mark bit index corresponding to the given byte       */
        /* offset and size (in bytes).                                  */
# define MARK_BIT_OFFSET(sz) 1
        /* Spacing between useful mark bits.                            */
# define FINAL_MARK_BIT(sz) ((sz) > MAXOBJBYTES ? 1 : HBLK_OBJS(sz))
        /* Position of final, always set, mark bit.                     */
#else
# define MARK_BIT_NO(offset, sz) BYTES_TO_GRANULES(offset)
# define MARK_BIT_OFFSET(sz) BYTES_TO_GRANULES(sz)
# define FINAL_MARK_BIT(sz) \
                ((sz) > MAXOBJBYTES ? MARK_BITS_PER_HBLK \
                                : BYTES_TO_GRANULES((sz) * HBLK_OBJS(sz)))
#endif /* !MARK_BIT_PER_OBJ */

/* Important internal collector routines */

GC_INNER ptr_t GC_approx_sp(void);

GC_INNER GC_bool GC_should_collect(void);

GC_INNER struct hblk * GC_next_block(struct hblk *h, GC_bool allow_free);
                        /* Get the next block whose address is at least */
                        /* h.  Returned block is managed by GC.  The    */
                        /* block must be in use unless allow_free is    */
                        /* true.  Return 0 if there is no such block.   */
GC_INNER struct hblk * GC_prev_block(struct hblk * h);
                        /* Get the last (highest address) block whose   */
                        /* address is at most h.  Returned block is     */
                        /* managed by GC, but may or may not be in use. */
                        /* Return 0 if there is no such block.          */
GC_INNER void GC_mark_init(void);
GC_INNER void GC_clear_marks(void);
                        /* Clear mark bits for all heap objects.        */
GC_INNER void GC_invalidate_mark_state(void);
                                /* Tell the marker that marked          */
                                /* objects may point to unmarked        */
                                /* ones, and roots may point to         */
                                /* unmarked objects.  Reset mark stack. */
GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame);
                        /* Perform about one pages worth of marking     */
                        /* work of whatever kind is needed.  Returns    */
                        /* quickly if no collection is in progress.     */
                        /* Return TRUE if mark phase finished.          */
GC_INNER void GC_initiate_gc(void);
                                /* initiate collection.                 */
                                /* If the mark state is invalid, this   */
                                /* becomes full collection.  Otherwise  */
                                /* it's partial.                        */

GC_INNER GC_bool GC_collection_in_progress(void);
                        /* Collection is in progress, or was abandoned. */

/* Push contents of the symbol residing in the static roots area        */
/* excluded from scanning by the the collector for a reason.            */
/* Note: it should be used only for symbols of relatively small size    */
/* (containing one or several pointers).                                */
#define GC_PUSH_ALL_SYM(sym) GC_push_all_eager(&(sym), &(sym) + 1)

GC_INNER void GC_push_all_stack(ptr_t b, ptr_t t);
                                    /* As GC_push_all but consider      */
                                    /* interior pointers as valid.      */

#ifdef NO_VDB_FOR_STATIC_ROOTS
# define GC_push_conditional_static(b, t, all) \
                ((void)(all), GC_push_all(b, t))
#else
  /* Same as GC_push_conditional (does either of GC_push_all or         */
  /* GC_push_selected depending on the third argument) but the caller   */
  /* guarantees the region belongs to the registered static roots.      */
  GC_INNER void GC_push_conditional_static(void *b, void *t, GC_bool all);
#endif

#if defined(WRAP_MARK_SOME) && defined(PARALLEL_MARK)
  /* GC_mark_local does not handle memory protection faults yet.  So,   */
  /* the static data regions are scanned immediately by GC_push_roots.  */
  GC_INNER void GC_push_conditional_eager(void *bottom, void *top,
                                          GC_bool all);
#endif

  /* In the threads case, we push part of the current thread stack      */
  /* with GC_push_all_eager when we push the registers.  This gets the  */
  /* callee-save registers that may disappear.  The remainder of the    */
  /* stacks are scheduled for scanning in *GC_push_other_roots, which   */
  /* is thread-package-specific.                                        */

GC_INNER void GC_push_roots(GC_bool all, ptr_t cold_gc_frame);
                                        /* Push all or dirty roots.     */

GC_API_PRIV GC_push_other_roots_proc GC_push_other_roots;
                        /* Push system or application specific roots    */
                        /* onto the mark stack.  In some environments   */
                        /* (e.g. threads environments) this is          */
                        /* predefined to be non-zero.  A client         */
                        /* supplied replacement should also call the    */
                        /* original function.  Remains externally       */
                        /* visible as used by some well-known 3rd-party */
                        /* software (e.g., ECL) currently.              */

#ifdef THREADS
  void GC_push_thread_structures(void);
#endif
GC_EXTERN void (*GC_push_typed_structures)(void);
                        /* A pointer such that we can avoid linking in  */
                        /* the typed allocation support if unused.      */

typedef void (*GC_with_callee_saves_func)(ptr_t arg, void *context);
GC_INNER void GC_with_callee_saves_pushed(GC_with_callee_saves_func fn,
                                          ptr_t arg);

#if defined(IA64) || defined(SPARC)
  /* Cause all stacked registers to be saved in memory.  Return a       */
  /* pointer to the top of the corresponding memory stack.              */
  ptr_t GC_save_regs_in_stack(void);
#endif

#ifdef E2K
# include <errno.h>
# include <asm/e2k_syswork.h>
# include <sys/syscall.h>

# if defined(CPPCHECK)
    /* Workaround "Uninitialized bs_lo" and "obsolete alloca() called"  */
    /* false positive warnings.                                         */
#   define PS_ALLOCA_BUF(pbuf, sz) \
        (void)(GC_noop1_ptr(pbuf), *(pbuf) = __builtin_alloca(sz))
# else
#   define PS_ALLOCA_BUF(pbuf, sz) (void)(*(pbuf) = alloca(sz))
# endif

  /* Approximate size (in bytes) of the obtained procedure stack part   */
  /* belonging the syscall() itself.                                    */
# define PS_SYSCALL_TAIL_BYTES 0x100

  /* Determine the current size of the whole procedure stack.  The size */
  /* is valid only within the current function.                         */
# define GET_PROCEDURE_STACK_SIZE_INNER(psz_ull)                            \
        do {                                                                \
          *(psz_ull) = 0; /* might be redundant */                          \
          if (syscall(__NR_access_hw_stacks, E2K_GET_PROCEDURE_STACK_SIZE,  \
                      NULL, NULL, 0, psz_ull) == -1)                        \
            ABORT_ARG1("Cannot get size of procedure stack",                \
                       ": errno= %d", errno);                               \
          GC_ASSERT(*(psz_ull) > 0 && *(psz_ull) % sizeof(ptr_t) == 0);     \
        } while (0)

# ifdef THREADS
#   define PS_COMPUTE_ADJUSTED_OFS(padj_ps_ofs, ps_ofs, ofs_sz_ull) \
        do {                                                        \
          if ((ofs_sz_ull) <= (ps_ofs) /* && ofs_sz_ull > 0 */)     \
            ABORT_ARG2("Incorrect size of procedure stack",         \
                       ": ofs= %lu, size= %lu",                     \
                       (unsigned long)(ps_ofs),                     \
                       (unsigned long)(ofs_sz_ull));                \
          *(padj_ps_ofs) = (ps_ofs) > PS_SYSCALL_TAIL_BYTES ?       \
                            (ps_ofs) - PS_SYSCALL_TAIL_BYTES : 0;   \
        } while (0)
# else
    /* A simplified version of the above assuming ps_ofs is a zero const. */
#   define PS_COMPUTE_ADJUSTED_OFS(padj_ps_ofs, ps_ofs, ofs_sz_ull) \
        do {                                                        \
          GC_STATIC_ASSERT((ps_ofs) == 0);                          \
          (void)(ofs_sz_ull);                                       \
          *(padj_ps_ofs) = 0;                                       \
        } while (0)
# endif /* !THREADS */

  /* Copy procedure (register) stack to a stack-allocated buffer.       */
  /* Usable from a signal handler.  The buffer is valid only within     */
  /* the current function.  ps_ofs designates the offset in the         */
  /* procedure stack to copy the contents from.  Note: this macro       */
  /* cannot be changed to a function because alloca() and both          */
  /* syscall() should be called in the context of the caller.           */
# define GET_PROCEDURE_STACK_LOCAL(ps_ofs, pbuf, psz)                       \
        do {                                                                \
          unsigned long long ofs_sz_ull;                                    \
          size_t adj_ps_ofs;                                                \
                                                                            \
          GET_PROCEDURE_STACK_SIZE_INNER(&ofs_sz_ull);                      \
          PS_COMPUTE_ADJUSTED_OFS(&adj_ps_ofs, ps_ofs, ofs_sz_ull);         \
          *(psz) = (size_t)ofs_sz_ull - adj_ps_ofs;                         \
          /* Allocate buffer on the stack; cannot return NULL. */           \
          PS_ALLOCA_BUF(pbuf, *(psz));                                      \
          /* Copy the procedure stack at the given offset to the buffer. */ \
          for (;;) {                                                        \
            ofs_sz_ull = adj_ps_ofs;                                        \
            if (syscall(__NR_access_hw_stacks, E2K_READ_PROCEDURE_STACK_EX, \
                        &ofs_sz_ull, *(pbuf), *(psz), NULL) != -1)          \
              break;                                                        \
            if (errno != EAGAIN)                                            \
              ABORT_ARG2("Cannot read procedure stack",                     \
                         ": sz= %lu, errno= %d",                            \
                         (unsigned long)(*(psz)), errno);                   \
          }                                                                 \
        } while (0)
#endif /* E2K */

#if defined(E2K) && defined(USE_PTR_HWTAG)
  /* Load value and get tag of the target memory.   */
# if defined(__ptr64__)
#   define LOAD_TAGGED_VALUE(v, tag, p)         \
        do {                                    \
          ptr_t val;                            \
          __asm__ __volatile__ (                \
            "ldd, sm %[adr], 0x0, %[val]\n\t"   \
            "gettagd %[val], %[tag]\n"          \
            : [val] "=r" (val),                 \
              [tag] "=r" (tag)                  \
            : [adr] "r" (p));                   \
          v = val;                              \
        } while (0)
# elif !defined(CPPCHECK)
#   error Unsupported -march for e2k target
# endif

# define LOAD_PTR_OR_CONTINUE(v, p) \
        { \
          int tag LOCAL_VAR_INIT_OK; \
          LOAD_TAGGED_VALUE(v, tag, p); \
          if (tag != 0) continue; \
        }
#else
# define LOAD_PTR_OR_CONTINUE(v, p) (void)(v = *(ptr_t *)(p))
#endif /* !E2K */

#if defined(AMIGA) || defined(MACOS) || defined(GC_DARWIN_THREADS)
  void GC_push_one(word p);
                              /* If p points to an object, mark it    */
                              /* and push contents on the mark stack. */
                              /* Pointer recognition test always      */
                              /* accepts interior pointers, i.e. this */
                              /* is appropriate for pointers found on */
                              /* stack.                               */
#endif

#ifdef GC_WIN32_THREADS
  /* Same as GC_push_one but for a sequence of registers.       */
  GC_INNER void GC_push_many_regs(const word *regs, unsigned count);
#endif

#if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS)
  GC_INNER void GC_mark_and_push_stack(ptr_t p, ptr_t source);
                                /* Ditto, omits plausibility test       */
#else
  GC_INNER void GC_mark_and_push_stack(ptr_t p);
#endif

/* Is the block with the given header containing no pointers?   */
#define IS_PTRFREE(hhdr) (0 == (hhdr) -> hb_descr)

GC_INNER void GC_clear_hdr_marks(hdr * hhdr);
                                    /* Clear the mark bits in a header */
GC_INNER void GC_set_hdr_marks(hdr * hhdr);
                                    /* Set the mark bits in a header */
GC_INNER void GC_set_fl_marks(ptr_t);
                                    /* Set all mark bits associated with */
                                    /* a free list.                      */
#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
  void GC_check_fl_marks(void **);
                                    /* Check that all mark bits         */
                                    /* associated with a free list are  */
                                    /* set.  Abort if not.              */
#endif

#ifndef AMIGA
  GC_INNER
#endif
void GC_add_roots_inner(ptr_t b, ptr_t e, GC_bool tmp);

#ifdef USE_PROC_FOR_LIBRARIES
  GC_INNER void GC_remove_roots_subregion(ptr_t b, ptr_t e);
#endif
GC_INNER void GC_exclude_static_roots_inner(ptr_t start, ptr_t finish);
#if defined(DYNAMIC_LOADING) || defined(ANY_MSWIN) || defined(PCR)
  GC_INNER void GC_register_dynamic_libraries(void);
                /* Add dynamic library data sections to the root set. */
#endif
GC_INNER void GC_cond_register_dynamic_libraries(void);
                /* Remove and reregister dynamic libraries if we're     */
                /* configured to do that at each GC.                    */

/* Machine dependent startup routines */
ptr_t GC_get_main_stack_base(void);     /* Cold end of stack.           */
#ifdef IA64
  GC_INNER ptr_t GC_get_register_stack_base(void);
                                        /* Cold end of register stack.  */
#endif

void GC_register_data_segments(void);

#ifdef THREADS
  /* Both are invoked from GC_init only.        */
  GC_INNER void GC_thr_init(void);
  GC_INNER void GC_init_parallel(void);
# ifndef DONT_USE_ATEXIT
    GC_INNER GC_bool GC_is_main_thread(void);
# endif
#else
  GC_INNER GC_bool GC_is_static_root(ptr_t p);
                /* Is the address p in one of the registered static     */
                /* root sections?                                       */
# ifdef TRACE_BUF
    void GC_add_trace_entry(const char *caller_fn_name,
                            ptr_t arg1, ptr_t arg2);
# endif
#endif /* !THREADS */

/* Black listing: */
#ifdef PRINT_BLACK_LIST
  GC_INNER void GC_add_to_black_list_normal(ptr_t p, ptr_t source);
                        /* Register bits as a possible future false     */
                        /* reference from the heap or static data       */
# define GC_ADD_TO_BLACK_LIST_NORMAL(p, source) \
                if (GC_all_interior_pointers) { \
                  GC_add_to_black_list_stack(p, source); \
                } else \
                  GC_add_to_black_list_normal(p, source)
  GC_INNER void GC_add_to_black_list_stack(ptr_t p, ptr_t source);
# define GC_ADD_TO_BLACK_LIST_STACK(p, source) \
                GC_add_to_black_list_stack(p, source)
#else
  GC_INNER void GC_add_to_black_list_normal(ptr_t p);
# define GC_ADD_TO_BLACK_LIST_NORMAL(p, source) \
                if (GC_all_interior_pointers) { \
                  GC_add_to_black_list_stack(p); \
                } else \
                  GC_add_to_black_list_normal(p)
  GC_INNER void GC_add_to_black_list_stack(ptr_t p);
# define GC_ADD_TO_BLACK_LIST_STACK(p, source) \
                GC_add_to_black_list_stack(p)
#endif /* PRINT_BLACK_LIST */

GC_INNER void GC_promote_black_lists(void);
                        /* Declare an end to a black listing phase.     */
GC_INNER void GC_unpromote_black_lists(void);
                        /* Approximately undo the effect of the above.  */
                        /* This actually loses some information, but    */
                        /* only in a reasonably safe way.               */

GC_INNER ptr_t GC_scratch_alloc(size_t bytes);
                                /* GC internal memory allocation for    */
                                /* small objects.  Deallocation is not  */
                                /* possible.  May return NULL.          */

#ifdef GWW_VDB
  /* GC_scratch_recycle_no_gww() not used.      */
#else
# define GC_scratch_recycle_no_gww GC_scratch_recycle_inner
#endif
GC_INNER void GC_scratch_recycle_inner(void *ptr, size_t sz);
                                /* Reuse the memory region by the heap. */

#ifndef MARK_BIT_PER_OBJ
  /* Add a heap block map for objects of a size in granules to obj_map. */
  /* A size of 0 is used for large objects.  Returns FALSE on failure.  */
  GC_INNER GC_bool GC_add_map_entry(size_t lg);
#endif

GC_INNER void GC_register_displacement_inner(size_t offset);
                                /* Version of GC_register_displacement  */
                                /* that assumes the allocator lock is   */
                                /* already held.                        */

/* Allocate a new heap block for small objects of the given size (in    */
/* granules) and kind.  Add all of the block's objects to the free list */
/* for objects of that size.  Set all mark bits if objects are          */
/* uncollectible.  Will fail to do anything if out of memory.           */
GC_INNER void GC_new_hblk(size_t lg, int k);

/* Build a free list for objects of size lg (in granules) inside heap   */
/* block h.  Clear objects inside h if clear is set.  Add list to the   */
/* end of the free list we build.  Return the new free list.  Normally  */
/* called by GC_new_hblk, but this could also be called without the     */
/* allocator lock, if we ensure that there is no concurrent collection  */
/* which might reclaim objects that we have not yet allocated.          */
GC_INNER ptr_t GC_build_fl(struct hblk *h, ptr_t list, size_t lg,
                           GC_bool clear);

GC_INNER struct hblk * GC_allochblk(size_t lb_adjusted, int k, unsigned flags,
                                    size_t align_m1);
                                /* Allocate (and return pointer to)     */
                                /* a heap block for objects of the      */
                                /* given size and alignment (in bytes), */
                                /* searching over the appropriate free  */
                                /* block lists; inform the marker       */
                                /* that the found block is valid for    */
                                /* objects of the indicated size.       */
                                /* Assumes (as implied by the argument  */
                                /* name) that EXTRA_BYTES value is      */
                                /* already added to the size, if        */
                                /* needed.  The client is responsible   */
                                /* for clearing the block, if needed.   */
                                /* Note: we set obj_map field in the    */
                                /* header correctly; the caller is      */
                                /* responsible for building an object's */
                                /* free list in the block.              */

GC_INNER void GC_freehblk(struct hblk * p);
                                /* Deallocate a heap block and mark it  */
                                /* as invalid.                          */

/*  Miscellaneous GC routines.  */

GC_INNER GC_bool GC_expand_hp_inner(word n);
GC_INNER void GC_start_reclaim(GC_bool abort_if_found);
                                /* Restore unmarked objects to free     */
                                /* lists, or (if abort_if_found is      */
                                /* TRUE) report them.                   */
                                /* Sweeping of small object pages is    */
                                /* largely deferred.                    */
GC_INNER void GC_continue_reclaim(size_t lg, int k);
                                /* Sweep blocks of the indicated object */
                                /* size (in granules) and kind until    */
                                /* either the appropriate nonempty free */
                                /* list is found, or there are no more  */
                                /* blocks to sweep.                     */

GC_INNER GC_bool GC_reclaim_all(GC_stop_func stop_func, GC_bool ignore_old);
                                /* Reclaim all blocks.  Abort (in a     */
                                /* consistent state) if stop_func()     */
                                /* returns TRUE.                        */
GC_INNER ptr_t GC_reclaim_generic(struct hblk * hbp, hdr *hhdr, size_t sz,
                                  GC_bool init, ptr_t list, word *pcount);
                                /* Rebuild free list in hbp with        */
                                /* header hhdr, with objects of size sz */
                                /* bytes.  Add list to the end of the   */
                                /* free list.  Add the number of        */
                                /* reclaimed bytes to *pcount.          */
GC_INNER GC_bool GC_block_empty(const hdr *hhdr);
                                /* Block completely unmarked?   */
GC_INNER int GC_CALLBACK GC_never_stop_func(void);
                                /* Always returns 0 (FALSE).            */
GC_INNER GC_bool GC_try_to_collect_inner(GC_stop_func stop_func);
                                /* Collect; caller must have acquired   */
                                /* the allocator lock.  Collection is   */
                                /* aborted if stop_func() returns TRUE. */
                                /* Returns TRUE if it completes         */
                                /* successfully.   */
#define GC_gcollect_inner() \
                (void)GC_try_to_collect_inner(GC_never_stop_func)

#ifdef THREADS
  GC_EXTERN GC_bool GC_in_thread_creation;
        /* We may currently be in thread creation or destruction.       */
        /* Only set to TRUE while the allocator lock is held.           */
        /* When set, it is OK to run GC from unknown thread.            */
#endif

GC_EXTERN GC_bool GC_is_initialized; /* GC_init() has been run. */

GC_INNER void GC_collect_a_little_inner(int n);
                                /* Do n units worth of garbage          */
                                /* collection work, if appropriate.     */
                                /* A unit is an amount appropriate for  */
                                /* HBLKSIZE bytes of allocation.        */

GC_INNER void * GC_malloc_kind_aligned_global(size_t lb, int k,
                                              size_t align_m1);

GC_INNER void * GC_generic_malloc_aligned(size_t lb, int k, unsigned flags,
                                          size_t align_m1);

GC_INNER void * GC_generic_malloc_inner(size_t lb, int k, unsigned flags);
                                /* Allocate an object of the given kind */
                                /* but assuming the allocator lock is   */
                                /* already held.  Should not be used to */
                                /* directly allocate objects requiring  */
                                /* special handling on allocation.      */
                                /* The flags argument should be 0 or    */
                                /* IGNORE_OFF_PAGE.  In the latter case */
                                /* the client guarantees there will     */
                                /* always be a pointer to the beginning */
                                /* (i.e. within the first hblk) of the  */
                                /* object while it is live.             */

GC_INNER GC_bool GC_collect_or_expand(word needed_blocks, unsigned flags,
                                      GC_bool retry);

/* Make sure the indicated object free is not empty, and return its     */
/* head (the first object on the free list).  The object must be        */
/* removed from the free list by the caller.  The size is in granules.  */
GC_INNER ptr_t GC_allocobj(size_t lg, int k);

#ifdef GC_ADD_CALLER
  /* GC_DBG_EXTRAS is used by GC debug API functions (unlike GC_EXTRAS  */
  /* used by GC debug API macros) thus GC_RETURN_ADDR_PARENT (pointing  */
  /* to client caller) should be used if possible.                      */
# ifdef GC_HAVE_RETURN_ADDR_PARENT
#   define GC_DBG_EXTRAS GC_RETURN_ADDR_PARENT, NULL, 0
# else
#   define GC_DBG_EXTRAS GC_RETURN_ADDR, NULL, 0
# endif
#else
# define GC_DBG_EXTRAS "unknown", 0
#endif /* !GC_ADD_CALLER */

#ifdef GC_COLLECT_AT_MALLOC
  extern size_t GC_dbg_collect_at_malloc_min_lb;
                            /* variable visible outside for debugging   */
# define GC_DBG_COLLECT_AT_MALLOC(lb) \
                (void)((lb) >= GC_dbg_collect_at_malloc_min_lb ? \
                            (GC_gcollect(), 0) : 0)
#else
# define GC_DBG_COLLECT_AT_MALLOC(lb) (void)0
#endif /* !GC_COLLECT_AT_MALLOC */

/* Allocation routines that bypass the thread-local cache.      */
#if defined(THREAD_LOCAL_ALLOC) && defined(GC_GCJ_SUPPORT)
  GC_INNER void *GC_core_gcj_malloc(size_t lb, const void *vtable_ptr,
                                    unsigned flags);
#endif

GC_INNER void GC_init_headers(void);
GC_INNER hdr * GC_install_header(struct hblk *h);
                                /* Install a header for block h.        */
                                /* Return NULL on failure, or the       */
                                /* uninitialized header otherwise.      */
GC_INNER GC_bool GC_install_counts(struct hblk * h, size_t sz);
                                /* Set up forwarding counts for block   */
                                /* h of size sz.                        */
                                /* Return FALSE on failure.             */
GC_INNER void GC_remove_header(struct hblk * h);
                                /* Remove the header for block h.       */
GC_INNER void GC_remove_counts(struct hblk * h, size_t sz);
                                /* Remove forwarding counts for h.      */
GC_INNER hdr * GC_find_header(const void * h);

GC_INNER ptr_t GC_os_get_mem(size_t bytes);
                        /* Get HBLKSIZE-aligned heap memory chunk from  */
                        /* the OS and add the chunk to GC_our_memory.   */
                        /* Return NULL if out of memory.                */

GC_INNER void GC_print_all_errors(void);
                        /* Print smashed and leaked objects, if any.    */
                        /* Clear the lists of such objects.             */

GC_EXTERN void (*GC_check_heap)(void);
                        /* Check that all objects in the heap with      */
                        /* debugging info are intact.                   */
                        /* Add any that are not to GC_smashed list.     */
GC_EXTERN void (*GC_print_all_smashed)(void);
                        /* Print GC_smashed if it's not empty.          */
                        /* Clear GC_smashed list.                       */
GC_EXTERN void (*GC_print_heap_obj)(ptr_t p);
                        /* If possible print (using GC_err_printf)      */
                        /* a more detailed description (terminated with */
                        /* "\n") of the object referred to by p.        */

#if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG)
  void GC_print_address_map(void);
                        /* Print an address map of the process.         */
                        /* The caller should hold the allocator lock.   */
#endif

#ifndef SHORT_DBG_HDRS
  GC_EXTERN GC_bool GC_findleak_delay_free;
                        /* Do not immediately deallocate object on      */
                        /* free() in the leak-finding mode, just mark   */
                        /* it as freed (and deallocate it after GC).    */
  GC_INNER GC_bool GC_check_leaked(ptr_t base); /* from dbg_mlc.c */
#endif

#ifdef AO_HAVE_store
  GC_EXTERN volatile AO_t GC_have_errors;
# define GC_SET_HAVE_ERRORS() AO_store(&GC_have_errors, (AO_t)TRUE)
# define get_have_errors() ((GC_bool)AO_load(&GC_have_errors))
                                /* The barriers are not needed.         */
#else
  GC_EXTERN GC_bool GC_have_errors;
# define GC_SET_HAVE_ERRORS() (void)(GC_have_errors = TRUE)
# define get_have_errors() GC_have_errors
#endif                          /* We saw a smashed or leaked object.   */
                                /* Call error printing routine          */
                                /* occasionally.  It is OK to read it   */
                                /* not acquiring the allocator lock.    */
                                /* If set to true, it is never cleared. */

#define VERBOSE 2
#if !defined(NO_CLOCK) || !defined(SMALL_CONFIG)
  GC_EXTERN int GC_print_stats;
                        /* Value 1 generates basic GC log;              */
                        /* VERBOSE generates additional messages.       */
#else /* SMALL_CONFIG */
# define GC_print_stats 0
  /* Will this remove the message character strings from the executable? */
  /* With a particular level of optimizations, it should...              */
#endif

#ifdef KEEP_BACK_PTRS
  GC_EXTERN long GC_backtraces;
#endif

/* A trivial (linear congruential) pseudo-random numbers generator, */
/* safe for the concurrent usage.                                   */
#define GC_RAND_MAX ((int)(~0U >> 1))
#if defined(AO_HAVE_store) && defined(THREAD_SANITIZER)
#   define GC_RAND_STATE_T volatile AO_t
#   define GC_RAND_NEXT(pseed) GC_rand_next(pseed)
    GC_INLINE int GC_rand_next(GC_RAND_STATE_T *pseed)
    {
      AO_t next = (AO_t)((AO_load(pseed) * (unsigned32)1103515245UL + 12345)
                           & (unsigned32)((unsigned)GC_RAND_MAX));
      AO_store(pseed, next);
      return (int)next;
    }
#else
#   define GC_RAND_STATE_T unsigned32
#   define GC_RAND_NEXT(pseed) /* overflow and race are OK */ \
        (int)(*(pseed) = (*(pseed) * (unsigned32)1103515245UL + 12345) \
                         & (unsigned32)((unsigned)GC_RAND_MAX))
#endif

GC_EXTERN GC_bool GC_print_back_height;

#ifdef MAKE_BACK_GRAPH
  void GC_print_back_graph_stats(void);
#endif

#ifdef THREADS
  /* Explicitly deallocate the object when we already hold the      */
  /* allocator lock.  Only used for internally allocated objects.   */
  GC_INNER void GC_free_inner(void * p);
#endif

#ifdef VALGRIND_TRACKING
# define FREE_PROFILER_HOOK(p) GC_free_profiler_hook(p)
#else
# define FREE_PROFILER_HOOK(p) (void)(p)
#endif

/* Macros used for collector internal allocation.       */
/* These assume the allocator lock is held.             */
#ifdef DBG_HDRS_ALL
  GC_INNER void * GC_debug_generic_malloc_inner(size_t lb, int k,
                                                unsigned flags);
# define GC_INTERNAL_MALLOC(lb, k) GC_debug_generic_malloc_inner(lb, k, 0)
# define GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(lb, k) \
               GC_debug_generic_malloc_inner(lb, k, IGNORE_OFF_PAGE)
# ifdef THREADS
    GC_INNER void GC_debug_free_inner(void * p);
#   define GC_INTERNAL_FREE GC_debug_free_inner
# else
#   define GC_INTERNAL_FREE GC_debug_free
# endif
#else
# define GC_INTERNAL_MALLOC(lb, k) GC_generic_malloc_inner(lb, k, 0)
# define GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(lb, k) \
               GC_generic_malloc_inner(lb, k, IGNORE_OFF_PAGE)
# ifdef THREADS
#   define GC_INTERNAL_FREE GC_free_inner
# else
#   define GC_INTERNAL_FREE GC_free
# endif
#endif /* !DBG_HDRS_ALL */

#ifdef USE_MUNMAP
  /* Memory unmapping: */
  GC_INNER void GC_unmap_old(unsigned threshold);
  GC_INNER void GC_merge_unmapped(void);
  GC_INNER void GC_unmap(ptr_t start, size_t bytes);
  GC_INNER void GC_remap(ptr_t start, size_t bytes);
  GC_INNER void GC_unmap_gap(ptr_t start1, size_t bytes1, ptr_t start2,
                             size_t bytes2);
#endif /* USE_MUNMAP */

#ifdef CAN_HANDLE_FORK
  GC_EXTERN int GC_handle_fork;
                /* Fork-handling mode:                                  */
                /* 0 means no fork handling requested (but client could */
                /* anyway call fork() provided it is surrounded with    */
                /* GC_atfork_prepare/parent/child calls);               */
                /* -1 means GC tries to use pthread_at_fork if it is    */
                /* available (if it succeeds then GC_handle_fork value  */
                /* is changed to 1), client should nonetheless surround */
                /* fork() with GC_atfork_prepare/parent/child (for the  */
                /* case of pthread_at_fork failure or absence);         */
                /* 1 (or other values) means client fully relies on     */
                /* pthread_at_fork (so if it is missing or failed then  */
                /* abort occurs in GC_init), GC_atfork_prepare and the  */
                /* accompanying routines are no-op in such a case.      */
#endif

#ifdef NO_MANUAL_VDB
# define GC_manual_vdb FALSE
# define GC_auto_incremental GC_incremental
# define GC_dirty(p) (void)(p)
# define REACHABLE_AFTER_DIRTY(p) (void)(p)
#else
  GC_EXTERN GC_bool GC_manual_vdb;
                /* The incremental collection is in the manual VDB      */
                /* mode.  Assumes GC_incremental is true.  Should not   */
                /* be modified once GC_incremental is set to true.      */

# define GC_auto_incremental (GC_incremental && !GC_manual_vdb)
  GC_INNER void GC_dirty_inner(const void *p); /* does not require locking */
# define GC_dirty(p) (GC_manual_vdb ? GC_dirty_inner(p) : (void)0)
# define REACHABLE_AFTER_DIRTY(p) GC_reachable_here(p)
#endif /* !NO_MANUAL_VDB */

#ifdef GC_DISABLE_INCREMENTAL
# define GC_incremental FALSE
#else
  GC_EXTERN GC_bool GC_incremental;
                        /* Using incremental/generational collection.   */
                        /* Assumes dirty bits are being maintained.     */

  /* Virtual dirty bit implementation:            */
  /* Each implementation exports the following:   */
  GC_INNER void GC_read_dirty(GC_bool output_unneeded);
                        /* Retrieve dirty bits.  Set output_unneeded to */
                        /* indicate that reading of the retrieved dirty */
                        /* bits is not planned till the next retrieval. */
  GC_INNER GC_bool GC_page_was_dirty(struct hblk *h);
                        /* Read retrieved dirty bits.   */

  GC_INNER void GC_remove_protection(struct hblk *h, size_t nblocks,
                                     GC_bool is_ptrfree);
                /* Block h is about to be written or allocated shortly. */
                /* Ensure that all pages containing any part of the     */
                /* n hblks starting at h are no longer write protected  */
                /* (by the virtual dirty bit implementation).  I.e.,    */
                /* this is a call that:                                 */
                /* - hints that [h, h+nblocks) is about to be written;  */
                /* - guarantees that protection is removed;             */
                /* - may speed up some dirty bit implementations;       */
                /* - may be essential if we need to ensure that         */
                /* pointer-free system call buffers in the heap are     */
                /* not protected.                                       */

# if !defined(NO_VDB_FOR_STATIC_ROOTS) && !defined(PROC_VDB)
    GC_INNER GC_bool GC_is_vdb_for_static_roots(void);
                /* Is VDB working for static roots?                     */
# endif

# ifdef CAN_HANDLE_FORK
#   if defined(PROC_VDB) || defined(SOFT_VDB) \
       || (defined(MPROTECT_VDB) && defined(GC_DARWIN_THREADS))
      GC_INNER void GC_dirty_update_child(void);
                /* Update pid-specific resources (like /proc file       */
                /* descriptors) needed by the dirty bits implementation */
                /* after fork in the child process.                     */
#   else
#     define GC_dirty_update_child() (void)0
#   endif
# endif /* CAN_HANDLE_FORK */

# if defined(MPROTECT_VDB) && defined(DARWIN)
    EXTERN_C_END
#   include <pthread.h>
    EXTERN_C_BEGIN
#   ifdef THREADS
      GC_INNER int GC_inner_pthread_create(pthread_t *t,
                                GC_PTHREAD_CREATE_CONST pthread_attr_t *a,
                                void *(*fn)(void *), void *arg);
#   else
#     define GC_inner_pthread_create pthread_create
#   endif
# endif /* MPROTECT_VDB && DARWIN */

  GC_INNER GC_bool GC_dirty_init(void);
                /* Returns true if dirty bits are maintained (otherwise */
                /* it is OK to be called again if the client invokes    */
                /* GC_enable_incremental once more).                    */
#endif /* !GC_DISABLE_INCREMENTAL */

#if defined(COUNT_PROTECTED_REGIONS) && defined(MPROTECT_VDB)
  /* Do actions on heap growth, if needed, to prevent hitting the       */
  /* kernel limit on the VM map regions.                                */
  GC_INNER void GC_handle_protected_regions_limit(void);
#else
# define GC_handle_protected_regions_limit() (void)0
#endif

/* Same as GC_base but excepts and returns a pointer to const object.   */
#define GC_base_C(p) ((const void *)GC_base(GC_CAST_AWAY_CONST_PVOID(p)))

/* Debugging print routines: */
void GC_print_block_list(void);
void GC_print_hblkfreelist(void);
void GC_print_heap_sects(void);
void GC_print_static_roots(void);

#ifdef KEEP_BACK_PTRS
  GC_INNER void GC_store_back_pointer(ptr_t source, ptr_t dest);
  GC_INNER void GC_marked_for_finalization(ptr_t dest);
# define GC_STORE_BACK_PTR(source, dest) GC_store_back_pointer(source, dest)
# define GC_MARKED_FOR_FINALIZATION(dest) GC_marked_for_finalization(dest)
#else
# define GC_STORE_BACK_PTR(source, dest) (void)(source)
# define GC_MARKED_FOR_FINALIZATION(dest)
#endif /* !KEEP_BACK_PTRS */

/* Make arguments appear live to compiler */
void GC_noop6(word, word, word, word, word, word);

#ifndef GC_ATTR_FORMAT_PRINTF
# if GC_GNUC_PREREQ(3, 0)
#   define GC_ATTR_FORMAT_PRINTF(spec_argnum, first_checked) \
        __attribute__((__format__(__printf__, spec_argnum, first_checked)))
# else
#   define GC_ATTR_FORMAT_PRINTF(spec_argnum, first_checked)
# endif
#endif

/* Logging and diagnostic output:       */
/* GC_printf is used typically on client explicit print requests.       */
/* For all GC_X_printf routines, it is recommended to put "\n" at       */
/* 'format' string end (for output atomicity).                          */
GC_API_PRIV void GC_printf(const char * format, ...)
                        GC_ATTR_FORMAT_PRINTF(1, 2);
                        /* A version of printf that doesn't allocate,   */
                        /* 1 KB total output length.                    */
                        /* (We use sprintf.  Hopefully that doesn't     */
                        /* allocate for long arguments.)                */
GC_API_PRIV void GC_err_printf(const char * format, ...)
                        GC_ATTR_FORMAT_PRINTF(1, 2);

/* Basic logging routine.  Typically, GC_log_printf is called directly  */
/* only inside various DEBUG_x blocks.                                  */
GC_API_PRIV void GC_log_printf(const char * format, ...)
                        GC_ATTR_FORMAT_PRINTF(1, 2);

#ifndef GC_ANDROID_LOG
# define GC_PRINT_STATS_FLAG (GC_print_stats != 0)
# define GC_INFOLOG_PRINTF GC_COND_LOG_PRINTF
  /* GC_verbose_log_printf is called only if GC_print_stats is VERBOSE. */
# define GC_verbose_log_printf GC_log_printf
#else
  extern GC_bool GC_quiet;
# define GC_PRINT_STATS_FLAG (!GC_quiet)
  /* INFO/DBG loggers are enabled even if GC_print_stats is off. */
# ifndef GC_INFOLOG_PRINTF
#   define GC_INFOLOG_PRINTF if (GC_quiet) {} else GC_info_log_printf
# endif
  GC_INNER void GC_info_log_printf(const char *format, ...)
                        GC_ATTR_FORMAT_PRINTF(1, 2);
  GC_INNER void GC_verbose_log_printf(const char *format, ...)
                        GC_ATTR_FORMAT_PRINTF(1, 2);
#endif /* GC_ANDROID_LOG */

#if defined(SMALL_CONFIG) || defined(GC_ANDROID_LOG)
# define GC_ERRINFO_PRINTF GC_INFOLOG_PRINTF
#else
# define GC_ERRINFO_PRINTF GC_log_printf
#endif

/* Convenient macros for GC_[verbose_]log_printf invocation.    */
#define GC_COND_LOG_PRINTF \
                if (EXPECT(!GC_print_stats, TRUE)) {} else GC_log_printf
#define GC_VERBOSE_LOG_PRINTF \
    if (EXPECT(GC_print_stats != VERBOSE, TRUE)) {} else GC_verbose_log_printf
#ifndef GC_DBGLOG_PRINTF
# define GC_DBGLOG_PRINTF if (!GC_PRINT_STATS_FLAG) {} else GC_log_printf
#endif

void GC_err_puts(const char *s);
                        /* Write s to stderr, don't buffer, don't add   */
                        /* newlines, don't ...                          */

/* Handy macro for logging size values (of word type) in KiB (rounding  */
/* to nearest value).                                                   */
#define TO_KiB_UL(v) ((unsigned long)(((v) + ((1 << 9) - 1)) >> 10))

GC_EXTERN unsigned GC_fail_count;
                        /* How many consecutive GC/expansion failures?  */
                        /* Reset by GC_allochblk(); defined in alloc.c. */

GC_EXTERN long GC_large_alloc_warn_interval; /* defined in misc.c */

GC_EXTERN signed_word GC_bytes_found;
                /* Number of reclaimed bytes after garbage collection;  */
                /* protected by the allocator lock.                     */

#ifndef GC_GET_HEAP_USAGE_NOT_NEEDED
  GC_EXTERN word GC_reclaimed_bytes_before_gc;
                /* Number of bytes reclaimed before this        */
                /* collection cycle; used for statistics only.  */
#endif

#ifdef USE_MUNMAP
  GC_EXTERN unsigned GC_unmap_threshold; /* defined in alloc.c */
  GC_EXTERN GC_bool GC_force_unmap_on_gcollect; /* defined in misc.c */
#endif

#ifdef MSWIN32
  GC_EXTERN GC_bool GC_no_win32_dlls; /* defined in os_dep.c */
  GC_EXTERN GC_bool GC_wnt;     /* Is Windows NT derivative;    */
                                /* defined and set in os_dep.c. */
#endif

#ifdef THREADS
# if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)
    GC_EXTERN CRITICAL_SECTION GC_write_cs; /* defined in misc.c */
#   ifdef GC_ASSERTIONS
      GC_EXTERN GC_bool GC_write_disabled;
                                /* defined in win32_threads.c;  */
                                /* protected by GC_write_cs.    */

#   endif
# endif /* MSWIN32 || MSWINCE */
# if (!defined(NO_MANUAL_VDB) || defined(MPROTECT_VDB)) \
     && !defined(HAVE_LOCKFREE_AO_OR) && defined(AO_HAVE_test_and_set_acquire)
    /* Acquire the spin lock we use to update dirty bits.       */
    /* Threads should not get stopped holding it.  But we may   */
    /* acquire and release it during GC_remove_protection call. */
#   define GC_acquire_dirty_lock() \
        do { /* empty */ \
        } while (AO_test_and_set_acquire(&GC_fault_handler_lock) == AO_TS_SET)
#   define GC_release_dirty_lock() AO_CLEAR(&GC_fault_handler_lock)
    GC_EXTERN volatile AO_TS_t GC_fault_handler_lock;
                                        /* defined in os_dep.c */
# else
#   define GC_acquire_dirty_lock() (void)0
#   define GC_release_dirty_lock() (void)0
# endif /* NO_MANUAL_VDB && !MPROTECT_VDB || HAVE_LOCKFREE_AO_OR */
# ifdef MSWINCE
    GC_EXTERN GC_bool GC_dont_query_stack_min;
                                /* Defined and set in os_dep.c. */
# endif
#elif defined(IA64)
  GC_EXTERN ptr_t GC_save_regs_ret_val; /* defined in mach_dep.c. */
                        /* Previously set to backing store pointer.     */
#endif /* !THREADS */

#ifdef THREAD_LOCAL_ALLOC
  GC_EXTERN GC_bool GC_world_stopped; /* defined in alloc.c */
  GC_INNER void GC_mark_thread_local_free_lists(void);
#endif

#if defined(GLIBC_2_19_TSX_BUG) && defined(GC_PTHREADS_PARAMARK)
  /* Parse string like <major>[.<minor>[<tail>]] and return major value. */
  GC_INNER int GC_parse_version(int *pminor, const char *pverstr);
#endif

#if defined(MPROTECT_VDB) && defined(GWW_VDB)
    GC_INNER GC_bool GC_gww_dirty_init(void);
                        /* Returns TRUE if GetWriteWatch is available.  */
                        /* May be called repeatedly.  May be called     */
                        /* with or without the allocator lock held.     */
#endif

#if defined(CHECKSUMS) || defined(PROC_VDB)
  GC_INNER GC_bool GC_page_was_ever_dirty(struct hblk * h);
                        /* Could the page contain valid heap pointers?  */
#endif

#ifdef CHECKSUMS
# ifdef MPROTECT_VDB
    void GC_record_fault(struct hblk * h);
# endif
  void GC_check_dirty(void);
#endif

GC_INNER void GC_default_print_heap_obj_proc(ptr_t p);

GC_INNER void GC_setpagesize(void);

GC_INNER void GC_initialize_offsets(void);      /* defined in obj_map.c */

GC_INNER void GC_bl_init(void);
GC_INNER void GC_bl_init_no_interiors(void);    /* defined in blacklst.c */

GC_INNER void GC_start_debugging_inner(void);   /* defined in dbg_mlc.c. */
                        /* Should not be called if GC_debugging_started. */

/* Store debugging info into p.  Return displaced pointer.      */
/* Assume we hold the allocator lock.                           */
GC_INNER void *GC_store_debug_info_inner(void *p, size_t sz, const char *str,
                                         int linenum);

#if defined(REDIRECT_MALLOC) && !defined(REDIRECT_MALLOC_IN_HEADER) \
    && defined(GC_LINUX_THREADS)
  GC_INNER void GC_init_lib_bounds(void);
#else
# define GC_init_lib_bounds() (void)0
#endif

#ifdef REDIRECT_MALLOC
# ifdef GC_LINUX_THREADS
    GC_INNER GC_bool GC_text_mapping(const char *nm, ptr_t *startp,
                                     ptr_t *endp); /* from os_dep.c */
# endif
#elif defined(USE_WINALLOC)
  GC_INNER void GC_add_current_malloc_heap(void);
#endif /* USE_WINALLOC && !REDIRECT_MALLOC */

#ifdef MAKE_BACK_GRAPH
  GC_INNER void GC_build_back_graph(void);
  GC_INNER void GC_traverse_back_graph(void);
#endif

#ifdef MSWIN32
  GC_INNER void GC_init_win32(void);
#endif

#ifndef ANY_MSWIN
  GC_INNER void * GC_roots_present(ptr_t);
        /* The type is a lie, since the real type doesn't make sense here, */
        /* and we only test for NULL.                                      */
#endif

#ifdef GC_WIN32_THREADS
  /* Find stack with the lowest address which overlaps the interval     */
  /* [start, limit).  Return stack bounds in *plo and *phi.  If no such */
  /* stack is found, both *phi and *plo will be set to an address       */
  /* higher than limit.                                                 */
  GC_INNER void GC_get_next_stack(ptr_t start, ptr_t limit, ptr_t *plo,
                                  ptr_t *phi);

# if defined(MPROTECT_VDB) && !defined(CYGWIN32)
    GC_INNER void GC_set_write_fault_handler(void);
# endif
# if defined(WRAP_MARK_SOME) && !defined(GC_PTHREADS)
    GC_INNER GC_bool GC_started_thread_while_stopped(void);
        /* Did we invalidate mark phase with an unexpected thread start? */
# endif
#endif /* GC_WIN32_THREADS */

#if defined(GC_DARWIN_THREADS) && defined(MPROTECT_VDB)
  GC_INNER void GC_mprotect_stop(void);
  GC_INNER void GC_mprotect_resume(void);
# ifndef GC_NO_THREADS_DISCOVERY
    GC_INNER void GC_darwin_register_self_mach_handler(void);
# endif
#endif

#ifndef NOT_GCBUILD
  /* Iterate over forwarding addresses, if any, to get the beginning of */
  /* the block and its header.  Assumes *phhdr is non-NULL on entry,    */
  /* and guarantees *phhdr is non-NULL on return.                       */
  GC_INLINE struct hblk *GC_find_starting_hblk(struct hblk *h, hdr **phhdr)
  {
    hdr *hhdr = *phhdr;

    GC_ASSERT(HDR(h) == hhdr);
    for (; IS_FORWARDING_ADDR_OR_NIL(hhdr); hhdr = HDR(h)) {
      GC_ASSERT(hhdr != NULL);
      h = FORWARDED_ADDR(h, hhdr);
    }
    *phhdr = hhdr;
    return h;
  }
#endif /* !NOT_GCBUILD */

#ifdef THREADS
# ifndef GC_NO_FINALIZATION
    GC_INNER void GC_reset_finalizer_nested(void);
    GC_INNER unsigned char *GC_check_finalizer_nested(void);
# endif
  GC_INNER void GC_do_blocking_inner(ptr_t data, void * context);
  GC_INNER void GC_push_all_stacks(void);
# ifdef USE_PROC_FOR_LIBRARIES
    GC_INNER GC_bool GC_segment_is_thread_stack(ptr_t lo, ptr_t hi);
# endif
# if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
     && defined(IA64)
    GC_INNER ptr_t GC_greatest_stack_base_below(ptr_t bound);
# endif
#endif /* THREADS */

#ifdef DYNAMIC_LOADING
  GC_INNER GC_bool GC_register_main_static_data(void);
# ifdef DARWIN
    GC_INNER void GC_init_dyld(void);
# endif
#endif /* DYNAMIC_LOADING */

#ifdef SEARCH_FOR_DATA_START
  GC_INNER void GC_init_linux_data_start(void);
  void * GC_find_limit(void *, int);
#endif

#ifdef NEED_PROC_MAPS
# if defined(DYNAMIC_LOADING) && defined(USE_PROC_FOR_LIBRARIES) \
     || defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) \
     || (defined(CHECK_SOFT_VDB) && defined(MPROTECT_VDB)) \
     || (defined(REDIRECT_MALLOC) && defined(GC_LINUX_THREADS))
    GC_INNER const char *GC_parse_map_entry(const char *maps_ptr,
                                            ptr_t *start, ptr_t *end,
                                            const char **prot,
                                            unsigned *maj_dev,
                                            const char **mapping_name);
# endif
# if defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) \
     || (defined(CHECK_SOFT_VDB) && defined(MPROTECT_VDB))
    GC_INNER GC_bool GC_enclosing_writable_mapping(ptr_t addr, ptr_t *startp,
                                                   ptr_t *endp);
# endif
  GC_INNER const char *GC_get_maps(void);
#endif /* NEED_PROC_MAPS */

#ifdef GC_ASSERTIONS
  GC_INNER word GC_compute_large_free_bytes(void);
  GC_INNER word GC_compute_root_size(void);
#endif

/* Check a compile time assertion at compile time.      */
#if defined(_MSC_VER) && (_MSC_VER >= 1700)
# define GC_STATIC_ASSERT(expr) \
                static_assert(expr, "static assertion failed: " #expr)
#elif defined(static_assert) && !defined(CPPCHECK) \
      && (__STDC_VERSION__ >= 201112L)
# define GC_STATIC_ASSERT(expr) static_assert(expr, #expr)
#elif defined(mips) && !defined(__GNUC__) && !defined(CPPCHECK)
/* DOB: MIPSPro C gets an internal error taking the sizeof an array type.
   This code works correctly (ugliness is to avoid "unused var" warnings) */
# define GC_STATIC_ASSERT(expr) \
    do { if (0) { char j[(expr)? 1 : -1]; j[0]='\0'; j[0]=j[0]; } } while(0)
#else
  /* The error message for failure is a bit baroque, but ...    */
# define GC_STATIC_ASSERT(expr) (void)sizeof(char[(expr)? 1 : -1])
#endif

/* Runtime check for an argument declared as non-null is actually not null. */
#if GC_GNUC_PREREQ(4, 0)
  /* Workaround tautological-pointer-compare Clang warning.     */
# define NONNULL_ARG_NOT_NULL(arg) \
                (*CAST_THRU_UINTPTR(volatile void **, &(arg)) != NULL)
#else
# define NONNULL_ARG_NOT_NULL(arg) ((arg) != NULL)
#endif

#define COND_DUMP_CHECKS \
          do { \
            GC_ASSERT(I_HOLD_LOCK()); \
            GC_ASSERT(GC_compute_large_free_bytes() == GC_large_free_bytes); \
            GC_ASSERT(GC_compute_root_size() == GC_root_size); \
          } while (0)

#ifndef NO_DEBUGGING
  GC_EXTERN GC_bool GC_dump_regularly;
                                /* Generate regular debugging dumps.    */
# define COND_DUMP if (EXPECT(GC_dump_regularly, FALSE)) { \
                        GC_dump_named(NULL); \
                   } else COND_DUMP_CHECKS
#else
# define COND_DUMP COND_DUMP_CHECKS
#endif

#ifdef PARALLEL_MARK
  /* We need additional synchronization facilities from the thread      */
  /* support.  We believe these are less performance critical than      */
  /* the allocator lock; standard pthreads-based implementations        */
  /* should be sufficient.                                              */

# define GC_markers_m1 GC_parallel
                        /* Number of mark threads we would like to have */
                        /* excluding the initiating thread.             */

  GC_EXTERN GC_bool GC_parallel_mark_disabled;
                        /* A flag to temporarily avoid parallel marking.*/

  /* The mark lock and condition variable.  If the allocator lock is    */
  /* also acquired, it must be done first.  The mark lock is used to    */
  /* both protect some variables used by the parallel marker, and to    */
  /* protect GC_fl_builder_count, below.  GC_notify_all_marker() is     */
  /* called when the state of the parallel marker changes in some       */
  /* significant way (see gc_mark.h for details).  The latter set of    */
  /* events includes incrementing GC_mark_no.                           */
  /* GC_notify_all_builder() is called when GC_fl_builder_count         */
  /* reaches 0.                                                         */

  GC_INNER void GC_wait_for_markers_init(void);
  GC_INNER void GC_acquire_mark_lock(void);
  GC_INNER void GC_release_mark_lock(void);
  GC_INNER void GC_notify_all_builder(void);
  GC_INNER void GC_wait_for_reclaim(void);

  GC_EXTERN signed_word GC_fl_builder_count; /* protected by the mark lock */

  GC_INNER void GC_notify_all_marker(void);
  GC_INNER void GC_wait_marker(void);

  GC_EXTERN word GC_mark_no; /* protected by the mark lock */

  GC_INNER void GC_help_marker(word my_mark_no);
              /* Try to help out parallel marker for mark cycle         */
              /* my_mark_no.  Returns if the mark cycle finishes or     */
              /* was already done, or there was nothing to do for       */
              /* some other reason.                                     */

  GC_INNER void GC_start_mark_threads_inner(void);

# define INCR_MARKS(hhdr) \
            AO_store(&(hhdr)->hb_n_marks, AO_load(&(hhdr)->hb_n_marks) + 1)
#else
# define INCR_MARKS(hhdr) (void)(++(hhdr)->hb_n_marks)
#endif /* !PARALLEL_MARK */

#if defined(SIGNAL_BASED_STOP_WORLD) && !defined(SIG_SUSPEND)
  /* We define the thread suspension signal here, so that we can refer  */
  /* to it in the dirty bit implementation, if necessary.  Ideally we   */
  /* would allocate a (real-time?) signal using the standard mechanism. */
  /* unfortunately, there is no standard mechanism.  (There is one      */
  /* in Linux glibc, but it's not exported.)  Thus we continue to use   */
  /* the same hard-coded signals we've always used.                     */
# ifdef THREAD_SANITIZER
    /* Unfortunately, use of an asynchronous signal to suspend threads  */
    /* leads to the situation when the signal is not delivered (is      */
    /* stored to pending_signals in TSan runtime actually) while the    */
    /* destination thread is blocked in pthread_mutex_lock.  Thus, we   */
    /* use some synchronous one instead (which is again unlikely to be  */
    /* used by clients directly).                                       */
#   define SIG_SUSPEND SIGSYS
# elif (defined(GC_LINUX_THREADS) || defined(GC_DGUX386_THREADS)) \
       && !defined(GC_USESIGRT_SIGNALS)
#   if defined(SPARC) && !defined(SIGPWR)
      /* Linux/SPARC doesn't properly define SIGPWR in <signal.h>.      */
      /* It is aliased to SIGLOST in asm/signal.h, though.              */
#     define SIG_SUSPEND SIGLOST
#   else
      /* Linuxthreads itself uses SIGUSR1 and SIGUSR2.                  */
#     define SIG_SUSPEND SIGPWR
#   endif
# elif defined(GC_FREEBSD_THREADS) && defined(__GLIBC__) \
       && !defined(GC_USESIGRT_SIGNALS)
#   define SIG_SUSPEND (32+6)
# elif (defined(GC_FREEBSD_THREADS) || defined(HURD) || defined(RTEMS)) \
       && !defined(GC_USESIGRT_SIGNALS)
#   define SIG_SUSPEND SIGUSR1
        /* SIGTSTP and SIGCONT could be used alternatively on FreeBSD.  */
# elif defined(GC_OPENBSD_THREADS) && !defined(GC_USESIGRT_SIGNALS)
#     define SIG_SUSPEND SIGXFSZ
# elif defined(_SIGRTMIN) && !defined(CPPCHECK)
#   define SIG_SUSPEND _SIGRTMIN + 6
# else
#   define SIG_SUSPEND SIGRTMIN + 6
# endif
#endif /* GC_PTHREADS && !SIG_SUSPEND */

#if defined(GC_PTHREADS) && !defined(GC_SEM_INIT_PSHARED)
# define GC_SEM_INIT_PSHARED 0
#endif

/* Some macros for setjmp that works across signal handlers     */
/* were possible, and a couple of routines to facilitate        */
/* catching accesses to bad addresses when that's               */
/* possible/needed.                                             */
#if (defined(UNIX_LIKE) || (defined(NEED_FIND_LIMIT) && defined(CYGWIN32))) \
    && !defined(GC_NO_SIGSETJMP)
# if defined(SUNOS5SIGS) && !defined(FREEBSD) && !defined(LINUX)
    EXTERN_C_END
#   include <sys/siginfo.h>
    EXTERN_C_BEGIN
# endif
  /* Define SETJMP and friends to be the version that restores  */
  /* the signal mask.                                           */
# define SETJMP(env) sigsetjmp(env, 1)
# define LONGJMP(env, val) siglongjmp(env, val)
# define JMP_BUF sigjmp_buf
#else
# ifdef ECOS
#   define SETJMP(env) hal_setjmp(env)
# else
#   define SETJMP(env) setjmp(env)
# endif
# define LONGJMP(env, val) longjmp(env, val)
# define JMP_BUF jmp_buf
#endif /* !UNIX_LIKE || GC_NO_SIGSETJMP */

#if defined(DATASTART_USES_BSDGETDATASTART)
  EXTERN_C_END
# include <machine/trap.h>
  EXTERN_C_BEGIN
  GC_INNER ptr_t GC_FreeBSDGetDataStart(size_t, ptr_t);
# define DATASTART_IS_FUNC
#endif /* DATASTART_USES_BSDGETDATASTART */

#if defined(NEED_FIND_LIMIT) \
     || (defined(UNIX_LIKE) && !defined(NO_DEBUGGING)) \
     || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS)) \
     || (defined(WRAP_MARK_SOME) && defined(NO_SEH_AVAILABLE))
  typedef void (*GC_fault_handler_t)(int);
  GC_INNER void GC_set_and_save_fault_handler(GC_fault_handler_t);
#endif

#if defined(NEED_FIND_LIMIT) \
     || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS)) \
     || (defined(WRAP_MARK_SOME) && defined(NO_SEH_AVAILABLE))
  GC_EXTERN JMP_BUF GC_jmp_buf;

  /* Set up a handler for address faults which will longjmp to  */
  /* GC_jmp_buf.                                                */
  GC_INNER void GC_setup_temporary_fault_handler(void);

  /* Undo the effect of GC_setup_temporary_fault_handler.       */
  GC_INNER void GC_reset_fault_handler(void);
#endif /* NEED_FIND_LIMIT || USE_PROC_FOR_LIBRARIES || WRAP_MARK_SOME */

/* Some convenience macros for cancellation support. */
#ifdef CANCEL_SAFE
# if defined(GC_ASSERTIONS) \
     && (defined(USE_COMPILER_TLS) \
         || (defined(LINUX) && !defined(ARM32) && GC_GNUC_PREREQ(3, 3) \
             || defined(HPUX) /* and probably others ... */))
    extern __thread unsigned char GC_cancel_disable_count;
#   define NEED_CANCEL_DISABLE_COUNT
#   define INCR_CANCEL_DISABLE() ++GC_cancel_disable_count
#   define DECR_CANCEL_DISABLE() --GC_cancel_disable_count
#   define ASSERT_CANCEL_DISABLED() GC_ASSERT(GC_cancel_disable_count > 0)
# else
#   define INCR_CANCEL_DISABLE()
#   define DECR_CANCEL_DISABLE()
#   define ASSERT_CANCEL_DISABLED() (void)0
# endif /* !GC_ASSERTIONS */
# define DISABLE_CANCEL(state) \
        do { pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &state); \
          INCR_CANCEL_DISABLE(); } while (0)
# define RESTORE_CANCEL(state) \
        do { ASSERT_CANCEL_DISABLED(); \
          pthread_setcancelstate(state, NULL); \
          DECR_CANCEL_DISABLE(); } while (0)
#else
# define DISABLE_CANCEL(state) (void)0
# define RESTORE_CANCEL(state) (void)0
# define ASSERT_CANCEL_DISABLED() (void)0
#endif /* !CANCEL_SAFE */

/* Multiply 32-bit unsigned values (used by GC_push_contents_hdr).  */
#ifdef NO_LONGLONG64
# define LONG_MULT(hprod, lprod, x, y) \
    do { \
      unsigned32 lx = (x) & 0xffffU; \
      unsigned32 ly = (y) & 0xffffU; \
      unsigned32 hx = (x) >> 16; \
      unsigned32 hy = (y) >> 16; \
      unsigned32 lxhy = lx * hy; \
      unsigned32 mid = hx * ly + lxhy; /* may overflow */ \
      unsigned32 lxly = lx * ly; \
      \
      lprod = (mid << 16) + lxly; /* may overflow */ \
      hprod = hx * hy + ((lprod) < lxly ? 1U : 0) \
              + (mid < lxhy ? (unsigned32)0x10000UL : 0) + (mid >> 16); \
    } while (0)
#elif defined(I386) && defined(__GNUC__) && !defined(NACL)
# define LONG_MULT(hprod, lprod, x, y) \
    __asm__ __volatile__ ("mull %2" \
                          : "=a" (lprod), "=d" (hprod) \
                          : "r" (y), "0" (x))
#else
# if defined(__int64) && !defined(__GNUC__) && !defined(CPPCHECK)
#   define ULONG_MULT_T unsigned __int64
# else
#   define ULONG_MULT_T unsigned long long
# endif
# define LONG_MULT(hprod, lprod, x, y) \
    do { \
        ULONG_MULT_T prod = (ULONG_MULT_T)(x) * (ULONG_MULT_T)(y); \
        \
        GC_STATIC_ASSERT(sizeof(x) + sizeof(y) <= sizeof(prod)); \
        hprod = (unsigned32)(prod >> 32); \
        lprod = (unsigned32)prod; \
    } while (0)
#endif /* !I386 && !NO_LONGLONG64 */

EXTERN_C_END

#endif /* GC_PRIVATE_H */

#ifdef KEEP_BACK_PTRS
# include "gc/gc_backptr.h"
#endif

EXTERN_C_BEGIN

#ifndef GC_FREED_MEM_MARKER
# if CPP_WORDSZ == 32
#   define GC_FREED_MEM_MARKER (GC_uintptr_t)0xdeadbeef
# else
#   define GC_FREED_MEM_MARKER (GC_uintptr_t)GC_WORD_C(0xEFBEADDEdeadbeef)
# endif
#endif /* !GC_FREED_MEM_MARKER */

#if CPP_WORDSZ == 32
# define START_FLAG (GC_uintptr_t)0xfedcedcb
# define END_FLAG (GC_uintptr_t)0xbcdecdef
#else
# define START_FLAG (GC_uintptr_t)GC_WORD_C(0xFEDCEDCBfedcedcb)
# define END_FLAG (GC_uintptr_t)GC_WORD_C(0xBCDECDEFbcdecdef)
#endif
        /* Stored both one past the end of user object, and one before  */
        /* the end of the object as seen by the allocator.              */

#if defined(KEEP_BACK_PTRS) || defined(PRINT_BLACK_LIST)
  /* Pointer "source"s that aren't real locations.      */
  /* Used in oh_back_ptr fields and as "source"         */
  /* argument to some marking functions.                */
# define MARKED_FOR_FINALIZATION ((ptr_t)(GC_uintptr_t)2)
                /* Object was marked because it is finalizable. */
# define MARKED_FROM_REGISTER ((ptr_t)(GC_uintptr_t)4)
                /* Object was marked from a register.  Hence the        */
                /* source of the reference doesn't have an address.     */
# define NOT_MARKED ((ptr_t)(GC_uintptr_t)8)
#endif /* KEEP_BACK_PTRS || PRINT_BLACK_LIST */

/* Object debug header. */
typedef struct {
# if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
    /* We potentially keep two different kinds of back          */
    /* pointers.  KEEP_BACK_PTRS stores a single back           */
    /* pointer in each reachable object to allow reporting      */
    /* of why an object was retained.  MAKE_BACK_GRAPH          */
    /* builds a graph containing the inverse of all             */
    /* "points-to" edges including those involving              */
    /* objects that have just become unreachable. This          */
    /* allows detection of growing chains of unreachable        */
    /* objects.  It may be possible to eventually combine       */
    /* both, but for now we keep them separate.  Both           */
    /* kinds of back pointers are hidden using the              */
    /* following macros.  In both cases, the plain version      */
    /* is constrained to have the least significant bit of 1,   */
    /* to allow it to be distinguished from a free-list         */
    /* link.  This means the plain version must have the least  */
    /* significant bit of zero.  Note that blocks dropped by    */
    /* black-listing will also have the the least significant   */
    /* bit clear once debugging has started; we are careful     */
    /* never to overwrite such a value.                         */
#   if ALIGNMENT == 1
      /* Fudge back pointer to be even. */
#     define HIDE_BACK_PTR(p) \
                GC_HIDE_POINTER((ptr_t)(~(GC_uintptr_t)1 & (GC_uintptr_t)(p)))
#   else
#     define HIDE_BACK_PTR(p) GC_HIDE_POINTER(p)
#   endif
    GC_hidden_pointer oh_back_ptr;
    GC_hidden_pointer oh_bg_ptr;
                        /* Always define either none or both of the     */
                        /* fields to ensure double-pointer alignment.   */
# endif
  const char * oh_string;       /* object descriptor string (file name)    */
  signed_word oh_int;           /* object descriptor integer (line number) */
# ifdef NEED_CALLINFO
    struct callinfo oh_ci[NFRAMES];
# endif
# ifndef SHORT_DBG_HDRS
    GC_uintptr_t oh_sz;                 /* Original malloc argument.    */
    GC_uintptr_t oh_sf;                 /* Start flag.                  */
# endif
} oh;
/* The size of the above structure is assumed not to de-align things,   */
/* and to be a multiple of a double-pointer length.                     */

#ifdef SHORT_DBG_HDRS
# define DEBUG_BYTES sizeof(oh)
# define UNCOLLECTABLE_DEBUG_BYTES DEBUG_BYTES
#else
  /* Add space for END_FLAG, but use any extra space that was already   */
  /* added to catch off-the-end pointers.                               */
  /* For uncollectible objects, the extra byte is not added.            */
# define UNCOLLECTABLE_DEBUG_BYTES (sizeof(oh) + sizeof(GC_uintptr_t))
# define DEBUG_BYTES (UNCOLLECTABLE_DEBUG_BYTES - EXTRA_BYTES)
#endif

/* ADD_CALL_CHAIN stores a (partial) call chain into an object  */
/* header; it should be called with the allocator lock held.    */
/* PRINT_CALL_CHAIN prints the call chain stored in an object   */
/* to stderr; it requires we do not hold the allocator lock.    */
#if defined(SAVE_CALL_CHAIN)
# define ADD_CALL_CHAIN(base, ra) GC_save_callers(((oh *)(base)) -> oh_ci)
# if defined(REDIRECT_MALLOC) && defined(THREADS) && defined(DBG_HDRS_ALL) \
     && NARGS == 0 && NFRAMES % 2 == 0 && defined(GC_HAVE_BUILTIN_BACKTRACE)
    GC_INNER void GC_save_callers_no_unlock(struct callinfo info[NFRAMES]);
#   define ADD_CALL_CHAIN_INNER(base) \
                    GC_save_callers_no_unlock(((oh *)(base)) -> oh_ci)
# endif
# define PRINT_CALL_CHAIN(base) GC_print_callers(((oh *)(base)) -> oh_ci)
#elif defined(GC_ADD_CALLER)
# define ADD_CALL_CHAIN(base, ra) ((oh *)(base)) -> oh_ci[0].ci_pc = (ra)
# define PRINT_CALL_CHAIN(base) GC_print_callers(((oh *)(base)) -> oh_ci)
#else
# define ADD_CALL_CHAIN(base, ra)
# define PRINT_CALL_CHAIN(base)
#endif

#if !defined(ADD_CALL_CHAIN_INNER) && defined(DBG_HDRS_ALL)
  /* A variant of ADD_CALL_CHAIN() used for internal allocations.   */
# define ADD_CALL_CHAIN_INNER(base) ADD_CALL_CHAIN(base, GC_RETURN_ADDR)
#endif

#ifdef GC_ADD_CALLER
# define OPT_RA ra,
#else
# define OPT_RA
#endif

/* Check whether object with base pointer p has debugging info  */
/* p is assumed to point to a legitimate object in our part     */
/* of the heap.                                                 */
#ifdef SHORT_DBG_HDRS
# define GC_has_other_debug_info(base) 1
#else
  GC_INNER int GC_has_other_debug_info(ptr_t base);
#endif

#if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
# if defined(SHORT_DBG_HDRS) && !defined(CPPCHECK)
#   error Non-ptr stored in object results in GC_HAS_DEBUG_INFO malfunction
    /* We may mistakenly conclude that base has a debugging wrapper.    */
# endif
# if defined(PARALLEL_MARK) && defined(KEEP_BACK_PTRS)
#   define GC_HAS_DEBUG_INFO(base) \
        (((GC_uintptr_t)GC_cptr_load((volatile ptr_t *)(base)) & 1) != 0 \
         && GC_has_other_debug_info(base) > 0)
                        /* Atomic load is used as GC_store_back_pointer */
                        /* stores oh_back_ptr atomically (base might    */
                        /* point to the field); this prevents a TSan    */
                        /* warning.                                     */
# else
#   define GC_HAS_DEBUG_INFO(base) \
                ((*(GC_uintptr_t *)(base) & 1) != 0 \
                 && GC_has_other_debug_info(base) > 0)
# endif
#else
# define GC_HAS_DEBUG_INFO(base) (GC_has_other_debug_info(base) > 0)
#endif /* !KEEP_BACK_PTRS && !MAKE_BACK_GRAPH */

EXTERN_C_END

#endif /* GC_DBG_MLC_H */


/*
 * This implements a full, though not well-tuned, representation of the
 * backwards points-to graph.  This is used to test for non-GC-robust
 * data structures; the code is not used during normal garbage collection.
 *
 * One restriction is that we drop all back-edges from nodes with very
 * high in-degree, and simply add them add them to a list of such
 * nodes.  They are then treated as permanent roots.  If this by itself
 * doesn't introduce a space leak, then such nodes can't contribute to
 * a growing space leak.
 */

#ifdef MAKE_BACK_GRAPH

#define MAX_IN  10      /* Maximum in-degree we handle directly */

#if (!defined(DBG_HDRS_ALL) || (ALIGNMENT != CPP_PTRSZ / 8) \
     /* || !defined(UNIX_LIKE) */) && !defined(CPPCHECK)
# error The configuration does not support MAKE_BACK_GRAPH
#endif

/* We store single back pointers directly in the object's oh_bg_ptr field. */
/* If there is more than one ptr to an object, we store q or'ed with       */
/* FLAG_MANY, where q is a pointer to a back_edges object.                 */
/* Every once in a while we use a back_edges object even for a single      */
/* pointer, since we need the other fields in the back_edges structure to  */
/* be present in some fraction of the objects.  Otherwise we get serious   */
/* performance issues.                                                     */
#define FLAG_MANY 2

typedef struct back_edges_struct {
  word n_edges; /* Number of edges, including those in continuation     */
                /* structures.                                          */
  unsigned short flags;
#       define RETAIN 1 /* Directly points to a reachable object;       */
                        /* retain for next GC.                          */
  unsigned short height_gc_no;
                /* If height > 0, then the GC_gc_no value when it       */
                /* was computed.  If it was computed this cycle, then   */
                /* it is current.  If it was computed during the        */
                /* last cycle, then it represents the old height,       */
                /* which is only saved for live objects referenced by   */
                /* dead ones.  This may grow due to refs from newly     */
                /* dead objects.                                        */
  signed_word height;
                /* Longest path through unreachable nodes to this node  */
                /* that we found using depth first search.              */
# define HEIGHT_UNKNOWN      (-2)
# define HEIGHT_IN_PROGRESS  (-1)

  ptr_t edges[MAX_IN];
  struct back_edges_struct *cont;
                /* Pointer to continuation structure; we use only the   */
                /* edges field in the continuation.                     */
                /* also used as a free-list link.                       */
} back_edges;

/* Allocate a new back edge structure.  Should be more sophisticated    */
/* if this were production code.                                        */
#define MAX_BACK_EDGE_STRUCTS 100000
static back_edges *back_edge_space = 0;
STATIC int GC_n_back_edge_structs = 0;
                                /* Serves as pointer to never used      */
                                /* back_edges space.                    */
static back_edges *avail_back_edges = 0;
                                /* Pointer to free list of deallocated  */
                                /* back_edges structures.               */

static back_edges * new_back_edges(void)
{
  GC_ASSERT(I_HOLD_LOCK());
  if (0 == back_edge_space) {
    size_t bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(MAX_BACK_EDGE_STRUCTS
                                                   * sizeof(back_edges));

    GC_ASSERT(GC_page_size != 0);
    back_edge_space = (back_edges *)GC_os_get_mem(bytes_to_get);
    if (NULL == back_edge_space)
      ABORT("Insufficient memory for back edges");
  }
  if (0 != avail_back_edges) {
    back_edges * result = avail_back_edges;
    avail_back_edges = result -> cont;
    result -> cont = 0;
    return result;
  }
  if (GC_n_back_edge_structs >= MAX_BACK_EDGE_STRUCTS - 1) {
    ABORT("Needed too much space for back edges: adjust "
          "MAX_BACK_EDGE_STRUCTS");
  }
  return back_edge_space + (GC_n_back_edge_structs++);
}

/* Deallocate p and its associated continuation structures.     */
static void deallocate_back_edges(back_edges *p)
{
   back_edges *last = p;

   while (0 != last -> cont) last = last -> cont;
   last -> cont = avail_back_edges;
   avail_back_edges = p;
}

/* Table of objects that are currently on the depth-first search        */
/* stack.  Only objects with in-degree one are in this table.           */
/* Other objects are identified using HEIGHT_IN_PROGRESS.               */
/* FIXME: This data structure NEEDS IMPROVEMENT.                        */
#define INITIAL_IN_PROGRESS 10000
static ptr_t * in_progress_space = 0;
static size_t in_progress_size = 0;
static size_t n_in_progress = 0;

static void push_in_progress(ptr_t p)
{
  GC_ASSERT(I_HOLD_LOCK());
  if (n_in_progress >= in_progress_size) {
    ptr_t * new_in_progress_space;

    GC_ASSERT(GC_page_size != 0);
    if (NULL == in_progress_space) {
      in_progress_size = ROUNDUP_PAGESIZE_IF_MMAP(INITIAL_IN_PROGRESS
                                                        * sizeof(ptr_t))
                                / sizeof(ptr_t);
      new_in_progress_space =
                (ptr_t *)GC_os_get_mem(in_progress_size * sizeof(ptr_t));
    } else {
      in_progress_size *= 2;
      new_in_progress_space =
                (ptr_t *)GC_os_get_mem(in_progress_size * sizeof(ptr_t));
      if (new_in_progress_space != NULL)
        BCOPY(in_progress_space, new_in_progress_space,
              n_in_progress * sizeof(ptr_t));
    }
#   ifndef GWW_VDB
      GC_scratch_recycle_no_gww(in_progress_space,
                                n_in_progress * sizeof(ptr_t));
#   elif defined(LINT2)
      /* TODO: implement GWW-aware recycling as in alloc_mark_stack */
      GC_noop1_ptr(in_progress_space);
#   endif
    in_progress_space = new_in_progress_space;
  }
  if (in_progress_space == 0)
      ABORT("MAKE_BACK_GRAPH: Out of in-progress space: "
            "Huge linear data structure?");
  in_progress_space[n_in_progress++] = p;
}

static GC_bool is_in_progress(const char *p)
{
  size_t i;
  for (i = 0; i < n_in_progress; ++i) {
    if (in_progress_space[i] == p) return TRUE;
  }
  return FALSE;
}

GC_INLINE void pop_in_progress(ptr_t p)
{
# ifndef GC_ASSERTIONS
    UNUSED_ARG(p);
# endif
  --n_in_progress;
  GC_ASSERT(in_progress_space[n_in_progress] == p);
}

#define GET_OH_BG_PTR(p) \
                (ptr_t)GC_REVEAL_POINTER(((oh *)(p)) -> oh_bg_ptr)
#define SET_OH_BG_PTR(p,q) (((oh *)(p)) -> oh_bg_ptr = GC_HIDE_POINTER(q))

/* Ensure that p has a back_edges structure associated with it. */
static void ensure_struct(ptr_t p)
{
  ptr_t old_back_ptr = GET_OH_BG_PTR(p);

  GC_ASSERT(I_HOLD_LOCK());
  if ((ADDR(old_back_ptr) & FLAG_MANY) == 0) {
    back_edges *be = new_back_edges();

    be -> flags = 0;
#   if defined(CPPCHECK)
      GC_noop1_ptr(&old_back_ptr);
      /* Workaround a false positive that old_back_ptr cannot be null.  */
#   endif
    if (NULL == old_back_ptr) {
      be -> n_edges = 0;
    } else {
      be -> n_edges = 1;
      be -> edges[0] = old_back_ptr;
    }
    be -> height = HEIGHT_UNKNOWN;
    be -> height_gc_no = (unsigned short)(GC_gc_no - 1);
    GC_ASSERT(ADDR_GE((ptr_t)be, (ptr_t)back_edge_space));
    SET_OH_BG_PTR(p, CPTR_SET_FLAGS(be, FLAG_MANY));
  }
}

/* Add the (forward) edge from p to q to the backward graph.  Both p    */
/* q are pointers to the object base, i.e. pointers to an oh.           */
static void add_edge(ptr_t p, ptr_t q)
{
    ptr_t pred = GET_OH_BG_PTR(q);
    back_edges *be, *be_cont;
    word i;

    GC_ASSERT(p == GC_base(p) && q == GC_base(q));
    GC_ASSERT(I_HOLD_LOCK());
    if (!GC_HAS_DEBUG_INFO(q) || !GC_HAS_DEBUG_INFO(p)) {
      /* This is really a misinterpreted free-list link, since we saw   */
      /* a pointer to a free list.  Don't overwrite it!                 */
      return;
    }
#   if defined(CPPCHECK)
      GC_noop1_ptr(&pred);
#   endif
    if (NULL == pred) {
      static unsigned random_number = 13;
#     define GOT_LUCKY_NUMBER (((++random_number) & 0x7f) == 0)
        /* A not very random number we use to occasionally allocate a   */
        /* back_edges structure even for a single backward edge.  This  */
        /* prevents us from repeatedly tracing back through very long   */
        /* chains, since we will have some place to store height and    */
        /* in_progress flags along the way.                             */

        SET_OH_BG_PTR(q, p);
        if (GOT_LUCKY_NUMBER) ensure_struct(q);
        return;
    }

    /* Check whether it was already in the list of predecessors. */
    {
      back_edges *e = (back_edges *)CPTR_CLEAR_FLAGS(pred, FLAG_MANY);
      word n_edges;
      word total;
      int local = 0;

      if ((ADDR(pred) & FLAG_MANY) != 0) {
        n_edges = e -> n_edges;
      } else if ((COVERT_DATAFLOW(ADDR(pred)) & 1) == 0) {
        /* A misinterpreted free-list link.     */
        n_edges = 1;
        local = -1;
      } else {
        n_edges = 0;
      }
      for (total = 0; total < n_edges; ++total) {
        if (local == MAX_IN) {
          e = e -> cont;
          local = 0;
        }
        if (local >= 0)
          pred = e -> edges[local++];
        if (pred == p)
          return;
      }
    }

    ensure_struct(q);
    be = (back_edges *)CPTR_CLEAR_FLAGS(GET_OH_BG_PTR(q), FLAG_MANY);
    for (i = be -> n_edges, be_cont = be; i > MAX_IN; i -= MAX_IN)
        be_cont = be_cont -> cont;
    if (i == MAX_IN) {
        be_cont -> cont = new_back_edges();
        be_cont = be_cont -> cont;
        i = 0;
    }
    be_cont -> edges[i] = p;
    be -> n_edges++;
#   ifdef DEBUG_PRINT_BIG_N_EDGES
      if (GC_print_stats == VERBOSE && be -> n_edges == 100) {
        GC_err_printf("The following object has big in-degree:\n");
        GC_print_heap_obj(q);
      }
#   endif
}

typedef void (*per_object_func)(ptr_t p, size_t sz, word descr);

static GC_CALLBACK void per_object_helper(struct hblk *h, void *fn_ptr)
{
  const hdr *hhdr = HDR(h);
  word descr = hhdr -> hb_descr;
  per_object_func fn = *(per_object_func *)fn_ptr;
  size_t sz = hhdr -> hb_sz;
  size_t i = 0;

  do {
    fn((ptr_t)(h -> hb_body + i), sz, descr);
    i += sz;
  } while (i + sz <= HBLKSIZE);
}

GC_INLINE void GC_apply_to_each_object(per_object_func fn)
{
  GC_apply_to_all_blocks(per_object_helper, &fn);
}

static void reset_back_edge(ptr_t p, size_t sz, word descr)
{
  UNUSED_ARG(sz);
  UNUSED_ARG(descr);
  GC_ASSERT(I_HOLD_LOCK());
  /* Skip any free-list links, or dropped blocks.   */
  if (GC_HAS_DEBUG_INFO(p)) {
    ptr_t old_back_ptr = GET_OH_BG_PTR(p);

    if ((ADDR(old_back_ptr) & FLAG_MANY) != 0) {
      back_edges *be = (back_edges *)CPTR_CLEAR_FLAGS(old_back_ptr, FLAG_MANY);

      if (!(be -> flags & RETAIN)) {
        deallocate_back_edges(be);
        SET_OH_BG_PTR(p, 0);
      } else {
        GC_ASSERT(GC_is_marked(p));

        /* Back edges may point to objects that will not be retained.   */
        /* Delete them for now, but remember the height.                */
        /* Some will be added back at next GC.                          */
        be -> n_edges = 0;
        if (0 != be -> cont) {
          deallocate_back_edges(be -> cont);
          be -> cont = 0;
        }

        GC_ASSERT(GC_is_marked(p));
        /* We only retain things for one GC cycle at a time.            */
        be -> flags &= (unsigned short)~RETAIN;
      }
    } else /* Simple back pointer */ {
      /* Clear to avoid dangling pointer. */
      SET_OH_BG_PTR(p, 0);
    }
  }
}

static void add_back_edges(ptr_t p, size_t sz, word descr)
{
  ptr_t current_p = p + sizeof(oh);

  /* For now, fix up non-length descriptors conservatively.     */
    if ((descr & GC_DS_TAGS) != GC_DS_LENGTH) {
      descr = sz;
    }

  for (; ADDR_LT(current_p, p + descr); current_p += sizeof(ptr_t)) {
    ptr_t q;

    LOAD_PTR_OR_CONTINUE(q, current_p);
    FIXUP_POINTER(q);
    if (GC_least_real_heap_addr < ADDR(q)
        && ADDR(q) < GC_greatest_real_heap_addr) {
      ptr_t target = (ptr_t)GC_base(q);

      if (target != NULL)
        add_edge(p, target);
    }
  }
}

/* Rebuild the representation of the backward reachability graph.       */
/* Does not examine mark bits.  Can be called before GC.                */
GC_INNER void GC_build_back_graph(void)
{
  GC_ASSERT(I_HOLD_LOCK());
  GC_apply_to_each_object(add_back_edges);
}

/* Return an approximation to the length of the longest simple path     */
/* through unreachable objects to p.  We refer to this as the height    */
/* of p.                                                                */
static word backwards_height(ptr_t p)
{
  word result;
  ptr_t pred = GET_OH_BG_PTR(p);
  back_edges *be;

  GC_ASSERT(I_HOLD_LOCK());
# if defined(CPPCHECK)
    GC_noop1_ptr(&pred);
# endif
  if (NULL == pred)
    return 1;
  if ((ADDR(pred) & FLAG_MANY) == 0) {
    if (is_in_progress(p)) return 0; /* DFS back edge, i.e. we followed */
                                     /* an edge to an object already    */
                                     /* on our stack: ignore            */
    push_in_progress(p);
    result = backwards_height(pred) + 1;
    pop_in_progress(p);
    return result;
  }
  be = (back_edges *)CPTR_CLEAR_FLAGS(pred, FLAG_MANY);
  if (be -> height >= 0 && be -> height_gc_no == (unsigned short)GC_gc_no)
      return (word)(be -> height);
  /* Ignore back edges in DFS */
    if (be -> height == HEIGHT_IN_PROGRESS) return 0;
  result = be -> height > 0 ? (word)(be -> height) : 1U;
  be -> height = HEIGHT_IN_PROGRESS;

  {
      back_edges *e = be;
      word n_edges;
      word total;
      int local = 0;

      if ((ADDR(pred) & FLAG_MANY) != 0) {
        n_edges = e -> n_edges;
      } else if ((ADDR(pred) & 1) == 0) {
        /* A misinterpreted free-list link.     */
        n_edges = 1;
        local = -1;
      } else {
        n_edges = 0;
      }
      for (total = 0; total < n_edges; ++total) {
        word this_height;
        if (local == MAX_IN) {
          e = e -> cont;
          local = 0;
        }
        if (local >= 0)
          pred = e -> edges[local++];

        /* Execute the following once for each predecessor pred of p    */
        /* in the points-to graph.                                      */
        if (GC_is_marked(pred) && (ADDR(GET_OH_BG_PTR(p)) & FLAG_MANY) == 0) {
          GC_COND_LOG_PRINTF("Found bogus pointer from %p to %p\n",
                             (void *)pred, (void *)p);
            /* Reachable object "points to" unreachable one.            */
            /* Could be caused by our lax treatment of GC descriptors.  */
          this_height = 1;
        } else {
          this_height = backwards_height(pred);
        }
        if (this_height >= result)
          result = this_height + 1;
      }
  }

  be -> height = (signed_word)result;
  be -> height_gc_no = (unsigned short)GC_gc_no;
  return result;
}

STATIC word GC_max_height = 0;
STATIC ptr_t GC_deepest_obj = NULL;

/* Compute the maximum height of every unreachable predecessor p of a   */
/* reachable object.  Arrange to save the heights of all such objects p */
/* so that they can be used in calculating the height of objects in the */
/* next GC.                                                             */
/* Set GC_max_height to be the maximum height we encounter, and         */
/* GC_deepest_obj to be the corresponding object.                       */
static void update_max_height(ptr_t p, size_t sz, word descr)
{
  UNUSED_ARG(sz);
  UNUSED_ARG(descr);
  GC_ASSERT(I_HOLD_LOCK());
  if (GC_is_marked(p) && GC_HAS_DEBUG_INFO(p)) {
    word p_height = 0;
    ptr_t p_deepest_obj = 0;
    ptr_t back_ptr;
    back_edges *be = 0;

    /* If we remembered a height last time, use it as a minimum.        */
    /* It may have increased due to newly unreachable chains pointing   */
    /* to p, but it can't have decreased.                               */
    back_ptr = GET_OH_BG_PTR(p);
#   if defined(CPPCHECK)
      GC_noop1_ptr(&back_ptr);
#   endif
    if (back_ptr != NULL && (ADDR(back_ptr) & FLAG_MANY) != 0) {
      be = (back_edges *)CPTR_CLEAR_FLAGS(back_ptr, FLAG_MANY);
      if (be -> height != HEIGHT_UNKNOWN)
        p_height = (word)(be -> height);
    }

    {
      ptr_t pred = back_ptr;
      back_edges *e = (back_edges *)CPTR_CLEAR_FLAGS(pred, FLAG_MANY);
      word n_edges;
      word total;
      int local = 0;

      if ((ADDR(pred) & FLAG_MANY) != 0) {
        n_edges = e -> n_edges;
      } else if (pred != NULL && (ADDR(pred) & 1) == 0) {
        /* A misinterpreted free-list link.     */
        n_edges = 1;
        local = -1;
      } else {
        n_edges = 0;
      }
      for (total = 0; total < n_edges; ++total) {
        if (local == MAX_IN) {
          e = e -> cont;
          local = 0;
        }
        if (local >= 0)
          pred = e -> edges[local++];

        /* Execute the following once for each predecessor pred of p    */
        /* in the points-to graph.                                      */
        if (!GC_is_marked(pred) && GC_HAS_DEBUG_INFO(pred)) {
          word this_height = backwards_height(pred);

          if (this_height > p_height) {
            p_height = this_height;
            p_deepest_obj = pred;
          }
        }
      }
    }

    if (p_height > 0) {
      /* Remember the height for next time. */
        if (NULL == be) {
          ensure_struct(p);
          back_ptr = GET_OH_BG_PTR(p);
          be = (back_edges *)CPTR_CLEAR_FLAGS(back_ptr, FLAG_MANY);
        }
        be -> flags |= RETAIN;
        be -> height = (signed_word)p_height;
        be -> height_gc_no = (unsigned short)GC_gc_no;
    }
    if (p_height > GC_max_height) {
        GC_max_height = p_height;
        GC_deepest_obj = p_deepest_obj;
    }
  }
}

STATIC word GC_max_max_height = 0;

GC_INNER void GC_traverse_back_graph(void)
{
  GC_ASSERT(I_HOLD_LOCK());
  GC_max_height = 0;
  GC_apply_to_each_object(update_max_height);
  if (0 != GC_deepest_obj)
    GC_set_mark_bit(GC_deepest_obj);  /* Keep it until we can print it. */
}

void GC_print_back_graph_stats(void)
{
  GC_ASSERT(I_HOLD_LOCK());
  GC_printf("Maximum backwards height of reachable objects"
            " at GC #%lu is %lu\n",
            (unsigned long)GC_gc_no, (unsigned long)GC_max_height);
  if (GC_max_height > GC_max_max_height) {
    ptr_t obj = GC_deepest_obj;

    GC_max_max_height = GC_max_height;
    UNLOCK();
    GC_err_printf(
            "The following unreachable object is last in a longest chain "
            "of unreachable objects:\n");
    GC_print_heap_obj(obj);
    LOCK();
  }
  GC_COND_LOG_PRINTF("Needed max total of %d back-edge structs\n",
                     GC_n_back_edge_structs);
  GC_apply_to_each_object(reset_back_edge);
  GC_deepest_obj = 0;
}

#endif /* MAKE_BACK_GRAPH */

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


/*
 * We maintain several hash tables of hblks that have had false hits.
 * Each contains one bit per hash bucket;  If any page in the bucket
 * has had a false hit, we assume that all of them have.
 * See the definition of page_hash_table in gc_priv.h.
 * False hits from the stack(s) are much more dangerous than false hits
 * from elsewhere, since the former can pin a large object that spans the
 * block, even though it does not start on the dangerous block.
 */

/* Externally callable routines are:    */
/* - GC_add_to_black_list_normal,       */
/* - GC_add_to_black_list_stack,        */
/* - GC_promote_black_lists.            */

/* Pointers to individual tables.  We replace one table by another by   */
/* switching these pointers.                                            */
STATIC word * GC_old_normal_bl = NULL;
                /* Nonstack false references seen at last full          */
                /* collection.                                          */
STATIC word * GC_incomplete_normal_bl = NULL;
                /* Nonstack false references seen since last            */
                /* full collection.                                     */
STATIC word * GC_old_stack_bl = NULL;
STATIC word * GC_incomplete_stack_bl = NULL;

STATIC word GC_total_stack_black_listed = 0;
                        /* Number of bytes on stack blacklist.  */

GC_INNER word GC_black_list_spacing = MINHINCR * HBLKSIZE;
                        /* Initial rough guess. */

STATIC void GC_clear_bl(word *);

GC_INNER void GC_default_print_heap_obj_proc(ptr_t p)
{
    ptr_t base = (ptr_t)GC_base(p);
    int kind = HDR(base) -> hb_obj_kind;

    GC_err_printf("object at %p of appr. %lu bytes (%s)\n",
                  (void *)base, (unsigned long)GC_size(base),
                  kind == PTRFREE ? "atomic" :
                    IS_UNCOLLECTABLE(kind) ? "uncollectable" : "composite");
}

GC_INNER void (*GC_print_heap_obj)(ptr_t p) = GC_default_print_heap_obj_proc;

#ifdef PRINT_BLACK_LIST
  STATIC void GC_print_blacklisted_ptr(ptr_t p, ptr_t source,
                                       const char *kind_str)
  {
    ptr_t base = (ptr_t)GC_base(source);

    if (0 == base) {
        GC_err_printf("Black listing (%s) %p referenced from %p in %s\n",
                      kind_str, (void *)p, (void *)source,
                      NULL != source ? "root set" : "register");
    } else {
        /* FIXME: We can't call the debug version of GC_print_heap_obj  */
        /* (with PRINT_CALL_CHAIN) here because the allocator lock is   */
        /* held and the world is stopped.                               */
        GC_err_printf("Black listing (%s) %p referenced from %p in"
                      " object at %p of appr. %lu bytes\n",
                      kind_str, (void *)p, (void *)source,
                      (void *)base, (unsigned long)GC_size(base));
    }
  }
#endif /* PRINT_BLACK_LIST */

GC_INNER void GC_bl_init_no_interiors(void)
{
  GC_ASSERT(I_HOLD_LOCK());
  if (NULL == GC_incomplete_normal_bl) {
    GC_old_normal_bl = (word *)GC_scratch_alloc(sizeof(page_hash_table));
    GC_incomplete_normal_bl = (word *)GC_scratch_alloc(
                                                  sizeof(page_hash_table));
    if (NULL == GC_old_normal_bl || NULL == GC_incomplete_normal_bl) {
      GC_err_printf("Insufficient memory for black list\n");
      EXIT();
    }
    GC_clear_bl(GC_old_normal_bl);
    GC_clear_bl(GC_incomplete_normal_bl);
  }
}

GC_INNER void GC_bl_init(void)
{
    GC_ASSERT(I_HOLD_LOCK());
    if (!GC_all_interior_pointers) {
        GC_bl_init_no_interiors();
    }
    GC_ASSERT(NULL == GC_old_stack_bl && NULL == GC_incomplete_stack_bl);
    GC_old_stack_bl = (word *)GC_scratch_alloc(sizeof(page_hash_table));
    GC_incomplete_stack_bl = (word *)GC_scratch_alloc(sizeof(page_hash_table));
    if (NULL == GC_old_stack_bl || NULL == GC_incomplete_stack_bl) {
        GC_err_printf("Insufficient memory for black list\n");
        EXIT();
    }
    GC_clear_bl(GC_old_stack_bl);
    GC_clear_bl(GC_incomplete_stack_bl);
}

STATIC void GC_clear_bl(word *bl)
{
    BZERO(bl, sizeof(page_hash_table));
}

STATIC void GC_copy_bl(const word *old, word *dest)
{
    BCOPY(old, dest, sizeof(page_hash_table));
}

static word total_stack_black_listed(void);

/* Signal the completion of a collection.  Turn the incomplete black    */
/* lists into new black lists, etc.                                     */
GC_INNER void GC_promote_black_lists(void)
{
    word * very_old_normal_bl = GC_old_normal_bl;
    word * very_old_stack_bl = GC_old_stack_bl;

    GC_ASSERT(I_HOLD_LOCK());
    GC_old_normal_bl = GC_incomplete_normal_bl;
    GC_old_stack_bl = GC_incomplete_stack_bl;
    if (!GC_all_interior_pointers) {
      GC_clear_bl(very_old_normal_bl);
    }
    GC_clear_bl(very_old_stack_bl);
    GC_incomplete_normal_bl = very_old_normal_bl;
    GC_incomplete_stack_bl = very_old_stack_bl;
    GC_total_stack_black_listed = total_stack_black_listed();
    GC_VERBOSE_LOG_PRINTF(
                "%lu bytes in heap blacklisted for interior pointers\n",
                (unsigned long)GC_total_stack_black_listed);
    if (GC_total_stack_black_listed != 0) {
        GC_black_list_spacing =
                HBLKSIZE*(GC_heapsize/GC_total_stack_black_listed);
    }
    if (GC_black_list_spacing < 3 * HBLKSIZE) {
        GC_black_list_spacing = 3 * HBLKSIZE;
    }
    if (GC_black_list_spacing > MAXHINCR * HBLKSIZE) {
        GC_black_list_spacing = MAXHINCR * HBLKSIZE;
        /* Makes it easier to allocate really huge blocks, which otherwise */
        /* may have problems with nonuniform blacklist distributions.      */
        /* This way we should always succeed immediately after growing the */
        /* heap.                                                           */
    }
}

GC_INNER void GC_unpromote_black_lists(void)
{
    if (!GC_all_interior_pointers) {
      GC_copy_bl(GC_old_normal_bl, GC_incomplete_normal_bl);
    }
    GC_copy_bl(GC_old_stack_bl, GC_incomplete_stack_bl);
}

#if defined(PARALLEL_MARK) && defined(THREAD_SANITIZER)
# define backlist_set_pht_entry_from_index(db, index) \
                        set_pht_entry_from_index_concurrent(db, index)
#else
  /* It is safe to set a bit in a blacklist even without        */
  /* synchronization, the only drawback is that we might have   */
  /* to redo blacklisting sometimes.                            */
# define backlist_set_pht_entry_from_index(bl, index) \
                        set_pht_entry_from_index(bl, index)
#endif

/* P is not a valid pointer reference, but it falls inside      */
/* the plausible heap bounds.                                   */
/* Add it to the normal incomplete black list if appropriate.   */
#ifdef PRINT_BLACK_LIST
  GC_INNER void GC_add_to_black_list_normal(ptr_t p, ptr_t source)
#else
  GC_INNER void GC_add_to_black_list_normal(ptr_t p)
#endif
{
# ifndef PARALLEL_MARK
    GC_ASSERT(I_HOLD_LOCK());
# endif
  if (GC_modws_valid_offsets[ADDR(p) & (sizeof(ptr_t)-1)]) {
    size_t index = PHT_HASH(p);

    if (NULL == HDR(p) || get_pht_entry_from_index(GC_old_normal_bl, index)) {
#     ifdef PRINT_BLACK_LIST
        if (!get_pht_entry_from_index(GC_incomplete_normal_bl, index)) {
          GC_print_blacklisted_ptr(p, source, "normal");
        }
#     endif
      backlist_set_pht_entry_from_index(GC_incomplete_normal_bl, index);
    } /* else this is probably just an interior pointer to an allocated */
      /* object, and isn't worth black listing.                         */
  }
}

/* And the same for false pointers from the stack. */
#ifdef PRINT_BLACK_LIST
  GC_INNER void GC_add_to_black_list_stack(ptr_t p, ptr_t source)
#else
  GC_INNER void GC_add_to_black_list_stack(ptr_t p)
#endif
{
  size_t index = PHT_HASH(p);

# ifndef PARALLEL_MARK
    GC_ASSERT(I_HOLD_LOCK());
# endif
  if (NULL == HDR(p) || get_pht_entry_from_index(GC_old_stack_bl, index)) {
#   ifdef PRINT_BLACK_LIST
      if (!get_pht_entry_from_index(GC_incomplete_stack_bl, index)) {
        GC_print_blacklisted_ptr(p, source, "stack");
      }
#   endif
    backlist_set_pht_entry_from_index(GC_incomplete_stack_bl, index);
  }
}

/* Is the block starting at h of size len bytes black-listed?  If so,   */
/* return the address of the next plausible r such that (r,len) might   */
/* not be black-listed.  (Pointer r may not actually be in the heap.    */
/* We guarantee only that every smaller value of r after h is also      */
/* black-listed.)  If (h,len) is not, then return NULL.  Knows about    */
/* the structure of the black list hash tables.  Assumes the allocator  */
/* lock is held but no assertion about it by design.                    */
GC_API struct GC_hblk_s *GC_CALL GC_is_black_listed(struct GC_hblk_s *h,
                                                    size_t len)
{
    size_t index = PHT_HASH(h);
    size_t i, nblocks;

    if (!GC_all_interior_pointers
        && (get_pht_entry_from_index(GC_old_normal_bl, index)
            || get_pht_entry_from_index(GC_incomplete_normal_bl, index))) {
      return h + 1;
    }

    nblocks = divHBLKSZ(len);
    for (i = 0;;) {
        if (GC_old_stack_bl[divWORDSZ(index)] == 0
            && GC_incomplete_stack_bl[divWORDSZ(index)] == 0) {
          /* An easy case. */
          i += CPP_WORDSZ - modWORDSZ(index);
        } else {
          if (get_pht_entry_from_index(GC_old_stack_bl, index)
              || get_pht_entry_from_index(GC_incomplete_stack_bl, index)) {
            return &h[i + 1];
          }
          i++;
        }
        if (i >= nblocks) break;
        index = PHT_HASH(h + i);
    }
    return NULL;
}

/* Return the number of blacklisted blocks in a given range.    */
/* Used only for statistical purposes.                          */
/* Looks only at the GC_incomplete_stack_bl.                    */
STATIC word GC_number_stack_black_listed(struct hblk *start,
                                         struct hblk *endp1)
{
    struct hblk * h;
    word result = 0;

    for (h = start; ADDR_LT((ptr_t)h, (ptr_t)endp1); h++) {
        size_t index = PHT_HASH(h);

        if (get_pht_entry_from_index(GC_old_stack_bl, index)) result++;
    }
    return result;
}

/* Return the total number of (stack) black-listed bytes. */
static word total_stack_black_listed(void)
{
    size_t i;
    word total = 0;

    for (i = 0; i < GC_n_heap_sects; i++) {
        struct hblk * start = (struct hblk *) GC_heap_sects[i].hs_start;
        struct hblk * endp1 = start + divHBLKSZ(GC_heap_sects[i].hs_bytes);

        total += GC_number_stack_black_listed(start, endp1);
    }
    return total * HBLKSIZE;
}

/*
 * Copyright (c) 1992-1994 by Xerox Corporation.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#ifdef CHECKSUMS

/* This is debugging code intended to verify the results of dirty bit   */
/* computations.  Works only in a single threaded environment.          */
# define NSUMS 10000
# define OFFSET 0x10000

typedef struct {
        GC_bool new_valid;
        word old_sum;
        word new_sum;
        struct hblk * block;    /* Block to which this refers + OFFSET  */
                                /* to hide it from collector.           */
} page_entry;

page_entry GC_sums[NSUMS];

STATIC word GC_faulted[NSUMS] = { 0 };
                /* Record of pages on which we saw a write fault.       */

STATIC size_t GC_n_faulted = 0;

#ifdef MPROTECT_VDB
  void GC_record_fault(struct hblk * h)
  {
    GC_ASSERT(GC_page_size != 0);
    if (GC_n_faulted >= NSUMS) ABORT("write fault log overflowed");
    GC_faulted[GC_n_faulted++] = ADDR(HBLK_PAGE_ALIGNED(h));
  }
#endif

STATIC GC_bool GC_was_faulted(struct hblk *h)
{
    size_t i;
    word page = ADDR(HBLK_PAGE_ALIGNED(h));

    for (i = 0; i < GC_n_faulted; ++i) {
        if (GC_faulted[i] == page) return TRUE;
    }
    return FALSE;
}

STATIC word GC_checksum(struct hblk *h)
{
    word *p;
    word *lim = (word *)(h + 1);
    word result = 0;

    for (p = (word *)h; ADDR_LT((ptr_t)p, (ptr_t)lim); p++) {
        result += *p;
    }
    return result | SIGNB; /* does not look like pointer */
}

int GC_n_dirty_errors = 0;
int GC_n_faulted_dirty_errors = 0;
unsigned long GC_n_clean = 0;
unsigned long GC_n_dirty = 0;

STATIC void GC_update_check_page(struct hblk *h, int index)
{
    page_entry *pe = GC_sums + index;
    hdr * hhdr = HDR(h);

    if (pe -> block != 0 && pe -> block != h + OFFSET) ABORT("goofed");
    pe -> old_sum = pe -> new_sum;
    pe -> new_sum = GC_checksum(h);
#   if !defined(MSWIN32) && !defined(MSWINCE)
        if (pe -> new_sum != SIGNB && !GC_page_was_ever_dirty(h)) {
            GC_err_printf("GC_page_was_ever_dirty(%p) is wrong\n", (void *)h);
        }
#   endif
    if (GC_page_was_dirty(h)) {
        GC_n_dirty++;
    } else {
        GC_n_clean++;
    }
    if (hhdr != NULL) {
        (void)GC_find_starting_hblk(h, &hhdr);
        if (pe -> new_valid
#           ifdef SOFT_VDB
              && !HBLK_IS_FREE(hhdr)
#           endif
            && !IS_PTRFREE(hhdr) && pe -> old_sum != pe -> new_sum) {
            if (!GC_page_was_dirty(h) || !GC_page_was_ever_dirty(h)) {
                GC_bool was_faulted = GC_was_faulted(h);
                /* Set breakpoint here */GC_n_dirty_errors++;
                if (was_faulted) GC_n_faulted_dirty_errors++;
            }
        }
    }
    pe -> new_valid = TRUE;
    pe -> block = h + OFFSET;
}

/* Should be called immediately after GC_read_dirty.    */
void GC_check_dirty(void)
{
    int index;
    size_t i;

    GC_n_dirty_errors = 0;
    GC_n_faulted_dirty_errors = 0;
    GC_n_clean = 0;
    GC_n_dirty = 0;

    index = 0;
    for (i = 0; i < GC_n_heap_sects; i++) {
        ptr_t start = GC_heap_sects[i].hs_start;
        struct hblk *h;

        for (h = (struct hblk *)start;
             ADDR_LT((ptr_t)h, start + GC_heap_sects[i].hs_bytes); h++) {
            GC_update_check_page(h, index);
            index++;
            if (index >= NSUMS) {
                i = GC_n_heap_sects;
                break;
            }
        }
    }

    GC_COND_LOG_PRINTF("Checked %lu clean and %lu dirty pages\n",
                       GC_n_clean, GC_n_dirty);
    if (GC_n_dirty_errors > 0) {
        GC_err_printf("Found %d dirty bit errors (%d were faulted)\n",
                      GC_n_dirty_errors, GC_n_faulted_dirty_errors);
    }
    for (i = 0; i < GC_n_faulted; ++i) {
        GC_faulted[i] = 0; /* Don't expose block pointers to GC */
    }
    GC_n_faulted = 0;
}

#endif /* CHECKSUMS */

/*
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/*
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 2001 by Hewlett-Packard Company. All rights reserved.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/* Private declarations of GC marker data structures and macros */

/*
 * Declarations of mark stack.  Needed by marker and client supplied mark
 * routines.  Transitively include gc_priv.h.
 */
#ifndef GC_PMARK_H
#define GC_PMARK_H

#if defined(HAVE_CONFIG_H) && !defined(GC_PRIVATE_H)
  /* When gc_pmark.h is included from gc_priv.h, some of macros might   */
  /* be undefined in gcconfig.h, so skip config.h in this case.         */
#endif

#ifndef GC_BUILD
# define GC_BUILD
#endif

#if (defined(__linux__) || defined(__GLIBC__) || defined(__GNU__)) \
    && !defined(_GNU_SOURCE) && defined(GC_PTHREADS) \
    && !defined(GC_NO_PTHREAD_SIGMASK)
# define _GNU_SOURCE 1
#endif

#if defined(KEEP_BACK_PTRS) || defined(PRINT_BLACK_LIST)
#endif




EXTERN_C_BEGIN

/* The real declarations of the following is in gc_priv.h, so that      */
/* we can avoid scanning GC_mark_procs table.                           */

#ifndef MARK_DESCR_OFFSET
# define MARK_DESCR_OFFSET sizeof(ptr_t)
#endif

/* Mark descriptor stuff that should remain private for now, mostly     */
/* because it's hard to export CPP_WORDSZ without including gcconfig.h. */
#define BITMAP_BITS (CPP_WORDSZ - GC_DS_TAG_BITS)
#define PROC(descr) \
      (GC_mark_procs[((descr) >> GC_DS_TAG_BITS) & (GC_MAX_MARK_PROCS-1)])
#define ENV(descr) \
      ((descr) >> (GC_DS_TAG_BITS + GC_LOG_MAX_MARK_PROCS))
#define MAX_ENV (((word)1 << (BITMAP_BITS - GC_LOG_MAX_MARK_PROCS)) - 1)

GC_EXTERN unsigned GC_n_mark_procs;

/* Number of mark stack entries to discard on overflow. */
#define GC_MARK_STACK_DISCARDS (INITIAL_MARK_STACK_SIZE/8)

#ifdef PARALLEL_MARK
    /*
     * Allow multiple threads to participate in the marking process.
     * This works roughly as follows:
     *  The main mark stack never shrinks, but it can grow.
     *
     *  The initiating threads holds the allocator lock, sets GC_help_wanted.
     *
     *  Other threads:
     *     1) update helper_count (while holding the mark lock).
     *     2) allocate a local mark stack
     *     repeatedly:
     *          3) Steal a global mark stack entry by atomically replacing
     *             its descriptor with 0.
     *          4) Copy it to the local stack.
     *          5) Mark on the local stack until it is empty, or
     *             it may be profitable to copy it back.
     *          6) If necessary, copy local stack to global one,
     *             holding the mark lock.
     *    7) Stop when the global mark stack is empty.
     *    8) decrement helper_count (holding the mark lock).
     *
     * This is an experiment to see if we can do something along the lines
     * of the University of Tokyo SGC in a less intrusive, though probably
     * also less performant, way.
     */

    /* GC_mark_stack_top is protected by the mark lock. */

    /*
     * GC_notify_all_marker() is used when GC_help_wanted is first set,
     * when the last helper becomes inactive,
     * when something is added to the global mark stack, and just after
     * GC_mark_no is incremented.
     * This could be split into multiple CVs (and probably should be to
     * scale to really large numbers of processors.)
     */
#endif /* PARALLEL_MARK */

/* Push the object obj with corresponding heap block header hhdr onto   */
/* the mark stack.  Returns the updated mark_stack_top value.           */
GC_INLINE mse * GC_push_obj(ptr_t obj, const hdr * hhdr, mse * mark_stack_top,
                            mse * mark_stack_limit)
{
  GC_ASSERT(!HBLK_IS_FREE(hhdr));
  if (!IS_PTRFREE(hhdr)) {
    mark_stack_top = GC_custom_push_proc(hhdr -> hb_descr, obj,
                                         mark_stack_top, mark_stack_limit);
  }
  return mark_stack_top;
}

/* Push the contents of current onto the mark stack if it is a valid    */
/* ptr to a currently unmarked object.  Mark it.                        */
#define PUSH_CONTENTS(current, mark_stack_top, mark_stack_limit, source) \
  do { \
    hdr * my_hhdr; \
    HC_GET_HDR(current, my_hhdr, source); /* contains "break" */ \
    mark_stack_top = GC_push_contents_hdr(current, mark_stack_top, \
                                          mark_stack_limit, \
                                          source, my_hhdr, TRUE); \
  } while (0)

/* Set mark bit, exit (using "break" statement) if it is already set.   */
#ifdef USE_MARK_BYTES
# if defined(PARALLEL_MARK) && defined(AO_HAVE_char_store) \
     && !defined(BASE_ATOMIC_OPS_EMULATED)
    /* There is a race here, and we may set the bit twice in the        */
    /* concurrent case.  This can result in the object being pushed     */
    /* twice.  But that is only a performance issue.                    */
#   define SET_MARK_BIT_EXIT_IF_SET(hhdr, bit_no) \
      { /* cannot use do-while(0) here */ \
        volatile unsigned char *mark_byte_addr \
                        = (unsigned char *)((hhdr) -> hb_marks) + (bit_no); \
        /* Unordered atomic load and store are sufficient here. */ \
        if (AO_char_load(mark_byte_addr) != 0) \
          break; /* go to the enclosing loop end */ \
        AO_char_store(mark_byte_addr, 1); \
      }
# else
#   define SET_MARK_BIT_EXIT_IF_SET(hhdr, bit_no) \
      { /* cannot use do-while(0) here */ \
        ptr_t mark_byte_addr = (ptr_t)((hhdr) -> hb_marks) + (bit_no); \
        \
        if (*mark_byte_addr != 0) break; /* go to the enclosing loop end */ \
        *mark_byte_addr = 1; \
      }
# endif /* !PARALLEL_MARK */
#else
# if defined(PARALLEL_MARK) || (defined(THREAD_SANITIZER) && defined(THREADS))
#   ifdef THREAD_SANITIZER
#     define MARK_WORD_READ(addr) AO_load(addr)
#   else
#     define MARK_WORD_READ(addr) (*(addr))
#   endif
    /* This is used only if we explicitly set USE_MARK_BITS.            */
    /* The following may fail to exit even if the bit was already set.  */
    /* For our uses, that's benign:                                     */
#   define SET_MARK_BIT_EXIT_IF_SET(hhdr, bit_no) \
        { /* cannot use do-while(0) here */ \
          volatile AO_t *mark_word_addr \
                                = (hhdr) -> hb_marks + divWORDSZ(bit_no); \
          word my_bits = (word)1 << modWORDSZ(bit_no); \
          \
          if ((MARK_WORD_READ(mark_word_addr) & my_bits) != 0) \
            break; /* go to the enclosing loop end */ \
          AO_or(mark_word_addr, my_bits); \
        }
# else
#   define SET_MARK_BIT_EXIT_IF_SET(hhdr, bit_no) \
        { /* cannot use do-while(0) here */ \
          word *mark_word_addr = (hhdr) -> hb_marks + divWORDSZ(bit_no); \
          word old = *mark_word_addr; \
          word my_bits = (word)1 << modWORDSZ(bit_no); \
          \
          if ((old & my_bits) != 0) \
            break; /* go to the enclosing loop end */ \
          *(mark_word_addr) = old | my_bits; \
        }
# endif /* !PARALLEL_MARK */
#endif /* !USE_MARK_BYTES */

#ifdef ENABLE_TRACE
# define TRACE(source, cmd) \
        if (GC_trace_ptr != NULL && (ptr_t)(source) == GC_trace_ptr) cmd
# define TRACE_TARGET(target, cmd) \
        if (GC_trace_ptr != NULL && GC_is_heap_ptr(GC_trace_ptr) \
            && (target) == *(ptr_t *)GC_trace_ptr) cmd
#else
# define TRACE(source, cmd)
# define TRACE_TARGET(source, cmd)
#endif

/* If the mark bit corresponding to current is not set, set it, and     */
/* push the contents of the object on the mark stack.  Current points   */
/* to the beginning of the object.  We rely on the fact that the        */
/* preceding header calculation will succeed for a pointer past the     */
/* first page of an object, only if it is in fact a valid pointer       */
/* to the object.  Thus we can omit the otherwise necessary tests       */
/* here.  Note in particular that the "displ" value is the displacement */
/* from the beginning of the heap block, which may itself be in the     */
/* interior of a large object.                                          */
GC_INLINE mse * GC_push_contents_hdr(ptr_t current, mse * mark_stack_top,
                                     mse * mark_stack_limit, ptr_t source,
                                     hdr * hhdr, GC_bool do_offset_check)
{
  do {
    size_t displ = HBLKDISPL(current); /* Displacement in block; in bytes. */
    /* displ is always within range.  If current doesn't point to the   */
    /* first block, then we are in the all_interior_pointers case, and  */
    /* it is safe to use any displacement value.                        */
    ptr_t base = current;
#   ifdef MARK_BIT_PER_OBJ
      unsigned32 gran_displ; /* high_prod */
      unsigned32 inv_sz = hhdr -> hb_inv_sz;

#   else
      size_t gran_displ = BYTES_TO_GRANULES(displ);
      size_t gran_offset = hhdr -> hb_map[gran_displ];
      size_t byte_offset = displ & (GC_GRANULE_BYTES-1);

      /* The following always fails for large block references.         */
      if (EXPECT((gran_offset | byte_offset) != 0, FALSE))
#   endif
    {
#     ifdef MARK_BIT_PER_OBJ
        if (EXPECT(inv_sz == LARGE_INV_SZ, FALSE))
#     else
        if ((hhdr -> hb_flags & LARGE_BLOCK) != 0)
#     endif
      {
        /* gran_offset is bogus.        */
        size_t obj_displ;

        base = (ptr_t)(hhdr -> hb_block);
        obj_displ = (size_t)(current - base);
        if (obj_displ != displ) {
          GC_ASSERT(obj_displ < hhdr -> hb_sz);
          /* Must be in all_interior_pointer case, not first block      */
          /* already did validity check on cache miss.                  */
        } else if (do_offset_check && !GC_valid_offsets[obj_displ]) {
          GC_ADD_TO_BLACK_LIST_NORMAL(current, source);
          break;
        }
        GC_ASSERT(hhdr -> hb_sz > HBLKSIZE
                  || hhdr -> hb_block == HBLKPTR(current));
        GC_ASSERT(ADDR_GE(current, (ptr_t)(hhdr -> hb_block)));
        gran_displ = 0;
      } else {
#       ifndef MARK_BIT_PER_OBJ
          size_t obj_displ = GRANULES_TO_BYTES(gran_offset) + byte_offset;

#       else
          unsigned32 low_prod;

          LONG_MULT(gran_displ, low_prod, (unsigned32)displ, inv_sz);
          if ((low_prod >> 16) != 0)
#       endif
        {
#         ifdef MARK_BIT_PER_OBJ
            size_t obj_displ;

            /* Accurate enough if HBLKSIZE <= 2**15.    */
            GC_STATIC_ASSERT(HBLKSIZE <= (1 << 15));
            obj_displ = (((low_prod >> 16) + 1) * hhdr -> hb_sz) >> 16;
#         endif
          if (do_offset_check && !GC_valid_offsets[obj_displ]) {
            GC_ADD_TO_BLACK_LIST_NORMAL(current, source);
            break;
          }
#         ifndef MARK_BIT_PER_OBJ
            gran_displ -= gran_offset;
#         endif
          base -= obj_displ;
        }
      }
    }
#   ifdef MARK_BIT_PER_OBJ
      /* May get here for pointer to start of block not at the          */
      /* beginning of object.  If so, it is valid, and we are fine.     */
      GC_ASSERT(gran_displ <= HBLK_OBJS(hhdr -> hb_sz));
#   else
      GC_ASSERT(hhdr == GC_find_header(base));
      GC_ASSERT(gran_displ % BYTES_TO_GRANULES(hhdr -> hb_sz) == 0);
#   endif
    TRACE(source, GC_log_printf("GC #%lu: passed validity tests\n",
                                (unsigned long)GC_gc_no));
    SET_MARK_BIT_EXIT_IF_SET(hhdr, gran_displ); /* contains "break" */
    TRACE(source, GC_log_printf("GC #%lu: previously unmarked\n",
                                (unsigned long)GC_gc_no));
    TRACE_TARGET(base, GC_log_printf("GC #%lu: marking %p from %p instead\n",
                                     (unsigned long)GC_gc_no, (void *)base,
                                     (void *)source));
    INCR_MARKS(hhdr);
    GC_STORE_BACK_PTR(source, base);
    mark_stack_top = GC_push_obj(base, hhdr, mark_stack_top,
                                 mark_stack_limit);
  } while (0);
  return mark_stack_top;
}

#if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS)
# define PUSH_ONE_CHECKED_STACK(p, source) \
                        GC_mark_and_push_stack(p, (ptr_t)(source))
#else
# define PUSH_ONE_CHECKED_STACK(p, source) GC_mark_and_push_stack(p)
#endif

/* Push a single value onto mark stack. Mark from the object        */
/* pointed to by p.  The argument should be of word type.           */
/* Invoke FIXUP_POINTER() before any further processing.  p is      */
/* considered valid even if it is an interior pointer.  Previously  */
/* marked objects are not pushed.  Hence we make progress even      */
/* if the mark stack overflows.                                     */
#ifdef NEED_FIXUP_POINTER
    /* Try both the raw version and the fixed up one.   */
# define GC_PUSH_ONE_STACK(p, source) \
    do { \
      ptr_t pp = (p); \
      \
      if (ADDR_LT((ptr_t)GC_least_plausible_heap_addr, p) \
          && ADDR_LT(p, (ptr_t)GC_greatest_plausible_heap_addr)) { \
        PUSH_ONE_CHECKED_STACK(p, source); \
      } \
      FIXUP_POINTER(pp); \
      if (ADDR_LT((ptr_t)GC_least_plausible_heap_addr, pp) \
          && ADDR_LT(pp, (ptr_t)GC_greatest_plausible_heap_addr)) { \
        PUSH_ONE_CHECKED_STACK(pp, source); \
      } \
    } while (0)
#else /* !NEED_FIXUP_POINTER */
# define GC_PUSH_ONE_STACK(p, source) \
    do { \
      if (ADDR_LT((ptr_t)GC_least_plausible_heap_addr, p) \
          && ADDR_LT(p, (ptr_t)GC_greatest_plausible_heap_addr)) { \
        PUSH_ONE_CHECKED_STACK(p, source); \
      } \
    } while (0)
#endif

/* As above, but interior pointer recognition as for normal heap pointers. */
#define GC_PUSH_ONE_HEAP(p, source, mark_stack_top) \
    do { \
      FIXUP_POINTER(p); \
      if (ADDR_LT((ptr_t)GC_least_plausible_heap_addr, p) \
          && ADDR_LT(p, (ptr_t)GC_greatest_plausible_heap_addr)) \
        mark_stack_top = GC_mark_and_push(p, mark_stack_top, \
                                GC_mark_stack_limit, (void **)(source)); \
    } while (0)

/* Mark starting at mark stack entry top (incl.) down to        */
/* mark stack entry bottom (incl.).  Stop after performing      */
/* about one page worth of work.  Return the new mark stack     */
/* top entry.                                                   */
GC_INNER mse * GC_mark_from(mse * top, mse * bottom, mse *limit);

#define MARK_FROM_MARK_STACK() \
        GC_mark_stack_top = GC_mark_from(GC_mark_stack_top, \
                                         GC_mark_stack, \
                                         GC_mark_stack + GC_mark_stack_size);

#define GC_mark_stack_empty() \
                ADDR_LT((ptr_t)GC_mark_stack_top, (ptr_t)GC_mark_stack)

                                /* Current state of marking, as follows.*/

                                /* We say something is dirty if it was  */
                                /* written since the last time we       */
                                /* retrieved dirty bits.  We say it's   */
                                /* grungy if it was marked dirty in the */
                                /* last set of bits we retrieved.       */

                                /* Invariant "I": all roots and marked  */
                                /* objects p are either dirty, or point */
                                /* to objects q that are either marked  */
                                /* or a pointer to q appears in a range */
                                /* on the mark stack.                   */

#define MS_NONE 0               /* No marking in progress. "I" holds.   */
                                /* Mark stack is empty.                 */

#define MS_PUSH_RESCUERS 1      /* Rescuing objects are currently       */
                                /* being pushed.  "I" holds, except     */
                                /* that grungy roots may point to       */
                                /* unmarked objects, as may marked      */
                                /* grungy objects above GC_scan_ptr.    */

#define MS_PUSH_UNCOLLECTABLE 2 /* "I" holds, except that marked        */
                                /* uncollectible objects above          */
                                /* GC_scan_ptr may point to unmarked    */
                                /* objects.  Roots may point to         */
                                /* unmarked objects.                    */

#define MS_ROOTS_PUSHED 3       /* "I" holds, mark stack may be nonempty. */

#define MS_PARTIALLY_INVALID 4  /* "I" may not hold, e.g. because of    */
                                /* the mark stack overflow.  However,   */
                                /* marked heap objects below            */
                                /* GC_scan_ptr point to marked or       */
                                /* stacked objects.                     */

#define MS_INVALID 5            /* "I" may not hold.                    */

EXTERN_C_END

#endif  /* GC_PMARK_H */


#ifdef GC_GCJ_SUPPORT

/*
 * This is an allocator interface tuned for gcj (the GNU static
 * java compiler).
 *
 * Each allocated object has a pointer in its beginning to a vtable,
 * which for our purposes is simply a structure describing the type of
 * the object.  This descriptor structure contains a GC marking
 * descriptor at offset MARK_DESCR_OFFSET.
 *
 * It is hoped that this interface may also be useful for other systems,
 * possibly with some tuning of the constants.  But the immediate goal
 * is to get better gcj performance.
 *
 * We assume: counting on explicit initialization of this interface is OK.
 */

#include "gc/gc_gcj.h"

int GC_gcj_kind = 0;    /* Object kind for objects with descriptors     */
                        /* in "vtable".                                 */
int GC_gcj_debug_kind = 0;
                        /* The kind of objects that is always marked    */
                        /* with a mark proc call.                       */

STATIC struct GC_ms_entry *GC_CALLBACK GC_gcj_fake_mark_proc(word *addr,
                        struct GC_ms_entry *mark_stack_top,
                        struct GC_ms_entry *mark_stack_limit, word env)
{
    UNUSED_ARG(addr);
    UNUSED_ARG(mark_stack_limit);
    UNUSED_ARG(env);
#   if defined(FUNCPTR_IS_DATAPTR) && defined(CPPCHECK)
        GC_noop1((word)&GC_init_gcj_malloc);
#   endif
    ABORT_RET("No client gcj mark proc is specified");
    return mark_stack_top;
}

#ifdef FUNCPTR_IS_DATAPTR
  GC_API void GC_CALL GC_init_gcj_malloc(int mp_index, void *mp)
  {
    GC_init_gcj_malloc_mp((unsigned)mp_index,
                          CAST_THRU_UINTPTR(GC_mark_proc, mp));
  }
#endif /* FUNCPTR_IS_DATAPTR */

GC_API void GC_CALL GC_init_gcj_malloc_mp(unsigned mp_index, GC_mark_proc mp)
{
#   ifndef GC_IGNORE_GCJ_INFO
      GC_bool ignore_gcj_info;
#   endif

    if (mp == 0)        /* In case GC_DS_PROC is unused.        */
      mp = GC_gcj_fake_mark_proc;

    GC_init();  /* In case it's not already done.       */
    LOCK();
    if (GC_gcjobjfreelist != NULL) {
      /* Already initialized.   */
      UNLOCK();
      return;
    }
#   ifdef GC_IGNORE_GCJ_INFO
      /* This is useful for debugging on platforms with missing getenv(). */
#     define ignore_gcj_info TRUE
#   else
      ignore_gcj_info = (0 != GETENV("GC_IGNORE_GCJ_INFO"));
#   endif
    if (ignore_gcj_info) {
      GC_COND_LOG_PRINTF("Gcj-style type information is disabled!\n");
    }
    GC_ASSERT(GC_mark_procs[mp_index] == (GC_mark_proc)0); /* unused */
    GC_mark_procs[mp_index] = mp;
    if (mp_index >= GC_n_mark_procs)
        ABORT("GC_init_gcj_malloc_mp: bad index");
    /* Set up object kind gcj-style indirect descriptor. */
    GC_gcjobjfreelist = (ptr_t *)GC_new_free_list_inner();
    if (ignore_gcj_info) {
        /* Use a simple length-based descriptor, thus forcing a fully   */
        /* conservative scan.                                           */
        GC_gcj_kind = (int)GC_new_kind_inner((void **)GC_gcjobjfreelist,
                                             /* 0 | */ GC_DS_LENGTH,
                                             TRUE, TRUE);
        GC_gcj_debug_kind = GC_gcj_kind;
    } else {
        GC_gcj_kind = (int)GC_new_kind_inner(
                        (void **)GC_gcjobjfreelist,
                        (((word)(-(signed_word)MARK_DESCR_OFFSET
                                 - GC_INDIR_PER_OBJ_BIAS))
                         | GC_DS_PER_OBJECT),
                        FALSE, TRUE);
        /* Set up object kind for objects that require mark proc call.  */
        GC_gcj_debug_kind = (int)GC_new_kind_inner(GC_new_free_list_inner(),
                                GC_MAKE_PROC(mp_index,
                                        1 /* allocated with debug info */),
                                FALSE, TRUE);
    }
    UNLOCK();
#   undef ignore_gcj_info
}

/* A mechanism to release the allocator lock and invoke finalizers.     */
/* We don't really have an opportunity to do this on a rarely executed  */
/* path on which the allocator lock is not held.  Thus we check at      */
/* a rarely executed point at which it is safe to release the allocator */
/* lock; we do this even where we could just call GC_INVOKE_FINALIZERS, */
/* since it is probably cheaper and certainly more uniform.             */
/* TODO: Consider doing the same elsewhere? */
static void maybe_finalize(void)
{
   static word last_finalized_no = 0;

   GC_ASSERT(I_HOLD_LOCK());
   if (GC_gc_no == last_finalized_no ||
       !EXPECT(GC_is_initialized, TRUE)) return;
   UNLOCK();
   GC_INVOKE_FINALIZERS();
   LOCK();
   last_finalized_no = GC_gc_no;
}

/* Allocate an object, clear it, and store the pointer to the   */
/* type structure (vtable in gcj).  This adds a byte at the     */
/* end of the object if GC_malloc would.                        */
#ifdef THREAD_LOCAL_ALLOC
  GC_INNER
#else
  STATIC
#endif
void * GC_core_gcj_malloc(size_t lb, const void *vtable_ptr, unsigned flags)
{
    ptr_t op;
    size_t lg;

    GC_DBG_COLLECT_AT_MALLOC(lb);
    LOCK();
    if (SMALL_OBJ(lb) && (op = GC_gcjobjfreelist[lg = GC_size_map[lb]],
                          EXPECT(op != NULL, TRUE))) {
        GC_gcjobjfreelist[lg] = (ptr_t)obj_link(op);
        GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
        GC_ASSERT(NULL == ((void **)op)[1]);
    } else {
        maybe_finalize();
        op = (ptr_t)GC_generic_malloc_inner(lb, GC_gcj_kind, flags);
        if (NULL == op) {
            GC_oom_func oom_fn = GC_oom_fn;
            UNLOCK();
            return (*oom_fn)(lb);
        }
    }
    *(const void **)op = vtable_ptr;
    UNLOCK();
    GC_dirty(op);
    REACHABLE_AFTER_DIRTY(vtable_ptr);
    return GC_clear_stack(op);
}

#ifndef THREAD_LOCAL_ALLOC
  GC_API GC_ATTR_MALLOC void * GC_CALL GC_gcj_malloc(size_t lb,
                                                     const void *vtable_ptr)
  {
    return GC_core_gcj_malloc(lb, vtable_ptr, 0 /* flags */);
  }
#endif /* !THREAD_LOCAL_ALLOC */

GC_API GC_ATTR_MALLOC void * GC_CALL GC_gcj_malloc_ignore_off_page(size_t lb,
                                                    const void *vtable_ptr)
{
    return GC_core_gcj_malloc(lb, vtable_ptr, IGNORE_OFF_PAGE);
}

GC_API GC_ATTR_MALLOC void * GC_CALL GC_debug_gcj_malloc(size_t lb,
                                const void *vtable_ptr, GC_EXTRA_PARAMS)
{
    void *base, *result;

    /* We're careful to avoid extra calls, which could          */
    /* confuse the backtrace.                                   */
    LOCK();
    maybe_finalize();
    base = GC_generic_malloc_inner(SIZET_SAT_ADD(lb, DEBUG_BYTES),
                                   GC_gcj_debug_kind, 0 /* flags */);
    if (NULL == base) {
        GC_oom_func oom_fn = GC_oom_fn;
        UNLOCK();
        GC_err_printf("GC_debug_gcj_malloc(%lu, %p) returning NULL (%s:%d)\n",
                      (unsigned long)lb, vtable_ptr, s, i);
        return (*oom_fn)(lb);
    }
    *((const void **)((ptr_t)base + sizeof(oh))) = vtable_ptr;
    if (!GC_debugging_started) {
        GC_start_debugging_inner();
    }
    result = GC_store_debug_info_inner(base, lb, s, i);
    ADD_CALL_CHAIN(base, ra);
    UNLOCK();
    GC_dirty(result);
    REACHABLE_AFTER_DIRTY(vtable_ptr);
    return result;
}

#endif  /* GC_GCJ_SUPPORT */

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#if defined(KEEP_BACK_PTRS) && defined(GC_ASSERTIONS)
#endif
/*
 * This implements:
 * 1. allocation of heap block headers
 * 2. A map from addresses to heap block addresses to heap block headers
 *
 * Access speed is crucial.  We implement an index structure based on a 2
 * level tree.
 */

/* A non-macro version of header location routine.      */
GC_INNER hdr * GC_find_header(const void * h)
{
#   ifdef HASH_TL
        hdr * result;
        GET_HDR(h, result);
        return result;
#   else
        return HDR_INNER(h);
#   endif
}

/* Handle a header cache miss.  Returns a pointer to the        */
/* header corresponding to p, if p can possibly be a valid      */
/* object pointer, and 0 otherwise.                             */
/* GUARANTEED to return 0 for a pointer past the first page     */
/* of an object unless both GC_all_interior_pointers is set     */
/* and p is in fact a valid object pointer.                     */
/* Never returns a pointer to a free hblk.                      */
GC_INNER hdr *
#ifdef PRINT_BLACK_LIST
  GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce, ptr_t source)
#else
  GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce)
#endif
{
  hdr *hhdr;

  HC_MISS();
  GET_HDR(p, hhdr);
  if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
    if (GC_all_interior_pointers) {
      if (hhdr != NULL) {
        ptr_t current = (ptr_t)GC_find_starting_hblk(HBLKPTR(p), &hhdr);
                    /* current points to near the start of the large object */

        if (hhdr -> hb_flags & IGNORE_OFF_PAGE)
            return 0;
        if (HBLK_IS_FREE(hhdr)
                || p - current >= (signed_word)(hhdr -> hb_sz)) {
            GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
            /* Pointer past the end of the block */
            return 0;
        }
      } else {
        GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
        /* And return zero: */
      }
      GC_ASSERT(NULL == hhdr || !HBLK_IS_FREE(hhdr));
      return hhdr;
      /* Pointers past the first page are probably too rare     */
      /* to add them to the cache.  We don't.                   */
      /* And correctness relies on the fact that we don't.      */
    } else {
      if (NULL == hhdr) {
        GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
      }
      return 0;
    }
  } else {
    if (HBLK_IS_FREE(hhdr)) {
      GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
      return 0;
    } else {
      hce -> block_addr = ADDR(p) >> LOG_HBLKSIZE;
      hce -> hce_hdr = hhdr;
      return hhdr;
    }
  }
}

/* Routines to dynamically allocate collector data structures that will */
/* never be freed.                                                      */

GC_INNER ptr_t GC_scratch_alloc(size_t bytes)
{
    ptr_t result = GC_scratch_free_ptr;
    size_t bytes_to_get;

    GC_ASSERT(I_HOLD_LOCK());
    bytes = ROUNDUP_GRANULE_SIZE(bytes);
    for (;;) {
        GC_ASSERT(GC_scratch_end_addr >= ADDR(result));
        if (bytes <= GC_scratch_end_addr - ADDR(result)) {
            /* Unallocated space of scratch buffer has enough size. */
            GC_scratch_free_ptr = result + bytes;
            return result;
        }

        GC_ASSERT(GC_page_size != 0);
        if (bytes >= MINHINCR * HBLKSIZE) {
            bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(bytes);
            result = GC_os_get_mem(bytes_to_get);
            if (result != NULL) {
#             if defined(KEEP_BACK_PTRS) && (GC_GRANULE_BYTES < 0x10)
                GC_ASSERT(ADDR(result) > (word)NOT_MARKED);
#             endif
              /* No update of scratch free area pointer;        */
              /* get memory directly.                           */
#             ifdef USE_SCRATCH_LAST_END_PTR
                /* Update end point of last obtained area (needed only  */
                /* by GC_register_dynamic_libraries for some targets).  */
                GC_scratch_last_end_addr = ADDR(result) + bytes;
#             endif
            }
            return result;
        }

        bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(MINHINCR * HBLKSIZE);
                                                /* round up for safety */
        result = GC_os_get_mem(bytes_to_get);
        if (EXPECT(NULL == result, FALSE)) {
            WARN("Out of memory - trying to allocate requested amount"
                 " (%" WARN_PRIuPTR " bytes)...\n", bytes);
            bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(bytes);
            result = GC_os_get_mem(bytes_to_get);
            if (result != NULL) {
#             ifdef USE_SCRATCH_LAST_END_PTR
                GC_scratch_last_end_addr = ADDR(result) + bytes;
#             endif
            }
            return result;
        }

        /* TODO: some amount of unallocated space may remain unused forever */
        /* Update scratch area pointers and retry.      */
        GC_scratch_free_ptr = result;
        GC_scratch_end_addr = ADDR(GC_scratch_free_ptr) + bytes_to_get;
#       ifdef USE_SCRATCH_LAST_END_PTR
          GC_scratch_last_end_addr = GC_scratch_end_addr;
#       endif
    }
}

/* Return an uninitialized header */
static hdr * alloc_hdr(void)
{
    hdr * result;

    GC_ASSERT(I_HOLD_LOCK());
    if (NULL == GC_hdr_free_list) {
        result = (hdr *)GC_scratch_alloc(sizeof(hdr));
    } else {
        result = GC_hdr_free_list;
        GC_hdr_free_list = (hdr *)(result -> hb_next);
    }
    return result;
}

GC_INLINE void free_hdr(hdr * hhdr)
{
    hhdr -> hb_next = (struct hblk *)GC_hdr_free_list;
    GC_hdr_free_list = hhdr;
}

#ifdef COUNT_HDR_CACHE_HITS
  /* Used for debugging/profiling (the symbols are externally visible). */
  word GC_hdr_cache_hits = 0;
  word GC_hdr_cache_misses = 0;
#endif

GC_INNER void GC_init_headers(void)
{
    unsigned i;

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(NULL == GC_all_nils);
    GC_all_nils = (bottom_index *)GC_scratch_alloc(sizeof(bottom_index));
    if (GC_all_nils == NULL) {
      GC_err_printf("Insufficient memory for GC_all_nils\n");
      EXIT();
    }
    BZERO(GC_all_nils, sizeof(bottom_index));
    for (i = 0; i < TOP_SZ; i++) {
        GC_top_index[i] = GC_all_nils;
    }
}

/* Make sure that there is a bottom level index block for address addr. */
/* Return FALSE on failure.                                             */
static GC_bool get_index(word addr)
{
    word hi = addr >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE);
    bottom_index * r;
    bottom_index * p;
    bottom_index ** prev;
    bottom_index *pi; /* old_p */
    word i;

    GC_ASSERT(I_HOLD_LOCK());
#   ifdef HASH_TL
      i = TL_HASH(hi);

      pi = GC_top_index[i];
      for (p = pi; p != GC_all_nils; p = p -> hash_link) {
          if (p -> key == hi) return TRUE;
      }
#   else
      if (GC_top_index[hi] != GC_all_nils)
        return TRUE;
      i = hi;
#   endif
    r = (bottom_index *)GC_scratch_alloc(sizeof(bottom_index));
    if (EXPECT(NULL == r, FALSE))
      return FALSE;
    BZERO(r, sizeof(bottom_index));
    r -> key = hi;
#   ifdef HASH_TL
      r -> hash_link = pi;
#   endif

    /* Add it to the list of bottom indices */
      prev = &GC_all_bottom_indices;    /* pointer to p */
      pi = 0;                           /* bottom_index preceding p */
      while ((p = *prev) != 0 && p -> key < hi) {
        pi = p;
        prev = &(p -> asc_link);
      }
      r -> desc_link = pi;
      if (0 == p) {
        GC_all_bottom_indices_end = r;
      } else {
        p -> desc_link = r;
      }
      r -> asc_link = p;
      *prev = r;

      GC_top_index[i] = r;
    return TRUE;
}

GC_INNER hdr * GC_install_header(struct hblk *h)
{
    hdr * result;

    GC_ASSERT(I_HOLD_LOCK());
    if (EXPECT(!get_index(ADDR(h)), FALSE)) return NULL;

    result = alloc_hdr();
    if (EXPECT(result != NULL, TRUE)) {
      GC_ASSERT(!IS_FORWARDING_ADDR_OR_NIL(result));
      SET_HDR(h, result);
#     ifdef USE_MUNMAP
        result -> hb_last_reclaimed = (unsigned short)GC_gc_no;
#     endif
    }
    return result;
}

GC_INNER GC_bool GC_install_counts(struct hblk *h, size_t sz /* bytes */)
{
    struct hblk * hbp;

    for (hbp = h; ADDR_LT((ptr_t)hbp, (ptr_t)h + sz); hbp += BOTTOM_SZ) {
        if (!get_index(ADDR(hbp)))
            return FALSE;
        if (ADDR(hbp) > GC_WORD_MAX - (word)BOTTOM_SZ * HBLKSIZE)
            break; /* overflow of hbp+=BOTTOM_SZ is expected */
    }
    if (!get_index(ADDR(h) + sz - 1))
        return FALSE;
    GC_ASSERT(!IS_FORWARDING_ADDR_OR_NIL(HDR(h)));
    for (hbp = h + 1; ADDR_LT((ptr_t)hbp, (ptr_t)h + sz); hbp++) {
        word i = (word)HBLK_PTR_DIFF(hbp, h);

        SET_HDR(hbp, (hdr *)(i > MAX_JUMP ? MAX_JUMP : i));
    }
    return TRUE;
}

GC_INNER void GC_remove_header(struct hblk *h)
{
    hdr **ha;
    GET_HDR_ADDR(h, ha);
    free_hdr(*ha);
    *ha = 0;
}

GC_INNER void GC_remove_counts(struct hblk *h, size_t sz /* bytes */)
{
    struct hblk * hbp;

    if (sz <= HBLKSIZE) return;
    if (NULL == HDR(h + 1)) {
#     ifdef GC_ASSERTIONS
        for (hbp = h + 2; ADDR_LT((ptr_t)hbp, (ptr_t)h + sz); hbp++) {
          GC_ASSERT(NULL == HDR(hbp));
        }
#     endif
      return;
    }

    for (hbp = h + 1; ADDR_LT((ptr_t)hbp, (ptr_t)h + sz); hbp++) {
      SET_HDR(hbp, NULL);
    }
}

GC_API void GC_CALL GC_apply_to_all_blocks(GC_walk_hblk_fn fn,
                                           void *client_data)
{
    signed_word j;
    bottom_index * index_p;

    for (index_p = GC_all_bottom_indices; index_p != NULL;
         index_p = index_p -> asc_link) {
        for (j = BOTTOM_SZ-1; j >= 0;) {
            if (!IS_FORWARDING_ADDR_OR_NIL(index_p -> index[j])) {
                if (!HBLK_IS_FREE(index_p -> index[j])) {
                    fn((struct hblk *)
                              (((index_p -> key << LOG_BOTTOM_SZ) + (word)j)
                               << LOG_HBLKSIZE),
                       client_data);
                }
                j--;
            } else if (NULL == index_p -> index[j]) {
                j--;
            } else {
                j -= (signed_word)(index_p -> index[j]);
            }
        }
    }
}

GC_INNER struct hblk * GC_next_block(struct hblk *h, GC_bool allow_free)
{
    REGISTER bottom_index * bi;
    REGISTER size_t j = (size_t)(ADDR(h) >> LOG_HBLKSIZE) & (BOTTOM_SZ-1);

    GC_ASSERT(I_HOLD_READER_LOCK());
    GET_BI(h, bi);
    if (bi == GC_all_nils) {
        REGISTER word hi = ADDR(h) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE);

        bi = GC_all_bottom_indices;
        while (bi != 0 && bi -> key < hi) bi = bi -> asc_link;
        j = 0;
    }

    while (bi != 0) {
        while (j < BOTTOM_SZ) {
            hdr * hhdr = bi -> index[j];

            if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
                j++;
            } else {
                if (allow_free || !HBLK_IS_FREE(hhdr)) {
                    return (struct hblk *)(((bi -> key << LOG_BOTTOM_SZ)
                                            + j) << LOG_HBLKSIZE);
                } else {
                    j += divHBLKSZ(hhdr -> hb_sz);
                }
            }
        }
        j = 0;
        bi = bi -> asc_link;
    }
    return NULL;
}

GC_INNER struct hblk * GC_prev_block(struct hblk *h)
{
    bottom_index * bi;
    signed_word j = (ADDR(h) >> LOG_HBLKSIZE) & (BOTTOM_SZ-1);

    GC_ASSERT(I_HOLD_READER_LOCK());
    GET_BI(h, bi);
    if (bi == GC_all_nils) {
        word hi = ADDR(h) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE);

        bi = GC_all_bottom_indices_end;
        while (bi != NULL && bi -> key > hi)
            bi = bi -> desc_link;
        j = BOTTOM_SZ - 1;
    }
    for (; bi != NULL; bi = bi -> desc_link) {
        while (j >= 0) {
            hdr * hhdr = bi -> index[j];

            if (NULL == hhdr) {
                --j;
            } else if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
                j -= (signed_word)ADDR(hhdr);
            } else {
                return (struct hblk*)(((bi -> key << LOG_BOTTOM_SZ) + (word)j)
                                       << LOG_HBLKSIZE);
            }
        }
        j = BOTTOM_SZ - 1;
    }
    return NULL;
}

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 2000 by Hewlett-Packard Company.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


/*
 * This file contains the functions:
 *      ptr_t GC_build_flXXX(h, old_fl)
 *      void GC_new_hblk(size, kind)
 */

#ifndef SMALL_CONFIG
  /* Build a free list for two-pointer cleared objects inside the given */
  /* block.  Set the last link to be ofl.  Return a pointer to the      */
  /* first free-list entry.                                             */
  STATIC ptr_t GC_build_fl_clear2(struct hblk *h, ptr_t ofl)
  {
    ptr_t *p = (ptr_t *)(h -> hb_body);
    ptr_t plim = (ptr_t)(h + 1);

    p[0] = ofl;
    p[1] = NULL;
    p[2] = (ptr_t)p;
    p[3] = NULL;
    for (p += 4; ADDR_LT((ptr_t)p, plim); p += 4) {
      p[0] = (ptr_t)(p - 2);
      p[1] = NULL;
      p[2] = (ptr_t)p;
      p[3] = NULL;
    }
    return (ptr_t)(p - 2);
  }

  /* The same as above but uncleared objects.   */
  STATIC ptr_t GC_build_fl2(struct hblk *h, ptr_t ofl)
  {
    ptr_t *p = (ptr_t *)(h -> hb_body);
    ptr_t plim = (ptr_t)(h + 1);

    p[0] = ofl;
    p[2] = (ptr_t)p;
    for (p += 4; ADDR_LT((ptr_t)p, plim); p += 4) {
      p[0] = (ptr_t)(p - 2);
      p[2] = (ptr_t)p;
    }
    return (ptr_t)(p - 2);
  }

  /* The same as above but for four-pointer cleared objects.        */
  STATIC ptr_t GC_build_fl_clear4(struct hblk *h, ptr_t ofl)
  {
    ptr_t *p = (ptr_t *)(h -> hb_body);
    ptr_t plim = (ptr_t)(h + 1);

    p[0] = ofl;
    p[1] = NULL;
    p[2] = NULL;
    p[3] = NULL;
    for (p += 4; ADDR_LT((ptr_t)p, plim); p += 4) {
      GC_PREFETCH_FOR_WRITE((ptr_t)(p + 64));
      p[0] = (ptr_t)(p - 4);
      p[1] = NULL;
      CLEAR_DOUBLE(p + 2);
    }
    return (ptr_t)(p - 4);
  }

  /* The same as GC_build_fl_clear4() but uncleared objects.            */
  STATIC ptr_t GC_build_fl4(struct hblk *h, ptr_t ofl)
  {
    ptr_t *p = (ptr_t *)(h -> hb_body);
    ptr_t plim = (ptr_t)(h + 1);

    p[0] = ofl;
    p[4] = (ptr_t)p;
    for (p += 8; ADDR_LT((ptr_t)p, plim); p += 8) { /* unroll by 2 */
      GC_PREFETCH_FOR_WRITE((ptr_t)(p + 64));
      p[0] = (ptr_t)(p - 4);
      p[4] = (ptr_t)p;
    }
    return (ptr_t)(p - 4);
  }
#endif /* !SMALL_CONFIG */

GC_INNER ptr_t GC_build_fl(struct hblk *h, ptr_t list, size_t lg,
                           GC_bool clear)
{
  ptr_t *p, *prev;
  ptr_t plim; /* points to last object in new hblk */
  size_t lpw = GRANULES_TO_PTRS(lg);

  /* Do a few prefetches here, just because it's cheap.         */
  /* If we were more serious about it, these should go inside   */
  /* the loops.  But write prefetches usually don't seem to     */
  /* matter much.                                               */
  GC_PREFETCH_FOR_WRITE((ptr_t)h);
  GC_PREFETCH_FOR_WRITE((ptr_t)h + 128);
  GC_PREFETCH_FOR_WRITE((ptr_t)h + 256);
  GC_PREFETCH_FOR_WRITE((ptr_t)h + 378);
# ifndef SMALL_CONFIG
    /* Handle small objects sizes more efficiently.  For larger objects */
    /* the difference is less significant.                              */
    switch (lpw) {
    case 2:
      if (clear) {
        return GC_build_fl_clear2(h, list);
      } else {
        return GC_build_fl2(h, list);
      }
    case 4:
      if (clear) {
        return GC_build_fl_clear4(h, list);
      } else {
        return GC_build_fl4(h, list);
      }
    default:
      break;
    }
# endif /* !SMALL_CONFIG */

  /* Clear the page if necessary. */
  if (clear) BZERO(h, HBLKSIZE);

  /* Add objects to free list. */
  prev = (ptr_t *)(h -> hb_body); /* one object behind p */
  plim = (ptr_t)h + HBLKSIZE - lpw * sizeof(ptr_t);
                                /* last place for last object to start */

  /* Make a list of all objects in *h with head as last object. */
  for (p = prev + lpw; ADDR_GE(plim, (ptr_t)p); p += lpw) {
    /* current object's link points to last object */
    obj_link(p) = (ptr_t)prev;
    prev = p;
  }
  p -= lpw;   /* p now points to last object */

  /* Put p (which is now head of list of objects in *h) as first    */
  /* pointer in the appropriate free list for this size.            */
  *(ptr_t *)h = list;
  return (ptr_t)p;
}

GC_INNER void GC_new_hblk(size_t lg, int k)
{
  struct hblk *h;       /* the new heap block */
  size_t lb_adjusted = GRANULES_TO_BYTES(lg);

  GC_STATIC_ASSERT(sizeof(struct hblk) == HBLKSIZE);
  GC_ASSERT(I_HOLD_LOCK());
  /* Allocate a new heap block. */
  h = GC_allochblk(lb_adjusted, k, 0 /* flags */, 0 /* align_m1 */);
  if (EXPECT(NULL == h, FALSE)) return; /* out of memory */

  /* Mark all objects if appropriate. */
  if (IS_UNCOLLECTABLE(k)) GC_set_hdr_marks(HDR(h));

  /* Build the free list.       */
  GC_obj_kinds[k].ok_freelist[lg] =
        GC_build_fl(h, (ptr_t)GC_obj_kinds[k].ok_freelist[lg], lg,
                    GC_debugging_started || GC_obj_kinds[k].ok_init);
}

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991, 1992 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1999-2001 by Hewlett-Packard Company. All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


/* Routines for maintaining maps describing heap block
 * layouts for various object sizes.  Allows fast pointer validity checks
 * and fast location of object start locations on machines (such as SPARC)
 * with slow division.
 */

/* Consider pointers that are offset bytes displaced from the beginning */
/* of an object to be valid.                                            */

GC_API void GC_CALL GC_register_displacement(size_t offset)
{
    LOCK();
    GC_register_displacement_inner(offset);
    UNLOCK();
}

GC_INNER void GC_register_displacement_inner(size_t offset)
{
    GC_ASSERT(I_HOLD_LOCK());
    if (offset >= VALID_OFFSET_SZ) {
        ABORT("Bad argument to GC_register_displacement");
    }
    if (!GC_valid_offsets[offset]) {
      GC_valid_offsets[offset] = TRUE;
      GC_modws_valid_offsets[offset % sizeof(ptr_t)] = TRUE;
    }
}

#ifndef MARK_BIT_PER_OBJ
  GC_INNER GC_bool GC_add_map_entry(size_t lg)
  {
    size_t displ;
    unsigned short * new_map;

    GC_ASSERT(I_HOLD_LOCK());
    if (lg > BYTES_TO_GRANULES(MAXOBJBYTES)) lg = 0;
    if (GC_obj_map[lg] != 0) return TRUE;

    new_map = (unsigned short *)GC_scratch_alloc(OBJ_MAP_LEN * sizeof(short));
    if (EXPECT(NULL == new_map, FALSE)) return FALSE;

    GC_COND_LOG_PRINTF(
                "Adding block map for size of %u granules (%u bytes)\n",
                (unsigned)lg, (unsigned)GRANULES_TO_BYTES(lg));
    if (0 == lg) {
      for (displ = 0; displ < OBJ_MAP_LEN; displ++) {
        new_map[displ] = 1;  /* Nonzero to get us out of marker fast path. */
      }
    } else {
      for (displ = 0; displ < OBJ_MAP_LEN; displ++) {
        new_map[displ] = (unsigned short)(displ % lg);
      }
    }
    GC_obj_map[lg] = new_map;
    return TRUE;
  }
#endif /* !MARK_BIT_PER_OBJ */

GC_INNER void GC_initialize_offsets(void)
{
  size_t i;

  if (GC_all_interior_pointers) {
    for (i = 0; i < VALID_OFFSET_SZ; ++i)
      GC_valid_offsets[i] = TRUE;
  } else {
    BZERO(GC_valid_offsets, sizeof(GC_valid_offsets));
    for (i = 0; i < sizeof(ptr_t); ++i)
      GC_modws_valid_offsets[i] = FALSE;
  }
}

/*
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


/*
 * These are checking routines calls to which could be inserted by a
 * preprocessor to validate C pointer arithmetic.
 */

STATIC void GC_CALLBACK GC_default_same_obj_print_proc(void *p, void *q)
{
    ABORT_ARG2("GC_same_obj test failed",
               ": %p and %p are not in the same object", p, q);
}

GC_same_obj_print_proc_t GC_same_obj_print_proc =
                GC_default_same_obj_print_proc;

GC_API void * GC_CALL GC_same_obj(void *p, void *q)
{
    hdr *hhdr;
    ptr_t base, limit;
    size_t sz;

    if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
    hhdr = HDR(p);
    if (NULL == hhdr) {
        if (divHBLKSZ(ADDR(p)) != divHBLKSZ(ADDR(q)) && HDR(q) != NULL) {
          GC_same_obj_print_proc((ptr_t)p, (ptr_t)q);
        }
        return p;
    }
    /* If it's a pointer to the middle of a large object, move it       */
    /* to the beginning.                                                */
    if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
        struct hblk *h = GC_find_starting_hblk(HBLKPTR(p), &hhdr);

        limit = (ptr_t)h + hhdr -> hb_sz;
        if (ADDR_GE((ptr_t)p, limit) || ADDR_GE((ptr_t)q, limit)
            || ADDR_LT((ptr_t)q, (ptr_t)h)) {
          GC_same_obj_print_proc((ptr_t)p, (ptr_t)q);
        }
        return p;
    }
    sz = hhdr -> hb_sz;
    if (sz > MAXOBJBYTES) {
      base = (ptr_t)HBLKPTR(p);
      limit = base + sz;
      if (ADDR_GE((ptr_t)p, limit)) {
        GC_same_obj_print_proc((ptr_t)p, (ptr_t)q);
        return p;
      }
    } else {
      size_t offset;

      if (HBLKPTR(p) != HBLKPTR(q)) {
                /* W/o this check, we might miss an error if    */
                /* q points to the first object on a page, and  */
                /* points just before the page.                 */
        GC_same_obj_print_proc((ptr_t)p, (ptr_t)q);
        return p;
      }
      offset = HBLKDISPL(p) % sz;
      base = (ptr_t)p - offset;
      limit = base + sz;
    }
    /* [base, limit) delimits the object containing p, if any.  */
    /* If p is not inside a valid object, then either q is      */
    /* also outside any valid object, or it is outside          */
    /* [base, limit).                                           */
    if (!ADDR_INSIDE((ptr_t)q, base, limit)) {
      GC_same_obj_print_proc((ptr_t)p, (ptr_t)q);
    }
    return p;
}

STATIC void GC_CALLBACK GC_default_is_valid_displacement_print_proc(void *p)
{
    ABORT_ARG1("GC_is_valid_displacement test failed", ": %p not valid", p);
}

GC_valid_ptr_print_proc_t GC_is_valid_displacement_print_proc =
                GC_default_is_valid_displacement_print_proc;

GC_API void * GC_CALL GC_is_valid_displacement(void *p)
{
    hdr *hhdr;
    size_t offset;
    struct hblk *h;
    size_t sz;

    if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
    if (NULL == p) return NULL;
    hhdr = HDR(p);
    if (NULL == hhdr) return p;
    h = HBLKPTR(p);
    if (GC_all_interior_pointers) {
        h = GC_find_starting_hblk(h, &hhdr);
    } else if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
        GC_is_valid_displacement_print_proc((ptr_t)p);
        return p;
    }
    sz = hhdr -> hb_sz;
    offset = HBLKDISPL(p) % sz;
    if ((sz > MAXOBJBYTES && ADDR_GE((ptr_t)p, (ptr_t)h + sz))
        || !GC_valid_offsets[offset]
        || (ADDR_LT((ptr_t)(h + 1), (ptr_t)p + sz - offset)
            && !IS_FORWARDING_ADDR_OR_NIL(HDR(h + 1)))) {
        GC_is_valid_displacement_print_proc((ptr_t)p);
    }
    return p;
}

STATIC void GC_CALLBACK GC_default_is_visible_print_proc(void *p)
{
    ABORT_ARG1("GC_is_visible test failed", ": %p not GC-visible", p);
}

GC_valid_ptr_print_proc_t GC_is_visible_print_proc =
                GC_default_is_visible_print_proc;

#ifndef THREADS
/* Could p be a stack address? */
  STATIC GC_bool GC_on_stack(ptr_t p)
  {
    return HOTTER_THAN(p, GC_stackbottom) && !HOTTER_THAN(p, GC_approx_sp());
  }
#endif /* !THREADS */

GC_API void * GC_CALL GC_is_visible(void *p)
{
    const hdr *hhdr;

    if ((ADDR(p) & (ALIGNMENT-1)) != 0) goto fail;
    if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
#   ifdef THREADS
        hhdr = HDR(p);
        if (hhdr != NULL && NULL == GC_base(p)) {
            goto fail;
        } else {
            /* May be inside thread stack.  We can't do much. */
            return p;
        }
#   else
        /* Check stack first: */
        if (GC_on_stack((ptr_t)p)) return p;

        hhdr = HDR(p);
        if (NULL == hhdr) {
            if (GC_is_static_root((ptr_t)p)) return p;
            /* Else do it again correctly:      */
#           if defined(DYNAMIC_LOADING) || defined(ANY_MSWIN) || defined(PCR)
              if (!GC_no_dls) {
                GC_register_dynamic_libraries();
                if (GC_is_static_root((ptr_t)p)) return p;
              }
#           endif
            goto fail;
        } else {
            /* p points to the heap. */
            word descr;
            ptr_t base = (ptr_t)GC_base(p);
                        /* TODO: should GC_base be manually inlined? */

            if (NULL == base) goto fail;
            if (HBLKPTR(base) != HBLKPTR(p))
                hhdr = HDR(base);
            descr = hhdr -> hb_descr;
        retry:
            switch (descr & GC_DS_TAGS) {
                case GC_DS_LENGTH:
                    if ((word)((ptr_t)p - base) >= descr) goto fail;
                    break;
                case GC_DS_BITMAP:
                    if ((ptr_t)p - base
                            >= (ptrdiff_t)PTRS_TO_BYTES(BITMAP_BITS)
                        || (ADDR(p) & (sizeof(ptr_t)-1)) != 0) goto fail;
                    if (!(((word)1 << (CPP_WORDSZ-1 - (word)((ptr_t)p - base)))
                          & descr)) goto fail;
                    break;
                case GC_DS_PROC:
                    /* We could try to decipher this partially.         */
                    /* For now we just punt.                            */
                    break;
                case GC_DS_PER_OBJECT:
                    if (!(descr & SIGNB)) {
                      descr = *(word *)((ptr_t)base
                                        + (descr & ~(word)GC_DS_TAGS));
                    } else {
                      ptr_t type_descr = *(ptr_t *)base;

                      if (EXPECT(NULL == type_descr, FALSE))
                        goto fail; /* see comment in GC_mark_from */
                      descr = *(word *)(type_descr
                                - ((signed_word)descr + (GC_INDIR_PER_OBJ_BIAS
                                                         - GC_DS_PER_OBJECT)));
                    }
                    goto retry;
            }
            return p;
        }
#   endif
fail:
    GC_is_visible_print_proc((ptr_t)p);
    return p;
}

GC_API void * GC_CALL GC_pre_incr(void **p, ptrdiff_t how_much)
{
    void *initial = *p;
    void *result = GC_same_obj((ptr_t)initial + how_much, initial);

    if (!GC_all_interior_pointers) {
        (void)GC_is_valid_displacement(result);
    }
    *p = result;
    return result; /* updated pointer */
}

GC_API void * GC_CALL GC_post_incr(void **p, ptrdiff_t how_much)
{
    void *initial = *p;
    void *result = GC_same_obj((ptr_t)initial + how_much, initial);

    if (!GC_all_interior_pointers) {
        (void)GC_is_valid_displacement(result);
    }
    *p = result;
    return initial; /* original *p */
}

GC_API void GC_CALL GC_set_same_obj_print_proc(GC_same_obj_print_proc_t fn)
{
    GC_ASSERT(NONNULL_ARG_NOT_NULL(fn));
    GC_same_obj_print_proc = fn;
}

GC_API GC_same_obj_print_proc_t GC_CALL GC_get_same_obj_print_proc(void)
{
    return GC_same_obj_print_proc;
}

GC_API void GC_CALL GC_set_is_valid_displacement_print_proc(
                                        GC_valid_ptr_print_proc_t fn)
{
    GC_ASSERT(NONNULL_ARG_NOT_NULL(fn));
    GC_is_valid_displacement_print_proc = fn;
}

GC_API GC_valid_ptr_print_proc_t GC_CALL
GC_get_is_valid_displacement_print_proc(void)
{
    return GC_is_valid_displacement_print_proc;
}

GC_API void GC_CALL GC_set_is_visible_print_proc(GC_valid_ptr_print_proc_t fn)
{
    GC_ASSERT(NONNULL_ARG_NOT_NULL(fn));
    GC_is_visible_print_proc = fn;
}

GC_API GC_valid_ptr_print_proc_t GC_CALL GC_get_is_visible_print_proc(void)
{
    return GC_is_visible_print_proc;
}


/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1998-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#ifdef GC_USE_ENTIRE_HEAP
  int GC_use_entire_heap = TRUE;
#else
  int GC_use_entire_heap = FALSE;
#endif

/*
 * Free heap blocks are kept on one of several free lists,
 * depending on the size of the block.  Each free list is doubly linked.
 * Adjacent free blocks are coalesced.
 */

#define MAX_BLACK_LIST_ALLOC (2*HBLKSIZE)
                /* largest block we will allocate starting on a black   */
                /* listed block.  Must be >= HBLKSIZE.                  */


#define UNIQUE_THRESHOLD 32
        /* Sizes up to this many HBLKs each have their own free list    */
#define HUGE_THRESHOLD 256
        /* Sizes of at least this many heap blocks are mapped to a      */
        /* single free list.                                            */
#define FL_COMPRESSION 8
        /* In between sizes map this many distinct sizes to a single    */
        /* bin.                                                         */

#define N_HBLK_FLS ((HUGE_THRESHOLD - UNIQUE_THRESHOLD) / FL_COMPRESSION \
                     + UNIQUE_THRESHOLD)

#ifndef GC_GCJ_SUPPORT
  STATIC
#endif
  struct hblk * GC_hblkfreelist[N_HBLK_FLS+1] = { 0 };
                                /* List of completely empty heap blocks */
                                /* Linked through hb_next field of      */
                                /* header structure associated with     */
                                /* block.  Remains externally visible   */
                                /* as used by GNU GCJ currently.        */

GC_API void GC_CALL GC_iterate_free_hblks(GC_walk_free_blk_fn fn,
                                          void *client_data)
{
  int i;

  for (i = 0; i <= N_HBLK_FLS; ++i) {
    struct hblk *h;

    for (h = GC_hblkfreelist[i]; h != NULL; h = HDR(h) -> hb_next) {
      fn(h, i, client_data);
    }
  }
}

#ifndef GC_GCJ_SUPPORT
  STATIC
#endif
  word GC_free_bytes[N_HBLK_FLS+1] = { 0 };
        /* Number of free bytes on each list.  Remains visible to GCJ.  */

/* Return the largest n such that the number of free bytes on lists     */
/* n .. N_HBLK_FLS is greater or equal to GC_max_large_allocd_bytes     */
/* minus GC_large_allocd_bytes.  If there is no such n, return 0.       */
GC_INLINE size_t GC_enough_large_bytes_left(void)
{
    size_t n;
    word bytes = GC_large_allocd_bytes;

    GC_ASSERT(GC_max_large_allocd_bytes <= GC_heapsize);
    for (n = N_HBLK_FLS + 1; n > 0;) {
        n--;
        bytes += GC_free_bytes[n];
        if (bytes >= GC_max_large_allocd_bytes) break;
    }
    return n;
}

/* Map a number of blocks to the appropriate large block free-list index. */
STATIC size_t GC_hblk_fl_from_blocks(size_t blocks_needed)
{
    if (blocks_needed <= UNIQUE_THRESHOLD) return blocks_needed;
    if (blocks_needed >= HUGE_THRESHOLD) return N_HBLK_FLS;
    return (blocks_needed - UNIQUE_THRESHOLD) / FL_COMPRESSION
           + UNIQUE_THRESHOLD;
}

#define PHDR(hhdr) HDR((hhdr) -> hb_prev)
#define NHDR(hhdr) HDR((hhdr) -> hb_next)

#ifdef USE_MUNMAP
#   define IS_MAPPED(hhdr) (((hhdr) -> hb_flags & WAS_UNMAPPED) == 0)
#else
#   define IS_MAPPED(hhdr) TRUE
#endif /* !USE_MUNMAP */

#if !defined(NO_DEBUGGING) || defined(GC_ASSERTIONS)
  static void GC_CALLBACK add_hb_sz(struct hblk *h, int i,
                                    void *total_free_ptr)
  {
      UNUSED_ARG(i);
      *(word *)total_free_ptr += HDR(h) -> hb_sz;
#     if defined(CPPCHECK)
          GC_noop1_ptr(h);
#     endif
  }

  /* Should return the same value as GC_large_free_bytes.       */
  GC_INNER word GC_compute_large_free_bytes(void)
  {
      word total_free = 0;

      GC_iterate_free_hblks(add_hb_sz, &total_free);
      return total_free;
  }
#endif /* !NO_DEBUGGING || GC_ASSERTIONS */

#if !defined(NO_DEBUGGING)
  static void GC_CALLBACK print_hblkfreelist_item(struct hblk *h, int i,
                                                  void *prev_index_ptr)
  {
    hdr *hhdr = HDR(h);

#   if defined(CPPCHECK)
      GC_noop1_ptr(h);
#   endif
    if (i != *(int *)prev_index_ptr) {
      GC_printf("Free list %d (total size %lu):\n",
                i, (unsigned long)GC_free_bytes[i]);
      *(int *)prev_index_ptr = i;
    }

    GC_printf("\t%p size %lu %s black listed\n",
              (void *)h, (unsigned long)(hhdr -> hb_sz),
              GC_is_black_listed(h, HBLKSIZE) != NULL ? "start"
                : GC_is_black_listed(h, hhdr -> hb_sz) != NULL ? "partially"
                : "not");
  }

  void GC_print_hblkfreelist(void)
  {
    word total;
    int prev_index = -1;

    GC_iterate_free_hblks(print_hblkfreelist_item, &prev_index);
    GC_printf("GC_large_free_bytes: %lu\n",
              (unsigned long)GC_large_free_bytes);
    total = GC_compute_large_free_bytes();
    if (total != GC_large_free_bytes)
      GC_err_printf("GC_large_free_bytes INCONSISTENT!! Should be: %lu\n",
                    (unsigned long)total);
  }

  /* Return the free-list index on which the block described by the header */
  /* appears, or -1 if it appears nowhere.                                 */
  static int free_list_index_of(const hdr *wanted)
  {
    int i;

    for (i = 0; i <= N_HBLK_FLS; ++i) {
      const struct hblk * h;
      const hdr * hhdr;

      for (h = GC_hblkfreelist[i]; h != NULL; h = hhdr -> hb_next) {
        hhdr = HDR(h);
        if (hhdr == wanted) return i;
      }
    }
    return -1;
  }

  GC_API void GC_CALL GC_dump_regions(void)
  {
    size_t i;

    for (i = 0; i < GC_n_heap_sects; ++i) {
        ptr_t start = GC_heap_sects[i].hs_start;
        size_t bytes = GC_heap_sects[i].hs_bytes;
        ptr_t end = start + bytes;
        ptr_t p;

        /* Merge in contiguous sections.        */
          while (i+1 < GC_n_heap_sects && GC_heap_sects[i+1].hs_start == end) {
            ++i;
            end = GC_heap_sects[i].hs_start + GC_heap_sects[i].hs_bytes;
          }
        GC_printf("***Section from %p to %p\n", (void *)start, (void *)end);
        for (p = start; ADDR_LT(p, end); ) {
            hdr *hhdr = HDR(p);

            if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
                GC_printf("\t%p Missing header!!(%p)\n",
                          (void *)p, (void *)hhdr);
                p += HBLKSIZE;
                continue;
            }
            if (HBLK_IS_FREE(hhdr)) {
                int correct_index = (int)GC_hblk_fl_from_blocks(
                                                divHBLKSZ(hhdr -> hb_sz));
                int actual_index;

                GC_printf("\t%p\tfree block of size 0x%lx bytes%s\n",
                          (void *)p, (unsigned long)(hhdr -> hb_sz),
                          IS_MAPPED(hhdr) ? "" : " (unmapped)");
                actual_index = free_list_index_of(hhdr);
                if (-1 == actual_index) {
                    GC_printf("\t\tBlock not on free list %d!!\n",
                              correct_index);
                } else if (correct_index != actual_index) {
                    GC_printf("\t\tBlock on list %d, should be on %d!!\n",
                              actual_index, correct_index);
                }
                p += hhdr -> hb_sz;
            } else {
                GC_printf("\t%p\tused for blocks of size 0x%lx bytes\n",
                          (void *)p, (unsigned long)(hhdr -> hb_sz));
                p += HBLKSIZE * OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz);
            }
        }
    }
  }
#endif /* NO_DEBUGGING */

/* Initialize hdr for a block containing the indicated size and         */
/* kind of objects.  Return FALSE on failure.                           */
static GC_bool setup_header(hdr *hhdr, struct hblk *block, size_t lb_adjusted,
                            int k, unsigned flags)
{
    const struct obj_kind *ok;
    word descr;

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(lb_adjusted >= ALIGNMENT);
#   ifndef MARK_BIT_PER_OBJ
      if (lb_adjusted > MAXOBJBYTES)
        flags |= LARGE_BLOCK;
#   endif
    ok = &GC_obj_kinds[k];
#   ifdef ENABLE_DISCLAIM
      if (ok -> ok_disclaim_proc)
        flags |= HAS_DISCLAIM;
      if (ok -> ok_mark_unconditionally)
        flags |= MARK_UNCONDITIONALLY;
#   endif

    /* Set size, kind and mark proc fields.     */
    hhdr -> hb_sz = lb_adjusted;
    hhdr -> hb_obj_kind = (unsigned char)k;
    hhdr -> hb_flags = (unsigned char)flags;
    hhdr -> hb_block = block;
    descr = ok -> ok_descriptor;
#   if ALIGNMENT > GC_DS_TAGS
      /* An extra byte is not added in case of ignore-off-page  */
      /* allocated objects not smaller than HBLKSIZE.           */
      if (EXTRA_BYTES != 0 && (flags & IGNORE_OFF_PAGE) != 0
          && k == NORMAL && lb_adjusted >= HBLKSIZE)
        descr += ALIGNMENT; /* or set to 0 */
#   endif
    if (ok -> ok_relocate_descr) descr += lb_adjusted;
    hhdr -> hb_descr = descr;

#   ifdef MARK_BIT_PER_OBJ
      /* Set hb_inv_sz as portably as possible.  We set it to the       */
      /* smallest value such that lb_adjusted * inv_sz >= 2**32.        */
      /* This may be more precision than necessary.                     */
      if (lb_adjusted > MAXOBJBYTES) {
        hhdr -> hb_inv_sz = LARGE_INV_SZ;
      } else {
        unsigned32 inv_sz;

        GC_ASSERT(lb_adjusted > 1);
#       if CPP_WORDSZ > 32
          inv_sz = (unsigned32)(((word)1 << 32) / lb_adjusted);
          if (((inv_sz * (word)lb_adjusted) >> 32) == 0) ++inv_sz;
#       else
          inv_sz = (((unsigned32)1 << 31) / lb_adjusted) << 1;
          while ((inv_sz * lb_adjusted) > lb_adjusted)
            inv_sz++;
#       endif
#       if (CPP_WORDSZ == 32) && defined(__GNUC__)
          GC_ASSERT(((1ULL << 32) + lb_adjusted - 1) / lb_adjusted == inv_sz);
#       endif
        hhdr -> hb_inv_sz = inv_sz;
      }
#   else
      {
        size_t lg = BYTES_TO_GRANULES(lb_adjusted);

        if (EXPECT(!GC_add_map_entry(lg), FALSE)) {
          /* Make it look like a valid block.   */
          hhdr -> hb_sz = HBLKSIZE;
          hhdr -> hb_descr = 0;
          hhdr -> hb_flags |= LARGE_BLOCK;
          hhdr -> hb_map = 0;
          return FALSE;
        }
        hhdr -> hb_map = GC_obj_map[(hhdr -> hb_flags & LARGE_BLOCK) != 0 ?
                                    0 : lg];
      }
#   endif

    /* Clear mark bits */
    GC_clear_hdr_marks(hhdr);

    hhdr -> hb_last_reclaimed = (unsigned short)GC_gc_no;
    return TRUE;
}

/* Remove hhdr from the free list (it is assumed to specified by index). */
STATIC void GC_remove_from_fl_at(hdr *hhdr, size_t index)
{
    GC_ASSERT(modHBLKSZ(hhdr -> hb_sz) == 0);
    if (hhdr -> hb_prev == 0) {
        GC_ASSERT(HDR(GC_hblkfreelist[index]) == hhdr);
        GC_hblkfreelist[index] = hhdr -> hb_next;
    } else {
        hdr *phdr;
        GET_HDR(hhdr -> hb_prev, phdr);
        phdr -> hb_next = hhdr -> hb_next;
    }
    /* We always need index to maintain free counts.    */
    GC_ASSERT(GC_free_bytes[index] >= hhdr -> hb_sz);
    GC_free_bytes[index] -= hhdr -> hb_sz;
    if (0 != hhdr -> hb_next) {
        hdr *nhdr;

        GC_ASSERT(!IS_FORWARDING_ADDR_OR_NIL(NHDR(hhdr)));
        GET_HDR(hhdr -> hb_next, nhdr);
        nhdr -> hb_prev = hhdr -> hb_prev;
    }
}

/* Remove hhdr from the appropriate free list (we assume it is on the   */
/* size-appropriate free list).                                         */
GC_INLINE void GC_remove_from_fl(hdr *hhdr)
{
  GC_remove_from_fl_at(hhdr, GC_hblk_fl_from_blocks(divHBLKSZ(hhdr -> hb_sz)));
}

/* Return a pointer to the block ending just before h, if any.  */
static struct hblk * get_block_ending_at(struct hblk *h)
{
    struct hblk *p = h - 1;
    hdr *hhdr;

    GET_HDR(p, hhdr);
    if (hhdr != NULL) {
        return GC_find_starting_hblk(p, &hhdr);
    }
    p = GC_prev_block(p);
    if (p != NULL) {
        hhdr = HDR(p);
        if ((ptr_t)p + hhdr -> hb_sz == (ptr_t)h) {
            return p;
        }
    }
    return NULL;
}

/* Return a pointer to the free block ending just before h, if any.     */
STATIC struct hblk * GC_free_block_ending_at(struct hblk *h)
{
    struct hblk * p = get_block_ending_at(h);

    if (p /* != NULL */) { /* CPPCHECK */
      const hdr *hhdr = HDR(p);

      if (HBLK_IS_FREE(hhdr)) {
        return p;
      }
    }
    return 0;
}

/* Add hhdr to the appropriate free list.               */
/* We maintain individual free lists sorted by address. */
STATIC void GC_add_to_fl(struct hblk *h, hdr *hhdr)
{
    size_t index = GC_hblk_fl_from_blocks(divHBLKSZ(hhdr -> hb_sz));
    struct hblk *second = GC_hblkfreelist[index];

#   if defined(GC_ASSERTIONS) && !defined(USE_MUNMAP)
    {
      struct hblk *next = (struct hblk *)((ptr_t)h + hhdr -> hb_sz);
      const hdr *nexthdr = HDR(next);
      struct hblk *prev = GC_free_block_ending_at(h);
      const hdr *prevhdr = HDR(prev);

      GC_ASSERT(NULL == nexthdr || !HBLK_IS_FREE(nexthdr)
                || (GC_heapsize & SIGNB) != 0);
                /* In the last case, blocks may be too large to merge. */
      GC_ASSERT(NULL == prev || !HBLK_IS_FREE(prevhdr)
                || (GC_heapsize & SIGNB) != 0);
    }
#   endif
    GC_ASSERT(modHBLKSZ(hhdr -> hb_sz) == 0);
    GC_hblkfreelist[index] = h;
    GC_free_bytes[index] += hhdr -> hb_sz;
    GC_ASSERT(GC_free_bytes[index] <= GC_large_free_bytes);
    hhdr -> hb_next = second;
    hhdr -> hb_prev = 0;
    if (second /* != NULL */) { /* CPPCHECK */
      hdr * second_hdr;

      GET_HDR(second, second_hdr);
      second_hdr -> hb_prev = h;
    }
    hhdr -> hb_flags |= FREE_BLK;
}

#ifdef USE_MUNMAP

#ifdef COUNT_UNMAPPED_REGIONS
  /* GC_unmap_old will avoid creating more than this many unmapped regions, */
  /* but an unmapped region may be split again so exceeding the limit.      */

  /* Return the change in number of unmapped regions if the block h swaps   */
  /* from its current state of mapped/unmapped to the opposite state.       */
  static int calc_num_unmapped_regions_delta(struct hblk *h, hdr *hhdr)
  {
    struct hblk * prev = get_block_ending_at(h);
    struct hblk * next;
    GC_bool prev_unmapped = FALSE;
    GC_bool next_unmapped = FALSE;

    next = GC_next_block((struct hblk *)((ptr_t)h + hhdr -> hb_sz), TRUE);
    /* Ensure next is contiguous with h.        */
    if (next != HBLK_PAGE_ALIGNED((ptr_t)h + hhdr -> hb_sz)) {
      next = NULL;
    }
    if (prev != NULL) {
      const hdr *prevhdr = HDR(prev);
      prev_unmapped = !IS_MAPPED(prevhdr);
    }
    if (next != NULL) {
      const hdr *nexthdr = HDR(next);
      next_unmapped = !IS_MAPPED(nexthdr);
    }

    if (prev_unmapped && next_unmapped) {
      /* If h unmapped, merge two unmapped regions into one.    */
      /* If h remapped, split one unmapped region into two.     */
      return IS_MAPPED(hhdr) ? -1 : 1;
    }
    if (!prev_unmapped && !next_unmapped) {
      /* If h unmapped, create an isolated unmapped region.     */
      /* If h remapped, remove it.                              */
      return IS_MAPPED(hhdr) ? 1 : -1;
    }
    /* If h unmapped, merge it with previous or next unmapped region.   */
    /* If h remapped, reduce either previous or next unmapped region.   */
    /* In either way, no change to the number of unmapped regions.      */
    return 0;
  }
#endif /* COUNT_UNMAPPED_REGIONS */

/* Update GC_num_unmapped_regions assuming the block h changes      */
/* from its current state of mapped/unmapped to the opposite state. */
GC_INLINE void GC_adjust_num_unmapped(struct hblk *h, hdr *hhdr)
{
# ifdef COUNT_UNMAPPED_REGIONS
    GC_num_unmapped_regions += calc_num_unmapped_regions_delta(h, hhdr);
# else
    UNUSED_ARG(h);
    UNUSED_ARG(hhdr);
# endif
}

/* Unmap blocks that haven't been recently touched.  This is the only   */
/* way blocks are ever unmapped.                                        */
GC_INNER void GC_unmap_old(unsigned threshold)
{
    size_t i;

# ifdef COUNT_UNMAPPED_REGIONS
    /* Skip unmapping if we have already exceeded the soft limit.       */
    /* This forgoes any opportunities to merge unmapped regions though. */
    if (GC_num_unmapped_regions >= GC_UNMAPPED_REGIONS_SOFT_LIMIT)
      return;
# endif

    for (i = 0; i <= N_HBLK_FLS; ++i) {
      struct hblk * h;
      hdr * hhdr;

      for (h = GC_hblkfreelist[i]; 0 != h; h = hhdr -> hb_next) {
        hhdr = HDR(h);
        if (!IS_MAPPED(hhdr)) continue;

        /* Check that the interval is not smaller than the threshold.   */
        /* The truncated counter value wrapping is handled correctly.   */
        if ((unsigned short)(GC_gc_no - hhdr->hb_last_reclaimed)
            >= (unsigned short)threshold) {
#         ifdef COUNT_UNMAPPED_REGIONS
            /* Continue with unmapping the block only if it will not    */
            /* create too many unmapped regions, or if unmapping        */
            /* reduces the number of regions.                           */
            int delta = calc_num_unmapped_regions_delta(h, hhdr);
            signed_word regions = GC_num_unmapped_regions + delta;

            if (delta >= 0 && regions >= GC_UNMAPPED_REGIONS_SOFT_LIMIT) {
              GC_COND_LOG_PRINTF("Unmapped regions limit reached!\n");
              return;
            }
            GC_num_unmapped_regions = regions;
#         endif
          GC_unmap((ptr_t)h, hhdr -> hb_sz);
          hhdr -> hb_flags |= WAS_UNMAPPED;
        }
      }
    }
}

/* Merge all unmapped blocks that are adjacent to other free            */
/* blocks.  This may involve remapping, since all blocks are either     */
/* fully mapped or fully unmapped.                                      */
GC_INNER void GC_merge_unmapped(void)
{
    size_t i;

    for (i = 0; i <= N_HBLK_FLS; ++i) {
      struct hblk *h = GC_hblkfreelist[i];

      while (h != 0) {
        struct hblk *next;
        hdr *hhdr, *nexthdr;
        size_t size, next_size;

        GET_HDR(h, hhdr);
        size = hhdr -> hb_sz;
        next = (struct hblk *)((ptr_t)h + size);
        GET_HDR(next, nexthdr);
        /* Coalesce with successor, if possible. */
        if (nexthdr != NULL && HBLK_IS_FREE(nexthdr)
              && ((size + (next_size = nexthdr -> hb_sz)) & SIZET_SIGNB) == 0
                 /* no overflow */) {
            /* Note that we usually try to avoid adjacent free blocks   */
            /* that are either both mapped or both unmapped.  But that  */
            /* isn't guaranteed to hold since we remap blocks when we   */
            /* split them, and don't merge at that point.  It may also  */
            /* not hold if the merged block would be too big.           */
            if (IS_MAPPED(hhdr) && !IS_MAPPED(nexthdr)) {
              /* Make both consistent, so that we can merge.    */
                if (size > next_size) {
                  GC_adjust_num_unmapped(next, nexthdr);
                  GC_remap((ptr_t)next, next_size);
                } else {
                  GC_adjust_num_unmapped(h, hhdr);
                  GC_unmap((ptr_t)h, size);
                  GC_unmap_gap((ptr_t)h, size, (ptr_t)next, next_size);
                  hhdr -> hb_flags |= WAS_UNMAPPED;
                }
            } else if (IS_MAPPED(nexthdr) && !IS_MAPPED(hhdr)) {
              if (size > next_size) {
                GC_adjust_num_unmapped(next, nexthdr);
                GC_unmap((ptr_t)next, next_size);
                GC_unmap_gap((ptr_t)h, size, (ptr_t)next, next_size);
              } else {
                GC_adjust_num_unmapped(h, hhdr);
                GC_remap((ptr_t)h, size);
                hhdr -> hb_flags &= (unsigned char)~WAS_UNMAPPED;
                hhdr -> hb_last_reclaimed = nexthdr -> hb_last_reclaimed;
              }
            } else if (!IS_MAPPED(hhdr) && !IS_MAPPED(nexthdr)) {
              /* Unmap any gap in the middle */
                GC_unmap_gap((ptr_t)h, size, (ptr_t)next, next_size);
            }
            /* If they are both unmapped, we merge, but leave unmapped. */
            GC_remove_from_fl_at(hhdr, i);
            GC_remove_from_fl(nexthdr);
            hhdr -> hb_sz += nexthdr -> hb_sz;
            GC_remove_header(next);
            GC_add_to_fl(h, hhdr);
            /* Start over at beginning of list. */
            h = GC_hblkfreelist[i];
        } else /* not mergeable with successor */ {
            h = hhdr -> hb_next;
        }
      } /* while (h != 0) ... */
    } /* for ... */
}

#endif /* USE_MUNMAP */

/*
 * Return a pointer to a block starting at h of length bytes.
 * Memory for the block is mapped.
 * Remove the block from its free list, and return the remainder (if any)
 * to its appropriate free list.
 * May fail by returning 0.
 * The header for the returned block must be set up by the caller.
 * If the return value is not 0, then hhdr is the header for it.
 */
STATIC struct hblk * GC_get_first_part(struct hblk *h, hdr *hhdr,
                                       size_t size_needed, size_t index)
{
    size_t total_size;
    struct hblk * rest;
    hdr * rest_hdr;

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(modHBLKSZ(size_needed) == 0);
    total_size = hhdr -> hb_sz;
    GC_ASSERT(modHBLKSZ(total_size) == 0);
    GC_remove_from_fl_at(hhdr, index);
    if (total_size == size_needed) return h;

    rest = (struct hblk *)((ptr_t)h + size_needed);
    rest_hdr = GC_install_header(rest);
    if (EXPECT(NULL == rest_hdr, FALSE)) {
        /* FIXME: This is likely to be very bad news ... */
        WARN("Header allocation failed: dropping block\n", 0);
        return NULL;
    }
    rest_hdr -> hb_sz = total_size - size_needed;
    rest_hdr -> hb_flags = 0;
#   ifdef GC_ASSERTIONS
      /* Mark h not free, to avoid assertion about adjacent free blocks. */
        hhdr -> hb_flags &= (unsigned char)~FREE_BLK;
#   endif
    GC_add_to_fl(rest, rest_hdr);
    return h;
}

/* Split the block.  hbp is a free block; last_hbp points at an address */
/* inside it; a new header for last_hbp is assumed to be already set    */
/* up.  Fix up the header of hbp to reflect the fact that it is being   */
/* split, move it to the appropriate free list.  last_hbp replaces hbp  */
/* in the original free list.  last_hdr is not completely filled in,    */
/* since it is about to be allocated.  It may in fact end up on the     */
/* wrong free list for its size.  That is not a disaster, since         */
/* last_hbp is to be allocated by our caller.  (Hence adding it to      */
/* a free list is silly.  But this path is hopefully rare enough that   */
/* it does not matter.  The code is cleaner this way.)                  */
STATIC void GC_split_block(struct hblk *hbp, hdr *hhdr, struct hblk *last_hbp,
                           hdr *last_hdr, size_t index /* of free list */)
{
    size_t h_size = (size_t)((ptr_t)last_hbp - (ptr_t)hbp);
    struct hblk *prev = hhdr -> hb_prev;
    struct hblk *next = hhdr -> hb_next;

    /* Replace hbp with last_hbp on its free list.  */
    last_hdr -> hb_prev = prev;
    last_hdr -> hb_next = next;
    last_hdr -> hb_sz = hhdr -> hb_sz - h_size;
    last_hdr -> hb_flags = 0;
    if (prev /* != NULL */) { /* CPPCHECK */
      HDR(prev) -> hb_next = last_hbp;
    } else {
      GC_hblkfreelist[index] = last_hbp;
    }
    if (next /* != NULL */) {
      HDR(next) -> hb_prev = last_hbp;
    }
    GC_ASSERT(GC_free_bytes[index] > h_size);
    GC_free_bytes[index] -= h_size;
#   ifdef USE_MUNMAP
      hhdr -> hb_last_reclaimed = (unsigned short)GC_gc_no;
#   endif
    hhdr -> hb_sz = h_size;
    GC_add_to_fl(hbp, hhdr);
    last_hdr -> hb_flags |= FREE_BLK;
}

STATIC struct hblk *GC_allochblk_nth(size_t lb_adjusted, int k, unsigned flags,
                                     size_t index, int may_split,
                                     size_t align_m1);

#ifdef USE_MUNMAP
# define AVOID_SPLIT_REMAPPED 2
#endif

GC_INNER struct hblk *GC_allochblk(size_t lb_adjusted, int k,
                                   unsigned flags /* IGNORE_OFF_PAGE or 0 */,
                                   size_t align_m1)
{
    size_t blocks, start_list;
    struct hblk *result;
    int may_split;
    size_t split_limit; /* highest index of free list whose blocks we split */

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT((lb_adjusted & (GC_GRANULE_BYTES-1)) == 0);
    blocks = OBJ_SZ_TO_BLOCKS_CHECKED(lb_adjusted);
    if (EXPECT(SIZET_SAT_ADD(blocks * HBLKSIZE, align_m1)
                >= (GC_SIZE_MAX >> 1), FALSE))
      return NULL; /* overflow */

    start_list = GC_hblk_fl_from_blocks(blocks);
    /* Try for an exact match first.    */
    result = GC_allochblk_nth(lb_adjusted, k, flags, start_list, FALSE,
                              align_m1);
    if (result != NULL) return result;

    may_split = TRUE;
    if (GC_use_entire_heap || GC_dont_gc
        || GC_heapsize - GC_large_free_bytes < GC_requested_heapsize
        || GC_incremental || !GC_should_collect()) {
        /* Should use more of the heap, even if it requires splitting. */
        split_limit = N_HBLK_FLS;
    } else if (GC_finalizer_bytes_freed > (GC_heapsize >> 4)) {
          /* If we are deallocating lots of memory from         */
          /* finalizers, fail and collect sooner rather         */
          /* than later.                                        */
          split_limit = 0;
    } else {
          /* If we have enough large blocks left to cover any   */
          /* previous request for large blocks, we go ahead     */
          /* and split.  Assuming a steady state, that should   */
          /* be safe.  It means that we can use the full        */
          /* heap if we allocate only small objects.            */
          split_limit = GC_enough_large_bytes_left();
#         ifdef USE_MUNMAP
            if (split_limit > 0)
              may_split = AVOID_SPLIT_REMAPPED;
#         endif
    }
    if (start_list < UNIQUE_THRESHOLD && 0 == align_m1) {
      /* No reason to try start_list again, since all blocks are exact  */
      /* matches.                                                       */
      ++start_list;
    }
    for (; start_list <= split_limit; ++start_list) {
      result = GC_allochblk_nth(lb_adjusted, k, flags, start_list, may_split,
                                align_m1);
      if (result != NULL) break;
    }
    return result;
}

STATIC long GC_large_alloc_warn_suppressed = 0;
                        /* Number of warnings suppressed so far.        */

STATIC unsigned GC_drop_blacklisted_count = 0;
                        /* Counter of the cases when found block by     */
                        /* GC_allochblk_nth is blacklisted completely.  */

#define ALIGN_PAD_SZ(p, align_m1) \
               (((align_m1) + 1 - (size_t)ADDR(p)) & (align_m1))

static GC_bool next_hblk_fits_better(const hdr *hhdr, size_t size_avail,
                                     size_t size_needed, size_t align_m1)
{
  const hdr *nexthdr;
  size_t next_size;
  size_t next_ofs;
  struct hblk *next_hbp = hhdr -> hb_next;

  if (NULL == next_hbp) return FALSE; /* no next block */
  GET_HDR(next_hbp, nexthdr);
  next_size = nexthdr -> hb_sz;
  if (size_avail <= next_size) return FALSE; /* not enough size */

  next_ofs = ALIGN_PAD_SZ(next_hbp, align_m1);
  return next_size >= size_needed + next_ofs
         && !GC_is_black_listed(next_hbp + divHBLKSZ(next_ofs), size_needed);
}

static struct hblk *find_nonbl_hblk(struct hblk *last_hbp, size_t size_remain,
                                    size_t eff_size_needed, size_t align_m1)
{
  ptr_t search_end = PTR_ALIGN_DOWN((ptr_t)last_hbp + size_remain,
                                    align_m1 + 1);

  do {
    struct hblk *next_hbp;

    last_hbp += divHBLKSZ(ALIGN_PAD_SZ(last_hbp, align_m1));
    next_hbp = GC_is_black_listed(last_hbp, eff_size_needed);
    if (NULL == next_hbp) return last_hbp; /* not black-listed */
    last_hbp = next_hbp;
  } while (ADDR_GE(search_end, (ptr_t)last_hbp));
  return NULL;
}

/* Allocate and drop the block in small chunks, to maximize the chance  */
/* that we will recover some later.  hhdr should correspond to hbp.     */
static void drop_hblk_in_chunks(size_t n, struct hblk *hbp, hdr *hhdr)
{
  size_t total_size = hhdr -> hb_sz;
  const struct hblk *limit = hbp + divHBLKSZ(total_size);

  GC_ASSERT(HDR(hbp) == hhdr);
  GC_ASSERT(modHBLKSZ(total_size) == 0 && total_size > 0);
  GC_large_free_bytes -= total_size;
  GC_bytes_dropped += total_size;
  GC_remove_from_fl_at(hhdr, n);
  do {
    (void)setup_header(hhdr, hbp, HBLKSIZE, PTRFREE, 0); /* cannot fail */
    if (GC_debugging_started) BZERO(hbp, HBLKSIZE);
    hbp++;
    if (ADDR_GE(hbp, limit)) break;

    hhdr = GC_install_header(hbp);
  } while (EXPECT(hhdr != NULL, TRUE)); /* no header allocation failure? */
}

#if defined(MPROTECT_VDB) && defined(DONT_PROTECT_PTRFREE)
  static GC_bool is_hblks_mix_in_page(struct hblk *hbp, GC_bool is_ptrfree)
  {
    struct hblk *h = HBLK_PAGE_ALIGNED(hbp);
    size_t i, cnt = divHBLKSZ(GC_page_size);

    /* Iterate over blocks in the page to check if all the    */
    /* occupied blocks are pointer-free if we are going to    */
    /* allocate a pointer-free one, and vice versa.           */
    for (i = 0; i < cnt; i++) {
      hdr *hhdr;

      GET_HDR(&h[i], hhdr);
      if (NULL == hhdr) continue;
      (void)GC_find_starting_hblk(&h[i], &hhdr);
      if (!HBLK_IS_FREE(hhdr)) {
        /* It is OK to check only the first found occupied block.   */
        return IS_PTRFREE(hhdr) != is_ptrfree;
      }
    }
    return FALSE; /* all blocks are free */
  }
#endif /* MPROTECT_VDB && DONT_PROTECT_PTRFREE */

/* The same as GC_allochblk, but with search restricted to the index-th */
/* free list.  flags should be IGNORE_OFF_PAGE or zero; may_split       */
/* indicates whether it is OK to split larger blocks; size is in bytes. */
/* If may_split is set to AVOID_SPLIT_REMAPPED, then memory remapping   */
/* followed by splitting should be generally avoided.  Rounded-up       */
/* lb_adjusted plus align_m1 value should be less than GC_SIZE_MAX / 2. */
STATIC struct hblk *GC_allochblk_nth(size_t lb_adjusted, int k, unsigned flags,
                                     size_t index, int may_split,
                                     size_t align_m1)
{
    struct hblk *hbp, *last_hbp;
    hdr *hhdr; /* header corresponding to hbp */
    size_t size_needed = HBLKSIZE * OBJ_SZ_TO_BLOCKS_CHECKED(lb_adjusted);
                                /* number of bytes in requested objects */

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(((align_m1 + 1) & align_m1) == 0 && lb_adjusted > 0);
    GC_ASSERT(0 == align_m1 || modHBLKSZ(align_m1 + 1) == 0);
  retry:
    /* Search for a big enough block in free list.      */
    for (hbp = GC_hblkfreelist[index];; hbp = hhdr -> hb_next) {
      size_t size_avail; /* bytes available in this block */
      size_t align_ofs;

      if (hbp /* != NULL */) {
        /* CPPCHECK */
      } else {
        return NULL;
      }
      GET_HDR(hbp, hhdr); /* set hhdr value */
      size_avail = hhdr -> hb_sz;
      if (!may_split && size_avail != size_needed) continue;

      align_ofs = ALIGN_PAD_SZ(hbp, align_m1);
      if (size_avail < size_needed + align_ofs)
        continue; /* the block is too small */

      if (size_avail != size_needed) {
        /* If the next heap block is obviously better, go on.   */
        /* This prevents us from disassembling a single large   */
        /* block to get tiny blocks.                            */
        if (next_hblk_fits_better(hhdr, size_avail, size_needed, align_m1))
          continue;
      }

#     if defined(MPROTECT_VDB) && defined(DONT_PROTECT_PTRFREE)
        /* Avoid write-protecting pointer-free blocks (only the */
        /* case if page size is larger than the block size).    */
        GC_ASSERT(GC_page_size != 0);
        if (GC_page_size != HBLKSIZE
            && (!GC_incremental /* not enabled yet */
                || GC_incremental_protection_needs() != GC_PROTECTS_NONE)
            && is_hblks_mix_in_page(hbp, k == PTRFREE))
          continue;
#     endif

      if (IS_UNCOLLECTABLE(k)
          || (k == PTRFREE && size_needed <= MAX_BLACK_LIST_ALLOC)) {
        last_hbp = hbp + divHBLKSZ(align_ofs);
        break;
      }

      last_hbp = find_nonbl_hblk(hbp, size_avail - size_needed,
                    (flags & IGNORE_OFF_PAGE) != 0 ? HBLKSIZE : size_needed,
                    align_m1);
      /* Is non-blacklisted part of enough size?        */
      if (last_hbp != NULL) {
#       ifdef USE_MUNMAP
          /* Avoid remapping followed by splitting.     */
          if (may_split == AVOID_SPLIT_REMAPPED && last_hbp != hbp
              && !IS_MAPPED(hhdr))
            continue;
#       endif
        break;
      }

      /* The block is completely blacklisted.  If so, we need to        */
      /* drop some such blocks, since otherwise we spend all our        */
      /* time traversing them if pointer-free blocks are unpopular.     */
      /* A dropped block will be reconsidered at next GC.               */
      if (size_needed == HBLKSIZE && 0 == align_m1
          && !GC_find_leak && IS_MAPPED(hhdr)
          && (++GC_drop_blacklisted_count & 3) == 0) {
        const struct hblk *prev = hhdr -> hb_prev;

        drop_hblk_in_chunks(index, hbp, hhdr);
        if (NULL == prev) goto retry;
        /* Restore hhdr to point at free block. */
        hhdr = HDR(prev);
        continue;
      }

      if (size_needed > BL_LIMIT && size_avail - size_needed > BL_LIMIT) {
        /* Punt, since anything else risks unreasonable heap growth.    */
        if (++GC_large_alloc_warn_suppressed
            >= GC_large_alloc_warn_interval) {
          WARN("Repeated allocation of very large block"
               " (appr. size %" WARN_PRIuPTR " KiB):\n"
               "\tMay lead to memory leak and poor performance\n",
               size_needed >> 10);
          GC_large_alloc_warn_suppressed = 0;
        }
        last_hbp = hbp + divHBLKSZ(align_ofs);
        break;
      }
    }

    GC_ASSERT((ADDR(last_hbp) & align_m1) == 0);
    if (last_hbp != hbp) {
      hdr *last_hdr = GC_install_header(last_hbp);

      if (EXPECT(NULL == last_hdr, FALSE)) return NULL;
      /* Make sure it's mapped before we mangle it.     */
#     ifdef USE_MUNMAP
        if (!IS_MAPPED(hhdr)) {
          GC_adjust_num_unmapped(hbp, hhdr);
          GC_remap((ptr_t)hbp, hhdr -> hb_sz);
          hhdr -> hb_flags &= (unsigned char)~WAS_UNMAPPED;
        }
#     endif
      /* Split the block at last_hbp. */
      GC_split_block(hbp, hhdr, last_hbp, last_hdr, index);
      /* We must now allocate last_hbp, since it may be on the  */
      /* wrong free list.                                       */
      hbp = last_hbp;
      hhdr = last_hdr;
    }
    GC_ASSERT(hhdr -> hb_sz >= size_needed);

#   ifdef USE_MUNMAP
      if (!IS_MAPPED(hhdr)) {
        GC_adjust_num_unmapped(hbp, hhdr);
        GC_remap((ptr_t)hbp, hhdr -> hb_sz);
        hhdr -> hb_flags &= (unsigned char)~WAS_UNMAPPED;
        /* Note: This may leave adjacent, mapped free blocks. */
      }
#   endif
    /* hbp may be on the wrong free list; the parameter index is important. */
    hbp = GC_get_first_part(hbp, hhdr, size_needed, index);
    if (EXPECT(NULL == hbp, FALSE)) return NULL;

    /* Add it to map of valid blocks.   */
    if (EXPECT(!GC_install_counts(hbp, size_needed), FALSE))
      return NULL; /* This leaks memory under very rare conditions. */

    /* Set up the header.       */
    GC_ASSERT(HDR(hbp) == hhdr);
#   ifdef MARK_BIT_PER_OBJ
      (void)setup_header(hhdr, hbp, lb_adjusted, k, flags);
      /* Result is always true, not checked to avoid a cppcheck warning. */
#   else
      if (EXPECT(!setup_header(hhdr, hbp, lb_adjusted, k, flags), FALSE)) {
        GC_remove_counts(hbp, size_needed);
        return NULL; /* ditto */
      }
#   endif

#   ifndef GC_DISABLE_INCREMENTAL
      /* Notify virtual dirty bit implementation that we are about to   */
      /* write.  Ensure that pointer-free objects are not protected     */
      /* if it is avoidable.  This also ensures that newly allocated    */
      /* blocks are treated as dirty - it is necessary since we do not  */
      /* protect free blocks.                                           */
      GC_ASSERT(modHBLKSZ(size_needed) == 0);
      GC_remove_protection(hbp, divHBLKSZ(size_needed), IS_PTRFREE(hhdr));
#   endif
    /* We just successfully allocated a block.  Restart count of        */
    /* consecutive failures.                                            */
    GC_fail_count = 0;

    GC_large_free_bytes -= size_needed;
    GC_ASSERT(IS_MAPPED(hhdr));
    return hbp;
}

#ifdef VALGRIND_TRACKING
  /* Note: this is intentionally defined in a file other than malloc.c  */
  /* and reclaim.c ones.                                                */
  GC_ATTR_NOINLINE
  GC_API void GC_CALLBACK GC_free_profiler_hook(void *p)
  {
#   ifndef PARALLEL_MARK
      GC_ASSERT(I_HOLD_LOCK());
#   endif
    /* Prevent treating this function by the compiler as a no-op one.   */
    GC_noop1_ptr(p);
  }
#endif /* VALGRIND_TRACKING */

/*
 * Free a heap block.
 *
 * Coalesce the block with its neighbors if possible.
 *
 * All mark words are assumed to be cleared.
 */
GC_INNER void GC_freehblk(struct hblk *hbp)
{
    struct hblk *next, *prev;
    hdr *hhdr, *prevhdr, *nexthdr;
    size_t size;

    GET_HDR(hbp, hhdr);
    size = HBLKSIZE * OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz);
    if ((size & SIZET_SIGNB) != 0)
      ABORT("Deallocating excessively large block.  Too large an allocation?");
      /* Probably possible if we try to allocate more than half the address */
      /* space at once.  If we don't catch it here, strange things happen   */
      /* later.                                                             */

    GC_remove_counts(hbp, size);
    hhdr -> hb_sz = size;
#   ifdef USE_MUNMAP
      hhdr -> hb_last_reclaimed = (unsigned short)GC_gc_no;
#   endif

    /* Check for duplicate deallocation in the easy case. */
    if (HBLK_IS_FREE(hhdr)) {
        ABORT_ARG1("Duplicate large block deallocation",
                   " of %p", (void *)hbp);
    }

    GC_ASSERT(IS_MAPPED(hhdr));
    hhdr -> hb_flags |= FREE_BLK;
    next = (struct hblk *)((ptr_t)hbp + size);
    GET_HDR(next, nexthdr);
    prev = GC_free_block_ending_at(hbp);
    /* Coalesce with successor, if possible.    */
    if (nexthdr != NULL && HBLK_IS_FREE(nexthdr) && IS_MAPPED(nexthdr)
          && ((hhdr -> hb_sz + nexthdr -> hb_sz) & SIZET_SIGNB) == 0
             /* no overflow */) {
        GC_remove_from_fl(nexthdr);
        hhdr -> hb_sz += nexthdr -> hb_sz;
        GC_remove_header(next);
    }

    /* Coalesce with predecessor, if possible. */
    if (prev /* != NULL */) { /* CPPCHECK */
        prevhdr = HDR(prev);
        if (IS_MAPPED(prevhdr)
            && ((hhdr -> hb_sz + prevhdr -> hb_sz) & SIZET_SIGNB) == 0) {
          GC_remove_from_fl(prevhdr);
          prevhdr -> hb_sz += hhdr -> hb_sz;
#         ifdef USE_MUNMAP
            prevhdr -> hb_last_reclaimed = (unsigned short)GC_gc_no;
#         endif
          GC_remove_header(hbp);
          hbp = prev;
          hhdr = prevhdr;
        }
    }
    /* FIXME: It is not clear we really always want to do these merges  */
    /* with USE_MUNMAP, since it updates ages and hence prevents        */
    /* unmapping.                                                       */

    GC_large_free_bytes += size;
    GC_add_to_fl(hbp, hhdr);
}

/*
 * Copyright (c) 1988-1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1996 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2011 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


/*
 * Separate free lists are maintained for different sized objects
 * up to MAXOBJBYTES.
 * The call GC_allocobj(lg, k) ensures that the free list for
 * kind k objects of size lg granules to a non-empty
 * free list. It returns a pointer to the first entry on the free list.
 * In a single-threaded world, GC_allocobj may be called to allocate
 * an object of small size lb (and NORMAL kind) as follows
 * (GC_generic_malloc_inner is a wrapper over GC_allocobj which also
 * fills in GC_size_map if needed):
 *
 *   lg = GC_size_map[lb];
 *   op = GC_objfreelist[lg];
 *   if (NULL == op) {
 *     op = GC_generic_malloc_inner(lb, NORMAL, 0);
 *   } else {
 *     GC_objfreelist[lg] = obj_link(op);
 *     GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
 *   }
 *
 * Note that this is very fast if the free list is non-empty; it should
 * only involve the execution of 4 or 5 simple instructions.
 * All composite objects on freelists are cleared, except for
 * their first "pointer-sized" word.
 */

/*
 * The allocator uses GC_allochblk to allocate large chunks of objects.
 * These chunks all start on addresses which are multiples of
 * HBLKSZ.   Each allocated chunk has an associated header,
 * which can be located quickly based on the address of the chunk.
 * (See headers.c for details.)
 * This makes it possible to check quickly whether an
 * arbitrary address corresponds to an object administered by the
 * allocator.
 */

word GC_non_gc_bytes = 0;  /* Number of bytes not intended to be collected */

word GC_gc_no = 0;

#ifndef NO_CLOCK
  static unsigned long full_gc_total_time = 0; /* in ms, may wrap */
  static unsigned long stopped_mark_total_time = 0;
  static unsigned32 full_gc_total_ns_frac = 0; /* fraction of 1 ms */
  static unsigned32 stopped_mark_total_ns_frac = 0;
  static GC_bool measure_performance = FALSE;
                /* Do performance measurements if set to true (e.g.,    */
                /* accumulation of the total time of full collections). */

  GC_API void GC_CALL GC_start_performance_measurement(void)
  {
    measure_performance = TRUE;
  }

  GC_API unsigned long GC_CALL GC_get_full_gc_total_time(void)
  {
    return full_gc_total_time;
  }

  GC_API unsigned long GC_CALL GC_get_stopped_mark_total_time(void)
  {
    return stopped_mark_total_time;
  }
#endif /* !NO_CLOCK */

#ifndef GC_DISABLE_INCREMENTAL
  GC_INNER GC_bool GC_incremental = FALSE; /* By default, stop the world. */
  STATIC GC_bool GC_should_start_incremental_collection = FALSE;
#endif

GC_API int GC_CALL GC_is_incremental_mode(void)
{
  return (int)GC_incremental;
}

#ifdef THREADS
  int GC_parallel = FALSE;      /* By default, parallel GC is off.      */
#endif

#if defined(GC_FULL_FREQ) && !defined(CPPCHECK)
  int GC_full_freq = GC_FULL_FREQ;
#else
  int GC_full_freq = 19;   /* Every 20th collection is a full   */
                           /* collection, whether we need it    */
                           /* or not.                           */
#endif

STATIC GC_bool GC_need_full_gc = FALSE;
                           /* Need full GC due to heap growth.  */

#ifdef THREAD_LOCAL_ALLOC
  GC_INNER GC_bool GC_world_stopped = FALSE;
#endif

STATIC GC_bool GC_disable_automatic_collection = FALSE;

GC_API void GC_CALL GC_set_disable_automatic_collection(int value)
{
  LOCK();
  GC_disable_automatic_collection = (GC_bool)value;
  UNLOCK();
}

GC_API int GC_CALL GC_get_disable_automatic_collection(void)
{
  int value;

  READER_LOCK();
  value = (int)GC_disable_automatic_collection;
  READER_UNLOCK();
  return value;
}

STATIC word GC_used_heap_size_after_full = 0;

/* Version macros are now defined in gc_version.h, which is included by */
/* gc.h, which is included by gc_priv.h.                                */
#ifndef GC_NO_VERSION_VAR
  EXTERN_C_BEGIN
  extern const GC_VERSION_VAL_T GC_version;
  EXTERN_C_END

  const GC_VERSION_VAL_T GC_version =
                ((GC_VERSION_VAL_T)GC_VERSION_MAJOR << 16)
                | (GC_VERSION_MINOR << 8) | GC_VERSION_MICRO;
#endif

GC_API GC_VERSION_VAL_T GC_CALL GC_get_version(void)
{
  return ((GC_VERSION_VAL_T)GC_VERSION_MAJOR << 16)
         | (GC_VERSION_MINOR << 8) | GC_VERSION_MICRO;
}

GC_API int GC_CALL GC_get_dont_add_byte_at_end(void)
{
# ifdef DONT_ADD_BYTE_AT_END
    return 1;
# else
    return 0; /* meaningful only if GC_all_interior_pointers */
# endif
}

/* some more variables */

#ifdef GC_DONT_EXPAND
  int GC_dont_expand = TRUE;
#else
  int GC_dont_expand = FALSE;
#endif

#if defined(GC_FREE_SPACE_DIVISOR) && !defined(CPPCHECK)
  word GC_free_space_divisor = GC_FREE_SPACE_DIVISOR; /* must be > 0 */
#else
  word GC_free_space_divisor = 3;
#endif

GC_INNER int GC_CALLBACK GC_never_stop_func(void)
{
  return FALSE;
}

#if defined(GC_TIME_LIMIT) && !defined(CPPCHECK)
  unsigned long GC_time_limit = GC_TIME_LIMIT;
                           /* We try to keep pause times from exceeding  */
                           /* this by much. In milliseconds.             */
#elif defined(PARALLEL_MARK)
  unsigned long GC_time_limit = GC_TIME_UNLIMITED;
                        /* The parallel marker cannot be interrupted for */
                        /* now, so the time limit is absent by default.  */
#else
  unsigned long GC_time_limit = 15;
#endif

#ifndef NO_CLOCK
  STATIC unsigned long GC_time_lim_nsec = 0;
                        /* The nanoseconds add-on to GC_time_limit      */
                        /* value.  Not updated by GC_set_time_limit().  */
                        /* Ignored if the value of GC_time_limit is     */
                        /* GC_TIME_UNLIMITED.                           */

# define TV_NSEC_LIMIT (1000UL * 1000) /* amount of nanoseconds in 1 ms */

  GC_API void GC_CALL GC_set_time_limit_tv(struct GC_timeval_s tv)
  {
    GC_ASSERT(tv.tv_ms <= GC_TIME_UNLIMITED);
    GC_ASSERT(tv.tv_nsec < TV_NSEC_LIMIT);
    GC_time_limit = tv.tv_ms;
    GC_time_lim_nsec = tv.tv_nsec;
  }

  GC_API struct GC_timeval_s GC_CALL GC_get_time_limit_tv(void)
  {
    struct GC_timeval_s tv;

    tv.tv_ms = GC_time_limit;
    tv.tv_nsec = GC_time_lim_nsec;
    return tv;
  }

  STATIC CLOCK_TYPE GC_start_time = CLOCK_TYPE_INITIALIZER;
                                /* Time at which we stopped world.      */
                                /* used only in GC_timeout_stop_func.   */
#endif /* !NO_CLOCK */

STATIC int GC_n_attempts = 0;   /* Number of attempts at finishing      */
                                /* collection within GC_time_limit.     */

STATIC GC_stop_func GC_default_stop_func = GC_never_stop_func;
                                /* Accessed holding the allocator lock. */

GC_API void GC_CALL GC_set_stop_func(GC_stop_func stop_func)
{
  GC_ASSERT(NONNULL_ARG_NOT_NULL(stop_func));
  LOCK();
  GC_default_stop_func = stop_func;
  UNLOCK();
}

GC_API GC_stop_func GC_CALL GC_get_stop_func(void)
{
  GC_stop_func stop_func;

  READER_LOCK();
  stop_func = GC_default_stop_func;
  READER_UNLOCK();
  return stop_func;
}

#if defined(GC_DISABLE_INCREMENTAL) || defined(NO_CLOCK)
# define GC_timeout_stop_func GC_default_stop_func
#else
  STATIC int GC_CALLBACK GC_timeout_stop_func(void)
  {
    CLOCK_TYPE current_time;
    static unsigned count = 0;
    unsigned long time_diff, nsec_diff;

    GC_ASSERT(I_HOLD_LOCK());
    if (GC_default_stop_func())
      return TRUE;

    if (GC_time_limit == GC_TIME_UNLIMITED || (count++ & 3) != 0)
      return FALSE;

    GET_TIME(current_time);
    time_diff = MS_TIME_DIFF(current_time, GC_start_time);
    nsec_diff = NS_FRAC_TIME_DIFF(current_time, GC_start_time);
#   if defined(CPPCHECK)
      GC_noop1_ptr(&nsec_diff);
#   endif
    if (time_diff >= GC_time_limit
        && (time_diff > GC_time_limit || nsec_diff >= GC_time_lim_nsec)) {
      GC_COND_LOG_PRINTF("Abandoning stopped marking after %lu ms %lu ns"
                         " (attempt %d)\n",
                         time_diff, nsec_diff, GC_n_attempts);
      return TRUE;
    }

    return FALSE;
  }
#endif /* !GC_DISABLE_INCREMENTAL */

#ifdef THREADS
  GC_INNER word GC_total_stacksize = 0; /* updated on every push_all_stacks */
#endif

static size_t min_bytes_allocd_minimum = 1;
                        /* The lowest value returned by min_bytes_allocd(). */

GC_API void GC_CALL GC_set_min_bytes_allocd(size_t value)
{
    GC_ASSERT(value > 0);
    min_bytes_allocd_minimum = value;
}

GC_API size_t GC_CALL GC_get_min_bytes_allocd(void)
{
    return min_bytes_allocd_minimum;
}

/* Return the minimum number of bytes that must be allocated between    */
/* collections to amortize the collection cost.  Should be non-zero.    */
static word min_bytes_allocd(void)
{
    word result;
    word stack_size;
    word total_root_size;       /* includes double stack size,  */
                                /* since the stack is expensive */
                                /* to scan.                     */
    word scan_size;             /* Estimate of memory to be scanned     */
                                /* during normal GC.                    */

    GC_ASSERT(I_HOLD_LOCK());
#   ifdef THREADS
      if (GC_need_to_lock) {
        /* We are multi-threaded... */
        stack_size = GC_total_stacksize;
        /* For now, we just use the value computed during the latest GC. */
#       ifdef DEBUG_THREADS
          GC_log_printf("Total stacks size: %lu\n",
                        (unsigned long)stack_size);
#       endif
      } else
#   endif
    /* else*/ {
#     ifdef STACK_NOT_SCANNED
        stack_size = 0;
#     elif defined(STACK_GROWS_UP)
        stack_size = (word)(GC_approx_sp() - GC_stackbottom);
#     else
        stack_size = (word)(GC_stackbottom - GC_approx_sp());
#     endif
    }

    total_root_size = 2 * stack_size + GC_root_size;
    scan_size = 2 * GC_composite_in_use + GC_atomic_in_use / 4
                + total_root_size;
    result = scan_size / GC_free_space_divisor;
    if (GC_incremental) {
      result /= 2;
    }
    return result > min_bytes_allocd_minimum
            ? result : min_bytes_allocd_minimum;
}

STATIC word GC_non_gc_bytes_at_gc = 0;
                /* Number of explicitly managed bytes of storage        */
                /* at last collection.                                  */

/* Return the number of bytes allocated, adjusted for explicit storage  */
/* management, etc.  This number is used in deciding when to trigger    */
/* collections.                                                         */
STATIC word GC_adj_bytes_allocd(void)
{
    signed_word result;
    signed_word expl_managed = (signed_word)GC_non_gc_bytes
                                - (signed_word)GC_non_gc_bytes_at_gc;

    /* Don't count what was explicitly freed, or newly allocated for    */
    /* explicit management.  Note that deallocating an explicitly       */
    /* managed object should not alter result, assuming the client      */
    /* is playing by the rules.                                         */
    result = (signed_word)GC_bytes_allocd
             + (signed_word)GC_bytes_dropped
             - (signed_word)GC_bytes_freed
             + (signed_word)GC_finalizer_bytes_freed
             - expl_managed;
    if (result > (signed_word)GC_bytes_allocd) {
        result = (signed_word)GC_bytes_allocd;
        /* probably client bug or unfortunate scheduling */
    }
    result += (signed_word)GC_bytes_finalized;
        /* We count objects enqueued for finalization as though they    */
        /* had been reallocated this round. Finalization is user        */
        /* visible progress.  And if we don't count this, we have       */
        /* stability problems for programs that finalize all objects.   */
    if (result < (signed_word)(GC_bytes_allocd >> 3)) {
        /* Always count at least 1/8 of the allocations.  We don't want */
        /* to collect too infrequently, since that would inhibit        */
        /* coalescing of free storage blocks.                           */
        /* This also makes us partially robust against client bugs.     */
        result = (signed_word)(GC_bytes_allocd >> 3);
    }
    return (word)result;
}


/* Clear up a few frames worth of garbage left at the top of the stack. */
/* This is used to prevent us from accidentally treating garbage left   */
/* on the stack by other parts of the collector as roots.  This         */
/* differs from the code in misc.c, which actually tries to keep the    */
/* stack clear of long-lived, client-generated garbage.                 */
STATIC void GC_clear_a_few_frames(void)
{
#   ifndef CLEAR_STACK_NPTRS
#     define CLEAR_STACK_NPTRS 64 /* pointers */
#   endif
    volatile ptr_t frames[CLEAR_STACK_NPTRS];

    BZERO(CAST_AWAY_VOLATILE_PVOID(frames), sizeof(frames));
}

GC_API void GC_CALL GC_start_incremental_collection(void)
{
# ifndef GC_DISABLE_INCREMENTAL
    LOCK();
    if (GC_incremental) {
      GC_should_start_incremental_collection = TRUE;
      if (!GC_dont_gc) {
        ENTER_GC();
        GC_collect_a_little_inner(1);
        EXIT_GC();
      }
    }
    UNLOCK();
# endif
}

/* Have we allocated enough to amortize a collection? */
GC_INNER GC_bool GC_should_collect(void)
{
    static word last_min_bytes_allocd;
    static word last_gc_no;

    GC_ASSERT(I_HOLD_LOCK());
    if (last_gc_no != GC_gc_no) {
      last_min_bytes_allocd = min_bytes_allocd();
      last_gc_no = GC_gc_no;
    }
# ifndef GC_DISABLE_INCREMENTAL
    if (GC_should_start_incremental_collection) {
      GC_should_start_incremental_collection = FALSE;
      return TRUE;
    }
# endif
    if (GC_disable_automatic_collection) return FALSE;

    if (GC_last_heap_growth_gc_no == GC_gc_no)
      return TRUE; /* avoid expanding past limits used by blacklisting  */

    return GC_adj_bytes_allocd() >= last_min_bytes_allocd;
}

/* STATIC */ GC_start_callback_proc GC_start_call_back = 0;
                        /* Called at start of full collections.         */
                        /* Not called if 0.  Called with the allocator  */
                        /* lock held.  Not used by GC itself.           */

GC_API void GC_CALL GC_set_start_callback(GC_start_callback_proc fn)
{
    LOCK();
    GC_start_call_back = fn;
    UNLOCK();
}

GC_API GC_start_callback_proc GC_CALL GC_get_start_callback(void)
{
    GC_start_callback_proc fn;

    READER_LOCK();
    fn = GC_start_call_back;
    READER_UNLOCK();
    return fn;
}

GC_INLINE void GC_notify_full_gc(void)
{
    if (GC_start_call_back != 0) {
        (*GC_start_call_back)();
    }
}

STATIC GC_bool GC_is_full_gc = FALSE;

STATIC GC_bool GC_stopped_mark(GC_stop_func stop_func);
STATIC void GC_finish_collection(void);

/* Initiate a garbage collection if appropriate.  Choose judiciously    */
/* between partial, full, and stop-world collections.                   */
STATIC void GC_maybe_gc(void)
{
  static int n_partial_gcs = 0;

  GC_ASSERT(I_HOLD_LOCK());
  ASSERT_CANCEL_DISABLED();
  if (!GC_should_collect()) return;

  if (!GC_incremental) {
    GC_gcollect_inner();
    return;
  }

  GC_ASSERT(!GC_collection_in_progress());
# ifdef PARALLEL_MARK
    if (GC_parallel)
      GC_wait_for_reclaim();
# endif
  if (GC_need_full_gc || n_partial_gcs >= GC_full_freq) {
    GC_COND_LOG_PRINTF(
                "***>Full mark for collection #%lu after %lu allocd bytes\n",
                (unsigned long)GC_gc_no + 1, (unsigned long)GC_bytes_allocd);
    GC_promote_black_lists();
    (void)GC_reclaim_all((GC_stop_func)0, TRUE);
    GC_notify_full_gc();
    GC_clear_marks();
    n_partial_gcs = 0;
    GC_is_full_gc = TRUE;
  } else {
    n_partial_gcs++;
  }

  /* Try to mark with the world stopped.  If we run out of      */
  /* time, this turns into an incremental marking.              */
# ifndef NO_CLOCK
    if (GC_time_limit != GC_TIME_UNLIMITED) GET_TIME(GC_start_time);
# endif
  if (GC_stopped_mark(GC_timeout_stop_func)) {
    SAVE_CALLERS_TO_LAST_STACK();
    GC_finish_collection();
  } else if (!GC_is_full_gc) {
    /* Count this as the first attempt. */
    GC_n_attempts++;
  }
}

STATIC GC_on_collection_event_proc GC_on_collection_event = 0;

GC_API void GC_CALL GC_set_on_collection_event(GC_on_collection_event_proc fn)
{
    /* fn may be 0 (means no event notifier). */
    LOCK();
    GC_on_collection_event = fn;
    UNLOCK();
}

GC_API GC_on_collection_event_proc GC_CALL GC_get_on_collection_event(void)
{
    GC_on_collection_event_proc fn;

    READER_LOCK();
    fn = GC_on_collection_event;
    READER_UNLOCK();
    return fn;
}

/* Stop the world garbage collection.  If stop_func is not      */
/* GC_never_stop_func then abort if stop_func returns TRUE.     */
/* Return TRUE if we successfully completed the collection.     */
GC_INNER GC_bool GC_try_to_collect_inner(GC_stop_func stop_func)
{
#   ifndef NO_CLOCK
      CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER;
      GC_bool start_time_valid;
#   endif

    ASSERT_CANCEL_DISABLED();
    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(GC_is_initialized);
    if (GC_dont_gc || (*stop_func)()) return FALSE;
    if (GC_on_collection_event)
      GC_on_collection_event(GC_EVENT_START);
    if (GC_incremental && GC_collection_in_progress()) {
      GC_COND_LOG_PRINTF(
            "GC_try_to_collect_inner: finishing collection in progress\n");
      /* Just finish collection already in progress.    */
        do {
            if ((*stop_func)()) {
              /* TODO: Notify GC_EVENT_ABANDON */
              return FALSE;
            }
            ENTER_GC();
            GC_collect_a_little_inner(1);
            EXIT_GC();
        } while (GC_collection_in_progress());
    }
    GC_notify_full_gc();
#   ifndef NO_CLOCK
      start_time_valid = FALSE;
      if ((GC_print_stats | (int)measure_performance) != 0) {
        if (GC_print_stats)
          GC_log_printf("Initiating full world-stop collection!\n");
        start_time_valid = TRUE;
        GET_TIME(start_time);
      }
#   endif
    GC_promote_black_lists();
    /* Make sure all blocks have been reclaimed, so sweep routines      */
    /* don't see cleared mark bits.                                     */
    /* If we're guaranteed to finish, then this is unnecessary.         */
    /* In the find_leak case, we have to finish to guarantee that       */
    /* previously unmarked objects are not reported as leaks.           */
#       ifdef PARALLEL_MARK
          if (GC_parallel)
            GC_wait_for_reclaim();
#       endif
        if ((GC_find_leak || stop_func != GC_never_stop_func)
            && !GC_reclaim_all(stop_func, FALSE)) {
            /* Aborted.  So far everything is still consistent. */
            /* TODO: Notify GC_EVENT_ABANDON */
            return FALSE;
        }
    GC_invalidate_mark_state();  /* Flush mark stack.   */
    GC_clear_marks();
    SAVE_CALLERS_TO_LAST_STACK();
    GC_is_full_gc = TRUE;
    if (!GC_stopped_mark(stop_func)) {
      if (!GC_incremental) {
        /* We're partially done and have no way to complete or use      */
        /* current work.  Reestablish invariants as cheaply as          */
        /* possible.                                                    */
        GC_invalidate_mark_state();
        GC_unpromote_black_lists();
      } /* else we claim the world is already still consistent.  We'll  */
        /* finish incrementally.                                        */
      /* TODO: Notify GC_EVENT_ABANDON */
      return FALSE;
    }
    GC_finish_collection();
#   ifndef NO_CLOCK
      if (start_time_valid) {
        CLOCK_TYPE current_time;
        unsigned long time_diff, ns_frac_diff;

        GET_TIME(current_time);
        time_diff = MS_TIME_DIFF(current_time, start_time);
        ns_frac_diff = NS_FRAC_TIME_DIFF(current_time, start_time);
        if (measure_performance) {
          full_gc_total_time += time_diff; /* may wrap */
          full_gc_total_ns_frac += (unsigned32)ns_frac_diff;
          if (full_gc_total_ns_frac >= (unsigned32)1000000UL) {
            /* Overflow of the nanoseconds part. */
            full_gc_total_ns_frac -= (unsigned32)1000000UL;
            full_gc_total_time++;
          }
        }
        if (GC_print_stats)
          GC_log_printf("Complete collection took %lu ms %lu ns\n",
                        time_diff, ns_frac_diff);
      }
#   endif
    if (GC_on_collection_event)
      GC_on_collection_event(GC_EVENT_END);
    return TRUE;
}

/* The number of extra calls to GC_mark_some that we have made. */
STATIC int GC_deficit = 0;

/* The default value of GC_rate.        */
#ifndef GC_RATE
# define GC_RATE 10
#endif

/* When GC_collect_a_little_inner() performs n units of GC work, a unit */
/* is intended to touch roughly GC_rate pages.  (But, every once in     */
/* a while, we do more than that.)  This needs to be a fairly large     */
/* number with our current incremental GC strategy, since otherwise we  */
/* allocate too much during GC, and the cleanup gets expensive.         */
STATIC int GC_rate = GC_RATE;

GC_API void GC_CALL GC_set_rate(int value)
{
    GC_ASSERT(value > 0);
    GC_rate = value;
}

GC_API int GC_CALL GC_get_rate(void)
{
    return GC_rate;
}

/* The default maximum number of prior attempts at world stop marking.  */
#ifndef MAX_PRIOR_ATTEMPTS
# define MAX_PRIOR_ATTEMPTS 3
#endif

/* The maximum number of prior attempts at world stop marking.          */
/* A value of 1 means that we finish the second time, no matter how     */
/* long it takes.  Does not count the initial root scan for a full GC.  */
static int max_prior_attempts = MAX_PRIOR_ATTEMPTS;

GC_API void GC_CALL GC_set_max_prior_attempts(int value)
{
    GC_ASSERT(value >= 0);
    max_prior_attempts = value;
}

GC_API int GC_CALL GC_get_max_prior_attempts(void)
{
    return max_prior_attempts;
}

GC_INNER void GC_collect_a_little_inner(int n)
{
    IF_CANCEL(int cancel_state;)

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(GC_is_initialized);
    DISABLE_CANCEL(cancel_state);
    if (GC_incremental && GC_collection_in_progress()) {
        int i;
        int max_deficit = GC_rate * n;

#       ifdef PARALLEL_MARK
            if (GC_time_limit != GC_TIME_UNLIMITED)
                GC_parallel_mark_disabled = TRUE;
#       endif
        for (i = GC_deficit; i < max_deficit; i++) {
            if (GC_mark_some(NULL))
                break;
        }
#       ifdef PARALLEL_MARK
            GC_parallel_mark_disabled = FALSE;
#       endif

        if (i < max_deficit && !GC_dont_gc) {
            GC_ASSERT(!GC_collection_in_progress());
            /* Need to follow up with a full collection.        */
            SAVE_CALLERS_TO_LAST_STACK();
#           ifdef PARALLEL_MARK
                if (GC_parallel)
                    GC_wait_for_reclaim();
#           endif
#           ifndef NO_CLOCK
                if (GC_time_limit != GC_TIME_UNLIMITED
                        && GC_n_attempts < max_prior_attempts)
                    GET_TIME(GC_start_time);
#           endif
            if (GC_stopped_mark(GC_n_attempts < max_prior_attempts ?
                                GC_timeout_stop_func : GC_never_stop_func)) {
                GC_finish_collection();
            } else {
                GC_n_attempts++;
            }
        }
        if (GC_deficit > 0) {
            GC_deficit -= max_deficit;
            if (GC_deficit < 0)
                GC_deficit = 0;
        }
    } else if (!GC_dont_gc) {
        GC_maybe_gc();
    }
    RESTORE_CANCEL(cancel_state);
}

GC_INNER void (*GC_check_heap)(void) = 0;
GC_INNER void (*GC_print_all_smashed)(void) = 0;

GC_API int GC_CALL GC_collect_a_little(void)
{
    int result;

    if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
    LOCK();
    ENTER_GC();
    /* Note: if the collection is in progress, this may do marking (not */
    /* stopping the world) even in case of disabled GC.                 */
    GC_collect_a_little_inner(1);
    EXIT_GC();
    result = (int)GC_collection_in_progress();
    UNLOCK();
    if (!result && GC_debugging_started) GC_print_all_smashed();
    return result;
}

#ifdef THREADS
  GC_API void GC_CALL GC_stop_world_external(void)
  {
    GC_ASSERT(GC_is_initialized);
    LOCK();
#   ifdef THREAD_LOCAL_ALLOC
      GC_ASSERT(!GC_world_stopped);
#   endif
    STOP_WORLD();
#   ifdef THREAD_LOCAL_ALLOC
      GC_world_stopped = TRUE;
#   endif
  }

  GC_API void GC_CALL GC_start_world_external(void)
  {
#   ifdef THREAD_LOCAL_ALLOC
      GC_ASSERT(GC_world_stopped);
      GC_world_stopped = FALSE;
#   else
      GC_ASSERT(GC_is_initialized);
#   endif
    START_WORLD();
    UNLOCK();
  }
#endif /* THREADS */

#ifndef NO_CLOCK
  /* Variables for world-stop average delay time statistic computation. */
  /* "divisor" is incremented every world-stop and halved when reached  */
  /* its maximum (or upon "total_time" overflow).                       */
  static unsigned world_stopped_total_time = 0;
  static unsigned world_stopped_total_divisor = 0;
# ifndef MAX_TOTAL_TIME_DIVISOR
    /* We shall not use big values here (so "outdated" delay time       */
    /* values would have less impact on "average" delay time value than */
    /* newer ones).                                                     */
#   define MAX_TOTAL_TIME_DIVISOR 1000
# endif
#endif /* !NO_CLOCK */

#ifdef USE_MUNMAP
# ifndef MUNMAP_THRESHOLD
#   define MUNMAP_THRESHOLD 7
# endif
  GC_INNER unsigned GC_unmap_threshold = MUNMAP_THRESHOLD;

# define IF_USE_MUNMAP(x) x
# define COMMA_IF_USE_MUNMAP(x) /* comma */, x
#else
# define IF_USE_MUNMAP(x) /* empty */
# define COMMA_IF_USE_MUNMAP(x) /* empty */
#endif

/* We stop the world and mark from all roots.  If stop_func() ever      */
/* returns TRUE, we may fail and return FALSE.  Increment GC_gc_no if   */
/* we succeed.                                                          */
STATIC GC_bool GC_stopped_mark(GC_stop_func stop_func)
{
    int abandoned_at;
    ptr_t cold_gc_frame = GC_approx_sp();
#   ifndef NO_CLOCK
      CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER;
      GC_bool start_time_valid = FALSE;
#   endif

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(GC_is_initialized);
#   if !defined(REDIRECT_MALLOC) && defined(USE_WINALLOC)
        GC_add_current_malloc_heap();
#   endif
#   if defined(REGISTER_LIBRARIES_EARLY)
        GC_cond_register_dynamic_libraries();
#   endif

#   if !defined(GC_NO_FINALIZATION) && !defined(GC_TOGGLE_REFS_NOT_NEEDED)
      GC_process_togglerefs();
#   endif

        /* Output blank line for convenience here.      */
    GC_COND_LOG_PRINTF(
              "\n--> Marking for collection #%lu after %lu allocated bytes\n",
              (unsigned long)GC_gc_no + 1, (unsigned long)GC_bytes_allocd);
#   ifndef NO_CLOCK
      if (GC_PRINT_STATS_FLAG || measure_performance) {
        GET_TIME(start_time);
        start_time_valid = TRUE;
      }
#   endif
#   ifdef THREADS
      if (GC_on_collection_event)
        GC_on_collection_event(GC_EVENT_PRE_STOP_WORLD);
#   endif
    STOP_WORLD();
#   ifdef THREADS
      if (GC_on_collection_event)
        GC_on_collection_event(GC_EVENT_POST_STOP_WORLD);
#     ifdef THREAD_LOCAL_ALLOC
        GC_world_stopped = TRUE;
#     elif defined(CPPCHECK)
        (void)0; /* workaround a warning about adjacent same "if" condition */
#     endif
#   endif

#   ifdef MAKE_BACK_GRAPH
      if (GC_print_back_height) {
        GC_build_back_graph();
      }
#   endif

    /* Notify about marking from all roots.     */
        if (GC_on_collection_event)
          GC_on_collection_event(GC_EVENT_MARK_START);

    /* Minimize junk left in my registers and on the stack.     */
            GC_clear_a_few_frames();
            GC_noop6(0,0,0,0,0,0);

        GC_initiate_gc();
#       ifdef PARALLEL_MARK
          if (stop_func != GC_never_stop_func)
            GC_parallel_mark_disabled = TRUE;
#       endif
        for (abandoned_at = 0; !(*stop_func)(); abandoned_at++) {
          if (GC_mark_some(cold_gc_frame)) {
#           ifdef PARALLEL_MARK
              if (GC_parallel && GC_parallel_mark_disabled) {
                GC_COND_LOG_PRINTF("Stopped marking done after %d iterations"
                                   " with disabled parallel marker\n",
                                   abandoned_at);
              }
#           endif
            abandoned_at = -1;
            break;
          }
        }
#       ifdef PARALLEL_MARK
          GC_parallel_mark_disabled = FALSE;
#       endif

    if (abandoned_at >= 0) {
      GC_deficit = abandoned_at; /* Give the mutator a chance. */
      /* TODO: Notify GC_EVENT_MARK_ABANDON */
    } else {
      GC_gc_no++;
      /* Check all debugged objects for consistency.    */
      if (GC_debugging_started) {
        (*GC_check_heap)();
      }
      if (GC_on_collection_event)
        GC_on_collection_event(GC_EVENT_MARK_END);
    }

#   ifdef THREADS
      if (GC_on_collection_event)
        GC_on_collection_event(GC_EVENT_PRE_START_WORLD);
#   endif
#   ifdef THREAD_LOCAL_ALLOC
      GC_world_stopped = FALSE;
#   endif
    START_WORLD();
#   ifdef THREADS
      if (GC_on_collection_event)
        GC_on_collection_event(GC_EVENT_POST_START_WORLD);
#   endif

#   ifndef NO_CLOCK
      if (start_time_valid) {
        CLOCK_TYPE current_time;
        unsigned long time_diff, ns_frac_diff;

        /* TODO: Avoid code duplication from GC_try_to_collect_inner */
        GET_TIME(current_time);
        time_diff = MS_TIME_DIFF(current_time, start_time);
        ns_frac_diff = NS_FRAC_TIME_DIFF(current_time, start_time);
        if (measure_performance) {
          stopped_mark_total_time += time_diff; /* may wrap */
          stopped_mark_total_ns_frac += (unsigned32)ns_frac_diff;
          if (stopped_mark_total_ns_frac >= (unsigned32)1000000UL) {
            stopped_mark_total_ns_frac -= (unsigned32)1000000UL;
            stopped_mark_total_time++;
          }
        }

        if (GC_PRINT_STATS_FLAG) {
          unsigned total_time = world_stopped_total_time;
          unsigned divisor = world_stopped_total_divisor;

          /* Compute new world-stop delay total time.   */
          if (total_time > (((unsigned)-1) >> 1)
              || divisor >= MAX_TOTAL_TIME_DIVISOR) {
            /* Halve values if overflow occurs. */
            total_time >>= 1;
            divisor >>= 1;
          }
          total_time += time_diff < (((unsigned)-1) >> 1) ?
                        (unsigned)time_diff : ((unsigned)-1) >> 1;
          /* Update old world_stopped_total_time and its divisor.   */
          world_stopped_total_time = total_time;
          world_stopped_total_divisor = ++divisor;
          if (abandoned_at < 0) {
            GC_ASSERT(divisor != 0);
            GC_log_printf("World-stopped marking took %lu ms %lu ns"
                          " (%u ms in average)\n", time_diff, ns_frac_diff,
                          total_time / divisor);
          }
        }
      }
#   endif

    if (abandoned_at >= 0) {
      GC_COND_LOG_PRINTF("Abandoned stopped marking after %d iterations\n",
                         abandoned_at);
      return FALSE;
    }
    return TRUE;
}

GC_INNER void GC_set_fl_marks(ptr_t q)
{
#   ifdef GC_ASSERTIONS
        ptr_t q2;
#   endif
    struct hblk *h = HBLKPTR(q);
    const struct hblk *last_h = h;
    hdr *hhdr;
#   ifdef MARK_BIT_PER_OBJ
        size_t sz;
#   endif

    GC_ASSERT(q != NULL);
    hhdr = HDR(h);
#   ifdef MARK_BIT_PER_OBJ
        sz = hhdr -> hb_sz;
#   endif
#   ifdef GC_ASSERTIONS
        q2 = (ptr_t)obj_link(q);
#   endif
    for (;;) {
        size_t bit_no = MARK_BIT_NO((size_t)((ptr_t)q - (ptr_t)h), sz);

        if (!mark_bit_from_hdr(hhdr, bit_no)) {
          set_mark_bit_from_hdr(hhdr, bit_no);
          INCR_MARKS(hhdr);
        }
        q = (ptr_t)obj_link(q);
        if (NULL == q) break;
#       ifdef GC_ASSERTIONS
          /* Detect a cycle in the free list.  The algorithm is to  */
          /* have a second "twice faster" iterator over the list -  */
          /* the second iterator meets the first one in case of     */
          /* a cycle existing in the list.                          */
          if (q2 != NULL) {
            q2 = (ptr_t)obj_link(q2);
            GC_ASSERT(q2 != q);
            if (q2 != NULL) {
              q2 = (ptr_t)obj_link(q2);
              GC_ASSERT(q2 != q);
            }
          }
#       endif

        h = HBLKPTR(q);
        if (EXPECT(h != last_h, FALSE)) {
          last_h = h;
          /* Update hhdr and sz. */
          hhdr = HDR(h);
#         ifdef MARK_BIT_PER_OBJ
            sz = hhdr -> hb_sz;
#         endif
        }
    }
}

#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
  /* Check that all mark bits for the free list whose first entry is    */
  /* (*pfreelist) are set.  Check skipped if points to a special value. */
  void GC_check_fl_marks(void **pfreelist)
  {
    /* TODO: There is a data race with GC_FAST_MALLOC_GRANS (which does */
    /* not do atomic updates to the free-list).  The race seems to be   */
    /* harmless, and for now we just skip this check in case of TSan.   */
#   if defined(AO_HAVE_load_acquire_read) && !defined(THREAD_SANITIZER)
      ptr_t list = GC_cptr_load_acquire_read((volatile ptr_t *)pfreelist);
                /* Atomic operations are used because the world is running. */
      ptr_t p, prev, next;

      if (ADDR(list) <= HBLKSIZE) return;

      prev = (ptr_t)pfreelist;
      for (p = list; p != NULL; p = next) {
        if (!GC_is_marked(p)) {
          ABORT_ARG2("Unmarked local free-list entry",
                     ": object %p on list %p", (void *)p, (void *)list);
        }

        /* While traversing the free-list, it re-reads the pointer to   */
        /* the current node before accepting its next pointer and       */
        /* bails out if the latter has changed.  That way, it won't     */
        /* try to follow the pointer which might be been modified       */
        /* after the object was returned to the client.  It might       */
        /* perform the mark-check on the just allocated object but      */
        /* that should be harmless.                                     */
        next = GC_cptr_load_acquire_read((volatile ptr_t *)p);
        if (GC_cptr_load((volatile ptr_t *)prev) != p)
          break;
        prev = p;
      }
#   else
      /* FIXME: Not implemented (just skipped). */
      (void)pfreelist;
#   endif
  }
#endif /* GC_ASSERTIONS && THREAD_LOCAL_ALLOC */

/* Clear all mark bits for the free list (specified by the first        */
/* entry).  Decrement GC_bytes_found by number of bytes on free list.   */
STATIC void GC_clear_fl_marks(ptr_t q)
{
      struct hblk *h = HBLKPTR(q);
      const struct hblk *last_h = h;
      hdr *hhdr = HDR(h);
      size_t sz = hhdr -> hb_sz; /* Normally set only once. */

      for (;;) {
        size_t bit_no = MARK_BIT_NO((size_t)((ptr_t)q - (ptr_t)h), sz);

        if (mark_bit_from_hdr(hhdr, bit_no)) {
          size_t n_marks = hhdr -> hb_n_marks;

          GC_ASSERT(n_marks != 0);
          clear_mark_bit_from_hdr(hhdr, bit_no);
          n_marks--;
#         ifdef PARALLEL_MARK
            /* Appr. count, don't decrement to zero! */
            if (0 != n_marks || !GC_parallel) {
              hhdr -> hb_n_marks = n_marks;
            }
#         else
            hhdr -> hb_n_marks = n_marks;
#         endif
        }
        GC_bytes_found -= (signed_word)sz;

        q = (ptr_t)obj_link(q);
        if (NULL == q) break;

        h = HBLKPTR(q);
        if (EXPECT(h != last_h, FALSE)) {
          last_h = h;
          /* Update hhdr and sz. */
          hhdr = HDR(h);
          sz = hhdr -> hb_sz;
        }
      }
}

#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
  void GC_check_tls(void);
#endif

GC_on_heap_resize_proc GC_on_heap_resize = 0;

/* Used for logging only. */
GC_INLINE int GC_compute_heap_usage_percent(void)
{
  word used = GC_composite_in_use + GC_atomic_in_use + GC_bytes_allocd;
  word heap_sz = GC_heapsize - GC_unmapped_bytes;
# if defined(CPPCHECK)
    word limit = (GC_WORD_MAX >> 1) / 50; /* to avoid a false positive */
# else
    const word limit = GC_WORD_MAX / 100;
# endif

  return used >= heap_sz ? 0 : used < limit ?
                (int)((used * 100) / heap_sz) : (int)(used / (heap_sz / 100));
}

#define GC_DBGLOG_PRINT_HEAP_IN_USE() \
  GC_DBGLOG_PRINTF("In-use heap: %d%% (%lu KiB pointers + %lu KiB other)\n", \
                   GC_compute_heap_usage_percent(), \
                   TO_KiB_UL(GC_composite_in_use), \
                   TO_KiB_UL(GC_atomic_in_use + GC_bytes_allocd))

/* Finish up a collection.  Assumes mark bits are consistent, but the   */
/* world is otherwise running.                                          */
STATIC void GC_finish_collection(void)
{
#   ifndef NO_CLOCK
      CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER;
      CLOCK_TYPE finalize_time = CLOCK_TYPE_INITIALIZER;
#   endif

    GC_ASSERT(I_HOLD_LOCK());
#   if defined(GC_ASSERTIONS) \
       && defined(THREAD_LOCAL_ALLOC) && !defined(DBG_HDRS_ALL)
        /* Check that we marked some of our own data.           */
        /* TODO: Add more checks. */
        GC_check_tls();
#   endif

#   ifndef NO_CLOCK
      if (GC_print_stats)
        GET_TIME(start_time);
#   endif
    if (GC_on_collection_event)
      GC_on_collection_event(GC_EVENT_RECLAIM_START);

#   ifndef GC_GET_HEAP_USAGE_NOT_NEEDED
      if (GC_bytes_found > 0)
        GC_reclaimed_bytes_before_gc += (word)GC_bytes_found;
#   endif
    GC_bytes_found = 0;
#   if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG)
        if (GETENV("GC_PRINT_ADDRESS_MAP") != 0) {
          GC_print_address_map();
        }
#   endif
    COND_DUMP;
    if (GC_find_leak) {
      /* Mark all objects on the free list.  All objects should be      */
      /* marked when we're done.                                        */
      unsigned kind;

      for (kind = 0; kind < GC_n_kinds; kind++) {
        word size;  /* current object size */

        for (size = 1; size <= MAXOBJGRANULES; size++) {
          ptr_t q = (ptr_t)GC_obj_kinds[kind].ok_freelist[size];

          if (q != NULL)
            GC_set_fl_marks(q);
        }
      }
      GC_start_reclaim(TRUE);
        /* The above just checks; it doesn't really reclaim anything.   */
    }

#   ifndef GC_NO_FINALIZATION
      GC_finalize();
#   endif
#   ifndef NO_CLOCK
      if (GC_print_stats)
        GET_TIME(finalize_time);
#   endif

    if (GC_print_back_height) {
#     ifdef MAKE_BACK_GRAPH
        GC_traverse_back_graph();
#     elif !defined(SMALL_CONFIG)
        GC_err_printf("Back height not available: "
                      "Rebuild collector with -DMAKE_BACK_GRAPH\n");
#     endif
    }

    /* Clear free-list mark bits, in case they got accidentally marked   */
    /* (or GC_find_leak is set and they were intentionally marked).      */
    /* Also subtract memory remaining from GC_bytes_found count.         */
    /* Note that composite objects on free list are cleared.             */
    /* Thus accidentally marking a free list is not a problem;  only     */
    /* objects on the list itself will be marked, and that's fixed here. */
    {
      unsigned kind;

      for (kind = 0; kind < GC_n_kinds; kind++) {
        word size;  /* current object size */

        for (size = 1; size <= MAXOBJGRANULES; size++) {
          ptr_t q = (ptr_t)GC_obj_kinds[kind].ok_freelist[size];

          if (q != NULL)
            GC_clear_fl_marks(q);
        }
      }
    }

    GC_VERBOSE_LOG_PRINTF("Bytes recovered before sweep - f.l. count = %ld\n",
                          (long)GC_bytes_found);

    /* Reconstruct free lists to contain everything not marked */
    GC_start_reclaim(FALSE);

#   ifdef USE_MUNMAP
      if (GC_unmap_threshold > 0 /* unmapping enabled? */
          && EXPECT(GC_gc_no != 1, TRUE)) /* do not unmap during GC init */
        GC_unmap_old(GC_unmap_threshold);

      GC_ASSERT(GC_heapsize >= GC_unmapped_bytes);
#   endif
    GC_ASSERT(GC_our_mem_bytes >= GC_heapsize);
    GC_DBGLOG_PRINTF("GC #%lu freed %ld bytes, heap %lu KiB ("
                     IF_USE_MUNMAP("+ %lu KiB unmapped ")
                     "+ %lu KiB internal)\n",
                     (unsigned long)GC_gc_no, (long)GC_bytes_found,
                     TO_KiB_UL(GC_heapsize - GC_unmapped_bytes) /*, */
                     COMMA_IF_USE_MUNMAP(TO_KiB_UL(GC_unmapped_bytes)),
                     TO_KiB_UL(GC_our_mem_bytes - GC_heapsize
                               + sizeof(GC_arrays)));
    GC_DBGLOG_PRINT_HEAP_IN_USE();
    if (GC_is_full_gc) {
        GC_used_heap_size_after_full = GC_heapsize - GC_large_free_bytes;
        GC_need_full_gc = FALSE;
    } else {
        GC_need_full_gc = GC_heapsize - GC_used_heap_size_after_full
                          > min_bytes_allocd() + GC_large_free_bytes;
    }

    /* Reset or increment counters for next cycle */
    GC_n_attempts = 0;
    GC_is_full_gc = FALSE;
    GC_bytes_allocd_before_gc += GC_bytes_allocd;
    GC_non_gc_bytes_at_gc = GC_non_gc_bytes;
    GC_bytes_allocd = 0;
    GC_bytes_dropped = 0;
    GC_bytes_freed = 0;
    GC_finalizer_bytes_freed = 0;

    if (GC_on_collection_event)
      GC_on_collection_event(GC_EVENT_RECLAIM_END);
#   ifndef NO_CLOCK
      if (GC_print_stats) {
        CLOCK_TYPE done_time;

        GET_TIME(done_time);
#       if !defined(SMALL_CONFIG) && !defined(GC_NO_FINALIZATION)
          /* A convenient place to output finalization statistics.      */
          GC_print_finalization_stats();
#       endif
        GC_log_printf("Finalize and initiate sweep took %lu ms %lu ns"
                      " + %lu ms %lu ns\n",
                      MS_TIME_DIFF(finalize_time, start_time),
                      NS_FRAC_TIME_DIFF(finalize_time, start_time),
                      MS_TIME_DIFF(done_time, finalize_time),
                      NS_FRAC_TIME_DIFF(done_time, finalize_time));
      }
#   elif !defined(SMALL_CONFIG) && !defined(GC_NO_FINALIZATION)
      if (GC_print_stats)
        GC_print_finalization_stats();
#   endif
}

STATIC word GC_heapsize_at_forced_unmap = 0;
                                /* accessed with the allocator lock held */

/* If stop_func == 0 then GC_default_stop_func is used instead.         */
STATIC GC_bool GC_try_to_collect_general(GC_stop_func stop_func,
                                         GC_bool force_unmap)
{
    GC_bool result;
    IF_USE_MUNMAP(unsigned old_unmap_threshold;)
    IF_CANCEL(int cancel_state;)

    if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
    if (GC_debugging_started) GC_print_all_smashed();
    GC_INVOKE_FINALIZERS();
    LOCK();
    if (force_unmap) {
      /* Record current heap size to make heap growth more conservative */
      /* afterwards (as if the heap is growing from zero size again).   */
      GC_heapsize_at_forced_unmap = GC_heapsize;
    }
    DISABLE_CANCEL(cancel_state);
#   ifdef USE_MUNMAP
      old_unmap_threshold = GC_unmap_threshold;
      if (force_unmap ||
          (GC_force_unmap_on_gcollect && old_unmap_threshold > 0))
        GC_unmap_threshold = 1; /* unmap as much as possible */
#   endif
    ENTER_GC();
    /* Minimize junk left in my registers */
      GC_noop6(0,0,0,0,0,0);
    result = GC_try_to_collect_inner(stop_func != 0 ? stop_func :
                                     GC_default_stop_func);
    EXIT_GC();
    IF_USE_MUNMAP(GC_unmap_threshold = old_unmap_threshold); /* restore */
    RESTORE_CANCEL(cancel_state);
    UNLOCK();
    if (result) {
        if (GC_debugging_started) GC_print_all_smashed();
        GC_INVOKE_FINALIZERS();
    }
    return result;
}

/* Externally callable routines to invoke full, stop-the-world collection. */

GC_API int GC_CALL GC_try_to_collect(GC_stop_func stop_func)
{
    GC_ASSERT(NONNULL_ARG_NOT_NULL(stop_func));
    return (int)GC_try_to_collect_general(stop_func, FALSE);
}

GC_API void GC_CALL GC_gcollect(void)
{
    /* Zero is passed as stop_func to get GC_default_stop_func value    */
    /* while holding the allocator lock (to prevent data race).         */
    (void)GC_try_to_collect_general(0, FALSE);
    if (get_have_errors())
      GC_print_all_errors();
}

GC_API void GC_CALL GC_gcollect_and_unmap(void)
{
    /* Collect and force memory unmapping to OS. */
    (void)GC_try_to_collect_general(GC_never_stop_func, TRUE);
}

GC_INNER ptr_t GC_os_get_mem(size_t bytes)
{
  struct hblk *space = GET_MEM(bytes); /* HBLKSIZE-aligned */

  GC_ASSERT(I_HOLD_LOCK());
  if (EXPECT(NULL == space, FALSE)) return NULL;
# ifdef USE_PROC_FOR_LIBRARIES
    /* Add HBLKSIZE aligned, GET_MEM-generated block to GC_our_memory. */
    if (GC_n_memory >= MAX_HEAP_SECTS)
      ABORT("Too many GC-allocated memory sections: Increase MAX_HEAP_SECTS");
    GC_our_memory[GC_n_memory].hs_start = (ptr_t)space;
    GC_our_memory[GC_n_memory].hs_bytes = bytes;
    GC_n_memory++;
# endif
  GC_our_mem_bytes += bytes;
  GC_VERBOSE_LOG_PRINTF("Got %lu bytes from OS\n", (unsigned long)bytes);
  return (ptr_t)space;
}

/* Use the chunk of memory starting at h of size sz as part of the      */
/* heap.  Assumes h is HBLKSIZE aligned, sz is a multiple of HBLKSIZE.  */
STATIC void GC_add_to_heap(struct hblk *h, size_t sz)
{
    hdr *hhdr;
    ptr_t endp;
    size_t old_capacity = 0;
    void *old_heap_sects = NULL;
#   ifdef GC_ASSERTIONS
      size_t i;
#   endif

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(ADDR(h) % HBLKSIZE == 0);
    GC_ASSERT(sz % HBLKSIZE == 0);
    GC_ASSERT(sz > 0);
    GC_ASSERT(GC_all_nils != NULL);

    if (EXPECT(GC_n_heap_sects == GC_capacity_heap_sects, FALSE)) {
      /* Allocate new GC_heap_sects with sufficient capacity.   */
#     ifndef INITIAL_HEAP_SECTS
#       define INITIAL_HEAP_SECTS 32
#     endif
      size_t new_capacity = GC_n_heap_sects > 0
                                ? GC_n_heap_sects * 2 : INITIAL_HEAP_SECTS;
      void *new_heap_sects =
                GC_scratch_alloc(new_capacity * sizeof(struct HeapSect));

      if (NULL == new_heap_sects) {
        /* Retry with smaller yet sufficient capacity.  */
        new_capacity = GC_n_heap_sects + INITIAL_HEAP_SECTS;
        new_heap_sects =
                GC_scratch_alloc(new_capacity * sizeof(struct HeapSect));
        if (NULL == new_heap_sects)
          ABORT("Insufficient memory for heap sections");
      }
      old_capacity = GC_capacity_heap_sects;
      old_heap_sects = GC_heap_sects;
      /* Transfer GC_heap_sects contents to the newly allocated array.  */
      if (GC_n_heap_sects > 0)
        BCOPY(old_heap_sects, new_heap_sects,
              GC_n_heap_sects * sizeof(struct HeapSect));
      GC_capacity_heap_sects = new_capacity;
      GC_heap_sects = (struct HeapSect *)new_heap_sects;
      GC_COND_LOG_PRINTF("Grew heap sections array to %lu elements\n",
                         (unsigned long)new_capacity);
    }

    while (EXPECT(ADDR(h) <= HBLKSIZE, FALSE)) {
        /* Can't handle memory near address zero. */
        ++h;
        sz -= HBLKSIZE;
        if (0 == sz) return;
    }
    while (EXPECT(ADDR(h) >= GC_WORD_MAX - sz, FALSE)) {
        /* Prevent overflow when calculating endp.  */
        sz -= HBLKSIZE;
        if (0 == sz) return;
    }
    endp = (ptr_t)h + sz;

    hhdr = GC_install_header(h);
    if (EXPECT(NULL == hhdr, FALSE)) {
        /* This is extremely unlikely. Can't add it.  This will         */
        /* almost certainly result in a 0 return from the allocator,    */
        /* which is entirely appropriate.                               */
        return;
    }
#   ifdef GC_ASSERTIONS
      /* Ensure no intersection between sections.       */
      for (i = 0; i < GC_n_heap_sects; i++) {
        ptr_t hs_start = GC_heap_sects[i].hs_start;
        ptr_t hs_end = hs_start + GC_heap_sects[i].hs_bytes;

        GC_ASSERT(!(ADDR_INSIDE((ptr_t)h, hs_start, hs_end)
                    || (ADDR_LT(hs_start, endp) && ADDR_GE(hs_end, endp))
                    || (ADDR_LT((ptr_t)h, hs_start)
                            && ADDR_LT(hs_end, endp))));
      }
#   endif
    GC_heap_sects[GC_n_heap_sects].hs_start = (ptr_t)h;
    GC_heap_sects[GC_n_heap_sects].hs_bytes = sz;
    GC_n_heap_sects++;
    hhdr -> hb_sz = sz;
    hhdr -> hb_flags = 0;
    GC_freehblk(h);
    GC_heapsize += sz;

    if (ADDR_GE((ptr_t)GC_least_plausible_heap_addr, (ptr_t)h)
        || EXPECT(NULL == GC_least_plausible_heap_addr, FALSE)) {
        GC_least_plausible_heap_addr = (ptr_t)h - sizeof(ptr_t);
                /* Making it a little smaller than necessary prevents   */
                /* us from getting a false hit from the variable        */
                /* itself.  There's some unintentional reflection       */
                /* here.                                                */
    }
    if (ADDR_LT((ptr_t)GC_greatest_plausible_heap_addr, endp)) {
        GC_greatest_plausible_heap_addr = endp;
    }
#   ifdef SET_REAL_HEAP_BOUNDS
      if (ADDR(h) < GC_least_real_heap_addr
          || EXPECT(0 == GC_least_real_heap_addr, FALSE))
        GC_least_real_heap_addr = ADDR(h) - sizeof(ptr_t);
      if (GC_greatest_real_heap_addr < ADDR(endp)) {
#       ifdef INCLUDE_LINUX_THREAD_DESCR
          /* Avoid heap intersection with the static data roots. */
          GC_exclude_static_roots_inner((ptr_t)h, endp);
#       endif
        GC_greatest_real_heap_addr = ADDR(endp);
      }
#   endif
    GC_handle_protected_regions_limit();
    if (EXPECT(old_capacity > 0, FALSE)) {
#     ifndef GWW_VDB
        /* Recycling may call GC_add_to_heap() again but should not     */
        /* cause resizing of GC_heap_sects.                             */
        GC_scratch_recycle_no_gww(old_heap_sects,
                                  old_capacity * sizeof(struct HeapSect));
#     else
        /* TODO: implement GWW-aware recycling as in alloc_mark_stack */
        GC_noop1_ptr(old_heap_sects);
#     endif
    }
}

#if !defined(NO_DEBUGGING)
  void GC_print_heap_sects(void)
  {
    size_t i;

    GC_printf("Total heap size: %lu" IF_USE_MUNMAP(" (%lu unmapped)") "\n",
              (unsigned long)GC_heapsize /*, */
              COMMA_IF_USE_MUNMAP((unsigned long)GC_unmapped_bytes));

    for (i = 0; i < GC_n_heap_sects; i++) {
      ptr_t start = GC_heap_sects[i].hs_start;
      size_t len = GC_heap_sects[i].hs_bytes;
      struct hblk *h;
      unsigned nbl = 0;

      for (h = (struct hblk *)start; ADDR_LT((ptr_t)h, start + len); h++) {
        if (GC_is_black_listed(h, HBLKSIZE)) nbl++;
      }
      GC_printf("Section %u from %p to %p %u/%lu blacklisted\n",
                (unsigned)i, (void *)start, (void *)&start[len],
                nbl, (unsigned long)divHBLKSZ(len));
    }
  }
#endif

void * GC_least_plausible_heap_addr = MAKE_CPTR(GC_WORD_MAX);
void * GC_greatest_plausible_heap_addr = NULL;

STATIC word GC_max_heapsize = 0;

GC_API void GC_CALL GC_set_max_heap_size(GC_word n)
{
    GC_max_heapsize = n;
}

word GC_max_retries = 0;

GC_INNER void GC_scratch_recycle_inner(void *ptr, size_t sz)
{
  size_t page_offset;
  size_t displ = 0;
  size_t recycled_bytes;

  GC_ASSERT(I_HOLD_LOCK());
  if (NULL == ptr) return;

  GC_ASSERT(sz != 0);
  GC_ASSERT(GC_page_size != 0);
  /* TODO: Assert correct memory flags if GWW_VDB */
  page_offset = ADDR(ptr) & (GC_page_size-1);
  if (page_offset != 0)
    displ = GC_page_size - page_offset;
  recycled_bytes = sz > displ ? (sz - displ) & ~(GC_page_size - 1) : 0;
  GC_COND_LOG_PRINTF("Recycle %lu/%lu scratch-allocated bytes at %p\n",
                (unsigned long)recycled_bytes, (unsigned long)sz, ptr);
  if (recycled_bytes > 0)
    GC_add_to_heap((struct hblk *)((ptr_t)ptr + displ), recycled_bytes);
}

/* This explicitly increases the size of the heap.  It is used          */
/* internally, but may also be invoked from GC_expand_hp by the user.   */
/* The argument is in units of HBLKSIZE (zero is treated as 1).         */
/* Returns FALSE on failure.                                            */
GC_INNER GC_bool GC_expand_hp_inner(word n)
{
    size_t sz;
    struct hblk * space;
    word expansion_slop;        /* Number of bytes by which we expect   */
                                /* the heap to expand soon.             */

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(GC_page_size != 0);
    if (0 == n) n = 1;
    sz = ROUNDUP_PAGESIZE((size_t)n * HBLKSIZE);
    GC_DBGLOG_PRINT_HEAP_IN_USE();
    if (GC_max_heapsize != 0
        && (GC_max_heapsize < (word)sz
            || GC_heapsize > GC_max_heapsize - (word)sz)) {
        /* Exceeded self-imposed limit */
        return FALSE;
    }
    space = (struct hblk *)GC_os_get_mem(sz);
    if (EXPECT(NULL == space, FALSE)) {
        WARN("Failed to expand heap by %" WARN_PRIuPTR " KiB\n", sz >> 10);
        return FALSE;
    }
    GC_last_heap_growth_gc_no = GC_gc_no;
    GC_INFOLOG_PRINTF("Grow heap to %lu KiB after %lu bytes allocated\n",
                      TO_KiB_UL(GC_heapsize + sz),
                      (unsigned long)GC_bytes_allocd);

    /* Adjust heap limits generously for blacklisting to work better.   */
    /* GC_add_to_heap performs minimal adjustment needed for            */
    /* correctness.                                                     */
    expansion_slop = min_bytes_allocd() + 4 * MAXHINCR * HBLKSIZE;
    if ((0 == GC_last_heap_addr && (ADDR(space) & SIGNB) == 0)
        || (GC_last_heap_addr != 0 && GC_last_heap_addr < ADDR(space))) {
      /* Assume the heap is growing up. */
      if (EXPECT(ADDR(space) < GC_WORD_MAX - (sz + expansion_slop), TRUE)) {
        ptr_t new_limit = (ptr_t)space + sz + expansion_slop;

        if (ADDR_LT((ptr_t)GC_greatest_plausible_heap_addr, new_limit))
          GC_greatest_plausible_heap_addr = new_limit;
      }
    } else {
      /* Heap is growing down.  */
      if (EXPECT(ADDR(space) > expansion_slop + sizeof(ptr_t), TRUE)) {
        ptr_t new_limit = (ptr_t)space - expansion_slop - sizeof(ptr_t);

        if (ADDR_LT(new_limit, (ptr_t)GC_least_plausible_heap_addr))
          GC_least_plausible_heap_addr = new_limit;
      }
    }
    GC_last_heap_addr = ADDR(space);

    GC_add_to_heap(space, sz);
    if (GC_on_heap_resize)
        (*GC_on_heap_resize)(GC_heapsize);

    return TRUE;
}

/* Really returns a bool, but it's externally visible, so that's clumsy. */
GC_API int GC_CALL GC_expand_hp(size_t bytes)
{
    word n_blocks = OBJ_SZ_TO_BLOCKS_CHECKED(bytes);
    word old_heapsize;
    GC_bool result;

    if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
    LOCK();
    old_heapsize = GC_heapsize;
    result = GC_expand_hp_inner(n_blocks);
    if (result) {
      GC_requested_heapsize += bytes;
      if (GC_dont_gc) {
        /* Do not call WARN if the heap growth is intentional.  */
        GC_ASSERT(GC_heapsize >= old_heapsize);
        GC_heapsize_on_gc_disable += GC_heapsize - old_heapsize;
      }
    }
    UNLOCK();
    return (int)result;
}

GC_INNER unsigned GC_fail_count = 0;
                        /* How many consecutive GC/expansion failures?  */
                        /* Reset by GC_allochblk.                       */

/* The minimum value of the ratio of allocated bytes since the latest   */
/* GC to the amount of finalizers created since that GC which triggers  */
/* the collection instead heap expansion.  Has no effect in the         */
/* incremental mode.                                                    */
#if defined(GC_ALLOCD_BYTES_PER_FINALIZER) && !defined(CPPCHECK)
  STATIC word GC_allocd_bytes_per_finalizer = GC_ALLOCD_BYTES_PER_FINALIZER;
#else
  STATIC word GC_allocd_bytes_per_finalizer = 10000;
#endif

GC_API void GC_CALL GC_set_allocd_bytes_per_finalizer(GC_word value)
{
  GC_allocd_bytes_per_finalizer = value;
}

GC_API GC_word GC_CALL GC_get_allocd_bytes_per_finalizer(void)
{
  return GC_allocd_bytes_per_finalizer;
}

static word last_fo_entries = 0;
static word last_bytes_finalized = 0;

/* Collect or expand heap in an attempt make the indicated number of    */
/* free blocks available.  Should be called until the blocks are        */
/* available (setting retry value to TRUE unless this is the first call */
/* in a loop) or until it fails by returning FALSE.  The flags argument */
/* should be IGNORE_OFF_PAGE or 0.                                      */
GC_INNER GC_bool GC_collect_or_expand(word needed_blocks,
                                      unsigned flags,
                                      GC_bool retry)
{
    GC_bool gc_not_stopped = TRUE;
    word blocks_to_get;
    IF_CANCEL(int cancel_state;)

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(GC_is_initialized);
    DISABLE_CANCEL(cancel_state);
    if (!GC_incremental && !GC_dont_gc &&
        ((GC_dont_expand && GC_bytes_allocd > 0)
         || (GC_fo_entries > last_fo_entries
             && (last_bytes_finalized | GC_bytes_finalized) != 0
             && (GC_fo_entries - last_fo_entries)
                * GC_allocd_bytes_per_finalizer > GC_bytes_allocd)
         || GC_should_collect())) {
      /* Try to do a full collection using 'default' stop_func (unless  */
      /* nothing has been allocated since the latest collection or heap */
      /* expansion is disabled).                                        */
      gc_not_stopped = GC_try_to_collect_inner(
                        GC_bytes_allocd > 0 && (!GC_dont_expand || !retry) ?
                        GC_default_stop_func : GC_never_stop_func);
      if (gc_not_stopped == TRUE || !retry) {
        /* Either the collection hasn't been aborted or this is the     */
        /* first attempt (in a loop).                                   */
        last_fo_entries = GC_fo_entries;
        last_bytes_finalized = GC_bytes_finalized;
        RESTORE_CANCEL(cancel_state);
        return TRUE;
      }
    }

    blocks_to_get = (GC_heapsize - GC_heapsize_at_forced_unmap)
                        / (HBLKSIZE * GC_free_space_divisor)
                    + needed_blocks;
    if (blocks_to_get > MAXHINCR) {
      word slop;

      /* Get the minimum required to make it likely that we can satisfy */
      /* the current request in the presence of black-listing.          */
      /* This will probably be more than MAXHINCR.                      */
      if ((flags & IGNORE_OFF_PAGE) != 0) {
        slop = 4;
      } else {
        slop = 2 * divHBLKSZ(BL_LIMIT);
        if (slop > needed_blocks) slop = needed_blocks;
      }
      if (needed_blocks + slop > MAXHINCR) {
        blocks_to_get = needed_blocks + slop;
      } else {
        blocks_to_get = MAXHINCR;
      }
      if (blocks_to_get > divHBLKSZ(GC_WORD_MAX))
        blocks_to_get = divHBLKSZ(GC_WORD_MAX);
    } else if (blocks_to_get < MINHINCR) {
      blocks_to_get = MINHINCR;
    }

    if (GC_max_heapsize > GC_heapsize) {
      word max_get_blocks = divHBLKSZ(GC_max_heapsize - GC_heapsize);
      if (blocks_to_get > max_get_blocks)
        blocks_to_get = max_get_blocks > needed_blocks
                        ? max_get_blocks : needed_blocks;
    }

#   ifdef USE_MUNMAP
      if (GC_unmap_threshold > 1) {
        /* Return as much memory to the OS as possible before   */
        /* trying to get memory from it.                        */
        GC_unmap_old(0);
      }
#   endif
    if (!GC_expand_hp_inner(blocks_to_get)
        && (blocks_to_get == needed_blocks
            || !GC_expand_hp_inner(needed_blocks))) {
      if (gc_not_stopped == FALSE) {
        /* Don't increment GC_fail_count here (and no warning).     */
        GC_gcollect_inner();
        GC_ASSERT(GC_bytes_allocd == 0);
      } else if (GC_fail_count++ < GC_max_retries) {
        WARN("Out of Memory!  Trying to continue...\n", 0);
        GC_gcollect_inner();
      } else {
#       if !defined(AMIGA) || !defined(GC_AMIGA_FASTALLOC)
#         ifdef USE_MUNMAP
            GC_ASSERT(GC_heapsize >= GC_unmapped_bytes);
#         endif
#         if !defined(SMALL_CONFIG) && (CPP_WORDSZ >= 32)
#           define MAX_HEAPSIZE_WARNED_IN_BYTES (5 << 20) /* 5 MB */
            if (GC_heapsize > (word)MAX_HEAPSIZE_WARNED_IN_BYTES) {
              WARN("Out of Memory! Heap size: %" WARN_PRIuPTR " MiB."
                   " Returning NULL!\n",
                   (GC_heapsize - GC_unmapped_bytes) >> 20);
            } else
#         endif
          /* else */ {
              WARN("Out of Memory! Heap size: %" WARN_PRIuPTR " bytes."
                   " Returning NULL!\n", GC_heapsize - GC_unmapped_bytes);
          }
#       endif
        RESTORE_CANCEL(cancel_state);
        return FALSE;
      }
    } else if (GC_fail_count) {
      GC_COND_LOG_PRINTF("Memory available again...\n");
    }
    RESTORE_CANCEL(cancel_state);
    return TRUE;
}

GC_INNER ptr_t GC_allocobj(size_t lg, int k)
{
    void **flh = &GC_obj_kinds[k].ok_freelist[lg];
    GC_bool tried_minor = FALSE;
    GC_bool retry = FALSE;

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(GC_is_initialized);
    if (EXPECT(0 == lg, FALSE)) return NULL;

    while (NULL == *flh) {
      ENTER_GC();
#     ifndef GC_DISABLE_INCREMENTAL
        if (GC_incremental && GC_time_limit != GC_TIME_UNLIMITED
            && !GC_dont_gc) {
          /* True incremental mode, not just generational.      */
          /* Do our share of marking work.                      */
          GC_collect_a_little_inner(1);
        }
#     endif
      /* Sweep blocks for objects of this size */
        GC_ASSERT(!GC_is_full_gc
                  || NULL == GC_obj_kinds[k].ok_reclaim_list
                  || NULL == GC_obj_kinds[k].ok_reclaim_list[lg]);
        GC_continue_reclaim(lg, k);
      EXIT_GC();
#     if defined(CPPCHECK)
        GC_noop1_ptr(&flh);
#     endif
      if (NULL == *flh) {
        GC_new_hblk(lg, k);
#       if defined(CPPCHECK)
          GC_noop1_ptr(&flh);
#       endif
        if (NULL == *flh) {
          ENTER_GC();
          if (GC_incremental && GC_time_limit == GC_TIME_UNLIMITED
              && !tried_minor && !GC_dont_gc) {
            GC_collect_a_little_inner(1);
            tried_minor = TRUE;
          } else {
            if (!GC_collect_or_expand(1, 0 /* flags */, retry)) {
              EXIT_GC();
              return NULL;
            }
            retry = TRUE;
          }
          EXIT_GC();
        }
      }
    }
    /* Successful allocation; reset failure count.      */
    GC_fail_count = 0;
    return (ptr_t)(*flh);
}

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1995 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1997 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2007 Free Software Foundation, Inc.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#ifndef MSWINCE
# include <errno.h>
#endif
#include <string.h>

#ifndef SHORT_DBG_HDRS
  /* Check whether object given by its base pointer has debugging   */
  /* info.  base is assumed to point to a legitimate object in our  */
  /* heap.  This excludes the check as to whether the back pointer  */
  /* is odd, which is added by the GC_HAS_DEBUG_INFO macro.         */
  /* Note that if DBG_HDRS_ALL is set, uncollectible objects on     */
  /* free lists may not have debug information set.  Thus, it is    */
  /* not always safe to return TRUE (1), even if the client does    */
  /* its part.  Return -1 if the object with debug info has been    */
  /* marked as deallocated.                                         */
  GC_INNER int GC_has_other_debug_info(ptr_t base)
  {
    ptr_t body = (ptr_t)((oh *)base + 1);
    size_t sz = GC_size(base);

    if (HBLKPTR(base) != HBLKPTR(body)
        || sz < DEBUG_BYTES + EXTRA_BYTES) {
      return 0;
    }
    if (((oh *)base) -> oh_sf != (START_FLAG ^ (GC_uintptr_t)body)
        && ((GC_uintptr_t *)base)[BYTES_TO_PTRS(sz) - 1]
            != (END_FLAG ^ (GC_uintptr_t)body)) {
      return 0;
    }
    if (((oh *)base) -> oh_sz == (GC_uintptr_t)sz) {
      /* Object may have had debug info, but has been deallocated.  */
      return -1;
    }
    return 1;
  }
#endif /* !SHORT_DBG_HDRS */

#ifdef KEEP_BACK_PTRS

  /* Use a custom trivial random() implementation as the standard   */
  /* one might lead to crashes (if used from a multi-threaded code) */
  /* or to a compiler warning about the deterministic result.       */
  static int GC_rand(void)
  {
    static GC_RAND_STATE_T seed;

    return GC_RAND_NEXT(&seed);
  }

# define RANDOM() (long)GC_rand()

  /* Store back pointer to source in dest, if that appears to be possible. */
  /* This is not completely safe, since we may mistakenly conclude that    */
  /* dest has a debugging wrapper.  But the error probability is very      */
  /* small, and this shouldn't be used in production code.                 */
  /* We assume that dest is the real base pointer.  Source will usually    */
  /* be a pointer to the interior of an object.                            */
  GC_INNER void GC_store_back_pointer(ptr_t source, ptr_t dest)
  {
    if (GC_HAS_DEBUG_INFO(dest)) {
#     ifdef PARALLEL_MARK
        GC_cptr_store((volatile ptr_t *)&(((oh *)dest) -> oh_back_ptr),
                      (ptr_t)HIDE_BACK_PTR(source));
#     else
        ((oh *)dest) -> oh_back_ptr = HIDE_BACK_PTR(source);
#     endif
    }
  }

  GC_INNER void GC_marked_for_finalization(ptr_t dest)
  {
    GC_store_back_pointer(MARKED_FOR_FINALIZATION, dest);
  }

  GC_API GC_ref_kind GC_CALL GC_get_back_ptr_info(void *dest, void **base_p,
                                                  size_t *offset_p)
  {
    oh * ohdr = (oh *)GC_base(dest);
    ptr_t bp, bp_base;

#   ifdef LINT2
      /* Explicitly instruct the code analysis tool that                */
      /* GC_get_back_ptr_info is not expected to be called with an      */
      /* incorrect "dest" value.                                        */
      if (!ohdr) ABORT("Invalid GC_get_back_ptr_info argument");
#   endif
    if (!GC_HAS_DEBUG_INFO((ptr_t)ohdr)) return GC_NO_SPACE;
    bp = (ptr_t)GC_REVEAL_POINTER(ohdr -> oh_back_ptr);
    if (MARKED_FOR_FINALIZATION == bp) return GC_FINALIZER_REFD;
    if (MARKED_FROM_REGISTER == bp) return GC_REFD_FROM_REG;
    if (NOT_MARKED == bp) return GC_UNREFERENCED;
#   if ALIGNMENT == 1
      /* Heuristically try to fix off-by-one errors we introduced by    */
      /* insisting on even addresses.                                   */
      {
        ptr_t alternate_ptr = bp + 1;
        ptr_t target = *(ptr_t *)bp;
        ptr_t alternate_target = *(ptr_t *)alternate_ptr;

        if (GC_least_real_heap_addr < ADDR(alternate_target)
            && ADDR(alternate_target) < GC_greatest_real_heap_addr
            && (GC_least_real_heap_addr >= ADDR(target)
                || ADDR(target) >= GC_greatest_real_heap_addr)) {
          bp = alternate_ptr;
        }
      }
#   endif
    bp_base = (ptr_t)GC_base(bp);
    if (NULL == bp_base) {
      *base_p = bp;
      *offset_p = 0;
      return GC_REFD_FROM_ROOT;
    } else {
      if (GC_HAS_DEBUG_INFO(bp_base)) bp_base += sizeof(oh);
      *base_p = bp_base;
      *offset_p = (size_t)(bp - bp_base);
      return GC_REFD_FROM_HEAP;
    }
  }

  /* Generate a random heap address.            */
  /* The resulting address is in the heap, but  */
  /* not necessarily inside a valid object.     */
  GC_API void * GC_CALL GC_generate_random_heap_address(void)
  {
    size_t i;
    word heap_offset = (word)RANDOM();

    if (GC_heapsize > (word)GC_RAND_MAX) {
        heap_offset *= GC_RAND_MAX;
        heap_offset += (word)RANDOM();
    }
    heap_offset %= GC_heapsize;
        /* This doesn't yield a uniform distribution, especially if     */
        /* e.g. RAND_MAX is 1.5*GC_heapsize.  But for typical cases,    */
        /* it's not too bad.                                            */

    for (i = 0;; ++i) {
        size_t size;

        if (i >= GC_n_heap_sects)
          ABORT("GC_generate_random_heap_address: size inconsistency");

        size = GC_heap_sects[i].hs_bytes;
        if (heap_offset < size) break;
        heap_offset -= size;
    }
    return GC_heap_sects[i].hs_start + heap_offset;
  }

  /* Generate a random address inside a valid marked heap object. */
  GC_API void * GC_CALL GC_generate_random_valid_address(void)
  {
    ptr_t result;
    ptr_t base;

    do {
      result = (ptr_t)GC_generate_random_heap_address();
      base = (ptr_t)GC_base(result);
    } while (NULL == base || !GC_is_marked(base));
    return result;
  }

  GC_API void GC_CALL GC_print_backtrace(void *p)
  {
    void *current = p;
    int i;

    GC_ASSERT(I_DONT_HOLD_LOCK());
    GC_print_heap_obj((ptr_t)GC_base(current));

    for (i = 0;; ++i) {
      void *base;
      size_t offset;
      GC_ref_kind source = GC_get_back_ptr_info(current, &base, &offset);

      if (GC_UNREFERENCED == source) {
        GC_err_printf("Reference could not be found\n");
        break;
      }
      if (GC_NO_SPACE == source) {
        GC_err_printf("No debug info in object: Can't find reference\n");
        break;
      }
      GC_err_printf("Reachable via %d levels of pointers from ", i);
      switch (source) {
      case GC_REFD_FROM_ROOT:
        GC_err_printf("root at %p\n\n", base);
        return;
      case GC_REFD_FROM_REG:
        GC_err_printf("root in register\n\n");
        return;
      case GC_FINALIZER_REFD:
        GC_err_printf("list of finalizable objects\n\n");
        return;
      case GC_REFD_FROM_HEAP:
        GC_err_printf("offset %ld in object:\n", (long)offset);
        /* Take GC_base(base) to get real base, i.e. header.    */
        GC_print_heap_obj((ptr_t)GC_base(base));
        break;
      default:
        GC_err_printf("INTERNAL ERROR: UNEXPECTED SOURCE!!!!\n");
        return;
      }
      current = base;
    }
  }

  GC_API void GC_CALL GC_generate_random_backtrace(void)
  {
    void *current;

    GC_ASSERT(I_DONT_HOLD_LOCK());
    if (GC_try_to_collect(GC_never_stop_func) == 0) {
      GC_err_printf("Cannot generate a backtrace: "
                    "garbage collection is disabled!\n");
      return;
    }

    /* Generate/print a backtrace from a random heap address.   */
    LOCK();
    current = GC_generate_random_valid_address();
    UNLOCK();
    GC_printf("\n****Chosen address %p in object\n", current);
    GC_print_backtrace(current);
  }

#endif /* KEEP_BACK_PTRS */

#define CROSSES_HBLK(p, sz) \
                ((ADDR((p) + sizeof(oh) + (sz) - 1) ^ ADDR(p)) >= HBLKSIZE)

GC_INNER void *GC_store_debug_info_inner(void *base, size_t sz,
                                         const char *string, int linenum)
{
    GC_uintptr_t *result = (GC_uintptr_t *)((oh *)base + 1);

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(GC_size(base) >= sizeof(oh) + sz);
    GC_ASSERT(!(SMALL_OBJ(sz) && CROSSES_HBLK((ptr_t)base, sz)));
#   ifdef KEEP_BACK_PTRS
      ((oh *)base) -> oh_back_ptr = HIDE_BACK_PTR(NOT_MARKED);
#   endif
#   ifdef MAKE_BACK_GRAPH
      ((oh *)base) -> oh_bg_ptr = HIDE_BACK_PTR((ptr_t)0);
#   endif
    ((oh *)base) -> oh_string = string;
    ((oh *)base) -> oh_int = linenum;
#   ifdef SHORT_DBG_HDRS
      UNUSED_ARG(sz);
#   else
      ((oh *)base) -> oh_sz = (GC_uintptr_t)sz;
      ((oh *)base) -> oh_sf = START_FLAG ^ (GC_uintptr_t)result;
      ((GC_uintptr_t *)base)[BYTES_TO_PTRS(GC_size(base)) - 1]
                        = result[BYTES_TO_PTRS_ROUNDUP(sz)]
                        = END_FLAG ^ (GC_uintptr_t)result;
#   endif
    return result;
}

/* Check the allocation is successful, store debugging info into base,  */
/* start the debugging mode (if not yet), and return displaced pointer. */
static void *store_debug_info(void *base, size_t lb,
                              const char *fn, GC_EXTRA_PARAMS)
{
    void *result;

    if (NULL == base) {
        GC_err_printf("%s(%lu) returning NULL (%s:%d)\n",
                      fn, (unsigned long)lb, s, i);
        return NULL;
    }
    LOCK();
    if (!GC_debugging_started)
        GC_start_debugging_inner();
    result = GC_store_debug_info_inner(base, lb, s, i);
    ADD_CALL_CHAIN(base, ra);
    UNLOCK();
    return result;
}

#ifndef SHORT_DBG_HDRS
  /* Check the object with debugging info at ohdr.      */
  /* Return NULL if it's OK.  Else return clobbered     */
  /* address.                                           */
  STATIC ptr_t GC_check_annotated_obj(oh *ohdr)
  {
    ptr_t body = (ptr_t)(ohdr + 1);
    size_t gc_sz = GC_size(ohdr);

    if (ohdr -> oh_sz + DEBUG_BYTES > (GC_uintptr_t)gc_sz) {
        return (ptr_t)(&(ohdr -> oh_sz));
    }
    if (ohdr -> oh_sf != (START_FLAG ^ (GC_uintptr_t)body)) {
        return (ptr_t)(&(ohdr -> oh_sf));
    }
    if (((GC_uintptr_t *)ohdr)[BYTES_TO_PTRS(gc_sz) - 1]
            != (END_FLAG ^ (GC_uintptr_t)body)) {
        return (ptr_t)(&((GC_uintptr_t *)ohdr)[BYTES_TO_PTRS(gc_sz) - 1]);
    }
    if (((GC_uintptr_t *)body)[BYTES_TO_PTRS_ROUNDUP((size_t)(ohdr -> oh_sz))]
            != (END_FLAG ^ (GC_uintptr_t)body)) {
        return (ptr_t)(&((GC_uintptr_t *)body)[BYTES_TO_PTRS_ROUNDUP(
                                                (size_t)(ohdr -> oh_sz))]);
    }
    return NULL;
  }
#endif /* !SHORT_DBG_HDRS */

STATIC GC_describe_type_fn GC_describe_type_fns[MAXOBJKINDS] = {0};

GC_API void GC_CALL GC_register_describe_type_fn(int k,
                                                 GC_describe_type_fn fn)
{
  GC_ASSERT((unsigned)k < MAXOBJKINDS);
  GC_describe_type_fns[k] = fn;
}

#define GET_OH_LINENUM(ohdr) ((int)((ohdr) -> oh_int))

#ifndef SHORT_DBG_HDRS
# define IF_NOT_SHORTDBG_HDRS(x) x
# define COMMA_IFNOT_SHORTDBG_HDRS(x) /* comma */, x
#else
# define IF_NOT_SHORTDBG_HDRS(x) /* empty */
# define COMMA_IFNOT_SHORTDBG_HDRS(x) /* empty */
#endif

/* Print a human-readable description of the object to stderr.  */
/* The object is assumed to have the debugging info.            */
STATIC void GC_print_obj(ptr_t base)
{
    oh *ohdr = (oh *)base;
    ptr_t q;
    hdr *hhdr;
    int k;
    const char *kind_str;
    char buffer[GC_TYPE_DESCR_LEN + 1];

    GC_ASSERT(I_DONT_HOLD_LOCK());
#   ifdef LINT2
      if (!ohdr) ABORT("Invalid GC_print_obj argument");
#   endif

    q = (ptr_t)(ohdr + 1);
    /* Print a type description for the object whose client-visible     */
    /* address is q.                                                    */
    hhdr = GC_find_header(q);
    k = hhdr -> hb_obj_kind;
    if (GC_describe_type_fns[k] != 0 && GC_is_marked(ohdr)) {
        /* This should preclude free-list objects except with   */
        /* thread-local allocation.                             */
        buffer[GC_TYPE_DESCR_LEN] = 0;
        (GC_describe_type_fns[k])(q, buffer);
        GC_ASSERT(buffer[GC_TYPE_DESCR_LEN] == 0);
        kind_str = buffer;
    } else {
        switch (k) {
        case PTRFREE:
            kind_str = "PTRFREE";
            break;
        case NORMAL:
            kind_str = "NORMAL";
            break;
        case UNCOLLECTABLE:
            kind_str = "UNCOLLECTABLE";
            break;
#       ifdef GC_ATOMIC_UNCOLLECTABLE
            case AUNCOLLECTABLE:
              kind_str = "ATOMIC_UNCOLLECTABLE";
              break;
#       endif
        default:
            kind_str = NULL;
                /* The alternative is to use snprintf(buffer) but it is */
                /* not quite portable (see vsnprintf in misc.c).        */
        }
    }

    if (NULL != kind_str) {
        GC_err_printf("%p (%s:%d," IF_NOT_SHORTDBG_HDRS(" sz= %lu,") " %s)\n",
                      (void *)((ptr_t)ohdr + sizeof(oh)),
                      ohdr -> oh_string, GET_OH_LINENUM(ohdr) /*, */
                      COMMA_IFNOT_SHORTDBG_HDRS((unsigned long)ohdr->oh_sz),
                      kind_str);
    } else {
        GC_err_printf("%p (%s:%d," IF_NOT_SHORTDBG_HDRS(" sz= %lu,")
                      " kind= %d, descr= 0x%lx)\n",
                      (void *)((ptr_t)ohdr + sizeof(oh)),
                      ohdr -> oh_string, GET_OH_LINENUM(ohdr) /*, */
                      COMMA_IFNOT_SHORTDBG_HDRS((unsigned long)ohdr->oh_sz),
                      k, (unsigned long)(hhdr -> hb_descr));
    }
    PRINT_CALL_CHAIN(ohdr);
}

STATIC void GC_debug_print_heap_obj_proc(ptr_t base)
{
    GC_ASSERT(I_DONT_HOLD_LOCK());
    if (GC_HAS_DEBUG_INFO(base)) {
        GC_print_obj(base);
    } else {
        GC_default_print_heap_obj_proc(base);
    }
}

#ifndef SHORT_DBG_HDRS
  /* Use GC_err_printf and friends to print a description of the object */
  /* whose client-visible address is p, and which was smashed at memory */
  /* location pointed by clobbered.                                     */
  STATIC void GC_print_smashed_obj(const char *msg, void *p, ptr_t clobbered)
  {
    oh * ohdr = (oh *)GC_base(p);

    GC_ASSERT(I_DONT_HOLD_LOCK());
#   ifdef LINT2
      if (!ohdr) ABORT("Invalid GC_print_smashed_obj argument");
#   endif
    if (ADDR_GE((ptr_t)(&(ohdr -> oh_sz)), clobbered)
            || NULL == ohdr -> oh_string) {
        GC_err_printf(
                "%s %p in or near object at %p(<smashed>, appr. sz= %lu)\n",
                msg, (void *)clobbered, p,
                (unsigned long)(GC_size(ohdr) - DEBUG_BYTES));
    } else {
        GC_err_printf("%s %p in or near object at %p (%s:%d, sz= %lu)\n",
                msg, (void *)clobbered, p,
                ADDR(ohdr -> oh_string) < HBLKSIZE ? "(smashed string)"
                    : ohdr -> oh_string[0] == '\0' ? "EMPTY(smashed?)"
                                                   : ohdr -> oh_string,
                GET_OH_LINENUM(ohdr), (unsigned long)(ohdr -> oh_sz));
        PRINT_CALL_CHAIN(ohdr);
    }
  }

  STATIC void GC_check_heap_proc(void);
  STATIC void GC_print_all_smashed_proc(void);
#else
  STATIC void GC_do_nothing(void) {}
#endif /* SHORT_DBG_HDRS */

GC_INNER void GC_start_debugging_inner(void)
{
  GC_ASSERT(I_HOLD_LOCK());
# ifndef SHORT_DBG_HDRS
    GC_check_heap = GC_check_heap_proc;
    GC_print_all_smashed = GC_print_all_smashed_proc;
# else
    GC_check_heap = GC_do_nothing;
    GC_print_all_smashed = GC_do_nothing;
# endif
  GC_print_heap_obj = GC_debug_print_heap_obj_proc;
  GC_debugging_started = TRUE;
  GC_register_displacement_inner(sizeof(oh));
# if defined(CPPCHECK)
    GC_noop1(GC_debug_header_size);
# endif
}

const size_t GC_debug_header_size = sizeof(oh);

GC_API size_t GC_CALL GC_get_debug_header_size(void) {
  return sizeof(oh);
}

GC_API void GC_CALL GC_debug_register_displacement(size_t offset)
{
  LOCK();
  GC_register_displacement_inner(offset);
  GC_register_displacement_inner(sizeof(oh) + offset);
  UNLOCK();
}

#ifdef GC_ADD_CALLER
# if defined(HAVE_DLADDR) && defined(GC_HAVE_RETURN_ADDR_PARENT) \
     && defined(FUNCPTR_IS_DATAPTR)
#   include <dlfcn.h>

    STATIC void GC_caller_func_offset(GC_return_addr_t ra, const char **symp,
                                      int *offp)
    {
      Dl_info caller;

      if (ra != 0 && dladdr((void *)ra, &caller)
          && caller.dli_sname != NULL) {
        *symp = caller.dli_sname;
        *offp = (int)((ptr_t)ra - (ptr_t)caller.dli_saddr);
      }
      if (NULL == *symp) {
        *symp = "unknown";
        /* Note: *offp is unchanged.    */
      }
    }
# else
#   define GC_caller_func_offset(ra, symp, offp) (void)(*(symp) = "unknown")
# endif
#endif /* GC_ADD_CALLER */

GC_API GC_ATTR_MALLOC void * GC_CALL GC_debug_malloc(size_t lb,
                                                     GC_EXTRA_PARAMS)
{
    void *base;

    /* Note that according to malloc() specification, if size is 0 then */
    /* malloc() returns either NULL, or a unique pointer value that can */
    /* later be successfully passed to free(). We always do the latter. */
#   if defined(_FORTIFY_SOURCE) && !defined(__clang__)
      /* Workaround to avoid "exceeds maximum object size" gcc warning. */
      base = GC_malloc(lb < GC_SIZE_MAX - DEBUG_BYTES ? lb + DEBUG_BYTES
                                                        : GC_SIZE_MAX >> 1);
#   else
      base = GC_malloc(SIZET_SAT_ADD(lb, DEBUG_BYTES));
#   endif
#   ifdef GC_ADD_CALLER
      if (NULL == s) {
        GC_caller_func_offset(ra, &s, &i);
      }
#   endif
    return store_debug_info(base, lb, "GC_debug_malloc", OPT_RA s, i);
}

GC_API GC_ATTR_MALLOC void * GC_CALL
    GC_debug_malloc_ignore_off_page(size_t lb, GC_EXTRA_PARAMS)
{
    void *base = GC_malloc_ignore_off_page(SIZET_SAT_ADD(lb, DEBUG_BYTES));

    return store_debug_info(base, lb, "GC_debug_malloc_ignore_off_page",
                            OPT_RA s, i);
}

GC_API GC_ATTR_MALLOC void * GC_CALL
    GC_debug_malloc_atomic_ignore_off_page(size_t lb, GC_EXTRA_PARAMS)
{
    void *base = GC_malloc_atomic_ignore_off_page(
                                SIZET_SAT_ADD(lb, DEBUG_BYTES));

    return store_debug_info(base, lb,
                            "GC_debug_malloc_atomic_ignore_off_page",
                            OPT_RA s, i);
}

STATIC void * GC_debug_generic_malloc(size_t lb, int k, GC_EXTRA_PARAMS)
{
    void *base = GC_generic_malloc_aligned(SIZET_SAT_ADD(lb, DEBUG_BYTES),
                                           k, 0 /* flags */, 0 /* align_m1 */);

    return store_debug_info(base, lb, "GC_debug_generic_malloc", OPT_RA s, i);
}

#ifdef DBG_HDRS_ALL
  /* An allocation function for internal use.  Normally internally      */
  /* allocated objects do not have debug information.  But in this      */
  /* case, we need to make sure that all objects have debug headers.    */
  GC_INNER void * GC_debug_generic_malloc_inner(size_t lb, int k,
                                                unsigned flags)
  {
    void *base, *result;

    GC_ASSERT(I_HOLD_LOCK());
    base = GC_generic_malloc_inner(SIZET_SAT_ADD(lb, DEBUG_BYTES), k, flags);
    if (NULL == base) {
        GC_err_printf("GC internal allocation (%lu bytes) returning NULL\n",
                      (unsigned long)lb);
        return NULL;
    }
    if (!GC_debugging_started)
        GC_start_debugging_inner();
    result = GC_store_debug_info_inner(base, lb, "INTERNAL", 0);
    ADD_CALL_CHAIN_INNER(base);
    return result;
  }
#endif /* DBG_HDRS_ALL */

#ifndef CPPCHECK
  GC_API void * GC_CALL GC_debug_malloc_stubborn(size_t lb, GC_EXTRA_PARAMS)
  {
    return GC_debug_malloc(lb, OPT_RA s, i);
  }

  GC_API void GC_CALL GC_debug_change_stubborn(const void *p)
  {
    UNUSED_ARG(p);
  }
#endif /* !CPPCHECK */

GC_API void GC_CALL GC_debug_end_stubborn_change(const void *p)
{
    const void * q = GC_base_C(p);

    if (NULL == q) {
        ABORT_ARG1("GC_debug_end_stubborn_change: bad arg", ": %p", p);
    }
    GC_end_stubborn_change(q);
}

GC_API void GC_CALL GC_debug_ptr_store_and_dirty(void *p, const void *q)
{
    *(void **)GC_is_visible(p)
                = GC_is_valid_displacement(GC_CAST_AWAY_CONST_PVOID(q));
    GC_debug_end_stubborn_change(p);
    REACHABLE_AFTER_DIRTY(q);
}

GC_API GC_ATTR_MALLOC void * GC_CALL GC_debug_malloc_atomic(size_t lb,
                                                            GC_EXTRA_PARAMS)
{
    void *base = GC_malloc_atomic(SIZET_SAT_ADD(lb, DEBUG_BYTES));

    return store_debug_info(base, lb, "GC_debug_malloc_atomic", OPT_RA s, i);
}

GC_API GC_ATTR_MALLOC char * GC_CALL GC_debug_strdup(const char *str,
                                                     GC_EXTRA_PARAMS)
{
  char *copy;
  size_t lb;
  if (str == NULL) {
    if (GC_find_leak)
      GC_err_printf("strdup(NULL) behavior is undefined\n");
    return NULL;
  }

  lb = strlen(str) + 1;
  copy = (char *)GC_debug_malloc_atomic(lb, OPT_RA s, i);
  if (copy == NULL) {
#   ifndef MSWINCE
      errno = ENOMEM;
#   endif
    return NULL;
  }
  BCOPY(str, copy, lb);
  return copy;
}

GC_API GC_ATTR_MALLOC char * GC_CALL GC_debug_strndup(const char *str,
                                                size_t size, GC_EXTRA_PARAMS)
{
  char *copy;
  size_t len = strlen(str); /* str is expected to be non-NULL  */
  if (len > size)
    len = size;
  copy = (char *)GC_debug_malloc_atomic(len + 1, OPT_RA s, i);
  if (copy == NULL) {
#   ifndef MSWINCE
      errno = ENOMEM;
#   endif
    return NULL;
  }
  if (len > 0)
    BCOPY(str, copy, len);
  copy[len] = '\0';
  return copy;
}

#ifdef GC_REQUIRE_WCSDUP
# include <wchar.h> /* for wcslen() */

  GC_API GC_ATTR_MALLOC wchar_t * GC_CALL GC_debug_wcsdup(const wchar_t *str,
                                                          GC_EXTRA_PARAMS)
  {
    size_t lb = (wcslen(str) + 1) * sizeof(wchar_t);
    wchar_t *copy = (wchar_t *)GC_debug_malloc_atomic(lb, OPT_RA s, i);
    if (copy == NULL) {
#     ifndef MSWINCE
        errno = ENOMEM;
#     endif
      return NULL;
    }
    BCOPY(str, copy, lb);
    return copy;
  }
#endif /* GC_REQUIRE_WCSDUP */

GC_API GC_ATTR_MALLOC void * GC_CALL GC_debug_malloc_uncollectable(size_t lb,
                                                        GC_EXTRA_PARAMS)
{
    void *base = GC_malloc_uncollectable(
                                SIZET_SAT_ADD(lb, UNCOLLECTABLE_DEBUG_BYTES));

    return store_debug_info(base, lb, "GC_debug_malloc_uncollectable",
                            OPT_RA s, i);
}

#ifdef GC_ATOMIC_UNCOLLECTABLE
  GC_API GC_ATTR_MALLOC void * GC_CALL
        GC_debug_malloc_atomic_uncollectable(size_t lb, GC_EXTRA_PARAMS)
  {
    void *base = GC_malloc_atomic_uncollectable(
                                SIZET_SAT_ADD(lb, UNCOLLECTABLE_DEBUG_BYTES));

    return store_debug_info(base, lb, "GC_debug_malloc_atomic_uncollectable",
                            OPT_RA s, i);
  }
#endif /* GC_ATOMIC_UNCOLLECTABLE */

#ifdef LINT2
/*
 * Copyright (c) 1996-1998 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 2018-2021 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/* This file is kept for a binary compatibility purpose only.   */

#ifndef GC_ALLOC_PTRS_H
#define GC_ALLOC_PTRS_H




#ifdef __cplusplus
  extern "C" {
#endif

#ifndef GC_API_PRIV
# define GC_API_PRIV GC_API
#endif

/* Some compilers do not accept "const" together with the dllimport     */
/* attribute, so the symbols below are exported as non-constant ones.   */
#ifndef GC_APIVAR_CONST
# if defined(GC_BUILD) || !defined(GC_DLL)
#   define GC_APIVAR_CONST const
# else
#   define GC_APIVAR_CONST /* empty */
# endif
#endif

GC_API_PRIV void ** GC_APIVAR_CONST GC_objfreelist_ptr;
GC_API_PRIV void ** GC_APIVAR_CONST GC_aobjfreelist_ptr;
GC_API_PRIV void ** GC_APIVAR_CONST GC_uobjfreelist_ptr;

#ifdef GC_ATOMIC_UNCOLLECTABLE
  GC_API_PRIV void ** GC_APIVAR_CONST GC_auobjfreelist_ptr;
#endif

/* Manually update the number of bytes allocated during the current     */
/* collection cycle and the number of explicitly deallocated bytes of   */
/* memory since the last collection, respectively.  Both functions are  */
/* unsynchronized, GC_call_with_alloc_lock() should be used to avoid    */
/* data race.                                                           */
GC_API_PRIV void GC_CALL GC_incr_bytes_allocd(size_t /* bytes */);
GC_API_PRIV void GC_CALL GC_incr_bytes_freed(size_t /* bytes */);

#ifdef __cplusplus
  } /* extern "C" */
#endif

#endif /* GC_ALLOC_PTRS_H */

#endif

GC_API void GC_CALL GC_debug_free(void * p)
{
    ptr_t base;
    if (0 == p) return;

    base = (ptr_t)GC_base(p);
    if (NULL == base) {
#     if defined(REDIRECT_MALLOC) \
         && ((defined(NEED_CALLINFO) && defined(GC_HAVE_BUILTIN_BACKTRACE)) \
             || defined(GC_LINUX_THREADS) || defined(GC_SOLARIS_THREADS) \
             || defined(MSWIN32))
        /* In some cases, we should ignore objects that do not belong   */
        /* to the GC heap.  See the comment in GC_free.                 */
        if (!GC_is_heap_ptr(p)) return;
#     endif
      ABORT_ARG1("Invalid pointer passed to free()", ": %p", p);
    }
    if ((word)((ptr_t)p - base) != sizeof(oh)) {
#     if defined(REDIRECT_FREE) && defined(USE_PROC_FOR_LIBRARIES)
        /* TODO: Suppress the warning if free() caller is in libpthread */
        /* or libdl.                                                    */
#     endif
      /* TODO: Suppress the warning for objects allocated by            */
      /* GC_memalign and friends (these ones do not have the debugging  */
      /* counterpart).                                                  */
      GC_err_printf(
               "GC_debug_free called on pointer %p w/o debugging info\n", p);
    } else {
#     ifndef SHORT_DBG_HDRS
        ptr_t clobbered = GC_check_annotated_obj((oh *)base);
        size_t sz = GC_size(base);

        if (clobbered != NULL) {
          GC_SET_HAVE_ERRORS(); /* no "release" barrier is needed */
          if (((oh *)base) -> oh_sz == (GC_uintptr_t)sz) {
            GC_print_smashed_obj(
                  "GC_debug_free: found previously deallocated (?) object at",
                  p, clobbered);
            return; /* ignore double free */
          } else {
            GC_print_smashed_obj("GC_debug_free: found smashed location at",
                                 p, clobbered);
          }
        }
        /* Invalidate size (mark the object as deallocated) */
        ((oh *)base) -> oh_sz = (GC_uintptr_t)sz;
#     endif /* !SHORT_DBG_HDRS */
    }
    if (GC_find_leak
#       ifndef SHORT_DBG_HDRS
          && ((word)((ptr_t)p - base) != sizeof(oh) || !GC_findleak_delay_free)
#       endif
        ) {
      GC_free(base);
    } else {
      const hdr *hhdr = HDR(p);
      if (hhdr -> hb_obj_kind == UNCOLLECTABLE
#         ifdef GC_ATOMIC_UNCOLLECTABLE
            || hhdr -> hb_obj_kind == AUNCOLLECTABLE
#         endif
          ) {
        GC_free(base);
      } else {
        size_t sz = hhdr -> hb_sz;
        size_t i;
        size_t lpw = BYTES_TO_PTRS(sz - sizeof(oh));

        for (i = 0; i < lpw; ++i)
          ((GC_uintptr_t *)p)[i] = GC_FREED_MEM_MARKER;
        GC_ASSERT((GC_uintptr_t *)p + i == (GC_uintptr_t *)(base + sz));
        /* Update the counter even though the real deallocation */
        /* is deferred.                                         */
        LOCK();
#       ifdef LINT2
          GC_incr_bytes_freed(sz);
#       else
          GC_bytes_freed += sz;
#       endif
        UNLOCK();
      }
    } /* !GC_find_leak */
}

#if defined(THREADS) && defined(DBG_HDRS_ALL)
  /* Used internally; we assume it's called correctly.    */
  GC_INNER void GC_debug_free_inner(void * p)
  {
    ptr_t base = (ptr_t)GC_base(p);

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT((word)((ptr_t)p - base) == sizeof(oh));
#   ifdef LINT2
      if (!base) ABORT("Invalid GC_debug_free_inner argument");
#   endif
#   ifndef SHORT_DBG_HDRS
      /* Invalidate size */
      ((oh *)base) -> oh_sz = (GC_uintptr_t)GC_size(base);
#   endif
    GC_free_inner(base);
  }
#endif

GC_API void * GC_CALL GC_debug_realloc(void * p, size_t lb, GC_EXTRA_PARAMS)
{
    ptr_t base;
    void * result;
    const hdr * hhdr;

    if (NULL == p) {
      return GC_debug_malloc(lb, OPT_RA s, i);
    }
    if (0 == lb) /* and p != NULL */ {
      GC_debug_free(p);
      return NULL;
    }

#   ifdef GC_ADD_CALLER
      if (NULL == s) {
        GC_caller_func_offset(ra, &s, &i);
      }
#   endif
    base = (ptr_t)GC_base(p);
    if (NULL == base) {
        ABORT_ARG1("Invalid pointer passed to realloc()", ": %p", p);
    }
    if ((word)((ptr_t)p - base) != sizeof(oh)) {
        GC_err_printf(
              "GC_debug_realloc called on pointer %p w/o debugging info\n", p);
        return GC_realloc(p, lb);
    }
    hhdr = HDR(base);
    result = GC_debug_generic_or_special_malloc(lb, hhdr -> hb_obj_kind,
                                                OPT_RA s, i);
    if (result != NULL) {
      size_t old_sz;
#     ifdef SHORT_DBG_HDRS
        old_sz = GC_size(base) - sizeof(oh);
#     else
        old_sz = (size_t)(((oh *)base) -> oh_sz);
#     endif
      if (old_sz > 0)
        BCOPY(p, result, old_sz < lb ? old_sz : lb);
      GC_debug_free(p);
    }
    return result;
}

GC_API GC_ATTR_MALLOC void * GC_CALL
    GC_debug_generic_or_special_malloc(size_t lb, int k, GC_EXTRA_PARAMS)
{
    switch (k) {
    case PTRFREE:
        return GC_debug_malloc_atomic(lb, OPT_RA s, i);
    case NORMAL:
        return GC_debug_malloc(lb, OPT_RA s, i);
    case UNCOLLECTABLE:
        return GC_debug_malloc_uncollectable(lb, OPT_RA s, i);
#   ifdef GC_ATOMIC_UNCOLLECTABLE
        case AUNCOLLECTABLE:
            return GC_debug_malloc_atomic_uncollectable(lb, OPT_RA s, i);
#   endif
    default:
        return GC_debug_generic_malloc(lb, k, OPT_RA s, i);
    }
}

#ifndef SHORT_DBG_HDRS

  /* List of smashed (clobbered) locations.  We defer printing these,   */
  /* since we cannot always print them nicely with the allocator lock   */
  /* held.  We put them here instead of in GC_arrays, since it may be   */
  /* useful to be able to look at them with the debugger.               */
# ifndef MAX_SMASHED
#   define MAX_SMASHED 20
# endif
  STATIC ptr_t GC_smashed[MAX_SMASHED] = {0};
  STATIC unsigned GC_n_smashed = 0;

  STATIC void GC_add_smashed(ptr_t smashed)
  {
    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(GC_is_marked(GC_base(smashed)));
    /* FIXME: Prevent adding an object while printing smashed list.     */
    GC_smashed[GC_n_smashed] = smashed;
    if (GC_n_smashed < MAX_SMASHED - 1) ++GC_n_smashed;
                /* In case of overflow, we keep the first MAX_SMASHED-1 */
                /* entries plus the last one.                           */
    GC_SET_HAVE_ERRORS();
  }

  /* Print all objects on the list.  Clear the list.    */
  STATIC void GC_print_all_smashed_proc(void)
  {
    unsigned i;

    GC_ASSERT(I_DONT_HOLD_LOCK());
    if (GC_n_smashed == 0) return;
    GC_err_printf("GC_check_heap_block: found %u smashed heap objects:\n",
                  GC_n_smashed);
    for (i = 0; i < GC_n_smashed; ++i) {
        ptr_t base = (ptr_t)GC_base(GC_smashed[i]);

#       ifdef LINT2
          if (!base) ABORT("Invalid GC_smashed element");
#       endif
        GC_print_smashed_obj("", base + sizeof(oh), GC_smashed[i]);
        GC_smashed[i] = 0;
    }
    GC_n_smashed = 0;
  }

  /* Check all marked objects in the given block for validity   */
  /* Avoid GC_apply_to_each_object for performance reasons.     */
  STATIC void GC_CALLBACK GC_check_heap_block(struct hblk *hbp, void *dummy)
  {
    const hdr *hhdr = HDR(hbp);
    ptr_t p = hbp -> hb_body;
    ptr_t plim;
    size_t sz = hhdr -> hb_sz;
    size_t bit_no;

    UNUSED_ARG(dummy);
    plim = sz > MAXOBJBYTES ? p : p + HBLKSIZE - sz;
    /* Go through all objects in block. */
    for (bit_no = 0; ADDR_GE(plim, p);
         bit_no += MARK_BIT_OFFSET(sz), p += sz) {
      if (mark_bit_from_hdr(hhdr, bit_no) && GC_HAS_DEBUG_INFO(p)) {
        ptr_t clobbered = GC_check_annotated_obj((oh *)p);

        if (clobbered != NULL) GC_add_smashed(clobbered);
      }
    }
  }

  /* This assumes that all accessible objects are marked.   */
  /* Normally called by collector.                          */
  STATIC void GC_check_heap_proc(void)
  {
    GC_ASSERT(I_HOLD_LOCK());
    GC_STATIC_ASSERT((sizeof(oh) & (GC_GRANULE_BYTES-1)) == 0);
    /* FIXME: Should we check for twice that alignment? */
    GC_apply_to_all_blocks(GC_check_heap_block, NULL);
  }

  GC_INNER GC_bool GC_check_leaked(ptr_t base)
  {
    size_t i;
    size_t lpw;
    ptr_t *p;

    if (
#       if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
          (*(GC_uintptr_t *)base & 1) != 0 &&
#       endif
        GC_has_other_debug_info(base) >= 0)
      return TRUE; /* object has leaked */

    /* Validate freed object's content. */
    p = (ptr_t *)(base + sizeof(oh));
    lpw = BYTES_TO_PTRS(HDR(base) -> hb_sz - sizeof(oh));
    for (i = 0; i < lpw; ++i)
      if ((GC_uintptr_t)p[i] != GC_FREED_MEM_MARKER) {
        GC_set_mark_bit(base); /* do not reclaim it in this cycle */
        GC_add_smashed((ptr_t)(&p[i])); /* alter-after-free detected */
        break; /* don't report any other smashed locations in the object */
      }

    return FALSE; /* GC_debug_free() has been called */
  }

#endif /* !SHORT_DBG_HDRS */

#ifndef GC_NO_FINALIZATION

  struct closure {
    GC_finalization_proc cl_fn;
    void * cl_data;
  };

  STATIC void * GC_make_closure(GC_finalization_proc fn, void *data)
  {
    struct closure * result =
#     ifdef DBG_HDRS_ALL
        (struct closure *)GC_debug_malloc(sizeof(struct closure), GC_EXTRAS);
#     else
        (struct closure *)GC_malloc(sizeof(struct closure));
#     endif
    if (result != NULL) {
      result -> cl_fn = fn;
      result -> cl_data = data;
    }
    return result;
  }

  /* An auxiliary function to make finalization work correctly with */
  /* displaced pointers introduced by the debugging allocators.     */
  STATIC void GC_CALLBACK GC_debug_invoke_finalizer(void *obj, void *data)
  {
    struct closure *cl = (struct closure *)data;

    cl -> cl_fn((ptr_t)obj + sizeof(oh), cl -> cl_data);
  }

  /* Special finalizer_proc value to detect GC_register_finalizer failure. */
# define OFN_UNSET ((GC_finalization_proc)~(GC_funcptr_uint)0)

  /* Set ofn and ocd to reflect the values we got back. */
  static void store_old(void *obj, GC_finalization_proc my_old_fn,
                        struct closure *my_old_cd, GC_finalization_proc *ofn,
                        void **ocd)
  {
    if (my_old_fn != 0) {
      if (my_old_fn == OFN_UNSET) {
        /* GC_register_finalizer() failed; (*ofn) and (*ocd) are unchanged. */
        return;
      }
      if (my_old_fn != GC_debug_invoke_finalizer) {
        GC_err_printf("Debuggable object at %p had a non-debug finalizer\n",
                      obj);
        /* This should probably be fatal. */
      } else {
        if (ofn) *ofn = my_old_cd -> cl_fn;
        if (ocd) *ocd = my_old_cd -> cl_data;
      }
    } else {
      if (ofn) *ofn = 0;
      if (ocd) *ocd = NULL;
    }
  }

  GC_API void GC_CALL GC_debug_register_finalizer(void *obj,
                                    GC_finalization_proc fn, void *cd,
                                    GC_finalization_proc *ofn, void **ocd)
  {
    GC_finalization_proc my_old_fn = OFN_UNSET;
    void * my_old_cd = NULL; /* to avoid "might be uninitialized" warning */
    ptr_t base = (ptr_t)GC_base(obj);
    if (NULL == base) {
      /* We will not collect it, hence finalizer wouldn't be run.   */
      if (ocd) *ocd = NULL;
      if (ofn) *ofn = 0;
      return;
    }
    if ((ptr_t)obj - base != sizeof(oh)) {
      GC_err_printf("GC_debug_register_finalizer called with"
                    " non-base-pointer %p\n", obj);
    }
    if (0 == fn) {
      GC_register_finalizer(base, 0, NULL, &my_old_fn, &my_old_cd);
    } else {
      cd = GC_make_closure(fn, cd);
      if (NULL == cd) return; /* out of memory; *ofn and *ocd are unchanged */
      GC_register_finalizer(base, GC_debug_invoke_finalizer, cd,
                            &my_old_fn, &my_old_cd);
    }
    store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd);
  }

  GC_API void GC_CALL GC_debug_register_finalizer_no_order(void *obj,
                                    GC_finalization_proc fn, void *cd,
                                    GC_finalization_proc *ofn, void **ocd)
  {
    GC_finalization_proc my_old_fn = OFN_UNSET;
    void * my_old_cd = NULL;
    ptr_t base = (ptr_t)GC_base(obj);
    if (NULL == base) {
      if (ocd) *ocd = NULL;
      if (ofn) *ofn = 0;
      return;
    }
    if ((ptr_t)obj - base != sizeof(oh)) {
      GC_err_printf("GC_debug_register_finalizer_no_order called with"
                    " non-base-pointer %p\n", obj);
    }
    if (0 == fn) {
      GC_register_finalizer_no_order(base, 0, NULL, &my_old_fn, &my_old_cd);
    } else {
      cd = GC_make_closure(fn, cd);
      if (NULL == cd) return; /* out of memory */
      GC_register_finalizer_no_order(base, GC_debug_invoke_finalizer, cd,
                                     &my_old_fn, &my_old_cd);
    }
    store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd);
  }

  GC_API void GC_CALL GC_debug_register_finalizer_unreachable(void *obj,
                                    GC_finalization_proc fn, void *cd,
                                    GC_finalization_proc *ofn, void **ocd)
  {
    GC_finalization_proc my_old_fn = OFN_UNSET;
    void * my_old_cd = NULL;
    ptr_t base = (ptr_t)GC_base(obj);
    if (NULL == base) {
      if (ocd) *ocd = NULL;
      if (ofn) *ofn = 0;
      return;
    }
    if ((ptr_t)obj - base != sizeof(oh)) {
      GC_err_printf("GC_debug_register_finalizer_unreachable called with"
                    " non-base-pointer %p\n", obj);
    }
    if (0 == fn) {
      GC_register_finalizer_unreachable(base, 0, NULL, &my_old_fn, &my_old_cd);
    } else {
      cd = GC_make_closure(fn, cd);
      if (NULL == cd) return; /* out of memory */
      GC_register_finalizer_unreachable(base, GC_debug_invoke_finalizer, cd,
                                        &my_old_fn, &my_old_cd);
    }
    store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd);
  }

  GC_API void GC_CALL GC_debug_register_finalizer_ignore_self(void *obj,
                                    GC_finalization_proc fn, void *cd,
                                    GC_finalization_proc *ofn, void **ocd)
  {
    GC_finalization_proc my_old_fn = OFN_UNSET;
    void * my_old_cd = NULL;
    ptr_t base = (ptr_t)GC_base(obj);
    if (NULL == base) {
      if (ocd) *ocd = NULL;
      if (ofn) *ofn = 0;
      return;
    }
    if ((ptr_t)obj - base != sizeof(oh)) {
      GC_err_printf("GC_debug_register_finalizer_ignore_self called with"
                    " non-base-pointer %p\n", obj);
    }
    if (0 == fn) {
      GC_register_finalizer_ignore_self(base, 0, NULL, &my_old_fn, &my_old_cd);
    } else {
      cd = GC_make_closure(fn, cd);
      if (NULL == cd) return; /* out of memory */
      GC_register_finalizer_ignore_self(base, GC_debug_invoke_finalizer, cd,
                                        &my_old_fn, &my_old_cd);
    }
    store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd);
  }

# ifndef GC_TOGGLE_REFS_NOT_NEEDED
    GC_API int GC_CALL GC_debug_toggleref_add(void *obj, int is_strong_ref)
    {
      ptr_t base = (ptr_t)GC_base(obj);

      if ((ptr_t)obj - base != sizeof(oh)) {
        GC_err_printf("GC_debug_toggleref_add called with"
                      " non-base-pointer %p\n", obj);
      }
      return GC_toggleref_add(base, is_strong_ref);
    }
# endif /* !GC_TOGGLE_REFS_NOT_NEEDED */

#endif /* !GC_NO_FINALIZATION */

GC_API GC_ATTR_MALLOC void * GC_CALL GC_debug_malloc_replacement(size_t lb)
{
    return GC_debug_malloc(lb, GC_DBG_EXTRAS);
}

GC_API void * GC_CALL GC_debug_realloc_replacement(void *p, size_t lb)
{
    return GC_debug_realloc(p, lb, GC_DBG_EXTRAS);
}

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1996 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 2007 Free Software Foundation, Inc.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#ifndef GC_NO_FINALIZATION
# include "gc/javaxfc.h" /* to get GC_finalize_all() as extern "C" */

/* Type of mark procedure used for marking from finalizable object.     */
/* This procedure normally does not mark the object, only its           */
/* descendants.                                                         */
typedef void (* finalization_mark_proc)(ptr_t /* finalizable_obj_ptr */);

#define HASH3(addr,size,log_size) \
            ((size_t)((ADDR(addr) >> 3) ^ (ADDR(addr) >> (3 + (log_size)))) \
             & ((size)-1))
#define HASH2(addr,log_size) HASH3(addr, (size_t)1 << (log_size), log_size)

struct hash_chain_entry {
    GC_hidden_pointer hidden_key;
    struct hash_chain_entry * next;
};

struct disappearing_link {
    struct hash_chain_entry prolog;
#   define dl_hidden_link prolog.hidden_key
                                /* Field to be cleared.         */
#   define dl_next(x) (struct disappearing_link *)((x) -> prolog.next)
#   define dl_set_next(x, y) \
                (void)((x)->prolog.next = (struct hash_chain_entry *)(y))
    GC_hidden_pointer dl_hidden_obj; /* pointer to object base */
};

struct finalizable_object {
    struct hash_chain_entry prolog;
#   define fo_hidden_base prolog.hidden_key
                                /* Pointer to object base.      */
                                /* No longer hidden once object */
                                /* is on finalize_now queue.    */
#   define fo_next(x) (struct finalizable_object *)((x) -> prolog.next)
#   define fo_set_next(x,y) ((x)->prolog.next = (struct hash_chain_entry *)(y))
    GC_finalization_proc fo_fn; /* finalizer */
    finalization_mark_proc fo_mark_proc; /* mark-through procedure */
    ptr_t fo_client_data;
    size_t fo_object_sz; /* in bytes */
};

#ifdef AO_HAVE_store
  /* Update finalize_now atomically as GC_should_invoke_finalizers does */
  /* not acquire the allocator lock.                                    */
# define SET_FINALIZE_NOW(fo) \
            GC_cptr_store((volatile ptr_t *)&GC_fnlz_roots.finalize_now, \
                          (ptr_t)(fo))
#else
# define SET_FINALIZE_NOW(fo) (void)(GC_fnlz_roots.finalize_now = (fo))
#endif /* !THREADS */

GC_API void GC_CALL GC_push_finalizer_structures(void)
{
  GC_ASSERT(ADDR(&GC_dl_hashtbl.head) % sizeof(ptr_t) == 0);
  GC_ASSERT(ADDR(&GC_fnlz_roots) % sizeof(ptr_t) == 0);
# ifndef GC_LONG_REFS_NOT_NEEDED
    GC_ASSERT(ADDR(&GC_ll_hashtbl.head) % sizeof(ptr_t) == 0);
    GC_PUSH_ALL_SYM(GC_ll_hashtbl.head);
# endif
  GC_PUSH_ALL_SYM(GC_dl_hashtbl.head);
  GC_PUSH_ALL_SYM(GC_fnlz_roots);
  /* GC_toggleref_arr is pushed specially by GC_mark_togglerefs.        */
}

/* Threshold of log_size to initiate full collection before growing     */
/* a hash table.                                                        */
#ifndef GC_ON_GROW_LOG_SIZE_MIN
# define GC_ON_GROW_LOG_SIZE_MIN LOG_HBLKSIZE
#endif

/* Double the size of a hash table. *log_size_ptr is the log of its     */
/* current size.  May be a no-op.  *table is a pointer to an array of   */
/* hash headers.  We update both *table and *log_size_ptr on success.   */
STATIC void GC_grow_table(struct hash_chain_entry ***table,
                          unsigned *log_size_ptr, const size_t *entries_ptr)
{
    size_t i;
    struct hash_chain_entry *p;
    unsigned log_old_size = *log_size_ptr;
    unsigned log_new_size = log_old_size + 1;
    size_t old_size = NULL == *table ? 0 : (size_t)1 << log_old_size;
    size_t new_size = (size_t)1 << log_new_size;
    /* FIXME: Power of 2 size often gets rounded up to one more page. */
    struct hash_chain_entry **new_table;

    GC_ASSERT(I_HOLD_LOCK());
    /* Avoid growing the table in case of at least 25% of entries can   */
    /* be deleted by enforcing a collection.  Ignored for small tables. */
    /* In incremental mode we skip this optimization, as we want to     */
    /* avoid triggering a full GC whenever possible.                    */
    if (log_old_size >= (unsigned)GC_ON_GROW_LOG_SIZE_MIN && !GC_incremental) {
      IF_CANCEL(int cancel_state;)

      DISABLE_CANCEL(cancel_state);
      GC_gcollect_inner();
      RESTORE_CANCEL(cancel_state);
      /* GC_finalize might decrease entries value.  */
      if (*entries_ptr < ((size_t)1 << log_old_size) - (*entries_ptr >> 2))
        return;
    }

    new_table = (struct hash_chain_entry **)
                    GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(
                        new_size * sizeof(struct hash_chain_entry *), NORMAL);
    if (NULL == new_table) {
        if (NULL == *table) {
            ABORT("Insufficient space for initial table allocation");
        } else {
            return;
        }
    }
    for (i = 0; i < old_size; i++) {
      for (p = (*table)[i]; p != NULL;) {
        ptr_t real_key = (ptr_t)GC_REVEAL_POINTER(p -> hidden_key);
        struct hash_chain_entry *next = p -> next;
        size_t new_hash = HASH3(real_key, new_size, log_new_size);

        p -> next = new_table[new_hash];
        GC_dirty(p);
        new_table[new_hash] = p;
        p = next;
      }
    }
    *log_size_ptr = log_new_size;
    *table = new_table;
    GC_dirty(new_table); /* entire object */
}

GC_API int GC_CALL GC_register_disappearing_link(void * * link)
{
    ptr_t base;

    base = (ptr_t)GC_base(link);
    if (base == 0)
        ABORT("Bad arg to GC_register_disappearing_link");
    return GC_general_register_disappearing_link(link, base);
}

STATIC int GC_register_disappearing_link_inner(
                        struct dl_hashtbl_s *dl_hashtbl, void **link,
                        const void *obj, const char *tbl_log_name)
{
    struct disappearing_link *curr_dl;
    size_t index;
    struct disappearing_link * new_dl;

    GC_ASSERT(GC_is_initialized);
    if (EXPECT(GC_find_leak, FALSE)) return GC_UNIMPLEMENTED;
#   ifdef GC_ASSERTIONS
      GC_noop1_ptr(*link); /* check accessibility */
#   endif
    LOCK();
    GC_ASSERT(obj != NULL && GC_base_C(obj) == obj);
    if (EXPECT(NULL == dl_hashtbl -> head, FALSE)
        || EXPECT(dl_hashtbl -> entries
                  > ((size_t)1 << dl_hashtbl -> log_size), FALSE)) {
        GC_grow_table((struct hash_chain_entry ***)&dl_hashtbl -> head,
                      &dl_hashtbl -> log_size, &dl_hashtbl -> entries);
        GC_COND_LOG_PRINTF("Grew %s table to %u entries\n", tbl_log_name,
                           1U << dl_hashtbl -> log_size);
    }
    index = HASH2(link, dl_hashtbl -> log_size);
    for (curr_dl = dl_hashtbl -> head[index]; curr_dl != 0;
         curr_dl = dl_next(curr_dl)) {
        if (curr_dl -> dl_hidden_link == GC_HIDE_POINTER(link)) {
            /* Alternatively, GC_HIDE_NZ_POINTER() could be used instead. */
            curr_dl -> dl_hidden_obj = GC_HIDE_POINTER(obj);
            UNLOCK();
            return GC_DUPLICATE;
        }
    }
    new_dl = (struct disappearing_link *)
        GC_INTERNAL_MALLOC(sizeof(struct disappearing_link), NORMAL);
    if (EXPECT(NULL == new_dl, FALSE)) {
      GC_oom_func oom_fn = GC_oom_fn;
      UNLOCK();
      new_dl = (struct disappearing_link *)
                (*oom_fn)(sizeof(struct disappearing_link));
      if (0 == new_dl) {
        return GC_NO_MEMORY;
      }
      /* It's not likely we'll make it here, but ... */
      LOCK();
      /* Recalculate index since the table may grow.    */
      index = HASH2(link, dl_hashtbl -> log_size);
      /* Check again that our disappearing link not in the table. */
      for (curr_dl = dl_hashtbl -> head[index]; curr_dl != 0;
           curr_dl = dl_next(curr_dl)) {
        if (curr_dl -> dl_hidden_link == GC_HIDE_POINTER(link)) {
          curr_dl -> dl_hidden_obj = GC_HIDE_POINTER(obj);
          UNLOCK();
#         ifndef DBG_HDRS_ALL
            /* Free unused new_dl returned by GC_oom_fn() */
            GC_free(new_dl);
#         endif
          return GC_DUPLICATE;
        }
      }
    }
    new_dl -> dl_hidden_obj = GC_HIDE_POINTER(obj);
    new_dl -> dl_hidden_link = GC_HIDE_POINTER(link);
    dl_set_next(new_dl, dl_hashtbl -> head[index]);
    GC_dirty(new_dl);
    dl_hashtbl -> head[index] = new_dl;
    dl_hashtbl -> entries++;
    GC_dirty(dl_hashtbl->head + index);
    UNLOCK();
    return GC_SUCCESS;
}

GC_API int GC_CALL GC_general_register_disappearing_link(void * * link,
                                                         const void * obj)
{
    if ((ADDR(link) & (ALIGNMENT-1)) != 0 || !NONNULL_ARG_NOT_NULL(link))
        ABORT("Bad arg to GC_general_register_disappearing_link");
    return GC_register_disappearing_link_inner(&GC_dl_hashtbl, link, obj,
                                               "dl");
}

#ifdef DBG_HDRS_ALL
# define FREE_DL_ENTRY(curr_dl) dl_set_next(curr_dl, NULL)
#else
# define FREE_DL_ENTRY(curr_dl) GC_free(curr_dl)
#endif

/* Unregisters given link and returns the link entry to free.   */
GC_INLINE struct disappearing_link *GC_unregister_disappearing_link_inner(
                                struct dl_hashtbl_s *dl_hashtbl, void **link)
{
    struct disappearing_link *curr_dl;
    struct disappearing_link *prev_dl = NULL;
    size_t index;

    GC_ASSERT(I_HOLD_LOCK());
    if (EXPECT(NULL == dl_hashtbl -> head, FALSE)) return NULL;

    index = HASH2(link, dl_hashtbl -> log_size);
    for (curr_dl = dl_hashtbl -> head[index]; curr_dl;
         curr_dl = dl_next(curr_dl)) {
        if (curr_dl -> dl_hidden_link == GC_HIDE_POINTER(link)) {
            /* Remove found entry from the table. */
            if (NULL == prev_dl) {
                dl_hashtbl -> head[index] = dl_next(curr_dl);
                GC_dirty(dl_hashtbl->head + index);
            } else {
                dl_set_next(prev_dl, dl_next(curr_dl));
                GC_dirty(prev_dl);
            }
            dl_hashtbl -> entries--;
            break;
        }
        prev_dl = curr_dl;
    }
    return curr_dl;
}

GC_API int GC_CALL GC_unregister_disappearing_link(void * * link)
{
    struct disappearing_link *curr_dl;

    if ((ADDR(link) & (ALIGNMENT-1)) != 0) return 0; /* nothing to do */

    LOCK();
    curr_dl = GC_unregister_disappearing_link_inner(&GC_dl_hashtbl, link);
    UNLOCK();
    if (NULL == curr_dl) return 0;
    FREE_DL_ENTRY(curr_dl);
    return 1;
}

/* Mark from one finalizable object using the specified mark proc.      */
/* May not mark the object pointed to by real_ptr (i.e, it is the job   */
/* of the caller, if appropriate).  Note that this is called with the   */
/* mutator running.  This is safe only if the mutator (client) gets     */
/* the allocator lock to reveal hidden pointers.                        */
GC_INLINE void GC_mark_fo(ptr_t real_ptr, finalization_mark_proc fo_mark_proc)
{
  GC_ASSERT(I_HOLD_LOCK());
  fo_mark_proc(real_ptr);
  /* Process objects pushed by the mark procedure.      */
  while (!GC_mark_stack_empty())
    MARK_FROM_MARK_STACK();
}

/* Complete a collection in progress, if any.   */
GC_INLINE void GC_complete_ongoing_collection(void) {
  if (EXPECT(GC_collection_in_progress(), FALSE)) {
    while (!GC_mark_some(NULL)) { /* empty */ }
  }
}

/* Toggle-ref support.  */
#ifndef GC_TOGGLE_REFS_NOT_NEEDED
  typedef union toggle_ref_u GCToggleRef;

  STATIC GC_toggleref_func GC_toggleref_callback = 0;

  GC_INNER void GC_process_togglerefs(void)
  {
    size_t i;
    size_t new_size = 0;
    GC_bool needs_barrier = FALSE;

    GC_ASSERT(I_HOLD_LOCK());
    for (i = 0; i < GC_toggleref_array_size; ++i) {
      GCToggleRef *r = &GC_toggleref_arr[i];
      void *obj = r -> strong_ref;

      if ((ADDR(obj) & 1) != 0) {
        obj = GC_REVEAL_POINTER(r -> weak_ref);
        GC_ASSERT((ADDR(obj) & 1) == 0);
      }
      if (NULL == obj) continue;

      switch (GC_toggleref_callback(obj)) {
      case GC_TOGGLE_REF_DROP:
        break;
      case GC_TOGGLE_REF_STRONG:
        GC_toggleref_arr[new_size++].strong_ref = obj;
        needs_barrier = TRUE;
        break;
      case GC_TOGGLE_REF_WEAK:
        GC_toggleref_arr[new_size++].weak_ref = GC_HIDE_POINTER(obj);
        break;
      default:
        ABORT("Bad toggle-ref status returned by callback");
      }
    }

    if (new_size < GC_toggleref_array_size) {
      BZERO(&GC_toggleref_arr[new_size],
            (GC_toggleref_array_size - new_size) * sizeof(GCToggleRef));
      GC_toggleref_array_size = new_size;
    }
    if (needs_barrier)
      GC_dirty(GC_toggleref_arr); /* entire object */
  }

  STATIC void GC_normal_finalize_mark_proc(ptr_t);

  STATIC void GC_mark_togglerefs(void)
  {
    size_t i;

    GC_ASSERT(I_HOLD_LOCK());
    if (NULL == GC_toggleref_arr)
      return;

    GC_set_mark_bit(GC_toggleref_arr);
    for (i = 0; i < GC_toggleref_array_size; ++i) {
      void *obj = GC_toggleref_arr[i].strong_ref;
      if (obj != NULL && (ADDR(obj) & 1) == 0) {
        /* Push and mark the object.    */
        GC_mark_fo((ptr_t)obj, GC_normal_finalize_mark_proc);
        GC_set_mark_bit(obj);
        GC_complete_ongoing_collection();
      }
    }
  }

  STATIC void GC_clear_togglerefs(void)
  {
    size_t i;

    GC_ASSERT(I_HOLD_LOCK());
    for (i = 0; i < GC_toggleref_array_size; ++i) {
      GCToggleRef *r = &GC_toggleref_arr[i];

      if ((ADDR(r -> strong_ref) & 1) != 0) {
        if (!GC_is_marked(GC_REVEAL_POINTER(r -> weak_ref))) {
          r -> weak_ref = 0;
        } else {
          /* No need to copy, BDWGC is a non-moving collector.    */
        }
      }
    }
  }

  GC_API void GC_CALL GC_set_toggleref_func(GC_toggleref_func fn)
  {
    LOCK();
    GC_toggleref_callback = fn;
    UNLOCK();
  }

  GC_API GC_toggleref_func GC_CALL GC_get_toggleref_func(void)
  {
    GC_toggleref_func fn;

    READER_LOCK();
    fn = GC_toggleref_callback;
    READER_UNLOCK();
    return fn;
  }

  static GC_bool ensure_toggleref_capacity(size_t capacity_inc)
  {
    GC_ASSERT(I_HOLD_LOCK());
    if (NULL == GC_toggleref_arr) {
      GC_toggleref_array_capacity = 32; /* initial capacity */
      GC_toggleref_arr = (GCToggleRef *)GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(
                        GC_toggleref_array_capacity * sizeof(GCToggleRef),
                        NORMAL);
      if (NULL == GC_toggleref_arr)
        return FALSE;
    }
    if (GC_toggleref_array_size + capacity_inc
        >= GC_toggleref_array_capacity) {
      GCToggleRef *new_array;
      while (GC_toggleref_array_capacity
              < GC_toggleref_array_size + capacity_inc) {
        GC_toggleref_array_capacity *= 2;
        if ((GC_toggleref_array_capacity
             & ((size_t)1 << (sizeof(size_t) * 8 - 1))) != 0)
          return FALSE; /* overflow */
      }

      new_array = (GCToggleRef *)GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(
                        GC_toggleref_array_capacity * sizeof(GCToggleRef),
                        NORMAL);
      if (NULL == new_array)
        return FALSE;
      if (EXPECT(GC_toggleref_array_size > 0, TRUE))
        BCOPY(GC_toggleref_arr, new_array,
              GC_toggleref_array_size * sizeof(GCToggleRef));
      GC_INTERNAL_FREE(GC_toggleref_arr);
      GC_toggleref_arr = new_array;
    }
    return TRUE;
  }

  GC_API int GC_CALL GC_toggleref_add(void *obj, int is_strong_ref)
  {
    int res = GC_SUCCESS;

    GC_ASSERT(NONNULL_ARG_NOT_NULL(obj));
    LOCK();
    GC_ASSERT((ADDR(obj) & 1) == 0 && obj == GC_base(obj));
    if (GC_toggleref_callback != 0) {
      if (!ensure_toggleref_capacity(1)) {
        res = GC_NO_MEMORY;
      } else {
        GCToggleRef *r = &GC_toggleref_arr[GC_toggleref_array_size];

        if (is_strong_ref) {
          r -> strong_ref = obj;
          GC_dirty(GC_toggleref_arr + GC_toggleref_array_size);
        } else {
          r -> weak_ref = GC_HIDE_POINTER(obj);
          GC_ASSERT((r -> weak_ref & 1) != 0);
        }
        GC_toggleref_array_size++;
      }
    }
    UNLOCK();
    return res;
  }
#endif /* !GC_TOGGLE_REFS_NOT_NEEDED */

/* Finalizer callback support. */
STATIC GC_await_finalize_proc GC_object_finalized_proc = 0;

GC_API void GC_CALL GC_set_await_finalize_proc(GC_await_finalize_proc fn)
{
  LOCK();
  GC_object_finalized_proc = fn;
  UNLOCK();
}

GC_API GC_await_finalize_proc GC_CALL GC_get_await_finalize_proc(void)
{
  GC_await_finalize_proc fn;

  READER_LOCK();
  fn = GC_object_finalized_proc;
  READER_UNLOCK();
  return fn;
}

#ifndef GC_LONG_REFS_NOT_NEEDED
  GC_API int GC_CALL GC_register_long_link(void * * link, const void * obj)
  {
    if ((ADDR(link) & (ALIGNMENT-1)) != 0 || !NONNULL_ARG_NOT_NULL(link))
        ABORT("Bad arg to GC_register_long_link");
    return GC_register_disappearing_link_inner(&GC_ll_hashtbl, link, obj,
                                               "long dl");
  }

  GC_API int GC_CALL GC_unregister_long_link(void * * link)
  {
    struct disappearing_link *curr_dl;

    if ((ADDR(link) & (ALIGNMENT-1)) != 0) return 0; /* nothing to do */

    LOCK();
    curr_dl = GC_unregister_disappearing_link_inner(&GC_ll_hashtbl, link);
    UNLOCK();
    if (NULL == curr_dl) return 0;
    FREE_DL_ENTRY(curr_dl);
    return 1;
  }
#endif /* !GC_LONG_REFS_NOT_NEEDED */

#ifndef GC_MOVE_DISAPPEARING_LINK_NOT_NEEDED
  STATIC int GC_move_disappearing_link_inner(
                                struct dl_hashtbl_s *dl_hashtbl,
                                void **link, void **new_link)
  {
    struct disappearing_link *curr_dl, *new_dl;
    struct disappearing_link *prev_dl = NULL;
    size_t curr_index, new_index;
    GC_hidden_pointer curr_hidden_link, new_hidden_link;

#   ifdef GC_ASSERTIONS
      GC_noop1_ptr(*new_link);
#   endif
    GC_ASSERT(I_HOLD_LOCK());
    if (EXPECT(NULL == dl_hashtbl -> head, FALSE)) return GC_NOT_FOUND;

    /* Find current link.       */
    curr_index = HASH2(link, dl_hashtbl -> log_size);
    curr_hidden_link = GC_HIDE_POINTER(link);
    for (curr_dl = dl_hashtbl -> head[curr_index]; curr_dl;
         curr_dl = dl_next(curr_dl)) {
      if (curr_dl -> dl_hidden_link == curr_hidden_link)
        break;
      prev_dl = curr_dl;
    }
    if (EXPECT(NULL == curr_dl, FALSE)) {
      return GC_NOT_FOUND;
    } else if (link == new_link) {
      return GC_SUCCESS; /* Nothing to do.      */
    }

    /* link found; now check new_link not present.      */
    new_index = HASH2(new_link, dl_hashtbl -> log_size);
    new_hidden_link = GC_HIDE_POINTER(new_link);
    for (new_dl = dl_hashtbl -> head[new_index]; new_dl;
         new_dl = dl_next(new_dl)) {
      if (new_dl -> dl_hidden_link == new_hidden_link) {
        /* Target already registered; bail.     */
        return GC_DUPLICATE;
      }
    }

    /* Remove from old, add to new, update link.        */
    if (NULL == prev_dl) {
      dl_hashtbl -> head[curr_index] = dl_next(curr_dl);
    } else {
      dl_set_next(prev_dl, dl_next(curr_dl));
      GC_dirty(prev_dl);
    }
    curr_dl -> dl_hidden_link = new_hidden_link;
    dl_set_next(curr_dl, dl_hashtbl -> head[new_index]);
    dl_hashtbl -> head[new_index] = curr_dl;
    GC_dirty(curr_dl);
    GC_dirty(dl_hashtbl->head); /* entire object */
    return GC_SUCCESS;
  }

  GC_API int GC_CALL GC_move_disappearing_link(void **link, void **new_link)
  {
    int result;

    if ((ADDR(new_link) & (ALIGNMENT-1)) != 0
        || !NONNULL_ARG_NOT_NULL(new_link))
      ABORT("Bad new_link arg to GC_move_disappearing_link");
    if ((ADDR(link) & (ALIGNMENT-1)) != 0)
      return GC_NOT_FOUND; /* Nothing to do. */

    LOCK();
    result = GC_move_disappearing_link_inner(&GC_dl_hashtbl, link, new_link);
    UNLOCK();
    return result;
  }

# ifndef GC_LONG_REFS_NOT_NEEDED
    GC_API int GC_CALL GC_move_long_link(void **link, void **new_link)
    {
      int result;

      if ((ADDR(new_link) & (ALIGNMENT-1)) != 0
          || !NONNULL_ARG_NOT_NULL(new_link))
        ABORT("Bad new_link arg to GC_move_long_link");
      if ((ADDR(link) & (ALIGNMENT-1)) != 0)
        return GC_NOT_FOUND; /* Nothing to do. */

      LOCK();
      result = GC_move_disappearing_link_inner(&GC_ll_hashtbl, link, new_link);
      UNLOCK();
      return result;
    }
# endif /* !GC_LONG_REFS_NOT_NEEDED */
#endif /* !GC_MOVE_DISAPPEARING_LINK_NOT_NEEDED */

/* Possible finalization_marker procedures.  Note that mark stack       */
/* overflow is handled by the caller, and is not a disaster.            */
#if defined(_MSC_VER) && defined(I386)
  GC_ATTR_NOINLINE
  /* Otherwise some optimizer bug is tickled in VC for x86 (v19, at least). */
#endif
STATIC void GC_normal_finalize_mark_proc(ptr_t p)
{
    GC_mark_stack_top = GC_push_obj(p, HDR(p), GC_mark_stack_top,
                                    GC_mark_stack + GC_mark_stack_size);
}

/* This only pays very partial attention to the mark descriptor.        */
/* It does the right thing for normal and atomic objects, and treats    */
/* most others as normal.                                               */
STATIC void GC_ignore_self_finalize_mark_proc(ptr_t p)
{
    const hdr *hhdr = HDR(p);
    word descr = hhdr -> hb_descr;
    ptr_t current_p;
    ptr_t scan_limit;
    ptr_t target_limit = p + hhdr -> hb_sz - 1;

    if ((descr & GC_DS_TAGS) == GC_DS_LENGTH) {
       scan_limit = p + descr - sizeof(ptr_t);
    } else {
       scan_limit = target_limit + 1 - sizeof(ptr_t);
    }
    for (current_p = p; ADDR_GE(scan_limit, current_p);
         current_p += ALIGNMENT) {
        ptr_t q;

        LOAD_PTR_OR_CONTINUE(q, current_p);
        if (ADDR_LT(q, p) || ADDR_LT(target_limit, q)) {
            GC_PUSH_ONE_HEAP(q, current_p, GC_mark_stack_top);
        }
    }
}

STATIC void GC_null_finalize_mark_proc(ptr_t p)
{
    UNUSED_ARG(p);
}

/* Possible finalization_marker procedures.  Note that mark stack       */
/* overflow is handled by the caller, and is not a disaster.            */

/* GC_unreachable_finalize_mark_proc is an alias for normal marking,    */
/* but it is explicitly tested for, and triggers different              */
/* behavior.  Objects registered in this way are not finalized          */
/* if they are reachable by other finalizable objects, even if those    */
/* other objects specify no ordering.                                   */
STATIC void GC_unreachable_finalize_mark_proc(ptr_t p)
{
    /* A dummy comparison to ensure the compiler not to optimize two    */
    /* identical functions into a single one (thus, to ensure a unique  */
    /* address of each).  Alternatively, GC_noop1_ptr(p) could be used. */
    if (EXPECT(NULL == p, FALSE)) return;

    GC_normal_finalize_mark_proc(p);
}

static GC_bool need_unreachable_finalization = FALSE;
        /* Avoid the work if this is not used.  */
        /* TODO: turn need_unreachable_finalization into a counter */

/* Register a finalization function.  See gc.h for details.     */
/* The last parameter is a procedure that determines            */
/* marking for finalization ordering.  Any objects marked       */
/* by that procedure will be guaranteed to not have been        */
/* finalized when this finalizer is invoked.                    */
STATIC void GC_register_finalizer_inner(void * obj,
                                        GC_finalization_proc fn, void *cd,
                                        GC_finalization_proc *ofn, void **ocd,
                                        finalization_mark_proc mp)
{
    struct finalizable_object * curr_fo;
    size_t index;
    struct finalizable_object *new_fo = 0;
    const hdr *hhdr = NULL; /* initialized to prevent warning. */

    GC_ASSERT(GC_is_initialized);
    if (EXPECT(GC_find_leak, FALSE)) {
      /* No-op.  *ocd and *ofn remain unchanged.    */
      return;
    }
    LOCK();
    GC_ASSERT(obj != NULL && GC_base_C(obj) == obj);
    if (mp == GC_unreachable_finalize_mark_proc)
        need_unreachable_finalization = TRUE;
    if (EXPECT(NULL == GC_fnlz_roots.fo_head, FALSE)
        || EXPECT(GC_fo_entries
                    > ((size_t)1 << GC_log_fo_table_size), FALSE)) {
        GC_grow_table((struct hash_chain_entry ***)&GC_fnlz_roots.fo_head,
                      &GC_log_fo_table_size, &GC_fo_entries);
        GC_COND_LOG_PRINTF("Grew fo table to %u entries\n",
                           1U << GC_log_fo_table_size);
    }
    for (;;) {
      struct finalizable_object *prev_fo = NULL;
      GC_oom_func oom_fn;

      index = HASH2(obj, GC_log_fo_table_size);
      curr_fo = GC_fnlz_roots.fo_head[index];
      while (curr_fo != NULL) {
        GC_ASSERT(GC_size(curr_fo) >= sizeof(struct finalizable_object));
        if (curr_fo -> fo_hidden_base == GC_HIDE_POINTER(obj)) {
          /* Interruption by a signal in the middle of this     */
          /* should be safe.  The client may see only *ocd      */
          /* updated, but we'll declare that to be his problem. */
          if (ocd) *ocd = curr_fo -> fo_client_data;
          if (ofn) *ofn = curr_fo -> fo_fn;
          /* Delete the structure for obj.      */
          if (prev_fo == 0) {
            GC_fnlz_roots.fo_head[index] = fo_next(curr_fo);
          } else {
            fo_set_next(prev_fo, fo_next(curr_fo));
            GC_dirty(prev_fo);
          }
          if (fn == 0) {
            GC_fo_entries--;
            /* May not happen if we get a signal.  But a high   */
            /* estimate will only make the table larger than    */
            /* necessary.                                       */
#           if !defined(THREADS) && !defined(DBG_HDRS_ALL)
              GC_free(curr_fo);
#           endif
          } else {
            curr_fo -> fo_fn = fn;
            curr_fo -> fo_client_data = (ptr_t)cd;
            curr_fo -> fo_mark_proc = mp;
            GC_dirty(curr_fo);
            /* Reinsert it.  We deleted it first to maintain    */
            /* consistency in the event of a signal.            */
            if (prev_fo == 0) {
              GC_fnlz_roots.fo_head[index] = curr_fo;
            } else {
              fo_set_next(prev_fo, curr_fo);
              GC_dirty(prev_fo);
            }
          }
          if (NULL == prev_fo)
            GC_dirty(GC_fnlz_roots.fo_head + index);
          UNLOCK();
#         ifndef DBG_HDRS_ALL
              /* Free unused new_fo returned by GC_oom_fn() */
              GC_free(new_fo);
#         endif
          return;
        }
        prev_fo = curr_fo;
        curr_fo = fo_next(curr_fo);
      }
      if (EXPECT(new_fo != 0, FALSE)) {
        /* new_fo is returned by GC_oom_fn().   */
        GC_ASSERT(fn != 0);
#       ifdef LINT2
          if (NULL == hhdr) ABORT("Bad hhdr in GC_register_finalizer_inner");
#       endif
        break;
      }
      if (fn == 0) {
        if (ocd) *ocd = 0;
        if (ofn) *ofn = 0;
        UNLOCK();
        return;
      }
      GET_HDR(obj, hhdr);
      if (EXPECT(NULL == hhdr, FALSE)) {
        /* We won't collect it, hence finalizer wouldn't be run. */
        if (ocd) *ocd = 0;
        if (ofn) *ofn = 0;
        UNLOCK();
        return;
      }
      new_fo = (struct finalizable_object *)
        GC_INTERNAL_MALLOC(sizeof(struct finalizable_object), NORMAL);
      if (EXPECT(new_fo != 0, TRUE))
        break;
      oom_fn = GC_oom_fn;
      UNLOCK();
      new_fo = (struct finalizable_object *)
                (*oom_fn)(sizeof(struct finalizable_object));
      if (0 == new_fo) {
        /* No enough memory.  *ocd and *ofn remain unchanged.   */
        return;
      }
      /* It's not likely we'll make it here, but ... */
      LOCK();
      /* Recalculate index since the table may grow and         */
      /* check again that our finalizer is not in the table.    */
    }
    GC_ASSERT(GC_size(new_fo) >= sizeof(struct finalizable_object));
    if (ocd) *ocd = 0;
    if (ofn) *ofn = 0;
    new_fo -> fo_hidden_base = GC_HIDE_POINTER(obj);
    new_fo -> fo_fn = fn;
    new_fo -> fo_client_data = (ptr_t)cd;
    new_fo -> fo_object_sz = hhdr -> hb_sz;
    new_fo -> fo_mark_proc = mp;
    fo_set_next(new_fo, GC_fnlz_roots.fo_head[index]);
    GC_dirty(new_fo);
    GC_fo_entries++;
    GC_fnlz_roots.fo_head[index] = new_fo;
    GC_dirty(GC_fnlz_roots.fo_head + index);
    UNLOCK();
}

GC_API void GC_CALL GC_register_finalizer(void * obj,
                                  GC_finalization_proc fn, void * cd,
                                  GC_finalization_proc *ofn, void ** ocd)
{
    GC_register_finalizer_inner(obj, fn, cd, ofn,
                                ocd, GC_normal_finalize_mark_proc);
}

GC_API void GC_CALL GC_register_finalizer_ignore_self(void * obj,
                               GC_finalization_proc fn, void * cd,
                               GC_finalization_proc *ofn, void ** ocd)
{
    GC_register_finalizer_inner(obj, fn, cd, ofn,
                                ocd, GC_ignore_self_finalize_mark_proc);
}

GC_API void GC_CALL GC_register_finalizer_no_order(void * obj,
                               GC_finalization_proc fn, void * cd,
                               GC_finalization_proc *ofn, void ** ocd)
{
    GC_register_finalizer_inner(obj, fn, cd, ofn,
                                ocd, GC_null_finalize_mark_proc);
}

GC_API void GC_CALL GC_register_finalizer_unreachable(void * obj,
                               GC_finalization_proc fn, void * cd,
                               GC_finalization_proc *ofn, void ** ocd)
{
    GC_ASSERT(GC_java_finalization);
    GC_register_finalizer_inner(obj, fn, cd, ofn,
                                ocd, GC_unreachable_finalize_mark_proc);
}

#ifndef NO_DEBUGGING
  STATIC void GC_dump_finalization_links(
                                const struct dl_hashtbl_s *dl_hashtbl)
  {
    size_t dl_size = (size_t)1 << dl_hashtbl -> log_size;
    size_t i;

    if (NULL == dl_hashtbl -> head) return; /* empty table  */

    for (i = 0; i < dl_size; i++) {
      struct disappearing_link *curr_dl;

      for (curr_dl = dl_hashtbl -> head[i]; curr_dl != 0;
           curr_dl = dl_next(curr_dl)) {
        ptr_t real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_dl -> dl_hidden_obj);
        ptr_t real_link = (ptr_t)GC_REVEAL_POINTER(curr_dl -> dl_hidden_link);

        GC_printf("Object: %p, link value: %p, link addr: %p\n",
                  (void *)real_ptr, *(void **)real_link, (void *)real_link);
      }
    }
  }

  GC_API void GC_CALL GC_dump_finalization(void)
  {
    struct finalizable_object * curr_fo;
    size_t i;
    size_t fo_size = GC_fnlz_roots.fo_head == NULL ? 0 :
                                (size_t)1 << GC_log_fo_table_size;

    GC_printf("\n***Disappearing (short) links:\n");
    GC_dump_finalization_links(&GC_dl_hashtbl);
#   ifndef GC_LONG_REFS_NOT_NEEDED
      GC_printf("\n***Disappearing long links:\n");
      GC_dump_finalization_links(&GC_ll_hashtbl);
#   endif
    GC_printf("\n***Finalizers:\n");
    for (i = 0; i < fo_size; i++) {
      for (curr_fo = GC_fnlz_roots.fo_head[i];
           curr_fo != NULL; curr_fo = fo_next(curr_fo)) {
        ptr_t real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo -> fo_hidden_base);

        GC_printf("Finalizable object: %p\n", (void *)real_ptr);
      }
    }
  }
#endif /* !NO_DEBUGGING */

#ifndef SMALL_CONFIG
  STATIC size_t GC_old_dl_entries = 0; /* for stats printing */
# ifndef GC_LONG_REFS_NOT_NEEDED
    STATIC size_t GC_old_ll_entries = 0;
# endif
#endif /* !SMALL_CONFIG */

#ifndef THREADS
  /* Global variables to minimize the level of recursion when a client  */
  /* finalizer allocates memory.                                        */
  STATIC int GC_finalizer_nested = 0;
                        /* Only the lowest byte is used, the rest is    */
                        /* padding for proper global data alignment     */
                        /* required for some compilers (like Watcom).   */
  STATIC unsigned GC_finalizer_skipped = 0;

  /* Checks and updates the level of finalizers recursion.              */
  /* Returns NULL if GC_invoke_finalizers() should not be called by the */
  /* collector (to minimize the risk of a deep finalizers recursion),   */
  /* otherwise returns a pointer to GC_finalizer_nested.                */
  STATIC unsigned char *GC_check_finalizer_nested(void)
  {
    unsigned nesting_level = *(unsigned char *)&GC_finalizer_nested;
    if (nesting_level) {
      /* We are inside another GC_invoke_finalizers().          */
      /* Skip some implicitly-called GC_invoke_finalizers()     */
      /* depending on the nesting (recursion) level.            */
      if (++GC_finalizer_skipped < (1U << nesting_level)) return NULL;
      GC_finalizer_skipped = 0;
    }
    *(char *)&GC_finalizer_nested = (char)(nesting_level + 1);
    return (unsigned char *)&GC_finalizer_nested;
  }
#endif /* !THREADS */

GC_INLINE void GC_make_disappearing_links_disappear(
                                        struct dl_hashtbl_s* dl_hashtbl,
                                        GC_bool is_remove_dangling)
{
  size_t i;
  size_t dl_size = (size_t)1 << dl_hashtbl -> log_size;
  GC_bool needs_barrier = FALSE;

  GC_ASSERT(I_HOLD_LOCK());
  if (NULL == dl_hashtbl -> head) return; /* empty table  */

  for (i = 0; i < dl_size; i++) {
    struct disappearing_link *curr_dl, *next_dl;
    struct disappearing_link *prev_dl = NULL;

    for (curr_dl = dl_hashtbl->head[i]; curr_dl != NULL; curr_dl = next_dl) {
      next_dl = dl_next(curr_dl);
#     if defined(GC_ASSERTIONS) && !defined(THREAD_SANITIZER)
         /* Check accessibility of the location pointed by link. */
        GC_noop1_ptr(*(ptr_t *)GC_REVEAL_POINTER(curr_dl -> dl_hidden_link));
#     endif
      if (is_remove_dangling) {
        ptr_t real_link = (ptr_t)GC_base(GC_REVEAL_POINTER(
                                                curr_dl -> dl_hidden_link));

        if (NULL == real_link || EXPECT(GC_is_marked(real_link), TRUE)) {
          prev_dl = curr_dl;
          continue;
        }
      } else {
        if (EXPECT(GC_is_marked((ptr_t)GC_REVEAL_POINTER(
                                        curr_dl -> dl_hidden_obj)), TRUE)) {
          prev_dl = curr_dl;
          continue;
        }
        *(ptr_t *)GC_REVEAL_POINTER(curr_dl -> dl_hidden_link) = NULL;
      }

      /* Delete curr_dl entry from dl_hashtbl.  */
      if (NULL == prev_dl) {
        dl_hashtbl -> head[i] = next_dl;
        needs_barrier = TRUE;
      } else {
        dl_set_next(prev_dl, next_dl);
        GC_dirty(prev_dl);
      }
      GC_clear_mark_bit(curr_dl);
      dl_hashtbl -> entries--;
    }
  }
  if (needs_barrier)
    GC_dirty(dl_hashtbl -> head); /* entire object */
}

/* Cause disappearing links to disappear and unreachable objects to be  */
/* enqueued for finalization.  Called with the world running.           */
GC_INNER void GC_finalize(void)
{
    struct finalizable_object * curr_fo, * prev_fo, * next_fo;
    ptr_t real_ptr;
    size_t i;
    size_t fo_size = GC_fnlz_roots.fo_head == NULL ? 0 :
                                (size_t)1 << GC_log_fo_table_size;
    GC_bool needs_barrier = FALSE;

    GC_ASSERT(I_HOLD_LOCK());
#   ifndef SMALL_CONFIG
      /* Save current GC_[dl/ll]_entries value for stats printing */
      GC_old_dl_entries = GC_dl_hashtbl.entries;
#     ifndef GC_LONG_REFS_NOT_NEEDED
        GC_old_ll_entries = GC_ll_hashtbl.entries;
#     endif
#   endif

#   ifndef GC_TOGGLE_REFS_NOT_NEEDED
      GC_mark_togglerefs();
#   endif
    GC_make_disappearing_links_disappear(&GC_dl_hashtbl, FALSE);

  /* Mark all objects reachable via chains of 1 or more pointers        */
  /* from finalizable objects.                                          */
    GC_ASSERT(!GC_collection_in_progress());
    for (i = 0; i < fo_size; i++) {
      for (curr_fo = GC_fnlz_roots.fo_head[i];
           curr_fo != NULL; curr_fo = fo_next(curr_fo)) {
        GC_ASSERT(GC_size(curr_fo) >= sizeof(struct finalizable_object));
        real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo -> fo_hidden_base);
        if (!GC_is_marked(real_ptr)) {
            GC_MARKED_FOR_FINALIZATION(real_ptr);
            GC_mark_fo(real_ptr, curr_fo -> fo_mark_proc);
            if (GC_is_marked(real_ptr)) {
                WARN("Finalization cycle involving %p\n", real_ptr);
            }
        }
      }
    }
  /* Enqueue for finalization all objects that are still                */
  /* unreachable.                                                       */
    GC_bytes_finalized = 0;
    for (i = 0; i < fo_size; i++) {
      curr_fo = GC_fnlz_roots.fo_head[i];
      prev_fo = NULL;
      while (curr_fo != NULL) {
        real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo -> fo_hidden_base);
        if (!GC_is_marked(real_ptr)) {
            if (!GC_java_finalization) {
              GC_set_mark_bit(real_ptr);
            }
            /* Delete from hash table.  */
              next_fo = fo_next(curr_fo);
              if (NULL == prev_fo) {
                GC_fnlz_roots.fo_head[i] = next_fo;
                if (GC_object_finalized_proc) {
                  GC_dirty(GC_fnlz_roots.fo_head + i);
                } else {
                  needs_barrier = TRUE;
                }
              } else {
                fo_set_next(prev_fo, next_fo);
                GC_dirty(prev_fo);
              }
              GC_fo_entries--;
              if (GC_object_finalized_proc)
                GC_object_finalized_proc(real_ptr);

            /* Add to list of objects awaiting finalization.    */
              fo_set_next(curr_fo, GC_fnlz_roots.finalize_now);
              GC_dirty(curr_fo);
              SET_FINALIZE_NOW(curr_fo);
            /* Unhide object pointer so any future collections will   */
            /* see it.                                                */
              curr_fo -> fo_hidden_base = (GC_hidden_pointer)GC_REVEAL_POINTER(
                                                curr_fo -> fo_hidden_base);
              GC_bytes_finalized += (word)(curr_fo -> fo_object_sz)
                                    + sizeof(struct finalizable_object);
            GC_ASSERT(GC_is_marked(GC_base(curr_fo)));
            curr_fo = next_fo;
        } else {
            prev_fo = curr_fo;
            curr_fo = fo_next(curr_fo);
        }
      }
    }

  if (GC_java_finalization) {
    /* Make sure we mark everything reachable from objects finalized    */
    /* using the no-order fo_mark_proc.                                 */
      for (curr_fo = GC_fnlz_roots.finalize_now;
           curr_fo != NULL; curr_fo = fo_next(curr_fo)) {
        real_ptr = (ptr_t)(curr_fo -> fo_hidden_base); /* revealed */
        if (!GC_is_marked(real_ptr)) {
            if (curr_fo -> fo_mark_proc == GC_null_finalize_mark_proc) {
                GC_mark_fo(real_ptr, GC_normal_finalize_mark_proc);
            }
            if (curr_fo -> fo_mark_proc != GC_unreachable_finalize_mark_proc) {
                GC_set_mark_bit(real_ptr);
            }
        }
      }

    /* Now revive finalize-when-unreachable objects reachable from      */
    /* other finalizable objects.                                       */
      if (need_unreachable_finalization) {
        curr_fo = GC_fnlz_roots.finalize_now;
        GC_ASSERT(NULL == curr_fo || GC_fnlz_roots.fo_head != NULL);
        for (prev_fo = NULL; curr_fo != NULL;
             prev_fo = curr_fo, curr_fo = next_fo) {
          next_fo = fo_next(curr_fo);
          if (curr_fo -> fo_mark_proc != GC_unreachable_finalize_mark_proc)
            continue;

          real_ptr = (ptr_t)(curr_fo -> fo_hidden_base); /* revealed */
          if (!GC_is_marked(real_ptr)) {
            GC_set_mark_bit(real_ptr);
            continue;
          }
          if (NULL == prev_fo) {
            SET_FINALIZE_NOW(next_fo);
          } else {
            fo_set_next(prev_fo, next_fo);
            GC_dirty(prev_fo);
          }
          curr_fo -> fo_hidden_base = GC_HIDE_POINTER(real_ptr);
          GC_bytes_finalized -= (word)(curr_fo -> fo_object_sz)
                                + sizeof(struct finalizable_object);

          i = HASH2(real_ptr, GC_log_fo_table_size);
          fo_set_next(curr_fo, GC_fnlz_roots.fo_head[i]);
          GC_dirty(curr_fo);
          GC_fo_entries++;
          GC_fnlz_roots.fo_head[i] = curr_fo;
          curr_fo = prev_fo;
          needs_barrier = TRUE;
        }
      }
  }
  if (needs_barrier)
    GC_dirty(GC_fnlz_roots.fo_head); /* entire object */

  /* Remove dangling disappearing links. */
  GC_make_disappearing_links_disappear(&GC_dl_hashtbl, TRUE);

# ifndef GC_TOGGLE_REFS_NOT_NEEDED
    GC_clear_togglerefs();
# endif
# ifndef GC_LONG_REFS_NOT_NEEDED
    GC_make_disappearing_links_disappear(&GC_ll_hashtbl, FALSE);
    GC_make_disappearing_links_disappear(&GC_ll_hashtbl, TRUE);
# endif

  if (GC_fail_count) {
    /* Don't prevent running finalizers if there has been an allocation */
    /* failure recently.                                                */
#   ifdef THREADS
      GC_reset_finalizer_nested();
#   else
      GC_finalizer_nested = 0;
#   endif
  }
}

/* Count of finalizers to run, at most, during a single invocation      */
/* of GC_invoke_finalizers(); zero means no limit.  Accessed with the   */
/* allocator lock held.                                                 */
STATIC unsigned GC_interrupt_finalizers = 0;

#ifndef JAVA_FINALIZATION_NOT_NEEDED

  /* Enqueue all remaining finalizers to be run.        */
  /* A collection in progress, if any, is completed     */
  /* when the first finalizer is enqueued.              */
  STATIC void GC_enqueue_all_finalizers(void)
  {
    size_t i;
    size_t fo_size = GC_fnlz_roots.fo_head == NULL ? 0 :
                                (size_t)1 << GC_log_fo_table_size;

    GC_ASSERT(I_HOLD_LOCK());
    GC_bytes_finalized = 0;
    for (i = 0; i < fo_size; i++) {
      struct finalizable_object * curr_fo = GC_fnlz_roots.fo_head[i];

      GC_fnlz_roots.fo_head[i] = NULL;
      while (curr_fo != NULL) {
          struct finalizable_object * next_fo;
          ptr_t real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo -> fo_hidden_base);

          GC_mark_fo(real_ptr, GC_normal_finalize_mark_proc);
          GC_set_mark_bit(real_ptr);
          GC_complete_ongoing_collection();
          next_fo = fo_next(curr_fo);

          /* Add to list of objects awaiting finalization.      */
          fo_set_next(curr_fo, GC_fnlz_roots.finalize_now);
          GC_dirty(curr_fo);
          SET_FINALIZE_NOW(curr_fo);

          /* Unhide object pointer so any future collections will       */
          /* see it.                                                    */
          curr_fo -> fo_hidden_base = (GC_hidden_pointer)GC_REVEAL_POINTER(
                                                curr_fo -> fo_hidden_base);
          GC_bytes_finalized += (word)(curr_fo -> fo_object_sz)
                                + sizeof(struct finalizable_object);
          curr_fo = next_fo;
      }
    }
    GC_fo_entries = 0;  /* all entries deleted from the hash table */
  }

  /* Invoke all remaining finalizers that haven't yet been run.
   * This is needed for strict compliance with the Java standard,
   * which can make the runtime guarantee that all finalizers are run.
   * Unfortunately, the Java standard implies we have to keep running
   * finalizers until there are no more left, a potential infinite loop.
   * YUCK.
   * Note that this is even more dangerous than the usual Java
   * finalizers, in that objects reachable from static variables
   * may have been finalized when these finalizers are run.
   * Finalizers run at this point must be prepared to deal with a
   * mostly broken world.
   */
  GC_API void GC_CALL GC_finalize_all(void)
  {
    LOCK();
    while (GC_fo_entries > 0) {
      GC_enqueue_all_finalizers();
      GC_interrupt_finalizers = 0; /* reset */
      UNLOCK();
      GC_invoke_finalizers();
      /* Running the finalizers in this thread is arguably not a good   */
      /* idea when we should be notifying another thread to run them.   */
      /* But otherwise we don't have a great way to wait for them to    */
      /* run.                                                           */
      LOCK();
    }
    UNLOCK();
  }

#endif /* !JAVA_FINALIZATION_NOT_NEEDED */

GC_API void GC_CALL GC_set_interrupt_finalizers(unsigned value)
{
  LOCK();
  GC_interrupt_finalizers = value;
  UNLOCK();
}

GC_API unsigned GC_CALL GC_get_interrupt_finalizers(void)
{
  unsigned value;

  READER_LOCK();
  value = GC_interrupt_finalizers;
  READER_UNLOCK();
  return value;
}

/* Returns true if it is worth calling GC_invoke_finalizers. (Useful if */
/* finalizers can only be called from some kind of "safe state" and     */
/* getting into that safe state is expensive.)                          */
GC_API int GC_CALL GC_should_invoke_finalizers(void)
{
# ifdef AO_HAVE_load
    return GC_cptr_load((volatile ptr_t *)&GC_fnlz_roots.finalize_now) != NULL;
# else
    return GC_fnlz_roots.finalize_now != NULL;
# endif /* !THREADS */
}

/* Invoke finalizers for all objects that are ready to be finalized.    */
GC_API int GC_CALL GC_invoke_finalizers(void)
{
    int count = 0;
    word bytes_freed_before = 0; /* initialized to prevent warning. */

    GC_ASSERT(I_DONT_HOLD_LOCK());
    while (GC_should_invoke_finalizers()) {
        struct finalizable_object * curr_fo;
        ptr_t real_ptr;

        LOCK();
        if (count == 0) {
            bytes_freed_before = GC_bytes_freed;
            /* Note: we hold the allocator lock here.   */
        } else if (EXPECT(GC_interrupt_finalizers != 0, FALSE)
                   && (unsigned)count >= GC_interrupt_finalizers) {
            UNLOCK();
            break;
        }
        curr_fo = GC_fnlz_roots.finalize_now;
#       ifdef THREADS
            if (EXPECT(NULL == curr_fo, FALSE)) {
                UNLOCK();
                break;
            }
#       endif
        SET_FINALIZE_NOW(fo_next(curr_fo));
        UNLOCK();
        fo_set_next(curr_fo, 0);
        real_ptr = (ptr_t)(curr_fo -> fo_hidden_base); /* revealed */
        curr_fo -> fo_fn(real_ptr, curr_fo -> fo_client_data);
        curr_fo -> fo_client_data = NULL;
        ++count;
        /* Explicit freeing of curr_fo is probably a bad idea.  */
        /* It throws off accounting if nearly all objects are   */
        /* finalizable.  Otherwise it should not matter.        */
    }
    /* bytes_freed_before is initialized whenever count != 0 */
    if (count != 0
#         if defined(THREADS) && !defined(THREAD_SANITIZER)
            /* A quick check whether some memory was freed.     */
            /* The race with GC_free() is safe to be ignored    */
            /* because we only need to know if the current      */
            /* thread has deallocated something.                */
            && bytes_freed_before != GC_bytes_freed
#         endif
       ) {
        LOCK();
        GC_finalizer_bytes_freed += (GC_bytes_freed - bytes_freed_before);
        UNLOCK();
    }
    return count;
}

static word last_finalizer_notification = 0;

GC_INNER void GC_notify_or_invoke_finalizers(void)
{
    GC_finalizer_notifier_proc notifier_fn = 0;
#   if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
      static word last_back_trace_gc_no = 1;    /* Skip first one. */
#   endif

#   if defined(THREADS) && !defined(KEEP_BACK_PTRS) \
       && !defined(MAKE_BACK_GRAPH)
      /* Quick check (while unlocked) for an empty finalization queue.  */
      if (!GC_should_invoke_finalizers())
        return;
#   endif
    LOCK();

    /* This is a convenient place to generate backtraces if appropriate, */
    /* since that code is not callable with the allocator lock.          */
#   if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
      if (GC_gc_no != last_back_trace_gc_no) {
#       ifdef KEEP_BACK_PTRS
          static GC_bool bt_in_progress = FALSE;

          if (!bt_in_progress) {
            long i;

            bt_in_progress = TRUE; /* prevent recursion or parallel usage */
            for (i = 0; i < GC_backtraces; ++i) {
              /* FIXME: This tolerates concurrent heap mutation, which  */
              /* may cause occasional mysterious results.  We need to   */
              /* release the allocator lock, since GC_print_callers()   */
              /* acquires it.  It probably shouldn't.                   */
              void *current = GC_generate_random_valid_address();

              UNLOCK();
              GC_printf("\n****Chosen address %p in object\n", current);
              GC_print_backtrace(current);
              LOCK();
            }
            bt_in_progress = FALSE;
          }
#       endif
        last_back_trace_gc_no = GC_gc_no;
#       ifdef MAKE_BACK_GRAPH
          if (GC_print_back_height) {
            GC_print_back_graph_stats();
          }
#       endif
      }
#   endif
    if (NULL == GC_fnlz_roots.finalize_now) {
      UNLOCK();
      return;
    }

    if (!GC_finalize_on_demand) {
      unsigned char *pnested;

#     ifdef THREADS
        if (EXPECT(GC_in_thread_creation, FALSE)) {
          UNLOCK();
          return;
        }
#     endif
      pnested = GC_check_finalizer_nested();
      UNLOCK();
      /* Skip GC_invoke_finalizers() if nested. */
      if (pnested != NULL) {
        (void)GC_invoke_finalizers();
        *pnested = 0; /* Reset since no more finalizers or interrupted. */
#       ifndef THREADS
          GC_ASSERT(NULL == GC_fnlz_roots.finalize_now
                    || GC_interrupt_finalizers > 0);
#       endif   /* Otherwise GC can run concurrently and add more */
      }
      return;
    }

    /* These variables require synchronization to avoid data race.  */
    if (last_finalizer_notification != GC_gc_no) {
        notifier_fn = GC_finalizer_notifier;
        last_finalizer_notification = GC_gc_no;
    }
    UNLOCK();
    if (notifier_fn != 0)
        (*notifier_fn)(); /* Invoke the notifier */
}

#ifndef SMALL_CONFIG
# ifndef GC_LONG_REFS_NOT_NEEDED
#   define IF_LONG_REFS_PRESENT_ELSE(x,y) (x)
# else
#   define IF_LONG_REFS_PRESENT_ELSE(x,y) (y)
# endif

  GC_INNER void GC_print_finalization_stats(void)
  {
    const struct finalizable_object *fo;
    unsigned long ready = 0;

    GC_log_printf("%lu finalization entries;"
                  " %lu/%lu short/long disappearing links alive\n",
                  (unsigned long)GC_fo_entries,
                  (unsigned long)GC_dl_hashtbl.entries,
                  (unsigned long)IF_LONG_REFS_PRESENT_ELSE(
                                                GC_ll_hashtbl.entries, 0));

    for (fo = GC_fnlz_roots.finalize_now; fo != NULL; fo = fo_next(fo))
      ++ready;
    GC_log_printf("%lu finalization-ready objects;"
                  " %ld/%ld short/long links cleared\n",
                  ready,
                  (long)GC_old_dl_entries - (long)GC_dl_hashtbl.entries,
                  (long)IF_LONG_REFS_PRESENT_ELSE(
                              GC_old_ll_entries - GC_ll_hashtbl.entries, 0));
  }
#endif /* !SMALL_CONFIG */

#endif /* !GC_NO_FINALIZATION */

/*
 * Copyright (c) 2011 by Hewlett-Packard Company.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#ifdef ENABLE_DISCLAIM

#include "gc/gc_disclaim.h"

#if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH)
  /* The first bit is already used for a debug purpose. */
# define FINALIZER_CLOSURE_FLAG 0x2
#else
# define FINALIZER_CLOSURE_FLAG 0x1
#endif

STATIC int GC_CALLBACK GC_finalized_disclaim(void *obj)
{
#   ifdef AO_HAVE_load
        ptr_t fc_p = GC_cptr_load((volatile ptr_t *)obj);
#   else
        ptr_t fc_p = *(ptr_t *)obj;
#   endif

    if ((ADDR(fc_p) & FINALIZER_CLOSURE_FLAG) != 0) {
        /* The disclaim function may be passed fragments from the       */
        /* free-list, on which it should not run finalization.          */
        /* To recognize this case, we use the fact that the value of    */
        /* the first pointer of such fragments is always, at least,     */
        /* multiple of a pointer size (a link to the next fragment, or  */
        /* NULL).  If it is desirable to have a finalizer which does    */
        /* not use the first pointer for storing the finalization       */
        /* information, GC_disclaim_and_reclaim() must be extended to   */
        /* clear fragments so that the assumption holds for the         */
        /* selected pointer location.                                   */
        const struct GC_finalizer_closure *fc
            = (struct GC_finalizer_closure *)CPTR_CLEAR_FLAGS(fc_p,
                                                FINALIZER_CLOSURE_FLAG);

        GC_ASSERT(!GC_find_leak);
        fc -> proc((ptr_t *)obj + 1, fc -> cd);
    }
    return 0;
}

STATIC void GC_register_disclaim_proc_inner(unsigned kind,
                                            GC_disclaim_proc proc,
                                            GC_bool mark_unconditionally)
{
    GC_ASSERT(kind < MAXOBJKINDS);
    if (EXPECT(GC_find_leak, FALSE)) return;

    GC_obj_kinds[kind].ok_disclaim_proc = proc;
    GC_obj_kinds[kind].ok_mark_unconditionally = mark_unconditionally;
}

GC_API void GC_CALL GC_init_finalized_malloc(void)
{
    GC_init();  /* In case it's not already done.       */
    LOCK();
    if (GC_finalized_kind != 0) {
        UNLOCK();
        return;
    }

    /* The finalizer closure is placed in the first pointer of the      */
    /* object in order to use the lower bits to distinguish live        */
    /* objects from objects on the free list.  The downside of this is  */
    /* that we need one-pointer offset interior pointers, and that      */
    /* GC_base() does not return the start of the user region.          */
    GC_register_displacement_inner(sizeof(ptr_t));

    /* And, the pointer to the finalizer closure object itself is       */
    /* displaced due to baking in this indicator.                       */
    GC_register_displacement_inner(FINALIZER_CLOSURE_FLAG);
    GC_register_displacement_inner(sizeof(oh) | FINALIZER_CLOSURE_FLAG);

    GC_finalized_kind = GC_new_kind_inner(GC_new_free_list_inner(),
                                          GC_DS_LENGTH, TRUE, TRUE);
    GC_ASSERT(GC_finalized_kind != 0);
    GC_register_disclaim_proc_inner(GC_finalized_kind, GC_finalized_disclaim,
                                    TRUE);
    UNLOCK();
}

GC_API void GC_CALL GC_register_disclaim_proc(int kind, GC_disclaim_proc proc,
                                              int mark_unconditionally)
{
    LOCK();
    GC_register_disclaim_proc_inner((unsigned)kind, proc,
                                    (GC_bool)mark_unconditionally);
    UNLOCK();
}

GC_API GC_ATTR_MALLOC void * GC_CALL GC_finalized_malloc(size_t lb,
                                const struct GC_finalizer_closure *fclos)
{
    void *op;
    ptr_t fc_p;

#   ifndef LINT2 /* no data race because the variable is set once */
      GC_ASSERT(GC_finalized_kind != 0);
#   endif
    GC_ASSERT(NONNULL_ARG_NOT_NULL(fclos));
    GC_ASSERT((ADDR(fclos) & FINALIZER_CLOSURE_FLAG) == 0);
    op = GC_malloc_kind(SIZET_SAT_ADD(lb, sizeof(ptr_t)),
                        (int)GC_finalized_kind);
    if (EXPECT(NULL == op, FALSE)) return NULL;

    /* Set the flag (w/o conversion to a numeric type) and store    */
    /* the finalizer closure.                                       */
    fc_p = CPTR_SET_FLAGS(GC_CAST_AWAY_CONST_PVOID(fclos),
                          FINALIZER_CLOSURE_FLAG);
#   ifdef AO_HAVE_store
        GC_cptr_store((volatile ptr_t *)op, fc_p);
#   else
        *(ptr_t *)op = fc_p;
#   endif
    GC_dirty(op);
    REACHABLE_AFTER_DIRTY(fc_p);

    return (ptr_t *)op + 1;
}

#endif /* ENABLE_DISCLAIM */

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#include <string.h>

/* Allocate reclaim list for the kind.  Returns TRUE on success.        */
STATIC GC_bool GC_alloc_reclaim_list(struct obj_kind *ok)
{
    struct hblk ** result;

    GC_ASSERT(I_HOLD_LOCK());
    result = (struct hblk **)GC_scratch_alloc(
                                (MAXOBJGRANULES+1) * sizeof(struct hblk *));
    if (EXPECT(NULL == result, FALSE)) return FALSE;

    BZERO(result, (MAXOBJGRANULES+1)*sizeof(struct hblk *));
    ok -> ok_reclaim_list = result;
    return TRUE;
}

/* Allocate a large block of size lb_adjusted bytes with the requested  */
/* alignment (align_m1 plus one).  The block is not cleared.  We assume */
/* that the size is non-zero and a multiple of GC_GRANULE_BYTES, and    */
/* that it already includes EXTRA_BYTES value.  The flags argument      */
/* should be IGNORE_OFF_PAGE or 0.  Calls GC_allochblk() to do the      */
/* actual allocation, but also triggers collection and/or heap          */
/* expansion as appropriate.  Updates value of GC_bytes_allocd; does    */
/* also other accounting.                                               */
STATIC ptr_t GC_alloc_large(size_t lb_adjusted, int k, unsigned flags,
                            size_t align_m1)
{
    struct hblk *h;
    size_t n_blocks; /* includes alignment */
    ptr_t result = NULL;
    GC_bool retry = FALSE;

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(lb_adjusted != 0 && (lb_adjusted & (GC_GRANULE_BYTES-1)) == 0);
    n_blocks = OBJ_SZ_TO_BLOCKS_CHECKED(SIZET_SAT_ADD(lb_adjusted, align_m1));
    if (!EXPECT(GC_is_initialized, TRUE)) {
      UNLOCK(); /* just to unset GC_lock_holder */
      GC_init();
      LOCK();
    }
    /* Do our share of marking work.    */
    if (GC_incremental && !GC_dont_gc) {
            ENTER_GC();
            GC_collect_a_little_inner((int)n_blocks);
            EXIT_GC();
    }

    h = GC_allochblk(lb_adjusted, k, flags, align_m1);
#   ifdef USE_MUNMAP
        if (NULL == h) {
            GC_merge_unmapped();
            h = GC_allochblk(lb_adjusted, k, flags, align_m1);
        }
#   endif
    while (NULL == h && GC_collect_or_expand(n_blocks, flags, retry)) {
        h = GC_allochblk(lb_adjusted, k, flags, align_m1);
        retry = TRUE;
    }
    if (EXPECT(h != NULL, TRUE)) {
        GC_bytes_allocd += lb_adjusted;
        if (lb_adjusted > HBLKSIZE) {
            GC_large_allocd_bytes += HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb_adjusted);
            if (GC_large_allocd_bytes > GC_max_large_allocd_bytes)
                GC_max_large_allocd_bytes = GC_large_allocd_bytes;
        }
        /* FIXME: Do we need some way to reset GC_max_large_allocd_bytes? */
        result = h -> hb_body;
        GC_ASSERT((ADDR(result) & align_m1) == 0);
    }
    return result;
}

/* Allocate a large block of given size in bytes, clear it if   */
/* appropriate.  We assume that the size is non-zero and        */
/* a multiple of GC_GRANULE_BYTES, and that it already includes */
/* EXTRA_BYTES value.  Update value of GC_bytes_allocd.         */
STATIC ptr_t GC_alloc_large_and_clear(size_t lb_adjusted, int k,
                                      unsigned flags)
{
    ptr_t result;

    GC_ASSERT(I_HOLD_LOCK());
    result = GC_alloc_large(lb_adjusted, k, flags, 0 /* align_m1 */);
    if (EXPECT(result != NULL, TRUE)
          && (GC_debugging_started || GC_obj_kinds[k].ok_init)) {
        /* Clear the whole block, in case of GC_realloc call. */
        BZERO(result, HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb_adjusted));
    }
    return result;
}

/* Fill in additional entries in GC_size_map, including the i-th one.   */
/* Note that a filled in section of the array ending at n always        */
/* has the length of at least n/4.                                      */
STATIC void GC_extend_size_map(size_t i)
{
  size_t original_lg = ALLOC_REQUEST_GRANS(i);
  size_t lg;
  size_t byte_sz = GRANULES_TO_BYTES(original_lg);
                        /* The size we try to preserve.         */
                        /* Close to i, unless this would        */
                        /* introduce too many distinct sizes.   */
  size_t smaller_than_i = byte_sz - (byte_sz >> 3);
  size_t low_limit; /* The lowest indexed entry we initialize.  */
  size_t number_of_objs;

  GC_ASSERT(I_HOLD_LOCK());
  GC_ASSERT(0 == GC_size_map[i]);
  if (0 == GC_size_map[smaller_than_i]) {
    low_limit = byte_sz - (byte_sz >> 2); /* much smaller than i */
    lg = original_lg;
    while (GC_size_map[low_limit] != 0)
      low_limit++;
  } else {
    low_limit = smaller_than_i + 1;
    while (GC_size_map[low_limit] != 0)
      low_limit++;

    lg = ALLOC_REQUEST_GRANS(low_limit);
    lg += lg >> 3;
    if (lg < original_lg) lg = original_lg;
  }

  /* For these larger sizes, we use an even number of granules.         */
  /* This makes it easier to, e.g., construct a 16-byte-aligned         */
  /* allocator even if GC_GRANULE_BYTES is 8.                           */
  lg = (lg + 1) & ~(size_t)1;
  if (lg > MAXOBJGRANULES) lg = MAXOBJGRANULES;

  /* If we can fit the same number of larger objects in a block, do so. */
  GC_ASSERT(lg != 0);
  number_of_objs = HBLK_GRANULES / lg;
  GC_ASSERT(number_of_objs != 0);
  lg = (HBLK_GRANULES / number_of_objs) & ~(size_t)1;

  byte_sz = GRANULES_TO_BYTES(lg) - EXTRA_BYTES;
                        /* We may need one extra byte; do not always    */
                        /* fill in GC_size_map[byte_sz].                */

  for (; low_limit <= byte_sz; low_limit++)
    GC_size_map[low_limit] = lg;
}

STATIC void * GC_generic_malloc_inner_small(size_t lb, int k)
{
  struct obj_kind *ok = &GC_obj_kinds[k];
  size_t lg = GC_size_map[lb];
  void **opp = &(ok -> ok_freelist[lg]);
  void *op = *opp;

  GC_ASSERT(I_HOLD_LOCK());
  if (EXPECT(NULL == op, FALSE)) {
    if (0 == lg) {
      if (!EXPECT(GC_is_initialized, TRUE)) {
        UNLOCK(); /* just to unset GC_lock_holder */
        GC_init();
        LOCK();
        lg = GC_size_map[lb];
      }
      if (0 == lg) {
        GC_extend_size_map(lb);
        lg = GC_size_map[lb];
        GC_ASSERT(lg != 0);
      }
      /* Retry */
      opp = &(ok -> ok_freelist[lg]);
      op = *opp;
    }
    if (NULL == op) {
      if (NULL == ok -> ok_reclaim_list
          && !GC_alloc_reclaim_list(ok))
        return NULL;
      op = GC_allocobj(lg, k);
      if (NULL == op) return NULL;
    }
  }
  *opp = obj_link(op);
  obj_link(op) = NULL;
  GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
  return op;
}

GC_INNER void * GC_generic_malloc_inner(size_t lb, int k, unsigned flags)
{
    size_t lb_adjusted;

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(k < MAXOBJKINDS);
    if (SMALL_OBJ(lb)) {
        return GC_generic_malloc_inner_small(lb, k);
    }

#   if MAX_EXTRA_BYTES > 0
      if ((flags & IGNORE_OFF_PAGE) != 0 && lb >= HBLKSIZE) {
        /* No need to add EXTRA_BYTES.  */
        lb_adjusted = lb;
      } else
#   endif
    /* else */ {
      lb_adjusted = ADD_EXTRA_BYTES(lb);
    }
    return GC_alloc_large_and_clear(ROUNDUP_GRANULE_SIZE(lb_adjusted),
                                    k, flags);
}

#ifdef GC_COLLECT_AT_MALLOC
  /* Parameter to force GC at every malloc of size greater or equal to  */
  /* the given value.  This might be handy during debugging.            */
# if defined(CPPCHECK)
    size_t GC_dbg_collect_at_malloc_min_lb = 16*1024; /* e.g. */
# else
    size_t GC_dbg_collect_at_malloc_min_lb = (GC_COLLECT_AT_MALLOC);
# endif
#endif

GC_INNER void * GC_generic_malloc_aligned(size_t lb, int k, unsigned flags,
                                          size_t align_m1)
{
    void * result;

    GC_ASSERT(k < MAXOBJKINDS);
    if (EXPECT(get_have_errors(), FALSE))
      GC_print_all_errors();
    GC_INVOKE_FINALIZERS();
    GC_DBG_COLLECT_AT_MALLOC(lb);
    if (SMALL_OBJ(lb) && EXPECT(align_m1 < GC_GRANULE_BYTES, TRUE)) {
        LOCK();
        result = GC_generic_malloc_inner_small(lb, k);
        UNLOCK();
    } else {
#       ifdef THREADS
          size_t lg;
#       endif
        size_t lb_adjusted;
        GC_bool init;

#       if MAX_EXTRA_BYTES > 0
          if ((flags & IGNORE_OFF_PAGE) != 0 && lb >= HBLKSIZE) {
            /* No need to add EXTRA_BYTES.      */
            lb_adjusted = ROUNDUP_GRANULE_SIZE(lb);
#           ifdef THREADS
              lg = BYTES_TO_GRANULES(lb_adjusted);
#           endif
          } else
#       endif
        /* else */ {
#         ifndef THREADS
            size_t lg; /* CPPCHECK */
#         endif

          if (EXPECT(0 == lb, FALSE)) lb = 1;
          lg = ALLOC_REQUEST_GRANS(lb);
          lb_adjusted = GRANULES_TO_BYTES(lg);
        }

        init = GC_obj_kinds[k].ok_init;
        if (EXPECT(align_m1 < GC_GRANULE_BYTES, TRUE)) {
          align_m1 = 0;
        } else if (align_m1 < HBLKSIZE) {
          align_m1 = HBLKSIZE - 1;
        }
        LOCK();
        result = GC_alloc_large(lb_adjusted, k, flags, align_m1);
        if (EXPECT(result != NULL, TRUE)) {
          if (GC_debugging_started
#             ifndef THREADS
                || init
#             endif
             ) {
            BZERO(result, HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb_adjusted));
          } else {
#           ifdef THREADS
              GC_ASSERT(GRANULES_TO_PTRS(lg) >= 2);
              /* Clear any memory that might be used for GC descriptors */
              /* before we release the allocator lock.                  */
                ((ptr_t *)result)[0] = NULL;
                ((ptr_t *)result)[1] = NULL;
                ((ptr_t *)result)[GRANULES_TO_PTRS(lg) - 1] = NULL;
                ((ptr_t *)result)[GRANULES_TO_PTRS(lg) - 2] = NULL;
#           endif
          }
        }
        UNLOCK();
#       ifdef THREADS
          if (init && !GC_debugging_started && result != NULL) {
            /* Clear the rest (i.e. excluding the initial 2 words). */
            BZERO((ptr_t *)result + 2,
                  HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb_adjusted) - 2*sizeof(ptr_t));
          }
#       endif
    }
    if (EXPECT(NULL == result, FALSE))
      result = (*GC_get_oom_fn())(lb); /* might be misaligned */
    return result;
}

GC_API GC_ATTR_MALLOC void * GC_CALL GC_generic_malloc(size_t lb, int k)
{
    return GC_generic_malloc_aligned(lb, k, 0 /* flags */, 0 /* align_m1 */);
}

GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_kind_global(size_t lb, int k)
{
    return GC_malloc_kind_aligned_global(lb, k, 0 /* align_m1 */);
}

GC_INNER void * GC_malloc_kind_aligned_global(size_t lb, int k,
                                              size_t align_m1)
{
    GC_ASSERT(k < MAXOBJKINDS);
    if (SMALL_OBJ(lb) && EXPECT(align_m1 < HBLKSIZE / 2, TRUE)) {
        void *op;
        void **opp;
        size_t lg;

        GC_DBG_COLLECT_AT_MALLOC(lb);
        LOCK();
        lg = GC_size_map[lb];
        opp = &GC_obj_kinds[k].ok_freelist[lg];
        op = *opp;
        if (EXPECT(align_m1 >= GC_GRANULE_BYTES, FALSE)) {
            /* TODO: Avoid linear search. */
            for (; (ADDR(op) & align_m1) != 0; op = *opp) {
                opp = &obj_link(op);
            }
        }
        if (EXPECT(op != NULL, TRUE)) {
            GC_ASSERT(PTRFREE == k || NULL == obj_link(op)
                      || (ADDR(obj_link(op)) < GC_greatest_real_heap_addr
                          && GC_least_real_heap_addr < ADDR(obj_link(op))));
            *opp = obj_link(op);
            if (k != PTRFREE)
                obj_link(op) = NULL;
            GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
            UNLOCK();
            GC_ASSERT((ADDR(op) & align_m1) == 0);
            return op;
        }
        UNLOCK();
    }

    /* We make the GC_clear_stack() call a tail one, hoping to get more */
    /* of the stack.                                                    */
    return GC_clear_stack(GC_generic_malloc_aligned(lb, k, 0 /* flags */,
                                                    align_m1));
}

#if defined(THREADS) && !defined(THREAD_LOCAL_ALLOC)
  GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_kind(size_t lb, int k)
  {
    return GC_malloc_kind_global(lb, k);
  }
#endif

/* Allocate lb bytes of atomic (pointer-free) data.     */
GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_atomic(size_t lb)
{
    return GC_malloc_kind(lb, PTRFREE);
}

/* Allocate lb bytes of composite (pointerful) data.    */
GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc(size_t lb)
{
    return GC_malloc_kind(lb, NORMAL);
}

GC_API GC_ATTR_MALLOC void * GC_CALL GC_generic_malloc_uncollectable(
                                                        size_t lb, int k)
{
    void *op;
    size_t lb_orig = lb;

    GC_ASSERT(k < MAXOBJKINDS);
    if (EXTRA_BYTES != 0 && EXPECT(lb != 0, TRUE)) lb--;
                /* We do not need the extra byte, since this will   */
                /* not be collected anyway.                         */

    if (SMALL_OBJ(lb)) {
        void **opp;
        size_t lg;

        if (EXPECT(get_have_errors(), FALSE))
          GC_print_all_errors();
        GC_INVOKE_FINALIZERS();
        GC_DBG_COLLECT_AT_MALLOC(lb_orig);
        LOCK();
        lg = GC_size_map[lb];
        opp = &GC_obj_kinds[k].ok_freelist[lg];
        op = *opp;
        if (EXPECT(op != NULL, TRUE)) {
            *opp = obj_link(op);
            obj_link(op) = 0;
            GC_bytes_allocd += GRANULES_TO_BYTES((word)lg);
            /* Mark bit was already set on free list.  It will be       */
            /* cleared only temporarily during a collection, as a       */
            /* result of the normal free-list mark bit clearing.        */
            GC_non_gc_bytes += GRANULES_TO_BYTES((word)lg);
        } else {
            op = GC_generic_malloc_inner_small(lb, k);
            if (NULL == op) {
              GC_oom_func oom_fn = GC_oom_fn;
              UNLOCK();
              return (*oom_fn)(lb_orig);
            }
            /* For small objects, the free lists are completely marked. */
        }
        GC_ASSERT(GC_is_marked(op));
        UNLOCK();
    } else {
      op = GC_generic_malloc_aligned(lb, k, 0 /* flags */, 0 /* align_m1 */);
      if (op /* != NULL */) { /* CPPCHECK */
        hdr * hhdr = HDR(op);

        GC_ASSERT(HBLKDISPL(op) == 0); /* large block */
        /* We do not need to acquire the allocator lock before HDR(op), */
        /* since we have an undisguised pointer, but we need it while   */
        /* we adjust the mark bits.                                     */
        LOCK();
        set_mark_bit_from_hdr(hhdr, 0); /* Only object. */
#       ifndef THREADS
          GC_ASSERT(hhdr -> hb_n_marks == 0);
                /* This is not guaranteed in the multi-threaded case    */
                /* because the counter could be updated before locking. */
#       endif
        hhdr -> hb_n_marks = 1;
        UNLOCK();
      }
    }
    return op;
}

/* Allocate lb bytes of pointerful, traced, but not collectible data.   */
GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_uncollectable(size_t lb)
{
  return GC_generic_malloc_uncollectable(lb, UNCOLLECTABLE);
}

#ifdef GC_ATOMIC_UNCOLLECTABLE
  /* Allocate lb bytes of pointer-free, untraced, uncollectible data    */
  /* This is normally roughly equivalent to the system malloc.          */
  /* But it may be useful if malloc is redefined.                       */
  GC_API GC_ATTR_MALLOC void * GC_CALL
        GC_malloc_atomic_uncollectable(size_t lb)
  {
    return GC_generic_malloc_uncollectable(lb, AUNCOLLECTABLE);
  }
#endif /* GC_ATOMIC_UNCOLLECTABLE */

#if defined(REDIRECT_MALLOC) && !defined(REDIRECT_MALLOC_IN_HEADER)

# ifndef MSWINCE
#   include <errno.h>
# endif

  /* Avoid unnecessary nested procedure calls here, by #defining some   */
  /* malloc replacements.  Otherwise we end up saving a meaningless     */
  /* return address in the object.  It also speeds things up, but it is */
  /* admittedly quite ugly.                                             */
# define GC_debug_malloc_replacement(lb) GC_debug_malloc(lb, GC_DBG_EXTRAS)

# if defined(CPPCHECK)
#   define REDIRECT_MALLOC_F GC_malloc /* e.g. */
# else
#   define REDIRECT_MALLOC_F REDIRECT_MALLOC
# endif

  void * malloc(size_t lb)
  {
    /* It might help to manually inline the GC_malloc call here.        */
    /* But any decent compiler should reduce the extra procedure call   */
    /* to at most a jump instruction in this case.                      */
#   if defined(I386) && defined(GC_SOLARIS_THREADS)
      /* Thread initialization can call malloc before we are ready for. */
      /* It is not clear that this is enough to help matters.           */
      /* The thread implementation may well call malloc at other        */
      /* inopportune times.                                             */
      if (!EXPECT(GC_is_initialized, TRUE)) return sbrk(lb);
#   endif
    return (void *)REDIRECT_MALLOC_F(lb);
  }

# if defined(GC_LINUX_THREADS)
#   ifdef HAVE_LIBPTHREAD_SO
      STATIC ptr_t GC_libpthread_start = NULL;
      STATIC ptr_t GC_libpthread_end = NULL;
#   endif
    STATIC ptr_t GC_libld_start = NULL;
    STATIC ptr_t GC_libld_end = NULL;
    static GC_bool lib_bounds_set = FALSE;

    GC_INNER void GC_init_lib_bounds(void)
    {
      IF_CANCEL(int cancel_state;)

      /* This test does not need to ensure memory visibility, since     */
      /* the bounds will be set when/if we create another thread.       */
      if (EXPECT(lib_bounds_set, TRUE)) return;

      DISABLE_CANCEL(cancel_state);
      GC_init(); /* if not called yet */
#     if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)
        LOCK(); /* just to set GC_lock_holder */
#     endif
#     ifdef HAVE_LIBPTHREAD_SO
        if (!GC_text_mapping("libpthread-",
                             &GC_libpthread_start, &GC_libpthread_end)) {
          WARN("Failed to find libpthread.so text mapping: Expect crash\n", 0);
          /* This might still work with some versions of libpthread,    */
          /* so we do not abort.                                        */
        }
#     endif
      if (!GC_text_mapping("ld-", &GC_libld_start, &GC_libld_end)) {
          WARN("Failed to find ld.so text mapping: Expect crash\n", 0);
      }
#     if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)
        UNLOCK();
#     endif
      RESTORE_CANCEL(cancel_state);
      lib_bounds_set = TRUE;
    }
# endif /* GC_LINUX_THREADS */

  void * calloc(size_t n, size_t lb)
  {
    if (EXPECT((lb | n) > GC_SQRT_SIZE_MAX, FALSE) /* fast initial test */
        && lb && n > GC_SIZE_MAX / lb)
      return (*GC_get_oom_fn())(GC_SIZE_MAX); /* n*lb overflow */
#   if defined(GC_LINUX_THREADS)
      /* The linker may allocate some memory that is only pointed to by */
      /* mmapped thread stacks.  Make sure it is not collectible.       */
      {
        ptr_t caller = (ptr_t)__builtin_return_address(0);

        GC_init_lib_bounds();
        if (ADDR_INSIDE(caller, GC_libld_start, GC_libld_end)
#           ifdef HAVE_LIBPTHREAD_SO
              || ADDR_INSIDE(caller, GC_libpthread_start, GC_libpthread_end)
                    /* The two ranges are actually usually adjacent,    */
                    /* so there may be a way to speed this up.          */
#           endif
           ) {
          return GC_generic_malloc_uncollectable(n * lb, UNCOLLECTABLE);
        }
      }
#   endif
    return (void *)REDIRECT_MALLOC_F(n * lb);
  }

# ifndef strdup
    char *strdup(const char *s)
    {
      size_t lb = strlen(s) + 1;
      char *result = (char *)REDIRECT_MALLOC_F(lb);

      if (EXPECT(NULL == result, FALSE)) {
        errno = ENOMEM;
        return NULL;
      }
      BCOPY(s, result, lb);
      return result;
    }
# endif /* !defined(strdup) */
 /* If strdup is macro defined, we assume that it actually calls malloc, */
 /* and thus the right thing will happen even without overriding it.     */
 /* This seems to be true on most Linux systems.                         */

# ifndef strndup
    /* This is similar to strdup().     */
    char *strndup(const char *str, size_t size)
    {
      char *copy;
      size_t len = strlen(str);
      if (EXPECT(len > size, FALSE))
        len = size;
      copy = (char *)REDIRECT_MALLOC_F(len + 1);
      if (EXPECT(NULL == copy, FALSE)) {
        errno = ENOMEM;
        return NULL;
      }
      if (EXPECT(len > 0, TRUE))
        BCOPY(str, copy, len);
      copy[len] = '\0';
      return copy;
    }
# endif /* !strndup */

# undef GC_debug_malloc_replacement

#endif /* REDIRECT_MALLOC */

/* Explicitly deallocate the object.  hhdr should correspond to p.      */
static void free_internal(void *p, const hdr *hhdr)
{
  size_t lb = hhdr -> hb_sz;  /* size in bytes */
  size_t lg = BYTES_TO_GRANULES(lb);    /* size in granules */
  int k = hhdr -> hb_obj_kind;

  GC_bytes_freed += lb;
  if (IS_UNCOLLECTABLE(k)) GC_non_gc_bytes -= lb;
  if (EXPECT(lg <= MAXOBJGRANULES, TRUE)) {
    struct obj_kind *ok = &GC_obj_kinds[k];
    void **flh;

    /* It is unnecessary to clear the mark bit.  If the object is       */
    /* reallocated, it does not matter.  Otherwise, the collector will  */
    /* do it, since it is on a free list.                               */
    if (ok -> ok_init && EXPECT(lb > sizeof(ptr_t), TRUE)) {
      BZERO((ptr_t *)p + 1, lb - sizeof(ptr_t));
    }

    flh = &(ok -> ok_freelist[lg]);
    obj_link(p) = *flh;
    *flh = (ptr_t)p;
  } else {
    if (lb > HBLKSIZE) {
      GC_large_allocd_bytes -= HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb);
    }
    GC_freehblk(HBLKPTR(p));
  }
}

GC_API void GC_CALL GC_free(void * p)
{
    const hdr *hhdr;

    if (p /* != NULL */) {
        /* CPPCHECK */
    } else {
        /* Required by ANSI.  It's not my fault ...     */
        return;
    }

#   ifdef LOG_ALLOCS
      GC_log_printf("GC_free(%p) after GC #%lu\n",
                    p, (unsigned long)GC_gc_no);
#   endif
    hhdr = HDR(p);
#   if defined(REDIRECT_MALLOC) && \
        ((defined(NEED_CALLINFO) && defined(GC_HAVE_BUILTIN_BACKTRACE)) \
         || defined(GC_SOLARIS_THREADS) || defined(GC_LINUX_THREADS) \
         || defined(MSWIN32))
        /* This might be called indirectly by GC_print_callers to free  */
        /* the result of backtrace_symbols.                             */
        /* For Solaris, we have to redirect malloc calls during         */
        /* initialization.  For the others, this seems to happen        */
        /* implicitly.                                                  */
        /* Don't try to deallocate that memory.                         */
        if (EXPECT(NULL == hhdr, FALSE)) return;
#   endif
    GC_ASSERT(GC_base(p) == p);
    LOCK();
    free_internal(p, hhdr);
    FREE_PROFILER_HOOK(p);
    UNLOCK();
}

#ifdef THREADS
  GC_INNER void GC_free_inner(void * p)
  {
    GC_ASSERT(I_HOLD_LOCK());
    free_internal(p, HDR(p));
  }
#endif /* THREADS */

#if defined(REDIRECT_MALLOC) && !defined(REDIRECT_FREE)
# define REDIRECT_FREE GC_free
#endif

#if defined(REDIRECT_FREE) && !defined(REDIRECT_MALLOC_IN_HEADER)

# if defined(CPPCHECK)
#   define REDIRECT_FREE_F GC_free /* e.g. */
# else
#   define REDIRECT_FREE_F REDIRECT_FREE
# endif

  void free(void * p)
  {
#   ifdef IGNORE_FREE
      UNUSED_ARG(p);
#   else
#     if defined(GC_LINUX_THREADS) && !defined(USE_PROC_FOR_LIBRARIES)
        /* Don't bother with initialization checks.  If nothing         */
        /* has been initialized, the check fails, and that's safe,      */
        /* since we have not allocated uncollectible objects neither.   */
        ptr_t caller = (ptr_t)__builtin_return_address(0);
        /* This test does not need to ensure memory visibility, since   */
        /* the bounds will be set when/if we create another thread.     */
        if (ADDR_INSIDE(caller, GC_libld_start, GC_libld_end)
#           ifdef HAVE_LIBPTHREAD_SO
              || ADDR_INSIDE(caller, GC_libpthread_start, GC_libpthread_end)
#           endif
           ) {
          GC_free(p);
          return;
        }
#     endif
      REDIRECT_FREE_F(p);
#   endif
  }
#endif /* REDIRECT_FREE */

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 2000 by Hewlett-Packard Company.  All rights reserved.
 * Copyright (c) 2009-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


/*
 * These are extra allocation routines which are likely to be less
 * frequently used than those in malloc.c.  They are separate in the
 * hope that the .o file will be excluded from statically linked
 * executables.  We should probably break this up further.
 */

#include <string.h>

#ifndef MSWINCE
# include <errno.h>
#endif

/* Some externally visible but unadvertised variables to allow access to */
/* free lists from inlined allocators without including gc_priv.h        */
/* or introducing dependencies on internal data structure layouts.       */
void ** const GC_objfreelist_ptr = GC_objfreelist;
void ** const GC_aobjfreelist_ptr = GC_aobjfreelist;
void ** const GC_uobjfreelist_ptr = GC_uobjfreelist;
# ifdef GC_ATOMIC_UNCOLLECTABLE
    void ** const GC_auobjfreelist_ptr = GC_auobjfreelist;
# endif

GC_API int GC_CALL GC_get_kind_and_size(const void * p, size_t * psize)
{
    const hdr *hhdr = HDR(p);

    if (psize != NULL) {
        *psize = hhdr -> hb_sz;
    }
    return hhdr -> hb_obj_kind;
}

GC_API GC_ATTR_MALLOC void * GC_CALL GC_generic_or_special_malloc(size_t lb,
                                                                  int k)
{
    switch (k) {
        case PTRFREE:
        case NORMAL:
            return GC_malloc_kind(lb, k);
        case UNCOLLECTABLE:
#       ifdef GC_ATOMIC_UNCOLLECTABLE
          case AUNCOLLECTABLE:
#       endif
            return GC_generic_malloc_uncollectable(lb, k);
        default:
            return GC_generic_malloc_aligned(lb, k, 0 /* flags */, 0);
    }
}

/* Change the size of the block pointed to by p to contain at least   */
/* lb bytes.  The object may be (and quite likely will be) moved.     */
/* The kind (e.g. atomic) is the same as that of the old.             */
/* Shrinking of large blocks is not implemented well.                 */
GC_API void * GC_CALL GC_realloc(void * p, size_t lb)
{
    hdr * hhdr;
    void * result;
#   if defined(_FORTIFY_SOURCE) && defined(__GNUC__) && !defined(__clang__)
      /* Use cleared_p instead of p as a workaround to avoid        */
      /* passing alloc_size(lb) attribute associated with p to      */
      /* memset (including a memset call inside GC_free).           */
      volatile GC_uintptr_t cleared_p = (GC_uintptr_t)p;
#   else
#     define cleared_p p
#   endif
    size_t sz;      /* Current size in bytes    */
    size_t orig_sz; /* Original sz in bytes     */
    int obj_kind;

    if (NULL == p) return GC_malloc(lb);  /* Required by ANSI */
    if (0 == lb) /* and p != NULL */ {
#     ifndef IGNORE_FREE
        GC_free(p);
#     endif
      return NULL;
    }
    hhdr = HDR(HBLKPTR(p));
    sz = hhdr -> hb_sz;
    obj_kind = hhdr -> hb_obj_kind;
    orig_sz = sz;

    if (sz > MAXOBJBYTES) {
        const struct obj_kind * ok = &GC_obj_kinds[obj_kind];
        word descr = ok -> ok_descriptor;

        /* Round it up to the next whole heap block.    */
        sz = (sz + HBLKSIZE-1) & ~(HBLKSIZE-1);
#       if ALIGNMENT > GC_DS_TAGS
          /* An extra byte is not added in case of ignore-off-page  */
          /* allocated objects not smaller than HBLKSIZE.           */
          GC_ASSERT(sz >= HBLKSIZE);
          if (EXTRA_BYTES != 0 && (hhdr -> hb_flags & IGNORE_OFF_PAGE) != 0
              && obj_kind == NORMAL)
            descr += ALIGNMENT; /* or set to 0 */
#       endif
        if (ok -> ok_relocate_descr)
          descr += sz;
        /* GC_realloc might be changing the block size while            */
        /* GC_reclaim_block or GC_clear_hdr_marks is examining it.      */
        /* The change to the size field is benign, in that GC_reclaim   */
        /* (and GC_clear_hdr_marks) would work correctly with either    */
        /* value, since we are not changing the number of objects in    */
        /* the block.  But seeing a half-updated value (though unlikely */
        /* to occur in practice) could be probably bad.                 */
        /* Using unordered atomic accesses on the size and hb_descr     */
        /* fields would solve the issue.  (The alternate solution might */
        /* be to initially overallocate large objects, so we do not     */
        /* have to adjust the size in GC_realloc, if they still fit.    */
        /* But that is probably more expensive, since we may end up     */
        /* scanning a bunch of zeros during GC.)                        */
#       ifdef AO_HAVE_store
          AO_store(&(hhdr -> hb_sz), sz);
          AO_store((AO_t *)&(hhdr -> hb_descr), descr);
#       else
          {
            LOCK();
            hhdr -> hb_sz = sz;
            hhdr -> hb_descr = descr;
            UNLOCK();
          }
#       endif

#         ifdef MARK_BIT_PER_OBJ
            GC_ASSERT(hhdr -> hb_inv_sz == LARGE_INV_SZ);
#         else
            GC_ASSERT((hhdr -> hb_flags & LARGE_BLOCK) != 0
                        && hhdr -> hb_map[ANY_INDEX] == 1);
#         endif
          if (IS_UNCOLLECTABLE(obj_kind)) GC_non_gc_bytes += (sz - orig_sz);
          /* Extra area is already cleared by GC_alloc_large_and_clear. */
    }
    if (ADD_EXTRA_BYTES(lb) <= sz) {
        if (lb >= (sz >> 1)) {
            if (orig_sz > lb) {
              /* Clear unneeded part of object to avoid bogus pointer */
              /* tracing.                                             */
                BZERO((ptr_t)cleared_p + lb, orig_sz - lb);
            }
            return p;
        }
        /* shrink */
        sz = lb;
    }
    result = GC_generic_or_special_malloc((word)lb, obj_kind);
    if (EXPECT(result != NULL, TRUE)) {
      /* In case of shrink, it could also return original object.       */
      /* But this gives the client warning of imminent disaster.        */
      BCOPY(p, result, sz);
#     ifndef IGNORE_FREE
        GC_free((ptr_t)cleared_p);
#     endif
    }
    return result;
#   undef cleared_p
}

# if defined(REDIRECT_MALLOC) && !defined(REDIRECT_REALLOC)
#   define REDIRECT_REALLOC GC_realloc
# endif

# ifdef REDIRECT_REALLOC

/* As with malloc, avoid two levels of extra calls here.        */
# define GC_debug_realloc_replacement(p, lb) \
        GC_debug_realloc(p, lb, GC_DBG_EXTRAS)

# if !defined(REDIRECT_MALLOC_IN_HEADER)
    void * realloc(void * p, size_t lb)
    {
      return REDIRECT_REALLOC(p, lb);
    }
# endif

# undef GC_debug_realloc_replacement
# endif /* REDIRECT_REALLOC */

/* Allocate memory such that only pointers to near the beginning of */
/* the object are considered.  We avoid holding the allocator lock  */
/* while we clear the memory.                                       */
GC_API GC_ATTR_MALLOC void * GC_CALL
    GC_generic_malloc_ignore_off_page(size_t lb, int k)
{
  return GC_generic_malloc_aligned(lb, k, IGNORE_OFF_PAGE, 0 /* align_m1 */);
}

GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_ignore_off_page(size_t lb)
{
    return GC_generic_malloc_aligned(lb, NORMAL, IGNORE_OFF_PAGE, 0);
}

GC_API GC_ATTR_MALLOC void * GC_CALL
    GC_malloc_atomic_ignore_off_page(size_t lb)
{
    return GC_generic_malloc_aligned(lb, PTRFREE, IGNORE_OFF_PAGE, 0);
}

/* Increment GC_bytes_allocd from code that doesn't have direct access  */
/* to GC_arrays.                                                        */
void GC_CALL GC_incr_bytes_allocd(size_t n)
{
    GC_bytes_allocd += n;
}

/* The same for GC_bytes_freed.                         */
void GC_CALL GC_incr_bytes_freed(size_t n)
{
    GC_bytes_freed += n;
}

GC_API size_t GC_CALL GC_get_expl_freed_bytes_since_gc(void)
{
    return (size_t)GC_bytes_freed;
}

# ifdef PARALLEL_MARK
    /* Number of bytes of memory allocated since we released the        */
    /* allocator lock.  Instead of reacquiring the allocator lock just  */
    /* to add this in, we add it in the next time we reacquire the      */
    /* allocator lock.  (Atomically adding it does not work, since we   */
    /* would have to atomically update it in GC_malloc, which is too    */
    /* expensive.)                                                      */
    STATIC volatile AO_t GC_bytes_allocd_tmp = 0;
# endif /* PARALLEL_MARK */

GC_API void GC_CALL GC_generic_malloc_many(size_t lb_adjusted, int k,
                                           void **result)
{
    void *op;
    void *p;
    void **opp;
    size_t lg;  /* lb_adjusted value converted to granules */
    word my_bytes_allocd = 0;
    struct obj_kind *ok;
    struct hblk **rlh;

    GC_ASSERT(lb_adjusted != 0 && (lb_adjusted & (GC_GRANULE_BYTES-1)) == 0);
    /* Currently a single object is always allocated if manual VDB. */
    /* TODO: GC_dirty should be called for each linked object (but  */
    /* the last one) to support multiple objects allocation.        */
    if (!EXPECT(lb_adjusted <= MAXOBJBYTES, TRUE) || GC_manual_vdb) {
        op = GC_generic_malloc_aligned(lb_adjusted - EXTRA_BYTES, k,
                                       0 /* flags */, 0 /* align_m1 */);
        if (EXPECT(op != NULL, TRUE))
          obj_link(op) = NULL;
        *result = op;
#       ifndef NO_MANUAL_VDB
          if (GC_manual_vdb && GC_is_heap_ptr(result)) {
            GC_dirty_inner(result);
            REACHABLE_AFTER_DIRTY(op);
          }
#       endif
        return;
    }
    GC_ASSERT(k < MAXOBJKINDS);
    lg = BYTES_TO_GRANULES(lb_adjusted);
    if (EXPECT(get_have_errors(), FALSE))
      GC_print_all_errors();
    GC_INVOKE_FINALIZERS();
    GC_DBG_COLLECT_AT_MALLOC(lb_adjusted - EXTRA_BYTES);
    if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
    LOCK();
    /* Do our share of marking work */
    if (GC_incremental && !GC_dont_gc) {
        ENTER_GC();
        GC_collect_a_little_inner(1);
        EXIT_GC();
    }

    /* First see if we can reclaim a page of objects waiting to be */
    /* reclaimed.                                                  */
    ok = &GC_obj_kinds[k];
    rlh = ok -> ok_reclaim_list;
    if (rlh != NULL) {
        struct hblk * hbp;
        hdr * hhdr;

        while ((hbp = rlh[lg]) != NULL) {
            hhdr = HDR(hbp);
            rlh[lg] = hhdr -> hb_next;
            GC_ASSERT(hhdr -> hb_sz == lb_adjusted);
            hhdr -> hb_last_reclaimed = (unsigned short)GC_gc_no;
#           ifdef PARALLEL_MARK
              if (GC_parallel) {
                  signed_word my_bytes_allocd_tmp =
                                (signed_word)AO_load(&GC_bytes_allocd_tmp);
                  GC_ASSERT(my_bytes_allocd_tmp >= 0);
                  /* We only decrement it while holding the allocator   */
                  /* lock.  Thus, we cannot accidentally adjust it down */
                  /* in more than one thread simultaneously.            */
                  if (my_bytes_allocd_tmp != 0) {
                    (void)AO_fetch_and_add(&GC_bytes_allocd_tmp,
                                           (AO_t)(-my_bytes_allocd_tmp));
                    GC_bytes_allocd += (word)my_bytes_allocd_tmp;
                  }
                  GC_acquire_mark_lock();
                  ++GC_fl_builder_count;
                  UNLOCK();
                  GC_release_mark_lock();
              }
#           endif
            op = GC_reclaim_generic(hbp, hhdr, lb_adjusted,
                                    ok -> ok_init, 0, &my_bytes_allocd);
            if (op != 0) {
#             ifdef PARALLEL_MARK
                if (GC_parallel) {
                  *result = op;
                  (void)AO_fetch_and_add(&GC_bytes_allocd_tmp,
                                         (AO_t)my_bytes_allocd);
                  GC_acquire_mark_lock();
                  --GC_fl_builder_count;
                  if (GC_fl_builder_count == 0) GC_notify_all_builder();
#                 ifdef THREAD_SANITIZER
                    GC_release_mark_lock();
                    LOCK();
                    GC_bytes_found += (signed_word)my_bytes_allocd;
                    UNLOCK();
#                 else
                    GC_bytes_found += (signed_word)my_bytes_allocd;
                                        /* The result may be inaccurate. */
                    GC_release_mark_lock();
#                 endif
                  (void)GC_clear_stack(0);
                  return;
                }
#             endif
              /* We also reclaimed memory, so we need to adjust       */
              /* that count.                                          */
              GC_bytes_found += (signed_word)my_bytes_allocd;
              GC_bytes_allocd += my_bytes_allocd;
              goto out;
            }
#           ifdef PARALLEL_MARK
              if (GC_parallel) {
                GC_acquire_mark_lock();
                --GC_fl_builder_count;
                if (GC_fl_builder_count == 0) GC_notify_all_builder();
                GC_release_mark_lock();
                LOCK();
                /* The allocator lock is needed for access to the       */
                /* reclaim list.  We must decrement fl_builder_count    */
                /* before reacquiring the allocator lock.  Hopefully    */
                /* this path is rare.                                   */

                rlh = ok -> ok_reclaim_list; /* reload rlh after locking */
                if (NULL == rlh) break;
              }
#           endif
        }
    }
    /* Next try to use prefix of global free list if there is one.      */
    /* We don't refill it, but we need to use it up before allocating   */
    /* a new block ourselves.                                           */
    opp = &(ok -> ok_freelist[lg]);
    if ((op = *opp) != NULL) {
        *opp = NULL;
        my_bytes_allocd = 0;
        for (p = op; p != NULL; p = obj_link(p)) {
          my_bytes_allocd += lb_adjusted;
          if ((word)my_bytes_allocd >= HBLKSIZE) {
            *opp = obj_link(p);
            obj_link(p) = NULL;
            break;
          }
        }
        GC_bytes_allocd += my_bytes_allocd;
        goto out;
    }

    /* Next try to allocate a new block worth of objects of this size.  */
    {
        struct hblk *h = GC_allochblk(lb_adjusted, k, 0 /* flags */,
                                      0 /* align_m1 */);

        if (h /* != NULL */) { /* CPPCHECK */
          if (IS_UNCOLLECTABLE(k)) GC_set_hdr_marks(HDR(h));
          GC_bytes_allocd += HBLKSIZE - (HBLKSIZE % lb_adjusted);
#         ifdef PARALLEL_MARK
            if (GC_parallel) {
              GC_acquire_mark_lock();
              ++GC_fl_builder_count;
              UNLOCK();
              GC_release_mark_lock();

              op = GC_build_fl(h, NULL, lg,
                               ok -> ok_init || GC_debugging_started);
              *result = op;
              GC_acquire_mark_lock();
              --GC_fl_builder_count;
              if (GC_fl_builder_count == 0) GC_notify_all_builder();
              GC_release_mark_lock();
              (void)GC_clear_stack(0);
              return;
            }
#         endif
          op = GC_build_fl(h, NULL, lg, ok -> ok_init || GC_debugging_started);
          goto out;
        }
    }

    /* As a last attempt, try allocating a single object.  Note that    */
    /* this may trigger a collection or expand the heap.                */
    op = GC_generic_malloc_inner(lb_adjusted - EXTRA_BYTES, k, 0 /* flags */);
    if (op != NULL) obj_link(op) = NULL;

  out:
    *result = op;
    UNLOCK();
    (void)GC_clear_stack(0);
}

GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_many(size_t lb)
{
    void *result;
    size_t lg, lb_adjusted;

    if (EXPECT(0 == lb, FALSE)) lb = 1;
    lg = ALLOC_REQUEST_GRANS(lb);
    lb_adjusted = GRANULES_TO_BYTES(lg);
    GC_generic_malloc_many(lb_adjusted, NORMAL, &result);
    return result;
}

/* TODO: The debugging version of GC_memalign and friends is tricky     */
/* and currently missing.  The major difficulty is:                     */
/* - store_debug_info() should return the pointer of the object with    */
/* the requested alignment (unlike the object header).                  */

GC_API GC_ATTR_MALLOC void * GC_CALL GC_memalign(size_t align, size_t lb)
{
    size_t align_m1 = align - 1;

    /* Check the alignment argument.    */
    if (EXPECT(0 == align || (align & align_m1) != 0, FALSE)) return NULL;
    /* TODO: use thread-local allocation */
    if (align <= GC_GRANULE_BYTES) return GC_malloc(lb);
    return GC_malloc_kind_aligned_global(lb, NORMAL, align_m1);
}

/* This one exists largely to redirect posix_memalign for leaks finding. */
GC_API int GC_CALL GC_posix_memalign(void **memptr, size_t align, size_t lb)
{
  void *p;
  size_t align_minus_one = align - 1; /* to workaround a cppcheck warning */

  /* Check alignment properly.  */
  if (EXPECT(align < sizeof(void *)
             || (align_minus_one & align) != 0, FALSE)) {
#   ifdef MSWINCE
      return ERROR_INVALID_PARAMETER;
#   else
      return EINVAL;
#   endif
  }

  p = GC_memalign(align, lb);
  if (EXPECT(NULL == p, FALSE)) {
#   ifdef MSWINCE
      return ERROR_NOT_ENOUGH_MEMORY;
#   else
      return ENOMEM;
#   endif
  }
  *memptr = p;
  return 0; /* success */
}

#ifndef GC_NO_VALLOC
  GC_API GC_ATTR_MALLOC void * GC_CALL GC_valloc(size_t lb)
  {
    if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
    GC_ASSERT(GC_real_page_size != 0);
    return GC_memalign(GC_real_page_size, lb);
  }

  GC_API GC_ATTR_MALLOC void * GC_CALL GC_pvalloc(size_t lb)
  {
    if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
    GC_ASSERT(GC_real_page_size != 0);
    lb = SIZET_SAT_ADD(lb, GC_real_page_size - 1) & ~(GC_real_page_size - 1);
    return GC_memalign(GC_real_page_size, lb);
  }
#endif /* !GC_NO_VALLOC */

/* Provide a version of strdup() that uses the collector to allocate    */
/* the copy of the string.                                              */
GC_API GC_ATTR_MALLOC char * GC_CALL GC_strdup(const char *s)
{
  char *copy;
  size_t lb;
  if (s == NULL) return NULL;
  lb = strlen(s) + 1;
  copy = (char *)GC_malloc_atomic(lb);
  if (EXPECT(NULL == copy, FALSE)) {
#   ifndef MSWINCE
      errno = ENOMEM;
#   endif
    return NULL;
  }
  BCOPY(s, copy, lb);
  return copy;
}

GC_API GC_ATTR_MALLOC char * GC_CALL GC_strndup(const char *str, size_t size)
{
  char *copy;
  size_t len = strlen(str); /* str is expected to be non-NULL  */
  if (EXPECT(len > size, FALSE))
    len = size;
  copy = (char *)GC_malloc_atomic(len + 1);
  if (EXPECT(NULL == copy, FALSE)) {
#   ifndef MSWINCE
      errno = ENOMEM;
#   endif
    return NULL;
  }
  if (EXPECT(len > 0, TRUE))
    BCOPY(str, copy, len);
  copy[len] = '\0';
  return copy;
}

#ifdef GC_REQUIRE_WCSDUP
# include <wchar.h> /* for wcslen() */

  GC_API GC_ATTR_MALLOC wchar_t * GC_CALL GC_wcsdup(const wchar_t *str)
  {
    size_t lb = (wcslen(str) + 1) * sizeof(wchar_t);
    wchar_t *copy = (wchar_t *)GC_malloc_atomic(lb);

    if (EXPECT(NULL == copy, FALSE)) {
#     ifndef MSWINCE
        errno = ENOMEM;
#     endif
      return NULL;
    }
    BCOPY(str, copy, lb);
    return copy;
  }
#endif /* GC_REQUIRE_WCSDUP */

#ifndef CPPCHECK
  GC_API void * GC_CALL GC_malloc_stubborn(size_t lb)
  {
    return GC_malloc(lb);
  }

  GC_API void GC_CALL GC_change_stubborn(const void *p)
  {
    UNUSED_ARG(p);
  }
#endif /* !CPPCHECK */

GC_API void GC_CALL GC_end_stubborn_change(const void *p)
{
  GC_dirty(p); /* entire object */
}

GC_API void GC_CALL GC_ptr_store_and_dirty(void *p, const void *q)
{
  *(const void **)p = q;
  GC_dirty(p);
  REACHABLE_AFTER_DIRTY(q);
}

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1995 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 2000 by Hewlett-Packard Company.  All rights reserved.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


/* Make arguments appear live to compiler.  Put here to minimize the    */
/* risk of inlining.  Used to minimize junk left in registers.          */
GC_ATTR_NOINLINE
void GC_noop6(word arg1, word arg2, word arg3, word arg4, word arg5, word arg6)
{
  UNUSED_ARG(arg1);
  UNUSED_ARG(arg2);
  UNUSED_ARG(arg3);
  UNUSED_ARG(arg4);
  UNUSED_ARG(arg5);
  UNUSED_ARG(arg6);
  /* Avoid GC_noop6 calls to be optimized away. */
# if defined(AO_HAVE_compiler_barrier) && !defined(BASE_ATOMIC_OPS_EMULATED)
    AO_compiler_barrier(); /* to serve as a special side-effect */
# else
    GC_noop1(0);
# endif
}

/* Make the argument appear live to compiler.  This is similar  */
/* to GC_noop6(), but with a single argument.  Robust against   */
/* whole program analysis.                                      */
GC_API void GC_CALL GC_noop1(GC_word x)
{
# if defined(AO_HAVE_store) && defined(THREAD_SANITIZER)
    AO_store(&GC_noop_sink, (AO_t)x);
# else
    GC_noop_sink = x;
# endif
}

GC_API void GC_CALL GC_noop1_ptr(volatile void *p)
{
# if CPP_PTRSZ > CPP_WORDSZ
#   if defined(AO_HAVE_store) && defined(THREAD_SANITIZER)
      GC_cptr_store(&GC_noop_sink_ptr, (ptr_t)CAST_AWAY_VOLATILE_PVOID(p));
#   else
      GC_noop_sink_ptr = (ptr_t)CAST_AWAY_VOLATILE_PVOID(p);
#   endif
# else
    GC_noop1(ADDR(p));
# endif
}

/* Initialize GC_obj_kinds properly and standard free lists properly.   */
/* This must be done statically since they may be accessed before       */
/* GC_init is called.                                                   */
/* It's done here, since we need to deal with mark descriptors.         */
GC_INNER struct obj_kind GC_obj_kinds[MAXOBJKINDS] = {
/* PTRFREE */ { &GC_aobjfreelist[0], 0 /* filled in dynamically */,
                /* 0 | */ GC_DS_LENGTH, FALSE, FALSE
                /*, */ OK_DISCLAIM_INITZ },
/* NORMAL */  { &GC_objfreelist[0], 0,
                /* 0 | */ GC_DS_LENGTH,
                                /* adjusted in GC_init for EXTRA_BYTES  */
                TRUE /* add length to descr */, TRUE
                /*, */ OK_DISCLAIM_INITZ },
/* UNCOLLECTABLE */
              { &GC_uobjfreelist[0], 0,
                /* 0 | */ GC_DS_LENGTH, TRUE /* add length to descr */, TRUE
                /*, */ OK_DISCLAIM_INITZ },
# ifdef GC_ATOMIC_UNCOLLECTABLE
              { &GC_auobjfreelist[0], 0,
                /* 0 | */ GC_DS_LENGTH, FALSE, FALSE
                /*, */ OK_DISCLAIM_INITZ },
# endif
};

#ifndef INITIAL_MARK_STACK_SIZE
#   define INITIAL_MARK_STACK_SIZE (1*HBLKSIZE)
                /* INITIAL_MARK_STACK_SIZE * sizeof(mse) should be a    */
                /* multiple of HBLKSIZE.                                */
                /* The incremental collector actually likes a larger    */
                /* size, since it wants to push all marked dirty        */
                /* objects before marking anything new.  Currently we   */
                /* let it grow dynamically.                             */
#endif /* !INITIAL_MARK_STACK_SIZE */

#if !defined(GC_DISABLE_INCREMENTAL)
  STATIC word GC_n_rescuing_pages = 0;
                                /* Number of dirty pages we marked      */
                                /* from, excluding pointer-free pages,  */
                                /* etc.  Used for logging only.         */
#endif

#ifdef PARALLEL_MARK
  GC_INNER GC_bool GC_parallel_mark_disabled = FALSE;
#endif

GC_API void GC_CALL GC_set_pointer_mask(GC_word value)
{
# ifdef DYNAMIC_POINTER_MASK
    GC_ASSERT(value >= 0xff); /* a simple sanity check */
    GC_pointer_mask = value;
# else
    if (value
#       ifdef POINTER_MASK
          != (word)(POINTER_MASK)
#       else
          != GC_WORD_MAX
#       endif
       ) {
      ABORT("Dynamic pointer mask/shift is unsupported");
    }
# endif
}

GC_API GC_word GC_CALL GC_get_pointer_mask(void)
{
# ifdef DYNAMIC_POINTER_MASK
    GC_word value = GC_pointer_mask;

    if (0 == value) {
      GC_ASSERT(!GC_is_initialized);
      value = GC_WORD_MAX;
    }
    return value;
# elif defined(POINTER_MASK)
    return POINTER_MASK;
# else
    return GC_WORD_MAX;
# endif
}

GC_API void GC_CALL GC_set_pointer_shift(unsigned value)
{
# ifdef DYNAMIC_POINTER_MASK
    GC_ASSERT(value < CPP_WORDSZ);
    GC_pointer_shift = (unsigned char)value;
# else
    if (value
#       ifdef POINTER_SHIFT
          != (unsigned)(POINTER_SHIFT)
#       endif /* else is not zero */
       ) {
      ABORT("Dynamic pointer mask/shift is unsupported");
    }
# endif
}

GC_API unsigned GC_CALL GC_get_pointer_shift(void)
{
# ifdef DYNAMIC_POINTER_MASK
    return GC_pointer_shift;
# elif defined(POINTER_SHIFT)
    GC_STATIC_ASSERT((unsigned)(POINTER_SHIFT) < CPP_WORDSZ);
    return POINTER_SHIFT;
# else
    return 0;
# endif
}

/* Is a collection in progress?  Note that this can return true in the  */
/* non-incremental case, if a collection has been abandoned and the     */
/* mark state is now MS_INVALID.                                        */
GC_INNER GC_bool GC_collection_in_progress(void)
{
    return GC_mark_state != MS_NONE;
}

/* Clear all mark bits in the header.   */
GC_INNER void GC_clear_hdr_marks(hdr *hhdr)
{
  size_t last_bit;

# ifdef AO_HAVE_load
    /* Atomic access is used to avoid racing with GC_realloc.   */
    last_bit = FINAL_MARK_BIT(AO_load(&(hhdr -> hb_sz)));
# else
    /* No race as GC_realloc holds the allocator lock while updating hb_sz. */
    last_bit = FINAL_MARK_BIT(hhdr -> hb_sz);
# endif

    BZERO(CAST_AWAY_VOLATILE_PVOID(hhdr -> hb_marks),
          sizeof(hhdr -> hb_marks));
    set_mark_bit_from_hdr(hhdr, last_bit);
    hhdr -> hb_n_marks = 0;
}

/* Set all mark bits in the header.  Used for uncollectible blocks. */
GC_INNER void GC_set_hdr_marks(hdr *hhdr)
{
    size_t i;
    size_t sz = hhdr -> hb_sz;
    size_t n_marks = FINAL_MARK_BIT(sz);

#   ifdef USE_MARK_BYTES
      for (i = 0; i <= n_marks; i += MARK_BIT_OFFSET(sz)) {
        hhdr -> hb_marks[i] = 1;
      }
#   else
      /* Note that all bits are set even in case of not MARK_BIT_PER_OBJ,   */
      /* instead of setting every n-th bit where n is MARK_BIT_OFFSET(sz).  */
      /* This is done for a performance reason.                             */
      for (i = 0; i < divWORDSZ(n_marks); ++i) {
        hhdr -> hb_marks[i] = GC_WORD_MAX;
      }
      /* Set the remaining bits near the end (plus one bit past the end).   */
      hhdr -> hb_marks[i] = ((((word)1 << modWORDSZ(n_marks)) - 1) << 1) | 1;
#   endif
#   ifdef MARK_BIT_PER_OBJ
      hhdr -> hb_n_marks = n_marks;
#   else
      hhdr -> hb_n_marks = HBLK_OBJS(sz);
#   endif
}

/* Clear all mark bits associated with block h. */
static void GC_CALLBACK clear_marks_for_block(struct hblk *h, void *dummy)
{
    hdr * hhdr = HDR(h);

    UNUSED_ARG(dummy);
    if (IS_UNCOLLECTABLE(hhdr -> hb_obj_kind)) return;
        /* Mark bit for these is cleared only once the object is        */
        /* explicitly deallocated.  This either frees the block, or     */
        /* the bit is cleared once the object is on the free list.      */
    GC_clear_hdr_marks(hhdr);
#   if defined(CPPCHECK)
        GC_noop1_ptr(h);
#   endif
}

/* Slow but general routines for setting/clearing/asking about mark bits. */
GC_API void GC_CALL GC_set_mark_bit(const void *p)
{
    struct hblk *h = HBLKPTR(p);
    hdr * hhdr = HDR(h);
    size_t bit_no = MARK_BIT_NO((size_t)((ptr_t)p - (ptr_t)h), hhdr -> hb_sz);

    if (!mark_bit_from_hdr(hhdr, bit_no)) {
      set_mark_bit_from_hdr(hhdr, bit_no);
      INCR_MARKS(hhdr);
    }
}

GC_API void GC_CALL GC_clear_mark_bit(const void *p)
{
    struct hblk *h = HBLKPTR(p);
    hdr * hhdr = HDR(h);
    size_t bit_no = MARK_BIT_NO((size_t)((ptr_t)p - (ptr_t)h), hhdr -> hb_sz);

    if (mark_bit_from_hdr(hhdr, bit_no)) {
      size_t n_marks = hhdr -> hb_n_marks;

      GC_ASSERT(n_marks != 0);
      clear_mark_bit_from_hdr(hhdr, bit_no);
      n_marks--;
#     ifdef PARALLEL_MARK
        if (n_marks != 0 || !GC_parallel)
          hhdr -> hb_n_marks = n_marks;
        /* Don't decrement to zero.  The counts are approximate due to  */
        /* concurrency issues, but we need to ensure that a count of    */
        /* zero implies an empty block.                                 */
#     else
          hhdr -> hb_n_marks = n_marks;
#     endif
    }
}

GC_API int GC_CALL GC_is_marked(const void *p)
{
    struct hblk *h = HBLKPTR(p);
    hdr * hhdr = HDR(h);
    size_t bit_no = MARK_BIT_NO((size_t)((ptr_t)p - (ptr_t)h), hhdr -> hb_sz);

    return (int)mark_bit_from_hdr(hhdr, bit_no); /* 0 or 1 */
}

/* Clear mark bits in all allocated heap blocks.  This invalidates the  */
/* marker invariant, and sets GC_mark_state to reflect this.  (This     */
/* implicitly starts marking to reestablish the invariant.)             */
GC_INNER void GC_clear_marks(void)
{
    GC_ASSERT(GC_is_initialized); /* needed for GC_push_roots */
    GC_apply_to_all_blocks(clear_marks_for_block, NULL);
    GC_objects_are_marked = FALSE;
    GC_mark_state = MS_INVALID;
    GC_scan_ptr = NULL;
}

/* Initiate a garbage collection.  Initiates a full collection if the   */
/* mark state is invalid.                                               */
GC_INNER void GC_initiate_gc(void)
{
    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(GC_is_initialized);
#   ifndef GC_DISABLE_INCREMENTAL
        if (GC_incremental) {
#         ifdef CHECKSUMS
            GC_read_dirty(FALSE);
            GC_check_dirty();
#         else
            GC_read_dirty(GC_mark_state == MS_INVALID);
#         endif
        }
        GC_n_rescuing_pages = 0;
#   endif
    if (GC_mark_state == MS_NONE) {
        GC_mark_state = MS_PUSH_RESCUERS;
    } else {
        GC_ASSERT(GC_mark_state == MS_INVALID);
        /* This is really a full collection, and mark bits are invalid. */
    }
    GC_scan_ptr = NULL;
}

#ifdef PARALLEL_MARK
    STATIC void GC_do_parallel_mark(void); /* Initiate parallel marking. */
#endif /* PARALLEL_MARK */

#ifdef GC_DISABLE_INCREMENTAL
# define GC_push_next_marked_dirty(h) GC_push_next_marked(h)
#else
  STATIC struct hblk * GC_push_next_marked_dirty(struct hblk *h);
                /* Invoke GC_push_marked on next dirty block above h.   */
                /* Return a pointer just past the end of this block.    */
#endif /* !GC_DISABLE_INCREMENTAL */
STATIC struct hblk * GC_push_next_marked(struct hblk *h);
                /* Ditto, but also mark from clean pages.       */
STATIC struct hblk * GC_push_next_marked_uncollectable(struct hblk *h);
                /* Ditto, but mark only from uncollectible pages.       */

static void alloc_mark_stack(size_t);

static void push_roots_and_advance(GC_bool push_all, ptr_t cold_gc_frame)
{
  if (GC_scan_ptr != NULL) return; /* not ready to push */

  GC_push_roots(push_all, cold_gc_frame);
  GC_objects_are_marked = TRUE;
  if (GC_mark_state != MS_INVALID)
    GC_mark_state = MS_ROOTS_PUSHED;
}

STATIC GC_on_mark_stack_empty_proc GC_on_mark_stack_empty;

GC_API void GC_CALL GC_set_on_mark_stack_empty(GC_on_mark_stack_empty_proc fn)
{
  LOCK();
  GC_on_mark_stack_empty = fn;
  UNLOCK();
}

GC_API GC_on_mark_stack_empty_proc GC_CALL GC_get_on_mark_stack_empty(void)
{
  GC_on_mark_stack_empty_proc fn;

  READER_LOCK();
  fn = GC_on_mark_stack_empty;
  READER_UNLOCK();
  return fn;
}

/* Perform a small amount of marking.                   */
/* We try to touch roughly a page of memory.            */
/* Return TRUE if we just finished a mark phase.        */
/* Cold_gc_frame is an address inside a GC frame that   */
/* remains valid until all marking is complete.         */
/* A zero value indicates that it's OK to miss some     */
/* register values.  In the case of an incremental      */
/* collection, the world may be running.                */
#ifdef WRAP_MARK_SOME
  /* For Win32, this is called after we establish a structured  */
  /* exception (or signal) handler, in case Windows unmaps one  */
  /* of our root segments.  Note that this code should never    */
  /* generate an incremental GC write fault.                    */
  STATIC GC_bool GC_mark_some_inner(ptr_t cold_gc_frame)
#else
  GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame)
#endif
{
    GC_ASSERT(I_HOLD_LOCK());
    switch (GC_mark_state) {
        case MS_NONE:
            return TRUE;

        case MS_PUSH_RESCUERS:
            if (ADDR_GE((ptr_t)GC_mark_stack_top,
                  (ptr_t)(GC_mark_stack_limit - INITIAL_MARK_STACK_SIZE/2))) {
                /* Go ahead and mark, even though that might cause us to */
                /* see more marked dirty objects later on.  Avoid this   */
                /* in the future.                                        */
                GC_mark_stack_too_small = TRUE;
                MARK_FROM_MARK_STACK();
            } else {
                GC_scan_ptr = GC_push_next_marked_dirty(GC_scan_ptr);
#               ifndef GC_DISABLE_INCREMENTAL
                  if (NULL == GC_scan_ptr) {
                    GC_COND_LOG_PRINTF("Marked from %lu dirty pages\n",
                                       (unsigned long)GC_n_rescuing_pages);
                  }
#               endif
                push_roots_and_advance(FALSE, cold_gc_frame);
            }
            GC_ASSERT(GC_mark_state == MS_PUSH_RESCUERS
                      || GC_mark_state == MS_ROOTS_PUSHED
                      || GC_mark_state == MS_INVALID);
            break;

        case MS_PUSH_UNCOLLECTABLE:
            if (ADDR_GE((ptr_t)GC_mark_stack_top,
                        (ptr_t)(GC_mark_stack + GC_mark_stack_size/4))) {
#               ifdef PARALLEL_MARK
                  /* Avoid this, since we don't parallelize the marker  */
                  /* here.                                              */
                  if (GC_parallel) GC_mark_stack_too_small = TRUE;
#               endif
                MARK_FROM_MARK_STACK();
            } else {
                GC_scan_ptr = GC_push_next_marked_uncollectable(GC_scan_ptr);
                push_roots_and_advance(TRUE, cold_gc_frame);
            }
            GC_ASSERT(GC_mark_state == MS_PUSH_UNCOLLECTABLE
                      || GC_mark_state == MS_ROOTS_PUSHED
                      || GC_mark_state == MS_INVALID);
            break;

        case MS_ROOTS_PUSHED:
#           ifdef PARALLEL_MARK
              /* Eventually, incremental marking should run             */
              /* asynchronously in multiple threads, without acquiring  */
              /* the allocator lock.                                    */
              /* For now, parallel marker is disabled if there is       */
              /* a chance that marking could be interrupted by          */
              /* a client-supplied time limit or custom stop function.  */
                if (GC_parallel && !GC_parallel_mark_disabled) {
                  GC_do_parallel_mark();
                  GC_ASSERT(ADDR_LT((ptr_t)GC_mark_stack_top,
                                    GC_first_nonempty));
                  GC_mark_stack_top = GC_mark_stack - 1;
                  if (GC_mark_stack_too_small) {
                    alloc_mark_stack(2*GC_mark_stack_size);
                  }
                  if (GC_mark_state == MS_ROOTS_PUSHED) {
                    GC_mark_state = MS_NONE;
                    return TRUE;
                  }
                  GC_ASSERT(GC_mark_state == MS_INVALID);
                  break;
                }
#           endif
            if (ADDR_GE((ptr_t)GC_mark_stack_top, (ptr_t)GC_mark_stack)) {
                MARK_FROM_MARK_STACK();
            } else {
                GC_on_mark_stack_empty_proc on_ms_empty =
                                                GC_on_mark_stack_empty;

                if (on_ms_empty != 0) {
                    GC_mark_stack_top = on_ms_empty(GC_mark_stack_top,
                                                    GC_mark_stack_limit);
                    /* If we pushed new items, we need to continue  */
                    /* processing.                                  */
                    if (ADDR_GE((ptr_t)GC_mark_stack_top,
                                (ptr_t)GC_mark_stack))
                        break;
                }
                if (GC_mark_stack_too_small) {
                    alloc_mark_stack(2*GC_mark_stack_size);
                }
                GC_mark_state = MS_NONE;
                return TRUE;
            }
            GC_ASSERT(GC_mark_state == MS_ROOTS_PUSHED
                      || GC_mark_state == MS_INVALID);
            break;

        case MS_INVALID:
        case MS_PARTIALLY_INVALID:
            if (!GC_objects_are_marked) {
                GC_mark_state = MS_PUSH_UNCOLLECTABLE;
                break;
            }
            if (ADDR_GE((ptr_t)GC_mark_stack_top, (ptr_t)GC_mark_stack)) {
                MARK_FROM_MARK_STACK();
                GC_ASSERT(GC_mark_state == MS_PARTIALLY_INVALID
                          || GC_mark_state == MS_INVALID);
                break;
            }
            if (NULL == GC_scan_ptr && GC_mark_state == MS_INVALID) {
                /* About to start a heap scan for marked objects. */
                /* Mark stack is empty.  OK to reallocate.        */
                if (GC_mark_stack_too_small) {
                    alloc_mark_stack(2*GC_mark_stack_size);
                }
                GC_mark_state = MS_PARTIALLY_INVALID;
            }
            GC_scan_ptr = GC_push_next_marked(GC_scan_ptr);
            if (GC_mark_state == MS_PARTIALLY_INVALID)
                push_roots_and_advance(TRUE, cold_gc_frame);
            GC_ASSERT(GC_mark_state == MS_ROOTS_PUSHED
                      || GC_mark_state == MS_PARTIALLY_INVALID
                      || GC_mark_state == MS_INVALID);
            break;

        default:
            ABORT("GC_mark_some: bad state");
    }
    return FALSE;
}

#ifdef WRAP_MARK_SOME
  GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame)
  {
      GC_bool ret_val;

      if (GC_no_dls) {
        ret_val = GC_mark_some_inner(cold_gc_frame);
      } else {
        /* Windows appears to asynchronously create and remove      */
        /* writable memory mappings, for reasons we haven't yet     */
        /* understood.  Since we look for writable regions to       */
        /* determine the root set, we may try to mark from an       */
        /* address range that disappeared since we started the      */
        /* collection.  Thus we have to recover from faults here.   */
        /* This code seems to be necessary for WinCE (at least in   */
        /* the case we'd decide to add MEM_PRIVATE sections to      */
        /* data roots in GC_register_dynamic_libraries()).          */
        /* It's conceivable that this is the same issue as with     */
        /* terminating threads that we see with Linux and           */
        /* USE_PROC_FOR_LIBRARIES.                                  */
#       ifndef NO_SEH_AVAILABLE
          __try {
            ret_val = GC_mark_some_inner(cold_gc_frame);
          } __except (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION ?
                  EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
            goto handle_ex;
          }
#       else
#         if defined(USE_PROC_FOR_LIBRARIES) && !defined(DEFAULT_VDB)
            if (GC_auto_incremental) {
              static GC_bool is_warned = FALSE;

              if (!is_warned) {
                is_warned = TRUE;
                WARN("Incremental GC incompatible with /proc roots\n", 0);
              }
              /* I'm not sure if this could still work ...  */
            }
#         endif
          /* If USE_PROC_FOR_LIBRARIES, we are handling the case in     */
          /* which /proc is used for root finding, and we have threads. */
          /* We may find a stack for a thread that is in the process of */
          /* exiting, and disappears while we are marking it.           */
          /* This seems extremely difficult to avoid otherwise.         */
          GC_setup_temporary_fault_handler();
          if (SETJMP(GC_jmp_buf) != 0) goto handle_ex;
          ret_val = GC_mark_some_inner(cold_gc_frame);
          GC_reset_fault_handler();
#       endif
      }

#     if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS)
        /* With DllMain-based thread tracking, a thread may have        */
        /* started while we were marking.  This is logically equivalent */
        /* to the exception case; our results are invalid and we have   */
        /* to start over.  This cannot be prevented since we can't      */
        /* block in DllMain.                                            */
        if (GC_started_thread_while_stopped())
          goto handle_thr_start;
#     endif
      return ret_val;

    handle_ex:
    /* Exception handler starts here for all cases. */
#     if defined(NO_SEH_AVAILABLE)
        GC_reset_fault_handler();
#     endif
      {
        static word warned_gc_no;

        /* Report caught ACCESS_VIOLATION, once per collection. */
        if (warned_gc_no != GC_gc_no) {
          GC_COND_LOG_PRINTF("Memory mapping disappeared at collection #%lu\n",
                             (unsigned long)GC_gc_no + 1);
          warned_gc_no = GC_gc_no;
        }
      }
#   if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS)
      handle_thr_start:
#   endif
      /* We have bad roots on the mark stack - discard it.      */
      /* Rescan from marked objects.  Redetermine roots.        */
#     ifdef REGISTER_LIBRARIES_EARLY
        START_WORLD();
        GC_cond_register_dynamic_libraries();
        STOP_WORLD();
#     endif
      GC_invalidate_mark_state();
      GC_scan_ptr = NULL;
      return FALSE;
  }
#endif /* WRAP_MARK_SOME */

GC_INNER void GC_invalidate_mark_state(void)
{
    GC_mark_state = MS_INVALID;
    GC_mark_stack_top = GC_mark_stack-1;
}

STATIC mse * GC_signal_mark_stack_overflow(mse *msp)
{
    GC_mark_state = MS_INVALID;
#   ifdef PARALLEL_MARK
      /* We are using a local_mark_stack in parallel mode, so   */
      /* do not signal the global mark stack to be resized.     */
      /* That will be done if required in GC_return_mark_stack. */
      if (!GC_parallel)
        GC_mark_stack_too_small = TRUE;
#   else
      GC_mark_stack_too_small = TRUE;
#   endif
    GC_COND_LOG_PRINTF("Mark stack overflow; current size: %lu entries\n",
                       (unsigned long)GC_mark_stack_size);
#   if defined(CPPCHECK)
      GC_noop1_ptr(msp);
#   endif
    return msp - GC_MARK_STACK_DISCARDS;
}

/*
 * Mark objects pointed to by the regions described by
 * mark stack entries between mark_stack and mark_stack_top,
 * inclusive.  Assumes the upper limit of a mark stack entry
 * is never 0.  A mark stack entry never has size 0.
 * We try to traverse on the order of a hblk of memory before we return.
 * Caller is responsible for calling this until the mark stack is empty.
 * Note that this is the most performance critical routine in the
 * collector.  Hence it contains all sorts of ugly hacks to speed
 * things up.  In particular, we avoid procedure calls on the common
 * path, we take advantage of peculiarities of the mark descriptor
 * encoding, we optionally maintain a cache for the block address to
 * header mapping, we prefetch when an object is "grayed", etc.
 */
GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY GC_ATTR_NO_SANITIZE_THREAD
GC_INNER mse * GC_mark_from(mse *mark_stack_top, mse *mark_stack,
                            mse *mark_stack_limit)
{
  signed_word credit = HBLKSIZE;  /* Remaining credit for marking work. */
  word descr;
  ptr_t current_p;      /* Pointer to current candidate ptr.            */
  ptr_t q;              /* Candidate pointer.                           */
  ptr_t limit = 0;      /* (Incl) limit of current candidate range.     */
  ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
  ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
  DECLARE_HDR_CACHE;

# define SPLIT_RANGE_PTRS 128 /* must be power of 2 */

  GC_objects_are_marked = TRUE;
  INIT_HDR_CACHE;
# if defined(OS2) || CPP_PTRSZ > CPP_WORDSZ
        /* OS/2: use untweaked version to circumvent a compiler problem. */
    while (ADDR_GE((ptr_t)mark_stack_top, (ptr_t)mark_stack) && credit >= 0)
# else
    while (((((word)mark_stack_top - (word)mark_stack) | (word)credit)
            & SIGNB) == 0)
# endif
  {
    current_p = mark_stack_top -> mse_start;
    descr = mark_stack_top -> mse_descr;
  retry:
    /* current_p and descr describe the current object.                 */
    /* (*mark_stack_top) is vacant.                                     */
    /* The following is 0 only for small objects described by a simple  */
    /* length descriptor.  For many applications this is the common     */
    /* case, so we try to detect it quickly.                            */
    if (descr & (~(word)(PTRS_TO_BYTES(SPLIT_RANGE_PTRS)-1) | GC_DS_TAGS)) {
      word tag = descr & GC_DS_TAGS;

      GC_STATIC_ASSERT(GC_DS_TAGS == 0x3);
      switch (tag) {
        case GC_DS_LENGTH:
          /* Large length.  Process part of the range to avoid pushing  */
          /* too much on the stack.                                     */

          /* Either it is a heap object or a region outside the heap.   */
          GC_ASSERT(descr < GC_greatest_real_heap_addr
                            - GC_least_real_heap_addr
                    || GC_least_real_heap_addr + sizeof(ptr_t)
                        >= ADDR(current_p) + descr
                    || ADDR(current_p) >= GC_greatest_real_heap_addr);
#         ifdef PARALLEL_MARK
#           define SHARE_BYTES 2048
            if (descr > SHARE_BYTES && GC_parallel
                && ADDR_LT((ptr_t)mark_stack_top,
                           (ptr_t)(mark_stack_limit - 1))) {
              word new_size = (descr >> 1) & ~(word)(sizeof(ptr_t)-1);

              mark_stack_top -> mse_start = current_p;
              mark_stack_top -> mse_descr
                        = (new_size + sizeof(ptr_t)) | GC_DS_LENGTH;
                                        /* This makes sure we handle    */
                                        /* misaligned pointers.         */
              mark_stack_top++;
#             ifdef ENABLE_TRACE
                if (ADDR_INSIDE(GC_trace_ptr, current_p, current_p + descr)) {
                  GC_log_printf("GC #%lu: large section; start %p, len %lu,"
                                " splitting (parallel) at %p\n",
                                (unsigned long)GC_gc_no, (void *)current_p,
                                (unsigned long)descr,
                                (void *)(current_p + new_size));
                }
#             endif
              current_p += new_size;
              descr -= new_size;
              goto retry;
            }
#         endif /* PARALLEL_MARK */
          limit = current_p + PTRS_TO_BYTES(SPLIT_RANGE_PTRS-1);
          mark_stack_top -> mse_start = limit;
          mark_stack_top -> mse_descr
                                = descr - PTRS_TO_BYTES(SPLIT_RANGE_PTRS-1);
#         ifdef ENABLE_TRACE
            if (ADDR_INSIDE(GC_trace_ptr, current_p, current_p + descr)) {
              GC_log_printf("GC #%lu: large section; start %p, len %lu,"
                            " splitting at %p\n",
                            (unsigned long)GC_gc_no, (void *)current_p,
                            (unsigned long)descr, (void *)limit);
            }
#         endif
          /* Make sure that pointers overlapping the two ranges are     */
          /* considered.                                                */
          limit += sizeof(ptr_t) - ALIGNMENT;
          break;
        case GC_DS_BITMAP:
          mark_stack_top--;
#         ifdef ENABLE_TRACE
            if (ADDR_INSIDE(GC_trace_ptr, current_p,
                            current_p + PTRS_TO_BYTES(BITMAP_BITS))) {
              GC_log_printf("GC #%lu: tracing from %p bitmap descr 0x%lx\n",
                            (unsigned long)GC_gc_no, (void *)current_p,
                            (unsigned long)descr);
            }
#         endif
          descr &= ~(word)GC_DS_TAGS;
          credit -= (signed_word)PTRS_TO_BYTES(CPP_PTRSZ / 2); /* guess */
          for (; descr != 0; descr <<= 1, current_p += sizeof(ptr_t)) {
            if ((descr & SIGNB) == 0) continue;
            LOAD_PTR_OR_CONTINUE(q, current_p);
            FIXUP_POINTER(q);
            if (ADDR_LT(least_ha, q) && ADDR_LT(q, greatest_ha)) {
                PREFETCH(q);
#               ifdef ENABLE_TRACE
                  if (GC_trace_ptr == current_p) {
                    GC_log_printf("GC #%lu: considering(3) %p -> %p\n",
                                  (unsigned long)GC_gc_no, (void *)current_p,
                                  (void *)q);
                  }
#               endif
                PUSH_CONTENTS(q, mark_stack_top, mark_stack_limit, current_p);
            }
          }
          continue;
        case GC_DS_PROC:
          mark_stack_top--;
#         ifdef ENABLE_TRACE
            if (ADDR_GE(GC_trace_ptr, current_p)) {
              const void *base = GC_base(current_p);

              if (base != NULL && GC_base(GC_trace_ptr) == base) {
                GC_log_printf("GC #%lu: tracing from %p, proc descr 0x%lx\n",
                              (unsigned long)GC_gc_no, (void *)current_p,
                              (unsigned long)descr);
              }
            }
#         endif
          credit -= GC_PROC_BYTES;
          mark_stack_top = (*PROC(descr))((word *)current_p, mark_stack_top,
                                          mark_stack_limit, ENV(descr));
          continue;
        case GC_DS_PER_OBJECT:
          if (!(descr & SIGNB)) {
            /* Descriptor is in the object.     */
            descr = *(word *)(current_p + descr - GC_DS_PER_OBJECT);
          } else {
            /* Descriptor is in the type descriptor pointed to by the   */
            /* first "pointer-sized" word of the object.                */
            ptr_t type_descr = *(ptr_t *)current_p;

            /* type_descr is either a valid pointer to the descriptor   */
            /* structure, or this object was on a free list.            */
            /* If it was anything but the last object on the free list, */
            /* we will misinterpret the next object on the free list as */
            /* the type descriptor, and get a 0 GC descriptor, which    */
            /* is ideal.  Unfortunately, we need to check for the last  */
            /* object case explicitly.                                  */
            if (EXPECT(NULL == type_descr, FALSE)) {
                mark_stack_top--;
                continue;
            }
            descr = *(word *)(type_descr
                              - ((signed_word)descr + (GC_INDIR_PER_OBJ_BIAS
                                                       - GC_DS_PER_OBJECT)));
          }
          if (0 == descr) {
              /* Can happen either because we generated a 0 descriptor  */
              /* or we saw a pointer to a free object.                  */
              mark_stack_top--;
              continue;
          }
          goto retry;
      }
    } else {
      /* Small object with length descriptor.   */
      mark_stack_top--;
#     ifndef SMALL_CONFIG
        if (descr < sizeof(ptr_t))
          continue;
#     endif
#     ifdef ENABLE_TRACE
        if (ADDR_INSIDE(GC_trace_ptr, current_p, current_p + descr)) {
          GC_log_printf("GC #%lu: small object; start %p, len %lu\n",
                        (unsigned long)GC_gc_no, (void *)current_p,
                        (unsigned long)descr);
        }
#     endif
      limit = current_p + descr;
    }
    /* The simple case in which we're scanning a range. */
    GC_ASSERT((ADDR(current_p) & (ALIGNMENT-1)) == 0);
    credit -= limit - current_p;
    limit -= sizeof(ptr_t);
    {
#     define PREF_DIST 4

#     if !defined(SMALL_CONFIG) && !defined(USE_PTR_HWTAG)
        ptr_t deferred;

        /* Try to prefetch the next pointer to be examined ASAP.        */
        /* Empirically, this also seems to help slightly without        */
        /* prefetches, at least on Linux/i686.  Presumably this loop    */
        /* ends up with less register pressure, and gcc thus ends up    */
        /* generating slightly better code.  Overall gcc code quality   */
        /* for this loop is still not great.                            */
        for (;;) {
          PREFETCH(limit - PREF_DIST*CACHE_LINE_SIZE);
          GC_ASSERT(ADDR_GE(limit, current_p));
          deferred = *(ptr_t *)limit;
          FIXUP_POINTER(deferred);
          limit -= ALIGNMENT;
          if (ADDR_LT(least_ha, deferred) && ADDR_LT(deferred, greatest_ha)) {
            PREFETCH(deferred);
            break;
          }
          if (ADDR_LT(limit, current_p)) goto next_object;
          /* Unroll once, so we don't do too many of the prefetches     */
          /* based on limit.                                            */
          deferred = *(ptr_t *)limit;
          FIXUP_POINTER(deferred);
          limit -= ALIGNMENT;
          if (ADDR_LT(least_ha, deferred) && ADDR_LT(deferred, greatest_ha)) {
            PREFETCH(deferred);
            break;
          }
          if (ADDR_LT(limit, current_p)) goto next_object;
        }
#     endif

      for (; ADDR_GE(limit, current_p); current_p += ALIGNMENT) {
        /* Empirically, unrolling this loop doesn't help a lot. */
        /* Since PUSH_CONTENTS expands to a lot of code,        */
        /* we don't.                                            */
        LOAD_PTR_OR_CONTINUE(q, current_p);
        FIXUP_POINTER(q);
        PREFETCH(current_p + PREF_DIST*CACHE_LINE_SIZE);
        if (ADDR_LT(least_ha, q) && ADDR_LT(q, greatest_ha)) {
          /* Prefetch the contents of the object we just pushed.  It's  */
          /* likely we will need them soon.                             */
          PREFETCH(q);
#         ifdef ENABLE_TRACE
            if (GC_trace_ptr == current_p) {
              GC_log_printf("GC #%lu: considering(1) %p -> %p\n",
                            (unsigned long)GC_gc_no, (void *)current_p,
                            (void *)q);
            }
#         endif
          PUSH_CONTENTS(q, mark_stack_top, mark_stack_limit, current_p);
        }
      }

#     if !defined(SMALL_CONFIG) && !defined(USE_PTR_HWTAG)
        /* We still need to mark the entry we previously prefetched.    */
        /* We already know that it passes the preliminary pointer       */
        /* validity test.                                               */
#       ifdef ENABLE_TRACE
            if (GC_trace_ptr == current_p) {
              GC_log_printf("GC #%lu: considering(2) %p -> %p\n",
                            (unsigned long)GC_gc_no, (void *)current_p,
                            (void *)deferred);
            }
#       endif
        PUSH_CONTENTS(deferred, mark_stack_top, mark_stack_limit, current_p);
        next_object:;
#     endif
    }
  }
  return mark_stack_top;
}

#ifdef PARALLEL_MARK

STATIC GC_bool GC_help_wanted = FALSE;  /* Protected by the mark lock.  */
STATIC unsigned GC_helper_count = 0;    /* Number of running helpers.   */
                                        /* Protected by the mark lock.  */
STATIC unsigned GC_active_count = 0;    /* Number of active helpers.    */
                                        /* Protected by the mark lock.  */
                                        /* May increase and decrease    */
                                        /* within each mark cycle.  But */
                                        /* once it returns to 0, it     */
                                        /* stays zero for the cycle.    */

GC_INNER word GC_mark_no = 0;

#ifdef LINT2
# define LOCAL_MARK_STACK_SIZE (HBLKSIZE / 8)
#else
# define LOCAL_MARK_STACK_SIZE HBLKSIZE
        /* Under normal circumstances, this is big enough to guarantee  */
        /* we don't overflow half of it in a single call to             */
        /* GC_mark_from.                                                */
#endif

/* Wait all markers to finish initialization (i.e. store        */
/* marker_[b]sp, marker_mach_threads, GC_marker_Id).            */
GC_INNER void GC_wait_for_markers_init(void)
{
  signed_word count;

  GC_ASSERT(I_HOLD_LOCK());
  if (GC_markers_m1 == 0)
    return;

  /* Allocate the local mark stack for the thread that holds    */
  /* the allocator lock.                                        */
# ifndef CAN_HANDLE_FORK
    GC_ASSERT(NULL == GC_main_local_mark_stack);
# else
    if (NULL == GC_main_local_mark_stack)
# endif
  {
    size_t bytes_to_get =
                ROUNDUP_PAGESIZE_IF_MMAP(LOCAL_MARK_STACK_SIZE * sizeof(mse));

    GC_ASSERT(GC_page_size != 0);
    GC_main_local_mark_stack = (mse *)GC_os_get_mem(bytes_to_get);
    if (NULL == GC_main_local_mark_stack)
      ABORT("Insufficient memory for main local_mark_stack");
  }

  /* Reuse the mark lock and builders count to synchronize      */
  /* marker threads startup.                                    */
  GC_acquire_mark_lock();
  GC_fl_builder_count += GC_markers_m1;
  count = GC_fl_builder_count;
  GC_release_mark_lock();
  if (count != 0) {
    GC_ASSERT(count > 0);
    GC_wait_for_reclaim();
  }
}

/* Steal mark stack entries starting at mse low into mark stack local   */
/* until we either steal mse high, or we have n_to_get entries.         */
/* Return a pointer to the top of the local mark stack.                 */
/* (*next) is replaced by a pointer to the next unscanned mark stack    */
/* entry.                                                               */
STATIC mse * GC_steal_mark_stack(mse * low, mse * high, mse * local,
                                 size_t n_to_get, mse **next)
{
    mse *p;
    mse *top = local - 1;
    size_t i = 0;

    GC_ASSERT(ADDR_GE((ptr_t)high, (ptr_t)(low - 1))
              && (word)(high - low + 1) <= GC_mark_stack_size);
    for (p = low; ADDR_GE((ptr_t)high, (ptr_t)p) && i <= n_to_get; ++p) {
        word descr = AO_load(&(p -> mse_descr));

        if (descr != 0) {
            /* Must be ordered after read of descr: */
            AO_store_release_write(&(p -> mse_descr), 0);
            /* More than one thread may get this entry, but that's only */
            /* a minor performance problem.                             */
            ++top;
            top -> mse_start = p -> mse_start;
            top -> mse_descr = descr;
            GC_ASSERT((descr & GC_DS_TAGS) != GC_DS_LENGTH /* 0 */
                    || descr < GC_greatest_real_heap_addr
                                - GC_least_real_heap_addr
                    || GC_least_real_heap_addr + sizeof(ptr_t)
                        >= ADDR(p -> mse_start) + descr
                    || ADDR(p -> mse_start) >= GC_greatest_real_heap_addr);
            /* If this is a big object, count it as size/256 + 1 objects. */
            ++i;
            if ((descr & GC_DS_TAGS) == GC_DS_LENGTH)
              i += (size_t)(descr >> 8);
        }
    }
    *next = p;
#   if defined(CPPCHECK)
        GC_noop1_ptr(local);
#   endif
    return top;
}

/* Copy back a local mark stack.        */
/* low and high are inclusive bounds.   */
STATIC void GC_return_mark_stack(mse * low, mse * high)
{
    mse * my_top;
    mse * my_start;
    size_t stack_size;

    if (ADDR_LT((ptr_t)high, (ptr_t)low)) return;
    stack_size = high - low + 1;
    GC_acquire_mark_lock();
    my_top = GC_mark_stack_top; /* Concurrent modification impossible. */
    my_start = my_top + 1;
    if ((word)(my_start - GC_mark_stack + stack_size)
                > (word)GC_mark_stack_size) {
      GC_COND_LOG_PRINTF("No room to copy back mark stack\n");
      GC_mark_state = MS_INVALID;
      GC_mark_stack_too_small = TRUE;
      /* We drop the local mark stack.  We'll fix things later. */
    } else {
      BCOPY(low, my_start, stack_size * sizeof(mse));
      GC_ASSERT((mse *)GC_cptr_load((volatile ptr_t *)&GC_mark_stack_top)
                == my_top);
      GC_cptr_store_release_write((volatile ptr_t *)&GC_mark_stack_top,
                                  (ptr_t)(my_top + stack_size));
                /* Ensures visibility of previously written stack contents. */
    }
    GC_release_mark_lock();
    GC_notify_all_marker();
}

#ifndef N_LOCAL_ITERS
# define N_LOCAL_ITERS 1
#endif

/* This function is only called when the local  */
/* and the main mark stacks are both empty.     */
static GC_bool has_inactive_helpers(void)
{
  GC_bool res;

  GC_acquire_mark_lock();
  res = GC_active_count < GC_helper_count;
  GC_release_mark_lock();
  return res;
}

/* Mark from the local mark stack.              */
/* On return, the local mark stack is empty.    */
/* But this may be achieved by copying the      */
/* local mark stack back into the global one.   */
/* We do not hold the mark lock.                */
STATIC void GC_do_local_mark(mse *local_mark_stack, mse *local_top)
{
    unsigned n;

    for (;;) {
        for (n = 0; n < N_LOCAL_ITERS; ++n) {
            local_top = GC_mark_from(local_top, local_mark_stack,
                                     local_mark_stack + LOCAL_MARK_STACK_SIZE);
            if (ADDR_LT((ptr_t)local_top, (ptr_t)local_mark_stack)) return;
            if ((word)(local_top - local_mark_stack)
                        >= LOCAL_MARK_STACK_SIZE / 2) {
                GC_return_mark_stack(local_mark_stack, local_top);
                return;
            }
        }
        if (ADDR_LT(GC_cptr_load((volatile ptr_t *)&GC_mark_stack_top),
                    GC_cptr_load(&GC_first_nonempty))
            && ADDR_LT((ptr_t)(local_mark_stack + 1), (ptr_t)local_top)
            && has_inactive_helpers()) {
            /* Try to share the load, since the main stack is empty,    */
            /* and helper threads are waiting for a refill.             */
            /* The entries near the bottom of the stack are likely      */
            /* to require more work.  Thus we return those, even though */
            /* it's harder.                                             */
            mse *new_bottom = local_mark_stack
                                + (local_top - local_mark_stack)/2;

            GC_ASSERT(ADDR_LT((ptr_t)local_mark_stack, (ptr_t)new_bottom)
                      && ADDR_LT((ptr_t)new_bottom, (ptr_t)local_top));
            GC_return_mark_stack(local_mark_stack, new_bottom - 1);
            memmove(local_mark_stack, new_bottom,
                    (local_top - new_bottom + 1) * sizeof(mse));
            local_top -= new_bottom - local_mark_stack;
        }
    }
}

#ifndef ENTRIES_TO_GET
# define ENTRIES_TO_GET 5
#endif

/* Mark using the local mark stack until the global mark stack is empty */
/* and there are no active workers. Update GC_first_nonempty to reflect */
/* progress.  Caller holds the mark lock.                               */
/* Caller has already incremented GC_helper_count.  We decrement it,    */
/* and maintain GC_active_count.                                        */
STATIC void GC_mark_local(mse *local_mark_stack, int id)
{
    mse *my_first_nonempty;

    GC_active_count++;
    my_first_nonempty = (mse *)GC_cptr_load(&GC_first_nonempty);
    GC_ASSERT(ADDR_GE((ptr_t)my_first_nonempty, (ptr_t)GC_mark_stack));
    GC_ASSERT(ADDR_GE(GC_cptr_load((volatile ptr_t *)&GC_mark_stack_top)
                        + sizeof(mse), (ptr_t)my_first_nonempty));
    GC_VERBOSE_LOG_PRINTF("Starting mark helper %d\n", id);
    GC_release_mark_lock();
    for (;;) {
        size_t n_on_stack, n_to_get;
        mse *my_top, *local_top;
        mse *global_first_nonempty = (mse *)GC_cptr_load(&GC_first_nonempty);

        GC_ASSERT(ADDR_GE((ptr_t)my_first_nonempty, (ptr_t)GC_mark_stack)
            && ADDR_GE(GC_cptr_load((volatile ptr_t *)&GC_mark_stack_top)
                            + sizeof(mse), (ptr_t)my_first_nonempty));
        GC_ASSERT(ADDR_GE((ptr_t)global_first_nonempty, (ptr_t)GC_mark_stack));
        if (ADDR_LT((ptr_t)my_first_nonempty, (ptr_t)global_first_nonempty)) {
            my_first_nonempty = global_first_nonempty;
        } else if (ADDR_LT((ptr_t)global_first_nonempty,
                           (ptr_t)my_first_nonempty)) {
            (void)GC_cptr_compare_and_swap(&GC_first_nonempty,
                                           (ptr_t)global_first_nonempty,
                                           (ptr_t)my_first_nonempty);
            /* If this fails, we just go ahead, without updating        */
            /* GC_first_nonempty.                                       */
        }
        /* Perhaps we should also update GC_first_nonempty, if it */
        /* is less.  But that would require using atomic updates. */
        my_top = (mse *)GC_cptr_load_acquire(
                                (volatile ptr_t *)&GC_mark_stack_top);
        if (ADDR_LT((ptr_t)my_top, (ptr_t)my_first_nonempty)) {
            GC_acquire_mark_lock();
            my_top = GC_mark_stack_top;
                /* Asynchronous modification impossible here,   */
                /* since we hold the mark lock.                 */
            n_on_stack = my_top - my_first_nonempty + 1;
            if (0 == n_on_stack) {
                GC_active_count--;
                GC_ASSERT(GC_active_count <= GC_helper_count);
                /* Other markers may redeposit objects          */
                /* on the stack.                                */
                if (0 == GC_active_count) GC_notify_all_marker();
                while (GC_active_count > 0
                       && ADDR_LT((ptr_t)GC_mark_stack_top,
                                  GC_cptr_load(&GC_first_nonempty))) {
                    /* We will be notified if either GC_active_count    */
                    /* reaches zero, or if more objects are pushed on   */
                    /* the global mark stack.                           */
                    GC_wait_marker();
                }
                if (0 == GC_active_count
                    && ADDR_LT((ptr_t)GC_mark_stack_top,
                               GC_cptr_load(&GC_first_nonempty))) {
                    GC_bool need_to_notify = FALSE;

                    /* The above conditions can't be falsified while we */
                    /* hold the mark lock, since neither                */
                    /* GC_active_count nor GC_mark_stack_top can        */
                    /* change.  GC_first_nonempty can only be           */
                    /* incremented asynchronously.  Thus we know that   */
                    /* both conditions actually held simultaneously.    */
                    GC_helper_count--;
                    if (0 == GC_helper_count) need_to_notify = TRUE;
                    GC_VERBOSE_LOG_PRINTF("Finished mark helper %d\n", id);
                    if (need_to_notify) GC_notify_all_marker();
                    return;
                }
                /* Else there's something on the stack again, or        */
                /* another helper may push something.                   */
                GC_active_count++;
                GC_ASSERT(GC_active_count > 0);
                GC_release_mark_lock();
                continue;
            } else {
                GC_release_mark_lock();
            }
        } else {
            n_on_stack = my_top - my_first_nonempty + 1;
        }
        n_to_get = ENTRIES_TO_GET;
        if (n_on_stack < 2 * ENTRIES_TO_GET) n_to_get = 1;
        local_top = GC_steal_mark_stack(my_first_nonempty, my_top,
                                        local_mark_stack, n_to_get,
                                        &my_first_nonempty);
        GC_ASSERT(ADDR_GE((ptr_t)my_first_nonempty, (ptr_t)GC_mark_stack)
            && ADDR_GE(GC_cptr_load((volatile ptr_t *)&GC_mark_stack_top)
                            + sizeof(mse), (ptr_t)my_first_nonempty));
        GC_do_local_mark(local_mark_stack, local_top);
    }
}

/* Perform parallel mark.  We hold the allocator lock, but not the mark */
/* lock.  Currently runs until the mark stack is empty.                 */
STATIC void GC_do_parallel_mark(void)
{
    GC_ASSERT(I_HOLD_LOCK());
    GC_acquire_mark_lock();

    /* This could be a GC_ASSERT, but it seems safer to keep it on      */
    /* all the time, especially since it's cheap.                       */
    if (GC_help_wanted || GC_active_count != 0 || GC_helper_count != 0)
        ABORT("Tried to start parallel mark in bad state");
    GC_VERBOSE_LOG_PRINTF("Starting marking for mark phase number %lu\n",
                          (unsigned long)GC_mark_no);
    GC_cptr_store(&GC_first_nonempty, (ptr_t)GC_mark_stack);
    GC_active_count = 0;
    GC_helper_count = 1;
    GC_help_wanted = TRUE;
    GC_notify_all_marker();
        /* Wake up potential helpers.   */
    GC_mark_local(GC_main_local_mark_stack, 0);
    GC_help_wanted = FALSE;
    /* Done; clean up.  */
    while (GC_helper_count > 0) {
      GC_wait_marker();
    }
    /* GC_helper_count cannot be incremented while not GC_help_wanted.  */
    GC_VERBOSE_LOG_PRINTF("Finished marking for mark phase number %lu\n",
                          (unsigned long)GC_mark_no);
    GC_mark_no++;
    GC_release_mark_lock();
    GC_notify_all_marker();
}

/* Try to help out the marker, if it's running.  We hold the mark lock  */
/* only, the initiating thread holds the allocator lock.                */
GC_INNER void GC_help_marker(word my_mark_no)
{
#   define my_id my_id_mse.mse_descr
    mse my_id_mse;  /* align local_mark_stack explicitly */
    mse local_mark_stack[LOCAL_MARK_STACK_SIZE];
                /* Note: local_mark_stack is quite big (up to 128 KiB). */

    GC_ASSERT(I_DONT_HOLD_LOCK());
    GC_ASSERT(GC_parallel);
    while (GC_mark_no < my_mark_no
           || (!GC_help_wanted && GC_mark_no == my_mark_no)) {
      GC_wait_marker();
    }
    my_id = GC_helper_count;
    if (GC_mark_no != my_mark_no || my_id > (unsigned)GC_markers_m1) {
      /* Second test is useful only if original threads can also        */
      /* act as helpers.  Under Linux they can't.                       */
      return;
    }
    GC_helper_count = (unsigned)my_id + 1;
    GC_mark_local(local_mark_stack, (int)my_id);
    /* GC_mark_local decrements GC_helper_count. */
#   undef my_id
}

#endif /* PARALLEL_MARK */

/* Allocate or reallocate space for mark stack of size n entries.  */
/* May silently fail.                                              */
static void alloc_mark_stack(size_t n)
{
#   ifdef GWW_VDB
      static GC_bool GC_incremental_at_stack_alloc = FALSE;

      GC_bool recycle_old;
#   endif
    mse * new_stack;

    GC_ASSERT(I_HOLD_LOCK());
    new_stack = (mse *)GC_scratch_alloc(n * sizeof(struct GC_ms_entry));
#   ifdef GWW_VDB
      /* Don't recycle a stack segment obtained with the wrong flags.   */
      /* Win32 GetWriteWatch requires the right kind of memory.         */
      recycle_old = !GC_auto_incremental || GC_incremental_at_stack_alloc;
      GC_incremental_at_stack_alloc = GC_auto_incremental;
#   endif

    GC_mark_stack_too_small = FALSE;
    if (GC_mark_stack != NULL) {
        if (new_stack != 0) {
#         ifdef GWW_VDB
            if (recycle_old)
#         endif
          {
            /* Recycle old space.       */
            GC_scratch_recycle_inner(GC_mark_stack,
                        GC_mark_stack_size * sizeof(struct GC_ms_entry));
          }
          GC_mark_stack = new_stack;
          GC_mark_stack_size = n;
          /* FIXME: Do we need some way to reset GC_mark_stack_size?    */
          GC_mark_stack_limit = new_stack + n;
          GC_COND_LOG_PRINTF("Grew mark stack to %lu frames\n",
                             (unsigned long)GC_mark_stack_size);
        } else {
          WARN("Failed to grow mark stack to %" WARN_PRIuPTR " frames\n", n);
        }
    } else if (NULL == new_stack) {
        GC_err_printf("No space for mark stack\n");
        EXIT();
    } else {
        GC_mark_stack = new_stack;
        GC_mark_stack_size = n;
        GC_mark_stack_limit = new_stack + n;
    }
    GC_mark_stack_top = GC_mark_stack-1;
}

GC_INNER void GC_mark_init(void)
{
    alloc_mark_stack(INITIAL_MARK_STACK_SIZE);
}

/* Push all locations between bottom and top onto the mark stack.   */
/* bottom is the first location to be checked; top is one past the  */
/* last location to be checked.  Should only be used if there is    */
/* no possibility of mark stack overflow.                           */
GC_API void GC_CALL GC_push_all(void *bottom, void *top)
{
    mse * mark_stack_top;
    word length;

    bottom = PTR_ALIGN_UP((ptr_t)bottom, ALIGNMENT);
    top = PTR_ALIGN_DOWN((ptr_t)top, ALIGNMENT);
    if (ADDR_GE((ptr_t)bottom, (ptr_t)top)) return;

    mark_stack_top = GC_mark_stack_top + 1;
    if (ADDR_GE((ptr_t)mark_stack_top, (ptr_t)GC_mark_stack_limit)) {
        ABORT("Unexpected mark stack overflow");
    }
    length = (word)((ptr_t)top - (ptr_t)bottom);
#   if GC_DS_TAGS > ALIGNMENT - 1
        length = (length + GC_DS_TAGS) & ~(word)GC_DS_TAGS; /* round up */
#   endif
    mark_stack_top -> mse_start = (ptr_t)bottom;
    mark_stack_top -> mse_descr = length | GC_DS_LENGTH;
    GC_mark_stack_top = mark_stack_top;
}

GC_API struct GC_ms_entry * GC_CALL GC_custom_push_range(void *bottom,
                                void *top, struct GC_ms_entry *mark_stack_top,
                                struct GC_ms_entry *mark_stack_limit)
{
    word length;

    bottom = PTR_ALIGN_UP((ptr_t)bottom, ALIGNMENT);
    top = PTR_ALIGN_DOWN((ptr_t)top, ALIGNMENT);
    if (ADDR_GE((ptr_t)bottom, (ptr_t)top)) return mark_stack_top;

    length = (word)((ptr_t)top - (ptr_t)bottom);
#   if GC_DS_TAGS > ALIGNMENT - 1
        length = (length + GC_DS_TAGS) & ~(word)GC_DS_TAGS; /* round up */
#   endif
    return GC_custom_push_proc(length | GC_DS_LENGTH, bottom, mark_stack_top,
                               mark_stack_limit);
}

GC_API struct GC_ms_entry * GC_CALL GC_custom_push_proc(GC_word descr,
                                void *obj, struct GC_ms_entry *mark_stack_top,
                                struct GC_ms_entry *mark_stack_limit)
{
    mark_stack_top++;
    if (ADDR_GE((ptr_t)mark_stack_top, (ptr_t)mark_stack_limit)) {
        mark_stack_top = GC_signal_mark_stack_overflow(mark_stack_top);
    }
    mark_stack_top -> mse_start = (ptr_t)obj;
    mark_stack_top -> mse_descr = descr;
    return mark_stack_top;
}

GC_API void GC_CALL GC_push_proc(GC_word descr, void *obj)
{
    GC_mark_stack_top = GC_custom_push_proc(descr, obj, GC_mark_stack_top,
                                            GC_mark_stack_limit);
}

#ifndef GC_DISABLE_INCREMENTAL

  /* Analogous to the above, but push only those pages h with           */
  /* dirty_fn(h) != 0.  We use GC_push_all to actually push the block.  */
  /* Used both to selectively push dirty pages, or to push a block in   */
  /* piecemeal fashion, to allow for more marking concurrency.          */
  /* Will not overflow mark stack if GC_push_all pushes a small fixed   */
  /* number of entries.  (This is invoked only if GC_push_all pushes    */
  /* a single entry, or if it marks each object before pushing it, thus */
  /* ensuring progress in the event of a stack overflow.)               */
  STATIC void GC_push_selected(ptr_t bottom, ptr_t top,
                               GC_bool (*dirty_fn)(struct hblk *))
  {
    struct hblk * h;

    bottom = PTR_ALIGN_UP(bottom, ALIGNMENT);
    top = PTR_ALIGN_DOWN(top, ALIGNMENT);
    if (ADDR_GE(bottom, top)) return;

    h = HBLKPTR(bottom + HBLKSIZE);
    if (ADDR_GE((ptr_t)h, top)) {
        if ((*dirty_fn)(h-1)) {
            GC_push_all(bottom, top);
        }
        return;
    }
    if ((*dirty_fn)(h-1)) {
        if ((word)(GC_mark_stack_top - GC_mark_stack)
            > 3 * GC_mark_stack_size / 4) {
            GC_push_all(bottom, top);
            return;
        }
        GC_push_all(bottom, h);
    }

    while (ADDR_GE(top, (ptr_t)(h + 1))) {
        if ((*dirty_fn)(h)) {
            if ((word)(GC_mark_stack_top - GC_mark_stack)
                > 3 * GC_mark_stack_size / 4) {
                /* Danger of mark stack overflow.       */
                GC_push_all(h, top);
                return;
            } else {
                GC_push_all(h, h + 1);
            }
        }
        h++;
    }

    if ((ptr_t)h != top && (*dirty_fn)(h)) {
       GC_push_all(h, top);
    }
  }

  GC_API void GC_CALL GC_push_conditional(void *bottom, void *top, int all)
  {
    if (!all) {
      GC_push_selected((ptr_t)bottom, (ptr_t)top, GC_page_was_dirty);
    } else {
#     ifdef PROC_VDB
        if (GC_auto_incremental) {
          /* Pages that were never dirtied cannot contain pointers.     */
          GC_push_selected((ptr_t)bottom, (ptr_t)top, GC_page_was_ever_dirty);
        } else
#     endif
      /* else */ {
        GC_push_all(bottom, top);
      }
    }
  }

# ifndef NO_VDB_FOR_STATIC_ROOTS
#   ifndef PROC_VDB
      /* Same as GC_page_was_dirty but h is allowed to point to some    */
      /* page in the registered static roots only.  Not used if         */
      /* manual VDB is on.                                              */
      STATIC GC_bool GC_static_page_was_dirty(struct hblk *h)
      {
        return get_pht_entry_from_index(GC_grungy_pages, PHT_HASH(h));
      }
#   endif

    GC_INNER void GC_push_conditional_static(void *bottom, void *top,
                                             GC_bool all)
    {
#     ifdef PROC_VDB
        /* Just redirect to the generic routine because PROC_VDB        */
        /* implementation gets the dirty bits map for the whole         */
        /* process memory.                                              */
        GC_push_conditional(bottom, top, all);
#     else
        if (all || !GC_is_vdb_for_static_roots()) {
          GC_push_all(bottom, top);
        } else {
          GC_push_selected((ptr_t)bottom, (ptr_t)top,
                           GC_static_page_was_dirty);
        }
#     endif
    }
# endif /* !NO_VDB_FOR_STATIC_ROOTS */

#else
  GC_API void GC_CALL GC_push_conditional(void *bottom, void *top, int all)
  {
    UNUSED_ARG(all);
    GC_push_all(bottom, top);
  }
#endif /* GC_DISABLE_INCREMENTAL */

#if defined(AMIGA) || defined(MACOS) || defined(GC_DARWIN_THREADS)
  void GC_push_one(word p)
  {
    GC_PUSH_ONE_STACK((ptr_t)p, MARKED_FROM_REGISTER);
  }
#endif

#ifdef GC_WIN32_THREADS
  GC_INNER void GC_push_many_regs(const word *regs, unsigned count)
  {
    unsigned i;

    for (i = 0; i < count; i++)
      GC_PUSH_ONE_STACK((ptr_t)regs[i], MARKED_FROM_REGISTER);
  }
#endif

GC_API struct GC_ms_entry * GC_CALL GC_mark_and_push(void *obj,
                        mse *mark_stack_top, mse *mark_stack_limit, void **src)
{
    hdr * hhdr;

    PREFETCH(obj);
    GET_HDR(obj, hhdr);
    if ((EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr), FALSE)
         && (!GC_all_interior_pointers
             || NULL == (hhdr = GC_find_header(GC_base(obj)))))
        || EXPECT(HBLK_IS_FREE(hhdr), FALSE)) {
      GC_ADD_TO_BLACK_LIST_NORMAL((ptr_t)obj, (ptr_t)src);
      return mark_stack_top;
    }
    return GC_push_contents_hdr((ptr_t)obj, mark_stack_top, mark_stack_limit,
                                (ptr_t)src, hhdr, TRUE);
}

/* Mark and push (i.e. gray) a single object p onto the main    */
/* mark stack.  Consider p to be valid if it is an interior     */
/* pointer.                                                     */
/* The object p has passed a preliminary pointer validity       */
/* test, but we do not definitely know whether it is valid.     */
/* Mark bits are NOT atomically updated.  Thus this must be the */
/* only thread setting them.                                    */
GC_ATTR_NO_SANITIZE_ADDR
GC_INNER void
# if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS)
    GC_mark_and_push_stack(ptr_t p, ptr_t source)
# else
    GC_mark_and_push_stack(ptr_t p)
#   define source ((ptr_t)0)
# endif
{
    hdr * hhdr;
    ptr_t r = p;

    PREFETCH(p);
    GET_HDR(p, hhdr);
    if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr), FALSE)) {
      if (NULL == hhdr
            || (r = (ptr_t)GC_base(p)) == NULL
            || (hhdr = HDR(r)) == NULL) {
        GC_ADD_TO_BLACK_LIST_STACK(p, source);
        return;
      }
    }
    if (EXPECT(HBLK_IS_FREE(hhdr), FALSE)) {
        GC_ADD_TO_BLACK_LIST_NORMAL(p, source);
        return;
    }
#   ifdef THREADS
      /* Pointer is on the stack.  We may have dirtied the object       */
      /* it points to, but have not called GC_dirty yet.                */
      GC_dirty(p); /* entire object */
#   endif
    GC_mark_stack_top = GC_push_contents_hdr(r, GC_mark_stack_top,
                                             GC_mark_stack_limit,
                                             source, hhdr, FALSE);
    /* We silently ignore pointers to near the end of a block,  */
    /* which is very mildly suboptimal.                         */
    /* FIXME: We should probably add a header word to address   */
    /* this.                                                    */
#   undef source
}

#ifdef TRACE_BUF
# ifndef TRACE_ENTRIES
#   define TRACE_ENTRIES 1000
# endif

  struct trace_entry {
    const char *caller_fn_name;
    word gc_no;
    word bytes_allocd;
    GC_hidden_pointer arg1;
    GC_hidden_pointer arg2;
  } GC_trace_buf[TRACE_ENTRIES] = { { (const char *)NULL, 0, 0, 0, 0 } };

  void GC_add_trace_entry(const char *caller_fn_name, ptr_t arg1, ptr_t arg2)
  {
    size_t i = GC_trace_buf_pos;

    GC_trace_buf[i].caller_fn_name = caller_fn_name;
    GC_trace_buf[i].gc_no = GC_gc_no;
    GC_trace_buf[i].bytes_allocd = GC_bytes_allocd;
    GC_trace_buf[i].arg1 = GC_HIDE_POINTER(arg1);
    GC_trace_buf[i].arg2 = GC_HIDE_POINTER(arg2);
    i++;
    if (i >= TRACE_ENTRIES) i = 0;
    GC_trace_buf_pos = i;
  }

  GC_API void GC_CALL GC_print_trace_inner(GC_word gc_no)
  {
    size_t i;

    for (i = GC_trace_buf_pos;; i--) {
        struct trace_entry *p;

        if (0 == i) i = TRACE_ENTRIES;
        p = &GC_trace_buf[i - 1];
        /* Compare gc_no values (p->gc_no is less than given gc_no) */
        /* taking into account that the counter may overflow.       */
        if ((((p -> gc_no) - gc_no) & SIGNB) != 0
            || NULL == p -> caller_fn_name) {
          return;
        }
        GC_printf("Trace:%s (gc:%lu, bytes:%lu) %p, %p\n",
                  p -> caller_fn_name, (unsigned long)(p -> gc_no),
                  (unsigned long)(p -> bytes_allocd),
                  GC_REVEAL_POINTER(p -> arg1), GC_REVEAL_POINTER(p -> arg2));
        if (i == GC_trace_buf_pos + 1) break;
    }
    GC_printf("Trace incomplete\n");
  }

  GC_API void GC_CALL GC_print_trace(GC_word gc_no)
  {
    READER_LOCK();
    GC_print_trace_inner(gc_no);
    READER_UNLOCK();
  }
#endif /* TRACE_BUF */

/* A version of GC_push_all that treats all interior pointers as valid  */
/* and scans the entire region immediately, in case the contents change.*/
GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY GC_ATTR_NO_SANITIZE_THREAD
GC_API void GC_CALL GC_push_all_eager(void *bottom, void *top)
{
    REGISTER ptr_t current_p;
    REGISTER ptr_t lim;
    REGISTER ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
    REGISTER ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
#   define GC_greatest_plausible_heap_addr greatest_ha
#   define GC_least_plausible_heap_addr least_ha

    if (NULL == top) return;
    /* Check all pointers in range and push if they appear to be valid. */
    current_p = PTR_ALIGN_UP((ptr_t)bottom, ALIGNMENT);
    lim = PTR_ALIGN_DOWN((ptr_t)top, ALIGNMENT) - sizeof(ptr_t);
    for (; ADDR_GE(lim, current_p); current_p += ALIGNMENT) {
      REGISTER ptr_t q;

      LOAD_PTR_OR_CONTINUE(q, current_p);
      GC_PUSH_ONE_STACK(q, current_p);
    }
#   undef GC_greatest_plausible_heap_addr
#   undef GC_least_plausible_heap_addr
}

GC_INNER void GC_push_all_stack(ptr_t bottom, ptr_t top)
{
    GC_ASSERT(I_HOLD_LOCK());
#   ifndef NEED_FIXUP_POINTER
      if (GC_all_interior_pointers
#         if defined(THREADS) && defined(MPROTECT_VDB)
            && !GC_auto_incremental
#         endif
          && ADDR_LT((ptr_t)GC_mark_stack_top,
                (ptr_t)(GC_mark_stack_limit - INITIAL_MARK_STACK_SIZE/8))) {
        GC_push_all(bottom, top);
      } else
#   endif
    /* else */ {
      GC_push_all_eager(bottom, top);
    }
}

#if defined(WRAP_MARK_SOME) && defined(PARALLEL_MARK)
  /* Similar to GC_push_conditional but scans the whole region immediately. */
  GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY
  GC_ATTR_NO_SANITIZE_THREAD
  GC_INNER void GC_push_conditional_eager(void *bottom, void *top,
                                          GC_bool all)
  {
    REGISTER ptr_t current_p;
    REGISTER ptr_t lim;
    REGISTER ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
    REGISTER ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
#   define GC_greatest_plausible_heap_addr greatest_ha
#   define GC_least_plausible_heap_addr least_ha

    if (NULL == top) return;
    (void)all; /* TODO: If !all then scan only dirty pages. */

    current_p = PTR_ALIGN_UP((ptr_t)bottom, ALIGNMENT);
    lim = PTR_ALIGN_DOWN((ptr_t)top, ALIGNMENT) - sizeof(ptr_t);
    for (; ADDR_GE(lim, current_p); current_p += ALIGNMENT) {
      REGISTER ptr_t q;

      LOAD_PTR_OR_CONTINUE(q, current_p);
      GC_PUSH_ONE_HEAP(q, current_p, GC_mark_stack_top);
    }
#   undef GC_greatest_plausible_heap_addr
#   undef GC_least_plausible_heap_addr
  }
#endif /* WRAP_MARK_SOME && PARALLEL_MARK */

#if !defined(SMALL_CONFIG) && !defined(USE_MARK_BYTES) \
    && !defined(MARK_BIT_PER_OBJ) && GC_GRANULE_PTRS <= 4
# define USE_PUSH_MARKED_ACCELERATORS
# if GC_GRANULE_PTRS == 1
#   define PUSH_GRANULE(q) \
                do { \
                  ptr_t qcontents = (q)[0]; \
                  GC_PUSH_ONE_HEAP(qcontents, q, GC_mark_stack_top); \
                } while (0)
# elif GC_GRANULE_PTRS == 2
#   define PUSH_GRANULE(q) \
                do { \
                  ptr_t qcontents = (q)[0]; \
                  GC_PUSH_ONE_HEAP(qcontents, q, GC_mark_stack_top); \
                  qcontents = (q)[1]; \
                  GC_PUSH_ONE_HEAP(qcontents, (q)+1, GC_mark_stack_top); \
                } while (0)
# else
#   define PUSH_GRANULE(q) \
                do { \
                  ptr_t qcontents = (q)[0]; \
                  GC_PUSH_ONE_HEAP(qcontents, q, GC_mark_stack_top); \
                  qcontents = (q)[1]; \
                  GC_PUSH_ONE_HEAP(qcontents, (q)+1, GC_mark_stack_top); \
                  qcontents = (q)[2]; \
                  GC_PUSH_ONE_HEAP(qcontents, (q)+2, GC_mark_stack_top); \
                  qcontents = (q)[3]; \
                  GC_PUSH_ONE_HEAP(qcontents, (q)+3, GC_mark_stack_top); \
                } while (0)
# endif

  /* Push all objects reachable from marked objects in the given block  */
  /* containing objects of size 1 granule.                              */
  GC_ATTR_NO_SANITIZE_THREAD
  STATIC void GC_push_marked1(struct hblk *h, const hdr *hhdr)
  {
    const word *mark_word_addr
                        = (word *)CAST_AWAY_VOLATILE_PVOID(hhdr -> hb_marks);
    ptr_t *p;
    ptr_t plim;

    /* Allow registers to be used for some frequently accessed  */
    /* global variables.  Otherwise aliasing issues are likely  */
    /* to prevent that.                                         */
    ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
    ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
    mse * mark_stack_top = GC_mark_stack_top;
    mse * mark_stack_limit = GC_mark_stack_limit;

#   undef GC_mark_stack_top
#   undef GC_mark_stack_limit
#   define GC_mark_stack_top mark_stack_top
#   define GC_mark_stack_limit mark_stack_limit
#   define GC_greatest_plausible_heap_addr greatest_ha
#   define GC_least_plausible_heap_addr least_ha

    p = (ptr_t *)(h -> hb_body);
    plim = (ptr_t)h + HBLKSIZE;

    /* Go through all granules in block.    */
    while (ADDR_LT((ptr_t)p, plim)) {
      word mark_word = *mark_word_addr++;
      ptr_t *q;

      for (q = p; mark_word != 0; mark_word >>= 1) {
        if ((mark_word & 1) != 0) PUSH_GRANULE(q);
        q += GC_GRANULE_PTRS;
      }
      p += CPP_WORDSZ * GC_GRANULE_PTRS;
    }

#   undef GC_greatest_plausible_heap_addr
#   undef GC_least_plausible_heap_addr
#   undef GC_mark_stack_top
#   undef GC_mark_stack_limit
#   define GC_mark_stack_limit GC_arrays._mark_stack_limit
#   define GC_mark_stack_top GC_arrays._mark_stack_top
    GC_mark_stack_top = mark_stack_top;
  }

# ifndef UNALIGNED_PTRS
    /* Push all objects reachable from marked objects in the given  */
    /* block of size 2 (granules) objects.                          */
    GC_ATTR_NO_SANITIZE_THREAD
    STATIC void GC_push_marked2(struct hblk *h, const hdr *hhdr)
    {
      const word *mark_word_addr
                        = (word *)CAST_AWAY_VOLATILE_PVOID(hhdr -> hb_marks);
      ptr_t *p;
      ptr_t plim;
      ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
      ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
      mse * mark_stack_top = GC_mark_stack_top;
      mse * mark_stack_limit = GC_mark_stack_limit;

#     undef GC_mark_stack_top
#     undef GC_mark_stack_limit
#     define GC_mark_stack_top mark_stack_top
#     define GC_mark_stack_limit mark_stack_limit
#     define GC_greatest_plausible_heap_addr greatest_ha
#     define GC_least_plausible_heap_addr least_ha

      p = (ptr_t *)(h -> hb_body);
      plim = (ptr_t)h + HBLKSIZE;

      /* Go through all granules in block.  */
      while (ADDR_LT((ptr_t)p, plim)) {
        word mark_word = *mark_word_addr++;
        ptr_t *q;

        for (q = p; mark_word != 0; mark_word >>= 2) {
          if (mark_word & 1) {
            PUSH_GRANULE(q);
            PUSH_GRANULE(q + GC_GRANULE_PTRS);
          }
          q += 2 * GC_GRANULE_PTRS;
        }
        p += CPP_WORDSZ * GC_GRANULE_PTRS;
      }

#     undef GC_greatest_plausible_heap_addr
#     undef GC_least_plausible_heap_addr
#     undef GC_mark_stack_top
#     undef GC_mark_stack_limit
#     define GC_mark_stack_limit GC_arrays._mark_stack_limit
#     define GC_mark_stack_top GC_arrays._mark_stack_top
      GC_mark_stack_top = mark_stack_top;
    }

#   if GC_GRANULE_PTRS < 4
      /* Push all objects reachable from marked objects in the given    */
      /* block of size 4 (granules) objects.  There is a risk of mark   */
      /* stack overflow here.  But we handle that.  And only unmarked   */
      /* objects get pushed, so it's not very likely.                   */
      GC_ATTR_NO_SANITIZE_THREAD
      STATIC void GC_push_marked4(struct hblk *h, const hdr *hhdr)
      {
        const word *mark_word_addr
                        = (word *)CAST_AWAY_VOLATILE_PVOID(hhdr -> hb_marks);
        ptr_t *p;
        ptr_t plim;
        ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
        ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
        mse * mark_stack_top = GC_mark_stack_top;
        mse * mark_stack_limit = GC_mark_stack_limit;

#       undef GC_mark_stack_top
#       undef GC_mark_stack_limit
#       define GC_mark_stack_top mark_stack_top
#       define GC_mark_stack_limit mark_stack_limit
#       define GC_greatest_plausible_heap_addr greatest_ha
#       define GC_least_plausible_heap_addr least_ha

        p = (ptr_t *)(h -> hb_body);
        plim = (ptr_t)h + HBLKSIZE;

        /* Go through all granules in block.    */
        while (ADDR_LT((ptr_t)p, plim)) {
          word mark_word = *mark_word_addr++;
          ptr_t *q;

          for (q = p; mark_word != 0; mark_word >>= 4) {
            if (mark_word & 1) {
              PUSH_GRANULE(q);
              PUSH_GRANULE(q + GC_GRANULE_PTRS);
              PUSH_GRANULE(q + 2 * GC_GRANULE_PTRS);
              PUSH_GRANULE(q + 3 * GC_GRANULE_PTRS);
            }
            q += 4 * GC_GRANULE_PTRS;
          }
          p += CPP_WORDSZ * GC_GRANULE_PTRS;
        }
#       undef GC_greatest_plausible_heap_addr
#       undef GC_least_plausible_heap_addr
#       undef GC_mark_stack_top
#       undef GC_mark_stack_limit
#       define GC_mark_stack_limit GC_arrays._mark_stack_limit
#       define GC_mark_stack_top GC_arrays._mark_stack_top
        GC_mark_stack_top = mark_stack_top;
      }
#   endif /* GC_GRANULE_PTRS < 4 */
# endif /* UNALIGNED_PTRS */
#endif /* !USE_MARK_BYTES && !MARK_BIT_PER_OBJ && !SMALL_CONFIG */

/* Push all objects reachable from marked objects in the given block.   */
STATIC void GC_push_marked(struct hblk *h, const hdr *hhdr)
{
    size_t sz = hhdr -> hb_sz;
    ptr_t p;
    size_t bit_no;
    ptr_t plim;
    mse * mark_stack_top;
    mse * mark_stack_limit = GC_mark_stack_limit;

    /* Some quick shortcuts: */
    if ((/* 0 | */ GC_DS_LENGTH) == hhdr -> hb_descr) return;
    if (GC_block_empty(hhdr)/* nothing marked */) return;

#   if !defined(GC_DISABLE_INCREMENTAL)
      GC_n_rescuing_pages++;
#   endif
    GC_objects_are_marked = TRUE;
    switch (BYTES_TO_GRANULES(sz)) {
#   ifdef USE_PUSH_MARKED_ACCELERATORS
      case 1:
        GC_push_marked1(h, hhdr);
        break;
#     ifndef UNALIGNED_PTRS
        case 2:
          GC_push_marked2(h, hhdr);
          break;
#       if GC_GRANULE_PTRS < 4
          case 4:
            GC_push_marked4(h, hhdr);
            break;
#       endif
#     endif /* !UNALIGNED_PTRS */
#   else
      case 1: /* to suppress "switch statement contains no case" warning */
#   endif
    default:
      plim = sz > MAXOBJBYTES ? h -> hb_body
                : CAST_THRU_UINTPTR(ptr_t, (h + 1) -> hb_body) - sz;
      mark_stack_top = GC_mark_stack_top;
      for (p = h -> hb_body, bit_no = 0; ADDR_GE(plim, p);
           p += sz, bit_no += MARK_BIT_OFFSET(sz)) {
        /* Mark from fields inside the object.  */
        if (mark_bit_from_hdr(hhdr, bit_no)) {
          mark_stack_top = GC_push_obj(p, hhdr, mark_stack_top,
                                       mark_stack_limit);
        }
      }
      GC_mark_stack_top = mark_stack_top;
    }
}

#ifdef ENABLE_DISCLAIM
  /* Unconditionally mark from all objects which have not been          */
  /* reclaimed.  This is useful in order to retain pointers reachable   */
  /* from the disclaim notifiers.                                       */
  /* To determine whether an object has been reclaimed, we require that */
  /* any live object has a non-zero as one of the two least significant */
  /* bits of the first "pointer-sized" word.  On the other hand, the    */
  /* reclaimed object is a member of free lists, and thus contains      */
  /* a pointer-aligned next-pointer as the first "pointer-sized" word.  */
  GC_ATTR_NO_SANITIZE_THREAD
  STATIC void GC_push_unconditionally(struct hblk *h, const hdr *hhdr)
  {
    size_t sz = hhdr -> hb_sz;
    ptr_t p;
    ptr_t plim;
    mse * mark_stack_top;
    mse * mark_stack_limit = GC_mark_stack_limit;

    if ((/* 0 | */ GC_DS_LENGTH) == hhdr -> hb_descr) return;

#   if !defined(GC_DISABLE_INCREMENTAL)
      GC_n_rescuing_pages++;
#   endif
    GC_objects_are_marked = TRUE;
    plim = sz > MAXOBJBYTES ? h -> hb_body
                : CAST_THRU_UINTPTR(ptr_t, (h + 1) -> hb_body) - sz;
    mark_stack_top = GC_mark_stack_top;
    for (p = h -> hb_body; ADDR_GE(plim, p); p += sz) {
      if ((ADDR(*(ptr_t *)p) & 0x3) != 0) {
        mark_stack_top = GC_push_obj(p, hhdr, mark_stack_top,
                                     mark_stack_limit);
      }
    }
    GC_mark_stack_top = mark_stack_top;
  }
#endif /* ENABLE_DISCLAIM */

#ifndef GC_DISABLE_INCREMENTAL
  /* Test whether any page in the given block is dirty.   */
  STATIC GC_bool GC_block_was_dirty(struct hblk *h, const hdr *hhdr)
  {
    size_t sz;
    ptr_t p;

#   ifdef AO_HAVE_load
      /* Atomic access is used to avoid racing with GC_realloc. */
      sz = AO_load(&(hhdr -> hb_sz));
#   else
      sz = hhdr -> hb_sz;
#   endif
    if (sz <= MAXOBJBYTES) {
      return GC_page_was_dirty(h);
    }

    for (p = (ptr_t)h; ADDR_LT(p, (ptr_t)h + sz); p += HBLKSIZE) {
      if (GC_page_was_dirty((struct hblk *)p)) return TRUE;
    }
    return FALSE;
  }
#endif /* GC_DISABLE_INCREMENTAL */

/* Similar to GC_push_marked, but skip over unallocated blocks  */
/* and return address of next plausible block.                  */
STATIC struct hblk * GC_push_next_marked(struct hblk *h)
{
    hdr * hhdr = HDR(h);

    if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr) || HBLK_IS_FREE(hhdr), FALSE)) {
      h = GC_next_block(h, FALSE);
      if (NULL == h) return NULL;
      hhdr = GC_find_header(h);
    } else {
#     ifdef LINT2
        if (NULL == h) ABORT("Bad HDR() definition");
#     endif
    }
    GC_push_marked(h, hhdr);
    return h + OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz);
}

#ifndef GC_DISABLE_INCREMENTAL
  /* Identical to above, but mark only from dirty pages.        */
  STATIC struct hblk * GC_push_next_marked_dirty(struct hblk *h)
  {
    hdr * hhdr;

    GC_ASSERT(I_HOLD_LOCK());
    if (!GC_incremental) ABORT("Dirty bits not set up");
    for (;; h += OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz)) {
      hhdr = HDR(h);
      if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr)
                 || HBLK_IS_FREE(hhdr), FALSE)) {
        h = GC_next_block(h, FALSE);
        if (NULL == h) return NULL;
        hhdr = GC_find_header(h);
      } else {
#       ifdef LINT2
          if (NULL == h) ABORT("Bad HDR() definition");
#       endif
      }
      if (GC_block_was_dirty(h, hhdr))
        break;
    }
#   ifdef ENABLE_DISCLAIM
      if ((hhdr -> hb_flags & MARK_UNCONDITIONALLY) != 0) {
        GC_push_unconditionally(h, hhdr);

        /* Then we may ask, why not also add the MARK_UNCONDITIONALLY   */
        /* case to GC_push_next_marked, which is also applied to        */
        /* uncollectible blocks?  But it seems to me that the function  */
        /* does not need to scan uncollectible (and unconditionally     */
        /* marked) blocks since those are already handled in the        */
        /* MS_PUSH_UNCOLLECTABLE phase.                                 */
      } else
#   endif
    /* else */ {
      GC_push_marked(h, hhdr);
    }
    return h + OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz);
  }
#endif /* !GC_DISABLE_INCREMENTAL */

/* Similar to above, but for uncollectible pages.  Needed since we      */
/* do not clear marks for such pages, even for full collections.        */
STATIC struct hblk * GC_push_next_marked_uncollectable(struct hblk *h)
{
    hdr * hhdr = HDR(h);

    for (;;) {
        if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr)
                   || HBLK_IS_FREE(hhdr), FALSE)) {
          h = GC_next_block(h, FALSE);
          if (NULL == h) return NULL;
          hhdr = GC_find_header(h);
        } else {
#         ifdef LINT2
            if (NULL == h) ABORT("Bad HDR() definition");
#         endif
        }
        if (hhdr -> hb_obj_kind == UNCOLLECTABLE) {
            GC_push_marked(h, hhdr);
            break;
        }
#       ifdef ENABLE_DISCLAIM
            if ((hhdr -> hb_flags & MARK_UNCONDITIONALLY) != 0) {
                GC_push_unconditionally(h, hhdr);
                break;
            }
#       endif
        h += OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz);
        hhdr = HDR(h);
    }
    return h + OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz);
}

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 2009-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#if defined(E2K) && !defined(THREADS)
# include <alloca.h>
#endif

/* Data structure for list of root sets.                                */
/* We keep a hash table, so that we can filter out duplicate additions. */
/* Under Win32, we need to do a better job of filtering overlaps, so    */
/* we resort to sequential search, and pay the price.                   */
/* This is really declared in gc_priv.h:
struct roots {
        ptr_t r_start;
        ptr_t r_end;
#       ifndef ANY_MSWIN
          struct roots * r_next;
#       endif
        GC_bool r_tmp;
                -- Delete before registering new dynamic libraries
};

struct roots GC_static_roots[MAX_ROOT_SETS];
*/

int GC_no_dls = 0;      /* Register dynamic library data segments.      */

#if !defined(NO_DEBUGGING) || defined(GC_ASSERTIONS)
  /* Should return the same value as GC_root_size.      */
  GC_INNER word GC_compute_root_size(void)
  {
    size_t i;
    word size = 0;

    for (i = 0; i < n_root_sets; i++) {
      size += (word)(GC_static_roots[i].r_end - GC_static_roots[i].r_start);
    }
    return size;
  }
#endif /* !NO_DEBUGGING || GC_ASSERTIONS */

#if !defined(NO_DEBUGGING)
  /* For debugging:     */
  void GC_print_static_roots(void)
  {
    size_t i;
    word size;

    for (i = 0; i < n_root_sets; i++) {
        GC_printf("From %p to %p%s\n",
                  (void *)GC_static_roots[i].r_start,
                  (void *)GC_static_roots[i].r_end,
                  GC_static_roots[i].r_tmp ? " (temporary)" : "");
    }
    GC_printf("GC_root_size= %lu\n", (unsigned long)GC_root_size);

    if ((size = GC_compute_root_size()) != GC_root_size)
      GC_err_printf("GC_root_size incorrect!! Should be: %lu\n",
                    (unsigned long)size);
  }
#endif /* !NO_DEBUGGING */

#ifndef THREADS
  /* Primarily for debugging support:     */
  /* Is the address p in one of the registered static root sections?      */
  GC_INNER GC_bool GC_is_static_root(ptr_t p)
  {
    static size_t last_static_root_set = MAX_ROOT_SETS;
    size_t i;

#   if defined(CPPCHECK)
      if (n_root_sets > MAX_ROOT_SETS) ABORT("Bad n_root_sets");
#   endif
    if (last_static_root_set < n_root_sets
        && ADDR_INSIDE(p, GC_static_roots[last_static_root_set].r_start,
                       GC_static_roots[last_static_root_set].r_end))
      return TRUE;
    for (i = 0; i < n_root_sets; i++) {
        if (ADDR_INSIDE(p, GC_static_roots[i].r_start,
                        GC_static_roots[i].r_end)) {
          last_static_root_set = i;
          return TRUE;
        }
    }
    return FALSE;
  }
#endif /* !THREADS */

#ifndef ANY_MSWIN
/*
#   define LOG_RT_SIZE 6
#   define RT_SIZE (1 << LOG_RT_SIZE)  -- Power of 2, may be != MAX_ROOT_SETS

    struct roots * GC_root_index[RT_SIZE];
        -- Hash table header.  Used only to check whether a range is
        -- already present.
        -- really defined in gc_priv.h
*/

  GC_INLINE size_t rt_hash(ptr_t addr)
  {
    word val = ADDR(addr);

#   if CPP_WORDSZ > 4*LOG_RT_SIZE
#     if CPP_WORDSZ > 8*LOG_RT_SIZE
        val ^= val >> (8*LOG_RT_SIZE);
#     endif
      val ^= val >> (4*LOG_RT_SIZE);
#   endif
    val ^= val >> (2*LOG_RT_SIZE);
    return (size_t)((val >> LOG_RT_SIZE) ^ val) & (RT_SIZE-1);
  }

  /* Is a range starting at b already in the table? If so, return a     */
  /* pointer to it, else NULL.                                          */
  GC_INNER void * GC_roots_present(ptr_t b)
  {
    size_t h;
    struct roots *p;

    GC_ASSERT(I_HOLD_READER_LOCK());
    h = rt_hash(b);
    for (p = GC_root_index[h]; p != NULL; p = p -> r_next) {
        if (p -> r_start == (ptr_t)b) break;
    }
    return p;
  }

  /* Add the given root structure to the index. */
  GC_INLINE void add_roots_to_index(struct roots *p)
  {
    size_t h = rt_hash(p -> r_start);

    p -> r_next = GC_root_index[h];
    GC_root_index[h] = p;
  }
#endif /* !ANY_MSWIN */

GC_INNER word GC_root_size = 0;

GC_API void GC_CALL GC_add_roots(void *b, void *e)
{
    if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
    LOCK();
    GC_add_roots_inner((ptr_t)b, (ptr_t)e, FALSE);
    UNLOCK();
}


/* Add [b,e) to the root set.  Adding the same interval a second time   */
/* is a moderately fast no-op, and hence benign.  We do not handle      */
/* different but overlapping intervals efficiently.  (We do handle      */
/* them correctly.)                                                     */
/* Tmp specifies that the interval may be deleted before                */
/* re-registering dynamic libraries.                                    */
#ifndef AMIGA
  GC_INNER
#endif
void GC_add_roots_inner(ptr_t b, ptr_t e, GC_bool tmp)
{
    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(ADDR_GE(e, b));
    b = PTR_ALIGN_UP(b, sizeof(ptr_t));
    e = PTR_ALIGN_DOWN(e, sizeof(ptr_t));
    if (ADDR_GE(b, e)) return; /* nothing to do */

#   ifdef ANY_MSWIN
      /* Spend the time to ensure that there are no overlapping */
      /* or adjacent intervals.                                 */
      /* This could be done faster with e.g. a                  */
      /* balanced tree.  But the execution time here is         */
      /* virtually guaranteed to be dominated by the time it    */
      /* takes to scan the roots.                               */
      {
        size_t i;
        struct roots * old = NULL; /* initialized to prevent warning. */

        for (i = 0; i < n_root_sets; i++) {
            old = GC_static_roots + i;
            if (ADDR_GE(old -> r_end, b) && ADDR_GE(e, old -> r_start)) {
                if (ADDR_LT(b, old -> r_start)) {
                    GC_root_size += (word)(old -> r_start - b);
                    old -> r_start = b;
                }
                if (ADDR_LT(old -> r_end, e)) {
                    GC_root_size += (word)(e - old -> r_end);
                    old -> r_end = e;
                }
                old -> r_tmp &= tmp;
                break;
            }
        }
        if (i < n_root_sets) {
          /* merge other overlapping intervals */
            struct roots *other;

            for (i++; i < n_root_sets; i++) {
              other = GC_static_roots + i;
              b = other -> r_start;
              e = other -> r_end;
              if (ADDR_GE(old -> r_end, b) && ADDR_GE(e, old -> r_start)) {
                if (ADDR_LT(b, old -> r_start)) {
                    GC_root_size += (word)(old -> r_start - b);
                    old -> r_start = b;
                }
                if (ADDR_LT(old -> r_end, e)) {
                    GC_root_size += (word)(e - old -> r_end);
                    old -> r_end = e;
                }
                old -> r_tmp &= other -> r_tmp;
                /* Delete this entry. */
                  GC_root_size -= (word)(other -> r_end - other -> r_start);
                  other -> r_start = GC_static_roots[n_root_sets-1].r_start;
                  other -> r_end = GC_static_roots[n_root_sets-1].r_end;
                  n_root_sets--;
              }
            }
          return;
        }
      }
#   else
      {
        struct roots * old = (struct roots *)GC_roots_present(b);

        if (old != NULL) {
          if (ADDR_GE(old -> r_end, e)) {
            old -> r_tmp &= tmp;
            return; /* already there */
          }
          if (old -> r_tmp == tmp || !tmp) {
            /* Extend the existing root. */
            GC_root_size += (word)(e - old -> r_end);
            old -> r_end = e;
            old -> r_tmp = tmp;
            return;
          }
          b = old -> r_end;
        }
      }
#   endif
    if (n_root_sets == MAX_ROOT_SETS) {
        ABORT("Too many root sets");
    }

#   ifdef DEBUG_ADD_DEL_ROOTS
      GC_log_printf("Adding data root section %u: %p .. %p%s\n",
                    (unsigned)n_root_sets, (void *)b, (void *)e,
                    tmp ? " (temporary)" : "");
#   endif
    GC_static_roots[n_root_sets].r_start = (ptr_t)b;
    GC_static_roots[n_root_sets].r_end = (ptr_t)e;
    GC_static_roots[n_root_sets].r_tmp = tmp;
#   ifndef ANY_MSWIN
      GC_static_roots[n_root_sets].r_next = 0;
      add_roots_to_index(GC_static_roots + n_root_sets);
#   endif
    GC_root_size += (word)(e - b);
    n_root_sets++;
}

GC_API void GC_CALL GC_clear_roots(void)
{
    if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
    LOCK();
#   ifdef THREADS
      GC_roots_were_cleared = TRUE;
#   endif
    n_root_sets = 0;
    GC_root_size = 0;
#   ifndef ANY_MSWIN
      BZERO(GC_root_index, RT_SIZE * sizeof(void *));
#   endif
#   ifdef DEBUG_ADD_DEL_ROOTS
      GC_log_printf("Clear all data root sections\n");
#   endif
    UNLOCK();
}

STATIC void GC_remove_root_at_pos(size_t i)
{
    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(i < n_root_sets);
#   ifdef DEBUG_ADD_DEL_ROOTS
      GC_log_printf("Remove data root section at %u: %p .. %p%s\n",
                    (unsigned)i, (void *)GC_static_roots[i].r_start,
                    (void *)GC_static_roots[i].r_end,
                    GC_static_roots[i].r_tmp ? " (temporary)" : "");
#   endif
    GC_root_size -= (word)(GC_static_roots[i].r_end -
                            GC_static_roots[i].r_start);
    GC_static_roots[i].r_start = GC_static_roots[n_root_sets-1].r_start;
    GC_static_roots[i].r_end = GC_static_roots[n_root_sets-1].r_end;
    GC_static_roots[i].r_tmp = GC_static_roots[n_root_sets-1].r_tmp;
    n_root_sets--;
}

#ifndef ANY_MSWIN
  STATIC void GC_rebuild_root_index(void)
  {
    size_t i;

    BZERO(GC_root_index, RT_SIZE * sizeof(void *));
    for (i = 0; i < n_root_sets; i++)
        add_roots_to_index(GC_static_roots + i);
  }
#endif /* !ANY_MSWIN */

#if defined(DYNAMIC_LOADING) || defined(ANY_MSWIN) || defined(PCR)
  STATIC void GC_remove_tmp_roots(void)
  {
    size_t i;
#   ifndef ANY_MSWIN
      size_t old_n_roots = n_root_sets;
#   endif

    GC_ASSERT(I_HOLD_LOCK());
    for (i = 0; i < n_root_sets; ) {
        if (GC_static_roots[i].r_tmp) {
            GC_remove_root_at_pos(i);
        } else {
            i++;
        }
    }
#   ifndef ANY_MSWIN
      if (n_root_sets < old_n_roots)
        GC_rebuild_root_index();
#   endif
  }
#endif /* DYNAMIC_LOADING || ANY_MSWIN || PCR */

STATIC void GC_remove_roots_inner(ptr_t b, ptr_t e);

GC_API void GC_CALL GC_remove_roots(void *b, void *e)
{
    /* A quick check whether has nothing to do. */
    if (ADDR_GE(PTR_ALIGN_UP((ptr_t)b, sizeof(ptr_t)),
                PTR_ALIGN_DOWN((ptr_t)e, sizeof(ptr_t))))
      return;

    LOCK();
    GC_remove_roots_inner((ptr_t)b, (ptr_t)e);
    UNLOCK();
}

STATIC void GC_remove_roots_inner(ptr_t b, ptr_t e)
{
    size_t i;
#   ifndef ANY_MSWIN
      size_t old_n_roots = n_root_sets;
#   endif

    GC_ASSERT(I_HOLD_LOCK());
    for (i = 0; i < n_root_sets; ) {
        if (ADDR_GE(GC_static_roots[i].r_start, b)
            && ADDR_GE(e, GC_static_roots[i].r_end)) {
          GC_remove_root_at_pos(i);
        } else {
          i++;
        }
    }
#   ifndef ANY_MSWIN
      if (n_root_sets < old_n_roots)
        GC_rebuild_root_index();
#   endif
}

#ifdef USE_PROC_FOR_LIBRARIES
  /* Exchange the elements of the roots table.  Requires rebuild of     */
  /* the roots index table after the swap.                              */
  GC_INLINE void swap_static_roots(size_t i, size_t j)
  {
    ptr_t r_start = GC_static_roots[i].r_start;
    ptr_t r_end = GC_static_roots[i].r_end;
    GC_bool r_tmp = GC_static_roots[i].r_tmp;

    GC_static_roots[i].r_start = GC_static_roots[j].r_start;
    GC_static_roots[i].r_end = GC_static_roots[j].r_end;
    GC_static_roots[i].r_tmp = GC_static_roots[j].r_tmp;
    /* No need to swap r_next values.   */
    GC_static_roots[j].r_start = r_start;
    GC_static_roots[j].r_end = r_end;
    GC_static_roots[j].r_tmp = r_tmp;
  }

  /* Remove given range from every static root which intersects with    */
  /* the range.  It is assumed GC_remove_tmp_roots is called before     */
  /* this function is called repeatedly by GC_register_map_entries.     */
  GC_INNER void GC_remove_roots_subregion(ptr_t b, ptr_t e)
  {
    size_t i;
    GC_bool rebuild = FALSE;

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(ADDR(b) % sizeof(ptr_t) == 0 && ADDR(e) % sizeof(ptr_t) == 0);
    for (i = 0; i < n_root_sets; i++) {
      ptr_t r_start, r_end;

      if (GC_static_roots[i].r_tmp) {
        /* The remaining roots are skipped as they are all temporary. */
#       ifdef GC_ASSERTIONS
          size_t j;

          for (j = i + 1; j < n_root_sets; j++) {
            GC_ASSERT(GC_static_roots[j].r_tmp);
          }
#       endif
        break;
      }
      r_start = GC_static_roots[i].r_start;
      r_end = GC_static_roots[i].r_end;
      if (!EXPECT(ADDR_GE(r_start, e) || ADDR_GE(b, r_end), TRUE)) {
#       ifdef DEBUG_ADD_DEL_ROOTS
          GC_log_printf("Removing %p .. %p from root section %u (%p .. %p)\n",
                        (void *)b, (void *)e,
                        (unsigned)i, (void *)r_start, (void *)r_end);
#       endif
        if (ADDR_LT(r_start, b)) {
          GC_root_size -= (word)(r_end - b);
          GC_static_roots[i].r_end = b;
          /* No need to rebuild as hash does not use r_end value. */
          if (ADDR_LT(e, r_end)) {
            size_t j;

            if (rebuild) {
              GC_rebuild_root_index();
              rebuild = FALSE;
            }
            GC_add_roots_inner(e, r_end, FALSE); /* updates n_root_sets */
            for (j = i + 1; j < n_root_sets; j++)
              if (GC_static_roots[j].r_tmp)
                break;
            if (j < n_root_sets-1 && !GC_static_roots[n_root_sets-1].r_tmp) {
              /* Exchange the roots to have all temporary ones at the end. */
              swap_static_roots(j, n_root_sets - 1);
              rebuild = TRUE;
            }
          }
        } else {
          if (ADDR_LT(e, r_end)) {
            GC_root_size -= (word)(e - r_start);
            GC_static_roots[i].r_start = e;
          } else {
            GC_remove_root_at_pos(i);
            if (i + 1 < n_root_sets && GC_static_roots[i].r_tmp
                && !GC_static_roots[i + 1].r_tmp) {
              size_t j;

              for (j = i + 2; j < n_root_sets; j++)
                if (GC_static_roots[j].r_tmp)
                  break;
              /* Exchange the roots to have all temporary ones at the end. */
              swap_static_roots(i, j - 1);
            }
            i--;
          }
          rebuild = TRUE;
        }
      }
    }
    if (rebuild)
      GC_rebuild_root_index();
  }
#endif /* USE_PROC_FOR_LIBRARIES */

#if !defined(NO_DEBUGGING)
  /* For the debugging purpose only.                                    */
  /* Workaround for the OS mapping and unmapping behind our back:       */
  /* Is the address p in one of the temporary static root sections?     */
  GC_API int GC_CALL GC_is_tmp_root(void *p)
  {
#   ifndef HAS_REAL_READER_LOCK
      static size_t last_root_set; /* initialized to 0; no shared access */
#   elif defined(AO_HAVE_load) || defined(AO_HAVE_store)
      static volatile AO_t last_root_set;
#   else
      static volatile size_t last_root_set;
                        /* A race is acceptable, it's just a cached index. */
#   endif
    size_t i;
    int res;

    READER_LOCK();
    /* First try the cached root. */
#   if defined(AO_HAVE_load) && defined(HAS_REAL_READER_LOCK)
      i = AO_load(&last_root_set);
#   else
      i = last_root_set;
#   endif
    if (i < n_root_sets
        && ADDR_INSIDE((ptr_t)p, GC_static_roots[i].r_start,
                       GC_static_roots[i].r_end)) {
      res = (int)GC_static_roots[i].r_tmp;
    } else {
      res = 0;
      for (i = 0; i < n_root_sets; i++) {
        if (ADDR_INSIDE((ptr_t)p, GC_static_roots[i].r_start,
                        GC_static_roots[i].r_end)) {
          res = (int)GC_static_roots[i].r_tmp;
#         if defined(AO_HAVE_store) && defined(HAS_REAL_READER_LOCK)
            AO_store(&last_root_set, i);
#         else
            last_root_set = i;
#         endif
          break;
        }
      }
    }
    READER_UNLOCK();
    return res;
  }
#endif /* !NO_DEBUGGING */

GC_INNER ptr_t GC_approx_sp(void)
{
    volatile ptr_t sp;
#   if ((defined(E2K) && defined(__clang__)) \
        || (defined(S390) && (__clang_major__ < 8))) && !defined(CPPCHECK)
        /* Workaround some bugs in clang:                                   */
        /* "undefined reference to llvm.frameaddress" error (clang-9/e2k);  */
        /* a crash in SystemZTargetLowering of libLLVM-3.8 (S390).          */
        sp = (ptr_t)(&sp);
#   elif defined(CPPCHECK) || (__GNUC__ >= 4 /* GC_GNUC_PREREQ(4, 0) */ \
                               && !defined(STACK_NOT_SCANNED))
        /* TODO: Use GC_GNUC_PREREQ after fixing a bug in cppcheck. */
        sp = (ptr_t)__builtin_frame_address(0);
#   else
        sp = (ptr_t)(&sp);
#   endif
                /* Also force stack to grow if necessary. Otherwise the */
                /* later accesses might cause the kernel to think we're */
                /* doing something wrong.                               */
    return (/* no volatile */ ptr_t)sp;
}

/*
 * Data structure for excluded static roots.
 * Real declaration is in gc_priv.h.

struct exclusion {
    ptr_t e_start;
    ptr_t e_end;
};

struct exclusion GC_excl_table[MAX_EXCLUSIONS];
                                        -- Array of exclusions, ascending
                                        -- address order.
*/

/* Clear the number of entries in the exclusion table.  The caller  */
/* should acquire the allocator lock (to avoid data race) but no    */
/* assertion about it by design.                                    */
GC_API void GC_CALL GC_clear_exclusion_table(void)
{
#   ifdef DEBUG_ADD_DEL_ROOTS
      GC_log_printf("Clear static root exclusions (%u elements)\n",
                    (unsigned)GC_excl_table_entries);
#   endif
    GC_excl_table_entries = 0;
}

/* Return the first exclusion range that includes an address not    */
/* lower than start_addr.                                           */
STATIC struct exclusion * GC_next_exclusion(ptr_t start_addr)
{
    size_t low = 0;
    size_t high;

    if (EXPECT(0 == GC_excl_table_entries, FALSE)) return NULL;
    high = GC_excl_table_entries - 1;
    while (high > low) {
        size_t mid = (low + high) >> 1;

        /* low <= mid < high    */
        if (ADDR_GE(start_addr, GC_excl_table[mid].e_end)) {
            low = mid + 1;
        } else {
            high = mid;
        }
    }
    if (ADDR_GE(start_addr, GC_excl_table[low].e_end)) return NULL;

    return GC_excl_table + low;
}

/* The range boundaries should be properly aligned and valid.   */
GC_INNER void GC_exclude_static_roots_inner(ptr_t start, ptr_t finish)
{
    struct exclusion * next;
    size_t next_index;

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(ADDR(start) % sizeof(ptr_t) == 0);
    GC_ASSERT(ADDR_LT(start, finish));

    next = GC_next_exclusion(start);
    if (next != NULL) {
      if (ADDR_LT(next -> e_start, finish)) {
        /* Incomplete error check.      */
        ABORT("Exclusion ranges overlap");
      }
      if (ADDR(next -> e_start) == ADDR(finish)) {
        /* Extend old range backwards.  */
        next -> e_start = start;
#       ifdef DEBUG_ADD_DEL_ROOTS
          GC_log_printf("Updating static root exclusion to %p .. %p\n",
                        (void *)start, (void *)(next -> e_end));
#       endif
        return;
      }
    }

    next_index = GC_excl_table_entries;
    if (next_index >= MAX_EXCLUSIONS) ABORT("Too many exclusions");
    if (next != NULL) {
      size_t i;

      next_index = (size_t)(next - GC_excl_table);
      for (i = GC_excl_table_entries; i > next_index; --i) {
        GC_excl_table[i] = GC_excl_table[i-1];
      }
    }
#   ifdef DEBUG_ADD_DEL_ROOTS
      GC_log_printf("Adding static root exclusion at %u: %p .. %p\n",
                    (unsigned)next_index, (void *)start, (void *)finish);
#   endif
    GC_excl_table[next_index].e_start = start;
    GC_excl_table[next_index].e_end = finish;
    ++GC_excl_table_entries;
}

GC_API void GC_CALL GC_exclude_static_roots(void *b, void *e)
{
    if (b == e) return;  /* nothing to exclude? */

    /* Round boundaries in direction reverse to that of GC_add_roots. */
    b = PTR_ALIGN_DOWN((ptr_t)b, sizeof(ptr_t));
    e = EXPECT(ADDR(e) > ~(word)(sizeof(ptr_t)-1), FALSE)
            ? PTR_ALIGN_DOWN((ptr_t)e, sizeof(ptr_t)) /* overflow */
            : PTR_ALIGN_UP((ptr_t)e, sizeof(ptr_t));

    LOCK();
    GC_exclude_static_roots_inner((ptr_t)b, (ptr_t)e);
    UNLOCK();
}

#if defined(WRAP_MARK_SOME) && defined(PARALLEL_MARK)
# define GC_PUSH_CONDITIONAL(b, t, all) \
                (GC_parallel \
                    ? GC_push_conditional_eager(b, t, all) \
                    : GC_push_conditional_static(b, t, all))
#else
# define GC_PUSH_CONDITIONAL(b, t, all) GC_push_conditional_static(b, t, all)
#endif

/* Invoke push_conditional on ranges that are not excluded. */
STATIC void GC_push_conditional_with_exclusions(ptr_t bottom, ptr_t top,
                                                GC_bool all)
{
    while (ADDR_LT(bottom, top)) {
        struct exclusion *next = GC_next_exclusion(bottom);
        ptr_t excl_start = top;

        if (next != NULL) {
          if (ADDR_GE(next -> e_start, top)) {
            next = NULL;
          } else {
            excl_start = next -> e_start;
          }
        }
        if (ADDR_LT(bottom, excl_start))
          GC_PUSH_CONDITIONAL(bottom, excl_start, all);
        if (NULL == next) break;
        bottom = next -> e_end;
    }
}

#ifdef IA64
  /* Similar to GC_push_all_stack_sections() but for IA-64 registers store. */
  GC_INNER void GC_push_all_register_sections(ptr_t bs_lo, ptr_t bs_hi,
                        GC_bool eager,
                        struct GC_traced_stack_sect_s *traced_stack_sect)
  {
    GC_ASSERT(I_HOLD_LOCK());
    while (traced_stack_sect != NULL) {
        ptr_t frame_bs_lo = traced_stack_sect -> backing_store_end;

        GC_ASSERT(ADDR_GE(bs_hi, frame_bs_lo));
        if (eager) {
            GC_push_all_eager(frame_bs_lo, bs_hi);
        } else {
            GC_push_all_stack(frame_bs_lo, bs_hi);
        }
        bs_hi = traced_stack_sect -> saved_backing_store_ptr;
        traced_stack_sect = traced_stack_sect -> prev;
    }
    GC_ASSERT(ADDR_GE(bs_hi, bs_lo));
    if (eager) {
        GC_push_all_eager(bs_lo, bs_hi);
    } else {
        GC_push_all_stack(bs_lo, bs_hi);
    }
  }
#endif /* IA64 */

#ifdef THREADS

  GC_INNER void GC_push_all_stack_sections(
                        ptr_t lo /* top */, ptr_t hi /* bottom */,
                        struct GC_traced_stack_sect_s *traced_stack_sect)
  {
    GC_ASSERT(I_HOLD_LOCK());
    while (traced_stack_sect != NULL) {
        GC_ASSERT(HOTTER_THAN(lo, (ptr_t)traced_stack_sect));
#       ifdef STACK_GROWS_UP
            GC_push_all_stack((ptr_t)traced_stack_sect, lo);
#       else
            GC_push_all_stack(lo, (ptr_t)traced_stack_sect);
#       endif
        lo = traced_stack_sect -> saved_stack_ptr;
        GC_ASSERT(lo != NULL);
        traced_stack_sect = traced_stack_sect -> prev;
    }
    GC_ASSERT(!HOTTER_THAN(hi, lo));
#   ifdef STACK_GROWS_UP
        /* We got them backwards! */
        GC_push_all_stack(hi, lo);
#   else
        GC_push_all_stack(lo, hi);
#   endif
  }

#else /* !THREADS */

                        /* Similar to GC_push_all_eager, but only the   */
                        /* part hotter than cold_gc_frame is scanned    */
                        /* immediately.  Needed to ensure that callee-  */
                        /* save registers are not missed.               */

  /* A version of GC_push_all that treats all interior pointers as  */
  /* valid and scans part of the area immediately, to make sure     */
  /* that saved register values are not lost.  Cold_gc_frame        */
  /* delimits the stack section that must be scanned eagerly.       */
  /* A zero value indicates that no eager scanning is needed.       */
  /* We do not need to worry about the manual VDB case here, since  */
  /* this is only called in the single-threaded case.  We assume    */
  /* that we cannot collect between an assignment and the           */
  /* corresponding GC_dirty() call.                                 */
  STATIC void GC_push_all_stack_partially_eager(ptr_t bottom, ptr_t top,
                                                ptr_t cold_gc_frame)
  {
#   ifndef NEED_FIXUP_POINTER
      if (GC_all_interior_pointers) {
        /* Push the hot end of the stack eagerly, so that register values   */
        /* saved inside GC frames are marked before they disappear.         */
        /* The rest of the marking can be deferred until later.             */
        if (0 == cold_gc_frame) {
          GC_push_all_stack(bottom, top);
          return;
        }
        GC_ASSERT(ADDR_GE(cold_gc_frame, bottom)
                  && ADDR_GE(top, cold_gc_frame));
#       ifdef STACK_GROWS_UP
          GC_push_all(bottom, cold_gc_frame + sizeof(ptr_t));
          GC_push_all_eager(cold_gc_frame, top);
#       else
          GC_push_all(cold_gc_frame - sizeof(ptr_t), top);
          GC_push_all_eager(bottom, cold_gc_frame);
#       endif
      } else
#   endif
    /* else */ {
      GC_push_all_eager(bottom, top);
    }
#   ifdef TRACE_BUF
      GC_add_trace_entry("GC_push_all_stack", bottom, top);
#   endif
  }

  /* Similar to GC_push_all_stack_sections() but also uses cold_gc_frame. */
  STATIC void GC_push_all_stack_part_eager_sections(
        ptr_t lo /* top */, ptr_t hi /* bottom */, ptr_t cold_gc_frame,
        struct GC_traced_stack_sect_s *traced_stack_sect)
  {
    GC_ASSERT(traced_stack_sect == NULL || cold_gc_frame == NULL
              || HOTTER_THAN(cold_gc_frame, (ptr_t)traced_stack_sect));

    while (traced_stack_sect != NULL) {
        GC_ASSERT(HOTTER_THAN(lo, (ptr_t)traced_stack_sect));
#       ifdef STACK_GROWS_UP
            GC_push_all_stack_partially_eager((ptr_t)traced_stack_sect, lo,
                                              cold_gc_frame);
#       else
            GC_push_all_stack_partially_eager(lo, (ptr_t)traced_stack_sect,
                                              cold_gc_frame);
#       endif
        lo = traced_stack_sect -> saved_stack_ptr;
        GC_ASSERT(lo != NULL);
        traced_stack_sect = traced_stack_sect -> prev;
        cold_gc_frame = NULL; /* Use at most once.      */
    }

    GC_ASSERT(!HOTTER_THAN(hi, lo));
#   ifdef STACK_GROWS_UP
        /* We got them backwards! */
        GC_push_all_stack_partially_eager(hi, lo, cold_gc_frame);
#   else
        GC_push_all_stack_partially_eager(lo, hi, cold_gc_frame);
#   endif
  }

#endif /* !THREADS */

/* Push enough of the current stack eagerly to ensure that callee-save  */
/* registers saved in GC frames are scanned.  In the non-threads case,  */
/* schedule entire stack for scanning.  The 2nd argument is a pointer   */
/* to the (possibly null) thread context, for (currently hypothetical)  */
/* more precise stack scanning.  In the presence of threads, push       */
/* enough of the current stack to ensure that callee-save registers     */
/* saved in collector frames have been seen.                            */
/* TODO: Merge it with per-thread stuff. */
STATIC void GC_push_current_stack(ptr_t cold_gc_frame, void *context)
{
    UNUSED_ARG(context);
    GC_ASSERT(I_HOLD_LOCK());
#   if defined(THREADS)
        /* cold_gc_frame is non-NULL.   */
#       ifdef STACK_GROWS_UP
          GC_push_all_eager(cold_gc_frame, GC_approx_sp());
#       else
          GC_push_all_eager(GC_approx_sp(), cold_gc_frame);
          /* For IA64, the register stack backing store is handled      */
          /* in the thread-specific code.                               */
#       endif
#   else
        GC_push_all_stack_part_eager_sections(GC_approx_sp(), GC_stackbottom,
                                        cold_gc_frame, GC_traced_stack_sect);
#       ifdef IA64
            /* We also need to push the register stack backing store.   */
            /* This should really be done in the same way as the        */
            /* regular stack.  For now we fudge it a bit.               */
            /* Note that the backing store grows up, so we can't use    */
            /* GC_push_all_stack_partially_eager.                       */
            {
                ptr_t bsp = GC_save_regs_ret_val;
                ptr_t cold_gc_bs_pointer = bsp - 2048;
                if (GC_all_interior_pointers
                    && ADDR_LT(GC_register_stackbottom, cold_gc_bs_pointer)) {
                  /* Adjust cold_gc_bs_pointer if below our innermost   */
                  /* "traced stack section" in backing store.           */
                  if (GC_traced_stack_sect != NULL
                      && ADDR_LT(cold_gc_bs_pointer,
                                 GC_traced_stack_sect -> backing_store_end)) {
                    cold_gc_bs_pointer =
                                GC_traced_stack_sect -> backing_store_end;
                  }
                  GC_push_all_register_sections(GC_register_stackbottom,
                        cold_gc_bs_pointer, FALSE, GC_traced_stack_sect);
                  GC_push_all_eager(cold_gc_bs_pointer, bsp);
                } else {
                  GC_push_all_register_sections(GC_register_stackbottom, bsp,
                                TRUE /* eager */, GC_traced_stack_sect);
                }
                /* All values should be sufficiently aligned that we    */
                /* don't have to worry about the boundary.              */
            }
#       elif defined(E2K)
          /* We also need to push procedure stack store.        */
          /* Procedure stack grows up.                          */
          {
            ptr_t bs_lo;
            size_t stack_size;

            /* TODO: support ps_ofs here and in GC_do_blocking_inner */
            GET_PROCEDURE_STACK_LOCAL(0, &bs_lo, &stack_size);
            GC_push_all_eager(bs_lo, bs_lo + stack_size);
          }
#       endif
#   endif /* !THREADS */
}

GC_INNER void (*GC_push_typed_structures)(void) = 0;

GC_INNER void GC_cond_register_dynamic_libraries(void)
{
  GC_ASSERT(I_HOLD_LOCK());
# if defined(DYNAMIC_LOADING) && !defined(MSWIN_XBOX1) \
     || defined(ANY_MSWIN) || defined(PCR)
    GC_remove_tmp_roots();
    if (!GC_no_dls) GC_register_dynamic_libraries();
# else
    GC_no_dls = TRUE;
# endif
}

STATIC void GC_push_regs_and_stack(ptr_t cold_gc_frame)
{
    GC_ASSERT(I_HOLD_LOCK());
#   ifdef THREADS
      if (NULL == cold_gc_frame)
        return; /* GC_push_all_stacks should push registers and stack */
#   endif
    GC_with_callee_saves_pushed(GC_push_current_stack, cold_gc_frame);
}

/* Call the mark routines (GC_push_one for a single pointer,            */
/* GC_push_conditional on groups of pointers) on every top level        */
/* accessible pointer.  If all is false, arrange to push only possibly  */
/* altered values.  Cold_gc_frame is an address inside a GC frame that  */
/* remains valid until all marking is complete; a NULL value indicates  */
/* that it is OK to miss some register values.                          */
GC_INNER void GC_push_roots(GC_bool all, ptr_t cold_gc_frame)
{
    size_t i;
    unsigned kind;

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(GC_is_initialized); /* needed for GC_push_all_stacks */

    /* Next push static data.  This must happen early on, since it is   */
    /* not robust against mark stack overflow.                          */
    /* Re-register dynamic libraries, in case one got added.            */
    /* There is some argument for doing this as late as possible,       */
    /* especially on Win32, where it can change asynchronously.         */
    /* In those cases, we do it here.  But on other platforms, it's     */
    /* not safe with the world stopped, so we do it earlier.            */
#   if !defined(REGISTER_LIBRARIES_EARLY)
        GC_cond_register_dynamic_libraries();
#   endif

    /* Mark everything in static data areas.                            */
    for (i = 0; i < n_root_sets; i++) {
        GC_push_conditional_with_exclusions(
                             GC_static_roots[i].r_start,
                             GC_static_roots[i].r_end, all);
    }

    /* Mark all free-list header blocks, if those were allocated from   */
    /* the garbage collected heap.  This makes sure they don't          */
    /* disappear if we are not marking from static data.  It also       */
    /* saves us the trouble of scanning them, and possibly that of      */
    /* marking the freelists.                                           */
    for (kind = 0; kind < GC_n_kinds; kind++) {
        const void *base = GC_base(GC_obj_kinds[kind].ok_freelist);

        if (base != NULL) {
            GC_set_mark_bit(base);
        }
    }

    /* Mark from GC internal roots if those might otherwise have        */
    /* been excluded.                                                   */
#   ifndef GC_NO_FINALIZATION
        GC_push_finalizer_structures();
#   endif
#   ifdef THREADS
        if (GC_no_dls || GC_roots_were_cleared)
            GC_push_thread_structures();
#   endif
    if (GC_push_typed_structures)
        GC_push_typed_structures();

    /* Mark thread-local free lists, even if their mark        */
    /* descriptor excludes the link field.                     */
    /* If the world is not stopped, this is unsafe.  It is     */
    /* also unnecessary, since we will do this again with the  */
    /* world stopped.                                          */
#   if defined(THREAD_LOCAL_ALLOC)
        if (GC_world_stopped)
            GC_mark_thread_local_free_lists();
#   endif

    /* Now traverse stacks, and mark from register contents.    */
    /* These must be done last, since they can legitimately     */
    /* overflow the mark stack.  This is usually done by saving */
    /* the current context on the stack, and then just tracing  */
    /* from the stack.                                          */
#   ifdef STACK_NOT_SCANNED
        UNUSED_ARG(cold_gc_frame);
#   else
        GC_push_regs_and_stack(cold_gc_frame);
#   endif

    if (GC_push_other_roots != 0) {
        /* In the threads case, this also pushes thread stacks. */
        /* Note that without interior pointer recognition lots  */
        /* of stuff may have been pushed already, and this      */
        /* should be careful about mark stack overflows.        */
        (*GC_push_other_roots)();
    }
}

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1996 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
 * Copyright (c) 2009-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#ifdef ENABLE_DISCLAIM


#endif

GC_INNER signed_word GC_bytes_found = 0;
                        /* Number of bytes of memory reclaimed     */
                        /* minus the number of bytes originally    */
                        /* on free lists which we had to drop.     */

#if defined(PARALLEL_MARK)
  GC_INNER signed_word GC_fl_builder_count = 0;
        /* Number of threads currently building free lists without      */
        /* holding the allocator lock.  It is not safe to collect if    */
        /* this is nonzero.  Also, together with the mark lock, it is   */
        /* used as a semaphore during marker threads startup.           */
#endif /* PARALLEL_MARK */

/* We defer printing of leaked objects until we're done with the GC     */
/* cycle, since the routine for printing objects needs to run outside   */
/* the collector, e.g. without the allocator lock.                      */
#ifndef MAX_LEAKED
# define MAX_LEAKED 40
#endif
STATIC ptr_t GC_leaked[MAX_LEAKED] = { NULL };
STATIC unsigned GC_n_leaked = 0;

#ifdef AO_HAVE_store
  GC_INNER volatile AO_t GC_have_errors = 0;
#else
  GC_INNER GC_bool GC_have_errors = FALSE;
#endif

#if !defined(EAGER_SWEEP) && defined(ENABLE_DISCLAIM)
  STATIC void GC_reclaim_unconditionally_marked(void);
#endif

GC_INLINE void GC_add_leaked(ptr_t leaked)
{
    GC_ASSERT(I_HOLD_LOCK());
#   ifndef SHORT_DBG_HDRS
      if (GC_findleak_delay_free && !GC_check_leaked(leaked))
        return;
#   endif

    GC_SET_HAVE_ERRORS();
    if (GC_n_leaked < MAX_LEAKED) {
      GC_leaked[GC_n_leaked++] = leaked;
      /* Make sure it's not reclaimed this cycle */
      GC_set_mark_bit(leaked);
    }
}

/* Print all objects on the list after printing any smashed objects.    */
/* Clear both lists.  Called without the allocator lock held.           */
GC_INNER void GC_print_all_errors(void)
{
    static GC_bool printing_errors = FALSE;
    GC_bool have_errors;
    unsigned i, n_leaked;
    ptr_t leaked[MAX_LEAKED];

    LOCK();
    if (printing_errors) {
        UNLOCK();
        return;
    }
    have_errors = get_have_errors();
    printing_errors = TRUE;
    n_leaked = GC_n_leaked;
    if (n_leaked > 0) {
      GC_ASSERT(n_leaked <= MAX_LEAKED);
      BCOPY(GC_leaked, leaked, n_leaked * sizeof(ptr_t));
      GC_n_leaked = 0;
      BZERO(GC_leaked, n_leaked * sizeof(ptr_t));
    }
    UNLOCK();

    if (GC_debugging_started) {
      GC_print_all_smashed();
    } else {
      have_errors = FALSE;
    }

    if (n_leaked > 0) {
        GC_err_printf("Found %u leaked objects:\n", n_leaked);
        have_errors = TRUE;
    }
    for (i = 0; i < n_leaked; i++) {
        ptr_t p = leaked[i];
#       ifndef SKIP_LEAKED_OBJECTS_PRINTING
          GC_print_heap_obj(p);
#       endif
        GC_free(p);
    }

    if (have_errors
#       ifndef GC_ABORT_ON_LEAK
          && GETENV("GC_ABORT_ON_LEAK") != NULL
#       endif
        ) {
      ABORT("Leaked or smashed objects encountered");
    }

    LOCK();
    printing_errors = FALSE;
    UNLOCK();
}


/*
 * reclaim phase
 *
 */

/* Test whether a block is completely empty, i.e. contains no marked    */
/* objects.  This does not require the block to be in physical memory.  */
GC_INNER GC_bool GC_block_empty(const hdr *hhdr)
{
    return 0 == hhdr -> hb_n_marks;
}

STATIC GC_bool GC_block_nearly_full(const hdr *hhdr, size_t sz)
{
    return hhdr -> hb_n_marks > HBLK_OBJS(sz) * 7 / 8;
}

/* TODO: This should perhaps again be specialized for USE_MARK_BYTES    */
/* and USE_MARK_BITS cases.                                             */

GC_INLINE ptr_t GC_clear_block(ptr_t q, size_t sz, word *pcount)
{
  ptr_t *p = (ptr_t *)q;
  ptr_t plim = q + sz;

  /* Clear object, advance p to next object in the process.     */
# ifdef USE_MARK_BYTES
    GC_ASSERT((sz & 1) == 0);
    GC_ASSERT((ADDR(p) & (2*sizeof(ptr_t)-1)) == 0);
    p[1] = NULL; /* but do not clear link field */
    for (p += 2; ADDR_LT((ptr_t)p, plim); p += 2) {
      CLEAR_DOUBLE(p);
    }
# else
    p++; /* skip link field */
    while (ADDR_LT((ptr_t)p, plim)) {
      *p++ = NULL;
    }
# endif
  *pcount += sz;
  return (ptr_t)p;
}

/*
 * Restore unmarked small objects in h of size sz to the object
 * free list.  Returns the new list.
 * Clears unmarked objects.  Sz is in bytes.
 */
STATIC ptr_t GC_reclaim_clear(struct hblk *hbp, const hdr *hhdr, size_t sz,
                              ptr_t list, word *pcount)
{
    size_t bit_no;
    ptr_t p, plim;

    GC_ASSERT(hhdr == GC_find_header(hbp));
#   ifndef THREADS
      GC_ASSERT(sz == hhdr -> hb_sz);
#   else
      /* Skip the assertion because of a potential race with GC_realloc. */
#   endif
    GC_ASSERT((sz & (sizeof(ptr_t)-1)) == 0);

    /* Go through all objects in the block. */
    p = hbp -> hb_body;
    plim = p + HBLKSIZE - sz;
    for (bit_no = 0; ADDR_GE(plim, p); bit_no += MARK_BIT_OFFSET(sz)) {
        if (mark_bit_from_hdr(hhdr, bit_no)) {
            p += sz;
        } else {
            /* The object is available - put it on list. */
            obj_link(p) = list;
            list = p;
            FREE_PROFILER_HOOK(p);
            p = GC_clear_block(p, sz, pcount);
        }
    }
    return list;
}

/* The same thing, but don't clear objects: */
STATIC ptr_t GC_reclaim_uninit(struct hblk *hbp, const hdr *hhdr, size_t sz,
                               ptr_t list, word *pcount)
{
    size_t bit_no;
    word n_bytes_found = 0;
    ptr_t p, plim;

#   ifndef THREADS
      GC_ASSERT(sz == hhdr -> hb_sz);
#   endif

    /* Go through all objects in the block. */
    p = hbp -> hb_body;
    plim = (ptr_t)hbp + HBLKSIZE - sz;
    for (bit_no = 0; ADDR_GE(plim, p);
         bit_no += MARK_BIT_OFFSET(sz), p += sz) {
        if (!mark_bit_from_hdr(hhdr, bit_no)) {
            n_bytes_found += sz;
            /* The object is available - put it on list. */
            obj_link(p) = list;
            list = p;
            FREE_PROFILER_HOOK(p);
        }
    }
    *pcount += n_bytes_found;
    return list;
}

#ifdef ENABLE_DISCLAIM
  /* Call reclaim notifier for block's kind on each unmarked object in  */
  /* block, all within a pair of corresponding enter/leave callbacks.   */
  STATIC ptr_t GC_disclaim_and_reclaim(struct hblk *hbp, hdr *hhdr, size_t sz,
                                       ptr_t list, word *pcount)
  {
    size_t bit_no;
    ptr_t p, plim;
    int (GC_CALLBACK *disclaim)(void *) =
                GC_obj_kinds[hhdr -> hb_obj_kind].ok_disclaim_proc;

    GC_ASSERT(disclaim != 0);
#   ifndef THREADS
      GC_ASSERT(sz == hhdr -> hb_sz);
#   endif
    p = hbp -> hb_body;
    plim = p + HBLKSIZE - sz;

    for (bit_no = 0; ADDR_GE(plim, p); bit_no += MARK_BIT_OFFSET(sz)) {
        if (mark_bit_from_hdr(hhdr, bit_no)) {
            p += sz;
        } else if (disclaim(p)) {
            set_mark_bit_from_hdr(hhdr, bit_no);
            INCR_MARKS(hhdr);
            p += sz;
        } else {
            obj_link(p) = list;
            list = p;
            FREE_PROFILER_HOOK(p);
            p = GC_clear_block(p, sz, pcount);
        }
    }
    return list;
  }
#endif /* ENABLE_DISCLAIM */

/* Don't really reclaim objects, just check for unmarked ones: */
STATIC void GC_reclaim_check(struct hblk *hbp, const hdr *hhdr, size_t sz)
{
    size_t bit_no;
    ptr_t p, plim;

#   ifndef THREADS
      GC_ASSERT(sz == hhdr -> hb_sz);
#   endif
    /* Go through all objects in the block. */
    p = hbp -> hb_body;
    plim = p + HBLKSIZE - sz;
    for (bit_no = 0; ADDR_GE(plim, p);
         bit_no += MARK_BIT_OFFSET(sz), p += sz) {
      if (!mark_bit_from_hdr(hhdr, bit_no))
        GC_add_leaked(p);
    }
}

/* Is a pointer-free block?  Same as IS_PTRFREE() macro but uses    */
/* unordered atomic access to avoid racing with GC_realloc.         */
#ifdef AO_HAVE_load
# define IS_PTRFREE_SAFE(hhdr) (AO_load((AO_t *)&((hhdr) -> hb_descr)) == 0)
#else
  /* No race as GC_realloc holds the allocator lock when updating hb_descr. */
# define IS_PTRFREE_SAFE(hhdr) IS_PTRFREE(hhdr)
#endif

/* Generic procedure to rebuild a free list in hbp.  Also called    */
/* directly from GC_malloc_many.  sz is in bytes.                   */
GC_INNER ptr_t GC_reclaim_generic(struct hblk *hbp, hdr *hhdr, size_t sz,
                                  GC_bool init, ptr_t list, word *pcount)
{
    ptr_t result;

#   ifndef PARALLEL_MARK
      GC_ASSERT(I_HOLD_LOCK());
#   endif
    GC_ASSERT(GC_find_header(hbp) == hhdr);
#   ifndef GC_DISABLE_INCREMENTAL
      GC_remove_protection(hbp, 1, IS_PTRFREE_SAFE(hhdr));
#   endif
#   ifdef ENABLE_DISCLAIM
      if ((hhdr -> hb_flags & HAS_DISCLAIM) != 0) {
        result = GC_disclaim_and_reclaim(hbp, hhdr, sz, list, pcount);
      } else
#   endif
    /* else */ if (init || GC_debugging_started) {
      result = GC_reclaim_clear(hbp, hhdr, sz, list, pcount);
    } else {
#     ifndef AO_HAVE_load
        GC_ASSERT(IS_PTRFREE(hhdr));
#     endif
      result = GC_reclaim_uninit(hbp, hhdr, sz, list, pcount);
    }
    if (IS_UNCOLLECTABLE(hhdr -> hb_obj_kind)) GC_set_hdr_marks(hhdr);
    return result;
}

/*
 * Restore unmarked small objects in the block pointed to by hbp
 * to the appropriate object free list.
 * If entirely empty blocks are to be completely deallocated, then
 * caller should perform that check.
 */
STATIC void GC_reclaim_small_nonempty_block(struct hblk *hbp, size_t sz,
                                            GC_bool report_if_found)
{
    hdr *hhdr;

    GC_ASSERT(I_HOLD_LOCK());
    hhdr = HDR(hbp);
    hhdr -> hb_last_reclaimed = (unsigned short)GC_gc_no;
    if (report_if_found) {
        GC_reclaim_check(hbp, hhdr, sz);
    } else {
        struct obj_kind *ok = &GC_obj_kinds[hhdr -> hb_obj_kind];
        void **flh = &(ok -> ok_freelist[BYTES_TO_GRANULES(sz)]);

        *flh = GC_reclaim_generic(hbp, hhdr, sz, ok -> ok_init, (ptr_t)(*flh),
                                  (/* unsigned */ word *)&GC_bytes_found);
    }
}

#ifdef ENABLE_DISCLAIM
  STATIC void GC_disclaim_and_reclaim_or_free_small_block(struct hblk *hbp)
  {
    hdr *hhdr;
    size_t sz;
    struct obj_kind *ok;
    void **flh;
    void *flh_next;

    GC_ASSERT(I_HOLD_LOCK());
    hhdr = HDR(hbp);
    sz = hhdr -> hb_sz;
    ok = &GC_obj_kinds[hhdr -> hb_obj_kind];
    flh = &(ok -> ok_freelist[BYTES_TO_GRANULES(sz)]);

    hhdr -> hb_last_reclaimed = (unsigned short)GC_gc_no;
    flh_next = GC_reclaim_generic(hbp, hhdr, sz, ok -> ok_init, (ptr_t)(*flh),
                                  (/* unsigned */ word *)&GC_bytes_found);
    if (hhdr -> hb_n_marks) {
        *flh = flh_next;
    } else {
        GC_bytes_found += (signed_word)HBLKSIZE;
        GC_freehblk(hbp);
    }
  }
#endif /* ENABLE_DISCLAIM */

/* Restore an unmarked large object or an entirely empty blocks of      */
/* small objects to the heap block free list.  Otherwise enqueue the    */
/* block for later processing by GC_reclaim_small_nonempty_block.       */
/* If report_if_found is TRUE, then process any block immediately, and  */
/* simply report free objects; do not actually reclaim them.            */
STATIC void GC_CALLBACK GC_reclaim_block(struct hblk *hbp,
                                         void *report_if_found)
{
    hdr *hhdr;
    size_t sz;  /* size of objects in current block */
    struct obj_kind *ok;

    GC_ASSERT(I_HOLD_LOCK());
#   if defined(CPPCHECK)
        GC_noop1_ptr(report_if_found);
#   endif
    hhdr = HDR(hbp);
    ok = &GC_obj_kinds[hhdr -> hb_obj_kind];
#   ifdef AO_HAVE_load
        /* Atomic access is used to avoid racing with GC_realloc.       */
        sz = AO_load(&(hhdr -> hb_sz));
#   else
        /* No race as GC_realloc holds the allocator lock while */
        /* updating hb_sz.                                      */
        sz = hhdr -> hb_sz;
#   endif
    if (sz > MAXOBJBYTES) { /* 1 big object */
        if (!mark_bit_from_hdr(hhdr, 0)) {
            if (report_if_found) {
              GC_add_leaked((ptr_t)hbp);
            } else {
#             ifdef ENABLE_DISCLAIM
                if (EXPECT(hhdr -> hb_flags & HAS_DISCLAIM, 0)) {
                  if (ok -> ok_disclaim_proc(hbp)) {
                    /* Not disclaimed => resurrect the object. */
                    set_mark_bit_from_hdr(hhdr, 0);
                    goto in_use;
                  }
                }
#             endif
              if (sz > HBLKSIZE) {
                GC_large_allocd_bytes -= HBLKSIZE * OBJ_SZ_TO_BLOCKS(sz);
              }
              GC_bytes_found += (signed_word)sz;
              GC_freehblk(hbp);
              FREE_PROFILER_HOOK(hbp);
            }
        } else {
#        ifdef ENABLE_DISCLAIM
           in_use:
#        endif
            if (IS_PTRFREE_SAFE(hhdr)) {
              GC_atomic_in_use += sz;
            } else {
              GC_composite_in_use += sz;
            }
        }
    } else {
        GC_bool empty = GC_block_empty(hhdr);

#       ifdef PARALLEL_MARK
          /* Count can be low or one too high because we sometimes      */
          /* have to ignore decrements.  Objects can also potentially   */
          /* be repeatedly marked by each marker.                       */
          /* Here we assume 3 markers at most, but this is extremely    */
          /* unlikely to fail spuriously with more.  And if it does, it */
          /* should be looked at.                                       */
          GC_ASSERT(sz != 0 && (GC_markers_m1 > 1 ? 3 : GC_markers_m1 + 1)
                                * (HBLKSIZE/sz + 1) + 16 >= hhdr->hb_n_marks);
#       else
          GC_ASSERT(sz * hhdr -> hb_n_marks <= HBLKSIZE);
#       endif
#       ifdef VALGRIND_TRACKING
          /* Call GC_free_profiler_hook() on freed objects so that  */
          /* a profiling tool could track the allocations.          */
          {
            ptr_t p = hbp -> hb_body;
            ptr_t plim = p + HBLKSIZE - sz;
            size_t bit_no;

            for (bit_no = 0; ADDR_GE(plim, p);
                 bit_no += MARK_BIT_OFFSET(sz), p += sz) {
              if (!mark_bit_from_hdr(hhdr, bit_no))
                FREE_PROFILER_HOOK(p);
            }
          }
#       endif
        if (report_if_found) {
          GC_reclaim_small_nonempty_block(hbp, sz,
                                          TRUE /* report_if_found */);
        } else if (empty) {
#       ifdef ENABLE_DISCLAIM
          if ((hhdr -> hb_flags & HAS_DISCLAIM) != 0) {
            GC_disclaim_and_reclaim_or_free_small_block(hbp);
          } else
#       endif
          /* else */ {
            GC_bytes_found += (signed_word)HBLKSIZE;
            GC_freehblk(hbp);
            FREE_PROFILER_HOOK(hbp);
          }
        } else if (GC_find_leak || !GC_block_nearly_full(hhdr, sz)) {
          /* group of smaller objects, enqueue the real work */
          struct hblk **rlh = ok -> ok_reclaim_list;

          if (rlh != NULL) {
            rlh += BYTES_TO_GRANULES(sz);
            hhdr -> hb_next = *rlh;
            *rlh = hbp;
          }
        } /* else not worth salvaging. */
        /* We used to do the nearly_full check later, but we    */
        /* already have the right cache context here.  Also     */
        /* doing it here avoids some silly lock contention in   */
        /* GC_malloc_many.                                      */
        if (IS_PTRFREE_SAFE(hhdr)) {
          GC_atomic_in_use += (word)sz * hhdr -> hb_n_marks;
        } else {
          GC_composite_in_use += (word)sz * hhdr -> hb_n_marks;
        }
    }
}

#if !defined(NO_DEBUGGING)
  /* Routines to gather and print heap block info intended for      */
  /* debugging.  Otherwise should be called with the allocator lock */
  /* held.                                                          */

  struct Print_stats
  {
    size_t number_of_blocks;
    size_t total_bytes;
  };

  EXTERN_C_BEGIN /* to avoid "no previous prototype" clang warning */
  unsigned GC_n_set_marks(const hdr *);
  EXTERN_C_END

# ifdef USE_MARK_BYTES
    /* Return the number of set mark bits in the given header.      */
    /* Remains externally visible as used by GNU GCJ currently.     */
    /* There could be a race between GC_clear_hdr_marks and this    */
    /* function but the latter is for a debug purpose.              */
    GC_ATTR_NO_SANITIZE_THREAD
    unsigned GC_n_set_marks(const hdr *hhdr)
    {
      unsigned result = 0;
      size_t i;
      size_t offset = MARK_BIT_OFFSET(hhdr -> hb_sz);
      size_t limit = FINAL_MARK_BIT(hhdr -> hb_sz);

      for (i = 0; i < limit; i += offset) {
        result += (unsigned)(hhdr -> hb_marks[i]);
      }
      GC_ASSERT(hhdr -> hb_marks[limit]); /* the one set past the end */
      return result;
    }

# else
    /* Number of set bits in a word.  Not performance critical.     */
    static unsigned count_ones(word v)
    {
      unsigned result = 0;

      for (; v > 0; v >>= 1) {
        if (v & 1) result++;
      }
      return result;
    }

    unsigned GC_n_set_marks(const hdr *hhdr)
    {
      unsigned result = 0;
      size_t sz = hhdr -> hb_sz;
      size_t i;
#     ifdef MARK_BIT_PER_OBJ
        size_t n_objs = HBLK_OBJS(sz);
        size_t n_mark_words
                    = divWORDSZ(n_objs > 0 ? n_objs : 1); /* round down */

        for (i = 0; i <= n_mark_words; i++) {
          result += count_ones(hhdr -> hb_marks[i]);
        }
#     else

        for (i = 0; i < HB_MARKS_SZ; i++) {
          result += count_ones(hhdr -> hb_marks[i]);
        }
#     endif
      GC_ASSERT(result > 0);
      result--; /* exclude the one bit set past the end */
#     ifndef MARK_BIT_PER_OBJ
        if (IS_UNCOLLECTABLE(hhdr -> hb_obj_kind)) {
          size_t lg = BYTES_TO_GRANULES(sz);

          /* As mentioned in GC_set_hdr_marks(), all the bits are set   */
          /* instead of every n-th, thus the result should be adjusted. */
          GC_ASSERT((unsigned)lg != 0 && result % lg == 0);
          result /= (unsigned)lg;
        }
#     endif
      return result;
    }
# endif /* !USE_MARK_BYTES  */

  GC_API unsigned GC_CALL GC_count_set_marks_in_hblk(const void *p) {
    return GC_n_set_marks(HDR(p));
  }

  STATIC void GC_CALLBACK GC_print_block_descr(struct hblk *h, void *raw_ps)
  {
    const hdr *hhdr = HDR(h);
    size_t sz = hhdr -> hb_sz;
    struct Print_stats *ps = (struct Print_stats *)raw_ps;
    size_t n_marks = (size_t)GC_n_set_marks(hhdr);
    size_t n_objs = HBLK_OBJS(sz);

#   ifndef PARALLEL_MARK
      GC_ASSERT(hhdr -> hb_n_marks == n_marks);
#   endif
#   if defined(CPPCHECK)
        GC_noop1_ptr(h);
#   endif
    GC_ASSERT((n_objs > 0 ? n_objs : 1) >= n_marks);
    GC_printf("%u,%u,%u,%u\n", hhdr -> hb_obj_kind, (unsigned)sz,
              (unsigned)n_marks, (unsigned)n_objs);
    ps -> number_of_blocks++;
    ps -> total_bytes += (sz + HBLKSIZE-1) & ~(HBLKSIZE-1); /* round up */
  }

  void GC_print_block_list(void)
  {
    struct Print_stats pstats;

    GC_printf("kind(0=ptrfree/1=normal/2=unc.),"
              "obj_sz,#marks_set,#objs_in_block\n");
    BZERO(&pstats, sizeof(pstats));
    GC_apply_to_all_blocks(GC_print_block_descr, &pstats);
    GC_printf("blocks= %lu, total_bytes= %lu\n",
              (unsigned long)pstats.number_of_blocks,
              (unsigned long)pstats.total_bytes);
    if (pstats.total_bytes + GC_large_free_bytes != GC_heapsize)
      GC_err_printf("LOST SOME BLOCKS!! Total bytes should be: %lu\n",
                    (unsigned long)(GC_heapsize - GC_large_free_bytes));
  }

  GC_API void GC_CALL GC_print_free_list(int k, size_t lg)
  {
    void *flh_next;
    int n;

    GC_ASSERT(k < MAXOBJKINDS);
    GC_ASSERT(lg <= MAXOBJGRANULES);
    flh_next = GC_obj_kinds[k].ok_freelist[lg];
    for (n = 0; flh_next != NULL; n++) {
      GC_printf("Free object in heap block %p [%d]: %p\n",
                (void *)HBLKPTR(flh_next), n, flh_next);
      flh_next = obj_link(flh_next);
    }
  }
#endif /* !NO_DEBUGGING */

/*
 * Clear all obj_link pointers in the list of free objects *flp.
 * Clear *flp.
 * This must be done before dropping a list of free gcj-style objects,
 * since may otherwise end up with dangling "descriptor" pointers.
 * It may help for other pointer-containing objects.
 */
STATIC void GC_clear_fl_links(void **flp)
{
    void *next;

    for (next = *flp; next != NULL; next = *flp) {
       *flp = NULL;
       flp = &obj_link(next);
    }
}

/*
 * Perform GC_reclaim_block on the entire heap, after first clearing
 * small-object free lists (if we are not just looking for leaks).
 */
GC_INNER void GC_start_reclaim(GC_bool report_if_found)
{
    int k;

    GC_ASSERT(I_HOLD_LOCK());
#   if defined(PARALLEL_MARK)
      GC_ASSERT(0 == GC_fl_builder_count);
#   endif
    /* Reset in use counters.  GC_reclaim_block recomputes them. */
      GC_composite_in_use = 0;
      GC_atomic_in_use = 0;
    /* Clear reclaim- and free-lists.   */
    for (k = 0; k < (int)GC_n_kinds; k++) {
        struct hblk **rlist = GC_obj_kinds[k].ok_reclaim_list;
        GC_bool should_clobber = GC_obj_kinds[k].ok_descriptor != 0;

        if (NULL == rlist) continue; /* means this object kind is not used */

        if (!report_if_found) {
            void **fop;
            void **lim = &GC_obj_kinds[k].ok_freelist[MAXOBJGRANULES + 1];

            for (fop = GC_obj_kinds[k].ok_freelist;
                 ADDR_LT((ptr_t)fop, (ptr_t)lim); fop++) {
              if (*fop != NULL) {
                if (should_clobber) {
                  GC_clear_fl_links(fop);
                } else {
                  *fop = NULL;
                }
              }
            }
        } /* otherwise free-list objects are marked,    */
          /* and it's safe to leave them.               */
        BZERO(rlist, (MAXOBJGRANULES + 1) * sizeof(void *));
    }


    /* Go through all heap blocks (in hblklist) and reclaim unmarked    */
    /* objects or enqueue the block for later processing.               */
    GC_apply_to_all_blocks(GC_reclaim_block, (void *)(word)report_if_found);

# ifdef EAGER_SWEEP
    /* This is a very stupid thing to do.  We make it possible anyway,  */
    /* so that you can convince yourself that it really is very stupid. */
    GC_reclaim_all((GC_stop_func)0, FALSE);
# elif defined(ENABLE_DISCLAIM)
    /* However, make sure to clear reclaimable objects of kinds with    */
    /* unconditional marking enabled before we do any significant       */
    /* marking work.                                                    */
    GC_reclaim_unconditionally_marked();
# endif
# if defined(PARALLEL_MARK)
    GC_ASSERT(0 == GC_fl_builder_count);
# endif
}

GC_INNER void GC_continue_reclaim(size_t lg, int k)
{
    struct hblk *hbp;
    struct obj_kind *ok = &GC_obj_kinds[k];
    struct hblk **rlh = ok -> ok_reclaim_list;
    void **flh;

    GC_ASSERT(I_HOLD_LOCK());
    if (NULL == rlh)
        return; /* No blocks of this kind.      */

    flh = &(ok -> ok_freelist[lg]);
    for (rlh += lg; (hbp = *rlh) != NULL; ) {
        const hdr *hhdr = HDR(hbp);

        *rlh = hhdr -> hb_next;
        GC_reclaim_small_nonempty_block(hbp, hhdr -> hb_sz, FALSE);
        if (*flh != NULL)
            break; /* the appropriate free list is nonempty */
    }
}

/*
 * Reclaim all small blocks waiting to be reclaimed.
 * Abort and return FALSE when/if (*stop_func)() returns TRUE.
 * If this returns TRUE, then it's safe to restart the world
 * with incorrectly cleared mark bits.
 * If ignore_old is TRUE, then reclaim only blocks that have been
 * recently reclaimed, and discard the rest.
 * Stop_func may be 0.
 */
GC_INNER GC_bool GC_reclaim_all(GC_stop_func stop_func, GC_bool ignore_old)
{
    size_t lg;
    int k;
    const hdr * hhdr;
    struct hblk * hbp;
    struct hblk ** rlp;
    struct hblk ** rlh;
#   ifndef NO_CLOCK
      CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER;

      if (GC_print_stats == VERBOSE)
        GET_TIME(start_time);
#   endif
    GC_ASSERT(I_HOLD_LOCK());

    for (k = 0; k < (int)GC_n_kinds; k++) {
        rlp = GC_obj_kinds[k].ok_reclaim_list;
        if (rlp == 0) continue;
        for (lg = 1; lg <= MAXOBJGRANULES; lg++) {
            for (rlh = rlp + lg; (hbp = *rlh) != NULL; ) {
                if (stop_func != (GC_stop_func)0 && (*stop_func)()) {
                    return FALSE;
                }
                hhdr = HDR(hbp);
                *rlh = hhdr -> hb_next;
                if (!ignore_old
                    || (word)(hhdr -> hb_last_reclaimed) == GC_gc_no - 1) {
                    /* It's likely we'll need it this time, too */
                    /* It's been touched recently, so this      */
                    /* shouldn't trigger paging.                */
                    GC_reclaim_small_nonempty_block(hbp, hhdr -> hb_sz, FALSE);
                }
            }
        }
    }
#   ifndef NO_CLOCK
      if (GC_print_stats == VERBOSE) {
        CLOCK_TYPE done_time;

        GET_TIME(done_time);
        GC_verbose_log_printf(
                        "Disposing of reclaim lists took %lu ms %lu ns\n",
                        MS_TIME_DIFF(done_time, start_time),
                        NS_FRAC_TIME_DIFF(done_time, start_time));
      }
#   endif
    return TRUE;
}

#if !defined(EAGER_SWEEP) && defined(ENABLE_DISCLAIM)
/* We do an eager sweep on heap blocks where unconditional marking has  */
/* been enabled, so that any reclaimable objects have been reclaimed    */
/* before we start marking.  This is a simplified GC_reclaim_all        */
/* restricted to kinds where ok_mark_unconditionally is true.           */
  STATIC void GC_reclaim_unconditionally_marked(void)
  {
    int k;

    GC_ASSERT(I_HOLD_LOCK());
    for (k = 0; k < (int)GC_n_kinds; k++) {
        size_t lg;
        struct obj_kind *ok = &GC_obj_kinds[k];
        struct hblk **rlp = ok -> ok_reclaim_list;

        if (NULL == rlp || !(ok -> ok_mark_unconditionally)) continue;

        for (lg = 1; lg <= MAXOBJGRANULES; lg++) {
            struct hblk **rlh = rlp + lg;
            struct hblk *hbp;

            while ((hbp = *rlh) != NULL) {
                const hdr *hhdr = HDR(hbp);

                *rlh = hhdr -> hb_next;
                GC_reclaim_small_nonempty_block(hbp, hhdr -> hb_sz, FALSE);
            }
        }
    }
  }
#endif /* !EAGER_SWEEP && ENABLE_DISCLAIM */

struct enumerate_reachable_s {
  GC_reachable_object_proc proc;
  void *client_data;
};

STATIC void GC_CALLBACK GC_do_enumerate_reachable_objects(struct hblk *hbp,
                                                          void *ed_ptr)
{
  const hdr *hhdr = HDR(hbp);
  ptr_t p, plim;
  const struct enumerate_reachable_s *ped
                        = (struct enumerate_reachable_s *)ed_ptr;
  size_t sz = hhdr -> hb_sz;
  size_t bit_no;

  if (GC_block_empty(hhdr)) return;

  p = hbp -> hb_body;
  if (sz > MAXOBJBYTES) { /* one big object */
    plim = p;
  } else {
    plim = p + HBLKSIZE - sz;
  }
  /* Go through all objects in the block. */
  for (bit_no = 0; ADDR_GE(plim, p); bit_no += MARK_BIT_OFFSET(sz), p += sz) {
    if (mark_bit_from_hdr(hhdr, bit_no)) {
      ped -> proc(p, sz, ped -> client_data);
    }
  }
}

GC_API void GC_CALL GC_enumerate_reachable_objects_inner(
                                                GC_reachable_object_proc proc,
                                                void *client_data)
{
  struct enumerate_reachable_s ed;

  GC_ASSERT(I_HOLD_READER_LOCK());
  ed.proc = proc;
  ed.client_data = client_data;
  GC_apply_to_all_blocks(GC_do_enumerate_reachable_objects, &ed);
}

/*
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1999-2000 by Hewlett-Packard Company.  All rights reserved.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


/*
 * Some simple primitives for allocation with explicit type information.
 * Simple objects are allocated such that they contain a GC_descr at the
 * end (in the last allocated word).  This descriptor may be a procedure
 * which then examines an extended descriptor passed as its environment.
 *
 * Arrays are treated as simple objects if they have sufficiently simple
 * structure.  Otherwise they are allocated from an array kind that supplies
 * a special mark procedure.  These arrays contain a pointer to a
 * complex_descriptor as their last "pointer-sized" word.
 * This is done because the environment field is too small, and the collector
 * must trace the complex_descriptor.
 *
 * Note that descriptors inside objects may appear cleared, if we encounter
 * a false reference to an object on a free list.  In the case of a simple
 * object, this is OK, since a zero descriptor corresponds to examining no
 * fields.  In the complex_descriptor case, we explicitly check for that case.
 *
 * MAJOR PARTS OF THIS CODE HAVE NOT BEEN TESTED AT ALL and are not testable,
 * since they are not accessible through the current interface.
 */

#include "gc/gc_typed.h"

STATIC int GC_explicit_kind = 0;
                        /* Object kind for objects with indirect        */
                        /* (possibly extended) descriptors.             */

STATIC int GC_array_kind = 0;
                        /* Object kind for objects with complex         */
                        /* descriptors and GC_array_mark_proc.          */

#define ED_INITIAL_SIZE 100

STATIC unsigned GC_typed_mark_proc_index = 0;   /* Indices of the typed */
STATIC unsigned GC_array_mark_proc_index = 0;   /* mark procedures.     */

STATIC void GC_push_typed_structures_proc(void)
{
  GC_PUSH_ALL_SYM(GC_ext_descriptors);
}

/* Add a multi-word bitmap to GC_ext_descriptors arrays.        */
/* Returns starting index on success, -1 otherwise.             */
STATIC signed_word GC_add_ext_descriptor(const word * bm, size_t nbits)
{
    signed_word result;
    size_t i;
    size_t nwords = divWORDSZ(nbits + CPP_WORDSZ-1);

    LOCK();
    while (EXPECT(GC_avail_descr + nwords >= GC_ed_size, FALSE)) {
        typed_ext_descr_t *newExtD;
        size_t new_size;
        size_t ed_size = GC_ed_size;

        if (0 == ed_size) {
            GC_ASSERT(ADDR(&GC_ext_descriptors) % sizeof(ptr_t) == 0);
            GC_push_typed_structures = GC_push_typed_structures_proc;
            UNLOCK();
            new_size = ED_INITIAL_SIZE;
        } else {
            UNLOCK();
            new_size = 2 * ed_size;
            if (new_size > MAX_ENV) return -1;
        }
        newExtD = (typed_ext_descr_t*)GC_malloc_atomic(new_size
                                                * sizeof(typed_ext_descr_t));
        if (NULL == newExtD)
            return -1;
        LOCK();
        if (ed_size == GC_ed_size) {
            if (GC_avail_descr != 0) {
                BCOPY(GC_ext_descriptors, newExtD,
                      GC_avail_descr * sizeof(typed_ext_descr_t));
            }
            GC_ed_size = new_size;
            GC_ext_descriptors = newExtD;
        }  /* else another thread already resized it in the meantime */
    }
    result = (signed_word)GC_avail_descr;
    for (i = 0; i < nwords - 1; i++) {
        GC_ext_descriptors[(size_t)result + i].ed_bitmap = bm[i];
        GC_ext_descriptors[(size_t)result + i].ed_continued = TRUE;
    }
    /* Clear irrelevant (highest) bits for the last element.    */
    GC_ext_descriptors[(size_t)result + i].ed_bitmap =
                bm[i] & (GC_WORD_MAX >> (nwords * CPP_WORDSZ - nbits));
    GC_ext_descriptors[(size_t)result + i].ed_continued = FALSE;
    GC_avail_descr += nwords;
    GC_ASSERT(result >= 0);
    UNLOCK();
    return result;
}

/* Table of bitmap descriptors for n pointer-long all-pointer objects.  */
STATIC GC_descr GC_bm_table[CPP_WORDSZ / 2];

/* Return a descriptor for the concatenation of 2 objects, each one is  */
/* lpw pointers long and described by descriptor d.  The result is      */
/* known to be short enough to fit into a bitmap descriptor.            */
/* d is a GC_DS_LENGTH or GC_DS_BITMAP descriptor.                      */
STATIC GC_descr GC_double_descr(GC_descr d, size_t lpw)
{
    GC_ASSERT(GC_bm_table[0] == GC_DS_BITMAP); /* bm table is initialized */
    if ((d & GC_DS_TAGS) == GC_DS_LENGTH) {
        d = GC_bm_table[BYTES_TO_PTRS(d)];
    }
    d |= (d & ~(GC_descr)GC_DS_TAGS) >> lpw;
    return d;
}

STATIC mse *GC_CALLBACK GC_typed_mark_proc(word *addr, mse *mark_stack_top,
                                           mse *mark_stack_limit, word env);

STATIC mse *GC_CALLBACK GC_array_mark_proc(word *addr, mse *mark_stack_top,
                                           mse *mark_stack_limit, word env);

STATIC void GC_init_explicit_typing(void)
{
    unsigned i;

    /* Set up object kind with simple indirect descriptor.      */
    /* Descriptor is in the last word of the object.            */
    GC_typed_mark_proc_index = GC_new_proc_inner(GC_typed_mark_proc);
    GC_explicit_kind = (int)GC_new_kind_inner(GC_new_free_list_inner(),
                            (PTRS_TO_BYTES(GC_WORD_MAX) | GC_DS_PER_OBJECT),
                            TRUE, TRUE);

    /* Set up object kind with array descriptor. */
    GC_array_mark_proc_index = GC_new_proc_inner(GC_array_mark_proc);
    GC_array_kind = (int)GC_new_kind_inner(GC_new_free_list_inner(),
                            GC_MAKE_PROC(GC_array_mark_proc_index, 0),
                            FALSE, TRUE);

    GC_bm_table[0] = GC_DS_BITMAP;
    for (i = 1; i < CPP_WORDSZ / 2; i++) {
      GC_bm_table[i] = (GC_WORD_MAX << (CPP_WORDSZ - i)) | GC_DS_BITMAP;
    }
}

STATIC mse *GC_CALLBACK GC_typed_mark_proc(word *addr, mse *mark_stack_top,
                                           mse *mark_stack_limit, word env)
{
    word bm;
    ptr_t current_p = (ptr_t)addr;
    ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr;
    ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr;
    DECLARE_HDR_CACHE;

    /* The allocator lock is held by the collection initiating thread.  */
    GC_ASSERT(GC_get_parallel() || I_HOLD_LOCK());
    bm = GC_ext_descriptors[env].ed_bitmap;

    INIT_HDR_CACHE;
    for (; bm != 0; bm >>= 1, current_p += sizeof(ptr_t)) {
        if (bm & 1) {
            ptr_t q;

            LOAD_PTR_OR_CONTINUE(q, current_p);
            FIXUP_POINTER(q);
            if (ADDR_LT(least_ha, q) && ADDR_LT(q, greatest_ha)) {
                PUSH_CONTENTS(q, mark_stack_top, mark_stack_limit, current_p);
            }
        }
    }
    if (GC_ext_descriptors[env].ed_continued) {
        /* Push an entry with the rest of the descriptor back onto the  */
        /* stack.  Thus we never do too much work at once.  Note that   */
        /* we also can't overflow the mark stack unless we actually     */
        /* mark something.                                              */
        mark_stack_top = GC_custom_push_proc(
                            GC_MAKE_PROC(GC_typed_mark_proc_index, env + 1),
                            (ptr_t *)addr + CPP_WORDSZ, mark_stack_top,
                            mark_stack_limit);
    }
    return mark_stack_top;
}

GC_API GC_descr GC_CALL GC_make_descriptor(const GC_word * bm, size_t len)
{
    signed_word last_set_bit = (signed_word)len - 1;
    GC_descr d;

#   if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_store_release)
      if (!EXPECT(AO_load_acquire(&GC_explicit_typing_initialized), TRUE)) {
        LOCK();
        if (!GC_explicit_typing_initialized) {
          GC_init_explicit_typing();
          AO_store_release(&GC_explicit_typing_initialized, TRUE);
        }
        UNLOCK();
      }
#   else
      LOCK();
      if (!EXPECT(GC_explicit_typing_initialized, TRUE)) {
        GC_init_explicit_typing();
        GC_explicit_typing_initialized = TRUE;
      }
      UNLOCK();
#   endif

    while (last_set_bit >= 0 && !GC_get_bit(bm, (word)last_set_bit))
      last_set_bit--;
    if (last_set_bit < 0) return 0; /* no pointers */

#   if ALIGNMENT == CPP_PTRSZ / 8
      {
        signed_word i;

        for (i = 0; i < last_set_bit; i++) {
          if (!GC_get_bit(bm, (word)i))
            break;
        }
        if (i == last_set_bit) {
          /* The initial section contains all pointers; use the     */
          /* length descriptor.                                     */
          return PTRS_TO_BYTES((word)last_set_bit + 1) | GC_DS_LENGTH;
        }
      }
#   endif
    if (last_set_bit < BITMAP_BITS) {
        signed_word i;

        /* Hopefully the common case.  Build the bitmap descriptor  */
        /* (with the bits reversed).                                */
        d = SIGNB;
        for (i = last_set_bit - 1; i >= 0; i--) {
            d >>= 1;
            if (GC_get_bit(bm, (word)i)) d |= SIGNB;
        }
        d |= GC_DS_BITMAP;
    } else {
        signed_word index = GC_add_ext_descriptor(bm,
                                                  (size_t)last_set_bit + 1);

        if (EXPECT(index == -1, FALSE)) {
            /* Out of memory: use a conservative approximation. */
            return PTRS_TO_BYTES((word)last_set_bit + 1) | GC_DS_LENGTH;
        }
        d = GC_MAKE_PROC(GC_typed_mark_proc_index, index);
    }
    return d;
}

static void set_obj_descr(ptr_t op, GC_descr d)
{
  size_t sz;

  if (EXPECT(NULL == op, FALSE)) return;

  /* It is not safe to use GC_size_map[] here as the table might be */
  /* updated asynchronously.                                        */
  sz = GC_size(op);

  GC_ASSERT((sz & (GC_GRANULE_BYTES-1)) == 0 && sz > sizeof(GC_descr));
# ifdef AO_HAVE_store_release
    AO_store_release((volatile AO_t *)&op[sz - sizeof(GC_descr)], d);
# else
    *(GC_descr *)&op[sz - sizeof(GC_descr)] = d;
# endif
}

GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_explicitly_typed(size_t lb,
                                                                GC_descr d)
{
    ptr_t op;

    GC_ASSERT(GC_explicit_typing_initialized);
    if (EXPECT(lb < sizeof(ptr_t) - sizeof(GC_descr) + 1, FALSE)) {
       /* Ensure the descriptor does not occupy the first pointer place. */
      lb = sizeof(ptr_t) - sizeof(GC_descr) + 1;
    }
    op = (ptr_t)GC_malloc_kind(
                        SIZET_SAT_ADD(lb, sizeof(GC_descr) - EXTRA_BYTES),
                        GC_explicit_kind);
    set_obj_descr(op, d);
    return op;
}

GC_API GC_ATTR_MALLOC void * GC_CALL
    GC_malloc_explicitly_typed_ignore_off_page(size_t lb, GC_descr d)
{
    ptr_t op;

    if (lb < HBLKSIZE - sizeof(GC_descr))
      return GC_malloc_explicitly_typed(lb, d);

    GC_ASSERT(GC_explicit_typing_initialized);
    /* Note that ignore-off-page objects with the requested size    */
    /* of at least HBLKSIZE do not have EXTRA_BYTES added by        */
    /* GC_generic_malloc_aligned().                                 */
    op = (ptr_t)GC_clear_stack(GC_generic_malloc_aligned(
                                SIZET_SAT_ADD(lb, sizeof(GC_descr)),
                                GC_explicit_kind, IGNORE_OFF_PAGE, 0));
    set_obj_descr(op, d);
    return op;
}

/* Array descriptors.  GC_array_mark_proc understands these.    */
/* We may eventually need to add provisions for headers and     */
/* trailers.  Hence we provide for tree structured descriptors, */
/* though we don't really use them currently.                   */

struct LeafDescriptor {         /* Describes simple array.      */
  word ld_tag;
# define LEAF_TAG 1
  size_t ld_size;               /* Bytes per element; non-zero, */
                                /* multiple of ALIGNMENT.       */
  size_t ld_nelements;          /* Number of elements.          */
  GC_descr ld_descriptor;       /* A simple length, bitmap,     */
                                /* or procedure descriptor.     */
};

struct ComplexArrayDescriptor {
  word ad_tag;
# define ARRAY_TAG 2
  size_t ad_nelements;
  union ComplexDescriptor *ad_element_descr;
};

struct SequenceDescriptor {
  word sd_tag;
# define SEQUENCE_TAG 3
  union ComplexDescriptor *sd_first;
  union ComplexDescriptor *sd_second;
};

typedef union ComplexDescriptor {
  struct LeafDescriptor ld;
  struct ComplexArrayDescriptor ad;
  struct SequenceDescriptor sd;
} complex_descriptor;

STATIC complex_descriptor *GC_make_leaf_descriptor(size_t size,
                                                   size_t nelements,
                                                   GC_descr d)
{
  complex_descriptor *result = (complex_descriptor *)
                GC_malloc_atomic(sizeof(struct LeafDescriptor));

  GC_ASSERT(size != 0);
  if (EXPECT(NULL == result, FALSE)) return NULL;

  result -> ld.ld_tag = LEAF_TAG;
  result -> ld.ld_size = size;
  result -> ld.ld_nelements = nelements;
  result -> ld.ld_descriptor = d;
  return result;
}

STATIC complex_descriptor *GC_make_sequence_descriptor(
                                                complex_descriptor *first,
                                                complex_descriptor *second)
{
  struct SequenceDescriptor *result = (struct SequenceDescriptor *)
                GC_malloc(sizeof(struct SequenceDescriptor));
                /* Note: for a reason, the sanitizer runtime complains  */
                /* of insufficient space for complex_descriptor if the  */
                /* pointer type of result variable is changed to.       */

  if (EXPECT(NULL == result, FALSE)) return NULL;

  /* Can't result in overly conservative marking, since tags are        */
  /* very small integers. Probably faster than maintaining type info.   */
  result -> sd_tag = SEQUENCE_TAG;
  result -> sd_first = first;
  result -> sd_second = second;
  GC_dirty(result);
  REACHABLE_AFTER_DIRTY(first);
  REACHABLE_AFTER_DIRTY(second);
  return (complex_descriptor *)result;
}

#define NO_MEM  (-1)
#define SIMPLE  0
#define LEAF    1
#define COMPLEX 2

/* Build a descriptor for an array with nelements elements, each of     */
/* which can be described by a simple descriptor d.  We try to optimize */
/* some common cases.  If the result is COMPLEX, a complex_descriptor*  */
/* value is returned in *pcomplex_d.  If the result is LEAF, then a     */
/* LeafDescriptor value is built in the structure pointed to by pleaf.  */
/* The tag in the *pleaf structure is not set.  If the result is        */
/* SIMPLE, then a GC_descr value is returned in *psimple_d.  If the     */
/* result is NO_MEM, then we failed to allocate the descriptor.         */
/* The implementation assumes GC_DS_LENGTH is 0.  *pleaf, *pcomplex_d   */
/* and *psimple_d may be used as temporaries during the construction.   */
STATIC int GC_make_array_descriptor(size_t nelements, size_t size,
                                    GC_descr d, GC_descr *psimple_d,
                                    complex_descriptor **pcomplex_d,
                                    struct LeafDescriptor *pleaf)
{
# define OPT_THRESHOLD 50
        /* For larger arrays, we try to combine descriptors of adjacent */
        /* descriptors to speed up marking, and to reduce the amount    */
        /* of space needed on the mark stack.                           */

  GC_ASSERT(size != 0);
  if ((d & GC_DS_TAGS) == GC_DS_LENGTH) {
    if (d == (GC_descr)size) {
      *psimple_d = nelements * d; /* no overflow guaranteed by caller */
      return SIMPLE;
    } else if (0 == d) {
      *psimple_d = 0;
      return SIMPLE;
    }
  }

  if (nelements <= OPT_THRESHOLD) {
    if (nelements <= 1) {
      *psimple_d = nelements == 1 ? d : 0;
      return SIMPLE;
    }
  } else if (size <= BITMAP_BITS / 2
             && (d & GC_DS_TAGS) != GC_DS_PROC
             && (size & (sizeof(ptr_t)-1)) == 0) {
    complex_descriptor *one_element, *beginning;
    int result = GC_make_array_descriptor(nelements / 2, 2 * size,
                                GC_double_descr(d, BYTES_TO_PTRS(size)),
                                psimple_d, pcomplex_d, pleaf);

    if ((nelements & 1) == 0 || EXPECT(NO_MEM == result, FALSE))
      return result;

    one_element = GC_make_leaf_descriptor(size, 1, d);
    if (EXPECT(NULL == one_element, FALSE)) return NO_MEM;

    if (COMPLEX == result) {
      beginning = *pcomplex_d;
    } else {
      beginning = SIMPLE == result
                    ? GC_make_leaf_descriptor(size, 1, *psimple_d)
                    : GC_make_leaf_descriptor(pleaf -> ld_size,
                                              pleaf -> ld_nelements,
                                              pleaf -> ld_descriptor);
      if (EXPECT(NULL == beginning, FALSE)) return NO_MEM;
    }
    *pcomplex_d = GC_make_sequence_descriptor(beginning, one_element);
    if (EXPECT(NULL == *pcomplex_d, FALSE)) return NO_MEM;

    return COMPLEX;
  }

  pleaf -> ld_size = size;
  pleaf -> ld_nelements = nelements;
  pleaf -> ld_descriptor = d;
  return LEAF;
}

struct GC_calloc_typed_descr_s {
  complex_descriptor *complex_d; /* the first field, the only pointer */
  struct LeafDescriptor leaf;
  GC_descr simple_d;
  word alloc_lb; /* size_t actually */
  signed_word descr_type; /* int actually */
};

GC_API int GC_CALL GC_calloc_prepare_explicitly_typed(
                                struct GC_calloc_typed_descr_s *pctd,
                                size_t ctd_sz,
                                size_t n, size_t lb, GC_descr d)
{
    GC_STATIC_ASSERT(sizeof(struct GC_calloc_typed_descr_s)
        == GC_CALLOC_TYPED_DESCR_PTRS * sizeof(ptr_t)
            + (GC_CALLOC_TYPED_DESCR_WORDS - GC_CALLOC_TYPED_DESCR_PTRS)
                * sizeof(word));
    GC_ASSERT(GC_explicit_typing_initialized);
    GC_ASSERT(sizeof(struct GC_calloc_typed_descr_s) == ctd_sz);
    (void)ctd_sz; /* unused currently */
    if (EXPECT(0 == lb || 0 == n, FALSE)) lb = n = 1;
    if (EXPECT((lb | n) > GC_SQRT_SIZE_MAX, FALSE) /* fast initial check */
        && n > GC_SIZE_MAX / lb) {
      pctd -> alloc_lb = GC_SIZE_MAX; /* n*lb overflow */
      pctd -> descr_type = NO_MEM;
      /* The rest of the fields are unset. */
      return 0; /* failure */
    }

    pctd -> descr_type = GC_make_array_descriptor(n, lb, d,
                                                  &(pctd -> simple_d),
                                                  &(pctd -> complex_d),
                                                  &(pctd -> leaf));
    switch (pctd -> descr_type) {
    case NO_MEM:
    case SIMPLE:
      pctd -> alloc_lb = (word)lb * n;
      break;
    case LEAF:
      pctd -> alloc_lb
        = SIZET_SAT_ADD(lb * n,
                        (BYTES_TO_PTRS_ROUNDUP(sizeof(struct LeafDescriptor))
                         + 1) * sizeof(ptr_t) - EXTRA_BYTES);
      break;
    case COMPLEX:
      pctd -> alloc_lb = SIZET_SAT_ADD(lb * n, sizeof(ptr_t) - EXTRA_BYTES);
      break;
    }
    return 1; /* success */
}

GC_API GC_ATTR_MALLOC void * GC_CALL GC_calloc_do_explicitly_typed(
                                const struct GC_calloc_typed_descr_s *pctd,
                                size_t ctd_sz)
{
    void *op;
    size_t lpw_m1;

    GC_ASSERT(sizeof(struct GC_calloc_typed_descr_s) == ctd_sz);
    (void)ctd_sz; /* unused currently */
    switch (pctd -> descr_type) {
    case NO_MEM:
      return (*GC_get_oom_fn())((size_t)(pctd -> alloc_lb));
    case SIMPLE:
      return GC_malloc_explicitly_typed((size_t)(pctd -> alloc_lb),
                                        pctd -> simple_d);
    case LEAF:
    case COMPLEX:
      break;
    default:
      ABORT_RET("Bad descriptor type");
      return NULL;
    }
    op = GC_malloc_kind((size_t)(pctd -> alloc_lb), GC_array_kind);
    if (EXPECT(NULL == op, FALSE))
      return NULL;

    lpw_m1 = BYTES_TO_PTRS(GC_size(op)) - 1;
    if (pctd -> descr_type == LEAF) {
      /* Set up the descriptor inside the object itself.        */
      struct LeafDescriptor *lp
                = (struct LeafDescriptor *)((ptr_t *)op + lpw_m1
                    - BYTES_TO_PTRS_ROUNDUP(sizeof(struct LeafDescriptor)));

      lp -> ld_tag = LEAF_TAG;
      lp -> ld_size = pctd -> leaf.ld_size;
      lp -> ld_nelements = pctd -> leaf.ld_nelements;
      lp -> ld_descriptor = pctd -> leaf.ld_descriptor;
      /* Hold the allocator lock (in the reader mode which should be    */
      /* enough) while writing the descriptor word to the object to     */
      /* ensure that the descriptor contents are seen by                */
      /* GC_array_mark_proc as expected.                                */
      /* TODO: It should be possible to replace locking with the atomic */
      /* operations (with the release barrier here) but, in this case,  */
      /* avoiding the acquire barrier in GC_array_mark_proc seems to    */
      /* be tricky as GC_mark_some might be invoked with the world      */
      /* running.                                                       */
      READER_LOCK();
      ((struct LeafDescriptor **)op)[lpw_m1] = lp;
      READER_UNLOCK_RELEASE();
    } else {
#     ifndef GC_NO_FINALIZATION
        READER_LOCK();
        ((complex_descriptor **)op)[lpw_m1] = pctd -> complex_d;
        READER_UNLOCK_RELEASE();

        GC_dirty((ptr_t *)op + lpw_m1);
        REACHABLE_AFTER_DIRTY(pctd -> complex_d);

        /* Make sure the descriptor is cleared once there is any danger */
        /* it may have been collected.                                  */
        if (EXPECT(GC_general_register_disappearing_link(
                        (void **)op + lpw_m1, op) == GC_NO_MEMORY, FALSE))
#     endif
        {
            /* Couldn't register it due to lack of memory.  Punt.       */
            return (*GC_get_oom_fn())((size_t)(pctd -> alloc_lb));
        }
    }
    return op;
}

GC_API GC_ATTR_MALLOC void * GC_CALL GC_calloc_explicitly_typed(size_t n,
                                                                size_t lb,
                                                                GC_descr d)
{
  struct GC_calloc_typed_descr_s ctd;

  (void)GC_calloc_prepare_explicitly_typed(&ctd, sizeof(ctd), n, lb, d);
  return GC_calloc_do_explicitly_typed(&ctd, sizeof(ctd));
}

/* Return the size of the object described by complex_d.  It would be   */
/* faster to store this directly, or to compute it as part of           */
/* GC_push_complex_descriptor, but hopefully it does not matter.        */
STATIC size_t GC_descr_obj_size(complex_descriptor *complex_d)
{
  switch (complex_d -> ad.ad_tag) {
  case LEAF_TAG:
    return complex_d -> ld.ld_nelements * complex_d -> ld.ld_size;
  case ARRAY_TAG:
    return complex_d -> ad.ad_nelements
           * GC_descr_obj_size(complex_d -> ad.ad_element_descr);
  case SEQUENCE_TAG:
    return GC_descr_obj_size(complex_d -> sd.sd_first)
           + GC_descr_obj_size(complex_d -> sd.sd_second);
  default:
    ABORT_RET("Bad complex descriptor");
    return 0;
  }
}

/* Push descriptors for the object with complex descriptor onto the */
/* mark stack.  Return NULL if the mark stack overflowed.           */
STATIC mse *GC_push_complex_descriptor(ptr_t current,
                                       complex_descriptor *complex_d,
                                       mse *msp, mse *msl)
{
  size_t i, nelements;
  size_t sz;
  GC_descr d;
  complex_descriptor *element_descr;

  switch (complex_d -> ad.ad_tag) {
  case LEAF_TAG:
    d = complex_d -> ld.ld_descriptor;
    nelements = complex_d -> ld.ld_nelements;
    sz = complex_d -> ld.ld_size;

    if (EXPECT(msl - msp <= (signed_word)nelements, FALSE)) return NULL;
    GC_ASSERT(sz != 0);
    for (i = 0; i < nelements; i++) {
      msp++;
      msp -> mse_start = current;
      msp -> mse_descr = d;
      current += sz;
    }
    break;
  case ARRAY_TAG:
    element_descr = complex_d -> ad.ad_element_descr;
    nelements = complex_d -> ad.ad_nelements;
    sz = GC_descr_obj_size(element_descr);
    GC_ASSERT(sz != 0 || 0 == nelements);
    for (i = 0; i < nelements; i++) {
      msp = GC_push_complex_descriptor(current, element_descr, msp, msl);
      if (EXPECT(NULL == msp, FALSE)) return NULL;
      current += sz;
    }
    break;
  case SEQUENCE_TAG:
    sz = GC_descr_obj_size(complex_d -> sd.sd_first);
    msp = GC_push_complex_descriptor(current, complex_d -> sd.sd_first,
                                     msp, msl);
    if (EXPECT(NULL == msp, FALSE)) return NULL;
    GC_ASSERT(sz != 0);
    current += sz;
    msp = GC_push_complex_descriptor(current, complex_d -> sd.sd_second,
                                     msp, msl);
    break;
  default:
    ABORT("Bad complex descriptor");
  }
  return msp;
}

GC_ATTR_NO_SANITIZE_THREAD
static complex_descriptor *get_complex_descr(ptr_t *p, size_t lpw)
{
  return (complex_descriptor *)p[lpw - 1];
}

/* Used by GC_calloc_do_explicitly_typed via GC_array_kind.     */
STATIC mse *GC_CALLBACK GC_array_mark_proc(word *addr, mse *mark_stack_top,
                                           mse *mark_stack_limit, word env)
{
  size_t sz = HDR(addr) -> hb_sz;
  size_t lpw = BYTES_TO_PTRS(sz);
  complex_descriptor *complex_d = get_complex_descr((ptr_t *)addr, lpw);
  mse *orig_mark_stack_top = mark_stack_top;
  mse *new_mark_stack_top;

  UNUSED_ARG(env);
  if (NULL == complex_d) {
    /* Found a reference to a free-list entry.  Ignore it.      */
    return orig_mark_stack_top;
  }
  /* In use counts were already updated when array descriptor was       */
  /* pushed.  Here we only replace it by subobject descriptors, so      */
  /* no update is necessary.                                            */
  new_mark_stack_top = GC_push_complex_descriptor((ptr_t)addr, complex_d,
                                                  mark_stack_top,
                                                  mark_stack_limit - 1);
  if (NULL == new_mark_stack_top) {
    /* Explicitly instruct Clang Static Analyzer that ptr is non-null.  */
    if (NULL == mark_stack_top) ABORT("Bad mark_stack_top");

    /* Does not fit.  Conservatively push the whole array as a unit and */
    /* request a mark stack expansion.  This cannot cause a mark stack  */
    /* overflow, since it replaces the original array entry.            */
#   ifdef PARALLEL_MARK
      /* We might be using a local_mark_stack in parallel mode. */
      if (GC_mark_stack + GC_mark_stack_size == mark_stack_limit)
#   endif
    {
      GC_mark_stack_too_small = TRUE;
    }
    new_mark_stack_top = orig_mark_stack_top + 1;
    new_mark_stack_top -> mse_start = (ptr_t)addr;
    new_mark_stack_top -> mse_descr = sz | GC_DS_LENGTH;
  } else {
    /* Push descriptor itself.  */
    new_mark_stack_top++;
    new_mark_stack_top -> mse_start = (ptr_t)((ptr_t *)addr + lpw - 1);
    new_mark_stack_top -> mse_descr = sizeof(ptr_t) | GC_DS_LENGTH;
  }
  return new_mark_stack_top;
}


/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1999-2001 by Hewlett-Packard Company. All rights reserved.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#include <limits.h>
#include <stdarg.h>

#ifdef GC_SOLARIS_THREADS
# include <sys/syscall.h>
#endif

#if defined(UNIX_LIKE) || defined(CYGWIN32) || defined(SYMBIAN) \
    || (defined(CONSOLE_LOG) && defined(MSWIN32))
# include <fcntl.h>
# include <sys/stat.h>
#endif

#if defined(CONSOLE_LOG) && defined(MSWIN32) && !defined(__GNUC__)
# include <io.h>
#endif

#ifdef NONSTOP
# include <floss.h>
#endif

#ifdef THREADS
# ifdef PCR
#   include "il/PCR_IL.h"
    GC_INNER PCR_Th_ML GC_allocate_ml;
# elif defined(SN_TARGET_PSP2)
    GC_INNER WapiMutex GC_allocate_ml_PSP2 = { 0, NULL };
# elif defined(GC_DEFN_ALLOCATE_ML) && !defined(USE_RWLOCK) \
       || defined(SN_TARGET_PS3)
#   include <pthread.h>
    GC_INNER pthread_mutex_t GC_allocate_ml;
# endif
  /* For other platforms with threads, the allocator lock and possibly  */
  /* GC_lock_holder variables are defined in the thread support code.   */
#endif /* THREADS */

#ifdef DYNAMIC_LOADING
  /* We need to register the main data segment.  Returns TRUE unless    */
  /* this is done implicitly as part of dynamic library registration.   */
# define GC_REGISTER_MAIN_STATIC_DATA() GC_register_main_static_data()
#elif defined(GC_DONT_REGISTER_MAIN_STATIC_DATA)
# define GC_REGISTER_MAIN_STATIC_DATA() FALSE
#else
  /* Don't unnecessarily call GC_register_main_static_data() in case    */
  /* dyn_load.c isn't linked in.                                        */
# define GC_REGISTER_MAIN_STATIC_DATA() TRUE
#endif

#ifdef NEED_CANCEL_DISABLE_COUNT
  __thread unsigned char GC_cancel_disable_count = 0;
#endif

GC_FAR struct _GC_arrays GC_arrays /* = { 0 } */;

GC_INNER unsigned GC_n_mark_procs = GC_RESERVED_MARK_PROCS;

GC_INNER unsigned GC_n_kinds = GC_N_KINDS_INITIAL_VALUE;

GC_INNER GC_bool GC_debugging_started = FALSE;
                /* defined here so we don't have to load dbg_mlc.o */

ptr_t GC_stackbottom = 0;

#if defined(E2K) && defined(THREADS) || defined(IA64)
  GC_INNER ptr_t GC_register_stackbottom = NULL;
#endif

int GC_dont_gc = FALSE;

int GC_dont_precollect = FALSE;

GC_bool GC_quiet = 0; /* used also in pcr_interface.c */

#if !defined(NO_CLOCK) || !defined(SMALL_CONFIG)
  GC_INNER int GC_print_stats = 0;
#endif

#ifdef GC_PRINT_BACK_HEIGHT
  GC_INNER GC_bool GC_print_back_height = TRUE;
#else
  GC_INNER GC_bool GC_print_back_height = FALSE;
#endif

#ifndef NO_DEBUGGING
# ifdef GC_DUMP_REGULARLY
    GC_INNER GC_bool GC_dump_regularly = TRUE;
                                /* Generate regular debugging dumps. */
# else
    GC_INNER GC_bool GC_dump_regularly = FALSE;
# endif
# ifndef NO_CLOCK
    STATIC CLOCK_TYPE GC_init_time;
                /* The time that the GC was initialized at.     */
# endif
#endif /* !NO_DEBUGGING */

#ifdef KEEP_BACK_PTRS
  GC_INNER long GC_backtraces = 0;
                /* Number of random backtraces to generate for each GC. */
#endif

#ifdef FIND_LEAK
  int GC_find_leak = 1;
#else
  int GC_find_leak = 0;
#endif

#ifndef SHORT_DBG_HDRS
# ifdef GC_FINDLEAK_DELAY_FREE
    GC_INNER GC_bool GC_findleak_delay_free = TRUE;
# else
    GC_INNER GC_bool GC_findleak_delay_free = FALSE;
# endif
#endif /* !SHORT_DBG_HDRS */

#ifdef ALL_INTERIOR_POINTERS
  int GC_all_interior_pointers = 1;
#else
  int GC_all_interior_pointers = 0;
#endif

#ifdef FINALIZE_ON_DEMAND
  int GC_finalize_on_demand = 1;
#else
  int GC_finalize_on_demand = 0;
#endif

#ifdef JAVA_FINALIZATION
  int GC_java_finalization = 1;
#else
  int GC_java_finalization = 0;
#endif

/* All accesses to it should be synchronized to avoid data race.    */
GC_finalizer_notifier_proc GC_finalizer_notifier =
                                        (GC_finalizer_notifier_proc)0;

#ifdef GC_FORCE_UNMAP_ON_GCOLLECT
  /* Has no effect unless USE_MUNMAP.                           */
  /* Has no effect on implicitly-initiated garbage collections. */
  GC_INNER GC_bool GC_force_unmap_on_gcollect = TRUE;
#else
  GC_INNER GC_bool GC_force_unmap_on_gcollect = FALSE;
#endif

#ifndef GC_LARGE_ALLOC_WARN_INTERVAL
# define GC_LARGE_ALLOC_WARN_INTERVAL 5
#endif
GC_INNER long GC_large_alloc_warn_interval = GC_LARGE_ALLOC_WARN_INTERVAL;
                        /* Interval between unsuppressed warnings.      */

STATIC void * GC_CALLBACK GC_default_oom_fn(size_t bytes_requested)
{
    UNUSED_ARG(bytes_requested);
    return NULL;
}

/* All accesses to it should be synchronized to avoid data race.    */
GC_oom_func GC_oom_fn = GC_default_oom_fn;

#ifdef CAN_HANDLE_FORK
# ifdef HANDLE_FORK
    GC_INNER int GC_handle_fork = 1;
                        /* The value is examined by GC_thr_init.        */
# else
    GC_INNER int GC_handle_fork = FALSE;
# endif

#elif !defined(HAVE_NO_FORK)
  GC_API void GC_CALL GC_atfork_prepare(void)
  {
#   ifdef THREADS
      ABORT("fork() handling unsupported");
#   endif
  }

  GC_API void GC_CALL GC_atfork_parent(void)
  {
    /* empty */
  }

  GC_API void GC_CALL GC_atfork_child(void)
  {
    /* empty */
  }
#endif /* !CAN_HANDLE_FORK && !HAVE_NO_FORK */

/* Overrides the default automatic handle-fork mode.  Has effect only   */
/* if called before GC_INIT.                                            */
GC_API void GC_CALL GC_set_handle_fork(int value)
{
# ifdef CAN_HANDLE_FORK
    if (!GC_is_initialized)
      GC_handle_fork = value >= -1 ? value : 1;
                /* Map all negative values except for -1 to a positive one. */
# elif defined(THREADS) || (defined(DARWIN) && defined(MPROTECT_VDB))
    if (!GC_is_initialized && value) {
#     ifndef SMALL_CONFIG
        GC_init(); /* to initialize GC_manual_vdb and GC_stderr */
#       ifndef THREADS
          if (GC_manual_vdb)
            return;
#       endif
#     endif
      ABORT("fork() handling unsupported");
    }
# else
    /* No at-fork handler is needed in the single-threaded mode.        */
    UNUSED_ARG(value);
# endif
}

/* Set things up so that GC_size_map[i] >= granules(i),                 */
/* but not too much bigger                                              */
/* and so that size_map contains relatively few distinct entries        */
/* This was originally stolen from Russ Atkinson's Cedar                */
/* quantization algorithm (but we precompute it).                       */
STATIC void GC_init_size_map(void)
{
    size_t i = 1;

    /* Map size 0 to something bigger.                  */
    /* This avoids problems at lower levels.            */
      GC_size_map[0] = 1;

    for (; i <= GRANULES_TO_BYTES(GC_TINY_FREELISTS-1) - EXTRA_BYTES; i++) {
        GC_size_map[i] = ALLOC_REQUEST_GRANS(i);
#       ifndef _MSC_VER
          GC_ASSERT(GC_size_map[i] < GC_TINY_FREELISTS);
          /* Seems to tickle bug in VC++ 2008 for x64 */
#       endif
    }
    /* We leave the rest of the array to be filled in on demand. */
}

/*
 * The following is a gross hack to deal with a problem that can occur
 * on machines that are sloppy about stack frame sizes, notably SPARC.
 * Bogus pointers may be written to the stack and not cleared for
 * a LONG time, because they always fall into holes in stack frames
 * that are not written.  We partially address this by clearing
 * sections of the stack whenever we get control.
 */

#ifndef SMALL_CLEAR_SIZE
# define SMALL_CLEAR_SIZE 256   /* Clear this much every time.  */
#endif

#if defined(ALWAYS_SMALL_CLEAR_STACK) || defined(STACK_NOT_SCANNED)
  GC_API void * GC_CALL GC_clear_stack(void *arg)
  {
#   ifndef STACK_NOT_SCANNED
      volatile ptr_t dummy[SMALL_CLEAR_SIZE];

      BZERO(CAST_AWAY_VOLATILE_PVOID(dummy), sizeof(dummy));
#   endif
    return arg;
  }
#else

# ifdef THREADS
#   define BIG_CLEAR_SIZE 2048  /* Clear this much now and then.        */
# else
    STATIC word GC_stack_last_cleared = 0;
                        /* GC_gc_no value when we last did this.        */
    STATIC word GC_bytes_allocd_at_reset = 0;
    STATIC ptr_t GC_min_sp = NULL;
                        /* Coolest stack pointer value from which       */
                        /* we've already cleared the stack.             */
    STATIC ptr_t GC_high_water = NULL;
                        /* "hottest" stack pointer value we have seen   */
                        /* recently.  Degrades over time.               */
#   define DEGRADE_RATE 50
# endif

# if defined(__APPLE_CC__) && !GC_CLANG_PREREQ(6, 0)
#   define CLEARSTACK_LIMIT_MODIFIER volatile /* to workaround some bug */
# else
#   define CLEARSTACK_LIMIT_MODIFIER /* empty */
# endif

  EXTERN_C_BEGIN
  void *GC_clear_stack_inner(void *, CLEARSTACK_LIMIT_MODIFIER ptr_t);
  EXTERN_C_END

# ifndef ASM_CLEAR_CODE
    /* Clear the stack up to about limit.  Return arg.  This function   */
    /* is not static because it could also be erroneously defined in .S */
    /* file, so this error would be caught by the linker.               */
    void *GC_clear_stack_inner(void *arg,
                               CLEARSTACK_LIMIT_MODIFIER ptr_t limit)
    {
#     define CLEAR_SIZE 213 /* granularity */
      volatile ptr_t dummy[CLEAR_SIZE];

      BZERO(CAST_AWAY_VOLATILE_PVOID(dummy), sizeof(dummy));
      if (HOTTER_THAN((/* no volatile */ ptr_t)limit, GC_approx_sp())) {
        (void)GC_clear_stack_inner(arg, limit);
      }
      /* Make sure the recursive call is not a tail call, and the bzero */
      /* call is not recognized as dead code.                           */
#     if defined(CPPCHECK)
        GC_noop1(ADDR(dummy[0]));
#     else
        GC_noop1(COVERT_DATAFLOW(ADDR(dummy)));
#     endif
      return arg;
    }
# endif /* !ASM_CLEAR_CODE */

# ifdef THREADS
    /* Used to occasionally clear a bigger chunk.       */
    /* TODO: Should be more random than it is ...       */
    static unsigned next_random_no(void)
    {
#     ifdef AO_HAVE_fetch_and_add1
        static volatile AO_t random_no;

        return (unsigned)AO_fetch_and_add1(&random_no) % 13;
#     else
        static unsigned random_no = 0;

        return (random_no++) % 13;
#     endif
    }
# endif /* THREADS */

/* Clear some of the inaccessible part of the stack.  Returns its       */
/* argument, so it can be used in a tail call position, hence clearing  */
/* another frame.                                                       */
  GC_API void * GC_CALL GC_clear_stack(void *arg)
  {
    ptr_t sp = GC_approx_sp();  /* Hotter than actual sp */
#   ifdef THREADS
        volatile ptr_t dummy[SMALL_CLEAR_SIZE];
#   endif

#   define SLOP 400
        /* Extra bytes we clear every time.  This clears our own        */
        /* activation record, and should cause more frequent            */
        /* clearing near the cold end of the stack, a good thing.       */

#   define GC_SLOP 4000
        /* We make GC_high_water this much hotter than we really saw    */
        /* it, to cover for GC noise etc. above our current frame.      */

#   define CLEAR_THRESHOLD 100000
        /* We restart the clearing process after this many bytes of     */
        /* allocation.  Otherwise very heavily recursive programs       */
        /* with sparse stacks may result in heaps that grow almost      */
        /* without bounds.  As the heap gets larger, collection         */
        /* frequency decreases, thus clearing frequency would decrease, */
        /* thus more junk remains accessible, thus the heap gets        */
        /* larger ...                                                   */

#   ifdef THREADS
      if (next_random_no() == 0) {
        ptr_t limit = sp;

        MAKE_HOTTER(limit, BIG_CLEAR_SIZE * sizeof(ptr_t));
        limit = PTR_ALIGN_DOWN(limit, 0x10);
                        /* Make it sufficiently aligned for assembly    */
                        /* implementations of GC_clear_stack_inner.     */
        return GC_clear_stack_inner(arg, limit);
      }
      BZERO(CAST_AWAY_VOLATILE_PVOID(dummy), sizeof(dummy));
#   else
      if (GC_gc_no != GC_stack_last_cleared) {
        /* Start things over, so we clear the entire stack again.   */
        if (EXPECT(NULL == GC_high_water, FALSE))
          GC_high_water = (ptr_t)GC_stackbottom;
        GC_min_sp = GC_high_water;
        GC_stack_last_cleared = GC_gc_no;
        GC_bytes_allocd_at_reset = GC_bytes_allocd;
      }
      /* Adjust GC_high_water.  */
      GC_ASSERT(GC_high_water != NULL);
      MAKE_COOLER(GC_high_water, PTRS_TO_BYTES(DEGRADE_RATE) + GC_SLOP);
      if (HOTTER_THAN(sp, GC_high_water))
          GC_high_water = sp;
      MAKE_HOTTER(GC_high_water, GC_SLOP);
      {
        ptr_t limit = GC_min_sp;

        MAKE_HOTTER(limit, SLOP);
        if (HOTTER_THAN(limit, sp)) {
          limit = PTR_ALIGN_DOWN(limit, 0x10);
          GC_min_sp = sp;
          return GC_clear_stack_inner(arg, limit);
        }
      }
      if (GC_bytes_allocd - GC_bytes_allocd_at_reset > CLEAR_THRESHOLD) {
        /* Restart clearing process, but limit how much clearing we do. */
        GC_min_sp = sp;
        MAKE_HOTTER(GC_min_sp, CLEAR_THRESHOLD/4);
        if (HOTTER_THAN(GC_min_sp, GC_high_water))
          GC_min_sp = GC_high_water;
        GC_bytes_allocd_at_reset = GC_bytes_allocd;
      }
#   endif
    return arg;
  }

#endif /* !ALWAYS_SMALL_CLEAR_STACK && !STACK_NOT_SCANNED */

/* Return a pointer to the base address of p, given a pointer to a      */
/* an address within an object.  Return 0 otherwise.                    */
GC_API void * GC_CALL GC_base(void * p)
{
    ptr_t r = (ptr_t)p;
    struct hblk *h;
    bottom_index *bi;
    hdr *hhdr;
    ptr_t limit;
    size_t sz;

    if (!EXPECT(GC_is_initialized, TRUE)) return NULL;
    h = HBLKPTR(r);
    GET_BI(r, bi);
    hhdr = HDR_FROM_BI(bi, r);
    if (NULL == hhdr) return NULL;

    /* If it's a pointer to the middle of a large object, move it       */
    /* to the beginning.                                                */
    if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
      h = GC_find_starting_hblk(h, &hhdr);
      r = (ptr_t)h;
    }
    if (HBLK_IS_FREE(hhdr)) return NULL;

    /* Make sure r points to the beginning of the object.       */
    r = PTR_ALIGN_DOWN(r, sizeof(ptr_t));

    sz = hhdr -> hb_sz;
    r -= HBLKDISPL(r) % sz;
    limit = r + sz;
    if ((ADDR_LT((ptr_t)(h + 1), limit) && sz <= HBLKSIZE)
        || ADDR_GE((ptr_t)p, limit))
      return NULL;

    return r;
}

/* Return TRUE if and only if p points to somewhere in GC heap. */
GC_API int GC_CALL GC_is_heap_ptr(const void *p)
{
    bottom_index *bi;

    GC_ASSERT(GC_is_initialized);
    GET_BI(p, bi);
    return HDR_FROM_BI(bi, p) != 0;
}

GC_API size_t GC_CALL GC_size(const void * p)
{
    const hdr *hhdr;

    /* Accept NULL for compatibility with malloc_usable_size(). */
    if (EXPECT(NULL == p, FALSE)) return 0;

    hhdr = HDR(p);
    return hhdr -> hb_sz;
}

/* These getters remain unsynchronized for compatibility (since some    */
/* clients could call some of them from a GC callback holding the       */
/* allocator lock).                                                     */

GC_API size_t GC_CALL GC_get_heap_size(void)
{
    /* ignore the memory space returned to OS (i.e. count only the      */
    /* space owned by the garbage collector)                            */
    return (size_t)(GC_heapsize - GC_unmapped_bytes);
}

GC_API size_t GC_CALL GC_get_obtained_from_os_bytes(void)
{
    return (size_t)GC_our_mem_bytes;
}

GC_API size_t GC_CALL GC_get_free_bytes(void)
{
    /* ignore the memory space returned to OS */
    return (size_t)(GC_large_free_bytes - GC_unmapped_bytes);
}

GC_API size_t GC_CALL GC_get_unmapped_bytes(void)
{
    return (size_t)GC_unmapped_bytes;
}

GC_API size_t GC_CALL GC_get_bytes_since_gc(void)
{
    return (size_t)GC_bytes_allocd;
}

GC_API size_t GC_CALL GC_get_total_bytes(void)
{
    return (size_t)(GC_bytes_allocd + GC_bytes_allocd_before_gc);
}

#ifndef GC_GET_HEAP_USAGE_NOT_NEEDED

GC_API size_t GC_CALL GC_get_size_map_at(int i)
{
  if ((unsigned)i > MAXOBJBYTES)
    return GC_SIZE_MAX;
  return GRANULES_TO_BYTES(GC_size_map[i]);
}

/* Return the heap usage information.  This is a thread-safe (atomic)   */
/* alternative for the five above getters.  NULL pointer is allowed for */
/* any argument.  Returned (filled in) values are of word type.         */
GC_API void GC_CALL GC_get_heap_usage_safe(GC_word *pheap_size,
                        GC_word *pfree_bytes, GC_word *punmapped_bytes,
                        GC_word *pbytes_since_gc, GC_word *ptotal_bytes)
{
  READER_LOCK();
  if (pheap_size != NULL)
    *pheap_size = GC_heapsize - GC_unmapped_bytes;
  if (pfree_bytes != NULL)
    *pfree_bytes = GC_large_free_bytes - GC_unmapped_bytes;
  if (punmapped_bytes != NULL)
    *punmapped_bytes = GC_unmapped_bytes;
  if (pbytes_since_gc != NULL)
    *pbytes_since_gc = GC_bytes_allocd;
  if (ptotal_bytes != NULL)
    *ptotal_bytes = GC_bytes_allocd + GC_bytes_allocd_before_gc;
  READER_UNLOCK();
}

  GC_INNER word GC_reclaimed_bytes_before_gc = 0;

  /* Fill in GC statistics provided the destination is of enough size.  */
  static void fill_prof_stats(struct GC_prof_stats_s *pstats)
  {
    pstats->heapsize_full = GC_heapsize;
    pstats->free_bytes_full = GC_large_free_bytes;
    pstats->unmapped_bytes = GC_unmapped_bytes;
    pstats->bytes_allocd_since_gc = GC_bytes_allocd;
    pstats->allocd_bytes_before_gc = GC_bytes_allocd_before_gc;
    pstats->non_gc_bytes = GC_non_gc_bytes;
    pstats->gc_no = GC_gc_no; /* could be -1 */
#   ifdef PARALLEL_MARK
      pstats->markers_m1 = (word)((signed_word)GC_markers_m1);
#   else
      pstats->markers_m1 = 0; /* one marker */
#   endif
    pstats->bytes_reclaimed_since_gc = GC_bytes_found > 0 ?
                                        (word)GC_bytes_found : 0;
    pstats->reclaimed_bytes_before_gc = GC_reclaimed_bytes_before_gc;
    pstats->expl_freed_bytes_since_gc = GC_bytes_freed; /* since gc-7.7 */
    pstats->obtained_from_os_bytes = GC_our_mem_bytes; /* since gc-8.2 */
  }

# include <string.h> /* for memset() */

  GC_API size_t GC_CALL GC_get_prof_stats(struct GC_prof_stats_s *pstats,
                                          size_t stats_sz)
  {
    struct GC_prof_stats_s stats;

    READER_LOCK();
    fill_prof_stats(stats_sz >= sizeof(stats) ? pstats : &stats);
    READER_UNLOCK();

    if (stats_sz == sizeof(stats)) {
      return sizeof(stats);
    } else if (stats_sz > sizeof(stats)) {
      /* Fill in the remaining part with -1.    */
      memset((char *)pstats + sizeof(stats), 0xff, stats_sz - sizeof(stats));
      return sizeof(stats);
    } else {
      if (EXPECT(stats_sz > 0, TRUE))
        BCOPY(&stats, pstats, stats_sz);
      return stats_sz;
    }
  }

# ifdef THREADS
    /* The _unsafe version assumes the caller holds the allocator lock, */
    /* at least in the reader mode.                                     */
    GC_API size_t GC_CALL GC_get_prof_stats_unsafe(
                                            struct GC_prof_stats_s *pstats,
                                            size_t stats_sz)
    {
      struct GC_prof_stats_s stats;

      if (stats_sz >= sizeof(stats)) {
        fill_prof_stats(pstats);
        if (stats_sz > sizeof(stats))
          memset((char *)pstats + sizeof(stats), 0xff,
                 stats_sz - sizeof(stats));
        return sizeof(stats);
      } else {
        if (EXPECT(stats_sz > 0, TRUE)) {
          fill_prof_stats(&stats);
          BCOPY(&stats, pstats, stats_sz);
        }
        return stats_sz;
      }
    }
# endif /* THREADS */

#endif /* !GC_GET_HEAP_USAGE_NOT_NEEDED */

#if defined(THREADS) && !defined(SIGNAL_BASED_STOP_WORLD)
  /* GC does not use signals to suspend and restart threads.    */
  GC_API void GC_CALL GC_set_suspend_signal(int sig)
  {
    UNUSED_ARG(sig);
  }

  GC_API void GC_CALL GC_set_thr_restart_signal(int sig)
  {
    UNUSED_ARG(sig);
  }

  GC_API int GC_CALL GC_get_suspend_signal(void)
  {
    return -1;
  }

  GC_API int GC_CALL GC_get_thr_restart_signal(void)
  {
    return -1;
  }
#endif /* THREADS && !SIGNAL_BASED_STOP_WORLD */

#if !defined(_MAX_PATH) && defined(ANY_MSWIN)
# define _MAX_PATH MAX_PATH
#endif

#ifdef GC_READ_ENV_FILE
  /* This works for Win32/WinCE for now.  Really useful only for WinCE. */
  STATIC char *GC_envfile_content = NULL;
                        /* The content of the GC "env" file with CR and */
                        /* LF replaced to '\0'.  NULL if the file is    */
                        /* missing or empty.  Otherwise, always ends    */
                        /* with '\0'.                                   */
  STATIC unsigned GC_envfile_length = 0;
                        /* Length of GC_envfile_content (if non-NULL).  */

# ifndef GC_ENVFILE_MAXLEN
#   define GC_ENVFILE_MAXLEN 0x4000
# endif

# define GC_ENV_FILE_EXT ".gc.env"

  /* The routine initializes GC_envfile_content from the GC "env" file. */
  STATIC void GC_envfile_init(void)
  {
#   ifdef ANY_MSWIN
      HANDLE hFile;
      char *content;
      unsigned ofs;
      unsigned len;
      DWORD nBytesRead;
      TCHAR path[_MAX_PATH + 0x10]; /* buffer for path + ext */
      size_t bytes_to_get;

      GC_ASSERT(I_HOLD_LOCK());
      len = (unsigned)GetModuleFileName(NULL /* hModule */, path,
                                        _MAX_PATH + 1);
      /* If GetModuleFileName() has failed then len is 0. */
      if (len > 4 && path[len - 4] == (TCHAR)'.') {
        len -= 4; /* strip executable file extension */
      }
      BCOPY(TEXT(GC_ENV_FILE_EXT), &path[len], sizeof(TEXT(GC_ENV_FILE_EXT)));
      hFile = CreateFile(path, GENERIC_READ,
                         FILE_SHARE_READ | FILE_SHARE_WRITE,
                         NULL /* lpSecurityAttributes */, OPEN_EXISTING,
                         FILE_ATTRIBUTE_NORMAL, NULL /* hTemplateFile */);
      if (hFile == INVALID_HANDLE_VALUE)
        return; /* the file is absent or the operation is failed */
      len = (unsigned)GetFileSize(hFile, NULL);
      if (len <= 1 || len >= GC_ENVFILE_MAXLEN) {
        CloseHandle(hFile);
        return; /* invalid file length - ignoring the file content */
      }
      /* At this execution point, GC_setpagesize() and GC_init_win32()  */
      /* must already be called (for GET_MEM() to work correctly).      */
      GC_ASSERT(GC_page_size != 0);
      bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP((size_t)len + 1);
      content = GC_os_get_mem(bytes_to_get);
      if (content == NULL) {
        CloseHandle(hFile);
        return; /* allocation failure */
      }
      ofs = 0;
      nBytesRead = (DWORD)-1L;
          /* Last ReadFile() call should clear nBytesRead on success. */
      while (ReadFile(hFile, content + ofs, len - ofs + 1, &nBytesRead,
                      NULL /* lpOverlapped */) && nBytesRead != 0) {
        if ((ofs += nBytesRead) > len)
          break;
      }
      CloseHandle(hFile);
      if (ofs != len || nBytesRead != 0) {
        /* TODO: recycle content */
        return; /* read operation is failed - ignoring the file content */
      }
      content[ofs] = '\0';
      while (ofs-- > 0) {
       if (content[ofs] == '\r' || content[ofs] == '\n')
         content[ofs] = '\0';
      }
      GC_ASSERT(NULL == GC_envfile_content);
      GC_envfile_length = len + 1;
      GC_envfile_content = content;
#   endif
  }

  /* This routine scans GC_envfile_content for the specified            */
  /* environment variable (and returns its value if found).             */
  GC_INNER char * GC_envfile_getenv(const char *name)
  {
    char *p;
    const char *end_of_content;
    size_t namelen;

#   ifndef NO_GETENV
      p = getenv(name); /* try the standard getenv() first */
      if (p != NULL)
        return *p != '\0' ? p : NULL;
#   endif
    p = GC_envfile_content;
    if (NULL == p)
      return NULL; /* "env" file is absent (or empty) */
    namelen = strlen(name);
    if (0 == namelen) /* a sanity check */
      return NULL;
    for (end_of_content = p + GC_envfile_length;
         ADDR_LT((ptr_t)p, (ptr_t)end_of_content); p += strlen(p) + 1) {
      if (strncmp(p, name, namelen) == 0 && *(p += namelen) == '=') {
        p++; /* the match is found; skip '=' */
        return *p != '\0' ? p : NULL;
      }
      /* If not matching then skip to the next line. */
    }
    GC_ASSERT(p == end_of_content);
    return NULL; /* no match found */
  }
#endif /* GC_READ_ENV_FILE */

GC_INNER GC_bool GC_is_initialized = FALSE;

GC_API int GC_CALL GC_is_init_called(void)
{
  return (int)GC_is_initialized;
}

#if defined(GC_WIN32_THREADS) \
    && ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE))
  GC_INNER CRITICAL_SECTION GC_write_cs;
#endif

#ifndef DONT_USE_ATEXIT
# if !defined(PCR) && !defined(SMALL_CONFIG)
    /* A dedicated variable to avoid a garbage collection on abort.     */
    /* GC_find_leak cannot be used for this purpose as otherwise        */
    /* TSan finds a data race (between GC_default_on_abort and, e.g.,   */
    /* GC_finish_collection).                                           */
    static GC_bool skip_gc_atexit = FALSE;
# else
#   define skip_gc_atexit FALSE
# endif

  STATIC void GC_exit_check(void)
  {
    if (GC_find_leak && !skip_gc_atexit) {
#     ifdef THREADS
        /* Check that the thread executing at-exit functions is     */
        /* the same as the one performed the GC initialization,     */
        /* otherwise the latter thread might already be dead but    */
        /* still registered and this, as a consequence, might       */
        /* cause a signal delivery fail when suspending the threads */
        /* on platforms that do not guarantee ESRCH returned if     */
        /* the signal is not delivered.                             */
        /* It should also prevent "Collecting from unknown thread"  */
        /* abort in GC_push_all_stacks().                           */
        if (!GC_is_main_thread() || !GC_thread_is_registered()) return;
#     endif
      GC_gcollect();
    }
  }
#endif

#if defined(UNIX_LIKE) && !defined(NO_DEBUGGING)
  static void looping_handler(int sig)
  {
    GC_err_printf("Caught signal %d: looping in handler\n", sig);
    for (;;) {
       /* empty */
    }
  }

  static GC_bool installed_looping_handler = FALSE;

  static void maybe_install_looping_handler(void)
  {
    /* Install looping handler before the write fault handler, so we    */
    /* handle write faults correctly.                                   */
    if (!installed_looping_handler && 0 != GETENV("GC_LOOP_ON_ABORT")) {
      GC_set_and_save_fault_handler(looping_handler);
      installed_looping_handler = TRUE;
    }
  }

#else /* !UNIX_LIKE */
# define maybe_install_looping_handler()
#endif

#define GC_DEFAULT_STDERR_FD 2
#ifdef KOS
# define GC_DEFAULT_STDOUT_FD GC_DEFAULT_STDERR_FD
#else
# define GC_DEFAULT_STDOUT_FD 1
#endif

#if !defined(OS2) && !defined(MACOS) && !defined(GC_ANDROID_LOG) \
    && !defined(NN_PLATFORM_CTR) && !defined(NINTENDO_SWITCH) \
    && (!defined(MSWIN32) || defined(CONSOLE_LOG)) && !defined(MSWINCE)
  STATIC int GC_stdout = GC_DEFAULT_STDOUT_FD;
  STATIC int GC_stderr = GC_DEFAULT_STDERR_FD;
  STATIC int GC_log = GC_DEFAULT_STDERR_FD;

# ifndef MSWIN32
    GC_API void GC_CALL GC_set_log_fd(int fd)
    {
      GC_log = fd;
    }
# endif
#endif

#ifdef MSGBOX_ON_ERROR
  STATIC void GC_win32_MessageBoxA(const char *msg, const char *caption,
                                   unsigned flags)
  {
#   ifndef DONT_USE_USER32_DLL
      /* Use static binding to "user32.dll".    */
      (void)MessageBoxA(NULL, msg, caption, flags);
#   else
      /* This simplifies linking - resolve "MessageBoxA" at run-time. */
      HINSTANCE hU32 = LoadLibrary(TEXT("user32.dll"));
      if (hU32) {
        FARPROC pfn = GetProcAddress(hU32, "MessageBoxA");
        if (pfn)
          (void)(*(int (WINAPI *)(HWND, LPCSTR, LPCSTR, UINT))
                 (GC_funcptr_uint)pfn)(NULL /* hWnd */, msg, caption, flags);
        (void)FreeLibrary(hU32);
      }
#   endif
  }
#endif /* MSGBOX_ON_ERROR */

#if defined(THREADS) && defined(UNIX_LIKE) && !defined(NO_GETCONTEXT)
  static void callee_saves_pushed_dummy_fn(ptr_t data, void *context)
  {
    UNUSED_ARG(data);
    UNUSED_ARG(context);
  }
#endif

#ifdef MANUAL_VDB
    static GC_bool manual_vdb_allowed = TRUE;
#else
    static GC_bool manual_vdb_allowed = FALSE;
#endif

GC_API void GC_CALL GC_set_manual_vdb_allowed(int value)
{
    manual_vdb_allowed = (GC_bool)value;
}

GC_API int GC_CALL GC_get_manual_vdb_allowed(void)
{
    return (int)manual_vdb_allowed;
}

GC_API unsigned GC_CALL GC_get_supported_vdbs(void)
{
# ifdef GC_DISABLE_INCREMENTAL
    return GC_VDB_NONE;
# else
#   if defined(CPPCHECK)
      volatile unsigned zero = 0;
      return zero /* workaround a warning of redundant "or 0" */
#   else
      return 0
#   endif
#     ifndef NO_MANUAL_VDB
        | GC_VDB_MANUAL
#     endif
#     ifdef DEFAULT_VDB
        | GC_VDB_DEFAULT
#     endif
#     ifdef MPROTECT_VDB
        | GC_VDB_MPROTECT
#     endif
#     ifdef GWW_VDB
        | GC_VDB_GWW
#     endif
#     ifdef PCR_VDB
        | GC_VDB_PCR
#     endif
#     ifdef PROC_VDB
        | GC_VDB_PROC
#     endif
#     ifdef SOFT_VDB
        | GC_VDB_SOFT
#     endif
      ;
# endif
}

#ifndef GC_DISABLE_INCREMENTAL
  static void set_incremental_mode_on(void)
  {
    GC_ASSERT(I_HOLD_LOCK());
#   ifndef NO_MANUAL_VDB
      if (manual_vdb_allowed) {
        GC_manual_vdb = TRUE;
        GC_incremental = TRUE;
      } else
#   endif
    /* else */ {
      /* For GWW_VDB on Win32, this needs to happen before any  */
      /* heap memory is allocated.                              */
      GC_incremental = GC_dirty_init();
    }
  }
#endif /* !GC_DISABLE_INCREMENTAL */

STATIC word GC_parse_mem_size_arg(const char *str)
{
  word result;
  char *endptr;
  char ch;

  if ('\0' == *str) return GC_WORD_MAX; /* bad value */
  result = (word)STRTOULL(str, &endptr, 10);
  ch = *endptr;
  if (ch != '\0') {
    if (*(endptr + 1) != '\0') return GC_WORD_MAX;
    /* Allow k, M or G suffix.  */
    switch (ch) {
    case 'K':
    case 'k':
      result <<= 10;
      break;
# if CPP_WORDSZ >= 32
    case 'M':
    case 'm':
      result <<= 20;
      break;
    case 'G':
    case 'g':
      result <<= 30;
      break;
# endif
    default:
      result = GC_WORD_MAX;
    }
  }
  return result;
}

#define GC_LOG_STD_NAME "gc.log"

GC_API void GC_CALL GC_init(void)
{
    /* LOCK(); -- no longer does anything this early. */
    word initial_heap_sz;
    IF_CANCEL(int cancel_state;)

    if (EXPECT(GC_is_initialized, TRUE)) return;
#   ifdef REDIRECT_MALLOC
      {
        static GC_bool init_started = FALSE;
        if (init_started)
          ABORT("Redirected malloc() called during GC init");
        init_started = TRUE;
      }
#   endif

#   if defined(GC_INITIAL_HEAP_SIZE) && !defined(CPPCHECK)
      initial_heap_sz = GC_INITIAL_HEAP_SIZE;
#   else
      initial_heap_sz = MINHINCR * HBLKSIZE;
#   endif

    DISABLE_CANCEL(cancel_state);
    /* Note that although we are nominally called with the allocator    */
    /* lock held, now it is only really acquired once a second thread   */
    /* is forked.  And the initialization code needs to run before      */
    /* then.  Thus we really don't hold any locks, and can in fact      */
    /* safely initialize them here.                                     */
#   ifdef THREADS
#     ifndef GC_ALWAYS_MULTITHREADED
        GC_ASSERT(!GC_need_to_lock);
#     endif
#     ifdef SN_TARGET_PS3
        {
          pthread_mutexattr_t mattr;

          if (0 != pthread_mutexattr_init(&mattr)) {
            ABORT("pthread_mutexattr_init failed");
          }
          if (0 != pthread_mutex_init(&GC_allocate_ml, &mattr)) {
            ABORT("pthread_mutex_init failed");
          }
          (void)pthread_mutexattr_destroy(&mattr);
        }
#     endif
#   endif /* THREADS */
#   if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS)
#     ifndef SPIN_COUNT
#       define SPIN_COUNT 4000
#     endif
#     ifdef USE_RWLOCK
        /* TODO: probably use SRWLOCK_INIT instead */
        InitializeSRWLock(&GC_allocate_ml);
#     elif defined(MSWINRT_FLAVOR)
        InitializeCriticalSectionAndSpinCount(&GC_allocate_ml, SPIN_COUNT);
#     else
        {
#         ifndef MSWINCE
            FARPROC pfn = 0;
            HMODULE hK32 = GetModuleHandle(TEXT("kernel32.dll"));
            if (hK32)
              pfn = GetProcAddress(hK32,
                                   "InitializeCriticalSectionAndSpinCount");
            if (pfn) {
              (*(BOOL (WINAPI *)(LPCRITICAL_SECTION, DWORD))
               (GC_funcptr_uint)pfn)(&GC_allocate_ml, SPIN_COUNT);
            } else
#         endif /* !MSWINCE */
          /* else */ InitializeCriticalSection(&GC_allocate_ml);

        }
#     endif
#   endif /* GC_WIN32_THREADS && !GC_PTHREADS */
#   if defined(GC_WIN32_THREADS) \
       && ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE))
      InitializeCriticalSection(&GC_write_cs);
#   endif
#   if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)
      LOCK(); /* just to set GC_lock_holder */
#   endif
#   ifdef DYNAMIC_POINTER_MASK
      if (0 == GC_pointer_mask) GC_pointer_mask = GC_WORD_MAX;
#   endif
    GC_setpagesize();
#   ifdef MSWIN32
      GC_init_win32();
#   endif
#   ifdef GC_READ_ENV_FILE
      GC_envfile_init();
#   endif
#   if !defined(NO_CLOCK) || !defined(SMALL_CONFIG)
#     ifdef GC_PRINT_VERBOSE_STATS
        /* This is useful for debugging and profiling on platforms with */
        /* missing getenv() (like WinCE).                               */
        GC_print_stats = VERBOSE;
#     else
        if (0 != GETENV("GC_PRINT_VERBOSE_STATS")) {
          GC_print_stats = VERBOSE;
        } else if (0 != GETENV("GC_PRINT_STATS")) {
          GC_print_stats = 1;
        }
#     endif
#   endif
#   if ((defined(UNIX_LIKE) && !defined(GC_ANDROID_LOG)) \
        || (defined(CONSOLE_LOG) && defined(MSWIN32)) \
        || defined(CYGWIN32) || defined(SYMBIAN)) && !defined(SMALL_CONFIG)
        {
          const char *fname = TRUSTED_STRING(GETENV("GC_LOG_FILE"));
#         ifdef GC_LOG_TO_FILE_ALWAYS
            if (NULL == fname)
              fname = GC_LOG_STD_NAME;
#         else
            if (0 != fname)
#         endif
          {
#           if defined(_MSC_VER)
              int log_d = _open(fname, O_CREAT | O_WRONLY | O_APPEND);
#           else
              int log_d = open(fname, O_CREAT | O_WRONLY | O_APPEND, 0644);
#           endif
            if (log_d < 0) {
              GC_err_printf("Failed to open %s as log file\n", fname);
            } else {
              const char *str;
              GC_log = log_d;
              str = GETENV("GC_ONLY_LOG_TO_FILE");
#             ifdef GC_ONLY_LOG_TO_FILE
                /* The similar environment variable set to "0"  */
                /* overrides the effect of the macro defined.   */
                if (str != NULL && str[0] == '0' && str[1] == '\0')
#             else
                /* Otherwise setting the environment variable   */
                /* to anything other than "0" will prevent from */
                /* redirecting stdout/err to the log file.      */
                if (str == NULL || (str[0] == '0' && str[1] == '\0'))
#             endif
              {
                GC_stdout = log_d;
                GC_stderr = log_d;
              }
            }
          }
        }
#   endif
#   if !defined(NO_DEBUGGING) && !defined(GC_DUMP_REGULARLY)
      if (0 != GETENV("GC_DUMP_REGULARLY")) {
        GC_dump_regularly = TRUE;
      }
#   endif
#   ifdef KEEP_BACK_PTRS
      {
        const char *str = GETENV("GC_BACKTRACES");

        if (str != NULL) {
          GC_backtraces = atol(str);
          if (str[0] == '\0') GC_backtraces = 1;
        }
      }
#   endif
    if (0 != GETENV("GC_FIND_LEAK")) {
      GC_find_leak = 1;
    }
#   ifndef SHORT_DBG_HDRS
      if (0 != GETENV("GC_FINDLEAK_DELAY_FREE")) {
        GC_findleak_delay_free = TRUE;
      }
#   endif
    if (0 != GETENV("GC_ALL_INTERIOR_POINTERS")) {
      GC_all_interior_pointers = 1;
    }
    if (0 != GETENV("GC_DONT_GC")) {
#     if defined(LINT2) \
         && !(defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED))
        GC_disable();
#     else
        GC_dont_gc = 1;
#     endif
    }
    if (0 != GETENV("GC_PRINT_BACK_HEIGHT")) {
      GC_print_back_height = TRUE;
    }
    if (0 != GETENV("GC_NO_BLACKLIST_WARNING")) {
      GC_large_alloc_warn_interval = LONG_MAX;
    }
    {
      const char *str = GETENV("GC_TRACE");

      if (str != NULL) {
#       ifndef ENABLE_TRACE
          WARN("Tracing not enabled: Ignoring GC_TRACE value\n", 0);
#       else
          ptr_t p = MAKE_CPTR(STRTOULL(str, NULL, 16));

          if (ADDR(p) < 0x1000)
              WARN("Unlikely trace address: %p\n", p);
          GC_trace_ptr = p;
#       endif
      }
    }
#   ifdef GC_COLLECT_AT_MALLOC
      {
        const char *str = GETENV("GC_COLLECT_AT_MALLOC");

        if (str != NULL) {
          size_t min_lb = (size_t)STRTOULL(str, NULL, 10);

          if (min_lb > 0)
            GC_dbg_collect_at_malloc_min_lb = min_lb;
        }
      }
#   endif
#   if !defined(GC_DISABLE_INCREMENTAL) && !defined(NO_CLOCK)
      {
        const char *str = GETENV("GC_PAUSE_TIME_TARGET");

        if (str != NULL) {
          long time_limit = atol(str);

          if (time_limit > 0) {
            GC_time_limit = (unsigned long)time_limit;
          }
        }
      }
#   endif
#   ifndef SMALL_CONFIG
      {
        const char *str = GETENV("GC_FULL_FREQUENCY");

        if (str != NULL) {
          int full_freq = atoi(str);

          if (full_freq > 0)
            GC_full_freq = full_freq;
        }
      }
#   endif
    {
      char const *str = GETENV("GC_LARGE_ALLOC_WARN_INTERVAL");

      if (str != NULL) {
        long interval = atol(str);

        if (interval <= 0) {
          WARN("GC_LARGE_ALLOC_WARN_INTERVAL environment variable has"
               " bad value - ignoring\n", 0);
        } else {
          GC_large_alloc_warn_interval = interval;
        }
      }
    }
    {
        const char *str = GETENV("GC_FREE_SPACE_DIVISOR");

        if (str != NULL) {
          int space_divisor = atoi(str);

          if (space_divisor > 0)
            GC_free_space_divisor = (unsigned)space_divisor;
        }
    }
#   ifdef USE_MUNMAP
      {
        const char *str = GETENV("GC_UNMAP_THRESHOLD");

        if (str != NULL) {
          if (str[0] == '0' && str[1] == '\0') {
            /* "0" is used to disable unmapping. */
            GC_unmap_threshold = 0;
          } else {
            int unmap_threshold = atoi(str);

            if (unmap_threshold > 0)
              GC_unmap_threshold = (unsigned)unmap_threshold;
          }
        }
      }
      {
        const char *str = GETENV("GC_FORCE_UNMAP_ON_GCOLLECT");

        if (str != NULL) {
          if (str[0] == '0' && str[1] == '\0') {
            /* "0" is used to turn off the mode. */
            GC_force_unmap_on_gcollect = FALSE;
          } else {
            GC_force_unmap_on_gcollect = TRUE;
          }
        }
      }
      {
        const char *str = GETENV("GC_USE_ENTIRE_HEAP");

        if (str != NULL) {
          if (str[0] == '0' && str[1] == '\0') {
            /* "0" is used to turn off the mode. */
            GC_use_entire_heap = FALSE;
          } else {
            GC_use_entire_heap = TRUE;
          }
        }
      }
#   endif
#   if !defined(NO_DEBUGGING) && !defined(NO_CLOCK)
      GET_TIME(GC_init_time);
#   endif
    maybe_install_looping_handler();
#   if ALIGNMENT > GC_DS_TAGS
      /* Adjust normal object descriptor for extra allocation.  */
      if (EXTRA_BYTES != 0)
        GC_obj_kinds[NORMAL].ok_descriptor
                                = ((~(word)ALIGNMENT) + 1) | GC_DS_LENGTH;
#   endif
    GC_exclude_static_roots_inner(beginGC_arrays, endGC_arrays);
    GC_exclude_static_roots_inner(beginGC_obj_kinds, endGC_obj_kinds);
#   ifdef SEPARATE_GLOBALS
      GC_exclude_static_roots_inner(beginGC_objfreelist, endGC_objfreelist);
      GC_exclude_static_roots_inner(beginGC_aobjfreelist, endGC_aobjfreelist);
#   endif
#   if defined(USE_PROC_FOR_LIBRARIES) && defined(GC_LINUX_THREADS)
        /* TODO: USE_PROC_FOR_LIBRARIES+GC_LINUX_THREADS performs poorly! */
        /* If thread stacks are cached, they tend to be scanned in      */
        /* entirety as part of the root set.  This will grow them to    */
        /* maximum size, and is generally not desirable.                */
#   endif
#   if !defined(THREADS) || defined(GC_PTHREADS) \
        || defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH) \
        || defined(GC_WIN32_THREADS) || defined(GC_SOLARIS_THREADS)
      if (GC_stackbottom == 0) {
        GC_stackbottom = GC_get_main_stack_base();
#       if (defined(LINUX) || defined(HPUX)) && defined(IA64)
          GC_register_stackbottom = GC_get_register_stack_base();
#       endif
      } else {
#       if (defined(LINUX) || defined(HPUX)) && defined(IA64)
          if (GC_register_stackbottom == 0) {
            WARN("GC_register_stackbottom should be set with GC_stackbottom\n",
                 0);
            /* The following may fail, since we may rely on             */
            /* alignment properties that may not hold with a user set   */
            /* GC_stackbottom.                                          */
            GC_register_stackbottom = GC_get_register_stack_base();
          }
#       endif
      }
#   endif
#   if !defined(CPPCHECK)
      GC_STATIC_ASSERT(sizeof(size_t) <= sizeof(ptrdiff_t));
#     ifdef AO_HAVE_store
        /* As of now, hb/mse_descr and hb_marks[i] might be treated */
        /* as words but might be accessed atomically.               */
        GC_STATIC_ASSERT(sizeof(AO_t) == sizeof(word));
#     endif
      GC_STATIC_ASSERT(sizeof(ptrdiff_t) == sizeof(word));
      GC_STATIC_ASSERT(sizeof(signed_word) == sizeof(word));
      GC_STATIC_ASSERT(sizeof(word) * 8 == CPP_WORDSZ);
      GC_STATIC_ASSERT(sizeof(ptr_t) * 8 == CPP_PTRSZ);
      GC_STATIC_ASSERT(sizeof(ptr_t) == sizeof(GC_uintptr_t));
      GC_STATIC_ASSERT(sizeof(GC_oom_func) == sizeof(GC_funcptr_uint));
#     ifdef FUNCPTR_IS_DATAPTR
        GC_STATIC_ASSERT(sizeof(ptr_t) == sizeof(GC_funcptr_uint));
#     endif
#     if !defined(_AUX_SOURCE) || defined(__GNUC__)
        GC_STATIC_ASSERT((word)(-1) > (word)0);
        /* word should be unsigned */
#     endif
      /* We no longer check for ((void*)(-1) > NULL) since all pointers */
      /* are explicitly cast to word in every less/greater comparison.  */
      GC_STATIC_ASSERT((signed_word)(-1) < (signed_word)0);
#   endif
    GC_STATIC_ASSERT(sizeof(struct hblk) == HBLKSIZE);
#   ifndef THREADS
      GC_ASSERT(!HOTTER_THAN(GC_stackbottom, GC_approx_sp()));
#   endif
    GC_init_headers();
#   ifdef SEARCH_FOR_DATA_START
      /* For MPROTECT_VDB, the temporary fault handler should be        */
      /* installed first, before the write fault one in GC_dirty_init.  */
      if (GC_REGISTER_MAIN_STATIC_DATA()) GC_init_linux_data_start();
#   endif
#   ifndef GC_DISABLE_INCREMENTAL
      if (GC_incremental || 0 != GETENV("GC_ENABLE_INCREMENTAL")) {
        set_incremental_mode_on();
        GC_ASSERT(GC_bytes_allocd == 0);
      }
#   endif

    /* Add initial guess of root sets.  Do this first, since sbrk(0)    */
    /* might be used.                                                   */
      if (GC_REGISTER_MAIN_STATIC_DATA()) GC_register_data_segments();

    GC_bl_init();
    GC_mark_init();
    {
        const char *str = GETENV("GC_INITIAL_HEAP_SIZE");

        if (str != NULL) {
          word value = GC_parse_mem_size_arg(str);

          if (GC_WORD_MAX == value) {
            WARN("Bad initial heap size %s - ignoring\n", str);
          } else {
            initial_heap_sz = value;
          }
        }
    }
    {
        const char *str = GETENV("GC_MAXIMUM_HEAP_SIZE");

        if (str != NULL) {
          word max_heap_sz = GC_parse_mem_size_arg(str);

          if (max_heap_sz < initial_heap_sz || GC_WORD_MAX == max_heap_sz) {
            WARN("Bad maximum heap size %s - ignoring\n", str);
          } else {
            if (0 == GC_max_retries) GC_max_retries = 2;
            GC_set_max_heap_size(max_heap_sz);
          }
        }
    }
    if (initial_heap_sz != 0) {
      if (!GC_expand_hp_inner(divHBLKSZ(initial_heap_sz))) {
        GC_err_printf("Can't start up: not enough memory\n");
        EXIT();
      } else {
        GC_requested_heapsize += initial_heap_sz;
      }
    }
    if (GC_all_interior_pointers)
      GC_initialize_offsets();
    GC_register_displacement_inner(0);
#   if defined(GC_LINUX_THREADS) && defined(REDIRECT_MALLOC)
      if (!GC_all_interior_pointers) {
        /* TLS ABI uses pointer-sized offsets for dtv. */
        GC_register_displacement_inner(sizeof(void *));
      }
#   endif
    GC_init_size_map();
#   ifdef PCR
      if (PCR_IL_Lock(PCR_Bool_false, PCR_allSigsBlocked, PCR_waitForever)
          != PCR_ERes_okay) {
          ABORT("Can't lock load state");
      } else if (PCR_IL_Unlock() != PCR_ERes_okay) {
          ABORT("Can't unlock load state");
      }
      PCR_IL_Unlock();
      GC_pcr_install();
#   endif
    GC_is_initialized = TRUE;
#   ifdef THREADS
#       if defined(LINT2) \
           && !(defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED))
          LOCK();
          GC_thr_init();
          UNLOCK();
#       else
          GC_thr_init();
#       endif
#   endif
    COND_DUMP;
    /* Get black list set up and/or incremental GC started */
    if (!GC_dont_precollect || GC_incremental) {
#       if defined(DYNAMIC_LOADING) && defined(DARWIN)
          GC_ASSERT(0 == GC_bytes_allocd);
#       endif
        GC_gcollect_inner();
    }
#   if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)
        UNLOCK();
#   endif
#   if defined(THREADS) && defined(UNIX_LIKE) && !defined(NO_GETCONTEXT)
      /* Ensure getcontext_works is set to avoid potential data race.   */
      if (GC_dont_gc || GC_dont_precollect)
        GC_with_callee_saves_pushed(callee_saves_pushed_dummy_fn, NULL);
#   endif
#   ifndef DONT_USE_ATEXIT
      if (GC_find_leak) {
        /* This is to give us at least one chance to detect leaks.        */
        /* This may report some very benign leaks, but ...                */
        atexit(GC_exit_check);
      }
#   endif

    /* The rest of this again assumes we do not really hold     */
    /* the allocator lock.                                      */

#   ifdef THREADS
      /* Initialize thread-local allocation.    */
      GC_init_parallel();
#   endif

#   if defined(DYNAMIC_LOADING) && defined(DARWIN)
        /* This must be called WITHOUT the allocator lock held  */
        /* and before any threads are created.                  */
        GC_init_dyld();
#   endif
    RESTORE_CANCEL(cancel_state);
    /* It is not safe to allocate any object till completion of GC_init */
    /* (in particular by GC_thr_init), i.e. before GC_init_dyld() call  */
    /* and initialization of the incremental mode (if any).             */
#   if defined(GWW_VDB) && !defined(KEEP_BACK_PTRS)
      GC_ASSERT(GC_bytes_allocd + GC_bytes_allocd_before_gc == 0);
#   endif
}

GC_API void GC_CALL GC_enable_incremental(void)
{
# if !defined(GC_DISABLE_INCREMENTAL) && !defined(KEEP_BACK_PTRS)
    /* If we are keeping back pointers, the GC itself dirties all */
    /* pages on which objects have been marked, making            */
    /* incremental GC pointless.                                  */
    if (!GC_find_leak && 0 == GETENV("GC_DISABLE_INCREMENTAL")) {
      LOCK();
      if (!GC_incremental) {
        GC_setpagesize();
        /* TODO: Should we skip enabling incremental if win32s? */
        maybe_install_looping_handler(); /* Before write fault handler! */
        if (!GC_is_initialized) {
          GC_incremental = TRUE; /* indicate intention to turn it on */
          UNLOCK();
          GC_init();
          LOCK();
        } else {
          set_incremental_mode_on();
        }
        if (GC_incremental && !GC_dont_gc) {
                                /* Can't easily do it if GC_dont_gc.    */
          IF_CANCEL(int cancel_state;)

          DISABLE_CANCEL(cancel_state);
          if (GC_bytes_allocd > 0) {
            /* There may be unmarked reachable objects. */
            GC_gcollect_inner();
          } else {
            /* We are OK in assuming everything is      */
            /* clean since nothing can point to an      */
            /* unmarked object.                         */
#           ifdef CHECKSUMS
              GC_read_dirty(FALSE);
#           else
              GC_read_dirty(TRUE);
#           endif
          }
          RESTORE_CANCEL(cancel_state);
        }
      }
      UNLOCK();
      return;
    }
# endif
  GC_init();
}

GC_API void GC_CALL GC_start_mark_threads(void)
{
#   ifdef PARALLEL_MARK
      IF_CANCEL(int cancel_state;)

      DISABLE_CANCEL(cancel_state);
      LOCK();
      GC_start_mark_threads_inner();
      UNLOCK();
      RESTORE_CANCEL(cancel_state);
#   else
      /* No action since parallel markers are disabled (or no POSIX fork). */
      GC_ASSERT(I_DONT_HOLD_LOCK());
#   endif
}

  GC_API void GC_CALL GC_deinit(void)
  {
    if (GC_is_initialized) {
      /* Prevent duplicate resource close.  */
      GC_is_initialized = FALSE;
      GC_bytes_allocd = 0;
      GC_bytes_allocd_before_gc = 0;
#     if defined(GC_WIN32_THREADS) && (defined(MSWIN32) || defined(MSWINCE))
#       if !defined(CONSOLE_LOG) || defined(MSWINCE)
          DeleteCriticalSection(&GC_write_cs);
#       endif
#       if !defined(GC_PTHREADS) && !defined(USE_RWLOCK)
          DeleteCriticalSection(&GC_allocate_ml);
#       endif
#     endif
    }
  }

#if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)

  STATIC HANDLE GC_log = 0;

# ifdef THREADS
#   if defined(PARALLEL_MARK) && !defined(GC_ALWAYS_MULTITHREADED)
#     define IF_NEED_TO_LOCK(x) if (GC_parallel || GC_need_to_lock) x
#   else
#     define IF_NEED_TO_LOCK(x) if (GC_need_to_lock) x
#   endif
# else
#   define IF_NEED_TO_LOCK(x)
# endif /* !THREADS */

# ifdef MSWINRT_FLAVOR
#   include <windows.storage.h>

    /* This API is defined in roapi.h, but we cannot include it here    */
    /* since it does not compile in C.                                  */
    DECLSPEC_IMPORT HRESULT WINAPI RoGetActivationFactory(
                                        HSTRING activatableClassId,
                                        REFIID iid, void** factory);

    static GC_bool getWinRTLogPath(wchar_t* buf, size_t bufLen)
    {
      static const GUID kIID_IApplicationDataStatics = {
        0x5612147B, 0xE843, 0x45E3,
        0x94, 0xD8, 0x06, 0x16, 0x9E, 0x3C, 0x8E, 0x17
      };
      static const GUID kIID_IStorageItem = {
        0x4207A996, 0xCA2F, 0x42F7,
        0xBD, 0xE8, 0x8B, 0x10, 0x45, 0x7A, 0x7F, 0x30
      };
      GC_bool result = FALSE;
      HSTRING_HEADER appDataClassNameHeader;
      HSTRING appDataClassName;
      __x_ABI_CWindows_CStorage_CIApplicationDataStatics* appDataStatics = 0;

      GC_ASSERT(bufLen > 0);
      if (SUCCEEDED(WindowsCreateStringReference(
                      RuntimeClass_Windows_Storage_ApplicationData,
                      (sizeof(RuntimeClass_Windows_Storage_ApplicationData)-1)
                        / sizeof(wchar_t),
                      &appDataClassNameHeader, &appDataClassName))
          && SUCCEEDED(RoGetActivationFactory(appDataClassName,
                                              &kIID_IApplicationDataStatics,
                                              &appDataStatics))) {
        __x_ABI_CWindows_CStorage_CIApplicationData* appData = NULL;
        __x_ABI_CWindows_CStorage_CIStorageFolder* tempFolder = NULL;
        __x_ABI_CWindows_CStorage_CIStorageItem* tempFolderItem = NULL;
        HSTRING tempPath = NULL;

        if (SUCCEEDED(appDataStatics->lpVtbl->get_Current(appDataStatics,
                                                          &appData))
            && SUCCEEDED(appData->lpVtbl->get_TemporaryFolder(appData,
                                                              &tempFolder))
            && SUCCEEDED(tempFolder->lpVtbl->QueryInterface(tempFolder,
                                                        &kIID_IStorageItem,
                                                        &tempFolderItem))
            && SUCCEEDED(tempFolderItem->lpVtbl->get_Path(tempFolderItem,
                                                          &tempPath))) {
          UINT32 tempPathLen;
          const wchar_t* tempPathBuf =
                          WindowsGetStringRawBuffer(tempPath, &tempPathLen);

          buf[0] = '\0';
          if (wcsncat_s(buf, bufLen, tempPathBuf, tempPathLen) == 0
              && wcscat_s(buf, bufLen, L"\\") == 0
              && wcscat_s(buf, bufLen, TEXT(GC_LOG_STD_NAME)) == 0)
            result = TRUE;
          WindowsDeleteString(tempPath);
        }

        if (tempFolderItem != NULL)
          tempFolderItem->lpVtbl->Release(tempFolderItem);
        if (tempFolder != NULL)
          tempFolder->lpVtbl->Release(tempFolder);
        if (appData != NULL)
          appData->lpVtbl->Release(appData);
        appDataStatics->lpVtbl->Release(appDataStatics);
      }
      return result;
    }
# endif /* MSWINRT_FLAVOR */

  STATIC HANDLE GC_CreateLogFile(void)
  {
    HANDLE hFile;
# ifdef MSWINRT_FLAVOR
      TCHAR pathBuf[_MAX_PATH + 0x10]; /* buffer for path + ext */

      hFile = INVALID_HANDLE_VALUE;
      if (getWinRTLogPath(pathBuf, _MAX_PATH + 1)) {
        CREATEFILE2_EXTENDED_PARAMETERS extParams;

        BZERO(&extParams, sizeof(extParams));
        extParams.dwSize = sizeof(extParams);
        extParams.dwFileAttributes = FILE_ATTRIBUTE_NORMAL;
        extParams.dwFileFlags = GC_print_stats == VERBOSE ? 0
                                    : FILE_FLAG_WRITE_THROUGH;
        hFile = CreateFile2(pathBuf, GENERIC_WRITE, FILE_SHARE_READ,
                            CREATE_ALWAYS, &extParams);
      }

# else
    TCHAR *logPath;
#   if defined(NO_GETENV_WIN32) && defined(CPPCHECK)
#     define appendToFile FALSE
#   else
      BOOL appendToFile = FALSE;
#   endif
#   if !defined(NO_GETENV_WIN32) || !defined(OLD_WIN32_LOG_FILE)
      TCHAR pathBuf[_MAX_PATH + 0x10]; /* buffer for path + ext */

      logPath = pathBuf;
#   endif

    /* Use GetEnvironmentVariable instead of GETENV() for unicode support. */
#   ifndef NO_GETENV_WIN32
      if (GetEnvironmentVariable(TEXT("GC_LOG_FILE"), pathBuf,
                                 _MAX_PATH + 1) - 1U < (DWORD)_MAX_PATH) {
        appendToFile = TRUE;
      } else
#   endif
    /* else */ {
      /* Env var not found or its value too long.       */
#     ifdef OLD_WIN32_LOG_FILE
        logPath = TEXT(GC_LOG_STD_NAME);
#     else
        int len = (int)GetModuleFileName(NULL /* hModule */, pathBuf,
                                         _MAX_PATH + 1);
        /* If GetModuleFileName() has failed then len is 0. */
        if (len > 4 && pathBuf[len - 4] == (TCHAR)'.') {
          len -= 4; /* strip executable file extension */
        }
        BCOPY(TEXT(".") TEXT(GC_LOG_STD_NAME), &pathBuf[len],
              sizeof(TEXT(".") TEXT(GC_LOG_STD_NAME)));
#     endif
    }

    hFile = CreateFile(logPath, GENERIC_WRITE, FILE_SHARE_READ,
                       NULL /* lpSecurityAttributes */,
                       appendToFile ? OPEN_ALWAYS : CREATE_ALWAYS,
                       GC_print_stats == VERBOSE ? FILE_ATTRIBUTE_NORMAL :
                            /* immediately flush writes unless very verbose */
                            FILE_ATTRIBUTE_NORMAL | FILE_FLAG_WRITE_THROUGH,
                       NULL /* hTemplateFile */);

#   ifndef NO_GETENV_WIN32
      if (appendToFile && hFile != INVALID_HANDLE_VALUE) {
        LONG posHigh = 0;
        (void)SetFilePointer(hFile, 0, &posHigh, FILE_END);
                                  /* Seek to file end (ignoring any error) */
      }
#   endif
#   undef appendToFile
# endif
    return hFile;
  }

  STATIC int GC_write(const char *buf, size_t len)
  {
      BOOL res;
      DWORD written;
#     if defined(THREADS) && defined(GC_ASSERTIONS)
        static GC_bool inside_write = FALSE;
                        /* to prevent infinite recursion at abort.      */
        if (inside_write)
          return -1;
#     endif

      if (len == 0)
          return 0;
      IF_NEED_TO_LOCK(EnterCriticalSection(&GC_write_cs));
#     if defined(THREADS) && defined(GC_ASSERTIONS)
        if (GC_write_disabled) {
          inside_write = TRUE;
          ABORT("Assertion failure: GC_write called with write_disabled");
        }
#     endif
      if (GC_log == 0) {
        GC_log = GC_CreateLogFile();
      }
      if (GC_log == INVALID_HANDLE_VALUE) {
        IF_NEED_TO_LOCK(LeaveCriticalSection(&GC_write_cs));
#       ifdef NO_DEBUGGING
          /* Ignore open log failure (e.g., it might be caused by       */
          /* read-only folder of the client application).               */
          return 0;
#       else
          return -1;
#       endif
      }
      res = WriteFile(GC_log, buf, (DWORD)len, &written, NULL);
#     if defined(_MSC_VER) && defined(_DEBUG) && !defined(NO_CRT) \
         && !defined(NO_CRTDBGREPORT)
#         ifdef MSWINCE
              /* There is no CrtDbgReport() in WinCE */
              {
                  WCHAR wbuf[1024];
                  /* Always use Unicode variant of OutputDebugString() */
                  wbuf[MultiByteToWideChar(CP_ACP, 0 /* dwFlags */,
                                buf, len, wbuf,
                                sizeof(wbuf) / sizeof(wbuf[0]) - 1)] = 0;
                  OutputDebugStringW(wbuf);
              }
#         else
              _CrtDbgReport(_CRT_WARN, NULL, 0, NULL, "%.*s", len, buf);
#         endif
#     endif
      IF_NEED_TO_LOCK(LeaveCriticalSection(&GC_write_cs));
      return res ? (int)written : -1;
  }

  /* TODO: This is pretty ugly ... */
# define WRITE(f, buf, len) GC_write(buf, len)

#elif defined(OS2) || defined(MACOS)
  STATIC FILE * GC_stdout = NULL;
  STATIC FILE * GC_stderr = NULL;
  STATIC FILE * GC_log = NULL;

  /* Initialize GC_log (and the friends) passed to GC_write().  */
  STATIC void GC_set_files(void)
  {
    if (GC_stdout == NULL) {
      GC_stdout = stdout;
    }
    if (GC_stderr == NULL) {
      GC_stderr = stderr;
    }
    if (GC_log == NULL) {
      GC_log = stderr;
    }
  }

  GC_INLINE int GC_write(FILE *f, const char *buf, size_t len)
  {
    int res = fwrite(buf, 1, len, f);
    fflush(f);
    return res;
  }

# define WRITE(f, buf, len) (GC_set_files(), GC_write(f, buf, len))

#elif defined(GC_ANDROID_LOG)

# include <android/log.h>

# ifndef GC_ANDROID_LOG_TAG
#   define GC_ANDROID_LOG_TAG "BDWGC"
# endif

# define GC_stdout ANDROID_LOG_DEBUG
# define GC_stderr ANDROID_LOG_ERROR
# define GC_log GC_stdout

# define WRITE(level, buf, unused_len) \
                __android_log_write(level, GC_ANDROID_LOG_TAG, buf)

#elif defined(NN_PLATFORM_CTR)
  int n3ds_log_write(const char* text, int length);
# define WRITE(level, buf, len) n3ds_log_write(buf, len)

#elif defined(NINTENDO_SWITCH)
  int switch_log_write(const char* text, int length);
# define WRITE(level, buf, len) switch_log_write(buf, len)

#else

# if !defined(ECOS) && !defined(NOSYS) && !defined(PLATFORM_WRITE) \
     && !defined(SN_TARGET_PSP2)
#   include <errno.h>
# endif

  STATIC int GC_write(int fd, const char *buf, size_t len)
  {
#   if defined(ECOS) || defined(PLATFORM_WRITE) || defined(SN_TARGET_PSP2) \
       || defined(NOSYS)
#     ifdef ECOS
        /* FIXME: This seems to be defined nowhere at present.  */
        /* _Jv_diag_write(buf, len); */
#     else
        /* No writing.  */
#     endif
      return (int)len;
#   else
      int bytes_written = 0;
      IF_CANCEL(int cancel_state;)

      DISABLE_CANCEL(cancel_state);
      while ((unsigned)bytes_written < len) {
#        ifdef GC_SOLARIS_THREADS
             int result = syscall(SYS_write, fd, buf + bytes_written,
                                             len - bytes_written);
#        elif defined(_MSC_VER)
             int result = _write(fd, buf + bytes_written,
                                 (unsigned)(len - bytes_written));
#        else
             int result = (int)write(fd, buf + bytes_written,
                                     len - (size_t)bytes_written);
#        endif

         if (-1 == result) {
             if (EAGAIN == errno) /* Resource temporarily unavailable */
               continue;
             RESTORE_CANCEL(cancel_state);
             return result;
         }
         bytes_written += result;
      }
      RESTORE_CANCEL(cancel_state);
      return bytes_written;
#   endif
  }

# define WRITE(f, buf, len) GC_write(f, buf, len)
#endif /* !MSWINCE && !OS2 && !MACOS && !GC_ANDROID_LOG */

#define BUFSZ 1024

#if defined(DJGPP) || defined(__STRICT_ANSI__)
  /* vsnprintf is missing in DJGPP (v2.0.3) */
# define GC_VSNPRINTF(buf, bufsz, format, args) vsprintf(buf, format, args)
#elif defined(_MSC_VER)
# ifdef MSWINCE
    /* _vsnprintf is deprecated in WinCE */
#   define GC_VSNPRINTF StringCchVPrintfA
# else
#   define GC_VSNPRINTF _vsnprintf
# endif
#else
# define GC_VSNPRINTF vsnprintf
#endif

/* A version of printf that is unlikely to call malloc, and is thus safer */
/* to call from the collector in case malloc has been bound to GC_malloc. */
/* Floating point arguments and formats should be avoided, since FP       */
/* conversion is more likely to allocate memory.                          */
/* Assumes that no more than BUFSZ-1 characters are written at once.      */
#define GC_PRINTF_FILLBUF(buf, format) \
        do { \
          va_list args; \
          va_start(args, format); \
          (buf)[sizeof(buf) - 1] = 0x15; /* guard */ \
          (void)GC_VSNPRINTF(buf, sizeof(buf) - 1, format, args); \
          va_end(args); \
          if ((buf)[sizeof(buf) - 1] != 0x15) \
            ABORT("GC_printf clobbered stack"); \
        } while (0)

void GC_printf(const char *format, ...)
{
    if (!GC_quiet) {
      char buf[BUFSZ + 1];

      GC_PRINTF_FILLBUF(buf, format);
#     ifdef NACL
        (void)WRITE(GC_stdout, buf, strlen(buf));
        /* Ignore errors silently.      */
#     else
        if (WRITE(GC_stdout, buf, strlen(buf)) < 0
#           if defined(CYGWIN32) || (defined(CONSOLE_LOG) && defined(MSWIN32))
              && GC_stdout != GC_DEFAULT_STDOUT_FD
#           endif
           ) {
          ABORT("write to stdout failed");
        }
#     endif
    }
}

void GC_err_printf(const char *format, ...)
{
    char buf[BUFSZ + 1];

    GC_PRINTF_FILLBUF(buf, format);
    GC_err_puts(buf);
}

void GC_log_printf(const char *format, ...)
{
    char buf[BUFSZ + 1];

    GC_PRINTF_FILLBUF(buf, format);
#   ifdef NACL
      (void)WRITE(GC_log, buf, strlen(buf));
#   else
      if (WRITE(GC_log, buf, strlen(buf)) < 0
#         if defined(CYGWIN32) || (defined(CONSOLE_LOG) && defined(MSWIN32))
            && GC_log != GC_DEFAULT_STDERR_FD
#         endif
         ) {
        ABORT("write to GC log failed");
      }
#   endif
}

#ifndef GC_ANDROID_LOG

# define GC_warn_printf GC_err_printf

#else

  GC_INNER void GC_info_log_printf(const char *format, ...)
  {
    char buf[BUFSZ + 1];

    GC_PRINTF_FILLBUF(buf, format);
    (void)WRITE(ANDROID_LOG_INFO, buf, 0 /* unused */);
  }

  GC_INNER void GC_verbose_log_printf(const char *format, ...)
  {
    char buf[BUFSZ + 1];

    GC_PRINTF_FILLBUF(buf, format);
    (void)WRITE(ANDROID_LOG_VERBOSE, buf, 0); /* ignore write errors */
  }

  STATIC void GC_warn_printf(const char *format, ...)
  {
    char buf[BUFSZ + 1];

    GC_PRINTF_FILLBUF(buf, format);
    (void)WRITE(ANDROID_LOG_WARN, buf, 0);
  }

#endif /* GC_ANDROID_LOG */

void GC_err_puts(const char *s)
{
    (void)WRITE(GC_stderr, s, strlen(s)); /* ignore errors */
}

STATIC void GC_CALLBACK GC_default_warn_proc(const char *msg, GC_uintptr_t arg)
{
    /* TODO: Add assertion on arg comply with msg (format).     */
    GC_warn_printf(msg, arg);
}

GC_INNER GC_warn_proc GC_current_warn_proc = GC_default_warn_proc;

/* This is recommended for production code (release). */
GC_API void GC_CALLBACK GC_ignore_warn_proc(const char *msg, GC_uintptr_t arg)
{
    if (GC_print_stats) {
      /* Don't ignore warnings if stats printing is on. */
      GC_default_warn_proc(msg, arg);
    }
}

GC_API void GC_CALL GC_set_warn_proc(GC_warn_proc p)
{
    GC_ASSERT(NONNULL_ARG_NOT_NULL(p));
#   ifdef GC_WIN32_THREADS
#     ifdef CYGWIN32
        /* Need explicit GC_INIT call */
        GC_ASSERT(GC_is_initialized);
#     else
        if (!GC_is_initialized) GC_init();
#     endif
#   endif
    LOCK();
    GC_current_warn_proc = p;
    UNLOCK();
}

GC_API GC_warn_proc GC_CALL GC_get_warn_proc(void)
{
    GC_warn_proc result;

    READER_LOCK();
    result = GC_current_warn_proc;
    READER_UNLOCK();
    return result;
}

/* Print (or display) a message before abnormal exit (including     */
/* abort).  Invoked from ABORT(msg) macro (there msg is non-NULL)   */
/* and from EXIT() macro (msg is NULL in that case).                */
STATIC void GC_CALLBACK GC_default_on_abort(const char *msg)
{
# if !defined(PCR) && !defined(SMALL_CONFIG)
#   ifndef DONT_USE_ATEXIT
      skip_gc_atexit = TRUE; /* disable at-exit GC_gcollect() */
#   endif

    if (msg != NULL) {
#     ifdef MSGBOX_ON_ERROR
        GC_win32_MessageBoxA(msg, "Fatal error in GC", MB_ICONERROR | MB_OK);
        /* Also duplicate msg to GC log file.   */
#     endif

#   ifndef GC_ANDROID_LOG
      /* Avoid calling GC_err_printf() here, as GC_on_abort() could be  */
      /* called from it.  Note 1: this is not an atomic output.         */
      /* Note 2: possible write errors are ignored.                     */
#     if defined(GC_WIN32_THREADS) && defined(GC_ASSERTIONS) \
         && ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE))
        if (!GC_write_disabled)
#     endif
      {
        if (WRITE(GC_stderr, msg, strlen(msg)) >= 0)
          (void)WRITE(GC_stderr, "\n", 1);
      }
#   else
      __android_log_assert("*" /* cond */, GC_ANDROID_LOG_TAG, "%s\n", msg);
#   endif
    }

#   if !defined(NO_DEBUGGING) && !defined(GC_ANDROID_LOG)
      if (GETENV("GC_LOOP_ON_ABORT") != NULL) {
            /* In many cases it's easier to debug a running process.    */
            /* It's arguably nicer to sleep, but that makes it harder   */
            /* to look at the thread if the debugger doesn't know much  */
            /* about threads.                                           */
            for (;;) {
              /* Empty */
            }
      }
#   endif
# else
    UNUSED_ARG(msg);
# endif
}

#if !defined(PCR) && !defined(SMALL_CONFIG)
  GC_abort_func GC_on_abort = GC_default_on_abort;
#endif

GC_API void GC_CALL GC_set_abort_func(GC_abort_func fn)
{
  GC_ASSERT(NONNULL_ARG_NOT_NULL(fn));
  LOCK();
# if !defined(PCR) && !defined(SMALL_CONFIG)
    GC_on_abort = fn;
# else
    UNUSED_ARG(fn);
# endif
  UNLOCK();
}

GC_API GC_abort_func GC_CALL GC_get_abort_func(void)
{
  GC_abort_func fn;

  READER_LOCK();
# if !defined(PCR) && !defined(SMALL_CONFIG)
    fn = GC_on_abort;
    GC_ASSERT(fn != 0);
# else
    fn = GC_default_on_abort;
# endif
  READER_UNLOCK();
  return fn;
}

GC_API void GC_CALL GC_enable(void)
{
    LOCK();
    GC_ASSERT(GC_dont_gc != 0); /* ensure no counter underflow */
    GC_dont_gc--;
    if (!GC_dont_gc && GC_heapsize > GC_heapsize_on_gc_disable)
      WARN("Heap grown by %" WARN_PRIuPTR " KiB while GC was disabled\n",
           (GC_heapsize - GC_heapsize_on_gc_disable) >> 10);
    UNLOCK();
}

GC_API void GC_CALL GC_disable(void)
{
    LOCK();
    if (!GC_dont_gc)
      GC_heapsize_on_gc_disable = GC_heapsize;
    GC_dont_gc++;
    UNLOCK();
}

GC_API int GC_CALL GC_is_disabled(void)
{
    return GC_dont_gc != 0;
}

/* Helper procedures for new kind creation.     */
GC_API void ** GC_CALL GC_new_free_list_inner(void)
{
    void *result;

    GC_ASSERT(I_HOLD_LOCK());
    result = GC_INTERNAL_MALLOC((MAXOBJGRANULES+1) * sizeof(ptr_t), PTRFREE);
    if (NULL == result) ABORT("Failed to allocate free list for new kind");
    BZERO(result, (MAXOBJGRANULES+1) * sizeof(ptr_t));
    return (void **)result;
}

GC_API void ** GC_CALL GC_new_free_list(void)
{
    void ** result;

    LOCK();
    result = GC_new_free_list_inner();
    UNLOCK();
    return result;
}

GC_API unsigned GC_CALL GC_new_kind_inner(void **fl, GC_word descr,
                                          int adjust, int clear)
{
    unsigned result = GC_n_kinds;

    GC_ASSERT(NONNULL_ARG_NOT_NULL(fl));
    GC_ASSERT(adjust == FALSE || adjust == TRUE);
    /* If an object is not needed to be cleared (when moved to the      */
    /* free list) then its descriptor should be zero to denote          */
    /* a pointer-free object (and, as a consequence, the size of the    */
    /* object should not be added to the descriptor template).          */
    GC_ASSERT(clear == TRUE
              || (descr == 0 && adjust == FALSE && clear == FALSE));
    if (result < MAXOBJKINDS) {
      GC_ASSERT(result > 0);
      GC_n_kinds++;
      GC_obj_kinds[result].ok_freelist = fl;
      GC_obj_kinds[result].ok_reclaim_list = 0;
      GC_obj_kinds[result].ok_descriptor = descr;
      GC_obj_kinds[result].ok_relocate_descr = adjust;
      GC_obj_kinds[result].ok_init = (GC_bool)clear;
#     ifdef ENABLE_DISCLAIM
        GC_obj_kinds[result].ok_mark_unconditionally = FALSE;
        GC_obj_kinds[result].ok_disclaim_proc = 0;
#     endif
    } else {
      ABORT("Too many kinds");
    }
    return result;
}

GC_API unsigned GC_CALL GC_new_kind(void **fl, GC_word descr, int adjust,
                                    int clear)
{
    unsigned result;

    LOCK();
    result = GC_new_kind_inner(fl, descr, adjust, clear);
    UNLOCK();
    return result;
}

GC_API unsigned GC_CALL GC_new_proc_inner(GC_mark_proc proc)
{
    unsigned result = GC_n_mark_procs;

    if (result < MAX_MARK_PROCS) {
      GC_n_mark_procs++;
      GC_mark_procs[result] = proc;
    } else {
      ABORT("Too many mark procedures");
    }
    return result;
}

GC_API unsigned GC_CALL GC_new_proc(GC_mark_proc proc)
{
    unsigned result;

    LOCK();
    result = GC_new_proc_inner(proc);
    UNLOCK();
    return result;
}

GC_API void * GC_CALL GC_call_with_alloc_lock(GC_fn_type fn, void *client_data)
{
    void * result;

    LOCK();
    result = fn(client_data);
    UNLOCK();
    return result;
}

#ifdef THREADS
  GC_API void GC_CALL GC_alloc_lock(void)
  {
    LOCK();
  }

  GC_API void GC_CALL GC_alloc_unlock(void)
  {
    UNLOCK();
  }

  GC_API void *GC_CALL GC_call_with_reader_lock(GC_fn_type fn,
                                                void *client_data,
                                                int release)
  {
    void *result;

    READER_LOCK();
    result = fn(client_data);
#   ifdef HAS_REAL_READER_LOCK
      if (release) {
        READER_UNLOCK_RELEASE();
#       ifdef LINT2
          GC_noop1((unsigned)release);
#       endif
        return result;
      }
#   else
      UNUSED_ARG(release);
#   endif
    READER_UNLOCK();
    return result;
  }
#endif /* THREADS */

GC_API void * GC_CALL GC_call_with_stack_base(GC_stack_base_func fn, void *arg)
{
    struct GC_stack_base base;
    void *result;

    base.mem_base = &base;
#   ifdef IA64
      base.reg_base = GC_save_regs_in_stack();
      /* TODO: Unnecessarily flushes register stack,    */
      /* but that probably doesn't hurt.                */
#   elif defined(E2K)
      {
        unsigned long long sz_ull;

        GET_PROCEDURE_STACK_SIZE_INNER(&sz_ull);
        base.reg_base = (void *)(word)sz_ull;
      }
#   endif
    result = (*(GC_stack_base_func volatile *)&fn)(&base, arg);
    /* Strongly discourage the compiler from treating the above */
    /* as a tail call.                                          */
    GC_noop1(COVERT_DATAFLOW(ADDR(&base)));
    return result;
}

#ifndef THREADS

  GC_INNER ptr_t GC_blocked_sp = NULL;
        /* NULL value means we are not inside GC_do_blocking() call. */

# ifdef IA64
    STATIC ptr_t GC_blocked_register_sp = NULL;
# endif

  GC_INNER struct GC_traced_stack_sect_s *GC_traced_stack_sect = NULL;

  /* This is nearly the same as in pthread_support.c.   */
  GC_API void * GC_CALL GC_call_with_gc_active(GC_fn_type fn,
                                               void *client_data)
  {
    struct GC_traced_stack_sect_s stacksect;
    GC_ASSERT(GC_is_initialized);

    /* Adjust our stack bottom pointer (this could happen if    */
    /* GC_get_main_stack_base() is unimplemented or broken for  */
    /* the platform).                                           */
    if (HOTTER_THAN(GC_stackbottom, (ptr_t)(&stacksect)))
      GC_stackbottom = COVERT_DATAFLOW_P(&stacksect);

    if (GC_blocked_sp == NULL) {
      /* We are not inside GC_do_blocking() - do nothing more.  */
      client_data = (*(GC_fn_type volatile *)&fn)(client_data);
      /* Prevent treating the above as a tail call.     */
      GC_noop1(COVERT_DATAFLOW(ADDR(&stacksect)));
      return client_data; /* result */
    }

    /* Setup new "stack section".       */
    stacksect.saved_stack_ptr = GC_blocked_sp;
#   ifdef IA64
      /* This is the same as in GC_call_with_stack_base().      */
      stacksect.backing_store_end = GC_save_regs_in_stack();
      /* Unnecessarily flushes register stack,          */
      /* but that probably doesn't hurt.                */
      stacksect.saved_backing_store_ptr = GC_blocked_register_sp;
#   endif
    stacksect.prev = GC_traced_stack_sect;
    GC_blocked_sp = NULL;
    GC_traced_stack_sect = &stacksect;

    client_data = (*(GC_fn_type volatile *)&fn)(client_data);
    GC_ASSERT(GC_blocked_sp == NULL);
    GC_ASSERT(GC_traced_stack_sect == &stacksect);

#   if defined(CPPCHECK)
      GC_noop1_ptr(GC_traced_stack_sect);
      GC_noop1_ptr(GC_blocked_sp);
#   endif
    /* Restore original "stack section".        */
    GC_traced_stack_sect = stacksect.prev;
#   ifdef IA64
      GC_blocked_register_sp = stacksect.saved_backing_store_ptr;
#   endif
    GC_blocked_sp = stacksect.saved_stack_ptr;

    return client_data; /* result */
  }

  /* This is nearly the same as in pthread_support.c.   */
  STATIC void GC_do_blocking_inner(ptr_t data, void *context)
  {
    struct blocking_data * d = (struct blocking_data *)data;

    UNUSED_ARG(context);
    GC_ASSERT(GC_is_initialized);
    GC_ASSERT(GC_blocked_sp == NULL);
#   ifdef SPARC
        GC_blocked_sp = GC_save_regs_in_stack();
#   else
        GC_blocked_sp = (ptr_t) &d; /* save approx. sp */
#       ifdef IA64
            GC_blocked_register_sp = GC_save_regs_in_stack();
#       endif
#   endif

    d -> client_data = (d -> fn)(d -> client_data);

#   ifdef SPARC
        GC_ASSERT(GC_blocked_sp != NULL);
#   else
        GC_ASSERT(GC_blocked_sp == (ptr_t)(&d));
#   endif
#   if defined(CPPCHECK)
      GC_noop1_ptr(GC_blocked_sp);
#   endif
    GC_blocked_sp = NULL;
  }

  GC_API void GC_CALL GC_set_stackbottom(void *gc_thread_handle,
                                         const struct GC_stack_base *sb)
  {
    GC_ASSERT(sb -> mem_base != NULL);
    GC_ASSERT(NULL == gc_thread_handle || &GC_stackbottom == gc_thread_handle);
    GC_ASSERT(NULL == GC_blocked_sp
              && NULL == GC_traced_stack_sect); /* for now */
    UNUSED_ARG(gc_thread_handle);

    GC_stackbottom = (char *)(sb -> mem_base);
#   ifdef IA64
      GC_register_stackbottom = (ptr_t)(sb -> reg_base);
#   endif
  }

  GC_API void * GC_CALL GC_get_my_stackbottom(struct GC_stack_base *sb)
  {
    GC_ASSERT(GC_is_initialized);
    sb -> mem_base = GC_stackbottom;
#   ifdef IA64
      sb -> reg_base = GC_register_stackbottom;
#   elif defined(E2K)
      sb -> reg_base = NULL;
#   endif
    return &GC_stackbottom; /* gc_thread_handle */
  }

#endif /* !THREADS */

GC_API void * GC_CALL GC_do_blocking(GC_fn_type fn, void * client_data)
{
    struct blocking_data my_data;

    my_data.fn = fn;
    my_data.client_data = client_data;
    GC_with_callee_saves_pushed(GC_do_blocking_inner, (ptr_t)(&my_data));
    return my_data.client_data; /* result */
}

#if !defined(NO_DEBUGGING)
  GC_API void GC_CALL GC_dump(void)
  {
    READER_LOCK();
    GC_dump_named(NULL);
    READER_UNLOCK();
  }

  GC_API void GC_CALL GC_dump_named(const char *name)
  {
#   ifndef NO_CLOCK
      CLOCK_TYPE current_time;

      GET_TIME(current_time);
#   endif
    if (name != NULL) {
      GC_printf("\n***GC Dump %s\n", name);
    } else {
      GC_printf("\n***GC Dump collection #%lu\n", (unsigned long)GC_gc_no);
    }
#   ifndef NO_CLOCK
      /* Note that the time is wrapped in ~49 days if sizeof(long)==4.  */
      GC_printf("Time since GC init: %lu ms\n",
                MS_TIME_DIFF(current_time, GC_init_time));
#   endif

    GC_printf("\n***Static roots:\n");
    GC_print_static_roots();
    GC_printf("\n***Heap sections:\n");
    GC_print_heap_sects();
    GC_printf("\n***Free blocks:\n");
    GC_print_hblkfreelist();
    GC_printf("\n***Blocks in use:\n");
    GC_print_block_list();
#   ifndef GC_NO_FINALIZATION
      GC_dump_finalization();
#   endif
  }
#endif /* !NO_DEBUGGING */

GC_API GC_word GC_CALL GC_get_memory_use(void)
{
  word bytes;

  READER_LOCK();
  GC_ASSERT(GC_heapsize >= GC_large_free_bytes);
  bytes = GC_heapsize - GC_large_free_bytes;
  READER_UNLOCK();
  return bytes;
}

/* Getter functions for the public Read-only variables.                 */

GC_API GC_word GC_CALL GC_get_gc_no(void)
{
    return GC_gc_no;
}

#ifndef PARALLEL_MARK
  GC_API void GC_CALL GC_set_markers_count(unsigned markers)
  {
    UNUSED_ARG(markers);
  }
#endif

GC_API int GC_CALL GC_get_parallel(void)
{
# ifdef THREADS
    return GC_parallel;
# else
    return 0;
# endif
}

/* Setter and getter functions for the public R/W function variables.   */
/* These functions are synchronized (like GC_set_warn_proc() and        */
/* GC_get_warn_proc()).                                                 */

GC_API void GC_CALL GC_set_oom_fn(GC_oom_func fn)
{
    GC_ASSERT(NONNULL_ARG_NOT_NULL(fn));
    LOCK();
    GC_oom_fn = fn;
    UNLOCK();
}

GC_API GC_oom_func GC_CALL GC_get_oom_fn(void)
{
    GC_oom_func fn;

    READER_LOCK();
    fn = GC_oom_fn;
    READER_UNLOCK();
    return fn;
}

GC_API void GC_CALL GC_set_on_heap_resize(GC_on_heap_resize_proc fn)
{
    /* fn may be 0 (means no event notifier). */
    LOCK();
    GC_on_heap_resize = fn;
    UNLOCK();
}

GC_API GC_on_heap_resize_proc GC_CALL GC_get_on_heap_resize(void)
{
    GC_on_heap_resize_proc fn;

    READER_LOCK();
    fn = GC_on_heap_resize;
    READER_UNLOCK();
    return fn;
}

GC_API void GC_CALL GC_set_finalizer_notifier(GC_finalizer_notifier_proc fn)
{
    /* fn may be 0 (means no finalizer notifier). */
    LOCK();
    GC_finalizer_notifier = fn;
    UNLOCK();
}

GC_API GC_finalizer_notifier_proc GC_CALL GC_get_finalizer_notifier(void)
{
    GC_finalizer_notifier_proc fn;

    READER_LOCK();
    fn = GC_finalizer_notifier;
    READER_UNLOCK();
    return fn;
}

/* Setter and getter functions for the public numeric R/W variables.    */
/* It is safe to call these functions even before GC_INIT().            */
/* These functions are unsynchronized and, if called after GC_INIT(),   */
/* should be typically invoked inside the context of                    */
/* GC_call_with_alloc_lock() (or GC_call_with_reader_lock() in case of  */
/* the getters) to prevent data race (unless it is guaranteed the       */
/* collector is not multi-threaded at that execution point).            */

GC_API void GC_CALL GC_set_find_leak(int value)
{
    /* value is of boolean type. */
    GC_find_leak = value;
}

GC_API int GC_CALL GC_get_find_leak(void)
{
    return GC_find_leak;
}

GC_API void GC_CALL GC_set_all_interior_pointers(int value)
{
    GC_all_interior_pointers = value ? 1 : 0;
    if (GC_is_initialized) {
      /* It is not recommended to change GC_all_interior_pointers value */
      /* after GC is initialized but it seems GC could work correctly   */
      /* even after switching the mode.                                 */
      LOCK();
      GC_initialize_offsets(); /* NOTE: this resets manual offsets as well */
      if (!GC_all_interior_pointers)
        GC_bl_init_no_interiors();
      UNLOCK();
    }
}

GC_API int GC_CALL GC_get_all_interior_pointers(void)
{
    return GC_all_interior_pointers;
}

GC_API void GC_CALL GC_set_finalize_on_demand(int value)
{
    GC_ASSERT(value != -1); /* -1 was used to retrieve old value in gc-7.2 */
    /* value is of boolean type. */
    GC_finalize_on_demand = value;
}

GC_API int GC_CALL GC_get_finalize_on_demand(void)
{
    return GC_finalize_on_demand;
}

GC_API void GC_CALL GC_set_java_finalization(int value)
{
    GC_ASSERT(value != -1); /* -1 was used to retrieve old value in gc-7.2 */
    /* value is of boolean type. */
    GC_java_finalization = value;
}

GC_API int GC_CALL GC_get_java_finalization(void)
{
    return GC_java_finalization;
}

GC_API void GC_CALL GC_set_dont_expand(int value)
{
    GC_ASSERT(value != -1); /* -1 was used to retrieve old value in gc-7.2 */
    /* value is of boolean type. */
    GC_dont_expand = value;
}

GC_API int GC_CALL GC_get_dont_expand(void)
{
    return GC_dont_expand;
}

GC_API void GC_CALL GC_set_no_dls(int value)
{
    GC_ASSERT(value != -1); /* -1 was used to retrieve old value in gc-7.2 */
    /* value is of boolean type. */
    GC_no_dls = value;
}

GC_API int GC_CALL GC_get_no_dls(void)
{
    return GC_no_dls;
}

GC_API void GC_CALL GC_set_non_gc_bytes(GC_word value)
{
    GC_non_gc_bytes = value;
}

GC_API GC_word GC_CALL GC_get_non_gc_bytes(void)
{
    return GC_non_gc_bytes;
}

GC_API void GC_CALL GC_set_free_space_divisor(GC_word value)
{
    GC_ASSERT(value > 0);
    GC_free_space_divisor = value;
}

GC_API GC_word GC_CALL GC_get_free_space_divisor(void)
{
    return GC_free_space_divisor;
}

GC_API void GC_CALL GC_set_max_retries(GC_word value)
{
    GC_ASSERT((GC_signed_word)value != -1);
                        /* -1 was used to retrieve old value in gc-7.2 */
    GC_max_retries = value;
}

GC_API GC_word GC_CALL GC_get_max_retries(void)
{
    return GC_max_retries;
}

GC_API void GC_CALL GC_set_dont_precollect(int value)
{
    GC_ASSERT(value != -1); /* -1 was used to retrieve old value in gc-7.2 */
    /* value is of boolean type. */
    GC_dont_precollect = value;
}

GC_API int GC_CALL GC_get_dont_precollect(void)
{
    return GC_dont_precollect;
}

GC_API void GC_CALL GC_set_full_freq(int value)
{
    GC_ASSERT(value >= 0);
    GC_full_freq = value;
}

GC_API int GC_CALL GC_get_full_freq(void)
{
    return GC_full_freq;
}

GC_API void GC_CALL GC_set_time_limit(unsigned long value)
{
    GC_ASSERT((long)value != -1L);
                        /* -1 was used to retrieve old value in gc-7.2 */
    GC_time_limit = value;
}

GC_API unsigned long GC_CALL GC_get_time_limit(void)
{
    return GC_time_limit;
}

GC_API void GC_CALL GC_set_force_unmap_on_gcollect(int value)
{
    GC_force_unmap_on_gcollect = (GC_bool)value;
}

GC_API int GC_CALL GC_get_force_unmap_on_gcollect(void)
{
    return (int)GC_force_unmap_on_gcollect;
}

GC_API void GC_CALL GC_abort_on_oom(void)
{
    GC_err_printf("Insufficient memory for the allocation\n");
    EXIT();
}

GC_API size_t GC_CALL GC_get_hblk_size(void)
{
    return (size_t)HBLKSIZE;
}

/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1995 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1999 by Hewlett-Packard Company.  All rights reserved.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#if (defined(MPROTECT_VDB) && !defined(MSWIN32) && !defined(MSWINCE)) \
    || defined(GC_SOLARIS_THREADS) || defined(OPENBSD)
# include <signal.h>
#endif

#if defined(UNIX_LIKE) || defined(CYGWIN32) || defined(NACL) \
    || defined(SYMBIAN)
# include <fcntl.h>
#endif

#if defined(LINUX) || defined(LINUX_STACKBOTTOM)
# include <ctype.h>
#endif

/* Blatantly OS dependent routines, except for those that are related   */
/* to dynamic loading.                                                  */

#ifdef AMIGA
# define GC_AMIGA_DEF


/******************************************************************

  AmigaOS-specific routines for GC.
  This file is normally included from os_dep.c

******************************************************************/


#if !defined(GC_AMIGA_DEF) && !defined(GC_AMIGA_SB) && !defined(GC_AMIGA_DS) && !defined(GC_AMIGA_AM)
# include <stdio.h>
# include <signal.h>
# define GC_AMIGA_DEF
# define GC_AMIGA_SB
# define GC_AMIGA_DS
# define GC_AMIGA_AM
#endif


#ifdef GC_AMIGA_DEF

# ifndef __GNUC__
#   include <exec/exec.h>
# endif
# include <proto/exec.h>
# include <proto/dos.h>
# include <dos/dosextens.h>
# include <workbench/startup.h>

#endif




#ifdef GC_AMIGA_SB

/******************************************************************
   Find the base of the stack.
******************************************************************/

ptr_t GC_get_main_stack_base(void)
{
    struct Process *proc = (struct Process*)SysBase->ThisTask;

    /* Reference: Amiga Guru Book Pages: 42,567,574 */
    if (proc->pr_Task.tc_Node.ln_Type==NT_PROCESS
        && proc->pr_CLI != NULL) {
        /* first ULONG is StackSize */
        /*longPtr = proc->pr_ReturnAddr;
        size = longPtr[0];*/

        return (char *)proc->pr_ReturnAddr + sizeof(ULONG);
    } else {
        return (char *)proc->pr_Task.tc_SPUpper;
    }
}

#endif


#ifdef GC_AMIGA_DS
/******************************************************************
   Register data segments.
******************************************************************/

   void GC_register_data_segments(void)
   {
     struct Process     *proc;
     const struct CommandLineInterface *cli;
     BPTR myseglist;
     ULONG *data;

#     ifdef __GNUC__
        ULONG dataSegSize;
        GC_bool found_segment = FALSE;
        extern char __data_size[];

        dataSegSize=__data_size+8;
        /* Can`t find the Location of __data_size, because
           it`s possible that is it, inside the segment. */

#     endif

        proc= (struct Process*)SysBase->ThisTask;

        /* Reference: Amiga Guru Book Pages: 538ff,565,573
                     and XOper.asm */
        myseglist = proc->pr_SegList;
        if (proc->pr_Task.tc_Node.ln_Type==NT_PROCESS) {
          if (proc->pr_CLI != NULL) {
            /* ProcLoaded "Loaded as a command: " */
            cli = BADDR(proc->pr_CLI);
            myseglist = cli->cli_Module;
          }
        } else {
          ABORT("Not a Process.");
        }

        if (myseglist == NULL) {
            ABORT("Arrrgh.. can't find segments, aborting");
        }

        /* xoper hunks Shell Process */

        for (data = (ULONG *)BADDR(myseglist); data != NULL;
             data = (ULONG *)BADDR(data[0])) {
          if ((ULONG)GC_register_data_segments < (ULONG)(&data[1])
              || (ULONG)GC_register_data_segments > (ULONG)(&data[1])
                                                    + data[-1]) {
#             ifdef __GNUC__
                if (dataSegSize == data[-1]) {
                  found_segment = TRUE;
                }
#             endif
              GC_add_roots_inner((char *)&data[1],
                                 ((char *)&data[1]) + data[-1], FALSE);
          }
        } /* for */
#       ifdef __GNUC__
           if (!found_segment) {
             ABORT("Can`t find correct Segments.\nSolution: Use an newer version of ixemul.library");
           }
#       endif
   }

#endif



#ifdef GC_AMIGA_AM

#ifndef GC_AMIGA_FASTALLOC

void *GC_amiga_allocwrapper(size_t size,void *(*AllocFunction)(size_t size2)){
        return (*AllocFunction)(size);
}

void *(*GC_amiga_allocwrapper_do)(size_t size,void *(*AllocFunction)(size_t size2))
        =GC_amiga_allocwrapper;

#else




void *GC_amiga_allocwrapper_firsttime(size_t size,void *(*AllocFunction)(size_t size2));

void *(*GC_amiga_allocwrapper_do)(size_t size,void *(*AllocFunction)(size_t size2))
        =GC_amiga_allocwrapper_firsttime;


/******************************************************************
   Amiga-specific routines to obtain memory, and force GC to give
   back fast-mem whenever possible.
        These hacks makes gc-programs go many times faster when
   the Amiga is low on memory, and are therefore strictly necessary.

   -Kjetil S. Matheussen, 2000.
******************************************************************/



/* List-header for all allocated memory. */

struct GC_Amiga_AllocedMemoryHeader{
        ULONG size;
        struct GC_Amiga_AllocedMemoryHeader *next;
};
struct GC_Amiga_AllocedMemoryHeader *GC_AMIGAMEM=(struct GC_Amiga_AllocedMemoryHeader *)(int)~(NULL);



/* Type of memory. Once in the execution of a program, this might change to MEMF_ANY|MEMF_CLEAR */

ULONG GC_AMIGA_MEMF = MEMF_FAST | MEMF_CLEAR;


/* Prevents GC_amiga_get_mem from allocating memory if this one is TRUE. */
#ifndef GC_AMIGA_ONLYFAST
BOOL GC_amiga_dontalloc=FALSE;
#endif

#ifdef GC_AMIGA_PRINTSTATS
int succ=0,succ2=0;
int nsucc=0,nsucc2=0;
int nullretries=0;
int numcollects=0;
int chipa=0;
int allochip=0;
int allocfast=0;
int cur0=0;
int cur1=0;
int cur10=0;
int cur50=0;
int cur150=0;
int cur151=0;
int ncur0=0;
int ncur1=0;
int ncur10=0;
int ncur50=0;
int ncur150=0;
int ncur151=0;
#endif

/* Free everything at program-end. */

void GC_amiga_free_all_mem(void){
        struct GC_Amiga_AllocedMemoryHeader *gc_am=(struct GC_Amiga_AllocedMemoryHeader *)(~(int)(GC_AMIGAMEM));

#ifdef GC_AMIGA_PRINTSTATS
        printf("\n\n"
                "%d bytes of chip-mem, and %d bytes of fast-mem where allocated from the OS.\n",
                allochip,allocfast
        );
        printf(
                "%d bytes of chip-mem were returned from the GC_AMIGA_FASTALLOC supported allocating functions.\n",
                chipa
        );
        printf("\n");
        printf("GC_gcollect was called %d times to avoid returning NULL or start allocating with the MEMF_ANY flag.\n",numcollects);
        printf("%d of them was a success. (the others had to use allocation from the OS.)\n",nullretries);
        printf("\n");
        printf("Succeeded forcing %d gc-allocations (%d bytes) of chip-mem to be fast-mem.\n",succ,succ2);
        printf("Failed forcing %d gc-allocations (%d bytes) of chip-mem to be fast-mem.\n",nsucc,nsucc2);
        printf("\n");
        printf(
                "Number of retries before succeeding a chip->fast force:\n"
                "0: %d, 1: %d, 2-9: %d, 10-49: %d, 50-149: %d, >150: %d\n",
                cur0,cur1,cur10,cur50,cur150,cur151
        );
        printf(
                "Number of retries before giving up a chip->fast force:\n"
                "0: %d, 1: %d, 2-9: %d, 10-49: %d, 50-149: %d, >150: %d\n",
                ncur0,ncur1,ncur10,ncur50,ncur150,ncur151
        );
#endif

        while(gc_am!=NULL){
                struct GC_Amiga_AllocedMemoryHeader *temp = gc_am->next;
                FreeMem(gc_am,gc_am->size);
                gc_am=(struct GC_Amiga_AllocedMemoryHeader *)(~(int)(temp));
        }
}

#ifndef GC_AMIGA_ONLYFAST

/* All memory with address lower than this one is chip-mem. */

char *chipmax;


/*
 * Always set to the last size of memory tried to be allocated.
 * Needed to ensure allocation when the size is bigger than 100000.
 *
 */
size_t latestsize;

#endif


#ifdef GC_AMIGA_FASTALLOC

/*
 * The actual function that is called with the GET_MEM macro.
 *
 */

void *GC_amiga_get_mem(size_t size){
        struct GC_Amiga_AllocedMemoryHeader *gc_am;

#ifndef GC_AMIGA_ONLYFAST
        if(GC_amiga_dontalloc==TRUE){
                return NULL;
        }

        /* We really don't want to use chip-mem, but if we must, then as little as possible. */
        if(GC_AMIGA_MEMF==(MEMF_ANY|MEMF_CLEAR) && size>100000 && latestsize<50000) return NULL;
#endif

        gc_am=AllocMem((ULONG)(size + sizeof(struct GC_Amiga_AllocedMemoryHeader)),GC_AMIGA_MEMF);
        if(gc_am==NULL) return NULL;

        gc_am->next=GC_AMIGAMEM;
        gc_am->size=size + sizeof(struct GC_Amiga_AllocedMemoryHeader);
        GC_AMIGAMEM=(struct GC_Amiga_AllocedMemoryHeader *)(~(int)(gc_am));

#ifdef GC_AMIGA_PRINTSTATS
        if((char *)gc_am<chipmax){
                allochip+=size;
        }else{
                allocfast+=size;
        }
#endif

        return gc_am+1;

}

#endif


#ifndef GC_AMIGA_ONLYFAST

/* Tries very hard to force GC to find fast-mem to return. Done recursively
 * to hold the rejected memory-pointers reachable from the collector in an
 * easy way.
 *
 */
#ifdef GC_AMIGA_RETRY
void *GC_amiga_rec_alloc(size_t size,void *(*AllocFunction)(size_t size2),const int rec){
        void *ret;

        ret=(*AllocFunction)(size);

#ifdef GC_AMIGA_PRINTSTATS
        if((char *)ret>chipmax || ret==NULL){
                if(ret==NULL){
                        nsucc++;
                        nsucc2+=size;
                        if(rec==0) ncur0++;
                        if(rec==1) ncur1++;
                        if(rec>1 && rec<10) ncur10++;
                        if(rec>=10 && rec<50) ncur50++;
                        if(rec>=50 && rec<150) ncur150++;
                        if(rec>=150) ncur151++;
                }else{
                        succ++;
                        succ2+=size;
                        if(rec==0) cur0++;
                        if(rec==1) cur1++;
                        if(rec>1 && rec<10) cur10++;
                        if(rec>=10 && rec<50) cur50++;
                        if(rec>=50 && rec<150) cur150++;
                        if(rec>=150) cur151++;
                }
        }
#endif

        if (((char *)ret)<=chipmax && ret!=NULL && (rec<(size>500000?9:size/5000))){
                ret=GC_amiga_rec_alloc(size,AllocFunction,rec+1);
        }

        return ret;
}
#endif


/* The allocating-functions defined inside the Amiga-blocks in gc.h is called
 * via these functions.
 */


void *GC_amiga_allocwrapper_any(size_t size,void *(*AllocFunction)(size_t size2)){
        void *ret;

        GC_amiga_dontalloc=TRUE; /* Pretty tough thing to do, but it's indeed necessary. */
        latestsize=size;

        ret=(*AllocFunction)(size);

        if(((char *)ret) <= chipmax){
                if(ret==NULL){
                        /* Give GC access to allocate memory. */
#ifdef GC_AMIGA_GC
                        if(!GC_dont_gc){
                                GC_gcollect();
#ifdef GC_AMIGA_PRINTSTATS
                                numcollects++;
#endif
                                ret=(*AllocFunction)(size);
                        }
                        if(ret==NULL)
#endif
                        {
                                GC_amiga_dontalloc=FALSE;
                                ret=(*AllocFunction)(size);
                                if(ret==NULL){
                                        WARN("Out of Memory!  Returning NIL!\n", 0);
                                }
                        }
#ifdef GC_AMIGA_PRINTSTATS
                        else{
                                nullretries++;
                        }
                        if(ret!=NULL && (char *)ret<=chipmax) chipa+=size;
#endif
                }
#ifdef GC_AMIGA_RETRY
                else{
                        void *ret2;
                        /* We got chip-mem. Better try again and again and again etc., we might get fast-mem sooner or later... */
                        /* Using gctest to check the effectiveness of doing this, does seldom give a very good result. */
                        /* However, real programs doesn't normally rapidly allocate and deallocate. */
                        if(
                                AllocFunction!=GC_malloc_uncollectable
#ifdef GC_ATOMIC_UNCOLLECTABLE
                                && AllocFunction!=GC_malloc_atomic_uncollectable
#endif
                        ){
                                ret2=GC_amiga_rec_alloc(size,AllocFunction,0);
                        }else{
                                ret2=(*AllocFunction)(size);
#ifdef GC_AMIGA_PRINTSTATS
                                if((char *)ret2<chipmax || ret2==NULL){
                                        nsucc++;
                                        nsucc2+=size;
                                        ncur0++;
                                }else{
                                        succ++;
                                        succ2+=size;
                                        cur0++;
                                }
#endif
                        }
                        if(((char *)ret2)>chipmax){
                                GC_free(ret);
                                ret=ret2;
                        }else{
                                GC_free(ret2);
                        }
                }
#endif
        }

#   if defined(CPPCHECK)
      if (GC_amiga_dontalloc) /* variable is actually used by AllocFunction */
#   endif
        GC_amiga_dontalloc=FALSE;

        return ret;
}



void (*GC_amiga_toany)(void)=NULL;

void GC_amiga_set_toany(void (*func)(void)){
        GC_amiga_toany=func;
}

#endif /* !GC_AMIGA_ONLYFAST */


void *GC_amiga_allocwrapper_fast(size_t size,void *(*AllocFunction)(size_t size2)){
        void *ret;

        ret=(*AllocFunction)(size);

        if(ret==NULL){
                /* Enable chip-mem allocation. */
#ifdef GC_AMIGA_GC
                if(!GC_dont_gc){
                        GC_gcollect();
#ifdef GC_AMIGA_PRINTSTATS
                        numcollects++;
#endif
                        ret=(*AllocFunction)(size);
                }
                if(ret==NULL)
#endif
                {
#ifndef GC_AMIGA_ONLYFAST
                        GC_AMIGA_MEMF=MEMF_ANY | MEMF_CLEAR;
                        if(GC_amiga_toany!=NULL) (*GC_amiga_toany)();
                        GC_amiga_allocwrapper_do=GC_amiga_allocwrapper_any;
                        return GC_amiga_allocwrapper_any(size,AllocFunction);
#endif
                }
#ifdef GC_AMIGA_PRINTSTATS
                else{
                        nullretries++;
                }
#endif
        }

        return ret;
}

void *GC_amiga_allocwrapper_firsttime(size_t size,void *(*AllocFunction)(size_t size2)){
        atexit(&GC_amiga_free_all_mem);
        chipmax=(char *)SysBase->MaxLocMem; /* For people still having SysBase in chip-mem, this might speed up a bit. */
        GC_amiga_allocwrapper_do=GC_amiga_allocwrapper_fast;
        return GC_amiga_allocwrapper_fast(size,AllocFunction);
}


#endif /* GC_AMIGA_FASTALLOC */



/*
 * The wrapped realloc function.
 *
 */
void *GC_amiga_realloc(void *old_object,size_t new_size_in_bytes){
#ifndef GC_AMIGA_FASTALLOC
        return GC_realloc(old_object,new_size_in_bytes);
#else
        void *ret;
        latestsize=new_size_in_bytes;
        ret=GC_realloc(old_object,new_size_in_bytes);
        if(ret==NULL && new_size_in_bytes != 0
           && GC_AMIGA_MEMF==(MEMF_FAST | MEMF_CLEAR)){
                /* Out of fast-mem. */
#ifdef GC_AMIGA_GC
                if(!GC_dont_gc){
                        GC_gcollect();
#ifdef GC_AMIGA_PRINTSTATS
                        numcollects++;
#endif
                        ret=GC_realloc(old_object,new_size_in_bytes);
                }
                if(ret==NULL)
#endif
                {
#ifndef GC_AMIGA_ONLYFAST
                        GC_AMIGA_MEMF=MEMF_ANY | MEMF_CLEAR;
                        if(GC_amiga_toany!=NULL) (*GC_amiga_toany)();
                        GC_amiga_allocwrapper_do=GC_amiga_allocwrapper_any;
                        ret=GC_realloc(old_object,new_size_in_bytes);
#endif
                }
#ifdef GC_AMIGA_PRINTSTATS
                else{
                        nullretries++;
                }
#endif
        }
        if(ret==NULL && new_size_in_bytes != 0){
                WARN("Out of Memory!  Returning NIL!\n", 0);
        }
#ifdef GC_AMIGA_PRINTSTATS
        if(((char *)ret)<chipmax && ret!=NULL){
                chipa+=new_size_in_bytes;
        }
#endif
        return ret;
#endif
}

#endif /* GC_AMIGA_AM */

# undef GC_AMIGA_DEF
#endif

#ifdef MACOS
# include <Processes.h>
#endif

#ifdef IRIX5
# include <sys/uio.h>
# include <malloc.h>   /* for locking */
#endif

#if defined(MMAP_SUPPORTED) || defined(ADD_HEAP_GUARD_PAGES)
# if defined(USE_MUNMAP) && !defined(USE_MMAP) && !defined(CPPCHECK)
#   error Invalid config: USE_MUNMAP requires USE_MMAP
# endif
# include <sys/mman.h>
# include <sys/stat.h>
#endif

#if defined(ADD_HEAP_GUARD_PAGES) || defined(LINUX_STACKBOTTOM) \
    || defined(MMAP_SUPPORTED) || defined(NEED_PROC_MAPS)
# include <errno.h>
#endif

#if defined(DARWIN) && !defined(DYNAMIC_LOADING) \
    && !defined(GC_DONT_REGISTER_MAIN_STATIC_DATA)
  /* for get_etext and friends */
# include <mach-o/getsect.h>
#endif

#ifdef DJGPP
  /* Apparently necessary for djgpp 2.01.  May cause problems with      */
  /* other versions.                                                    */
  typedef long unsigned int caddr_t;
#endif

#ifdef PCR




# include "mm/PCR_MM.h"
#endif

#if !defined(NO_EXECUTE_PERMISSION)
  STATIC GC_bool GC_pages_executable = TRUE;
#else
  STATIC GC_bool GC_pages_executable = FALSE;
#endif
#define IGNORE_PAGES_EXECUTABLE 1
                        /* Undefined on GC_pages_executable real use.   */

#if ((defined(LINUX_STACKBOTTOM) || defined(NEED_PROC_MAPS) \
      || defined(PROC_VDB) || defined(SOFT_VDB)) && !defined(PROC_READ)) \
    || defined(CPPCHECK)
# define PROC_READ read
          /* Should probably call the real read, if read is wrapped.    */
#endif

#if defined(LINUX_STACKBOTTOM) || defined(NEED_PROC_MAPS)
  /* Repeatedly perform a read call until the buffer is filled  */
  /* up, or we encounter EOF or an error.                       */
  STATIC ssize_t GC_repeat_read(int fd, char *buf, size_t count)
  {
    ssize_t num_read = 0;

    ASSERT_CANCEL_DISABLED();
    while ((size_t)num_read < count) {
        ssize_t result = PROC_READ(fd, buf + num_read,
                                   count - (size_t)num_read);

        if (result < 0) return result;
        if (result == 0) break;
        num_read += result;
    }
    return num_read;
  }
#endif /* LINUX_STACKBOTTOM || NEED_PROC_MAPS */

#ifdef NEED_PROC_MAPS
/* We need to parse /proc/self/maps, either to find dynamic libraries,  */
/* and/or to find the register backing store base (IA64).  Do it once   */
/* here.                                                                */

#ifdef THREADS
  /* Determine the length of a file by incrementally reading it into a  */
  /* buffer.  This would be silly to use it on a file supporting lseek, */
  /* but Linux /proc files usually do not.                              */
  /* As of Linux 4.15.0, lseek(SEEK_END) fails for /proc/self/maps.     */
  STATIC size_t GC_get_file_len(int f)
  {
    size_t total = 0;
    ssize_t result;
#   define GET_FILE_LEN_BUF_SZ 500
    char buf[GET_FILE_LEN_BUF_SZ];

    do {
        result = PROC_READ(f, buf, sizeof(buf));
        if (result == -1) return 0;
        total += (size_t)result;
    } while (result > 0);
    return total;
  }

  STATIC size_t GC_get_maps_len(void)
  {
    int f = open("/proc/self/maps", O_RDONLY);
    size_t result;

    if (f < 0) return 0; /* treat missing file as empty */
    result = GC_get_file_len(f);
    close(f);
    return result;
  }
#endif /* THREADS */

/* Copy the contents of /proc/self/maps to a buffer in our address      */
/* space.  Return the address of the buffer.                            */
GC_INNER const char * GC_get_maps(void)
{
    ssize_t result;
    static char *maps_buf = NULL;
    static size_t maps_buf_sz = 1;
    size_t maps_size;
#   ifdef THREADS
      size_t old_maps_size = 0;
#   endif

    /* The buffer is essentially static, so there must be a single client. */
    GC_ASSERT(I_HOLD_LOCK());

    /* Note that in the presence of threads, the maps file can  */
    /* essentially shrink asynchronously and unexpectedly as    */
    /* threads that we already think of as dead release their   */
    /* stacks.  And there is no easy way to read the entire     */
    /* file atomically.  This is arguably a misfeature of the   */
    /* /proc/self/maps interface.                               */
    /* Since we expect the file can grow asynchronously in rare */
    /* cases, it should suffice to first determine              */
    /* the size (using read), and then to reread the file.      */
    /* If the size is inconsistent we have to retry.            */
    /* This only matters with threads enabled, and if we use    */
    /* this to locate roots (not the default).                  */

#   ifdef THREADS
        /* Determine the initial size of /proc/self/maps.       */
        maps_size = GC_get_maps_len();
        if (0 == maps_size)
          ABORT("Cannot determine length of /proc/self/maps");
#   else
        maps_size = 4000;       /* Guess */
#   endif

    /* Read /proc/self/maps, growing maps_buf as necessary.     */
    /* Note that we may not allocate conventionally, and        */
    /* thus can't use stdio.                                    */
        do {
            int f;

            while (maps_size >= maps_buf_sz) {
#             ifdef LINT2
                /* Workaround passing tainted maps_buf to a tainted sink. */
                GC_noop1_ptr(maps_buf);
#             else
                GC_scratch_recycle_no_gww(maps_buf, maps_buf_sz);
#             endif
              /* Grow only by powers of 2, since we leak "too small" buffers.*/
              while (maps_size >= maps_buf_sz) maps_buf_sz *= 2;
              maps_buf = GC_scratch_alloc(maps_buf_sz);
              if (NULL == maps_buf)
                ABORT_ARG1("Insufficient space for /proc/self/maps buffer",
                        ", %lu bytes requested", (unsigned long)maps_buf_sz);
#             ifdef THREADS
                /* Recompute initial length, since we allocated.        */
                /* This can only happen a few times per program         */
                /* execution.                                           */
                maps_size = GC_get_maps_len();
                if (0 == maps_size)
                  ABORT("Cannot determine length of /proc/self/maps");
#             endif
            }
            GC_ASSERT(maps_buf_sz >= maps_size + 1);
            f = open("/proc/self/maps", O_RDONLY);
            if (-1 == f)
              ABORT_ARG1("Cannot open /proc/self/maps",
                         ": errno= %d", errno);
#           ifdef THREADS
              old_maps_size = maps_size;
#           endif
            maps_size = 0;
            do {
                result = GC_repeat_read(f, maps_buf, maps_buf_sz-1);
                if (result < 0) {
                  ABORT_ARG1("Failed to read /proc/self/maps",
                             ": errno= %d", errno);
                }
                maps_size += (size_t)result;
            } while ((size_t)result == maps_buf_sz-1);
            close(f);
            if (0 == maps_size)
              ABORT("Empty /proc/self/maps");
#           ifdef THREADS
              if (maps_size > old_maps_size) {
                /* This might be caused by e.g. thread creation. */
                WARN("Unexpected asynchronous /proc/self/maps growth"
                     " (to %" WARN_PRIuPTR " bytes)\n", maps_size);
              }
#           endif
        } while (maps_size >= maps_buf_sz
#                ifdef THREADS
                   || maps_size < old_maps_size
#                endif
                );
        maps_buf[maps_size] = '\0';
        return maps_buf;
}

/*
 *  GC_parse_map_entry parses an entry from /proc/self/maps so we can
 *  locate all writable data segments that belong to shared libraries.
 *  The format of one of these entries and the fields we care about
 *  is as follows:
 *  XXXXXXXX-XXXXXXXX r-xp 00000000 30:05 260537     name of mapping...\n
 *  ^^^^^^^^ ^^^^^^^^ ^^^^          ^^
 *  start    end      prot          maj_dev
 *
 *  Note that since about august 2003 kernels, the columns no longer have
 *  fixed offsets on 64-bit kernels.  Hence we no longer rely on fixed offsets
 *  anywhere, which is safer anyway.
 */

/* Assign various fields of the first line in maps_ptr to (*start),     */
/* (*end), (*prot), (*maj_dev) and (*mapping_name).  mapping_name may   */
/* be NULL. (*prot) and (*mapping_name) are assigned pointers into the  */
/* original buffer.                                                     */
#if defined(DYNAMIC_LOADING) && defined(USE_PROC_FOR_LIBRARIES) \
    || defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) \
    || (defined(CHECK_SOFT_VDB) && defined(MPROTECT_VDB)) \
    || (defined(REDIRECT_MALLOC) && defined(GC_LINUX_THREADS))
  GC_INNER const char *GC_parse_map_entry(const char *maps_ptr,
                                          ptr_t *start, ptr_t *end,
                                          const char **prot, unsigned *maj_dev,
                                          const char **mapping_name)
  {
    const unsigned char *start_start, *end_start, *maj_dev_start;
    const unsigned char *p; /* unsigned for isspace, isxdigit */

    if (maps_ptr == NULL || *maps_ptr == '\0') {
        return NULL;
    }

    p = (const unsigned char *)maps_ptr;
    while (isspace(*p)) ++p;
    start_start = p;
    GC_ASSERT(isxdigit(*start_start));
    *start = (ptr_t)strtoul((const char *)start_start, (char **)&p, 16);
    GC_ASSERT(*p=='-');

    ++p;
    end_start = p;
    GC_ASSERT(isxdigit(*end_start));
    *end = (ptr_t)strtoul((const char *)end_start, (char **)&p, 16);
    GC_ASSERT(isspace(*p));

    while (isspace(*p)) ++p;
    GC_ASSERT(*p == 'r' || *p == '-');
    *prot = (const char *)p;
    /* Skip past protection field to offset field */
    while (!isspace(*p)) ++p;
    while (isspace(*p)) p++;
    GC_ASSERT(isxdigit(*p));
    /* Skip past offset field, which we ignore */
    while (!isspace(*p)) ++p;
    while (isspace(*p)) p++;
    maj_dev_start = p;
    GC_ASSERT(isxdigit(*maj_dev_start));
    *maj_dev = strtoul((const char *)maj_dev_start, NULL, 16);

    if (mapping_name != NULL) {
      while (*p && *p != '\n' && *p != '/' && *p != '[') p++;
      *mapping_name = (const char *)p;
    }
    while (*p && *p++ != '\n');
    return (const char *)p;
  }
#endif /* REDIRECT_MALLOC || DYNAMIC_LOADING || IA64 || ... */

#if defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) \
    || (defined(CHECK_SOFT_VDB) && defined(MPROTECT_VDB))
  /* Try to read the backing store base from /proc/self/maps.           */
  /* Return the bounds of the writable mapping with a 0 major device,   */
  /* which includes the address passed as data.                         */
  /* Return FALSE if there is no such mapping.                          */
  GC_INNER GC_bool GC_enclosing_writable_mapping(ptr_t addr, ptr_t *startp,
                                                 ptr_t *endp)
  {
    const char *prot;
    ptr_t my_start, my_end;
    const char *maps_ptr;
    unsigned maj_dev;

    GC_ASSERT(I_HOLD_LOCK());
    maps_ptr = GC_get_maps();
    for (;;) {
      maps_ptr = GC_parse_map_entry(maps_ptr, &my_start, &my_end,
                                    &prot, &maj_dev, 0);
      if (NULL == maps_ptr) break;

      if (ADDR_INSIDE(addr, my_start, my_end)) {
        if (prot[1] != 'w' || maj_dev != 0) break;
        *startp = my_start;
        *endp = my_end;
        return TRUE;
      }
    }
    return FALSE;
  }
#endif /* IA64 || INCLUDE_LINUX_THREAD_DESCR */

#if defined(REDIRECT_MALLOC) && defined(GC_LINUX_THREADS)
  /* Find the text(code) mapping for the library whose name, after      */
  /* stripping the directory part, starts with nm.                      */
  GC_INNER GC_bool GC_text_mapping(const char *nm, ptr_t *startp, ptr_t *endp)
  {
    size_t nm_len;
    const char *prot, *map_path;
    ptr_t my_start, my_end;
    unsigned int maj_dev;
    const char *maps_ptr;

    GC_ASSERT(I_HOLD_LOCK());
    maps_ptr = GC_get_maps();
    nm_len = strlen(nm);
    for (;;) {
      maps_ptr = GC_parse_map_entry(maps_ptr, &my_start, &my_end,
                                    &prot, &maj_dev, &map_path);
      if (NULL == maps_ptr) break;

      if (prot[0] == 'r' && prot[1] == '-' && prot[2] == 'x') {
          const char *p = map_path;

          /* Set p to point just past last slash, if any. */
            while (*p != '\0' && *p != '\n' && *p != ' ' && *p != '\t') ++p;
            while (ADDR_GE((ptr_t)p, (ptr_t)map_path) && *p != '/') --p;
            ++p;
          if (strncmp(nm, p, nm_len) == 0) {
            *startp = my_start;
            *endp = my_end;
            return TRUE;
          }
      }
    }
    return FALSE;
  }
#endif /* REDIRECT_MALLOC */

#ifdef IA64
  static ptr_t backing_store_base_from_proc(void)
  {
    ptr_t my_start, my_end;

    GC_ASSERT(I_HOLD_LOCK());
    if (!GC_enclosing_writable_mapping(GC_save_regs_in_stack(),
                                       &my_start, &my_end)) {
        GC_COND_LOG_PRINTF("Failed to find backing store base from /proc\n");
        return 0;
    }
    return my_start;
  }
#endif

#endif /* NEED_PROC_MAPS */

#if defined(SEARCH_FOR_DATA_START)
  /* The i686 case can be handled without a search.  The Alpha case     */
  /* used to be handled differently as well, but the rules changed      */
  /* for recent Linux versions.  This seems to be the easiest way to    */
  /* cover all versions.                                                */

# if defined(LINUX) || defined(HURD)
    /* Some Linux distributions arrange to define __data_start.  Some   */
    /* define data_start as a weak symbol.  The latter is technically   */
    /* broken, since the user program may define data_start, in which   */
    /* case we lose.  Nonetheless, we try both, preferring __data_start.*/
    /* We assume gcc-compatible pragmas.                                */
    EXTERN_C_BEGIN
#   pragma weak __data_start
#   pragma weak data_start
    extern int __data_start[], data_start[];
    EXTERN_C_END
# elif defined(NETBSD)
    EXTERN_C_BEGIN
    extern char **environ;
    EXTERN_C_END
# endif

  ptr_t GC_data_start = NULL;

  GC_INNER void GC_init_linux_data_start(void)
  {
    ptr_t data_end = DATAEND;

#   if (defined(LINUX) || defined(HURD)) && defined(USE_PROG_DATA_START)
      /* Try the easy approaches first: */
      /* However, this may lead to wrong data start value if libgc  */
      /* code is put into a shared library (directly or indirectly) */
      /* which is linked with -Bsymbolic-functions option.  Thus,   */
      /* the following is not used by default.                      */
      if (COVERT_DATAFLOW(ADDR(__data_start)) != 0) {
        GC_data_start = (ptr_t)(__data_start);
      } else {
        GC_data_start = (ptr_t)(data_start);
      }
      if (COVERT_DATAFLOW(ADDR(GC_data_start)) != 0) {
        if (ADDR_LT(data_end, GC_data_start))
          ABORT_ARG2("Wrong __data_start/_end pair",
                     ": %p .. %p", (void *)GC_data_start, (void *)data_end);
        return;
      }
#     ifdef DEBUG_ADD_DEL_ROOTS
        GC_log_printf("__data_start not provided\n");
#     endif
#   endif /* LINUX */

    if (GC_no_dls) {
      /* Not needed, avoids the SIGSEGV caused by       */
      /* GC_find_limit which complicates debugging.     */
      GC_data_start = data_end; /* set data root size to 0 */
      return;
    }

#   ifdef NETBSD
      /* This may need to be environ, without the underscore, for       */
      /* some versions.                                                 */
      GC_data_start = (ptr_t)GC_find_limit(&environ, FALSE);
#   else
      GC_data_start = (ptr_t)GC_find_limit(data_end, FALSE);
#   endif
  }
#endif /* SEARCH_FOR_DATA_START */

#ifdef ECOS

# ifndef ECOS_GC_MEMORY_SIZE
#   define ECOS_GC_MEMORY_SIZE (448 * 1024)
# endif /* ECOS_GC_MEMORY_SIZE */

  /* TODO: This is a simple way of allocating memory which is           */
  /* compatible with ECOS early releases.  Later releases use a more    */
  /* sophisticated means of allocating memory than this simple static   */
  /* allocator, but this method is at least bound to work.              */
  static char ecos_gc_memory[ECOS_GC_MEMORY_SIZE];
  static ptr_t ecos_gc_brk = ecos_gc_memory;

  static void *tiny_sbrk(ptrdiff_t increment)
  {
    void *p = ecos_gc_brk;

    ecos_gc_brk += increment;
    if (ADDR_LT((ptr_t)ecos_gc_memory + sizeof(ecos_gc_memory), ecos_gc_brk)) {
      ecos_gc_brk -= increment;
      return NULL;
    }
    return p;
  }
# define sbrk tiny_sbrk
#endif /* ECOS */

#if defined(ADDRESS_SANITIZER) && (defined(UNIX_LIKE) \
                    || defined(NEED_FIND_LIMIT) || defined(MPROTECT_VDB)) \
    && !defined(CUSTOM_ASAN_DEF_OPTIONS)
  EXTERN_C_BEGIN
  GC_API const char *__asan_default_options(void);
  EXTERN_C_END

  /* To tell ASan to allow GC to use its own SIGBUS/SEGV handlers.      */
  /* The function is exported just to be visible to ASan library.       */
  GC_API const char *__asan_default_options(void)
  {
    return "allow_user_segv_handler=1";
  }
#endif

#ifdef OPENBSD
  static struct sigaction old_segv_act;
  STATIC JMP_BUF GC_jmp_buf_openbsd;

  STATIC void GC_fault_handler_openbsd(int sig)
  {
     UNUSED_ARG(sig);
     LONGJMP(GC_jmp_buf_openbsd, 1);
  }

  static volatile int firstpass;

  /* Return first addressable location > p or bound.    */
  STATIC ptr_t GC_skip_hole_openbsd(ptr_t p, ptr_t bound)
  {
    static volatile ptr_t result;
    struct sigaction act;
    size_t pgsz;

    GC_ASSERT(I_HOLD_LOCK());
    pgsz = (size_t)sysconf(_SC_PAGESIZE);
    GC_ASSERT(ADDR(bound) >= (word)pgsz);

    act.sa_handler = GC_fault_handler_openbsd;
    sigemptyset(&act.sa_mask);
    act.sa_flags = SA_NODEFER | SA_RESTART;
    /* act.sa_restorer is deprecated and should not be initialized. */
    sigaction(SIGSEGV, &act, &old_segv_act);

    firstpass = 1;
    result = PTR_ALIGN_DOWN(p, pgsz);
    if (SETJMP(GC_jmp_buf_openbsd) != 0 || firstpass) {
      firstpass = 0;
      if (ADDR_GE(result, bound - pgsz)) {
        result = bound;
      } else {
        result = result + pgsz;
                    /* no overflow expected; do not use compound        */
                    /* assignment with volatile-qualified left operand  */
        GC_noop1((word)(unsigned char)(*result));
      }
    }

    sigaction(SIGSEGV, &old_segv_act, 0);
    return result;
  }
#endif /* OPENBSD */

#ifdef OS2

# include <stddef.h>

# if !defined(__IBMC__) && !defined(__WATCOMC__) /* e.g. EMX */

struct exe_hdr {
    unsigned short      magic_number;
    unsigned short      padding[29];
    long                new_exe_offset;
};

#define E_MAGIC(x)      (x).magic_number
#define EMAGIC          0x5A4D
#define E_LFANEW(x)     (x).new_exe_offset

struct e32_exe {
    unsigned char       magic_number[2];
    unsigned char       byte_order;
    unsigned char       word_order;
    unsigned long       exe_format_level;
    unsigned short      cpu;
    unsigned short      os;
    unsigned long       padding1[13];
    unsigned long       object_table_offset;
    unsigned long       object_count;
    unsigned long       padding2[31];
};

#define E32_MAGIC1(x)   (x).magic_number[0]
#define E32MAGIC1       'L'
#define E32_MAGIC2(x)   (x).magic_number[1]
#define E32MAGIC2       'X'
#define E32_BORDER(x)   (x).byte_order
#define E32LEBO         0
#define E32_WORDER(x)   (x).word_order
#define E32LEWO         0
#define E32_CPU(x)      (x).cpu
#define E32CPU286       1
#define E32_OBJTAB(x)   (x).object_table_offset
#define E32_OBJCNT(x)   (x).object_count

struct o32_obj {
    unsigned long       size;
    unsigned long       base;
    unsigned long       flags;
    unsigned long       pagemap;
    unsigned long       mapsize;
    unsigned long       reserved;
};

#define O32_FLAGS(x)    (x).flags
#define OBJREAD         0x0001L
#define OBJWRITE        0x0002L
#define OBJINVALID      0x0080L
#define O32_SIZE(x)     (x).size
#define O32_BASE(x)     (x).base

# else  /* IBM's compiler */

/* A kludge to get around what appears to be a header file bug */
# ifndef WORD
#   define WORD unsigned short
# endif
# ifndef DWORD
#   define DWORD unsigned long
# endif

# define EXE386 1
# include <newexe.h>
# include <exe386.h>

# endif  /* __IBMC__ */

# define INCL_DOSERRORS
# define INCL_DOSEXCEPTIONS
# define INCL_DOSFILEMGR
# define INCL_DOSMEMMGR
# define INCL_DOSMISC
# define INCL_DOSMODULEMGR
# define INCL_DOSPROCESS
# include <os2.h>

#endif /* OS2 */

/* Find the page size.  */
GC_INNER size_t GC_page_size = 0;
#ifdef REAL_PAGESIZE_NEEDED
  GC_INNER size_t GC_real_page_size = 0;
#endif

#ifdef SOFT_VDB
  STATIC unsigned GC_log_pagesize = 0;
#endif

#ifdef ANY_MSWIN

# ifndef VER_PLATFORM_WIN32_CE
#   define VER_PLATFORM_WIN32_CE 3
# endif

# if defined(MSWINCE) && defined(THREADS)
    GC_INNER GC_bool GC_dont_query_stack_min = FALSE;
# endif

  GC_INNER SYSTEM_INFO GC_sysinfo;

# ifndef CYGWIN32
#   define is_writable(prot) ((prot) == PAGE_READWRITE \
                            || (prot) == PAGE_WRITECOPY \
                            || (prot) == PAGE_EXECUTE_READWRITE \
                            || (prot) == PAGE_EXECUTE_WRITECOPY)
    /* Return the number of bytes that are writable starting at p.      */
    /* The pointer p is assumed to be page aligned.                     */
    /* If base is not 0, *base becomes the beginning of the             */
    /* allocation region containing p.                                  */
    STATIC word GC_get_writable_length(ptr_t p, ptr_t *base)
    {
      MEMORY_BASIC_INFORMATION buf;
      word result;
      word protect;

      result = VirtualQuery(p, &buf, sizeof(buf));
      if (result != sizeof(buf)) ABORT("Weird VirtualQuery result");
      if (base != 0) *base = (ptr_t)(buf.AllocationBase);
      protect = buf.Protect & ~(word)(PAGE_GUARD | PAGE_NOCACHE);
      if (!is_writable(protect) || buf.State != MEM_COMMIT) return 0;
      return buf.RegionSize;
    }

    /* Fill in the GC_stack_base structure with the stack bottom for    */
    /* this thread.  Should not acquire the allocator lock as the       */
    /* function is used by GC_DllMain.                                  */
    GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
    {
      ptr_t trunc_sp;
      word size;

      /* Set page size if it is not ready (so client can use this       */
      /* function even before GC is initialized).                       */
      if (!GC_page_size) GC_setpagesize();

      trunc_sp = PTR_ALIGN_DOWN(GC_approx_sp(), GC_page_size);
      /* FIXME: This won't work if called from a deeply recursive       */
      /* client code (and the committed stack space has grown).         */
      size = GC_get_writable_length(trunc_sp, 0);
      GC_ASSERT(size != 0);
      sb -> mem_base = trunc_sp + size;
      return GC_SUCCESS;
    }
# else /* CYGWIN32 */
    /* An alternate version for Cygwin (adapted from Dave Korn's        */
    /* gcc version of boehm-gc).                                        */
    GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
    {
#     ifdef X86_64
        sb -> mem_base = ((NT_TIB*)NtCurrentTeb())->StackBase;
#     else
        void * _tlsbase;

        __asm__ ("movl %%fs:4, %0"
                 : "=r" (_tlsbase));
        sb -> mem_base = _tlsbase;
#     endif
      return GC_SUCCESS;
    }
# endif /* CYGWIN32 */
# define HAVE_GET_STACK_BASE

#elif defined(OS2)

  static int os2_getpagesize(void)
  {
      ULONG result[1];

      if (DosQuerySysInfo(QSV_PAGE_SIZE, QSV_PAGE_SIZE,
                          (void *)result, sizeof(ULONG)) != NO_ERROR) {
        WARN("DosQuerySysInfo failed\n", 0);
        result[0] = 4096;
      }
      return (int)result[0];
  }

#endif /* !ANY_MSWIN && OS2 */

GC_INNER void GC_setpagesize(void)
{
# ifdef ANY_MSWIN
    GetSystemInfo(&GC_sysinfo);
#   ifdef ALT_PAGESIZE_USED
      /* Allocations made with mmap() are aligned to the allocation     */
      /* granularity, which (at least on Win64) is not the same as the  */
      /* page size.  Probably we could distinguish the allocation       */
      /* granularity from the actual page size, but in practice there   */
      /* is no good reason to make allocations smaller than             */
      /* dwAllocationGranularity, so we just use it instead of the      */
      /* actual page size here (as Cygwin itself does in many cases).   */
      GC_page_size = (size_t)GC_sysinfo.dwAllocationGranularity;
#     ifdef REAL_PAGESIZE_NEEDED
        GC_real_page_size = (size_t)GC_sysinfo.dwPageSize;
        GC_ASSERT(GC_page_size >= GC_real_page_size);
#     endif
#   else
      GC_page_size = (size_t)GC_sysinfo.dwPageSize;
#   endif
#   if defined(MSWINCE) && !defined(_WIN32_WCE_EMULATION)
      {
        OSVERSIONINFO verInfo;
        /* Check the current WinCE version.     */
        verInfo.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
        if (!GetVersionEx(&verInfo))
          ABORT("GetVersionEx failed");
        if (verInfo.dwPlatformId == VER_PLATFORM_WIN32_CE &&
            verInfo.dwMajorVersion < 6) {
          /* Only the first 32 MB of address space belongs to the       */
          /* current process (unless WinCE 6.0+ or emulation).          */
          GC_sysinfo.lpMaximumApplicationAddress = (LPVOID)((word)32 << 20);
#         ifdef THREADS
            /* On some old WinCE versions, it's observed that           */
            /* VirtualQuery calls don't work properly when used to      */
            /* get thread current stack committed minimum.              */
            if (verInfo.dwMajorVersion < 5)
              GC_dont_query_stack_min = TRUE;
#         endif
        }
      }
#   endif
# else
#   ifdef ALT_PAGESIZE_USED
#     ifdef REAL_PAGESIZE_NEEDED
        GC_real_page_size = (size_t)GETPAGESIZE();
#     endif
      /* It's acceptable to fake it.    */
      GC_page_size = HBLKSIZE;
#   else
      GC_page_size = (size_t)GETPAGESIZE();
#     if !defined(CPPCHECK)
        if (0 == GC_page_size)
          ABORT("getpagesize failed");
#     endif
#   endif
# endif /* !ANY_MSWIN */
# ifdef SOFT_VDB
    {
      size_t pgsize;
      unsigned log_pgsize = 0;

#     if !defined(CPPCHECK)
        if (((GC_page_size-1) & GC_page_size) != 0)
          ABORT("Invalid page size"); /* not a power of two */
#     endif
      for (pgsize = GC_page_size; pgsize > 1; pgsize >>= 1)
        log_pgsize++;
      GC_log_pagesize = log_pgsize;
    }
# endif
}

#ifdef EMBOX
# include <kernel/thread/thread_stack.h>
# include <pthread.h>

  GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
  {
    pthread_t self = pthread_self();
    void *stack_addr = thread_stack_get(self);

    /* TODO: use pthread_getattr_np, pthread_attr_getstack alternatively */
#   ifdef STACK_GROWS_UP
      sb -> mem_base = stack_addr;
#   else
      sb -> mem_base = (ptr_t)stack_addr + thread_stack_get_size(self);
#   endif
    return GC_SUCCESS;
  }
# define HAVE_GET_STACK_BASE
#endif /* EMBOX */

#ifdef HAIKU
# include <kernel/OS.h>

  GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
  {
    thread_info th;
    get_thread_info(find_thread(NULL),&th);
    sb->mem_base = th.stack_end;
    return GC_SUCCESS;
  }
# define HAVE_GET_STACK_BASE
#endif /* HAIKU */

#ifdef OS2
  GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
  {
    PTIB ptib; /* thread information block */
    PPIB ppib;
    if (DosGetInfoBlocks(&ptib, &ppib) != NO_ERROR) {
      WARN("DosGetInfoBlocks failed\n", 0);
      return GC_UNIMPLEMENTED;
    }
    sb->mem_base = ptib->tib_pstacklimit;
    return GC_SUCCESS;
  }
# define HAVE_GET_STACK_BASE
#endif /* OS2 */

#ifdef AMIGA
#   define GC_AMIGA_SB
#   undef GC_AMIGA_SB
#   define GET_MAIN_STACKBASE_SPECIAL
#endif /* AMIGA */

#if defined(NEED_FIND_LIMIT) \
    || (defined(UNIX_LIKE) && !defined(NO_DEBUGGING)) \
    || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS)) \
    || (defined(WRAP_MARK_SOME) && defined(NO_SEH_AVAILABLE))

#   include <signal.h>

#   ifdef USE_SEGV_SIGACT
#     ifndef OPENBSD
        static struct sigaction old_segv_act;
#     endif
#     ifdef USE_BUS_SIGACT
        static struct sigaction old_bus_act;
#     endif
#   else
      static GC_fault_handler_t old_segv_hand;
#     ifdef HAVE_SIGBUS
        static GC_fault_handler_t old_bus_hand;
#     endif
#   endif /* !USE_SEGV_SIGACT */

    GC_INNER void GC_set_and_save_fault_handler(GC_fault_handler_t h)
    {
#       ifdef USE_SEGV_SIGACT
          struct sigaction act;

          act.sa_handler = h;
#         ifdef SIGACTION_FLAGS_NODEFER_HACK
            /* Was necessary for Solaris 2.3 and very temporary */
            /* NetBSD bugs.                                     */
            act.sa_flags = SA_RESTART | SA_NODEFER;
#         else
            act.sa_flags = SA_RESTART;
#         endif

          (void)sigemptyset(&act.sa_mask);
          /* act.sa_restorer is deprecated and should not be initialized. */
#         ifdef GC_IRIX_THREADS
            /* Older versions have a bug related to retrieving and      */
            /* and setting a handler at the same time.                  */
            (void)sigaction(SIGSEGV, 0, &old_segv_act);
            (void)sigaction(SIGSEGV, &act, 0);
#         else
            (void)sigaction(SIGSEGV, &act, &old_segv_act);
#           ifdef USE_BUS_SIGACT
              /* Pthreads doesn't exist under Irix 5.x, so we   */
              /* don't have to worry in the threads case.       */
              (void)sigaction(SIGBUS, &act, &old_bus_act);
#           endif
#         endif /* !GC_IRIX_THREADS */
#       else
          old_segv_hand = signal(SIGSEGV, h);
#         ifdef HAVE_SIGBUS
            old_bus_hand = signal(SIGBUS, h);
#         endif
#       endif /* !USE_SEGV_SIGACT */
#       if defined(CPPCHECK) && defined(ADDRESS_SANITIZER)
          GC_noop1((word)(GC_funcptr_uint)&__asan_default_options);
#       endif
    }
#endif /* NEED_FIND_LIMIT || UNIX_LIKE || WRAP_MARK_SOME */

#if defined(NEED_FIND_LIMIT) \
    || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS)) \
    || (defined(WRAP_MARK_SOME) && defined(NO_SEH_AVAILABLE))
    GC_INNER JMP_BUF GC_jmp_buf;

    STATIC void GC_fault_handler(int sig)
    {
        UNUSED_ARG(sig);
        LONGJMP(GC_jmp_buf, 1);
    }

    GC_INNER void GC_setup_temporary_fault_handler(void)
    {
        /* Handler is process-wide, so this should only happen in       */
        /* one thread at a time.                                        */
        GC_ASSERT(I_HOLD_LOCK());
        GC_set_and_save_fault_handler(GC_fault_handler);
    }

    GC_INNER void GC_reset_fault_handler(void)
    {
#       ifdef USE_SEGV_SIGACT
          (void)sigaction(SIGSEGV, &old_segv_act, 0);
#         ifdef USE_BUS_SIGACT
            (void)sigaction(SIGBUS, &old_bus_act, 0);
#         endif
#       else
          (void)signal(SIGSEGV, old_segv_hand);
#         ifdef HAVE_SIGBUS
            (void)signal(SIGBUS, old_bus_hand);
#         endif
#       endif
    }
#endif /* NEED_FIND_LIMIT || USE_PROC_FOR_LIBRARIES || WRAP_MARK_SOME */

#if defined(NEED_FIND_LIMIT) \
     || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS))
#   define MIN_PAGE_SIZE 256 /* Smallest conceivable page size, in bytes. */

    /* Return the first non-addressable location > p (up) or    */
    /* the smallest location q s.t. [q,p) is addressable (!up). */
    /* We assume that p (up) or p-1 (!up) is addressable.       */
    GC_ATTR_NO_SANITIZE_ADDR
    STATIC ptr_t GC_find_limit_with_bound(ptr_t p, GC_bool up, ptr_t bound)
    {
        static volatile ptr_t result;
                /* Safer if static, since otherwise it may not be   */
                /* preserved across the longjmp.  Can safely be     */
                /* static since it's only called with the allocator */
                /* lock held.                                       */

        GC_ASSERT(up ? ADDR(bound) >= MIN_PAGE_SIZE
                     : ADDR(bound) <= ~(word)MIN_PAGE_SIZE);
        GC_ASSERT(I_HOLD_LOCK());
        GC_setup_temporary_fault_handler();
        if (SETJMP(GC_jmp_buf) == 0) {
            result = PTR_ALIGN_DOWN(p, MIN_PAGE_SIZE);
            for (;;) {
                if (up) {
                    if (ADDR_GE(result, bound - MIN_PAGE_SIZE)) {
                      result = bound;
                      break;
                    }
                    result = result + MIN_PAGE_SIZE;
                        /* no overflow expected; do not use compound        */
                        /* assignment with volatile-qualified left operand  */
                } else {
                    if (ADDR_GE(bound + MIN_PAGE_SIZE, result)) {
                      result = bound - MIN_PAGE_SIZE;
                                        /* This is to compensate        */
                                        /* further result increment (we */
                                        /* do not modify "up" variable  */
                                        /* since it might be clobbered  */
                                        /* by setjmp otherwise).        */
                      break;
                    }
                    result = result - MIN_PAGE_SIZE;
                        /* no underflow expected; do not use compound       */
                        /* assignment with volatile-qualified left operand  */
                }
                GC_noop1((word)(unsigned char)(*result));
            }
        }
        GC_reset_fault_handler();
        return up ? result : result + MIN_PAGE_SIZE;
    }

    void * GC_find_limit(void * p, int up)
    {
        return GC_find_limit_with_bound((ptr_t)p, (GC_bool)up,
                                        up ? MAKE_CPTR(GC_WORD_MAX) : 0);
    }
#endif /* NEED_FIND_LIMIT || USE_PROC_FOR_LIBRARIES */

#ifdef HPUX_MAIN_STACKBOTTOM
# include <sys/param.h>
# include <sys/pstat.h>

  STATIC ptr_t GC_hpux_main_stack_base(void)
  {
    struct pst_vm_status vm_status;
    int i = 0;

    while (pstat_getprocvm(&vm_status, sizeof(vm_status), 0, i++) == 1) {
      if (vm_status.pst_type == PS_STACK)
        return (ptr_t)vm_status.pst_vaddr;
    }

    /* Old way to get the stack bottom. */
#   ifdef STACK_GROWS_UP
      return (ptr_t)GC_find_limit(GC_approx_sp(), /* up= */ FALSE);
#   else /* not HP_PA */
      return (ptr_t)GC_find_limit(GC_approx_sp(), TRUE);
#   endif
  }
#endif /* HPUX_MAIN_STACKBOTTOM */

#ifdef HPUX_STACKBOTTOM

#include <sys/param.h>
#include <sys/pstat.h>

  GC_INNER ptr_t GC_get_register_stack_base(void)
  {
    struct pst_vm_status vm_status;

    int i = 0;
    while (pstat_getprocvm(&vm_status, sizeof(vm_status), 0, i++) == 1) {
      if (vm_status.pst_type == PS_RSESTACK) {
        return (ptr_t) vm_status.pst_vaddr;
      }
    }

    /* old way to get the register stackbottom */
    GC_ASSERT(GC_stackbottom != NULL);
    return PTR_ALIGN_DOWN(GC_stackbottom - BACKING_STORE_DISPLACEMENT - 1,
                          BACKING_STORE_ALIGNMENT);
  }

#endif /* HPUX_STACK_BOTTOM */

#ifdef LINUX_STACKBOTTOM

# include <sys/stat.h>

# define STAT_SKIP 27   /* Number of fields preceding startstack        */
                        /* field in /proc/self/stat                     */

# ifdef USE_LIBC_PRIVATES
    EXTERN_C_BEGIN
#   pragma weak __libc_stack_end
    extern ptr_t __libc_stack_end;
#   ifdef IA64
#     pragma weak __libc_ia64_register_backing_store_base
      extern ptr_t __libc_ia64_register_backing_store_base;
#   endif
    EXTERN_C_END
# endif

# ifdef IA64
    GC_INNER ptr_t GC_get_register_stack_base(void)
    {
      ptr_t result;

      GC_ASSERT(I_HOLD_LOCK());
#     ifdef USE_LIBC_PRIVATES
        {
          ptr_t *p_libc_ia64_register_backing_store_base
                                = &__libc_ia64_register_backing_store_base;

#         if defined(CPPCHECK)
            /* Workaround a warning that the address of the global  */
            /* symbol (which is a weak one) cannot be null.         */
            GC_noop1_ptr(&p_libc_ia64_register_backing_store_base);
#         endif
          if (p_libc_ia64_register_backing_store_base != NULL
              && __libc_ia64_register_backing_store_base != NULL) {
            /* glibc 2.2.4 has a bug such that for dynamically linked   */
            /* executables __libc_ia64_register_backing_store_base is   */
            /* defined but uninitialized during constructor calls.      */
            /* Hence we check for both nonzero address and value.       */
            return __libc_ia64_register_backing_store_base;
          }
        }
#     endif
      result = backing_store_base_from_proc();
      if (0 == result) {
          result = (ptr_t)GC_find_limit(GC_save_regs_in_stack(), FALSE);
          /* This works better than a constant displacement heuristic.  */
      }
      return result;
    }
# endif /* IA64 */

  STATIC ptr_t GC_linux_main_stack_base(void)
  {
    /* We read the stack bottom value from /proc/self/stat.  We do this */
    /* using direct I/O system calls in order to avoid calling malloc   */
    /* in case REDIRECT_MALLOC is defined.                              */
#     define STAT_BUF_SIZE 4096
    unsigned char stat_buf[STAT_BUF_SIZE];
    int f;
    word addr;
    ssize_t i, buf_offset = 0, len;

    /* First try the easy way.  This should work for glibc 2.2. */
    /* This fails in a prelinked ("prelink" command) executable */
    /* since the correct value of __libc_stack_end never        */
    /* becomes visible to us.  The second test works around     */
    /* this.                                                    */
#   ifdef USE_LIBC_PRIVATES
      ptr_t *p_libc_stack_end = &__libc_stack_end;

#     if defined(CPPCHECK)
        GC_noop1_ptr(&p_libc_stack_end);
#     endif
      if (p_libc_stack_end != NULL && __libc_stack_end != NULL) {
#       if defined(IA64)
          /* Some versions of glibc set the address 16 bytes too        */
          /* low while the initialization code is running.              */
          if ((ADDR(__libc_stack_end) & 0xfff) + 0x10 < 0x1000) {
            return __libc_stack_end + 0x10;
          } /* Otherwise it's not safe to add 16 bytes and we fall      */
            /* back to using /proc.                                     */
#       elif defined(SPARC)
          /* Older versions of glibc for 64-bit SPARC do not set this   */
          /* variable correctly, it gets set to either zero or one.     */
          if (ADDR(__libc_stack_end) != 1)
            return __libc_stack_end;
#       else
          return __libc_stack_end;
#       endif
      }
#   endif

    f = open("/proc/self/stat", O_RDONLY);
    if (-1 == f)
      ABORT_ARG1("Could not open /proc/self/stat", ": errno= %d", errno);
    len = GC_repeat_read(f, (char*)stat_buf, sizeof(stat_buf));
    if (len < 0)
      ABORT_ARG1("Failed to read /proc/self/stat",
                 ": errno= %d", errno);
    close(f);

    /* Skip the required number of fields.  This number is hopefully    */
    /* constant across all Linux implementations.                       */
    for (i = 0; i < STAT_SKIP; ++i) {
      while (buf_offset < len && isspace(stat_buf[buf_offset++])) {
        /* empty */
      }
      while (buf_offset < len && !isspace(stat_buf[buf_offset++])) {
        /* empty */
      }
    }
    /* Skip spaces.     */
    while (buf_offset < len && isspace(stat_buf[buf_offset])) {
      buf_offset++;
    }
    /* Find the end of the number and cut the buffer there.     */
    for (i = 0; buf_offset + i < len; i++) {
      if (!isdigit(stat_buf[buf_offset + i])) break;
    }
    if (buf_offset + i >= len) ABORT("Could not parse /proc/self/stat");
    stat_buf[buf_offset + i] = '\0';

    addr = (word)STRTOULL((char *)stat_buf + buf_offset, NULL, 10);
    if (addr < 0x100000 || (addr & (sizeof(ptr_t)-1)) != 0)
      ABORT_ARG1("Absurd stack bottom value", ": 0x%lx", (unsigned long)addr);
    return MAKE_CPTR(addr);
  }
#endif /* LINUX_STACKBOTTOM */

#ifdef QNX_STACKBOTTOM
  STATIC ptr_t GC_qnx_main_stack_base(void)
  {
    /* TODO: this approach is not very exact but it works for the       */
    /* tests, at least, unlike other available heuristics.              */
    return (ptr_t)__builtin_frame_address(0);
  }
#endif /* QNX_STACKBOTTOM */

#ifdef FREEBSD_STACKBOTTOM
  /* This uses an undocumented sysctl call, but at least one expert     */
  /* believes it will stay.                                             */

# include <sys/sysctl.h>

  STATIC ptr_t GC_freebsd_main_stack_base(void)
  {
    int nm[2] = {CTL_KERN, KERN_USRSTACK};
    ptr_t base;
    size_t len = sizeof(ptr_t);
    int r = sysctl(nm, 2, &base, &len, NULL, 0);
    if (r) ABORT("Error getting main stack base");
    return base;
  }
#endif /* FREEBSD_STACKBOTTOM */

#if defined(ECOS) || defined(NOSYS)
  ptr_t GC_get_main_stack_base(void)
  {
    return STACKBOTTOM;
  }
# define GET_MAIN_STACKBASE_SPECIAL
#elif defined(SYMBIAN)
  EXTERN_C_BEGIN
  extern int GC_get_main_symbian_stack_base(void);
  EXTERN_C_END

  ptr_t GC_get_main_stack_base(void)
  {
    return (ptr_t)GC_get_main_symbian_stack_base();
  }
# define GET_MAIN_STACKBASE_SPECIAL
#elif defined(EMSCRIPTEN)
# include <emscripten/stack.h>

  ptr_t GC_get_main_stack_base(void)
  {
    return (ptr_t)emscripten_stack_get_base();
  }
# define GET_MAIN_STACKBASE_SPECIAL
#elif !defined(AMIGA) && !defined(EMBOX) && !defined(HAIKU) && !defined(OS2) \
      && !defined(ANY_MSWIN) && !defined(GC_OPENBSD_THREADS) \
      && (!defined(GC_SOLARIS_THREADS) || defined(_STRICT_STDC))

# if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
     && (defined(THREADS) || defined(USE_GET_STACKBASE_FOR_MAIN))
#   include <pthread.h>
#   ifdef HAVE_PTHREAD_NP_H
#     include <pthread_np.h> /* for pthread_attr_get_np() */
#   endif
# elif defined(DARWIN) && !defined(NO_PTHREAD_GET_STACKADDR_NP)
    /* We could use pthread_get_stackaddr_np even in case of a  */
    /* single-threaded gclib (there is no -lpthread on Darwin). */
#   include <pthread.h>
#   undef STACKBOTTOM
#   define STACKBOTTOM (ptr_t)pthread_get_stackaddr_np(pthread_self())
# endif

  ptr_t GC_get_main_stack_base(void)
  {
    ptr_t result;
#   if (defined(HAVE_PTHREAD_ATTR_GET_NP) \
        || defined(HAVE_PTHREAD_GETATTR_NP)) \
       && (defined(USE_GET_STACKBASE_FOR_MAIN) \
           || (defined(THREADS) && !defined(REDIRECT_MALLOC)))
      pthread_attr_t attr;
      void *stackaddr;
      size_t size;

#     ifdef HAVE_PTHREAD_ATTR_GET_NP
        if (pthread_attr_init(&attr) == 0
            && (pthread_attr_get_np(pthread_self(), &attr) == 0
                ? TRUE : (pthread_attr_destroy(&attr), FALSE)))
#     else /* HAVE_PTHREAD_GETATTR_NP */
        if (pthread_getattr_np(pthread_self(), &attr) == 0)
#     endif
      {
        if (pthread_attr_getstack(&attr, &stackaddr, &size) == 0
            && stackaddr != NULL) {
          (void)pthread_attr_destroy(&attr);
#         ifndef STACK_GROWS_UP
            stackaddr = (char *)stackaddr + size;
#         endif
          return (ptr_t)stackaddr;
        }
        (void)pthread_attr_destroy(&attr);
      }
      WARN("pthread_getattr_np or pthread_attr_getstack failed"
           " for main thread\n", 0);
#   endif
#   ifdef STACKBOTTOM
      result = STACKBOTTOM;
#   else
#     ifdef HEURISTIC1
#       define STACKBOTTOM_ALIGNMENT_M1 ((word)STACK_GRAN - 1)
#       ifdef STACK_GROWS_UP
          result = PTR_ALIGN_DOWN(GC_approx_sp(),
                                  STACKBOTTOM_ALIGNMENT_M1 + 1);
#       else
          result = PTR_ALIGN_UP(GC_approx_sp(), STACKBOTTOM_ALIGNMENT_M1 + 1);
#       endif
#     elif defined(HPUX_MAIN_STACKBOTTOM)
        result = GC_hpux_main_stack_base();
#     elif defined(LINUX_STACKBOTTOM)
        result = GC_linux_main_stack_base();
#     elif defined(QNX_STACKBOTTOM)
        result = GC_qnx_main_stack_base();
#     elif defined(FREEBSD_STACKBOTTOM)
        result = GC_freebsd_main_stack_base();
#     elif defined(HEURISTIC2)
        {
          ptr_t sp = GC_approx_sp();

#         ifdef STACK_GROWS_UP
            result = (ptr_t)GC_find_limit(sp, /* up= */ FALSE);
#         else
            result = (ptr_t)GC_find_limit(sp, TRUE);
#         endif
#         if defined(HEURISTIC2_LIMIT) && !defined(CPPCHECK)
            if (HOTTER_THAN(HEURISTIC2_LIMIT, result)
                && HOTTER_THAN(sp, HEURISTIC2_LIMIT))
              result = HEURISTIC2_LIMIT;
#         endif
        }
#     elif defined(STACK_NOT_SCANNED) || defined(CPPCHECK)
        result = NULL;
#     else
#       error None of HEURISTIC* and *STACKBOTTOM defined!
#     endif
#     if !defined(STACK_GROWS_UP) && !defined(CPPCHECK)
        if (NULL == result)
          result = MAKE_CPTR((signed_word)(-sizeof(ptr_t)));
#     endif
#   endif
#   if !defined(CPPCHECK)
      GC_ASSERT(HOTTER_THAN(GC_approx_sp(), result));
#   endif
    return result;
  }
# define GET_MAIN_STACKBASE_SPECIAL
#endif /* !AMIGA && !ANY_MSWIN && !HAIKU && !GC_OPENBSD_THREADS && !OS2 */

#if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
    && defined(THREADS) && !defined(HAVE_GET_STACK_BASE)
# include <pthread.h>
# ifdef HAVE_PTHREAD_NP_H
#   include <pthread_np.h>
# endif

  GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b)
  {
    pthread_attr_t attr;
    size_t size;

#   ifdef HAVE_PTHREAD_ATTR_GET_NP
      if (pthread_attr_init(&attr) != 0)
        ABORT("pthread_attr_init failed");
      if (pthread_attr_get_np(pthread_self(), &attr) != 0) {
        WARN("pthread_attr_get_np failed\n", 0);
        (void)pthread_attr_destroy(&attr);
        return GC_UNIMPLEMENTED;
      }
#   else /* HAVE_PTHREAD_GETATTR_NP */
      if (pthread_getattr_np(pthread_self(), &attr) != 0) {
        WARN("pthread_getattr_np failed\n", 0);
        return GC_UNIMPLEMENTED;
      }
#   endif
    if (pthread_attr_getstack(&attr, &(b -> mem_base), &size) != 0) {
        ABORT("pthread_attr_getstack failed");
    }
    (void)pthread_attr_destroy(&attr);
#   ifndef STACK_GROWS_UP
        b -> mem_base = (char *)(b -> mem_base) + size;
#   endif
#   ifdef IA64
      /* We could try backing_store_base_from_proc, but that's safe     */
      /* only if no mappings are being asynchronously created.          */
      /* Subtracting the size from the stack base doesn't work for at   */
      /* least the main thread.                                         */
      LOCK();
      {
        IF_CANCEL(int cancel_state;)
        ptr_t bsp;
        ptr_t next_stack;

        DISABLE_CANCEL(cancel_state);
        bsp = GC_save_regs_in_stack();
        next_stack = GC_greatest_stack_base_below(bsp);
        if (0 == next_stack) {
          b -> reg_base = GC_find_limit(bsp, FALSE);
        } else {
          /* Avoid walking backwards into preceding memory stack and    */
          /* growing it.                                                */
          b -> reg_base = GC_find_limit_with_bound(bsp, FALSE, next_stack);
        }
        RESTORE_CANCEL(cancel_state);
      }
      UNLOCK();
#   elif defined(E2K)
      b -> reg_base = NULL;
#   endif
    return GC_SUCCESS;
  }
# define HAVE_GET_STACK_BASE
#endif /* THREADS && (HAVE_PTHREAD_ATTR_GET_NP || HAVE_PTHREAD_GETATTR_NP) */

#if defined(GC_DARWIN_THREADS) && !defined(NO_PTHREAD_GET_STACKADDR_NP)
# include <pthread.h>

  GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b)
  {
    /* pthread_get_stackaddr_np() should return stack bottom (highest   */
    /* stack address plus 1).                                           */
    b -> mem_base = pthread_get_stackaddr_np(pthread_self());
    GC_ASSERT(HOTTER_THAN(GC_approx_sp(), (ptr_t)(b -> mem_base)));
    return GC_SUCCESS;
  }
# define HAVE_GET_STACK_BASE
#endif /* GC_DARWIN_THREADS */

#ifdef GC_OPENBSD_THREADS
# include <sys/signal.h>
# include <pthread.h>
# include <pthread_np.h>

  /* Find the stack using pthread_stackseg_np(). */
  GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
  {
    stack_t stack;
    if (pthread_stackseg_np(pthread_self(), &stack))
      ABORT("pthread_stackseg_np(self) failed");
    sb->mem_base = stack.ss_sp;
    return GC_SUCCESS;
  }
# define HAVE_GET_STACK_BASE
#endif /* GC_OPENBSD_THREADS */

#if defined(GC_SOLARIS_THREADS) && !defined(_STRICT_STDC)

# include <thread.h>
# include <pthread.h>

  /* These variables are used to cache ss_sp value for the primordial   */
  /* thread (it's better not to call thr_stksegment() twice for this    */
  /* thread - see JDK bug #4352906).                                    */
  static pthread_t stackbase_main_self = 0;
                        /* 0 means stackbase_main_ss_sp value is unset. */
  static void *stackbase_main_ss_sp = NULL;

  GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b)
  {
    stack_t s;
    pthread_t self = pthread_self();

    if (self == stackbase_main_self)
      {
        /* If the client calls GC_get_stack_base() from the main thread */
        /* then just return the cached value.                           */
        b -> mem_base = stackbase_main_ss_sp;
        GC_ASSERT(b -> mem_base != NULL);
        return GC_SUCCESS;
      }

    if (thr_stksegment(&s)) {
      /* According to the manual, the only failure error code returned  */
      /* is EAGAIN meaning "the information is not available due to the */
      /* thread is not yet completely initialized or it is an internal  */
      /* thread" - this shouldn't happen here.                          */
      ABORT("thr_stksegment failed");
    }
    /* s.ss_sp holds the pointer to the stack bottom. */
    GC_ASSERT(HOTTER_THAN(GC_approx_sp(), (ptr_t)s.ss_sp));

    if (!stackbase_main_self && thr_main() != 0)
      {
        /* Cache the stack bottom pointer for the primordial thread     */
        /* (this is done during GC_init, so there is no race).          */
        stackbase_main_ss_sp = s.ss_sp;
        stackbase_main_self = self;
      }

    b -> mem_base = s.ss_sp;
    return GC_SUCCESS;
  }
# define HAVE_GET_STACK_BASE
#endif /* GC_SOLARIS_THREADS */

#ifdef GC_RTEMS_PTHREADS
  GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb)
  {
    sb->mem_base = rtems_get_stack_bottom();
    return GC_SUCCESS;
  }
# define HAVE_GET_STACK_BASE
#endif /* GC_RTEMS_PTHREADS */

#ifndef HAVE_GET_STACK_BASE
# ifdef NEED_FIND_LIMIT
    /* Retrieve the stack bottom.                                       */
    /* Using the GC_find_limit version is risky.                        */
    /* On IA64, for example, there is no guard page between the         */
    /* stack of one thread and the register backing store of the        */
    /* next.  Thus this is likely to identify way too large a           */
    /* "stack" and thus at least result in disastrous performance.      */
    /* TODO: Implement better strategies here. */
    GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b)
    {
      IF_CANCEL(int cancel_state;)

      LOCK();
      DISABLE_CANCEL(cancel_state);  /* May be unnecessary? */
#     ifdef STACK_GROWS_UP
        b -> mem_base = GC_find_limit(GC_approx_sp(), /* up= */ FALSE);
#     else
        b -> mem_base = GC_find_limit(GC_approx_sp(), TRUE);
#     endif
#     ifdef IA64
        b -> reg_base = GC_find_limit(GC_save_regs_in_stack(), FALSE);
#     elif defined(E2K)
        b -> reg_base = NULL;
#     endif
      RESTORE_CANCEL(cancel_state);
      UNLOCK();
      return GC_SUCCESS;
    }
# else
    GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b)
    {
#     if defined(GET_MAIN_STACKBASE_SPECIAL) && !defined(THREADS) \
         && !defined(IA64)
        b->mem_base = GC_get_main_stack_base();
        return GC_SUCCESS;
#     else
        UNUSED_ARG(b);
        return GC_UNIMPLEMENTED;
#     endif
    }
# endif /* !NEED_FIND_LIMIT */
#endif /* !HAVE_GET_STACK_BASE */

#ifndef GET_MAIN_STACKBASE_SPECIAL
  /* This is always called from the main thread.  Default implementation. */
  ptr_t GC_get_main_stack_base(void)
  {
    struct GC_stack_base sb;

    if (GC_get_stack_base(&sb) != GC_SUCCESS)
      ABORT("GC_get_stack_base failed");
    GC_ASSERT(HOTTER_THAN(GC_approx_sp(), (ptr_t)sb.mem_base));
    return (ptr_t)sb.mem_base;
  }
#endif /* !GET_MAIN_STACKBASE_SPECIAL */

/* Register static data segment(s) as roots.  If more data segments are */
/* added later then they need to be registered at that point (as we do  */
/* with SunOS dynamic loading), or GC_mark_roots needs to check for     */
/* them (as we do with PCR).                                            */

#ifdef OS2

  void GC_register_data_segments(void)
  {
    PTIB ptib;
    PPIB ppib;
    HMODULE module_handle;
#   define PBUFSIZ 512
    UCHAR path[PBUFSIZ];
    FILE * myexefile;
    struct exe_hdr hdrdos;      /* MSDOS header.        */
    struct e32_exe hdr386;      /* Real header for my executable */
    struct o32_obj seg;         /* Current segment */
    int nsegs;

#   if defined(CPPCHECK)
        hdrdos.padding[0] = 0; /* to prevent "field unused" warnings */
        hdr386.exe_format_level = 0;
        hdr386.os = 0;
        hdr386.padding1[0] = 0;
        hdr386.padding2[0] = 0;
        seg.pagemap = 0;
        seg.mapsize = 0;
        seg.reserved = 0;
#   endif
    if (DosGetInfoBlocks(&ptib, &ppib) != NO_ERROR) {
        ABORT("DosGetInfoBlocks failed");
    }
    module_handle = ppib -> pib_hmte;
    if (DosQueryModuleName(module_handle, PBUFSIZ, path) != NO_ERROR) {
        ABORT("DosQueryModuleName failed");
    }
    myexefile = fopen(path, "rb");
    if (myexefile == 0) {
        ABORT_ARG1("Failed to open executable", ": %s", path);
    }
    if (fread((char *)&hdrdos, 1, sizeof(hdrdos), myexefile)
          < sizeof(hdrdos)) {
        ABORT_ARG1("Could not read MSDOS header", " from: %s", path);
    }
    if (E_MAGIC(hdrdos) != EMAGIC) {
        ABORT_ARG1("Bad DOS magic number", " in file: %s", path);
    }
    if (fseek(myexefile, E_LFANEW(hdrdos), SEEK_SET) != 0) {
        ABORT_ARG1("Bad DOS magic number", " in file: %s", path);
    }
    if (fread((char *)&hdr386, 1, sizeof(hdr386), myexefile)
          < sizeof(hdr386)) {
        ABORT_ARG1("Could not read OS/2 header", " from: %s", path);
    }
    if (E32_MAGIC1(hdr386) != E32MAGIC1 || E32_MAGIC2(hdr386) != E32MAGIC2) {
        ABORT_ARG1("Bad OS/2 magic number", " in file: %s", path);
    }
    if (E32_BORDER(hdr386) != E32LEBO || E32_WORDER(hdr386) != E32LEWO) {
        ABORT_ARG1("Bad byte order in executable", " file: %s", path);
    }
    if (E32_CPU(hdr386) == E32CPU286) {
        ABORT_ARG1("GC cannot handle 80286 executables", ": %s", path);
    }
    if (fseek(myexefile, E_LFANEW(hdrdos) + E32_OBJTAB(hdr386),
              SEEK_SET) != 0) {
        ABORT_ARG1("Seek to object table failed", " in file: %s", path);
    }
    for (nsegs = E32_OBJCNT(hdr386); nsegs > 0; nsegs--) {
      int flags;
      if (fread((char *)&seg, 1, sizeof(seg), myexefile) < sizeof(seg)) {
        ABORT_ARG1("Could not read obj table entry", " from file: %s", path);
      }
      flags = O32_FLAGS(seg);
      if (!(flags & OBJWRITE)) continue;
      if (!(flags & OBJREAD)) continue;
      if (flags & OBJINVALID) {
          GC_err_printf("Object with invalid pages?\n");
          continue;
      }
      GC_add_roots_inner((ptr_t)O32_BASE(seg),
                         (ptr_t)(O32_BASE(seg)+O32_SIZE(seg)), FALSE);
    }
    (void)fclose(myexefile);
  }

#else /* !OS2 */

# if defined(GWW_VDB)
#   ifndef MEM_WRITE_WATCH
#     define MEM_WRITE_WATCH 0x200000
#   endif
#   ifndef WRITE_WATCH_FLAG_RESET
#     define WRITE_WATCH_FLAG_RESET 1
#   endif

    /* Since we can't easily check whether ULONG_PTR and SIZE_T are     */
    /* defined in Win32 basetsd.h, we define own ULONG_PTR.             */
#   define GC_ULONG_PTR word

    typedef UINT (WINAPI * GetWriteWatch_type)(
                                DWORD, PVOID, GC_ULONG_PTR /* SIZE_T */,
                                PVOID *, GC_ULONG_PTR *, PULONG);
    static FARPROC GetWriteWatch_func;
    static DWORD GetWriteWatch_alloc_flag;

#   define GC_GWW_AVAILABLE() (GetWriteWatch_func != 0)

    static void detect_GetWriteWatch(void)
    {
      static GC_bool done;
      HMODULE hK32;
      if (done)
        return;

#     if defined(MPROTECT_VDB)
        {
          char * str = GETENV("GC_USE_GETWRITEWATCH");
#         if defined(GC_PREFER_MPROTECT_VDB)
            if (str == NULL || (*str == '0' && *(str + 1) == '\0')) {
              /* GC_USE_GETWRITEWATCH is unset or set to "0".           */
              done = TRUE; /* falling back to MPROTECT_VDB strategy.    */
              /* This should work as if GWW_VDB is undefined. */
              return;
            }
#         else
            if (str != NULL && *str == '0' && *(str + 1) == '\0') {
              /* GC_USE_GETWRITEWATCH is set "0".                       */
              done = TRUE; /* falling back to MPROTECT_VDB strategy.    */
              return;
            }
#         endif
        }
#     endif

#     if defined(MSWINRT_FLAVOR) && defined(FUNCPTR_IS_DATAPTR)
        {
          MEMORY_BASIC_INFORMATION memInfo;
          SIZE_T result
                    = VirtualQuery(CAST_THRU_UINTPTR(void*, GetProcAddress),
                                   &memInfo, sizeof(memInfo));

          if (result != sizeof(memInfo))
            ABORT("Weird VirtualQuery result");
          hK32 = (HMODULE)memInfo.AllocationBase;
        }
#     else
        hK32 = GetModuleHandle(TEXT("kernel32.dll"));
#     endif
      if (hK32 != (HMODULE)0 &&
          (GetWriteWatch_func = GetProcAddress(hK32, "GetWriteWatch")) != 0) {
        /* Also check whether VirtualAlloc accepts MEM_WRITE_WATCH,   */
        /* as some versions of kernel32.dll have one but not the      */
        /* other, making the feature completely broken.               */
        void * page;

        GC_ASSERT(GC_page_size != 0);
        page = VirtualAlloc(NULL, GC_page_size, MEM_WRITE_WATCH | MEM_RESERVE,
                            PAGE_READWRITE);
        if (page != NULL) {
          PVOID pages[16];
          GC_ULONG_PTR count = sizeof(pages) / sizeof(PVOID);
          DWORD page_size;
          /* Check that it actually works.  In spite of some            */
          /* documentation it actually seems to exist on Win2K.         */
          /* This test may be unnecessary, but ...                      */
          if ((*(GetWriteWatch_type)(GC_funcptr_uint)GetWriteWatch_func)(
                                        WRITE_WATCH_FLAG_RESET, page,
                                        GC_page_size, pages, &count,
                                        &page_size) != 0) {
            /* GetWriteWatch always fails. */
            GetWriteWatch_func = 0;
          } else {
            GetWriteWatch_alloc_flag = MEM_WRITE_WATCH;
          }
          VirtualFree(page, 0 /* dwSize */, MEM_RELEASE);
        } else {
          /* GetWriteWatch will be useless. */
          GetWriteWatch_func = 0;
        }
      }
      done = TRUE;
    }

# else
#   define GetWriteWatch_alloc_flag 0
# endif /* !GWW_VDB */

# ifdef MSWIN32
    /* Unfortunately, we have to handle win32s very differently from    */
    /* NT, since VirtualQuery has very different semantics.  In         */
    /* particular, under win32s a VirtualQuery call on an unmapped page */
    /* returns an invalid result.  Under NT, GC_register_data_segments  */
    /* is a no-op and all real work is done by                          */
    /* GC_register_dynamic_libraries().  Under win32s, we cannot find   */
    /* the data segments associated with dll's.  We register the main   */
    /* data segment here.                                               */

    GC_INNER GC_bool GC_no_win32_dlls = FALSE;
                        /* This used to be set for gcc, to avoid        */
                        /* dealing with the structured exception        */
                        /* handling issues.  But we now have assembly   */
                        /* code to do that right.                       */

    GC_INNER GC_bool GC_wnt = FALSE;
                        /* This is a Windows NT derivative, i.e. NT,    */
                        /* Win2K, XP or later.                          */

    GC_INNER void GC_init_win32(void)
    {
#     if defined(_WIN64) || (defined(_MSC_VER) && _MSC_VER >= 1800)
        /* MS Visual Studio 2013 deprecates GetVersion, but on the      */
        /* other hand it cannot be used to target pre-Win2K.            */
        GC_wnt = TRUE;
#     else
        /* Set GC_wnt.  If we're running under win32s, assume that no     */
        /* DLLs will be loaded.  I doubt anyone still runs win32s, but... */
        DWORD v = GetVersion();

        GC_wnt = !(v & (DWORD)0x80000000UL);
        GC_no_win32_dlls |= ((!GC_wnt) && (v & 0xff) <= 3);
#     endif
#     ifdef USE_MUNMAP
        if (GC_no_win32_dlls) {
          /* Turn off unmapping for safety (since may not work well     */
          /* with GlobalAlloc).                                         */
          GC_unmap_threshold = 0;
        }
#     endif
    }

    /* Return the smallest address p such that VirtualQuery returns     */
    /* correct results for all addresses between p and start.  Assumes  */
    /* VirtualQuery() returns correct information for start.            */
    STATIC ptr_t GC_least_described_address(ptr_t start)
    {
      ptr_t limit = (ptr_t)GC_sysinfo.lpMinimumApplicationAddress;
      ptr_t p = PTR_ALIGN_DOWN(start, GC_page_size);

      GC_ASSERT(GC_page_size != 0);
      for (;;) {
        MEMORY_BASIC_INFORMATION buf;
        size_t result;
        ptr_t q;

        if (EXPECT(ADDR(p) <= (word)GC_page_size, FALSE))
          break; /* prevent underflow */
        q = p - GC_page_size;
        if (ADDR_LT(q, limit)) break;

        result = VirtualQuery((LPVOID)q, &buf, sizeof(buf));
        if (result != sizeof(buf) || 0 == buf.AllocationBase) break;
        p = (ptr_t)buf.AllocationBase;
      }
      return p;
    }

    STATIC void GC_register_root_section(ptr_t static_root)
    {
      ptr_t p, base, limit;

      GC_ASSERT(I_HOLD_LOCK());
      if (!GC_no_win32_dlls) return;

      p = GC_least_described_address(static_root);
      base = limit = p;
      while (ADDR_LT(p, (ptr_t)GC_sysinfo.lpMaximumApplicationAddress)) {
        MEMORY_BASIC_INFORMATION buf;
        size_t result = VirtualQuery((LPVOID)p, &buf, sizeof(buf));

        if (result != sizeof(buf) || 0 == buf.AllocationBase
            || GC_is_heap_base(buf.AllocationBase)) break;
        if (ADDR(p) > GC_WORD_MAX - buf.RegionSize) break; /* overflow */

        if (buf.State == MEM_COMMIT && is_writable(buf.Protect)) {
          if (p != limit) {
            if (base != limit) GC_add_roots_inner(base, limit, FALSE);
            base = p;
          }
          limit = p + buf.RegionSize;
        }
        p += buf.RegionSize;
      }
      if (base != limit) GC_add_roots_inner(base, limit, FALSE);
    }
# endif /* MSWIN32 */

# ifdef ANY_MSWIN

#   if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC)
      /* We maintain a linked list of AllocationBase values that we */
      /* know correspond to malloc heap sections.  Currently this   */
      /* is only called during a GC.  But there is some hope that   */
      /* for long running programs we will eventually see most heap */
      /* sections.                                                  */

      /* In the long run, it would be more reliable to occasionally */
      /* walk the malloc heap with HeapWalk on the default heap.    */
      /* But that apparently works only for NT-based Windows.       */

      STATIC size_t GC_max_root_size = 100000; /* approx. largest root size */

      /* In the long run, a better data structure would also be nice... */
      STATIC struct GC_malloc_heap_list {
        void * allocation_base;
        struct GC_malloc_heap_list *next;
      } *GC_malloc_heap_l = 0;

      /* Is p the base of one of the malloc heap sections we already    */
      /* know about?                                                    */
      STATIC GC_bool GC_is_malloc_heap_base(const void *p)
      {
        struct GC_malloc_heap_list *q;

        for (q = GC_malloc_heap_l; q != NULL; q = q -> next) {
          if (q -> allocation_base == p) return TRUE;
        }
        return FALSE;
      }

      STATIC void *GC_get_allocation_base(void *p)
      {
        MEMORY_BASIC_INFORMATION buf;
        size_t result = VirtualQuery(p, &buf, sizeof(buf));

        if (result != sizeof(buf)) {
          ABORT("Weird VirtualQuery result");
        }
        return buf.AllocationBase;
      }

      GC_INNER void GC_add_current_malloc_heap(void)
      {
        struct GC_malloc_heap_list *new_l = (struct GC_malloc_heap_list *)
                                malloc(sizeof(struct GC_malloc_heap_list));
        void *candidate;

        if (NULL == new_l) return;
        new_l -> allocation_base = NULL;
                        /* to suppress maybe-uninitialized gcc warning */

        candidate = GC_get_allocation_base(new_l);
        if (GC_is_malloc_heap_base(candidate)) {
          /* Try a little harder to find malloc heap.   */
          size_t req_size = 10000;

          do {
            void *p = malloc(req_size);

            if (NULL == p) {
              free(new_l);
              return;
            }
            candidate = GC_get_allocation_base(p);
            free(p);
            req_size *= 2;
          } while (GC_is_malloc_heap_base(candidate)
                   && req_size < GC_max_root_size / 10 && req_size < 500000);
          if (GC_is_malloc_heap_base(candidate)) {
            free(new_l);
            return;
          }
        }
        GC_COND_LOG_PRINTF("Found new system malloc AllocationBase at %p\n",
                           candidate);
        new_l -> allocation_base = candidate;
        new_l -> next = GC_malloc_heap_l;
        GC_malloc_heap_l = new_l;
      }

      /* Free all the linked list nodes.  Could be invoked at process   */
      /* exit to avoid memory leak complains of a dynamic code analysis */
      /* tool.                                                          */
      STATIC void GC_free_malloc_heap_list(void)
      {
        struct GC_malloc_heap_list *q = GC_malloc_heap_l;

        GC_malloc_heap_l = NULL;
        while (q != NULL) {
          struct GC_malloc_heap_list *next = q -> next;

          free(q);
          q = next;
        }
      }
#   endif /* USE_WINALLOC && !REDIRECT_MALLOC */

    /* Is p the start of either the malloc heap, or of one of our   */
    /* heap sections?                                               */
    GC_INNER GC_bool GC_is_heap_base(const void *p)
    {
      size_t i;

#     if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC)
        if (GC_root_size > GC_max_root_size)
          GC_max_root_size = GC_root_size;
        if (GC_is_malloc_heap_base(p))
          return TRUE;
#     endif
      for (i = 0; i < GC_n_heap_bases; i++) {
        if (GC_heap_bases[i] == p) return TRUE;
      }
      return FALSE;
    }

    void GC_register_data_segments(void)
    {
#     ifdef MSWIN32
        GC_register_root_section((ptr_t)&GC_pages_executable);
                            /* any other GC global variable would fit too */
#     endif
    }

# else /* !ANY_MSWIN */

#   if (defined(SVR4) || defined(AIX) || defined(DGUX)) && !defined(PCR)
      ptr_t GC_SysVGetDataStart(size_t max_page_size, ptr_t etext_addr)
      {
        word page_offset = ADDR(PTR_ALIGN_UP(etext_addr, sizeof(ptr_t)))
                            & ((word)max_page_size - 1);
        volatile ptr_t result = PTR_ALIGN_UP(etext_addr, max_page_size)
                                + page_offset;
        /* Note that this is not equivalent to just adding          */
        /* max_page_size to &etext if etext is at a page boundary.  */

        GC_ASSERT(max_page_size % sizeof(ptr_t) == 0);
        GC_setup_temporary_fault_handler();
        if (SETJMP(GC_jmp_buf) == 0) {
          /* Try writing to the address.    */
#         ifdef AO_HAVE_fetch_and_add
            volatile AO_t zero = 0;

            (void)AO_fetch_and_add((volatile AO_t *)result, zero);
#         else
            /* Fallback to non-atomic fetch-and-store.  */
            char v = *result;

#           if defined(CPPCHECK)
              GC_noop1_ptr(&v);
#           endif
            *result = v;
#         endif
          GC_reset_fault_handler();
        } else {
          GC_reset_fault_handler();
          /* We got here via a longjmp.  The address is not readable.   */
          /* This is known to happen under Solaris 2.4 + gcc, which     */
          /* places string constants in the text segment, but after     */
          /* etext.  Use plan B.  Note that we now know there is a gap  */
          /* between text and data segments, so plan A brought us       */
          /* something.                                                 */
          result = (char *)GC_find_limit(DATAEND, FALSE);
        }
        return (ptr_t)CAST_AWAY_VOLATILE_PVOID(result);
      }
#   endif /* SVR4 || AIX || DGUX */

#   ifdef DATASTART_USES_BSDGETDATASTART
      /* It's unclear whether this should be identical to the above, or */
      /* whether it should apply to non-x86 architectures.  For now we  */
      /* do not assume that there is always an empty page after etext.  */
      /* But in some cases there actually seems to be slightly more.    */
      /* It also deals with holes between read-only and writable data.  */
      GC_INNER ptr_t GC_FreeBSDGetDataStart(size_t max_page_size,
                                            ptr_t etext_addr)
      {
        volatile ptr_t result = PTR_ALIGN_UP(etext_addr, sizeof(ptr_t));
        volatile ptr_t next_page = PTR_ALIGN_UP(etext_addr, max_page_size);

        GC_ASSERT(max_page_size % sizeof(ptr_t) == 0);
        GC_setup_temporary_fault_handler();
        if (SETJMP(GC_jmp_buf) == 0) {
          /* Try reading at the address.                        */
          /* This should happen before there is another thread. */
          for (; ADDR_LT(next_page, DATAEND); next_page += max_page_size)
            GC_noop1((word)(*(volatile unsigned char *)next_page));
          GC_reset_fault_handler();
        } else {
          GC_reset_fault_handler();
          /* As above, we go to plan B. */
          result = (ptr_t)GC_find_limit(DATAEND, FALSE);
        }
        return result;
      }
#   endif /* DATASTART_USES_BSDGETDATASTART */

#   ifdef AMIGA
#     define GC_AMIGA_DS
#     undef GC_AMIGA_DS

#   elif defined(OPENBSD)
      /* Depending on arch alignment, there can be multiple holes       */
      /* between DATASTART and DATAEND.  Scan in DATASTART .. DATAEND   */
      /* and register each region.                                      */
      void GC_register_data_segments(void)
      {
        ptr_t region_start = DATASTART;

        GC_ASSERT(I_HOLD_LOCK());
        if (ADDR(region_start) - 1U >= ADDR(DATAEND))
          ABORT_ARG2("Wrong DATASTART/END pair",
                     ": %p .. %p", (void *)region_start, (void *)DATAEND);
        for (;;) {
          ptr_t region_end = GC_find_limit_with_bound(region_start, TRUE,
                                                      DATAEND);

          GC_add_roots_inner(region_start, region_end, FALSE);
          if (ADDR_GE(region_end, DATAEND))
            break;
          region_start = GC_skip_hole_openbsd(region_end, DATAEND);
        }
      }

#   else /* !AMIGA && !OPENBSD */
#     if !defined(PCR) && !defined(MACOS) && defined(REDIRECT_MALLOC) \
         && defined(GC_SOLARIS_THREADS)
        EXTERN_C_BEGIN
        extern caddr_t sbrk(int);
        EXTERN_C_END
#     endif

      void GC_register_data_segments(void)
      {
        GC_ASSERT(I_HOLD_LOCK());
#       if !defined(DYNAMIC_LOADING) \
           && defined(GC_DONT_REGISTER_MAIN_STATIC_DATA)
          /* Avoid even referencing DATASTART and DATAEND as they are   */
          /* unnecessary and cause linker errors when bitcode is        */
          /* enabled.  GC_register_data_segments is not called anyway.  */
#       elif defined(PCR) || (defined(DYNAMIC_LOADING) && defined(DARWIN))
          /* No-op.  GC_register_main_static_data() always returns false. */
#       elif defined(MACOS)
          {
#           if defined(THINK_C)
              extern void *GC_MacGetDataStart(void);

              /* Globals begin above stack and end at a5.   */
              GC_add_roots_inner((ptr_t)GC_MacGetDataStart(),
                                 (ptr_t)LMGetCurrentA5(), FALSE);
#           elif defined(__MWERKS__) && defined(M68K)
              extern void *GC_MacGetDataStart(void);
#             if __option(far_data)
                extern void *GC_MacGetDataEnd(void);

                /* Handle Far Globals (CW Pro 3) located after  */
                /* the QD globals.                              */
                GC_add_roots_inner((ptr_t)GC_MacGetDataStart(),
                                   (ptr_t)GC_MacGetDataEnd(), FALSE);
#             else
                GC_add_roots_inner((ptr_t)GC_MacGetDataStart(),
                                   (ptr_t)LMGetCurrentA5(), FALSE);
#             endif
#           elif defined(__MWERKS__) && defined(POWERPC)
              extern char __data_start__[], __data_end__[];

              GC_add_roots_inner((ptr_t)&__data_start__,
                                 (ptr_t)&__data_end__, FALSE);
#           endif
          }
#       elif defined(REDIRECT_MALLOC) && defined(GC_SOLARIS_THREADS)
            /* As of Solaris 2.3, the Solaris threads implementation    */
            /* allocates the data structure for the initial thread with */
            /* sbrk at process startup.  It needs to be scanned, so     */
            /* that we don't lose some malloc allocated data structures */
            /* hanging from it.  We're on thin ice here...              */
            GC_ASSERT(DATASTART);
            {
              ptr_t p = (ptr_t)sbrk(0);

              if (ADDR_LT(DATASTART, p))
                GC_add_roots_inner(DATASTART, p, FALSE);
            }
#       else
            if (ADDR(DATASTART) - 1U >= ADDR(DATAEND)) {
                            /* Subtract one to check also for NULL  */
                            /* without a compiler warning.          */
              ABORT_ARG2("Wrong DATASTART/END pair",
                         ": %p .. %p", (void *)DATASTART, (void *)DATAEND);
            }
            GC_add_roots_inner(DATASTART, DATAEND, FALSE);
#           ifdef GC_HAVE_DATAREGION2
              if (ADDR(DATASTART2) - 1U >= ADDR(DATAEND2))
                ABORT_ARG2("Wrong DATASTART/END2 pair",
                           ": %p .. %p", (void *)DATASTART2, (void *)DATAEND2);
              GC_add_roots_inner(DATASTART2, DATAEND2, FALSE);
#           endif
#       endif
        /* Dynamic libraries are added at every collection, since they  */
        /* may change.                                                  */
      }
#   endif /* !AMIGA && !OPENBSD */

# endif /* !ANY_MSWIN */
#endif /* !OS2 */

/*
 * Auxiliary routines for obtaining memory from OS.
 */

#ifndef NO_UNIX_GET_MEM

# define SBRK_ARG_T ptrdiff_t

#if defined(MMAP_SUPPORTED)

#ifdef USE_MMAP_FIXED
#   define GC_MMAP_FLAGS MAP_FIXED | MAP_PRIVATE
        /* Seems to yield better performance on Solaris 2, but can      */
        /* be unreliable if something is already mapped at the address. */
#else
#   define GC_MMAP_FLAGS MAP_PRIVATE
#endif

#ifdef USE_MMAP_ANON
# define zero_fd -1
# if defined(MAP_ANONYMOUS) && !defined(CPPCHECK)
#   define OPT_MAP_ANON MAP_ANONYMOUS
# else
#   define OPT_MAP_ANON MAP_ANON
# endif
#else
  static int zero_fd = -1;
# define OPT_MAP_ANON 0
#endif

# ifndef MSWIN_XBOX1
#   if defined(SYMBIAN) && !defined(USE_MMAP_ANON)
      EXTERN_C_BEGIN
      extern char *GC_get_private_path_and_zero_file(void);
      EXTERN_C_END
#   endif

  STATIC ptr_t GC_unix_mmap_get_mem(size_t bytes)
  {
    void *result;
    static ptr_t last_addr = HEAP_START;

#   ifndef USE_MMAP_ANON
      static GC_bool initialized = FALSE;

      if (!EXPECT(initialized, TRUE)) {
#       ifdef SYMBIAN
          char *path = GC_get_private_path_and_zero_file();
          if (path != NULL) {
            zero_fd = open(path, O_RDWR | O_CREAT, 0644);
            free(path);
          }
#       else
          zero_fd = open("/dev/zero", O_RDONLY);
#       endif
          if (zero_fd == -1)
            ABORT("Could not open /dev/zero");
          if (fcntl(zero_fd, F_SETFD, FD_CLOEXEC) == -1)
            WARN("Could not set FD_CLOEXEC for /dev/zero\n", 0);

          initialized = TRUE;
      }
#   endif

    GC_ASSERT(GC_page_size != 0);
    if (bytes & (GC_page_size-1)) ABORT("Bad GET_MEM arg");
    result = mmap(last_addr, bytes, (PROT_READ | PROT_WRITE)
                                    | (GC_pages_executable ? PROT_EXEC : 0),
                  GC_MMAP_FLAGS | OPT_MAP_ANON, zero_fd, 0/* offset */);
#   undef IGNORE_PAGES_EXECUTABLE

    if (EXPECT(MAP_FAILED == result, FALSE)) {
      if (HEAP_START == last_addr && GC_pages_executable
          && (EACCES == errno || EPERM == errno))
        ABORT("Cannot allocate executable pages");
      return NULL;
    }
    last_addr = PTR_ALIGN_UP((ptr_t)result + bytes, GC_page_size);
#   if !defined(LINUX)
      if (last_addr == 0) {
        /* Oops.  We got the end of the address space.  This isn't      */
        /* usable by arbitrary C code, since one-past-end pointers      */
        /* don't work, so we discard it and try again.                  */
        munmap(result, ~GC_page_size - (size_t)result + 1);
                        /* Leave last page mapped, so we can't repeat.  */
        return GC_unix_mmap_get_mem(bytes);
      }
#   else
      GC_ASSERT(last_addr != 0);
#   endif
    if ((ADDR(result) % HBLKSIZE) != 0)
      ABORT(
       "GC_unix_get_mem: Memory returned by mmap is not aligned to HBLKSIZE.");
    return (ptr_t)result;
  }
# endif  /* !MSWIN_XBOX1 */

#endif  /* MMAP_SUPPORTED */

#if defined(USE_MMAP)
  ptr_t GC_unix_get_mem(size_t bytes)
  {
    return GC_unix_mmap_get_mem(bytes);
  }
#else /* !USE_MMAP */

STATIC ptr_t GC_unix_sbrk_get_mem(size_t bytes)
{
  ptr_t result;
# ifdef IRIX5
    /* Bare sbrk isn't thread safe.  Play by malloc rules.      */
    /* The equivalent may be needed on other systems as well.   */
    __LOCK_MALLOC();
# endif
  {
    ptr_t cur_brk = (ptr_t)sbrk(0);
    SBRK_ARG_T lsbs = ADDR(cur_brk) & (GC_page_size-1);

    GC_ASSERT(GC_page_size != 0);
    if ((SBRK_ARG_T)bytes < 0) {
        result = 0; /* too big */
        goto out;
    }
    if (lsbs != 0) {
        if ((ptr_t)sbrk((SBRK_ARG_T)GC_page_size - lsbs) == (ptr_t)(-1)) {
            result = 0;
            goto out;
        }
    }
#   ifdef ADD_HEAP_GUARD_PAGES
      /* This is useful for catching severe memory overwrite problems that */
      /* span heap sections.  It shouldn't otherwise be turned on.         */
      {
        ptr_t guard = (ptr_t)sbrk((SBRK_ARG_T)GC_page_size);
        if (mprotect(guard, GC_page_size, PROT_NONE) != 0)
            ABORT("ADD_HEAP_GUARD_PAGES: mprotect failed");
      }
#   endif /* ADD_HEAP_GUARD_PAGES */
    result = (ptr_t)sbrk((SBRK_ARG_T)bytes);
    if (result == (ptr_t)(-1)) result = 0;
  }
 out:
# ifdef IRIX5
    __UNLOCK_MALLOC();
# endif
  return result;
}

ptr_t GC_unix_get_mem(size_t bytes)
{
# if defined(MMAP_SUPPORTED)
    /* By default, we try both sbrk and mmap, in that order.    */
    static GC_bool sbrk_failed = FALSE;
    ptr_t result = 0;

    if (GC_pages_executable) {
        /* If the allocated memory should have the execute permission   */
        /* then sbrk() cannot be used.                                  */
        return GC_unix_mmap_get_mem(bytes);
    }
    if (!sbrk_failed) result = GC_unix_sbrk_get_mem(bytes);
    if (0 == result) {
        sbrk_failed = TRUE;
        result = GC_unix_mmap_get_mem(bytes);
    }
    if (0 == result) {
        /* Try sbrk again, in case sbrk memory became available.        */
        result = GC_unix_sbrk_get_mem(bytes);
    }
    return result;
# else /* !MMAP_SUPPORTED */
    return GC_unix_sbrk_get_mem(bytes);
# endif
}

#endif /* !USE_MMAP */

#endif /* !NO_UNIX_GET_MEM */

#ifdef OS2
  void * os2_alloc(size_t bytes)
  {
    void * result;

    if (DosAllocMem(&result, bytes, (PAG_READ | PAG_WRITE | PAG_COMMIT)
                                    | (GC_pages_executable ? PAG_EXECUTE : 0))
                    != NO_ERROR) {
        return NULL;
    }
    /* FIXME: What's the purpose of this recursion?  (Probably, if      */
    /* DosAllocMem returns memory at 0 address then just retry once.)   */
    if (NULL == result) return os2_alloc(bytes);
    return result;
  }
#endif /* OS2 */

#ifdef MSWIN_XBOX1
    ptr_t GC_durango_get_mem(size_t bytes)
    {
      if (0 == bytes) return NULL;
      return (ptr_t)VirtualAlloc(NULL, bytes, MEM_COMMIT | MEM_TOP_DOWN,
                                 PAGE_READWRITE);
    }
#elif defined(MSWINCE)
  ptr_t GC_wince_get_mem(size_t bytes)
  {
    ptr_t result = 0; /* initialized to prevent warning. */
    size_t i;

    GC_ASSERT(GC_page_size != 0);
    bytes = ROUNDUP_PAGESIZE(bytes);

    /* Try to find reserved, uncommitted pages */
    for (i = 0; i < GC_n_heap_bases; i++) {
        if (((word)(-(signed_word)GC_heap_lengths[i])
             & (GC_sysinfo.dwAllocationGranularity-1))
            >= bytes) {
            result = GC_heap_bases[i] + GC_heap_lengths[i];
            break;
        }
    }

    if (i == GC_n_heap_bases) {
        /* Reserve more pages */
        size_t res_bytes =
            SIZET_SAT_ADD(bytes, (size_t)GC_sysinfo.dwAllocationGranularity-1)
            & ~((size_t)GC_sysinfo.dwAllocationGranularity-1);
        /* If we ever support MPROTECT_VDB here, we will probably need to    */
        /* ensure that res_bytes is strictly > bytes, so that VirtualProtect */
        /* never spans regions.  It seems to be OK for a VirtualFree         */
        /* argument to span regions, so we should be OK for now.             */
        result = (ptr_t) VirtualAlloc(NULL, res_bytes,
                                MEM_RESERVE | MEM_TOP_DOWN,
                                GC_pages_executable ? PAGE_EXECUTE_READWRITE :
                                                      PAGE_READWRITE);
        if (HBLKDISPL(result) != 0) ABORT("Bad VirtualAlloc result");
            /* If I read the documentation correctly, this can          */
            /* only happen if HBLKSIZE > 64 KB or not a power of 2.     */
        if (GC_n_heap_bases >= MAX_HEAP_SECTS) ABORT("Too many heap sections");
        if (result == NULL) return NULL;
        GC_heap_bases[GC_n_heap_bases] = result;
        GC_heap_lengths[GC_n_heap_bases] = 0;
        GC_n_heap_bases++;
    }

    /* Commit pages */
    result = (ptr_t) VirtualAlloc(result, bytes, MEM_COMMIT,
                              GC_pages_executable ? PAGE_EXECUTE_READWRITE :
                                                    PAGE_READWRITE);
#   undef IGNORE_PAGES_EXECUTABLE

    if (result != NULL) {
        if (HBLKDISPL(result) != 0) ABORT("Bad VirtualAlloc result");
        GC_heap_lengths[i] += bytes;
    }
    return result;
  }

#elif defined(USE_WINALLOC) /* && !MSWIN_XBOX1 */ || defined(CYGWIN32)

# ifdef USE_GLOBAL_ALLOC
#   define GLOBAL_ALLOC_TEST 1
# else
#   define GLOBAL_ALLOC_TEST GC_no_win32_dlls
# endif

# if (defined(GC_USE_MEM_TOP_DOWN) && defined(USE_WINALLOC)) \
     || defined(CPPCHECK)
    DWORD GC_mem_top_down = MEM_TOP_DOWN;
                           /* Use GC_USE_MEM_TOP_DOWN for better 64-bit */
                           /* testing.  Otherwise all addresses tend to */
                           /* end up in first 4 GB, hiding bugs.        */
# else
#   define GC_mem_top_down 0
# endif /* !GC_USE_MEM_TOP_DOWN */

  ptr_t GC_win32_get_mem(size_t bytes)
  {
    ptr_t result;

# ifndef USE_WINALLOC
    result = GC_unix_get_mem(bytes);
# else
#   if defined(MSWIN32) && !defined(MSWINRT_FLAVOR)
      if (GLOBAL_ALLOC_TEST) {
        /* VirtualAlloc doesn't like PAGE_EXECUTE_READWRITE.    */
        /* There are also unconfirmed rumors of other           */
        /* problems, so we dodge the issue.                     */
        result = (ptr_t)GlobalAlloc(0, SIZET_SAT_ADD(bytes, HBLKSIZE));
        /* Align it at HBLKSIZE boundary (NULL value remains unchanged). */
        result = PTR_ALIGN_UP(result, HBLKSIZE);
      } else
#   endif
    /* else */ {
        /* VirtualProtect only works on regions returned by a   */
        /* single VirtualAlloc call.  Thus we allocate one      */
        /* extra page, which will prevent merging of blocks     */
        /* in separate regions, and eliminate any temptation    */
        /* to call VirtualProtect on a range spanning regions.  */
        /* This wastes a small amount of memory, and risks      */
        /* increased fragmentation.  But better alternatives    */
        /* would require effort.                                */
#       ifdef MPROTECT_VDB
          /* We can't check for GC_incremental here (because    */
          /* GC_enable_incremental() might be called some time  */
          /* later after the GC initialization).                */
#         ifdef GWW_VDB
#           define VIRTUAL_ALLOC_PAD (GC_GWW_AVAILABLE() ? 0 : 1)
#         else
#           define VIRTUAL_ALLOC_PAD 1
#         endif
#       else
#         define VIRTUAL_ALLOC_PAD 0
#       endif
        /* Pass the MEM_WRITE_WATCH only if GetWriteWatch-based */
        /* VDBs are enabled and the GetWriteWatch function is   */
        /* available.  Otherwise we waste resources or possibly */
        /* cause VirtualAlloc to fail (observed in Windows 2000 */
        /* SP2).                                                */
        result = (ptr_t) VirtualAlloc(NULL,
                            SIZET_SAT_ADD(bytes, VIRTUAL_ALLOC_PAD),
                            GetWriteWatch_alloc_flag
                                | (MEM_COMMIT | MEM_RESERVE)
                                | GC_mem_top_down,
                            GC_pages_executable ? PAGE_EXECUTE_READWRITE :
                                                  PAGE_READWRITE);
#       undef IGNORE_PAGES_EXECUTABLE
    }
# endif /* USE_WINALLOC */
    if (HBLKDISPL(result) != 0) ABORT("Bad VirtualAlloc result");
        /* If I read the documentation correctly, this can      */
        /* only happen if HBLKSIZE > 64 KB or not a power of 2. */
    if (GC_n_heap_bases >= MAX_HEAP_SECTS) ABORT("Too many heap sections");
    if (result != NULL) GC_heap_bases[GC_n_heap_bases++] = result;
    return result;
  }
#endif /* USE_WINALLOC || CYGWIN32 */

#if defined(ANY_MSWIN) || defined(MSWIN_XBOX1)
  GC_API void GC_CALL GC_win32_free_heap(void)
  {
#   if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC) \
       && !defined(MSWIN_XBOX1)
      GC_free_malloc_heap_list();
#   endif
#   if (defined(USE_WINALLOC) && !defined(MSWIN_XBOX1) \
        && !defined(MSWINCE)) || defined(CYGWIN32)
#     ifndef MSWINRT_FLAVOR
#       ifndef CYGWIN32
          if (GLOBAL_ALLOC_TEST)
#       endif
        {
          while (GC_n_heap_bases > 0) {
            GC_n_heap_bases--;
#           ifdef CYGWIN32
              /* FIXME: Is it OK to use non-GC free() here? */
#           else
              GlobalFree(GC_heap_bases[GC_n_heap_bases]);
#           endif
            GC_heap_bases[GC_n_heap_bases] = 0;
          }
          return;
        }
#     endif /* !MSWINRT_FLAVOR */
#     ifndef CYGWIN32
        /* Avoiding VirtualAlloc leak.  */
        while (GC_n_heap_bases > 0) {
          VirtualFree(GC_heap_bases[--GC_n_heap_bases], 0, MEM_RELEASE);
          GC_heap_bases[GC_n_heap_bases] = 0;
        }
#     endif
#   endif /* USE_WINALLOC || CYGWIN32 */
  }
#endif /* ANY_MSWIN || MSWIN_XBOX1 */

#ifdef AMIGA
# define GC_AMIGA_AM
# undef GC_AMIGA_AM
#endif

#if defined(HAIKU)
# ifdef GC_LEAK_DETECTOR_H
#   undef posix_memalign /* to use the real one */
# endif
  ptr_t GC_haiku_get_mem(size_t bytes)
  {
    void* mem;

    GC_ASSERT(GC_page_size != 0);
    if (posix_memalign(&mem, GC_page_size, bytes) == 0)
      return mem;
    return NULL;
  }
#endif /* HAIKU */

#if (defined(USE_MUNMAP) || defined(MPROTECT_VDB)) && !defined(USE_WINALLOC)
# define ABORT_ON_REMAP_FAIL(C_msg_prefix, start_addr, len) \
        ABORT_ARG3(C_msg_prefix " failed", \
                   " at %p (length %lu), errno= %d", \
                   (void *)(start_addr), (unsigned long)(len), errno)
#endif

#ifdef USE_MUNMAP

/* For now, this only works on Win32/WinCE and some Unix-like   */
/* systems.  If you have something else, don't define           */
/* USE_MUNMAP.                                                  */

#if !defined(NN_PLATFORM_CTR) && !defined(MSWIN32) && !defined(MSWINCE) \
    && !defined(MSWIN_XBOX1)
# ifdef SN_TARGET_PS3
#   include <sys/memory.h>
# else
#   include <sys/mman.h>
# endif
# include <sys/stat.h>
#endif

/* Compute a page aligned starting address for the unmap        */
/* operation on a block of size bytes starting at start.        */
/* Return 0 if the block is too small to make this feasible.    */
STATIC ptr_t GC_unmap_start(ptr_t start, size_t bytes)
{
    ptr_t result;

    GC_ASSERT(GC_page_size != 0);
    result = PTR_ALIGN_UP(start, GC_page_size);
    if (ADDR_LT(start + bytes, result + GC_page_size)) return NULL;

    return result;
}

/* We assume that GC_remap is called on exactly the same range  */
/* as a previous call to GC_unmap.  It is safe to consistently  */
/* round the endpoints in both places.                          */

static void block_unmap_inner(ptr_t start_addr, size_t len)
{
    if (0 == start_addr) return;

#   ifdef USE_WINALLOC
      /* Under Win32/WinCE we commit (map) and decommit (unmap)         */
      /* memory using VirtualAlloc and VirtualFree.  These functions    */
      /* work on individual allocations of virtual memory, made         */
      /* previously using VirtualAlloc with the MEM_RESERVE flag.       */
      /* The ranges we need to (de)commit may span several of these     */
      /* allocations; therefore we use VirtualQuery to check            */
      /* allocation lengths, and split up the range as necessary.       */
      while (len != 0) {
          MEMORY_BASIC_INFORMATION mem_info;
          word free_len;

          if (VirtualQuery(start_addr, &mem_info, sizeof(mem_info))
              != sizeof(mem_info))
              ABORT("Weird VirtualQuery result");
          free_len = (len < mem_info.RegionSize) ? len : mem_info.RegionSize;
          if (!VirtualFree(start_addr, free_len, MEM_DECOMMIT))
              ABORT("VirtualFree failed");
          GC_unmapped_bytes += free_len;
          start_addr += free_len;
          len -= free_len;
      }
#   else
      if (len != 0) {
#       ifdef SN_TARGET_PS3
          ps3_free_mem(start_addr, len);
#       elif defined(AIX) || defined(CYGWIN32) || defined(HAIKU) \
             || (defined(LINUX) && !defined(PREFER_MMAP_PROT_NONE)) \
             || defined(HPUX)
          /* On AIX, mmap(PROT_NONE) fails with ENOMEM unless the       */
          /* environment variable XPG_SUS_ENV is set to ON.             */
          /* On Cygwin, calling mmap() with the new protection flags on */
          /* an existing memory map with MAP_FIXED is broken.           */
          /* However, calling mprotect() on the given address range     */
          /* with PROT_NONE seems to work fine.                         */
          /* On Linux, low RLIMIT_AS value may lead to mmap failure.    */
#         if defined(LINUX) && !defined(FORCE_MPROTECT_BEFORE_MADVISE)
            /* On Linux, at least, madvise() should be sufficient.      */
#         else
            if (mprotect(start_addr, len, PROT_NONE))
              ABORT_ON_REMAP_FAIL("unmap: mprotect", start_addr, len);
#         endif
#         if !defined(CYGWIN32)
            /* On Linux (and some other platforms probably),    */
            /* mprotect(PROT_NONE) is just disabling access to  */
            /* the pages but not returning them to OS.          */
            if (madvise(start_addr, len, MADV_DONTNEED) == -1)
              ABORT_ON_REMAP_FAIL("unmap: madvise", start_addr, len);
#         endif
#       else
          /* We immediately remap it to prevent an intervening mmap()   */
          /* from accidentally grabbing the same address space.         */
          void * result = mmap(start_addr, len, PROT_NONE,
                               MAP_PRIVATE | MAP_FIXED | OPT_MAP_ANON,
                               zero_fd, 0/* offset */);

          if (EXPECT(MAP_FAILED == result, FALSE))
            ABORT_ON_REMAP_FAIL("unmap: mmap", start_addr, len);
          if (result != start_addr)
            ABORT("unmap: mmap() result differs from start_addr");
#         if defined(CPPCHECK) || defined(LINT2)
            /* Explicitly store the resource handle to a global variable. */
            GC_noop1_ptr(result);
#         endif
#       endif
        GC_unmapped_bytes += len;
      }
#   endif
}

/* Compute end address for an unmap operation on the indicated block.   */
GC_INLINE ptr_t GC_unmap_end(ptr_t start, size_t bytes)
{
    return (ptr_t)HBLK_PAGE_ALIGNED(start + bytes);
}

GC_INNER void GC_unmap(ptr_t start, size_t bytes)
{
    ptr_t start_addr = GC_unmap_start(start, bytes);
    ptr_t end_addr = GC_unmap_end(start, bytes);

    block_unmap_inner(start_addr, (size_t)(end_addr - start_addr));
}

GC_INNER void GC_remap(ptr_t start, size_t bytes)
{
    ptr_t start_addr = GC_unmap_start(start, bytes);
    ptr_t end_addr = GC_unmap_end(start, bytes);
    word len = (word)(end_addr - start_addr);
    if (0 == start_addr) return;

    /* FIXME: Handle out-of-memory correctly (at least for Win32)       */
#   ifdef USE_WINALLOC
      while (len != 0) {
          MEMORY_BASIC_INFORMATION mem_info;
          word alloc_len;
          ptr_t result;

          if (VirtualQuery(start_addr, &mem_info, sizeof(mem_info))
              != sizeof(mem_info))
              ABORT("Weird VirtualQuery result");
          alloc_len = (len < mem_info.RegionSize) ? len : mem_info.RegionSize;
          result = (ptr_t)VirtualAlloc(start_addr, alloc_len, MEM_COMMIT,
                                       GC_pages_executable
                                                ? PAGE_EXECUTE_READWRITE
                                                : PAGE_READWRITE);
          if (result != start_addr) {
              if (GetLastError() == ERROR_NOT_ENOUGH_MEMORY ||
                  GetLastError() == ERROR_OUTOFMEMORY) {
                  ABORT("Not enough memory to process remapping");
              } else {
                  ABORT("VirtualAlloc remapping failed");
              }
          }
#         ifdef LINT2
            GC_noop1_ptr(result);
#         endif
          GC_ASSERT(GC_unmapped_bytes >= alloc_len);
          GC_unmapped_bytes -= alloc_len;
          start_addr += alloc_len;
          len -= alloc_len;
      }
#     undef IGNORE_PAGES_EXECUTABLE
#   else
      /* It was already remapped with PROT_NONE. */
      {
#       if !defined(SN_TARGET_PS3) && !defined(FORCE_MPROTECT_BEFORE_MADVISE) \
           && defined(LINUX) && !defined(PREFER_MMAP_PROT_NONE)
          /* Nothing to unprotect as madvise() is just a hint.  */
#       elif defined(NACL) || defined(NETBSD)
          /* NaCl does not expose mprotect, but mmap should work fine.  */
          /* In case of NetBSD, mprotect fails (unlike mmap) even       */
          /* without PROT_EXEC if PaX MPROTECT feature is enabled.      */
          void *result = mmap(start_addr, len, (PROT_READ | PROT_WRITE)
                                    | (GC_pages_executable ? PROT_EXEC : 0),
                                   MAP_PRIVATE | MAP_FIXED | OPT_MAP_ANON,
                                   zero_fd, 0 /* offset */);
          if (EXPECT(MAP_FAILED == result, FALSE))
            ABORT_ON_REMAP_FAIL("remap: mmap", start_addr, len);
          if (result != start_addr)
            ABORT("remap: mmap() result differs from start_addr");
#         if defined(CPPCHECK) || defined(LINT2)
            GC_noop1_ptr(result);
#         endif
#         undef IGNORE_PAGES_EXECUTABLE
#       else
          if (mprotect(start_addr, len, (PROT_READ | PROT_WRITE)
                            | (GC_pages_executable ? PROT_EXEC : 0)))
            ABORT_ON_REMAP_FAIL("remap: mprotect", start_addr, len);
#         undef IGNORE_PAGES_EXECUTABLE
#       endif /* !NACL */
      }
      GC_ASSERT(GC_unmapped_bytes >= len);
      GC_unmapped_bytes -= len;
#   endif
}

/* Two adjacent blocks have already been unmapped and are about to      */
/* be merged.  Unmap the whole block.  This typically requires          */
/* that we unmap a small section in the middle that was not previously  */
/* unmapped due to alignment constraints.                               */
GC_INNER void GC_unmap_gap(ptr_t start1, size_t bytes1, ptr_t start2,
                           size_t bytes2)
{
    ptr_t start1_addr = GC_unmap_start(start1, bytes1);
    ptr_t end1_addr = GC_unmap_end(start1, bytes1);
    ptr_t start2_addr = GC_unmap_start(start2, bytes2);
    ptr_t start_addr = end1_addr;
    ptr_t end_addr = start2_addr;

    GC_ASSERT(start1 + bytes1 == start2);
    if (0 == start1_addr) start_addr = GC_unmap_start(start1, bytes1 + bytes2);
    if (0 == start2_addr) end_addr = GC_unmap_end(start1, bytes1 + bytes2);
    block_unmap_inner(start_addr, (size_t)(end_addr - start_addr));
}

#endif /* USE_MUNMAP */

/* Routine for pushing any additional roots.  In THREADS        */
/* environment, this is also responsible for marking from       */
/* thread stacks.                                               */
#ifndef THREADS

# if defined(EMSCRIPTEN) && defined(EMSCRIPTEN_ASYNCIFY)
#   include <emscripten.h>

    static void scan_regs_cb(void *begin, void *end)
    {
      GC_push_all_stack((ptr_t)begin, (ptr_t)end);
    }

    STATIC void GC_CALLBACK GC_default_push_other_roots(void)
    {
      /* Note: this needs -sASYNCIFY linker flag. */
      emscripten_scan_registers(scan_regs_cb);
    }

# else
#   define GC_default_push_other_roots 0
# endif

#else /* THREADS */

# ifdef PCR
PCR_ERes GC_push_thread_stack(PCR_Th_T *t, PCR_Any dummy)
{
    struct PCR_ThCtl_TInfoRep info;
    PCR_ERes result;

    info.ti_stkLow = info.ti_stkHi = 0;
    result = PCR_ThCtl_GetInfo(t, &info);
    GC_push_all_stack((ptr_t)(info.ti_stkLow), (ptr_t)(info.ti_stkHi));
    return result;
}

/* Push the contents of an old object. We treat this as stack   */
/* data only because that makes it robust against mark stack    */
/* overflow.                                                    */
PCR_ERes GC_push_old_obj(void *p, size_t size, PCR_Any data)
{
    GC_push_all_stack((ptr_t)p, (ptr_t)p + size);
    return PCR_ERes_okay;
}

extern struct PCR_MM_ProcsRep * GC_old_allocator;
                                        /* defined in pcr_interface.c.  */

STATIC void GC_CALLBACK GC_default_push_other_roots(void)
{
    GC_ASSERT(I_HOLD_LOCK());
    /* Traverse data allocated by previous memory managers.             */
          if ((*(GC_old_allocator->mmp_enumerate))(PCR_Bool_false,
                                                   GC_push_old_obj, 0)
              != PCR_ERes_okay) {
              ABORT("Old object enumeration failed");
          }
    /* Traverse all thread stacks. */
        if (PCR_ERes_IsErr(
                PCR_ThCtl_ApplyToAllOtherThreads(GC_push_thread_stack,0))
            || PCR_ERes_IsErr(GC_push_thread_stack(PCR_Th_CurrThread(), 0))) {
          ABORT("Thread stack marking failed");
        }
}

# elif defined(SN_TARGET_PS3)
    STATIC void GC_CALLBACK GC_default_push_other_roots(void)
    {
      ABORT("GC_default_push_other_roots is not implemented");
    }

    void GC_push_thread_structures(void)
    {
      ABORT("GC_push_thread_structures is not implemented");
    }

# else /* GC_PTHREADS, or GC_WIN32_THREADS, etc.        */
    STATIC void GC_CALLBACK GC_default_push_other_roots(void)
    {
      GC_push_all_stacks();
    }
# endif

#endif /* THREADS */

GC_push_other_roots_proc GC_push_other_roots = GC_default_push_other_roots;

GC_API void GC_CALL GC_set_push_other_roots(GC_push_other_roots_proc fn)
{
    GC_push_other_roots = fn;
}

GC_API GC_push_other_roots_proc GC_CALL GC_get_push_other_roots(void)
{
    return GC_push_other_roots;
}

#if defined(SOFT_VDB) && !defined(NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK) \
    || (defined(GLIBC_2_19_TSX_BUG) && defined(GC_PTHREADS_PARAMARK))
  GC_INNER int GC_parse_version(int *pminor, const char *pverstr) {
    char *endp;
    unsigned long value = strtoul(pverstr, &endp, 10);
    int major = (int)value;

    if (major < 0 || (char *)pverstr == endp || (unsigned)major != value) {
      /* Parse error.   */
      return -1;
    }
    if (*endp != '.') {
      /* No minor part. */
      *pminor = -1;
    } else {
      value = strtoul(endp + 1, &endp, 10);
      *pminor = (int)value;
      if (*pminor < 0 || (unsigned)(*pminor) != value) {
        return -1;
      }
    }
    return major;
  }
#endif

/*
 * Routines for accessing dirty bits on virtual pages.
 * There are six ways to maintain this information:
 * DEFAULT_VDB: A simple dummy implementation that treats every page
 *              as possibly dirty.  This makes incremental collection
 *              useless, but the implementation is still correct.
 * Manual VDB:  Stacks and static data are always considered dirty.
 *              Heap pages are considered dirty if GC_dirty(p) has been
 *              called on some pointer p pointing to somewhere inside
 *              an object on that page.  A GC_dirty() call on a large
 *              object directly dirties only a single page, but for the
 *              manual VDB we are careful to treat an object with a dirty
 *              page as completely dirty.
 *              In order to avoid races, an object must be marked dirty
 *              after it is written, and a reference to the object
 *              must be kept on a stack or in a register in the interim.
 *              With threads enabled, an object directly reachable from the
 *              stack at the time of a collection is treated as dirty.
 *              In single-threaded mode, it suffices to ensure that no
 *              collection can take place between the pointer assignment
 *              and the GC_dirty() call.
 * PCR_VDB:     Use PPCRs virtual dirty bit facility.
 * PROC_VDB:    Use the /proc facility for reading dirty bits.  Only
 *              works under some SVR4 variants.  Even then, it may be
 *              too slow to be entirely satisfactory.  Requires reading
 *              dirty bits for entire address space.  Implementations tend
 *              to assume that the client is a (slow) debugger.
 * SOFT_VDB:    Use the /proc facility for reading soft-dirty PTEs.
 *              Works on Linux 3.18+ if the kernel is properly configured.
 *              The proposed implementation iterates over GC_heap_sects and
 *              GC_static_roots examining the soft-dirty bit of the words
 *              in /proc/self/pagemap corresponding to the pages of the
 *              sections; finally all soft-dirty bits of the process are
 *              cleared (by writing some special value to
 *              /proc/self/clear_refs file).  In case the soft-dirty bit is
 *              not supported by the kernel, MPROTECT_VDB may be defined as
 *              a fallback strategy.
 * MPROTECT_VDB:Protect pages and then catch the faults to keep track of
 *              dirtied pages.  The implementation (and implementability)
 *              is highly system dependent.  This usually fails when system
 *              calls write to a protected page.  We prevent the read system
 *              call from doing so.  It is the clients responsibility to
 *              make sure that other system calls are similarly protected
 *              or write only to the stack.
 * GWW_VDB:     Use the Win32 GetWriteWatch functions, if available, to
 *              read dirty bits.  In case it is not available (because we
 *              are running on Windows 95, Windows 2000 or earlier),
 *              MPROTECT_VDB may be defined as a fallback strategy.
 */

#if (defined(CHECKSUMS) && defined(GWW_VDB)) || defined(PROC_VDB)
    /* Add all pages in pht2 to pht1.   */
    STATIC void GC_or_pages(page_hash_table pht1, const word *pht2)
    {
      size_t i;

      for (i = 0; i < PHT_SIZE; i++) pht1[i] |= pht2[i];
    }
#endif /* CHECKSUMS && GWW_VDB || PROC_VDB */

#ifdef GWW_VDB

# define GC_GWW_BUF_LEN (MAXHINCR * HBLKSIZE / 4096 /* x86 page size */)
  /* Still susceptible to overflow, if there are very large allocations, */
  /* and everything is dirty.                                            */
  static PVOID gww_buf[GC_GWW_BUF_LEN];

#   ifndef MPROTECT_VDB
#     define GC_gww_dirty_init GC_dirty_init
#   endif

    GC_INNER GC_bool GC_gww_dirty_init(void)
    {
      /* No assumption about the allocator lock. */
      detect_GetWriteWatch();
      return GC_GWW_AVAILABLE();
    }

  GC_INLINE void GC_gww_read_dirty(GC_bool output_unneeded)
  {
    size_t i;

    GC_ASSERT(I_HOLD_LOCK());
    if (!output_unneeded)
      BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));

    for (i = 0; i < GC_n_heap_sects; ++i) {
      GC_ULONG_PTR count;

      do {
        PVOID * pages = gww_buf;
        DWORD page_size;

        count = GC_GWW_BUF_LEN;
        /* GetWriteWatch is documented as returning non-zero when it    */
        /* fails, but the documentation doesn't explicitly say why it   */
        /* would fail or what its behavior will be if it fails.  It     */
        /* does appear to fail, at least on recent Win2K instances, if  */
        /* the underlying memory was not allocated with the appropriate */
        /* flag.  This is common if GC_enable_incremental is called     */
        /* shortly after GC initialization.  To avoid modifying the     */
        /* interface, we silently work around such a failure, it only   */
        /* affects the initial (small) heap allocation. If there are    */
        /* more dirty pages than will fit in the buffer, this is not    */
        /* treated as a failure; we must check the page count in the    */
        /* loop condition. Since each partial call will reset the       */
        /* status of some pages, this should eventually terminate even  */
        /* in the overflow case.                                        */
        if ((*(GetWriteWatch_type)(GC_funcptr_uint)GetWriteWatch_func)(
                                        WRITE_WATCH_FLAG_RESET,
                                        GC_heap_sects[i].hs_start,
                                        GC_heap_sects[i].hs_bytes,
                                        pages, &count, &page_size) != 0) {
          static int warn_count = 0;
          struct hblk * start = (struct hblk *)GC_heap_sects[i].hs_start;
          static const struct hblk *last_warned = NULL;
          size_t nblocks = divHBLKSZ(GC_heap_sects[i].hs_bytes);

          if (i != 0 && last_warned != start && warn_count++ < 5) {
            last_warned = start;
            WARN("GC_gww_read_dirty unexpectedly failed at %p:"
                 " Falling back to marking all pages dirty\n", start);
          }
          if (!output_unneeded) {
            size_t j;

            for (j = 0; j < nblocks; ++j) {
              size_t index = PHT_HASH(start + j);

              set_pht_entry_from_index(GC_grungy_pages, index);
            }
          }
          count = 1;  /* Done with this section. */
        } else /* succeeded */ if (!output_unneeded) {
          const PVOID *pages_end = pages + count;

          while (pages != pages_end) {
            struct hblk * h = (struct hblk *)(*pages++);
            ptr_t h_end = (ptr_t)h + page_size;

            do {
              set_pht_entry_from_index(GC_grungy_pages, PHT_HASH(h));
              h++;
            } while (ADDR_LT((ptr_t)h, h_end));
          }
        }
      } while (count == GC_GWW_BUF_LEN);
      /* FIXME: It's unclear from Microsoft's documentation if this loop */
      /* is useful.  We suspect the call just fails if the buffer fills  */
      /* up.  But that should still be handled correctly.                */
    }

#   ifdef CHECKSUMS
      GC_ASSERT(!output_unneeded);
      GC_or_pages(GC_written_pages, GC_grungy_pages);
#   endif
  }

#elif defined(SOFT_VDB)
  static int clear_refs_fd = -1;
# define GC_GWW_AVAILABLE() (clear_refs_fd != -1)
#else
# define GC_GWW_AVAILABLE() FALSE
#endif /* !GWW_VDB && !SOFT_VDB */

#ifdef DEFAULT_VDB
  /* The client asserts that unallocated pages in the heap are never    */
  /* written.                                                           */

  /* Initialize virtual dirty bit implementation.       */
  GC_INNER GC_bool GC_dirty_init(void)
  {
    GC_VERBOSE_LOG_PRINTF("Initializing DEFAULT_VDB...\n");
    /* GC_dirty_pages and GC_grungy_pages are already cleared.  */
    return TRUE;
  }
#endif /* DEFAULT_VDB */

#if !defined(NO_MANUAL_VDB) || defined(MPROTECT_VDB)
# if !defined(THREADS) || defined(HAVE_LOCKFREE_AO_OR)
#   ifdef MPROTECT_VDB
#     define async_set_pht_entry_from_index(db, index) \
                set_pht_entry_from_index_concurrent_volatile(db, index)
#   else
#     define async_set_pht_entry_from_index(db, index) \
                set_pht_entry_from_index_concurrent(db, index)
#   endif
# elif defined(AO_HAVE_test_and_set_acquire)
    /* We need to lock around the bitmap update (in the write fault     */
    /* handler or GC_dirty) in order to avoid the risk of losing a bit. */
    /* We do this with a test-and-set spin lock if possible.            */
    GC_INNER volatile AO_TS_t GC_fault_handler_lock = AO_TS_INITIALIZER;

    static void async_set_pht_entry_from_index(volatile page_hash_table db,
                                               size_t index)
    {
      GC_acquire_dirty_lock();
      set_pht_entry_from_index(db, index);
      GC_release_dirty_lock();
    }
# else
#   error No test_and_set operation: Introduces a race.
# endif /* THREADS && !AO_HAVE_test_and_set_acquire */
#endif /* !NO_MANUAL_VDB || MPROTECT_VDB */

#ifdef MPROTECT_VDB
  /* This implementation maintains dirty bits itself by catching write  */
  /* faults and keeping track of them.  We assume nobody else catches   */
  /* SIGBUS or SIGSEGV.  We assume no write faults occur in system      */
  /* calls.  This means that clients must ensure that system calls do   */
  /* not write to the write-protected heap.  Probably the best way to   */
  /* do this is to ensure that system calls write at most to            */
  /* pointer-free objects in the heap, and do even that only if we are  */
  /* on a platform on which those are not protected (or the collector   */
  /* is built with DONT_PROTECT_PTRFREE defined).  We assume the page   */
  /* size is a multiple of HBLKSIZE.                                    */

# ifdef DARWIN
    /* #define BROKEN_EXCEPTION_HANDLING */

    /* Using vm_protect (mach syscall) over mprotect (BSD syscall) seems to
       decrease the likelihood of some of the problems described below. */
#   include <mach/vm_map.h>
    STATIC mach_port_t GC_task_self = 0;
#   define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
        if (vm_protect(GC_task_self, (vm_address_t)(addr), (vm_size_t)(len), \
                       FALSE, VM_PROT_READ \
                              | ((allow_write) ? VM_PROT_WRITE : 0) \
                              | (GC_pages_executable ? VM_PROT_EXECUTE : 0)) \
                == KERN_SUCCESS) {} else ABORT(C_msg_prefix \
                                               "vm_protect() failed")

# elif !defined(USE_WINALLOC)
#   include <sys/mman.h>
#   if !defined(AIX) && !defined(CYGWIN32) && !defined(HAIKU)
#     include <sys/syscall.h>
#   endif

#   define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
        if (mprotect((caddr_t)(addr), (size_t)(len), \
                     PROT_READ | ((allow_write) ? PROT_WRITE : 0) \
                     | (GC_pages_executable ? PROT_EXEC : 0)) >= 0) { \
        } else if (GC_pages_executable) { \
            ABORT_ON_REMAP_FAIL(C_msg_prefix \
                                    "mprotect vdb executable pages", \
                                addr, len); \
        } else ABORT_ON_REMAP_FAIL(C_msg_prefix "mprotect vdb", addr, len)
#   undef IGNORE_PAGES_EXECUTABLE

# else /* USE_WINALLOC */
    static DWORD protect_junk;
#   define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
        if (VirtualProtect(addr, len, \
                           GC_pages_executable ? \
                                ((allow_write) ? PAGE_EXECUTE_READWRITE : \
                                                 PAGE_EXECUTE_READ) : \
                                 (allow_write) ? PAGE_READWRITE : \
                                                 PAGE_READONLY, \
                           &protect_junk)) { \
        } else ABORT_ARG1(C_msg_prefix "VirtualProtect failed", \
                          ": errcode= 0x%X", (unsigned)GetLastError())
# endif /* USE_WINALLOC */

# define PROTECT(addr, len) PROTECT_INNER(addr, len, FALSE, "")
# define UNPROTECT(addr, len) PROTECT_INNER(addr, len, TRUE, "un-")

# if defined(MSWIN32)
    typedef LPTOP_LEVEL_EXCEPTION_FILTER SIG_HNDLR_PTR;
#   undef SIG_DFL
#   define SIG_DFL ((LPTOP_LEVEL_EXCEPTION_FILTER)~(GC_funcptr_uint)0)
# elif defined(MSWINCE)
    typedef LONG (WINAPI *SIG_HNDLR_PTR)(struct _EXCEPTION_POINTERS *);
#   undef SIG_DFL
#   define SIG_DFL ((SIG_HNDLR_PTR)~(GC_funcptr_uint)0)
# elif defined(DARWIN)
#   ifdef BROKEN_EXCEPTION_HANDLING
      typedef void (*SIG_HNDLR_PTR)();
#   endif
# else
    typedef void (*SIG_HNDLR_PTR)(int, siginfo_t *, void *);
    typedef void (*PLAIN_HNDLR_PTR)(int);
# endif /* !DARWIN && !MSWIN32 && !MSWINCE */

#ifndef DARWIN
  STATIC SIG_HNDLR_PTR GC_old_segv_handler = 0;
                        /* Also old MSWIN32 ACCESS_VIOLATION filter */
# ifdef USE_BUS_SIGACT
    STATIC SIG_HNDLR_PTR GC_old_bus_handler = 0;
    STATIC GC_bool GC_old_bus_handler_used_si = FALSE;
# endif
# if !defined(MSWIN32) && !defined(MSWINCE)
    STATIC GC_bool GC_old_segv_handler_used_si = FALSE;
# endif /* !MSWIN32 */
#endif /* !DARWIN */

#ifdef THREADS
  /* This function is used only by the fault handler.  Potential data   */
  /* race between this function and GC_install_header, GC_remove_header */
  /* should not be harmful because the added or removed header should   */
  /* be already unprotected.                                            */
  GC_ATTR_NO_SANITIZE_THREAD
  static GC_bool is_header_found_async(const void *p)
  {
#   ifdef HASH_TL
      hdr *result;

      GET_HDR(p, result);
      return result != NULL;
#   else
      return HDR_INNER(p) != NULL;
#   endif
  }
#else
# define is_header_found_async(p) (HDR(p) != NULL)
#endif /* !THREADS */

#ifndef DARWIN

# if !defined(MSWIN32) && !defined(MSWINCE)
#   include <errno.h>
#   ifdef USE_BUS_SIGACT
#     define SIG_OK (sig == SIGBUS || sig == SIGSEGV)
#   else
#     define SIG_OK (sig == SIGSEGV)
                            /* Catch SIGSEGV but ignore SIGBUS. */
#   endif
#   if defined(FREEBSD) || defined(OPENBSD)
#     ifndef SEGV_ACCERR
#       define SEGV_ACCERR 2
#     endif
#     if defined(AARCH64) || defined(ARM32) || defined(MIPS) \
         || (__FreeBSD__ >= 7 || defined(OPENBSD))
#       define CODE_OK (si -> si_code == SEGV_ACCERR)
#     elif defined(POWERPC)
#       define AIM  /* Pretend that we're AIM. */
#       include <machine/trap.h>
#       define CODE_OK (si -> si_code == EXC_DSI \
                        || si -> si_code == SEGV_ACCERR)
#     else
#       define CODE_OK (si -> si_code == BUS_PAGE_FAULT \
                        || si -> si_code == SEGV_ACCERR)
#     endif
#   elif defined(OSF1)
#     define CODE_OK (si -> si_code == 2 /* experimentally determined */)
#   elif defined(IRIX5)
#     define CODE_OK (si -> si_code == EACCES)
#   elif defined(AIX) || defined(CYGWIN32) || defined(HAIKU) || defined(HURD)
#     define CODE_OK TRUE
#   elif defined(LINUX)
#     define CODE_OK TRUE
      /* Empirically c.trapno == 14, on IA32, but is that useful?       */
      /* Should probably consider alignment issues on other             */
      /* architectures.                                                 */
#   elif defined(HPUX)
#     define CODE_OK (si -> si_code == SEGV_ACCERR \
                      || si -> si_code == BUS_ADRERR \
                      || si -> si_code == BUS_UNKNOWN \
                      || si -> si_code == SEGV_UNKNOWN \
                      || si -> si_code == BUS_OBJERR)
#   elif defined(SUNOS5SIGS)
#     define CODE_OK (si -> si_code == SEGV_ACCERR)
#   endif
#   ifndef NO_GETCONTEXT
#     include <ucontext.h>
#   endif
    STATIC void GC_write_fault_handler(int sig, siginfo_t *si, void *raw_sc)
# else
#   define SIG_OK (exc_info -> ExceptionRecord -> ExceptionCode \
                     == STATUS_ACCESS_VIOLATION)
#   define CODE_OK (exc_info -> ExceptionRecord -> ExceptionInformation[0] \
                      == 1) /* Write fault */
    STATIC LONG WINAPI GC_write_fault_handler(
                                struct _EXCEPTION_POINTERS *exc_info)
# endif /* MSWIN32 || MSWINCE */
  {
#   if !defined(MSWIN32) && !defined(MSWINCE)
        char *addr = (char *)si->si_addr;
#   else
        char *addr = (char *)(exc_info -> ExceptionRecord
                                -> ExceptionInformation[1]);
#   endif

    if (SIG_OK && CODE_OK) {
        struct hblk *h = HBLK_PAGE_ALIGNED(addr);
        GC_bool in_allocd_block;
        size_t i;

        GC_ASSERT(GC_page_size != 0);
#       ifdef CHECKSUMS
          GC_record_fault(h);
#       endif
#       ifdef SUNOS5SIGS
            /* Address is only within the correct physical page.        */
            in_allocd_block = FALSE;
            for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
              if (is_header_found_async(&h[i])) {
                in_allocd_block = TRUE;
                break;
              }
            }
#       else
            in_allocd_block = is_header_found_async(addr);
#       endif
        if (!in_allocd_block) {
            /* FIXME - We should make sure that we invoke the   */
            /* old handler with the appropriate calling         */
            /* sequence, which often depends on SA_SIGINFO.     */

            /* Heap blocks now begin and end on page boundaries */
            SIG_HNDLR_PTR old_handler;

#           if defined(MSWIN32) || defined(MSWINCE)
                old_handler = GC_old_segv_handler;
#           else
                GC_bool used_si;

#             ifdef USE_BUS_SIGACT
                if (sig == SIGBUS) {
                   old_handler = GC_old_bus_handler;
                   used_si = GC_old_bus_handler_used_si;
                } else
#             endif
                /* else */ {
                   old_handler = GC_old_segv_handler;
                   used_si = GC_old_segv_handler_used_si;
                }
#           endif

            if ((GC_funcptr_uint)old_handler == (GC_funcptr_uint)SIG_DFL) {
#               if !defined(MSWIN32) && !defined(MSWINCE)
                    ABORT_ARG1("Unexpected segmentation fault outside heap",
                               " at %p", (void *)addr);
#               else
                    return EXCEPTION_CONTINUE_SEARCH;
#               endif
            } else {
                /*
                 * FIXME: This code should probably check if the
                 * old signal handler used the traditional style and
                 * if so call it using that style.
                 */
#               if defined(MSWIN32) || defined(MSWINCE)
                    return (*old_handler)(exc_info);
#               else
                    if (used_si)
                      ((SIG_HNDLR_PTR)old_handler)(sig, si, raw_sc);
                    else
                      /* FIXME: should pass nonstandard args as well. */
                      ((PLAIN_HNDLR_PTR)(GC_funcptr_uint)old_handler)(sig);
                    return;
#               endif
            }
        }
        UNPROTECT(h, GC_page_size);
        /* We need to make sure that no collection occurs between       */
        /* the UNPROTECT and the setting of the dirty bit.  Otherwise   */
        /* a write by a third thread might go unnoticed.  Reversing     */
        /* the order is just as bad, since we would end up unprotecting */
        /* a page in a GC cycle during which it's not marked.           */
        /* Currently we do this by disabling the thread stopping        */
        /* signals while this handler is running.  An alternative might */
        /* be to record the fact that we're about to unprotect, or      */
        /* have just unprotected a page in the GC's thread structure,   */
        /* and then to have the thread stopping code set the dirty      */
        /* flag, if necessary.                                          */
        for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
            size_t index = PHT_HASH(h + i);

            async_set_pht_entry_from_index(GC_dirty_pages, index);
        }
        /* The write may not take place before dirty bits are read.     */
        /* But then we'll fault again ...                               */
#       if defined(MSWIN32) || defined(MSWINCE)
            return EXCEPTION_CONTINUE_EXECUTION;
#       else
            return;
#       endif
    }
#   if defined(MSWIN32) || defined(MSWINCE)
      return EXCEPTION_CONTINUE_SEARCH;
#   else
      ABORT_ARG1("Unexpected bus error or segmentation fault",
                 " at %p", (void *)addr);
#   endif
  }

# if defined(GC_WIN32_THREADS) && !defined(CYGWIN32)
    GC_INNER void GC_set_write_fault_handler(void)
    {
      SetUnhandledExceptionFilter(GC_write_fault_handler);
    }
# endif

# ifdef SOFT_VDB
    static GC_bool soft_dirty_init(void);
# endif

  GC_INNER GC_bool GC_dirty_init(void)
  {
#   if !defined(MSWIN32) && !defined(MSWINCE)
      struct sigaction act, oldact;
#   endif

    GC_ASSERT(I_HOLD_LOCK());
#   ifdef COUNT_PROTECTED_REGIONS
      GC_ASSERT(GC_page_size != 0);
      if ((signed_word)(GC_heapsize / (word)GC_page_size)
                >= ((signed_word)GC_UNMAPPED_REGIONS_SOFT_LIMIT
                    - GC_num_unmapped_regions) * 2) {
        GC_COND_LOG_PRINTF("Cannot turn on GC incremental mode"
                           " as heap contains too many pages\n");
        return FALSE;
      }
#   endif
#   if !defined(MSWIN32) && !defined(MSWINCE)
      act.sa_flags = SA_RESTART | SA_SIGINFO;
      act.sa_sigaction = GC_write_fault_handler;
      (void)sigemptyset(&act.sa_mask);
#     ifdef SIGNAL_BASED_STOP_WORLD
        /* Arrange to postpone the signal while we are in a write fault */
        /* handler.  This effectively makes the handler atomic w.r.t.   */
        /* stopping the world for GC.                                   */
        (void)sigaddset(&act.sa_mask, GC_get_suspend_signal());
#     endif
#   endif /* !MSWIN32 */
    GC_VERBOSE_LOG_PRINTF(
                "Initializing mprotect virtual dirty bit implementation\n");
    if (GC_page_size % HBLKSIZE != 0) {
        ABORT("Page size not multiple of HBLKSIZE");
    }
#   ifdef GWW_VDB
      if (GC_gww_dirty_init()) {
        GC_COND_LOG_PRINTF("Using GetWriteWatch()\n");
        return TRUE;
      }
#   elif defined(SOFT_VDB)
#     ifdef CHECK_SOFT_VDB
        if (!soft_dirty_init())
          ABORT("Soft-dirty bit support is missing");
#     else
        if (soft_dirty_init()) {
          GC_COND_LOG_PRINTF("Using soft-dirty bit feature\n");
          return TRUE;
        }
#     endif
#   endif
#   ifdef MSWIN32
      GC_old_segv_handler = SetUnhandledExceptionFilter(
                                        GC_write_fault_handler);
      if (GC_old_segv_handler != NULL) {
        GC_COND_LOG_PRINTF("Replaced other UnhandledExceptionFilter\n");
      } else {
          GC_old_segv_handler = SIG_DFL;
      }
#   elif defined(MSWINCE)
      /* MPROTECT_VDB is unsupported for WinCE at present.      */
      /* FIXME: implement it (if possible). */
#   else
      /* act.sa_restorer is deprecated and should not be initialized. */
#     if defined(GC_IRIX_THREADS)
        sigaction(SIGSEGV, 0, &oldact);
        sigaction(SIGSEGV, &act, 0);
#     else
        {
          int res = sigaction(SIGSEGV, &act, &oldact);
          if (res != 0) ABORT("Sigaction failed");
        }
#     endif
      if (oldact.sa_flags & SA_SIGINFO) {
        GC_old_segv_handler = oldact.sa_sigaction;
        GC_old_segv_handler_used_si = TRUE;
      } else {
        GC_old_segv_handler =
                        (SIG_HNDLR_PTR)(GC_funcptr_uint)oldact.sa_handler;
        GC_old_segv_handler_used_si = FALSE;
      }
      if ((GC_funcptr_uint)GC_old_segv_handler == (GC_funcptr_uint)SIG_IGN) {
        WARN("Previously ignored segmentation violation!?\n", 0);
        GC_old_segv_handler = (SIG_HNDLR_PTR)(GC_funcptr_uint)SIG_DFL;
      }
      if ((GC_funcptr_uint)GC_old_segv_handler != (GC_funcptr_uint)SIG_DFL) {
        GC_VERBOSE_LOG_PRINTF("Replaced other SIGSEGV handler\n");
      }
#     ifdef USE_BUS_SIGACT
        sigaction(SIGBUS, &act, &oldact);
        if ((oldact.sa_flags & SA_SIGINFO) != 0) {
          GC_old_bus_handler = oldact.sa_sigaction;
          GC_old_bus_handler_used_si = TRUE;
        } else {
          GC_old_bus_handler =
                        (SIG_HNDLR_PTR)(GC_funcptr_uint)oldact.sa_handler;
        }
        if ((GC_funcptr_uint)GC_old_bus_handler == (GC_funcptr_uint)SIG_IGN) {
          WARN("Previously ignored bus error!?\n", 0);
          GC_old_bus_handler = (SIG_HNDLR_PTR)(GC_funcptr_uint)SIG_DFL;
        } else if ((GC_funcptr_uint)GC_old_bus_handler
                   != (GC_funcptr_uint)SIG_DFL) {
          GC_VERBOSE_LOG_PRINTF("Replaced other SIGBUS handler\n");
        }
#     endif
#   endif /* !MSWIN32 && !MSWINCE */
#   if defined(CPPCHECK) && defined(ADDRESS_SANITIZER)
      GC_noop1((word)(GC_funcptr_uint)&__asan_default_options);
#   endif
    return TRUE;
  }
#endif /* !DARWIN */

#define PAGE_ALIGNED(x) ((ADDR(x) & (GC_page_size-1)) == 0)

STATIC void GC_protect_heap(void)
{
  size_t i;

  GC_ASSERT(GC_page_size != 0);
  for (i = 0; i < GC_n_heap_sects; i++) {
    ptr_t start = GC_heap_sects[i].hs_start;
    size_t len = GC_heap_sects[i].hs_bytes;
    struct hblk *current;
    struct hblk *current_start; /* start of block to be protected */
    ptr_t limit;

    GC_ASSERT(PAGE_ALIGNED(start));
    GC_ASSERT(PAGE_ALIGNED(len));
#   ifndef DONT_PROTECT_PTRFREE
      /* We avoid protecting pointer-free objects unless the page   */
      /* size differs from HBLKSIZE.                                */
      if (GC_page_size != HBLKSIZE) {
        PROTECT(start, len);
        continue;
      }
#   endif

    current_start = (struct hblk *)start;
    limit = start + len;
    for (current = current_start;;) {
      size_t nblocks = 0;
      GC_bool is_ptrfree = TRUE;

      if (ADDR_LT((ptr_t)current, limit)) {
        hdr *hhdr;

        GET_HDR(current, hhdr);
        if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
          /* This can happen only if we are at the beginning of */
          /* a heap segment, and a block spans heap segments.   */
          /* We will handle that block as part of the preceding */
          /* segment.                                           */
          GC_ASSERT(current_start == current);

          current_start = ++current;
          continue;
        }
        if (HBLK_IS_FREE(hhdr)) {
          GC_ASSERT(modHBLKSZ(hhdr -> hb_sz) == 0);
          nblocks = divHBLKSZ(hhdr -> hb_sz);
        } else {
          nblocks = OBJ_SZ_TO_BLOCKS(hhdr -> hb_sz);
          is_ptrfree = IS_PTRFREE(hhdr);
        }
      }
      if (is_ptrfree) {
        if (ADDR_LT((ptr_t)current_start, (ptr_t)current)) {
#         ifdef DONT_PROTECT_PTRFREE
            ptr_t cur_aligned = PTR_ALIGN_UP((ptr_t)current, GC_page_size);

            current_start = HBLK_PAGE_ALIGNED(current_start);
            /* Adjacent free blocks might be protected too because  */
            /* of the alignment by the page size.                   */
            PROTECT(current_start, cur_aligned - (ptr_t)current_start);
#         else
            PROTECT(current_start, (ptr_t)current - (ptr_t)current_start);
#         endif
        }
        if (ADDR_GE((ptr_t)current, limit)) break;
      }
      current += nblocks;
      if (is_ptrfree) current_start = current;
    }
  }
}

# if defined(CAN_HANDLE_FORK) && defined(DARWIN) && defined(THREADS) \
     || defined(COUNT_PROTECTED_REGIONS)
    /* Remove protection for the entire heap not updating GC_dirty_pages. */
    STATIC void GC_unprotect_all_heap(void)
    {
      size_t i;

      GC_ASSERT(I_HOLD_LOCK());
      GC_ASSERT(GC_auto_incremental);
      for (i = 0; i < GC_n_heap_sects; i++) {
        UNPROTECT(GC_heap_sects[i].hs_start, GC_heap_sects[i].hs_bytes);
      }
    }
# endif /* CAN_HANDLE_FORK && DARWIN && THREADS || COUNT_PROTECTED_REGIONS */

# ifdef COUNT_PROTECTED_REGIONS
    GC_INNER void GC_handle_protected_regions_limit(void)
    {
      GC_ASSERT(GC_page_size != 0);
      /* To prevent exceeding the limit of vm.max_map_count, the most */
      /* trivial (though highly restrictive) way is to turn off the   */
      /* incremental collection mode (based on mprotect) once the     */
      /* number of pages in the heap reaches that limit.              */
      if (GC_auto_incremental && !GC_GWW_AVAILABLE()
          && (signed_word)(GC_heapsize / (word)GC_page_size)
                >= ((signed_word)GC_UNMAPPED_REGIONS_SOFT_LIMIT
                    - GC_num_unmapped_regions) * 2) {
        GC_unprotect_all_heap();
#       ifdef DARWIN
          GC_task_self = 0;
#       endif
        GC_incremental = FALSE;
        WARN("GC incremental mode is turned off"
             " to prevent hitting VM maps limit\n", 0);
      }
    }
# endif /* COUNT_PROTECTED_REGIONS */

#endif /* MPROTECT_VDB */

#if !defined(THREADS) && (defined(PROC_VDB) || defined(SOFT_VDB))
  static pid_t saved_proc_pid; /* pid used to compose /proc file names */
#endif

#ifdef PROC_VDB
/* This implementation assumes a Solaris 2.X like /proc                 */
/* pseudo-file-system from which we can read page modified bits.  This  */
/* facility is far from optimal (e.g. we would like to get the info for */
/* only some of the address space), but it avoids intercepting system   */
/* calls.                                                               */

# include <errno.h>
# include <sys/signal.h>
# include <sys/syscall.h>
# include <sys/stat.h>

# ifdef GC_NO_SYS_FAULT_H
    /* This exists only to check PROC_VDB code compilation (on Linux).  */
#   define PG_MODIFIED 1
    struct prpageheader {
      int dummy[2]; /* pr_tstamp */
      unsigned long pr_nmap;
      unsigned long pr_npage;
    };
    struct prasmap {
      char *pr_vaddr;
      size_t pr_npage;
      char dummy1[64+8]; /* pr_mapname, pr_offset */
      unsigned pr_mflags;
      unsigned pr_pagesize;
      int dummy2[2];
    };
# else
#   include <sys/fault.h>
#   include <sys/procfs.h>
# endif

# define INITIAL_BUF_SZ 16384
  STATIC size_t GC_proc_buf_size = INITIAL_BUF_SZ;
  STATIC char *GC_proc_buf = NULL;
  STATIC int GC_proc_fd = -1;

  static GC_bool proc_dirty_open_files(void)
  {
    char buf[40];
    pid_t pid = getpid();

    (void)snprintf(buf, sizeof(buf), "/proc/%ld/pagedata", (long)pid);
    buf[sizeof(buf) - 1] = '\0';
    GC_proc_fd = open(buf, O_RDONLY);
    if (-1 == GC_proc_fd) {
      WARN("/proc open failed; cannot enable GC incremental mode\n", 0);
      return FALSE;
    }
    if (syscall(SYS_fcntl, GC_proc_fd, F_SETFD, FD_CLOEXEC) == -1)
      WARN("Could not set FD_CLOEXEC for /proc\n", 0);
#   ifndef THREADS
      saved_proc_pid = pid; /* updated on success only */
#   endif
    return TRUE;
  }

# ifdef CAN_HANDLE_FORK
    GC_INNER void GC_dirty_update_child(void)
    {
      GC_ASSERT(I_HOLD_LOCK());
      if (-1 == GC_proc_fd)
        return; /* GC incremental mode is off */

      close(GC_proc_fd);
      if (!proc_dirty_open_files())
        GC_incremental = FALSE; /* should be safe to turn it off */
    }
# endif /* CAN_HANDLE_FORK */

GC_INNER GC_bool GC_dirty_init(void)
{
    GC_ASSERT(I_HOLD_LOCK());
    if (GC_bytes_allocd != 0 || GC_bytes_allocd_before_gc != 0) {
      memset(GC_written_pages, 0xff, sizeof(page_hash_table));
      GC_VERBOSE_LOG_PRINTF(
                "Allocated %lu bytes: all pages may have been written\n",
                (unsigned long)(GC_bytes_allocd + GC_bytes_allocd_before_gc));
    }
    if (!proc_dirty_open_files())
      return FALSE;
    GC_proc_buf = GC_scratch_alloc(GC_proc_buf_size);
    if (GC_proc_buf == NULL)
      ABORT("Insufficient space for /proc read");
    return TRUE;
}

GC_INLINE void GC_proc_read_dirty(GC_bool output_unneeded)
{
    size_t i, nmaps;
    char * bufp = GC_proc_buf;

    GC_ASSERT(I_HOLD_LOCK());
#   ifndef THREADS
      /* If the current pid differs from the saved one, then we are in  */
      /* the forked (child) process, the current /proc file should be   */
      /* closed, the new one should be opened with the updated path.    */
      /* Note, this is not needed for multi-threaded case because       */
      /* fork_child_proc() reopens the file right after fork.           */
      if (getpid() != saved_proc_pid
          && (-1 == GC_proc_fd /* no need to retry */
              || (close(GC_proc_fd), !proc_dirty_open_files()))) {
        /* Failed to reopen the file.  Punt!    */
        if (!output_unneeded)
          memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
        memset(GC_written_pages, 0xff, sizeof(page_hash_table));
        return;
      }
#   endif

    BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));
    if (PROC_READ(GC_proc_fd, bufp, GC_proc_buf_size) <= 0) {
        /* Retry with larger buffer.    */
        size_t new_size = 2 * GC_proc_buf_size;
        char *new_buf;

        WARN("/proc read failed (buffer size is %" WARN_PRIuPTR " bytes)\n",
             GC_proc_buf_size);
        new_buf = GC_scratch_alloc(new_size);
        if (new_buf != 0) {
            GC_scratch_recycle_no_gww(bufp, GC_proc_buf_size);
            GC_proc_buf = bufp = new_buf;
            GC_proc_buf_size = new_size;
        }
        if (PROC_READ(GC_proc_fd, bufp, GC_proc_buf_size) <= 0) {
            WARN("Insufficient space for /proc read\n", 0);
            /* Punt:        */
            if (!output_unneeded)
              memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
            memset(GC_written_pages, 0xff, sizeof(page_hash_table));
            return;
        }
    }

    /* Copy dirty bits into GC_grungy_pages     */
    nmaps = (size_t)(((struct prpageheader *)bufp) -> pr_nmap);
#   ifdef DEBUG_DIRTY_BITS
      GC_log_printf("Proc VDB read: pr_nmap= %u, pr_npage= %lu\n",
                    (unsigned)nmaps, ((struct prpageheader *)bufp)->pr_npage);
#   endif
#   if defined(GC_NO_SYS_FAULT_H) && defined(CPPCHECK)
      GC_noop1(((struct prpageheader *)bufp)->dummy[0]);
#   endif
    bufp += sizeof(struct prpageheader);
    for (i = 0; i < nmaps; i++) {
        struct prasmap * map = (struct prasmap *)bufp;
        ptr_t vaddr = (ptr_t)(map -> pr_vaddr);
        unsigned long npages = map -> pr_npage;
        unsigned pagesize = map -> pr_pagesize;
        ptr_t limit;

#       if defined(GC_NO_SYS_FAULT_H) && defined(CPPCHECK)
          GC_noop1(map->dummy1[0] + map->dummy2[0]);
#       endif
#       ifdef DEBUG_DIRTY_BITS
          GC_log_printf(
                "pr_vaddr= %p, npage= %lu, mflags= 0x%x, pagesize= 0x%x\n",
                (void *)vaddr, npages, map->pr_mflags, pagesize);
#       endif

        bufp += sizeof(struct prasmap);
        limit = vaddr + pagesize * npages;
        for (; ADDR_LT(vaddr, limit); vaddr += pagesize) {
            if ((*bufp++) & PG_MODIFIED) {
                struct hblk * h;
                ptr_t next_vaddr = vaddr + pagesize;

#               ifdef DEBUG_DIRTY_BITS
                  GC_log_printf("dirty page at: %p\n", (void *)vaddr);
#               endif
                for (h = (struct hblk *)vaddr;
                     ADDR_LT((ptr_t)h, next_vaddr); h++) {
                    size_t index = PHT_HASH(h);

                    set_pht_entry_from_index(GC_grungy_pages, index);
                }
            }
        }
        bufp = PTR_ALIGN_UP(bufp, sizeof(long));
    }
#   ifdef DEBUG_DIRTY_BITS
      GC_log_printf("Proc VDB read done\n");
#   endif

    /* Update GC_written_pages (even if output_unneeded).       */
    GC_or_pages(GC_written_pages, GC_grungy_pages);
}

#endif /* PROC_VDB */

#ifdef SOFT_VDB
# ifndef VDB_BUF_SZ
#   define VDB_BUF_SZ 16384
# endif

  static int open_proc_fd(pid_t pid, const char *proc_filename, int mode)
  {
    int f;
    char buf[40];

    (void)snprintf(buf, sizeof(buf), "/proc/%ld/%s", (long)pid,
                   proc_filename);
    buf[sizeof(buf) - 1] = '\0';
    f = open(buf, mode);
    if (-1 == f) {
      WARN("/proc/self/%s open failed; cannot enable GC incremental mode\n",
           proc_filename);
    } else if (fcntl(f, F_SETFD, FD_CLOEXEC) == -1) {
      WARN("Could not set FD_CLOEXEC for /proc\n", 0);
    }
    return f;
  }

# include <stdint.h> /* for uint64_t */

  typedef uint64_t pagemap_elem_t;

  static pagemap_elem_t *soft_vdb_buf;
  static int pagemap_fd;

  static GC_bool soft_dirty_open_files(void)
  {
    pid_t pid = getpid();

    clear_refs_fd = open_proc_fd(pid, "clear_refs", O_WRONLY);
    if (-1 == clear_refs_fd)
      return FALSE;
    pagemap_fd = open_proc_fd(pid, "pagemap", O_RDONLY);
    if (-1 == pagemap_fd) {
      close(clear_refs_fd);
      clear_refs_fd = -1;
      return FALSE;
    }
#   ifndef THREADS
      saved_proc_pid = pid; /* updated on success only */
#   endif
    return TRUE;
  }

# ifdef CAN_HANDLE_FORK
    GC_INNER void GC_dirty_update_child(void)
    {
      GC_ASSERT(I_HOLD_LOCK());
      if (-1 == clear_refs_fd)
        return; /* GC incremental mode is off */

      close(clear_refs_fd);
      close(pagemap_fd);
      if (!soft_dirty_open_files())
        GC_incremental = FALSE;
    }
# endif /* CAN_HANDLE_FORK */

  /* Clear soft-dirty bits from the task's PTEs.        */
  static void clear_soft_dirty_bits(void)
  {
    ssize_t res = write(clear_refs_fd, "4\n", 2);

    if (res != 2)
      ABORT_ARG1("Failed to write to /proc/self/clear_refs",
                 ": errno= %d", res < 0 ? errno : 0);
  }

  /* The bit 55 of the 64-bit qword of pagemap file is the soft-dirty one. */
# define PM_SOFTDIRTY_MASK ((pagemap_elem_t)1 << 55)

  static GC_bool detect_soft_dirty_supported(ptr_t vaddr)
  {
    off_t fpos;
    pagemap_elem_t buf[1];

    GC_ASSERT(GC_log_pagesize != 0);
    *vaddr = 1; /* make it dirty */
    fpos = (off_t)((ADDR(vaddr) >> GC_log_pagesize) * sizeof(pagemap_elem_t));

    for (;;) {
      /* Read the relevant PTE from the pagemap file.   */
      if (lseek(pagemap_fd, fpos, SEEK_SET) == (off_t)(-1))
        return FALSE;
      if (PROC_READ(pagemap_fd, buf, sizeof(buf)) != (int)sizeof(buf))
        return FALSE;

      /* Is the soft-dirty bit unset?   */
      if ((buf[0] & PM_SOFTDIRTY_MASK) == 0) return FALSE;

      if (0 == *vaddr) break;
      /* Retry to check that writing to clear_refs works as expected.   */
      /* This malfunction of the soft-dirty bits implementation is      */
      /* observed on some Linux kernels on Power9 (e.g. in Fedora 36).  */
      clear_soft_dirty_bits();
      *vaddr = 0;
    }
    return TRUE; /* success */
  }

# ifndef NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK
#   include <sys/utsname.h>
#   include <string.h> /* for strcmp() */

    /* Ensure the linux (kernel) major/minor version is as given or higher. */
    static GC_bool ensure_min_linux_ver(int major, int minor) {
      struct utsname info;
      int actual_major;
      int actual_minor = -1;

      if (uname(&info) == -1) {
        return FALSE; /* uname() failed, should not happen actually. */
      }
      if (strcmp(info.sysname, "Linux")) {
        WARN("Cannot ensure Linux version as running on other OS: %s\n",
             info.sysname);
        return FALSE;
      }
      actual_major = GC_parse_version(&actual_minor, info.release);
      return actual_major > major
             || (actual_major == major && actual_minor >= minor);
    }
# endif

# ifdef MPROTECT_VDB
    static GC_bool soft_dirty_init(void)
# else
    GC_INNER GC_bool GC_dirty_init(void)
# endif
  {
#   if defined(MPROTECT_VDB) && !defined(CHECK_SOFT_VDB)
      char * str = GETENV("GC_USE_GETWRITEWATCH");
#     ifdef GC_PREFER_MPROTECT_VDB
        if (str == NULL || (*str == '0' && *(str + 1) == '\0'))
          return FALSE; /* the environment variable is unset or set to "0" */
#     else
        if (str != NULL && *str == '0' && *(str + 1) == '\0')
          return FALSE; /* the environment variable is set "0" */
#     endif
#   endif
    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(NULL == soft_vdb_buf);
#   ifndef NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK
      if (!ensure_min_linux_ver(3, 18)) {
        GC_COND_LOG_PRINTF(
            "Running on old kernel lacking correct soft-dirty bit support\n");
        return FALSE;
      }
#   endif
    if (!soft_dirty_open_files())
      return FALSE;
    soft_vdb_buf = (pagemap_elem_t *)GC_scratch_alloc(VDB_BUF_SZ);
    if (NULL == soft_vdb_buf)
      ABORT("Insufficient space for /proc pagemap buffer");
    if (!detect_soft_dirty_supported((ptr_t)soft_vdb_buf)) {
      GC_COND_LOG_PRINTF("Soft-dirty bit is not supported by kernel\n");
      /* Release the resources. */
      GC_scratch_recycle_no_gww(soft_vdb_buf, VDB_BUF_SZ);
      soft_vdb_buf = NULL;
      close(clear_refs_fd);
      clear_refs_fd = -1;
      close(pagemap_fd);
      return FALSE;
    }
    return TRUE;
  }

  static off_t pagemap_buf_fpos; /* valid only if pagemap_buf_len > 0 */
  static size_t pagemap_buf_len;

  /* Read bytes from /proc/self/pagemap at given file position.         */
  /* len - the maximum number of bytes to read; (*pres) - amount of     */
  /* bytes actually read, always bigger than 0 but never exceeds len;   */
  /* next_fpos_hint - the file position of the next bytes block to read */
  /* ahead if possible (0 means no information provided).               */
  static const pagemap_elem_t *pagemap_buffered_read(size_t *pres,
                                                     off_t fpos, size_t len,
                                                     off_t next_fpos_hint)
  {
    ssize_t res;
    size_t ofs;

    GC_ASSERT(GC_page_size != 0);
    GC_ASSERT(len > 0);
    if (pagemap_buf_fpos <= fpos
        && fpos < pagemap_buf_fpos + (off_t)pagemap_buf_len) {
      /* The requested data is already in the buffer.   */
      ofs = (size_t)(fpos - pagemap_buf_fpos);
      res = (ssize_t)(pagemap_buf_fpos + pagemap_buf_len - fpos);
    } else {
      off_t aligned_pos = fpos & ~(off_t)(GC_page_size < VDB_BUF_SZ
                                            ? GC_page_size-1 : VDB_BUF_SZ-1);

      for (;;) {
        size_t count;

        if ((0 == pagemap_buf_len
             || pagemap_buf_fpos + (off_t)pagemap_buf_len != aligned_pos)
            && lseek(pagemap_fd, aligned_pos, SEEK_SET) == (off_t)(-1))
          ABORT_ARG2("Failed to lseek /proc/self/pagemap",
                     ": offset= %lu, errno= %d", (unsigned long)fpos, errno);

        /* How much to read at once?    */
        ofs = (size_t)(fpos - aligned_pos);
        GC_ASSERT(ofs < VDB_BUF_SZ);
        if (next_fpos_hint > aligned_pos
            && next_fpos_hint - aligned_pos < VDB_BUF_SZ) {
          count = VDB_BUF_SZ;
        } else {
          count = len + ofs;
          if (count > VDB_BUF_SZ)
            count = VDB_BUF_SZ;
        }

        GC_ASSERT(count % sizeof(pagemap_elem_t) == 0);
        res = PROC_READ(pagemap_fd, soft_vdb_buf, count);
        if (res > (ssize_t)ofs)
          break;
        if (res <= 0)
          ABORT_ARG1("Failed to read /proc/self/pagemap",
                     ": errno= %d", res < 0 ? errno : 0);
        /* Retry (once) w/o page-alignment.     */
        aligned_pos = fpos;
      }

      /* Save the buffer (file window) position and size.       */
      pagemap_buf_fpos = aligned_pos;
      pagemap_buf_len = (size_t)res;
      res -= (ssize_t)ofs;
    }

    GC_ASSERT(ofs % sizeof(pagemap_elem_t) == 0);
    *pres = (size_t)res < len ? (size_t)res : len;
    return &soft_vdb_buf[ofs / sizeof(pagemap_elem_t)];
  }

  static void soft_set_grungy_pages(ptr_t start, ptr_t limit,
                                    ptr_t next_start_hint,
                                    GC_bool is_static_root)
  {
    ptr_t vaddr = (ptr_t)HBLK_PAGE_ALIGNED(start);
    off_t next_fpos_hint = (off_t)((ADDR(next_start_hint) >> GC_log_pagesize)
                                   * sizeof(pagemap_elem_t));

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(modHBLKSZ(ADDR(start)) == 0);
    GC_ASSERT(GC_log_pagesize != 0);
    while (ADDR_LT(vaddr, limit)) {
      size_t res;
      ptr_t limit_buf;
      const pagemap_elem_t *bufp = pagemap_buffered_read(&res,
                (off_t)((ADDR(vaddr) >> GC_log_pagesize)
                        * sizeof(pagemap_elem_t)),
                (size_t)(((ADDR(limit) - ADDR(vaddr)
                           + GC_page_size - 1) >> GC_log_pagesize)
                         * sizeof(pagemap_elem_t)),
                next_fpos_hint);

      if (res % sizeof(pagemap_elem_t) != 0) {
        /* Punt: */
        memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
        WARN("Incomplete read of pagemap, not multiple of entry size\n", 0);
        break;
      }

      limit_buf = vaddr + ((res / sizeof(pagemap_elem_t)) << GC_log_pagesize);
      for (; ADDR_LT(vaddr, limit_buf); vaddr += GC_page_size, bufp++) {
        if ((*bufp & PM_SOFTDIRTY_MASK) != 0) {
          struct hblk * h;
          ptr_t next_vaddr = vaddr + GC_page_size;

          if (EXPECT(ADDR_LT(limit, next_vaddr), FALSE))
            next_vaddr = limit;
          /* If the bit is set, the respective PTE was written to       */
          /* since clearing the soft-dirty bits.                        */
#         ifdef DEBUG_DIRTY_BITS
            if (is_static_root)
              GC_log_printf("static root dirty page at: %p\n", (void *)vaddr);
#         endif
          h = (struct hblk *)vaddr;
          if (EXPECT(ADDR_LT(vaddr, start), FALSE))
            h = (struct hblk *)start;
          for (; ADDR_LT((ptr_t)h, next_vaddr); h++) {
            size_t index = PHT_HASH(h);

            /* Filter out the blocks without pointers.  It might worth  */
            /* for the case when the heap is large enough for the hash  */
            /* collisions to occur frequently.  Thus, off by default.   */
#           if defined(FILTER_PTRFREE_HBLKS_IN_SOFT_VDB) \
               || defined(CHECKSUMS) || defined(DEBUG_DIRTY_BITS)
              if (!is_static_root) {
                hdr *hhdr;

#               ifdef CHECKSUMS
                  set_pht_entry_from_index(GC_written_pages, index);
#               endif
                GET_HDR(h, hhdr);
                if (NULL == hhdr) continue;

                (void)GC_find_starting_hblk(h, &hhdr);
                if (HBLK_IS_FREE(hhdr) || IS_PTRFREE(hhdr)) continue;
#               ifdef DEBUG_DIRTY_BITS
                  GC_log_printf("dirty page (hblk) at: %p\n", (void *)h);
#               endif
              }
#           else
              UNUSED_ARG(is_static_root);
#           endif
            set_pht_entry_from_index(GC_grungy_pages, index);
          }
        } else {
#         if defined(CHECK_SOFT_VDB) /* && MPROTECT_VDB */
            /* Ensure that each clean page according to the soft-dirty  */
            /* VDB is also identified such by the mprotect-based one.   */
            if (!is_static_root
                && get_pht_entry_from_index(GC_dirty_pages, PHT_HASH(vaddr))) {
              ptr_t my_start, my_end; /* the values are not used */

              /* There could be a hash collision, thus we need to       */
              /* verify the page is clean using slow GC_get_maps().     */
              if (GC_enclosing_writable_mapping(vaddr, &my_start, &my_end)) {
                ABORT("Inconsistent soft-dirty against mprotect dirty bits");
              }
            }
#         endif
        }
      }
      /* Read the next portion of pagemap file if incomplete.   */
    }
  }

  GC_INLINE void GC_soft_read_dirty(GC_bool output_unneeded)
  {
    GC_ASSERT(I_HOLD_LOCK());
#   ifndef THREADS
      /* Similar as for GC_proc_read_dirty.     */
      if (getpid() != saved_proc_pid
          && (-1 == clear_refs_fd /* no need to retry */
              || (close(clear_refs_fd), close(pagemap_fd),
                  !soft_dirty_open_files()))) {
        /* Failed to reopen the files.  */
        if (!output_unneeded) {
          /* Punt: */
          memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
#         ifdef CHECKSUMS
            memset(GC_written_pages, 0xff, sizeof(page_hash_table));
#         endif
        }
        return;
      }
#   endif

    if (!output_unneeded) {
      size_t i;

      BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));
      pagemap_buf_len = 0; /* invalidate soft_vdb_buf */

      for (i = 0; i < GC_n_heap_sects; ++i) {
        ptr_t start = GC_heap_sects[i].hs_start;

        soft_set_grungy_pages(start, start + GC_heap_sects[i].hs_bytes,
                              i + 1 < GC_n_heap_sects
                                    ? GC_heap_sects[i+1].hs_start : NULL,
                              FALSE);
      }

#     ifndef NO_VDB_FOR_STATIC_ROOTS
        for (i = 0; i < n_root_sets; ++i) {
          soft_set_grungy_pages((ptr_t)HBLKPTR(GC_static_roots[i].r_start),
                                GC_static_roots[i].r_end,
                                i + 1 < n_root_sets
                                    ? GC_static_roots[i+1].r_start : NULL,
                                TRUE);
        }
#     endif
    }

    clear_soft_dirty_bits();
  }
#endif /* SOFT_VDB */

#ifdef PCR_VDB

# include "vd/PCR_VD.h"

# define NPAGES (32*1024)       /* 128 MB */

PCR_VD_DB GC_grungy_bits[NPAGES];

STATIC ptr_t GC_vd_base = NULL;
                        /* Address corresponding to GC_grungy_bits[0]   */
                        /* HBLKSIZE aligned.                            */

GC_INNER GC_bool GC_dirty_init(void)
{
    /* For the time being, we assume the heap generally grows up */
    GC_vd_base = GC_heap_sects[0].hs_start;
    if (GC_vd_base == 0) {
        ABORT("Bad initial heap segment");
    }
    if (PCR_VD_Start(HBLKSIZE, GC_vd_base, NPAGES*HBLKSIZE)
        != PCR_ERes_okay) {
        ABORT("Dirty bit initialization failed");
    }
    return TRUE;
}
#endif /* PCR_VDB */

#ifndef NO_MANUAL_VDB
  GC_INNER GC_bool GC_manual_vdb = FALSE;

  /* Manually mark the page containing p as dirty.  Logically, this     */
  /* dirties the entire object.                                         */
  GC_INNER void GC_dirty_inner(const void *p)
  {
    size_t index = PHT_HASH(p);

#   if defined(MPROTECT_VDB)
      /* Do not update GC_dirty_pages if it should be followed by the   */
      /* page unprotection.                                             */
      GC_ASSERT(GC_manual_vdb);
#   endif
    async_set_pht_entry_from_index(GC_dirty_pages, index);
  }
#endif /* !NO_MANUAL_VDB */

#ifndef GC_DISABLE_INCREMENTAL
  /* Retrieve system dirty bits for the heap to a local buffer (unless  */
  /* output_unneeded).  Restore the systems notion of which pages are   */
  /* dirty.  We assume that either the world is stopped or it is OK to  */
  /* lose dirty bits while it is happening (GC_enable_incremental is    */
  /* the caller and output_unneeded is TRUE at least if multi-threading */
  /* support is on).                                                    */
  GC_INNER void GC_read_dirty(GC_bool output_unneeded)
  {
    GC_ASSERT(I_HOLD_LOCK());
#   ifdef DEBUG_DIRTY_BITS
      GC_log_printf("read dirty begin\n");
#   endif
    if (GC_manual_vdb
#       if defined(MPROTECT_VDB)
          || !GC_GWW_AVAILABLE()
#       endif
        ) {
      if (!output_unneeded)
        BCOPY(CAST_AWAY_VOLATILE_PVOID(GC_dirty_pages), GC_grungy_pages,
              sizeof(GC_dirty_pages));
      BZERO(CAST_AWAY_VOLATILE_PVOID(GC_dirty_pages), sizeof(GC_dirty_pages));
#     ifdef MPROTECT_VDB
        if (!GC_manual_vdb)
          GC_protect_heap();
#     endif
      return;
    }

#   ifdef GWW_VDB
      GC_gww_read_dirty(output_unneeded);
#   elif defined(PROC_VDB)
      GC_proc_read_dirty(output_unneeded);
#   elif defined(SOFT_VDB)
      GC_soft_read_dirty(output_unneeded);
#   elif defined(PCR_VDB)
      /* lazily enable dirty bits on newly added heap sects */
      {
        static size_t onhs = 0;
        size_t nhs = GC_n_heap_sects;

        for (; onhs < nhs; onhs++) {
            PCR_VD_WriteProtectEnable(
                    GC_heap_sects[onhs].hs_start,
                    GC_heap_sects[onhs].hs_bytes);
        }
      }
      if (PCR_VD_Clear(GC_vd_base, NPAGES*HBLKSIZE, GC_grungy_bits)
          != PCR_ERes_okay) {
        ABORT("Dirty bit read failed");
      }
#   endif
#   if defined(CHECK_SOFT_VDB) /* && MPROTECT_VDB */
      BZERO(CAST_AWAY_VOLATILE_PVOID(GC_dirty_pages), sizeof(GC_dirty_pages));
      GC_protect_heap();
#   endif
  }

# if !defined(NO_VDB_FOR_STATIC_ROOTS) && !defined(PROC_VDB)
    GC_INNER GC_bool GC_is_vdb_for_static_roots(void)
    {
      if (GC_manual_vdb) return FALSE;
#     if defined(MPROTECT_VDB)
        /* Currently used only in conjunction with SOFT_VDB.    */
        return GC_GWW_AVAILABLE();
#     else
#       ifndef LINT2
          GC_ASSERT(GC_incremental);
#       endif
        return TRUE;
#     endif
    }
# endif

  /* Is the HBLKSIZE sized page at h marked dirty in the local buffer?  */
  /* If the actual page size is different, this returns TRUE if any     */
  /* of the pages overlapping h are dirty.  This routine may err on the */
  /* side of labeling pages as dirty (and this implementation does).    */
  GC_INNER GC_bool GC_page_was_dirty(struct hblk *h)
  {
    size_t index;

#   ifdef PCR_VDB
      if (!GC_manual_vdb) {
        if (!ADDR_INSIDE((ptr_t)h, GC_vd_base, GC_vd_base + NPAGES * HBLKSIZE))
          return TRUE;
        return GC_grungy_bits[h-(struct hblk*)GC_vd_base] & PCR_VD_DB_dirtyBit;
      }
#   elif defined(DEFAULT_VDB)
      if (!GC_manual_vdb)
        return TRUE;
#   elif defined(PROC_VDB)
      /* Unless manual VDB is on, the bitmap covers all process memory. */
      if (GC_manual_vdb)
#   endif
      {
        if (NULL == HDR(h))
          return TRUE;
      }
    index = PHT_HASH(h);
    return get_pht_entry_from_index(GC_grungy_pages, index);
  }

# if defined(CHECKSUMS) || defined(PROC_VDB)
    /* Could any valid GC heap pointer ever have been written to this page? */
    GC_INNER GC_bool GC_page_was_ever_dirty(struct hblk *h)
    {
#     if defined(GWW_VDB) || defined(PROC_VDB) || defined(SOFT_VDB)
        size_t index;

#       ifdef MPROTECT_VDB
          if (!GC_GWW_AVAILABLE())
            return TRUE;
#       endif
#       if defined(PROC_VDB)
          if (GC_manual_vdb)
#       endif
        {
          if (NULL == HDR(h))
            return TRUE;
        }
        index = PHT_HASH(h);
        return get_pht_entry_from_index(GC_written_pages, index);
#     else
        /* TODO: implement me for MANUAL_VDB. */
        UNUSED_ARG(h);
        return TRUE;
#     endif
    }
# endif /* CHECKSUMS || PROC_VDB */

  GC_INNER void GC_remove_protection(struct hblk *h, size_t nblocks,
                                     GC_bool is_ptrfree)
  {
#   ifdef MPROTECT_VDB
      struct hblk * current;
      struct hblk * h_trunc;    /* Truncated to page boundary */
      ptr_t h_end;              /* Page boundary following block end */
#   endif

#   ifndef PARALLEL_MARK
      GC_ASSERT(I_HOLD_LOCK());
#   endif
#   ifdef MPROTECT_VDB
      /* Note it is not allowed to call GC_printf (and the friends) */
      /* in this function, see Win32 GC_stop_world for the details. */
#     ifdef DONT_PROTECT_PTRFREE
        if (is_ptrfree) return;
#     endif
      if (!GC_auto_incremental || GC_GWW_AVAILABLE())
        return;
      GC_ASSERT(GC_page_size != 0);
      h_trunc = HBLK_PAGE_ALIGNED(h);
      h_end = PTR_ALIGN_UP((ptr_t)(h + nblocks), GC_page_size);
      /* Note that we cannot examine GC_dirty_pages to check    */
      /* whether the page at h_trunc has already been marked    */
      /* dirty as there could be a hash collision.              */
      for (current = h_trunc; ADDR_LT((ptr_t)current, h_end); ++current) {
        size_t index = PHT_HASH(current);

#       ifndef DONT_PROTECT_PTRFREE
          if (!is_ptrfree
              || !ADDR_INSIDE((ptr_t)current, (ptr_t)h, (ptr_t)(h + nblocks)))
#       endif
        {
          async_set_pht_entry_from_index(GC_dirty_pages, index);
        }
      }
      UNPROTECT(h_trunc, h_end - (ptr_t)h_trunc);
#   elif defined(PCR_VDB)
      UNUSED_ARG(is_ptrfree);
      if (!GC_auto_incremental)
        return;
      PCR_VD_WriteProtectDisable(h, nblocks * HBLKSIZE);
      PCR_VD_WriteProtectEnable(h, nblocks * HBLKSIZE);
#   else
      /* Ignore write hints.  They don't help us here.  */
      UNUSED_ARG(h);
      UNUSED_ARG(nblocks);
      UNUSED_ARG(is_ptrfree);
#   endif
  }
#endif /* !GC_DISABLE_INCREMENTAL */

#if defined(MPROTECT_VDB) && defined(DARWIN)
/* The following sources were used as a "reference" for this exception
   handling code:
      1. Apple's mach/xnu documentation
      2. Timothy J. Wood's "Mach Exception Handlers 101" post to the
         omnigroup's macosx-dev list.
         www.omnigroup.com/mailman/archive/macosx-dev/2000-June/014178.html
      3. macosx-nat.c from Apple's GDB source code.
*/

/* The bug that caused all this trouble should now be fixed. This should
   eventually be removed if all goes well. */

#include <mach/mach.h>
#include <mach/mach_error.h>
#include <mach/exception.h>
#include <mach/task.h>

EXTERN_C_BEGIN

/* Some of the following prototypes are missing in any header, although */
/* they are documented.  Some are in mach/exc.h file.                   */
extern boolean_t
exc_server(mach_msg_header_t *, mach_msg_header_t *);

extern kern_return_t
exception_raise(mach_port_t, mach_port_t, mach_port_t, exception_type_t,
                exception_data_t, mach_msg_type_number_t);

extern kern_return_t
exception_raise_state(mach_port_t, mach_port_t, mach_port_t, exception_type_t,
                      exception_data_t, mach_msg_type_number_t,
                      thread_state_flavor_t*, thread_state_t,
                      mach_msg_type_number_t, thread_state_t,
                      mach_msg_type_number_t*);

extern kern_return_t
exception_raise_state_identity(mach_port_t, mach_port_t, mach_port_t,
                               exception_type_t, exception_data_t,
                               mach_msg_type_number_t, thread_state_flavor_t*,
                               thread_state_t, mach_msg_type_number_t,
                               thread_state_t, mach_msg_type_number_t*);

GC_API_OSCALL kern_return_t
catch_exception_raise(mach_port_t exception_port, mach_port_t thread,
                      mach_port_t task, exception_type_t exception,
                      exception_data_t code,
                      mach_msg_type_number_t code_count);

GC_API_OSCALL kern_return_t
catch_exception_raise_state(mach_port_name_t exception_port,
                int exception, exception_data_t code,
                mach_msg_type_number_t codeCnt, int flavor,
                thread_state_t old_state, int old_stateCnt,
                thread_state_t new_state, int new_stateCnt);

GC_API_OSCALL kern_return_t
catch_exception_raise_state_identity(mach_port_name_t exception_port,
                mach_port_t thread, mach_port_t task, int exception,
                exception_data_t code, mach_msg_type_number_t codeCnt,
                int flavor, thread_state_t old_state, int old_stateCnt,
                thread_state_t new_state, int new_stateCnt);

EXTERN_C_END

/* These should never be called, but just in case...  */
GC_API_OSCALL kern_return_t
catch_exception_raise_state(mach_port_name_t exception_port, int exception,
                            exception_data_t code,
                            mach_msg_type_number_t codeCnt, int flavor,
                            thread_state_t old_state, int old_stateCnt,
                            thread_state_t new_state, int new_stateCnt)
{
  UNUSED_ARG(exception_port);
  UNUSED_ARG(exception);
  UNUSED_ARG(code);
  UNUSED_ARG(codeCnt);
  UNUSED_ARG(flavor);
  UNUSED_ARG(old_state);
  UNUSED_ARG(old_stateCnt);
  UNUSED_ARG(new_state);
  UNUSED_ARG(new_stateCnt);
  ABORT_RET("Unexpected catch_exception_raise_state invocation");
  return KERN_INVALID_ARGUMENT;
}

GC_API_OSCALL kern_return_t
catch_exception_raise_state_identity(mach_port_name_t exception_port,
                                     mach_port_t thread, mach_port_t task,
                                     int exception, exception_data_t code,
                                     mach_msg_type_number_t codeCnt,
                                     int flavor, thread_state_t old_state,
                                     int old_stateCnt,
                                     thread_state_t new_state,
                                     int new_stateCnt)
{
  UNUSED_ARG(exception_port);
  UNUSED_ARG(thread);
  UNUSED_ARG(task);
  UNUSED_ARG(exception);
  UNUSED_ARG(code);
  UNUSED_ARG(codeCnt);
  UNUSED_ARG(flavor);
  UNUSED_ARG(old_state);
  UNUSED_ARG(old_stateCnt);
  UNUSED_ARG(new_state);
  UNUSED_ARG(new_stateCnt);
  ABORT_RET("Unexpected catch_exception_raise_state_identity invocation");
  return KERN_INVALID_ARGUMENT;
}

#define MAX_EXCEPTION_PORTS 16

static struct {
  mach_msg_type_number_t count;
  exception_mask_t      masks[MAX_EXCEPTION_PORTS];
  exception_handler_t   ports[MAX_EXCEPTION_PORTS];
  exception_behavior_t  behaviors[MAX_EXCEPTION_PORTS];
  thread_state_flavor_t flavors[MAX_EXCEPTION_PORTS];
} GC_old_exc_ports;

STATIC struct ports_s {
  void (*volatile os_callback[3])(void);
  mach_port_t exception;
# if defined(THREADS)
    mach_port_t reply;
# endif
} GC_ports = {
  {
    /* This is to prevent stripping these routines as dead.     */
    (void (*)(void))catch_exception_raise,
    (void (*)(void))catch_exception_raise_state,
    (void (*)(void))catch_exception_raise_state_identity
  },
# ifdef THREADS
    0, /* for 'exception' */
# endif
  0
};

typedef struct {
    mach_msg_header_t head;
} GC_msg_t;

typedef enum {
    GC_MP_NORMAL,
    GC_MP_DISCARDING,
    GC_MP_STOPPED
} GC_mprotect_state_t;

#ifdef THREADS
  /* FIXME: 1 and 2 seem to be safe to use in the msgh_id field, but it */
  /* is not documented.  Use the source and see if they should be OK.   */
# define ID_STOP 1
# define ID_RESUME 2

  /* This value is only used on the reply port. */
# define ID_ACK 3

  STATIC GC_mprotect_state_t GC_mprotect_state = GC_MP_NORMAL;

  /* The following should ONLY be called when the world is stopped.     */
  STATIC void GC_mprotect_thread_notify(mach_msg_id_t id)
  {
    struct buf_s {
      GC_msg_t msg;
      mach_msg_trailer_t trailer;
    } buf;
    mach_msg_return_t r;

    /* remote, local */
    buf.msg.head.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_MAKE_SEND, 0);
    buf.msg.head.msgh_size = sizeof(buf.msg);
    buf.msg.head.msgh_remote_port = GC_ports.exception;
    buf.msg.head.msgh_local_port = MACH_PORT_NULL;
    buf.msg.head.msgh_id = id;

    r = mach_msg(&buf.msg.head, MACH_SEND_MSG | MACH_RCV_MSG | MACH_RCV_LARGE,
                 sizeof(buf.msg), sizeof(buf), GC_ports.reply,
                 MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
    if (r != MACH_MSG_SUCCESS)
      ABORT("mach_msg failed in GC_mprotect_thread_notify");
    if (buf.msg.head.msgh_id != ID_ACK)
      ABORT("Invalid ack in GC_mprotect_thread_notify");
  }

  /* Should only be called by the mprotect thread */
  STATIC void GC_mprotect_thread_reply(void)
  {
    GC_msg_t msg;
    mach_msg_return_t r;
    /* remote, local */

    msg.head.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_MAKE_SEND, 0);
    msg.head.msgh_size = sizeof(msg);
    msg.head.msgh_remote_port = GC_ports.reply;
    msg.head.msgh_local_port = MACH_PORT_NULL;
    msg.head.msgh_id = ID_ACK;

    r = mach_msg(&msg.head, MACH_SEND_MSG, sizeof(msg), 0, MACH_PORT_NULL,
                 MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
    if (r != MACH_MSG_SUCCESS)
      ABORT("mach_msg failed in GC_mprotect_thread_reply");
  }

  GC_INNER void GC_mprotect_stop(void)
  {
    GC_mprotect_thread_notify(ID_STOP);
  }

  GC_INNER void GC_mprotect_resume(void)
  {
    GC_mprotect_thread_notify(ID_RESUME);
  }

# ifdef CAN_HANDLE_FORK
    GC_INNER void GC_dirty_update_child(void)
    {
      GC_ASSERT(I_HOLD_LOCK());
      if (0 == GC_task_self) return; /* GC incremental mode is off */

      GC_ASSERT(GC_mprotect_state == GC_MP_NORMAL);
      GC_task_self = mach_task_self(); /* needed by UNPROTECT() */
      GC_unprotect_all_heap();

      /* Restore the old task exception ports.  */
      /* TODO: Should we do it in fork_prepare/parent_proc? */
      if (GC_old_exc_ports.count > 0) {
        /* TODO: Should we check GC_old_exc_ports.count<=1? */
        if (task_set_exception_ports(GC_task_self, GC_old_exc_ports.masks[0],
                    GC_old_exc_ports.ports[0], GC_old_exc_ports.behaviors[0],
                    GC_old_exc_ports.flavors[0]) != KERN_SUCCESS)
          ABORT("task_set_exception_ports failed (in child)");
      }

      /* TODO: Re-enable incremental mode in child. */
      GC_task_self = 0;
      GC_incremental = FALSE;
    }
# endif /* CAN_HANDLE_FORK */

#else
  /* The compiler should optimize away any GC_mprotect_state computations */
# define GC_mprotect_state GC_MP_NORMAL
#endif /* !THREADS */

struct mp_reply_s {
  mach_msg_header_t head;
  char data[256];
};

struct mp_msg_s {
  mach_msg_header_t head;
  mach_msg_body_t msgh_body;
  char data[1024];
};

STATIC void *GC_mprotect_thread(void *arg)
{
  mach_msg_return_t r;
  /* These two structures contain some private kernel data.  We don't   */
  /* need to access any of it so we don't bother defining a proper      */
  /* struct.  The correct definitions are in the xnu source code.       */
  struct mp_reply_s reply;
  struct mp_msg_s msg;
  mach_msg_id_t id;

  if (ADDR(arg) == GC_WORD_MAX) return 0; /* to prevent a compiler warning */
# if defined(CPPCHECK)
    reply.data[0] = 0; /* to prevent "field unused" warnings */
    msg.data[0] = 0;
# endif

# if defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID)
    (void)pthread_setname_np("GC-mprotect");
# endif
# if defined(THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
    GC_darwin_register_self_mach_handler();
# endif

  for (;;) {
    r = mach_msg(&msg.head, MACH_RCV_MSG | MACH_RCV_LARGE |
                 (GC_mprotect_state == GC_MP_DISCARDING ? MACH_RCV_TIMEOUT
                  : 0), 0, sizeof(msg), GC_ports.exception,
                 GC_mprotect_state == GC_MP_DISCARDING ? 0
                 : MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
    id = r == MACH_MSG_SUCCESS ? msg.head.msgh_id : -1;

#   if defined(THREADS)
      if (GC_mprotect_state == GC_MP_DISCARDING) {
        if (r == MACH_RCV_TIMED_OUT) {
          GC_mprotect_state = GC_MP_STOPPED;
          GC_mprotect_thread_reply();
          continue;
        }
        if (r == MACH_MSG_SUCCESS && (id == ID_STOP || id == ID_RESUME))
          ABORT("Out of order mprotect thread request");
      }
#   endif /* THREADS */

    if (r != MACH_MSG_SUCCESS) {
      ABORT_ARG2("mach_msg failed",
                 ": errcode= %d (%s)", (int)r, mach_error_string(r));
    }

    switch (id) {
#     if defined(THREADS)
        case ID_STOP:
          if (GC_mprotect_state != GC_MP_NORMAL)
            ABORT("Called mprotect_stop when state wasn't normal");
          GC_mprotect_state = GC_MP_DISCARDING;
          break;
        case ID_RESUME:
          if (GC_mprotect_state != GC_MP_STOPPED)
            ABORT("Called mprotect_resume when state wasn't stopped");
          GC_mprotect_state = GC_MP_NORMAL;
          GC_mprotect_thread_reply();
          break;
#     endif /* THREADS */
        default:
          /* Handle the message (calls catch_exception_raise) */
          if (!exc_server(&msg.head, &reply.head))
            ABORT("exc_server failed");
          /* Send the reply */
          r = mach_msg(&reply.head, MACH_SEND_MSG, reply.head.msgh_size, 0,
                       MACH_PORT_NULL, MACH_MSG_TIMEOUT_NONE,
                       MACH_PORT_NULL);
          if (r != MACH_MSG_SUCCESS) {
            /* This will fail if the thread dies, but the thread */
            /* shouldn't die... */
#           ifdef BROKEN_EXCEPTION_HANDLING
              GC_err_printf("mach_msg failed with %d %s while sending "
                            "exc reply\n", (int)r, mach_error_string(r));
#           else
              ABORT("mach_msg failed while sending exception reply");
#           endif
          }
    } /* switch */
  } /* for */
}

/* All this SIGBUS code shouldn't be necessary. All protection faults should
   be going through the mach exception handler. However, it seems a SIGBUS is
   occasionally sent for some unknown reason. Even more odd, it seems to be
   meaningless and safe to ignore. */
#ifdef BROKEN_EXCEPTION_HANDLING

  /* Updates to this aren't atomic, but the SIGBUS'es seem pretty rare.    */
  /* Even if this doesn't get updated property, it isn't really a problem. */
  STATIC int GC_sigbus_count = 0;

  STATIC void GC_darwin_sigbus(int num, siginfo_t *sip, void *context)
  {
    if (num != SIGBUS)
      ABORT("Got a non-sigbus signal in the sigbus handler");

    /* Ugh... some seem safe to ignore, but too many in a row probably means
       trouble. GC_sigbus_count is reset for each mach exception that is
       handled */
    if (GC_sigbus_count >= 8)
      ABORT("Got many SIGBUS signals in a row!");
    GC_sigbus_count++;
    WARN("Ignoring SIGBUS\n", 0);
  }
#endif /* BROKEN_EXCEPTION_HANDLING */

GC_INNER GC_bool GC_dirty_init(void)
{
  kern_return_t r;
  mach_port_t me;
  pthread_t thread;
  pthread_attr_t attr;
  exception_mask_t mask;

  GC_ASSERT(I_HOLD_LOCK());
# if defined(CAN_HANDLE_FORK) && !defined(THREADS)
    if (GC_handle_fork) {
      /* To both support GC incremental mode and GC functions usage in  */
      /* the forked child, pthread_atfork should be used to install     */
      /* handlers that switch off GC_incremental in the child           */
      /* gracefully (unprotecting all pages and clearing                */
      /* GC_mach_handler_thread).  For now, we just disable incremental */
      /* mode if fork() handling is requested by the client.            */
      WARN("Can't turn on GC incremental mode as fork()"
           " handling requested\n", 0);
      return FALSE;
    }
# endif

  GC_VERBOSE_LOG_PRINTF("Initializing mach/darwin mprotect"
                        " virtual dirty bit implementation\n");
# ifdef BROKEN_EXCEPTION_HANDLING
    WARN("Enabling workarounds for various darwin exception handling bugs\n",
         0);
# endif
  if (GC_page_size % HBLKSIZE != 0) {
    ABORT("Page size not multiple of HBLKSIZE");
  }

  GC_task_self = me = mach_task_self();
  GC_ASSERT(me != 0);

  r = mach_port_allocate(me, MACH_PORT_RIGHT_RECEIVE, &GC_ports.exception);
  /* TODO: WARN and return FALSE in case of a failure. */
  if (r != KERN_SUCCESS)
    ABORT("mach_port_allocate failed (exception port)");

  r = mach_port_insert_right(me, GC_ports.exception, GC_ports.exception,
                             MACH_MSG_TYPE_MAKE_SEND);
  if (r != KERN_SUCCESS)
    ABORT("mach_port_insert_right failed (exception port)");

# if defined(THREADS)
    r = mach_port_allocate(me, MACH_PORT_RIGHT_RECEIVE, &GC_ports.reply);
    if (r != KERN_SUCCESS)
      ABORT("mach_port_allocate failed (reply port)");
# endif

  /* The exceptions we want to catch. */
  mask = EXC_MASK_BAD_ACCESS;
  r = task_get_exception_ports(me, mask, GC_old_exc_ports.masks,
                               &GC_old_exc_ports.count, GC_old_exc_ports.ports,
                               GC_old_exc_ports.behaviors,
                               GC_old_exc_ports.flavors);
  if (r != KERN_SUCCESS)
    ABORT("task_get_exception_ports failed");

  r = task_set_exception_ports(me, mask, GC_ports.exception, EXCEPTION_DEFAULT,
                               GC_MACH_THREAD_STATE);
  if (r != KERN_SUCCESS)
    ABORT("task_set_exception_ports failed");

  if (pthread_attr_init(&attr) != 0)
    ABORT("pthread_attr_init failed");
  if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0)
    ABORT("pthread_attr_setdetachedstate failed");
  /* This will call the real pthread function, not our wrapper. */
  if (GC_inner_pthread_create(&thread, &attr, GC_mprotect_thread, NULL) != 0)
    ABORT("pthread_create failed");
  (void)pthread_attr_destroy(&attr);

  /* Setup the sigbus handler for ignoring the meaningless SIGBUS signals. */
# ifdef BROKEN_EXCEPTION_HANDLING
    {
      struct sigaction sa, oldsa;
      sa.sa_handler = (SIG_HNDLR_PTR)GC_darwin_sigbus;
      sigemptyset(&sa.sa_mask);
      sa.sa_flags = SA_RESTART | SA_SIGINFO;
      /* sa.sa_restorer is deprecated and should not be initialized. */
      if (sigaction(SIGBUS, &sa, &oldsa) < 0)
        ABORT("sigaction failed");
      if ((GC_funcptr_uint)oldsa.sa_handler != (GC_funcptr_uint)SIG_DFL) {
        GC_VERBOSE_LOG_PRINTF("Replaced other SIGBUS handler\n");
      }
    }
# endif /* BROKEN_EXCEPTION_HANDLING  */
# if defined(CPPCHECK)
    GC_noop1((word)GC_ports.os_callback[0]);
# endif
  return TRUE;
}

/* The source code for Apple's GDB was used as a reference for the      */
/* exception forwarding code.  This code is similar to be GDB code only */
/* because there is only one way to do it.                              */
STATIC kern_return_t GC_forward_exception(mach_port_t thread, mach_port_t task,
                                          exception_type_t exception,
                                          exception_data_t data,
                                          mach_msg_type_number_t data_count)
{
  size_t i;
  kern_return_t r;
  mach_port_t port;
  exception_behavior_t behavior;
  thread_state_flavor_t flavor;

  thread_state_data_t thread_state;
  mach_msg_type_number_t thread_state_count = THREAD_STATE_MAX;

  for (i = 0; i < (size_t)GC_old_exc_ports.count; i++) {
    if ((GC_old_exc_ports.masks[i] & ((exception_mask_t)1 << exception)) != 0)
      break;
  }
  if (i == (size_t)GC_old_exc_ports.count)
    ABORT("No handler for exception!");

  port = GC_old_exc_ports.ports[i];
  behavior = GC_old_exc_ports.behaviors[i];
  flavor = GC_old_exc_ports.flavors[i];

  if (behavior == EXCEPTION_STATE || behavior == EXCEPTION_STATE_IDENTITY) {
    r = thread_get_state(thread, flavor, thread_state, &thread_state_count);
    if (r != KERN_SUCCESS)
      ABORT("thread_get_state failed in forward_exception");
  }

  switch (behavior) {
    case EXCEPTION_STATE:
      r = exception_raise_state(port, thread, task, exception, data,
                                data_count, &flavor, thread_state,
                                thread_state_count, thread_state,
                                &thread_state_count);
      break;
    case EXCEPTION_STATE_IDENTITY:
      r = exception_raise_state_identity(port, thread, task, exception, data,
                                         data_count, &flavor, thread_state,
                                         thread_state_count, thread_state,
                                         &thread_state_count);
      break;
    /* case EXCEPTION_DEFAULT: */ /* default signal handlers */
    default: /* user-supplied signal handlers */
      r = exception_raise(port, thread, task, exception, data, data_count);
  }

  if (behavior == EXCEPTION_STATE || behavior == EXCEPTION_STATE_IDENTITY) {
    r = thread_set_state(thread, flavor, thread_state, thread_state_count);
    if (r != KERN_SUCCESS)
      ABORT("thread_set_state failed in forward_exception");
  }
  return r;
}

#define FWD() GC_forward_exception(thread, task, exception, code, code_count)

#ifdef ARM32
# define DARWIN_EXC_STATE         ARM_EXCEPTION_STATE
# define DARWIN_EXC_STATE_COUNT   ARM_EXCEPTION_STATE_COUNT
# define DARWIN_EXC_STATE_T       arm_exception_state_t
# define DARWIN_EXC_STATE_DAR     THREAD_FLD_NAME(far)
#elif defined(AARCH64)
# define DARWIN_EXC_STATE         ARM_EXCEPTION_STATE64
# define DARWIN_EXC_STATE_COUNT   ARM_EXCEPTION_STATE64_COUNT
# define DARWIN_EXC_STATE_T       arm_exception_state64_t
# define DARWIN_EXC_STATE_DAR     THREAD_FLD_NAME(far)
#elif defined(POWERPC)
# if CPP_WORDSZ == 32
#   define DARWIN_EXC_STATE       PPC_EXCEPTION_STATE
#   define DARWIN_EXC_STATE_COUNT PPC_EXCEPTION_STATE_COUNT
#   define DARWIN_EXC_STATE_T     ppc_exception_state_t
# else
#   define DARWIN_EXC_STATE       PPC_EXCEPTION_STATE64
#   define DARWIN_EXC_STATE_COUNT PPC_EXCEPTION_STATE64_COUNT
#   define DARWIN_EXC_STATE_T     ppc_exception_state64_t
# endif
# define DARWIN_EXC_STATE_DAR     THREAD_FLD_NAME(dar)
#elif defined(I386) || defined(X86_64)
# if CPP_WORDSZ == 32
#   if defined(i386_EXCEPTION_STATE_COUNT) \
       && !defined(x86_EXCEPTION_STATE32_COUNT)
      /* Use old naming convention for i686.    */
#     define DARWIN_EXC_STATE           i386_EXCEPTION_STATE
#     define DARWIN_EXC_STATE_COUNT     i386_EXCEPTION_STATE_COUNT
#     define DARWIN_EXC_STATE_T         i386_exception_state_t
#   else
#     define DARWIN_EXC_STATE           x86_EXCEPTION_STATE32
#     define DARWIN_EXC_STATE_COUNT     x86_EXCEPTION_STATE32_COUNT
#     define DARWIN_EXC_STATE_T         x86_exception_state32_t
#   endif
# else
#   define DARWIN_EXC_STATE       x86_EXCEPTION_STATE64
#   define DARWIN_EXC_STATE_COUNT x86_EXCEPTION_STATE64_COUNT
#   define DARWIN_EXC_STATE_T     x86_exception_state64_t
# endif
# define DARWIN_EXC_STATE_DAR     THREAD_FLD_NAME(faultvaddr)
#elif !defined(CPPCHECK)
# error FIXME for non-arm/ppc/x86 darwin
#endif

/* This violates the namespace rules but there isn't anything that can  */
/* be done about it.  The exception handling stuff is hard coded to     */
/* call this.  catch_exception_raise, catch_exception_raise_state and   */
/* and catch_exception_raise_state_identity are called from OS.         */
GC_API_OSCALL kern_return_t
catch_exception_raise(mach_port_t exception_port, mach_port_t thread,
                      mach_port_t task, exception_type_t exception,
                      exception_data_t code, mach_msg_type_number_t code_count)
{
  kern_return_t r;
  char *addr;
  thread_state_flavor_t flavor = DARWIN_EXC_STATE;
  mach_msg_type_number_t exc_state_count = DARWIN_EXC_STATE_COUNT;
  DARWIN_EXC_STATE_T exc_state;

  UNUSED_ARG(exception_port);
  UNUSED_ARG(task);
  if (exception != EXC_BAD_ACCESS || code[0] != KERN_PROTECTION_FAILURE) {
#   ifdef DEBUG_EXCEPTION_HANDLING
      /* We aren't interested, pass it on to the old handler */
      GC_log_printf("Exception: 0x%x Code: 0x%x 0x%x in catch...\n",
                    exception, code_count > 0 ? code[0] : -1,
                    code_count > 1 ? code[1] : -1);
#   else
      UNUSED_ARG(code_count);
#   endif
    return FWD();
  }

  r = thread_get_state(thread, flavor, (natural_t*)&exc_state,
                       &exc_state_count);
  if (r != KERN_SUCCESS) {
    /* The thread is supposed to be suspended while the exception       */
    /* handler is called.  This shouldn't fail.                         */
#   ifdef BROKEN_EXCEPTION_HANDLING
      GC_err_printf("thread_get_state failed in catch_exception_raise\n");
      return KERN_SUCCESS;
#   else
      ABORT("thread_get_state failed in catch_exception_raise");
#   endif
  }

  /* This is the address that caused the fault */
  addr = (char*)exc_state.DARWIN_EXC_STATE_DAR;
  if (!is_header_found_async(addr)) {
    /* Ugh... just like the SIGBUS problem above, it seems we get       */
    /* a bogus KERN_PROTECTION_FAILURE every once and a while.  We wait */
    /* till we get a bunch in a row before doing anything about it.     */
    /* If a "real" fault ever occurs it'll just keep faulting over and  */
    /* over and we'll hit the limit pretty quickly.                     */
#   ifdef BROKEN_EXCEPTION_HANDLING
      static const char *last_fault;
      static int last_fault_count;

      if (addr != last_fault) {
        last_fault = addr;
        last_fault_count = 0;
      }
      if (++last_fault_count < 32) {
        if (last_fault_count == 1)
          WARN("Ignoring KERN_PROTECTION_FAILURE at %p\n", addr);
        return KERN_SUCCESS;
      }

      GC_err_printf("Unexpected KERN_PROTECTION_FAILURE at %p; aborting...\n",
                    (void *)addr);
      /* Can't pass it along to the signal handler because that is      */
      /* ignoring SIGBUS signals.  We also shouldn't call ABORT here as */
      /* signals don't always work too well from the exception handler. */
      EXIT();
#   else /* BROKEN_EXCEPTION_HANDLING */
      /* Pass it along to the next exception handler
         (which should call SIGBUS/SIGSEGV) */
      return FWD();
#   endif /* !BROKEN_EXCEPTION_HANDLING */
  }

# ifdef BROKEN_EXCEPTION_HANDLING
    /* Reset the number of consecutive SIGBUS signals.  */
    GC_sigbus_count = 0;
# endif

  GC_ASSERT(GC_page_size != 0);
  if (GC_mprotect_state == GC_MP_NORMAL) { /* common case */
    struct hblk *h = HBLK_PAGE_ALIGNED(addr);
    size_t i;

#   ifdef CHECKSUMS
      GC_record_fault(h);
#   endif
    UNPROTECT(h, GC_page_size);
    for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
      size_t index = PHT_HASH(h + i);

      async_set_pht_entry_from_index(GC_dirty_pages, index);
    }
  } else if (GC_mprotect_state == GC_MP_DISCARDING) {
    /* Lie to the thread for now. No sense UNPROTECT()ing the memory
       when we're just going to PROTECT() it again later. The thread
       will just fault again once it resumes */
  } else {
    /* Shouldn't happen, i don't think */
    GC_err_printf("KERN_PROTECTION_FAILURE while world is stopped\n");
    return FWD();
  }
  return KERN_SUCCESS;
}
#undef FWD

#ifndef NO_DESC_CATCH_EXCEPTION_RAISE
  /* These symbols should have REFERENCED_DYNAMICALLY (0x10) bit set to */
  /* let strip know they are not to be stripped.                        */
  __asm__(".desc _catch_exception_raise, 0x10");
  __asm__(".desc _catch_exception_raise_state, 0x10");
  __asm__(".desc _catch_exception_raise_state_identity, 0x10");
#endif

#endif /* DARWIN && MPROTECT_VDB */

GC_API int GC_CALL GC_incremental_protection_needs(void)
{
  GC_ASSERT(GC_is_initialized);
# ifdef MPROTECT_VDB
#   if defined(GWW_VDB) || (defined(SOFT_VDB) && !defined(CHECK_SOFT_VDB))
      /* Only if the incremental mode is already switched on.   */
      if (GC_GWW_AVAILABLE())
        return GC_PROTECTS_NONE;
#   endif
#   ifndef DONT_PROTECT_PTRFREE
      if (GC_page_size != HBLKSIZE)
        return GC_PROTECTS_POINTER_HEAP | GC_PROTECTS_PTRFREE_HEAP;
#   endif
    return GC_PROTECTS_POINTER_HEAP;
# else
    return GC_PROTECTS_NONE;
# endif
}

GC_API unsigned GC_CALL GC_get_actual_vdb(void)
{
# ifndef GC_DISABLE_INCREMENTAL
    if (GC_incremental) {
#     ifndef NO_MANUAL_VDB
        if (GC_manual_vdb) return GC_VDB_MANUAL;
#     endif
#     ifdef MPROTECT_VDB
#       ifdef GWW_VDB
          if (GC_GWW_AVAILABLE()) return GC_VDB_GWW;
#       endif
#       ifdef SOFT_VDB
          if (GC_GWW_AVAILABLE()) return GC_VDB_SOFT;
#       endif
        return GC_VDB_MPROTECT;
#     elif defined(GWW_VDB)
        return GC_VDB_GWW;
#     elif defined(SOFT_VDB)
        return GC_VDB_SOFT;
#     elif defined(PCR_VDB)
        return GC_VDB_PCR;
#     elif defined(PROC_VDB)
        return GC_VDB_PROC;
#     else /* DEFAULT_VDB */
        return GC_VDB_DEFAULT;
#     endif
    }
# endif
  return GC_VDB_NONE;
}

#ifdef ECOS
  /* Undo sbrk() redirection. */
# undef sbrk
#endif

/* If value is non-zero then allocate executable memory.        */
GC_API void GC_CALL GC_set_pages_executable(int value)
{
  GC_ASSERT(!GC_is_initialized);
  /* Even if IGNORE_PAGES_EXECUTABLE is defined, GC_pages_executable is */
  /* touched here to prevent a compiler warning.                        */
  GC_pages_executable = (GC_bool)(value != 0);
}

/* Returns non-zero if the GC-allocated memory is executable.   */
/* GC_get_pages_executable is defined after all the places      */
/* where GC_get_pages_executable is undefined.                  */
GC_API int GC_CALL GC_get_pages_executable(void)
{
# ifdef IGNORE_PAGES_EXECUTABLE
    return 1;   /* Always allocate executable memory. */
# else
    return (int)GC_pages_executable;
# endif
}

/* Call stack save code for debugging.  Should probably be in           */
/* mach_dep.c, but that requires reorganization.                        */
#ifdef NEED_CALLINFO

  /* I suspect the following works for most *nix i686 variants, so long */
  /* as the frame pointer is explicitly stored.  In the case of gcc,    */
  /* the client code should not be compiled with -fomit-frame-pointer.  */
# if defined(I386) && defined(LINUX) && defined(SAVE_CALL_CHAIN)
    struct frame {
        struct  frame *fr_savfp;
        long    fr_savpc;
#       if NARGS > 0
          long  fr_arg[NARGS];  /* All the arguments go here.   */
#       endif
    };
# endif

# if defined(SPARC)
#   if defined(LINUX)
#     if defined(SAVE_CALL_CHAIN)
        struct frame {
          long fr_local[8];
          long fr_arg[6];
          struct frame *fr_savfp;
          long fr_savpc;
#         ifndef __arch64__
            char *fr_stret;
#         endif
          long fr_argd[6];
          long fr_argx[0];
        };
#     endif
#   elif defined (DRSNX)
#     include <sys/sparc/frame.h>
#   elif defined(OPENBSD)
#     include <frame.h>
#   elif defined(FREEBSD) || defined(NETBSD)
#     include <machine/frame.h>
#   else
#     include <sys/frame.h>
#   endif
#   if NARGS > 6
#     error We only know how to get the first 6 arguments
#   endif
# endif /* SPARC */

  /* Fill in the pc and argument information for up to NFRAMES of my    */
  /* callers.  Ignore my frame and my callers frame.                    */

# if defined(GC_HAVE_BUILTIN_BACKTRACE)
#   ifdef _MSC_VER
      EXTERN_C_BEGIN
      int backtrace(void* addresses[], int count);
      char** backtrace_symbols(void* const addresses[], int count);
      EXTERN_C_END
#   else
#     include <execinfo.h>
#   endif
# endif /* GC_HAVE_BUILTIN_BACKTRACE */

# ifdef SAVE_CALL_CHAIN

#   if NARGS == 0 && NFRAMES % 2 == 0 /* No padding */ \
       && defined(GC_HAVE_BUILTIN_BACKTRACE)

#     ifdef REDIRECT_MALLOC
        /* Deal with possible malloc calls in backtrace by omitting */
        /* the infinitely recursing backtrace.                      */
        STATIC GC_bool GC_in_save_callers = FALSE;

#       if defined(THREADS) && defined(DBG_HDRS_ALL)

          /* A dummy version of GC_save_callers() which does not call   */
          /* backtrace().                                               */
          GC_INNER void GC_save_callers_no_unlock(
                                        struct callinfo info[NFRAMES])
          {
            GC_ASSERT(I_HOLD_LOCK());
            info[0].ci_pc = CAST_THRU_UINTPTR(GC_return_addr_t,
                                              GC_save_callers_no_unlock);
            BZERO(&info[1], sizeof(void *) * (NFRAMES - 1));
          }
#       endif
#     endif /* REDIRECT_MALLOC */

      GC_INNER void GC_save_callers(struct callinfo info[NFRAMES])
      {
        void * tmp_info[NFRAMES + 1];
        int npcs, i;

        GC_ASSERT(I_HOLD_LOCK());
                /* backtrace() may call dl_iterate_phdr which is also   */
                /* used by GC_register_dynamic_libraries(), and         */
                /* dl_iterate_phdr is not guaranteed to be reentrant.   */

        GC_STATIC_ASSERT(sizeof(struct callinfo) == sizeof(void *));
#       ifdef REDIRECT_MALLOC
          if (GC_in_save_callers) {
            info[0].ci_pc = CAST_THRU_UINTPTR(GC_return_addr_t,
                                              GC_save_callers);
            BZERO(&info[1], sizeof(void *) * (NFRAMES - 1));
            return;
          }
          GC_in_save_callers = TRUE;
          /* backtrace() might call a redirected malloc. */
          UNLOCK();
          npcs = backtrace((void **)tmp_info, NFRAMES + 1);
          LOCK();
#       else
          npcs = backtrace((void **)tmp_info, NFRAMES + 1);
#       endif
        /* We retrieve NFRAMES+1 pc values, but discard the first one,  */
        /* since it points to our own frame.                            */
        i = 0;
        if (npcs > 1) {
          i = npcs - 1;
          BCOPY(&tmp_info[1], info, (unsigned)i * sizeof(void *));
        }
        BZERO(&info[i], sizeof(void *) * (unsigned)(NFRAMES - i));
#       ifdef REDIRECT_MALLOC
          GC_in_save_callers = FALSE;
#       endif
      }

#   elif defined(I386) || defined(SPARC)

#     if defined(ANY_BSD) && defined(SPARC)
#       define FR_SAVFP fr_fp
#       define FR_SAVPC fr_pc
#     else
#       define FR_SAVFP fr_savfp
#       define FR_SAVPC fr_savpc
#     endif

#     if defined(SPARC) && (defined(__arch64__) || defined(__sparcv9))
#       define BIAS 2047
#     else
#       define BIAS 0
#     endif

      GC_INNER void GC_save_callers(struct callinfo info[NFRAMES])
      {
        struct frame *frame;
        struct frame *fp;
        int nframes = 0;
#       ifdef I386
          /* We assume this is turned on only with gcc as the compiler. */
          asm("movl %%ebp,%0" : "=r"(frame));
          fp = frame;
#       else /* SPARC */
          frame = (struct frame *)GC_save_regs_in_stack();
          fp = (struct frame *)((long)(frame -> FR_SAVFP) + BIAS);
#       endif

        for (; !HOTTER_THAN((ptr_t)fp, (ptr_t)frame)
#               ifndef THREADS
                  && !HOTTER_THAN(GC_stackbottom, (ptr_t)fp)
#               elif defined(STACK_GROWS_UP)
                  && fp != NULL
#               endif
                && nframes < NFRAMES;
              fp = (struct frame *)((long)(fp -> FR_SAVFP) + BIAS),
              nframes++) {
#         if NARGS > 0
            int i;
#         endif

          info[nframes].ci_pc = (GC_return_addr_t)(fp -> FR_SAVPC);
#         if NARGS > 0
            for (i = 0; i < NARGS; i++) {
              info[nframes].ci_arg[i] =
                        GC_HIDE_NZ_POINTER((void *)(fp -> fr_arg[i]));
            }
#         endif
        }
        if (nframes < NFRAMES) info[nframes].ci_pc = 0;
      }

#   endif /* !GC_HAVE_BUILTIN_BACKTRACE */

# endif /* SAVE_CALL_CHAIN */

  /* Print info to stderr.  We do not hold the allocator lock.  */
  GC_INNER void GC_print_callers(struct callinfo info[NFRAMES])
  {
    int i, reent_cnt;
#   if defined(AO_HAVE_fetch_and_add1) && defined(AO_HAVE_fetch_and_sub1)
      static volatile AO_t reentry_count = 0;

      /* Note: alternatively, if available, we may use a thread-local   */
      /* storage, thus, enabling concurrent usage of GC_print_callers;  */
      /* but practically this has little sense because printing is done */
      /* into a single output stream.                                   */
      GC_ASSERT(I_DONT_HOLD_LOCK());
      reent_cnt = (int)(signed_word)AO_fetch_and_add1(&reentry_count);
#   else
      static int reentry_count = 0;

      /* Note: this could use a different lock. */
      LOCK();
      reent_cnt = reentry_count++;
      UNLOCK();
#   endif
#   if NFRAMES == 1
      GC_err_printf("\tCaller at allocation:\n");
#   else
      GC_err_printf("\tCall chain at allocation:\n");
#   endif
    for (i = 0; i < NFRAMES; i++) {
#       if defined(LINUX) && !defined(SMALL_CONFIG)
          GC_bool stop = FALSE;
#       endif

        if (0 == info[i].ci_pc)
          break;
#       if NARGS > 0
          {
            int j;

            GC_err_printf("\t\targs: ");
            for (j = 0; j < NARGS; j++) {
              void *p = GC_REVEAL_NZ_POINTER(info[i].ci_arg[j]);

              if (j != 0) GC_err_printf(", ");
              GC_err_printf("%ld (%p)", (long)(signed_word)ADDR(p), p);
            }
            GC_err_printf("\n");
          }
#       endif
        if (reent_cnt > 0) {
          /* We were called either concurrently or during an allocation */
          /* by backtrace_symbols() called from GC_print_callers; punt. */
          GC_err_printf("\t\t##PC##= 0x%lx\n",
                        (unsigned long)info[i].ci_pc);
          continue;
        }

        {
          char buf[40];
          char *name;
#         if defined(GC_HAVE_BUILTIN_BACKTRACE) \
             && !defined(GC_BACKTRACE_SYMBOLS_BROKEN) \
             && defined(FUNCPTR_IS_DATAPTR)
            char **sym_name = backtrace_symbols((void **)&info[i].ci_pc, 1);
            if (sym_name != NULL) {
              name = sym_name[0];
            } else
#         endif
          /* else */ {
            (void)snprintf(buf, sizeof(buf), "##PC##= 0x%lx",
                           (unsigned long)info[i].ci_pc);
            buf[sizeof(buf) - 1] = '\0';
            name = buf;
          }
#         if defined(LINUX) && !defined(SMALL_CONFIG)
            /* Try for a line number. */
            do {
                FILE *pipe;
#               define EXE_SZ 100
                static char exe_name[EXE_SZ];
#               define CMD_SZ 200
                char cmd_buf[CMD_SZ];
#               define RESULT_SZ 200
                static char result_buf[RESULT_SZ];
                size_t result_len;
                const char *old_preload;
#               define PRELOAD_SZ 200
                char preload_buf[PRELOAD_SZ];
                static GC_bool found_exe_name = FALSE;
                static GC_bool will_fail = FALSE;

                /* Try to get it via a hairy and expensive scheme.      */
                /* First we get the name of the executable:             */
                if (will_fail)
                  break;
                if (!found_exe_name) {
                  int ret_code = readlink("/proc/self/exe", exe_name, EXE_SZ);

                  if (ret_code < 0 || ret_code >= EXE_SZ
                      || exe_name[0] != '/') {
                    will_fail = TRUE;   /* Don't try again. */
                    break;
                  }
                  exe_name[ret_code] = '\0';
                  found_exe_name = TRUE;
                }
                /* Then we use popen to start addr2line -e <exe> <addr> */
                /* There are faster ways to do this, but hopefully this */
                /* isn't time critical.                                 */
                (void)snprintf(cmd_buf, sizeof(cmd_buf),
                               "/usr/bin/addr2line -f -e %s 0x%lx",
                               exe_name, (unsigned long)info[i].ci_pc);
                cmd_buf[sizeof(cmd_buf) - 1] = '\0';
                old_preload = GETENV("LD_PRELOAD");
                if (old_preload != NULL) {
                  size_t old_len = strlen(old_preload);
                  if (old_len >= PRELOAD_SZ) {
                    will_fail = TRUE;
                    break;
                  }
                  BCOPY(old_preload, preload_buf, old_len + 1);
                  unsetenv("LD_PRELOAD");
                }
                pipe = popen(cmd_buf, "r");
                if (old_preload != NULL
                    && 0 != setenv("LD_PRELOAD", preload_buf, 0)) {
                  WARN("Failed to reset LD_PRELOAD\n", 0);
                }
                if (NULL == pipe) {
                  will_fail = TRUE;
                  break;
                }
                result_len = fread(result_buf, 1, RESULT_SZ - 1, pipe);
                (void)pclose(pipe);
                if (0 == result_len) {
                  will_fail = TRUE;
                  break;
                }
                if (result_buf[result_len - 1] == '\n') --result_len;
                result_buf[result_len] = 0;
                if (result_buf[0] == '?'
                    || (result_buf[result_len-2] == ':'
                        && result_buf[result_len-1] == '0'))
                  break;
                /* Get rid of embedded newline, if any.  Test for "main" */
                {
                  char * nl = strchr(result_buf, '\n');
                  if (nl != NULL && ADDR_LT(nl, result_buf + result_len)) {
                    *nl = ':';
                  }
                  if (strncmp(result_buf, "main",
                              nl != NULL
                                ? (size_t)(ADDR(nl) /* a cppcheck workaround */
                                           - COVERT_DATAFLOW(ADDR(result_buf)))
                                : result_len) == 0) {
                    stop = TRUE;
                  }
                }
                if (result_len < RESULT_SZ - 25) {
                  /* Add in hex address */
                  (void)snprintf(&result_buf[result_len],
                                 sizeof(result_buf) - result_len,
                                 " [0x%lx]", (unsigned long)info[i].ci_pc);
                  result_buf[sizeof(result_buf) - 1] = '\0';
                }
#               if defined(CPPCHECK)
                  GC_noop1((unsigned char)name[0]);
                                /* name computed previously is discarded */
#               endif
                name = result_buf;
            } while (0);
#         endif /* LINUX */
          GC_err_printf("\t\t%s\n", name);
#         if defined(GC_HAVE_BUILTIN_BACKTRACE) \
             && !defined(GC_BACKTRACE_SYMBOLS_BROKEN) \
             && defined(FUNCPTR_IS_DATAPTR)
            if (sym_name != NULL)
              free(sym_name);   /* May call GC_[debug_]free; that's OK  */
#         endif
        }
#       if defined(LINUX) && !defined(SMALL_CONFIG)
          if (stop)
            break;
#       endif
    }
#   if defined(AO_HAVE_fetch_and_add1) && defined(AO_HAVE_fetch_and_sub1)
      (void)AO_fetch_and_sub1(&reentry_count);
#   else
      LOCK();
      --reentry_count;
      UNLOCK();
#   endif
  }

#endif /* NEED_CALLINFO */

#if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG)
  /* Dump /proc/self/maps to GC_stderr, to enable looking up names for  */
  /* addresses in FIND_LEAK output.                                     */
  void GC_print_address_map(void)
  {
    const char *maps_ptr;

    GC_ASSERT(I_HOLD_LOCK());
    maps_ptr = GC_get_maps();
    GC_err_printf("---------- Begin address map ----------\n");
    GC_err_puts(maps_ptr);
    GC_err_printf("---------- End address map ----------\n");
  }
#endif /* LINUX && ELF */

/*
 * Copyright (c) 2000-2005 by Hewlett-Packard Company.  All rights reserved.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#if defined(THREAD_LOCAL_ALLOC)

#if !defined(THREADS) && !defined(CPPCHECK)
# error Invalid config - THREAD_LOCAL_ALLOC requires GC_THREADS
#endif

/*
 * Copyright (c) 2000-2005 by Hewlett-Packard Company.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/* Included indirectly from a thread-library-specific file.     */
/* This is the interface for thread-local allocation, whose     */
/* implementation is mostly thread-library-independent.         */
/* Here we describe only the interface that needs to be known   */
/* and invoked from the thread support layer; the actual        */
/* implementation also exports GC_malloc and friends, which     */
/* are declared in gc.h.                                        */

#ifndef GC_THREAD_LOCAL_ALLOC_H
#define GC_THREAD_LOCAL_ALLOC_H


#ifdef THREAD_LOCAL_ALLOC

#if defined(USE_HPUX_TLS)
# error USE_HPUX_TLS macro was replaced by USE_COMPILER_TLS
#endif

#include <stdlib.h>

EXTERN_C_BEGIN

#if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC) \
    && !defined(USE_WIN32_COMPILER_TLS) && !defined(USE_COMPILER_TLS) \
    && !defined(USE_CUSTOM_SPECIFIC)
# if defined(GC_WIN32_THREADS)
#   if defined(CYGWIN32) && GC_GNUC_PREREQ(4, 0)
#     if defined(__clang__)
        /* As of Cygwin clang3.5.2, thread-local storage is unsupported.    */
#       define USE_PTHREAD_SPECIFIC
#     else
#       define USE_COMPILER_TLS
#     endif
#   elif defined(__GNUC__) || defined(MSWINCE)
#     define USE_WIN32_SPECIFIC
#   else
#     define USE_WIN32_COMPILER_TLS
#   endif /* !GNU */

# elif defined(HOST_ANDROID)
#   if defined(ARM32) && (GC_GNUC_PREREQ(4, 6) \
                          || GC_CLANG_PREREQ_FULL(3, 8, 256229))
#     define USE_COMPILER_TLS
#   elif !defined(__clang__) && !defined(ARM32)
      /* TODO: Support clang/arm64 */
#     define USE_COMPILER_TLS
#   else
#     define USE_PTHREAD_SPECIFIC
#   endif

# elif defined(LINUX) && GC_GNUC_PREREQ(3, 3) /* && !HOST_ANDROID */
#   if defined(ARM32) || defined(AVR32)
      /* TODO: support Linux/arm */
#     define USE_PTHREAD_SPECIFIC
#   elif defined(AARCH64) && defined(__clang__) && !GC_CLANG_PREREQ(8, 0)
      /* To avoid "R_AARCH64_ABS64 used with TLS symbol" linker warnings. */
#     define USE_PTHREAD_SPECIFIC
#   else
#     define USE_COMPILER_TLS
#   endif

# elif (defined(FREEBSD) \
        || (defined(NETBSD) && __NetBSD_Version__ >= 600000000 /* 6.0 */)) \
       && (GC_GNUC_PREREQ(4, 4) || GC_CLANG_PREREQ(3, 9))
#   define USE_COMPILER_TLS

# elif defined(GC_HPUX_THREADS)
#   ifdef __GNUC__
#     define USE_PTHREAD_SPECIFIC
        /* Empirically, as of gcc 3.3, USE_COMPILER_TLS doesn't work.   */
#   else
#     define USE_COMPILER_TLS
#   endif

# elif defined(GC_IRIX_THREADS) || defined(GC_OPENBSD_THREADS) \
       || defined(GC_SOLARIS_THREADS) \
       || defined(NN_PLATFORM_CTR) || defined(NN_BUILD_TARGET_PLATFORM_NX)
#   define USE_CUSTOM_SPECIFIC  /* Use our own. */

# else
#   define USE_PTHREAD_SPECIFIC
# endif
#endif /* !USE_x_SPECIFIC */

#ifndef THREAD_FREELISTS_KINDS
# ifdef ENABLE_DISCLAIM
#   define THREAD_FREELISTS_KINDS (NORMAL+2)
# else
#   define THREAD_FREELISTS_KINDS (NORMAL+1)
# endif
#endif /* !THREAD_FREELISTS_KINDS */

/* The first GC_TINY_FREELISTS free lists correspond to the first   */
/* GC_TINY_FREELISTS multiples of GC_GRANULE_BYTES, i.e. we keep    */
/* separate free lists for each multiple of GC_GRANULE_BYTES up to  */
/* (GC_TINY_FREELISTS-1) * GC_GRANULE_BYTES.  After that they may   */
/* be spread out further.                                           */

/* One of these should be declared as the tlfs field in the     */
/* structure pointed to by a GC_thread.                         */
typedef struct thread_local_freelists {
  void * _freelists[THREAD_FREELISTS_KINDS][GC_TINY_FREELISTS];
# define ptrfree_freelists _freelists[PTRFREE]
# define normal_freelists _freelists[NORMAL]
        /* Note: Preserve *_freelists names for some clients.   */
# ifdef GC_GCJ_SUPPORT
    void * gcj_freelists[GC_TINY_FREELISTS];
#   define ERROR_FL GC_WORD_MAX
        /* Value used for gcj_freelists[-1]; allocation is      */
        /* erroneous.                                           */
# endif
  /* Free lists contain either a pointer or a small count       */
  /* reflecting the number of granules allocated at that        */
  /* size.                                                      */
  /* 0 ==> thread-local allocation in use, free list            */
  /*       empty.                                               */
  /* > 0, <= DIRECT_GRANULES ==> Using global allocation,       */
  /*       too few objects of this size have been               */
  /*       allocated by this thread.                            */
  /* >= HBLKSIZE  => pointer to nonempty free list.             */
  /* > DIRECT_GRANULES, < HBLKSIZE ==> transition to            */
  /*    local alloc, equivalent to 0.                           */
# define DIRECT_GRANULES (HBLKSIZE/GC_GRANULE_BYTES)
        /* Don't use local free lists for up to this much       */
        /* allocation.                                          */
} *GC_tlfs;

#if defined(USE_PTHREAD_SPECIFIC)
# define GC_getspecific pthread_getspecific
# define GC_setspecific pthread_setspecific
# define GC_key_create pthread_key_create
# define GC_remove_specific(key) (void)pthread_setspecific(key, NULL)
                        /* Explicitly delete the value to stop the TLS  */
                        /* destructor from being called repeatedly.     */
# define GC_remove_specific_after_fork(key, t) (void)0
                                        /* Should not need any action.  */
  typedef pthread_key_t GC_key_t;
#elif defined(USE_COMPILER_TLS) || defined(USE_WIN32_COMPILER_TLS)
# define GC_getspecific(x) (x)
# define GC_setspecific(key, v) ((key) = (v), 0)
# define GC_key_create(key, d) 0
# define GC_remove_specific(key) (void)GC_setspecific(key, NULL)
                        /* Just to clear the pointer to tlfs. */
# define GC_remove_specific_after_fork(key, t) (void)0
  typedef void * GC_key_t;
#elif defined(USE_WIN32_SPECIFIC)
# define GC_getspecific TlsGetValue
# define GC_setspecific(key, v) !TlsSetValue(key, v)
        /* We assume 0 == success, msft does the opposite.      */
# ifndef TLS_OUT_OF_INDEXES
        /* this is currently missing in WinCE   */
#   define TLS_OUT_OF_INDEXES (DWORD)0xFFFFFFFF
# endif
# define GC_key_create(key, d) \
        ((d) != 0 || (*(key) = TlsAlloc()) == TLS_OUT_OF_INDEXES ? -1 : 0)
# define GC_remove_specific(key) (void)GC_setspecific(key, NULL)
        /* Need TlsFree on process exit/detach?   */
# define GC_remove_specific_after_fork(key, t) (void)0
  typedef DWORD GC_key_t;
#elif defined(USE_CUSTOM_SPECIFIC)
  EXTERN_C_END
/*
 * Copyright (c) 2000 by Hewlett-Packard Company.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/*
 * This is a reimplementation of a subset of the pthread_get/setspecific
 * interface. This appears to outperform the standard linuxthreads one
 * by a significant margin.
 * The major restriction is that each thread may only make a single
 * pthread_setspecific call on a single key.  (The current data structure
 * doesn't really require that.  The restriction should be easily removable.)
 * We don't currently support the destruction functions, though that
 * could be done.
 * We also currently assume that only one pthread_setspecific call
 * can be executed at a time, though that assumption would be easy to remove
 * by adding a lock.
 */

#ifndef GC_SPECIFIC_H
#define GC_SPECIFIC_H

#if !defined(GC_THREAD_LOCAL_ALLOC_H)
# error specific.h should be included from thread_local_alloc.h
#endif

#include <errno.h>

EXTERN_C_BEGIN

/* Called during key creation or setspecific.           */
/* For the GC we already hold the allocator lock.       */
/* Currently allocated objects leak on thread exit.     */
/* That's hard to fix, but OK if we allocate garbage    */
/* collected memory.                                    */
#define MALLOC_CLEAR(n) GC_INTERNAL_MALLOC(n, NORMAL)

#define TS_CACHE_SIZE 1024
#define TS_CACHE_HASH(n) ((((n) >> 8) ^ (n)) & (TS_CACHE_SIZE-1))

#define TS_HASH_SIZE 1024
#define TS_HASH(p) ((unsigned)((ADDR(p) >> 8) ^ ADDR(p)) & (TS_HASH_SIZE-1))

#ifdef GC_ASSERTIONS
  /* Thread-local storage is not guaranteed to be scanned by GC.        */
  /* We hide values stored in "specific" entries for a test purpose.    */
  typedef GC_hidden_pointer ts_entry_value_t;
# define TS_HIDE_VALUE(p) GC_HIDE_NZ_POINTER(p)
# define TS_REVEAL_PTR(p) GC_REVEAL_NZ_POINTER(p)
#else
  typedef void * ts_entry_value_t;
# define TS_HIDE_VALUE(p) (p)
# define TS_REVEAL_PTR(p) (p)
#endif

/* An entry describing a thread-specific value for a given thread.      */
/* All such accessible structures preserve the invariant that if either */
/* thread is a valid pthread id or qtid is a valid "quick thread id"    */
/* for a thread, then value holds the corresponding thread specific     */
/* value.  This invariant must be preserved at ALL times, since         */
/* asynchronous reads are allowed.                                      */
typedef struct thread_specific_entry {
  volatile AO_t qtid;   /* quick thread id, only for cache */
  ts_entry_value_t value;
  struct thread_specific_entry *next;
  pthread_t thread;
} tse;

/* We represent each thread-specific datum as two tables.  The first is */
/* a cache, indexed by a "quick thread identifier".  The "quick" thread */
/* identifier is an easy to compute value, which is guaranteed to       */
/* determine the thread, though a thread may correspond to more than    */
/* one value.  We typically use the address of a page in the stack.     */
/* The second is a hash table, indexed by pthread_self().  It is used   */
/* only as a backup.                                                    */

/* Return the "quick thread id".  Default version.  Assumes page size,  */
/* or at least thread stack separation, is at least 4 KB.               */
/* Must be defined so that it never returns 0.  (Page 0 can't really be */
/* part of any stack, since that would make 0 a valid stack pointer.)   */
#define ts_quick_thread_id() ((size_t)(ADDR(GC_approx_sp()) >> 12))

#define INVALID_QTID ((size_t)0)
#define INVALID_THREADID ((pthread_t)0)

typedef struct thread_specific_data {
  tse *volatile cache[TS_CACHE_SIZE]; /* a faster index to the hash table */
  tse *hash[TS_HASH_SIZE];
  pthread_mutex_t lock;
} tsd;

typedef tsd * GC_key_t;

#define GC_key_create(key, d) GC_key_create_inner(key)
GC_INNER int GC_key_create_inner(tsd ** key_ptr);
GC_INNER int GC_setspecific(tsd * key, void * value);
#define GC_remove_specific(key) \
                        GC_remove_specific_after_fork(key, pthread_self())
GC_INNER void GC_remove_specific_after_fork(tsd * key, pthread_t t);

/* An internal version of getspecific that assumes a cache miss.        */
GC_INNER void * GC_slow_getspecific(tsd * key, size_t qtid,
                                    tse * volatile * cache_entry);

GC_INLINE void * GC_getspecific(tsd * key)
{
  size_t qtid = ts_quick_thread_id();
  tse * volatile * entry_ptr = &(key -> cache[TS_CACHE_HASH(qtid)]);
  const tse * entry = *entry_ptr; /* must be loaded only once */

  GC_ASSERT(qtid != INVALID_QTID);
  if (EXPECT(entry -> qtid == qtid, TRUE)) {
    GC_ASSERT(entry -> thread == pthread_self());
    return TS_REVEAL_PTR(entry -> value);
  }

  return GC_slow_getspecific(key, qtid, entry_ptr);
}

EXTERN_C_END

#endif /* GC_SPECIFIC_H */

  EXTERN_C_BEGIN
#else
# error implement me
#endif

/* Each thread structure must be initialized.   */
/* This call must be made from the new thread.  */
/* Caller should hold the allocator lock.       */
GC_INNER void GC_init_thread_local(GC_tlfs p);

/* Called when a thread is unregistered, or exits.      */
/* Caller should hold the allocator lock.               */
GC_INNER void GC_destroy_thread_local(GC_tlfs p);

/* The thread support layer must arrange to mark thread-local   */
/* free lists explicitly, since the link field is often         */
/* invisible to the marker.  It knows how to find all threads;  */
/* we take care of an individual thread free-list structure.    */
GC_INNER void GC_mark_thread_local_fls_for(GC_tlfs p);

#ifdef GC_ASSERTIONS
  GC_bool GC_is_thread_tsd_valid(void *tsd);
  void GC_check_tls_for(GC_tlfs p);
# if defined(USE_CUSTOM_SPECIFIC)
    void GC_check_tsd_marks(tsd *key);
# endif
#endif /* GC_ASSERTIONS */

#ifndef GC_ATTR_TLS_FAST
# define GC_ATTR_TLS_FAST /* empty */
#endif

extern
#if defined(USE_COMPILER_TLS)
  __thread GC_ATTR_TLS_FAST
#elif defined(USE_WIN32_COMPILER_TLS)
  __declspec(thread) GC_ATTR_TLS_FAST
#endif
  GC_key_t GC_thread_key;
/* This is set up by the thread_local_alloc implementation.  No need    */
/* for cleanup on thread exit.  But the thread support layer makes sure */
/* that GC_thread_key is traced, if necessary.                          */

EXTERN_C_END

#endif /* THREAD_LOCAL_ALLOC */

#endif /* GC_THREAD_LOCAL_ALLOC_H */


#if defined(USE_COMPILER_TLS)
  __thread GC_ATTR_TLS_FAST
#elif defined(USE_WIN32_COMPILER_TLS)
  __declspec(thread) GC_ATTR_TLS_FAST
#endif
GC_key_t GC_thread_key;

static GC_bool keys_initialized;

/* Return a single nonempty free list fl to the global one pointed to   */
/* by gfl.                                                              */
static void return_single_freelist(void *fl, void **gfl)
{
    if (NULL == *gfl) {
        *gfl = fl;
    } else {
        void *q = fl;
        void **qptr;

        GC_ASSERT(GC_size(fl) == GC_size(*gfl));
        /* Concatenate: */
        do {
            qptr = &obj_link(q);
            q = *qptr;
        } while (ADDR(q) >= HBLKSIZE);
        GC_ASSERT(NULL == q);
        *qptr = *gfl;
        *gfl = fl;
    }
}

/* Recover the contents of the free-list array fl into the global one gfl. */
static void return_freelists(void **fl, void **gfl)
{
    int i;

    for (i = 1; i < GC_TINY_FREELISTS; ++i) {
        if (ADDR(fl[i]) >= HBLKSIZE) {
          return_single_freelist(fl[i], &gfl[i]);
        }
        /* Clear fl[i], since the thread structure may hang around.     */
        /* Do it in a way that is likely to trap if we access it.       */
        fl[i] = (ptr_t)(GC_uintptr_t)HBLKSIZE;
    }
    /* The 0 granule free list really contains 1 granule objects.       */
    if (ADDR(fl[0]) >= HBLKSIZE
#       ifdef GC_GCJ_SUPPORT
          && ADDR(fl[0]) != ERROR_FL
#       endif
       ) {
        return_single_freelist(fl[0], &gfl[1]);
    }
}

#ifdef USE_PTHREAD_SPECIFIC
  /* Re-set the TLS value on thread cleanup to allow thread-local       */
  /* allocations to happen in the TLS destructors.                      */
  /* GC_unregister_my_thread (and similar routines) will finally set    */
  /* the GC_thread_key to NULL preventing this destructor from being    */
  /* called repeatedly.                                                 */
  static void reset_thread_key(void* v) {
    pthread_setspecific(GC_thread_key, v);
  }
#else
# define reset_thread_key 0
#endif

GC_INNER void GC_init_thread_local(GC_tlfs p)
{
    int k, j, res;

    GC_ASSERT(I_HOLD_LOCK());
    if (!EXPECT(keys_initialized, TRUE)) {
#       ifdef USE_CUSTOM_SPECIFIC
          /* Ensure proper alignment of a "pushed" GC symbol.   */
          GC_ASSERT(ADDR(&GC_thread_key) % sizeof(ptr_t) == 0);
#       endif
        res = GC_key_create(&GC_thread_key, reset_thread_key);
        if (COVERT_DATAFLOW(res) != 0) {
            ABORT("Failed to create key for local allocator");
        }
        keys_initialized = TRUE;
    }
    res = GC_setspecific(GC_thread_key, p);
    if (COVERT_DATAFLOW(res) != 0) {
        ABORT("Failed to set thread specific allocation pointers");
    }
    for (j = 0; j < GC_TINY_FREELISTS; ++j) {
        for (k = 0; k < THREAD_FREELISTS_KINDS; ++k) {
            p -> _freelists[k][j] = (void *)(GC_uintptr_t)1;
        }
#       ifdef GC_GCJ_SUPPORT
            p -> gcj_freelists[j] = (void *)(GC_uintptr_t)1;
#       endif
    }
    /* The zero-sized free list is handled like the regular free list,  */
    /* to ensure that the explicit deallocation works.  However, an     */
    /* allocation of a size 0 "gcj" object is always an error.          */
#   ifdef GC_GCJ_SUPPORT
        p -> gcj_freelists[0] = MAKE_CPTR(ERROR_FL);
#   endif
}

GC_INNER void GC_destroy_thread_local(GC_tlfs p)
{
    int k;

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(GC_getspecific(GC_thread_key) == p);
    /* We currently only do this from the thread itself.        */
    GC_STATIC_ASSERT(THREAD_FREELISTS_KINDS <= MAXOBJKINDS);
    for (k = 0; k < THREAD_FREELISTS_KINDS; ++k) {
        if (k == (int)GC_n_kinds)
            break; /* kind is not created */
        return_freelists(p -> _freelists[k], GC_obj_kinds[k].ok_freelist);
    }
#   ifdef GC_GCJ_SUPPORT
        return_freelists(p -> gcj_freelists, (void **)GC_gcjobjfreelist);
#   endif
}

STATIC void *GC_get_tlfs(void)
{
# if !defined(USE_PTHREAD_SPECIFIC) && !defined(USE_WIN32_SPECIFIC)
    GC_key_t k = GC_thread_key;

    if (EXPECT(0 == k, FALSE)) {
      /* We have not yet run GC_init_parallel.  That means we also  */
      /* are not locking, so GC_malloc_kind_global is fairly cheap. */
      return NULL;
    }
    return GC_getspecific(k);
# else
    if (EXPECT(!keys_initialized, FALSE)) return NULL;

    return GC_getspecific(GC_thread_key);
# endif
}

GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_kind(size_t lb, int k)
{
    size_t lg;
    void *tsd;
    void *result;

#   if MAXOBJKINDS > THREAD_FREELISTS_KINDS
      if (EXPECT(k >= THREAD_FREELISTS_KINDS, FALSE)) {
        return GC_malloc_kind_global(lb, k);
      }
#   endif
    tsd = GC_get_tlfs();
    if (EXPECT(NULL == tsd, FALSE)) {
        return GC_malloc_kind_global(lb, k);
    }
    GC_ASSERT(GC_is_initialized);
    GC_ASSERT(GC_is_thread_tsd_valid(tsd));
    lg = ALLOC_REQUEST_GRANS(lb);
#   if defined(CPPCHECK)
#     define MALLOC_KIND_PTRFREE_INIT (void*)1
#   else
#     define MALLOC_KIND_PTRFREE_INIT NULL
#   endif
    GC_FAST_MALLOC_GRANS(result, lg,
                         ((GC_tlfs)tsd) -> _freelists[k], DIRECT_GRANULES,
                         k, GC_malloc_kind_global(lb, k),
                         (void)(k == PTRFREE ? MALLOC_KIND_PTRFREE_INIT
                                               : (obj_link(result) = 0)));
#   ifdef LOG_ALLOCS
      GC_log_printf("GC_malloc_kind(%lu, %d) returned %p, recent GC #%lu\n",
                    (unsigned long)lb, k, result, (unsigned long)GC_gc_no);
#   endif
    return result;
}

#ifdef GC_GCJ_SUPPORT




/* Gcj-style allocation without locks is extremely tricky.  The         */
/* fundamental issue is that we may end up marking a free list, which   */
/* has free-list links instead of "vtable" pointers.  That is usually   */
/* OK, since the next object on the free list will be cleared, and      */
/* will thus be interpreted as containing a zero descriptor.  That's    */
/* fine if the object has not yet been initialized.  But there are      */
/* interesting potential races.                                         */
/* In the case of incremental collection, this seems hopeless, since    */
/* the marker may run asynchronously, and may pick up the pointer to    */
/* the next free-list entry (which it thinks is a vtable pointer), get  */
/* suspended for a while, and then see an allocated object instead      */
/* of the vtable.  This may be avoidable with either a handshake with   */
/* the collector or, probably more easily, by moving the free list      */
/* links to the second word of each object.  The latter isn't a         */
/* universal win, since on architecture like Itanium, nonzero offsets   */
/* are not necessarily free.  And there may be cache fill order issues. */
/* For now, we punt with incremental GC.  This probably means that      */
/* incremental GC should be enabled before we fork a second thread.     */
/* Unlike the other thread-local allocation calls, we assume that the   */
/* collector has been explicitly initialized.                           */
GC_API GC_ATTR_MALLOC void * GC_CALL GC_gcj_malloc(size_t lb,
                                                   const void *vtable_ptr)
{
  if (EXPECT(GC_incremental, FALSE)) {
    return GC_core_gcj_malloc(lb, vtable_ptr, 0 /* flags */);
  } else {
    size_t lg = ALLOC_REQUEST_GRANS(lb);
    void *result;
    void **tiny_fl;

    GC_ASSERT(GC_gcjobjfreelist != NULL);
    tiny_fl = ((GC_tlfs)GC_getspecific(GC_thread_key))->gcj_freelists;
    GC_FAST_MALLOC_GRANS(result, lg, tiny_fl, DIRECT_GRANULES, GC_gcj_kind,
                         GC_core_gcj_malloc(lb, vtable_ptr, 0 /* flags */),
                         do { AO_compiler_barrier();
                           *(const void **)result = vtable_ptr; } while(0));
        /* This forces the initialization of the "method ptr".          */
        /* This is necessary to ensure some very subtle properties      */
        /* required if a GC is run in the middle of such an allocation. */
        /* Here we implicitly also assume atomicity for the free list.  */
        /* and method pointer assignments.                              */
        /* We must update the free list before we store the pointer.    */
        /* Otherwise a GC at this point would see a corrupted           */
        /* free list.                                                   */
        /* A real memory barrier is not needed, since the               */
        /* action of stopping this thread will cause prior writes       */
        /* to complete.                                                 */
        /* We assert that any concurrent marker will stop us.           */
        /* Thus it is impossible for a mark procedure to see the        */
        /* allocation of the next object, but to see this object        */
        /* still containing a free-list pointer.  Otherwise the         */
        /* marker, by misinterpreting the free-list link as a vtable    */
        /* pointer, might find a random "mark descriptor" in the next   */
        /* object.                                                      */
    return result;
  }
}

#endif /* GC_GCJ_SUPPORT */

/* The thread support layer must arrange to mark thread-local   */
/* free lists explicitly, since the link field is often         */
/* invisible to the marker.  It knows how to find all threads;  */
/* we take care of an individual thread free-list structure.    */
GC_INNER void GC_mark_thread_local_fls_for(GC_tlfs p)
{
    ptr_t q;
    int k, j;

    for (j = 0; j < GC_TINY_FREELISTS; ++j) {
      for (k = 0; k < THREAD_FREELISTS_KINDS; ++k) {
        /* Load the pointer atomically as it might be updated   */
        /* concurrently by GC_FAST_MALLOC_GRANS.                */
        q = GC_cptr_load((volatile ptr_t *)&(p -> _freelists[k][j]));
        if (ADDR(q) > HBLKSIZE)
          GC_set_fl_marks(q);
      }
#     ifdef GC_GCJ_SUPPORT
        if (EXPECT(j > 0, TRUE)) {
          q = GC_cptr_load((volatile ptr_t *)&(p -> gcj_freelists[j]));
          if (ADDR(q) > HBLKSIZE)
            GC_set_fl_marks(q);
        }
#     endif
    }
}

#if defined(GC_ASSERTIONS)
    /* Check that all thread-local free-lists in p are completely marked. */
    void GC_check_tls_for(GC_tlfs p)
    {
        int k, j;

        for (j = 1; j < GC_TINY_FREELISTS; ++j) {
          for (k = 0; k < THREAD_FREELISTS_KINDS; ++k) {
            GC_check_fl_marks(&p->_freelists[k][j]);
          }
#         ifdef GC_GCJ_SUPPORT
            GC_check_fl_marks(&p->gcj_freelists[j]);
#         endif
        }
    }
#endif /* GC_ASSERTIONS */

#endif /* THREAD_LOCAL_ALLOC */


/* Most platform-specific files go here... */
/*
 * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
 * Copyright (c) 2000-2010 by Hewlett-Packard Development Company.
 * All rights reserved.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/*
 * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
 * Copyright (c) 2000-2009 by Hewlett-Packard Development Company.
 * All rights reserved.
 * Copyright (c) 2009-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/* Private declarations for threads support.    */

#ifndef GC_PTHREAD_SUPPORT_H
#define GC_PTHREAD_SUPPORT_H


#ifdef THREADS

#if defined(GC_PTHREADS) || defined(GC_PTHREADS_PARAMARK)
# include <pthread.h>
#endif

#ifdef GC_DARWIN_THREADS
# include <mach/mach.h>
# include <mach/thread_act.h>
#endif

#ifdef THREAD_LOCAL_ALLOC
#endif

#ifdef THREAD_SANITIZER
#endif

EXTERN_C_BEGIN

typedef struct GC_StackContext_Rep {
# if defined(THREAD_SANITIZER) && defined(SIGNAL_BASED_STOP_WORLD)
    char dummy[sizeof(oh)];     /* A dummy field to avoid TSan false    */
                                /* positive about the race between      */
                                /* GC_has_other_debug_info and          */
                                /* GC_suspend_handler_inner (which      */
                                /* sets stack_ptr).                     */
# endif

# if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
    volatile
# endif
  ptr_t stack_end;              /* Cold end of the stack (except for    */
                                /* main thread on non-Windows).         */
                                /* On Windows: 0 means entry invalid;   */
                                /* not in_use implies stack_end is 0.   */

  ptr_t stack_ptr;      /* Valid only in some platform-specific states. */

# ifdef GC_WIN32_THREADS
#   define ADDR_LIMIT ((ptr_t)GC_WORD_MAX)
    ptr_t last_stack_min;       /* Last known minimum (hottest) address */
                                /* in stack or ADDR_LIMIT if unset.     */
#   ifdef I386
      ptr_t initial_stack_base; /* The cold end of the stack saved by   */
                                /* GC_record_stack_base (never modified */
                                /* by GC_set_stackbottom); used for the */
                                /* old way of the coroutines support.   */
#   endif
# elif defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
    ptr_t topOfStack;           /* Result of GC_FindTopOfStack(0);      */
                                /* valid only if the thread is blocked; */
                                /* non-NULL value means already set.    */
# endif

# if defined(E2K) || defined(IA64)
    ptr_t backing_store_end;
    ptr_t backing_store_ptr;
# endif

# ifdef GC_WIN32_THREADS
    /* For now, alt-stack is not implemented for Win32. */
# else
    ptr_t altstack;             /* The start of the alt-stack if there  */
                                /* is one, NULL otherwise.              */
    ptr_t normstack;            /* Same for the "normal" stack (set by  */
                                /* GC_register_altstack).               */
    size_t altstack_size;       /* The size of the alt-stack if exists. */
    size_t normstack_size;
# endif

# ifdef E2K
    size_t ps_ofs; /* the current offset in the procedure stack */
# endif

# ifndef GC_NO_FINALIZATION
    unsigned char finalizer_nested;
    char fnlz_pad[1];           /* Explicit alignment (for some rare    */
                                /* compilers such as bcc32 and wcc32).  */
    unsigned short finalizer_skipped;
                                /* Used by GC_check_finalizer_nested()  */
                                /* to minimize the level of recursion   */
                                /* when a client finalizer allocates    */
                                /* memory (initially both are 0).       */
# endif

  struct GC_traced_stack_sect_s *traced_stack_sect;
                                /* Points to the "frame" data held in   */
                                /* stack by the innermost               */
                                /* GC_call_with_gc_active() of this     */
                                /* stack (thread); may be NULL.         */

} *GC_stack_context_t;

#ifdef GC_WIN32_THREADS
  typedef DWORD thread_id_t;
# define thread_id_self() GetCurrentThreadId()
# define THREAD_ID_EQUAL(id1, id2) ((id1) == (id2))
#else
  typedef pthread_t thread_id_t;
# define thread_id_self() pthread_self()
# define THREAD_ID_EQUAL(id1, id2) THREAD_EQUAL(id1, id2)
#endif

typedef struct GC_Thread_Rep {
  union {
#   if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
      volatile AO_t in_use;     /* Updated without a lock.  We assert   */
                                /* that each unused entry has invalid   */
                                /* id of zero and zero stack_end.       */
                                /* Used only with GC_win32_dll_threads. */
      LONG long_in_use;         /* The same but of the type that        */
                                /* matches the first argument of        */
                                /* InterlockedExchange(); volatile is   */
                                /* omitted because the ancient version  */
                                /* of the prototype lacked the          */
                                /* qualifier.                           */
#   endif
    struct GC_Thread_Rep *next; /* Hash table link without              */
                                /* GC_win32_dll_threads.                */
                                /* More recently allocated threads      */
                                /* with a given pthread id come         */
                                /* first.  (All but the first are       */
                                /* guaranteed to be dead, but we may    */
                                /* not yet have registered the join.)   */
  } tm; /* table_management */

  GC_stack_context_t crtn;

  thread_id_t id; /* hash table key */
# ifdef GC_DARWIN_THREADS
    mach_port_t mach_thread;
# elif defined(GC_WIN32_THREADS) && defined(GC_PTHREADS)
    pthread_t pthread_id;
# elif defined(USE_TKILL_ON_ANDROID)
    pid_t kernel_id;
# endif

# ifdef MSWINCE
    /* According to MSDN specs for WinCE targets:                       */
    /* - DuplicateHandle() is not applicable to thread handles; and     */
    /* - the value returned by GetCurrentThreadId() could be used as    */
    /* a "real" thread handle (for SuspendThread(), ResumeThread()      */
    /* and GetThreadContext()).                                         */
#   define THREAD_HANDLE(p) ((HANDLE)(word)(p) -> id)
# elif defined(GC_WIN32_THREADS)
    HANDLE handle;
#   define THREAD_HANDLE(p) ((p) -> handle)
# endif /* GC_WIN32_THREADS && !MSWINCE */

  unsigned char flags;          /* Protected by the allocator lock.     */
# define FINISHED       0x1     /* Thread has exited (pthreads only).   */
# ifndef GC_PTHREADS
#   define KNOWN_FINISHED(p) FALSE
# else
#   define KNOWN_FINISHED(p) (((p) -> flags & FINISHED) != 0)
#   define DETACHED     0x2     /* Thread is treated as detached.       */
                                /* Thread may really be detached, or    */
                                /* it may have been explicitly          */
                                /* registered, in which case we can     */
                                /* deallocate its GC_Thread_Rep once    */
                                /* it unregisters itself, since it      */
                                /* may not return a GC pointer.         */
# endif
# if (defined(GC_HAVE_PTHREAD_EXIT) || !defined(GC_NO_PTHREAD_CANCEL)) \
     && defined(GC_PTHREADS)
#   define DISABLED_GC  0x10    /* Collections are disabled while the   */
                                /* thread is exiting.                   */
# endif
# define DO_BLOCKING    0x20    /* Thread is in the do-blocking state.  */
                                /* If set, the thread will acquire the  */
                                /* allocator lock before any pointer    */
                                /* manipulation, and has set its SP     */
                                /* value.  Thus, it does not need       */
                                /* a signal sent to stop it.            */
# ifdef GC_WIN32_THREADS
#   define IS_SUSPENDED 0x40    /* Thread is suspended by SuspendThread. */
# endif

  char flags_pad[sizeof(word) - 1 /* sizeof(flags) */];
                                /* Explicit alignment (for some rare    */
                                /* compilers such as bcc32 and wcc32).  */

# ifdef SIGNAL_BASED_STOP_WORLD
    volatile AO_t last_stop_count;
                                /* The value of GC_stop_count when the  */
                                /* thread last successfully handled     */
                                /* a suspend signal.                    */
#   ifdef GC_ENABLE_SUSPEND_THREAD
      volatile AO_t ext_suspend_cnt;
                                /* An odd value means thread was        */
                                /* suspended externally; incremented on */
                                /* every call of GC_suspend_thread()    */
                                /* and GC_resume_thread(); updated with */
                                /* the allocator lock held, but could   */
                                /* be read from a signal handler.       */
#   endif
# endif

# ifdef GC_PTHREADS
    void *status;               /* The value returned from the thread.  */
                                /* Used only to avoid premature         */
                                /* reclamation of any data it might     */
                                /* reference.                           */
                                /* This is unfortunately also the       */
                                /* reason we need to intercept join     */
                                /* and detach.                          */
# endif

# ifdef THREAD_LOCAL_ALLOC
    struct thread_local_freelists tlfs GC_ATTR_PTRT_ALIGNED;
# endif

# ifdef NACL
    /* Grab NACL_GC_REG_STORAGE_SIZE pointers off the stack when        */
    /* going into a syscall.  20 is more than we need, but it's an      */
    /* overestimate in case the instrumented function uses any callee   */
    /* saved registers, they may be pushed to the stack much earlier.   */
    /* Also, on x86_64 'push' puts 8 bytes on the stack even though     */
    /* our pointers are 4 bytes.                                        */
#   ifdef ARM32
      /* Space for r4-r8, r10-r12, r14.       */
#     define NACL_GC_REG_STORAGE_SIZE 9
#   else
#     define NACL_GC_REG_STORAGE_SIZE 20
#   endif
    ptr_t reg_storage[NACL_GC_REG_STORAGE_SIZE];
# elif defined(PLATFORM_HAVE_GC_REG_STORAGE_SIZE)
    word registers[PLATFORM_GC_REG_STORAGE_SIZE]; /* used externally */
# endif

# if defined(WOW64_THREAD_CONTEXT_WORKAROUND) && defined(MSWINRT_FLAVOR)
    PNT_TIB tib;
# endif

# ifdef RETRY_GET_THREAD_CONTEXT /* && GC_WIN32_THREADS */
    ptr_t context_sp;
    word context_regs[PUSHED_REGS_COUNT];
                                /* Populated as part of GC_suspend() as */
                                /* resume/suspend loop may be needed    */
                                /* for GetThreadContext() to succeed.   */
# endif
} * GC_thread;

#ifndef THREAD_TABLE_SZ
# define THREAD_TABLE_SZ 256    /* Power of 2 (for speed). */
#endif

#ifdef GC_WIN32_THREADS
# define THREAD_TABLE_INDEX(id) /* id is of DWORD type */ \
                (int)((((id) >> 8) ^ (id)) % THREAD_TABLE_SZ)
#elif CPP_WORDSZ > 32
# define THREAD_TABLE_INDEX(id) \
    (int)(((((NUMERIC_THREAD_ID(id) >> 8) ^ NUMERIC_THREAD_ID(id)) >> 16) \
          ^ ((NUMERIC_THREAD_ID(id) >> 8) ^ NUMERIC_THREAD_ID(id))) \
         % THREAD_TABLE_SZ)
#else
# define THREAD_TABLE_INDEX(id) \
                (int)(((NUMERIC_THREAD_ID(id) >> 16) \
                       ^ (NUMERIC_THREAD_ID(id) >> 8) \
                       ^ NUMERIC_THREAD_ID(id)) % THREAD_TABLE_SZ)
#endif

/* The set of all known threads.  We intercept thread creation and      */
/* join/detach.  Protected by the allocator lock.                       */
GC_EXTERN GC_thread GC_threads[THREAD_TABLE_SZ];

#ifndef MAX_MARKERS
# define MAX_MARKERS 16
#endif

#ifdef GC_ASSERTIONS
  GC_EXTERN GC_bool GC_thr_initialized;
#endif

#ifdef STACKPTR_CORRECTOR_AVAILABLE
  GC_EXTERN GC_sp_corrector_proc GC_sp_corrector;
#endif

GC_EXTERN GC_on_thread_event_proc GC_on_thread_event;

#ifdef GC_WIN32_THREADS

# ifdef GC_NO_THREADS_DISCOVERY
#   define GC_win32_dll_threads FALSE
# elif defined(GC_DISCOVER_TASK_THREADS)
#   define GC_win32_dll_threads TRUE
# else
    GC_EXTERN GC_bool GC_win32_dll_threads;
# endif

# ifdef PARALLEL_MARK
    GC_EXTERN int GC_available_markers_m1;
    GC_EXTERN unsigned GC_required_markers_cnt;
    GC_EXTERN ptr_t GC_marker_sp[MAX_MARKERS - 1];
    GC_EXTERN ptr_t GC_marker_last_stack_min[MAX_MARKERS - 1];
#   ifndef GC_PTHREADS_PARAMARK
      GC_EXTERN thread_id_t GC_marker_Id[MAX_MARKERS - 1];
#   endif
#   if !defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) && !defined(MSWINCE)
      GC_INNER void GC_init_win32_thread_naming(HMODULE hK32);
#   endif
#   ifdef GC_PTHREADS_PARAMARK
      GC_INNER void *GC_mark_thread(void *);
#   elif defined(MSWINCE)
      GC_INNER DWORD WINAPI GC_mark_thread(LPVOID);
#   else
      GC_INNER unsigned __stdcall GC_mark_thread(void *);
#   endif
# endif /* PARALLEL_MARK */

  GC_INNER GC_thread GC_new_thread(thread_id_t);
  GC_INNER void GC_record_stack_base(GC_stack_context_t crtn,
                                     const struct GC_stack_base *sb);
  GC_INNER GC_thread GC_register_my_thread_inner(
                                        const struct GC_stack_base *sb,
                                        thread_id_t self_id);

# ifdef GC_PTHREADS
    GC_INNER void GC_win32_cache_self_pthread(thread_id_t);
# else
    GC_INNER void GC_delete_thread(GC_thread);
# endif

# ifdef CAN_HANDLE_FORK
    GC_INNER void GC_setup_atfork(void);
# endif

# if !defined(DONT_USE_ATEXIT) || !defined(GC_NO_THREADS_DISCOVERY)
    GC_EXTERN thread_id_t GC_main_thread_id;
# endif

# ifndef GC_NO_THREADS_DISCOVERY
    GC_INNER GC_thread GC_win32_dll_lookup_thread(thread_id_t);
# endif

# ifdef MPROTECT_VDB
    /* Make sure given thread descriptor is not protected by the VDB    */
    /* implementation.  Used to prevent write faults when the world     */
    /* is (partially) stopped, since it may have been stopped with      */
    /* a system lock held, and that lock may be required for fault      */
    /* handling.                                                        */
    GC_INNER void GC_win32_unprotect_thread(GC_thread);
# else
#   define GC_win32_unprotect_thread(t) (void)(t)
# endif /* !MPROTECT_VDB */

#else
# define GC_win32_dll_threads FALSE
#endif /* !GC_WIN32_THREADS */

#ifdef GC_PTHREADS
# if defined(GC_WIN32_THREADS) && !defined(CYGWIN32) \
     && (defined(GC_WIN32_PTHREADS) || defined(GC_PTHREADS_PARAMARK)) \
     && !defined(__WINPTHREADS_VERSION_MAJOR)
#   define GC_PTHREAD_PTRVAL(pthread_id) pthread_id.p
# else
#   define GC_PTHREAD_PTRVAL(pthread_id) pthread_id
# endif /* !GC_WIN32_THREADS || CYGWIN32 */
# ifdef GC_WIN32_THREADS
    GC_INNER GC_thread GC_lookup_by_pthread(pthread_t);
# else
#   define GC_lookup_by_pthread(t) GC_lookup_thread(t)
# endif
#endif /* GC_PTHREADS */

GC_INNER GC_thread GC_lookup_thread(thread_id_t);
#define GC_self_thread_inner() GC_lookup_thread(thread_id_self())

GC_INNER void GC_wait_for_gc_completion(GC_bool);

#ifdef NACL
  GC_INNER void GC_nacl_initialize_gc_thread(GC_thread);
  GC_INNER void GC_nacl_shutdown_gc_thread(void);
#endif

#if defined(PTHREAD_STOP_WORLD_IMPL) && !defined(NO_SIGNALS_UNBLOCK_IN_MAIN) \
    || defined(GC_EXPLICIT_SIGNALS_UNBLOCK)
  GC_INNER void GC_unblock_gc_signals(void);
#endif

#if defined(GC_ENABLE_SUSPEND_THREAD) && defined(SIGNAL_BASED_STOP_WORLD)
  GC_INNER void GC_suspend_self_inner(GC_thread me, size_t suspend_cnt);

  GC_INNER void GC_suspend_self_blocked(ptr_t thread_me, void *context);
                                /* Wrapper over GC_suspend_self_inner.  */
#endif

#if defined(GC_PTHREADS) \
    && !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)

# ifdef GC_PTHREAD_START_STANDALONE
#   define GC_INNER_PTHRSTART /* empty */
# else
#   define GC_INNER_PTHRSTART GC_INNER
# endif

  GC_INNER_PTHRSTART void *GC_CALLBACK GC_pthread_start_inner(
                                        struct GC_stack_base *sb, void *arg);
  GC_INNER_PTHRSTART GC_thread GC_start_rtn_prepare_thread(
                                        void *(**pstart)(void *),
                                        void **pstart_arg,
                                        struct GC_stack_base *sb, void *arg);
  GC_INNER_PTHRSTART void GC_thread_exit_proc(void *);
#endif /* GC_PTHREADS */

#ifdef GC_DARWIN_THREADS
# ifndef DARWIN_DONT_PARSE_STACK
    GC_INNER ptr_t GC_FindTopOfStack(unsigned long);
# endif
# if defined(PARALLEL_MARK) && !defined(GC_NO_THREADS_DISCOVERY)
    GC_INNER GC_bool GC_is_mach_marker(thread_act_t);
# endif
#endif /* GC_DARWIN_THREADS */

#ifdef PTHREAD_STOP_WORLD_IMPL
  GC_INNER void GC_stop_init(void);
#endif

EXTERN_C_END

#endif /* THREADS */

#endif /* GC_PTHREAD_SUPPORT_H */


/* This probably needs more porting work to ppc64. */

#if defined(GC_DARWIN_THREADS)

#include <sys/sysctl.h>
#include <mach/machine.h>

#if defined(ARM32) && defined(ARM_THREAD_STATE32)
# include <CoreFoundation/CoreFoundation.h>
#endif

/* From "Inside Mac OS X - Mach-O Runtime Architecture" published by Apple
   Page 49:
   "The space beneath the stack pointer, where a new stack frame would normally
   be allocated, is called the red zone. This area as shown in Figure 3-2 may
   be used for any purpose as long as a new stack frame does not need to be
   added to the stack."

   Page 50: "If a leaf procedure's red zone usage would exceed 224 bytes, then
   it must set up a stack frame just like routines that call other routines."
*/
#ifdef POWERPC
# if CPP_WORDSZ == 32
#   define PPC_RED_ZONE_SIZE 224
# elif CPP_WORDSZ == 64
#   define PPC_RED_ZONE_SIZE 320
# endif
#endif

#ifndef DARWIN_DONT_PARSE_STACK

typedef struct StackFrame {
  unsigned long savedSP;
  unsigned long savedCR;
  unsigned long savedLR;
  unsigned long reserved[2];
  unsigned long savedRTOC;
} StackFrame;

GC_INNER ptr_t GC_FindTopOfStack(unsigned long stack_start)
{
  StackFrame *frame = (StackFrame *)stack_start;

  if (stack_start == 0) {
#   ifdef POWERPC
#     if CPP_WORDSZ == 32
        __asm__ __volatile__ ("lwz %0,0(r1)" : "=r" (frame));
#     else
        __asm__ __volatile__ ("ld %0,0(r1)" : "=r" (frame));
#     endif
#   elif defined(ARM32)
        volatile ptr_t sp_reg;
        __asm__ __volatile__ ("mov %0, r7\n" : "=r" (sp_reg));
        frame = (StackFrame *)sp_reg;
#   elif defined(AARCH64)
        volatile ptr_t sp_reg;
        __asm__ __volatile__ ("mov %0, x29\n" : "=r" (sp_reg));
        frame = (StackFrame *)sp_reg;
#   else
#     if defined(CPPCHECK)
        GC_noop1_ptr(&frame);
#     endif
      ABORT("GC_FindTopOfStack(0) is not implemented");
#   endif
  }

# ifdef DEBUG_THREADS_EXTRA
    GC_log_printf("FindTopOfStack start at sp= %p\n", (void *)frame);
# endif
  while (frame->savedSP != 0) { /* stop if no more stack frames */
    unsigned long maskedLR;

    frame = (StackFrame*)frame->savedSP;

    /* we do these next two checks after going to the next frame
       because the LR for the first stack frame in the loop
       is not set up on purpose, so we shouldn't check it. */
    maskedLR = frame -> savedLR & ~0x3UL;
    if (0 == maskedLR || ~0x3UL == maskedLR)
      break; /* if the next LR is bogus, stop */
  }
# ifdef DEBUG_THREADS_EXTRA
    GC_log_printf("FindTopOfStack finish at sp= %p\n", (void *)frame);
# endif
  return (ptr_t)frame;
}

#endif /* !DARWIN_DONT_PARSE_STACK */

/* GC_query_task_threads controls whether to obtain the list of */
/* the threads from the kernel or to use GC_threads table.      */
#ifdef GC_NO_THREADS_DISCOVERY
# define GC_query_task_threads FALSE
#elif defined(GC_DISCOVER_TASK_THREADS)
# define GC_query_task_threads TRUE
#else
  STATIC GC_bool GC_query_task_threads = FALSE;
#endif /* !GC_NO_THREADS_DISCOVERY */

/* Use implicit threads registration (all task threads excluding the GC */
/* special ones are stopped and scanned).  Should be called before      */
/* GC_INIT() (or, at least, before going multi-threaded).  Deprecated.  */
GC_API void GC_CALL GC_use_threads_discovery(void)
{
# ifdef GC_NO_THREADS_DISCOVERY
    ABORT("Darwin task-threads-based stop and push unsupported");
# else
#   ifndef GC_ALWAYS_MULTITHREADED
      GC_ASSERT(!GC_need_to_lock);
#   endif
#   ifndef GC_DISCOVER_TASK_THREADS
      GC_query_task_threads = TRUE;
#   endif
    GC_init();
# endif
}

#ifndef kCFCoreFoundationVersionNumber_iOS_8_0
# define kCFCoreFoundationVersionNumber_iOS_8_0 1140.1
#endif

/* Evaluates the stack range for a given thread.  Returns the lower     */
/* bound and sets *phi to the upper one.  Sets *pfound_me to TRUE if    */
/* this is current thread, otherwise the value is not changed.          */
STATIC ptr_t GC_stack_range_for(ptr_t *phi, thread_act_t thread, GC_thread p,
                                mach_port_t my_thread, ptr_t *paltstack_lo,
                                ptr_t *paltstack_hi, GC_bool *pfound_me)
{
# ifdef DARWIN_DONT_PARSE_STACK
    GC_stack_context_t crtn;
# endif
  ptr_t lo;

  GC_ASSERT(I_HOLD_LOCK());
  if (thread == my_thread) {
    GC_ASSERT(NULL == p || (p -> flags & DO_BLOCKING) == 0);
    lo = GC_approx_sp();
#   ifndef DARWIN_DONT_PARSE_STACK
      *phi = GC_FindTopOfStack(0);
#   endif
    *pfound_me = TRUE;
  } else if (p != NULL && (p -> flags & DO_BLOCKING) != 0) {
    lo = p -> crtn -> stack_ptr;
#   ifndef DARWIN_DONT_PARSE_STACK
      *phi = p -> crtn -> topOfStack;
#   endif

  } else {
    /* MACHINE_THREAD_STATE_COUNT does not seem to be defined       */
    /* everywhere.  Hence we use our own version.  Alternatively,   */
    /* we could use THREAD_STATE_MAX (but seems to be not optimal). */
    kern_return_t kern_result;
    GC_THREAD_STATE_T state;

#   if defined(ARM32) && defined(ARM_THREAD_STATE32)
      /* Use ARM_UNIFIED_THREAD_STATE on iOS8+ 32-bit targets and on    */
      /* 64-bit H/W (iOS7+ 32-bit mode).                                */
      size_t size;
      static cpu_type_t cputype = 0;

      if (cputype == 0) {
        sysctlbyname("hw.cputype", &cputype, &size, NULL, 0);
      }
      if (cputype == CPU_TYPE_ARM64
          || kCFCoreFoundationVersionNumber
             >= kCFCoreFoundationVersionNumber_iOS_8_0) {
        arm_unified_thread_state_t unified_state;
        mach_msg_type_number_t unified_thread_state_count
                                        = ARM_UNIFIED_THREAD_STATE_COUNT;
#       if defined(CPPCHECK)
#         define GC_ARM_UNIFIED_THREAD_STATE 1
#       else
#         define GC_ARM_UNIFIED_THREAD_STATE ARM_UNIFIED_THREAD_STATE
#       endif
        kern_result = thread_get_state(thread, GC_ARM_UNIFIED_THREAD_STATE,
                                       (natural_t *)&unified_state,
                                       &unified_thread_state_count);
#       if !defined(CPPCHECK)
          if (unified_state.ash.flavor != ARM_THREAD_STATE32) {
            ABORT("unified_state flavor should be ARM_THREAD_STATE32");
          }
#       endif
        state = unified_state;
      } else
#   endif
    /* else */ {
      mach_msg_type_number_t thread_state_count = GC_MACH_THREAD_STATE_COUNT;

      /* Get the thread state (registers, etc.) */
      do {
        kern_result = thread_get_state(thread, GC_MACH_THREAD_STATE,
                                       (natural_t *)&state,
                                       &thread_state_count);
      } while (kern_result == KERN_ABORTED);
    }
#   ifdef DEBUG_THREADS
      GC_log_printf("thread_get_state returns %d\n", kern_result);
#   endif
    if (kern_result != KERN_SUCCESS)
      ABORT("thread_get_state failed");

#   if defined(I386)
      lo = (ptr_t)state.THREAD_FLD(esp);
#     ifndef DARWIN_DONT_PARSE_STACK
        *phi = GC_FindTopOfStack(state.THREAD_FLD(esp));
#     endif
      GC_push_one(state.THREAD_FLD(eax));
      GC_push_one(state.THREAD_FLD(ebx));
      GC_push_one(state.THREAD_FLD(ecx));
      GC_push_one(state.THREAD_FLD(edx));
      GC_push_one(state.THREAD_FLD(edi));
      GC_push_one(state.THREAD_FLD(esi));
      GC_push_one(state.THREAD_FLD(ebp));

#   elif defined(X86_64)
      lo = (ptr_t)state.THREAD_FLD(rsp);
#     ifndef DARWIN_DONT_PARSE_STACK
        *phi = GC_FindTopOfStack(state.THREAD_FLD(rsp));
#     endif
      GC_push_one(state.THREAD_FLD(rax));
      GC_push_one(state.THREAD_FLD(rbx));
      GC_push_one(state.THREAD_FLD(rcx));
      GC_push_one(state.THREAD_FLD(rdx));
      GC_push_one(state.THREAD_FLD(rdi));
      GC_push_one(state.THREAD_FLD(rsi));
      GC_push_one(state.THREAD_FLD(rbp));
      /* rsp is skipped.        */
      GC_push_one(state.THREAD_FLD(r8));
      GC_push_one(state.THREAD_FLD(r9));
      GC_push_one(state.THREAD_FLD(r10));
      GC_push_one(state.THREAD_FLD(r11));
      GC_push_one(state.THREAD_FLD(r12));
      GC_push_one(state.THREAD_FLD(r13));
      GC_push_one(state.THREAD_FLD(r14));
      GC_push_one(state.THREAD_FLD(r15));

#   elif defined(POWERPC)
      lo = (ptr_t)(state.THREAD_FLD(r1) - PPC_RED_ZONE_SIZE);
#     ifndef DARWIN_DONT_PARSE_STACK
        *phi = GC_FindTopOfStack(state.THREAD_FLD(r1));
#     endif
      GC_push_one(state.THREAD_FLD(r0));
      /* r1 is skipped. */
      GC_push_one(state.THREAD_FLD(r2));
      GC_push_one(state.THREAD_FLD(r3));
      GC_push_one(state.THREAD_FLD(r4));
      GC_push_one(state.THREAD_FLD(r5));
      GC_push_one(state.THREAD_FLD(r6));
      GC_push_one(state.THREAD_FLD(r7));
      GC_push_one(state.THREAD_FLD(r8));
      GC_push_one(state.THREAD_FLD(r9));
      GC_push_one(state.THREAD_FLD(r10));
      GC_push_one(state.THREAD_FLD(r11));
      GC_push_one(state.THREAD_FLD(r12));
      GC_push_one(state.THREAD_FLD(r13));
      GC_push_one(state.THREAD_FLD(r14));
      GC_push_one(state.THREAD_FLD(r15));
      GC_push_one(state.THREAD_FLD(r16));
      GC_push_one(state.THREAD_FLD(r17));
      GC_push_one(state.THREAD_FLD(r18));
      GC_push_one(state.THREAD_FLD(r19));
      GC_push_one(state.THREAD_FLD(r20));
      GC_push_one(state.THREAD_FLD(r21));
      GC_push_one(state.THREAD_FLD(r22));
      GC_push_one(state.THREAD_FLD(r23));
      GC_push_one(state.THREAD_FLD(r24));
      GC_push_one(state.THREAD_FLD(r25));
      GC_push_one(state.THREAD_FLD(r26));
      GC_push_one(state.THREAD_FLD(r27));
      GC_push_one(state.THREAD_FLD(r28));
      GC_push_one(state.THREAD_FLD(r29));
      GC_push_one(state.THREAD_FLD(r30));
      GC_push_one(state.THREAD_FLD(r31));

#   elif defined(ARM32)
      lo = (ptr_t)state.THREAD_FLD(sp);
#     ifndef DARWIN_DONT_PARSE_STACK
        *phi = GC_FindTopOfStack(state.THREAD_FLD(r[7])); /* fp */
#     endif
      {
        int j;
        for (j = 0; j < 7; j++)
          GC_push_one(state.THREAD_FLD(r[j]));
        j++; /* "r7" is skipped (iOS uses it as a frame pointer) */
        for (; j <= 12; j++)
          GC_push_one(state.THREAD_FLD(r[j]));
      }
      /* "cpsr", "pc" and "sp" are skipped */
      GC_push_one(state.THREAD_FLD(lr));

#   elif defined(AARCH64)
      lo = (ptr_t)state.THREAD_FLD(sp);
#     ifndef DARWIN_DONT_PARSE_STACK
        *phi = GC_FindTopOfStack(state.THREAD_FLD(fp));
#     endif
      {
        int j;
        for (j = 0; j <= 28; j++) {
          GC_push_one(state.THREAD_FLD(x[j]));
        }
      }
      /* "cpsr", "fp", "pc" and "sp" are skipped */
      GC_push_one(state.THREAD_FLD(lr));

#   elif defined(CPPCHECK)
      lo = NULL;
#   else
#     error FIXME for non-arm/ppc/x86 architectures
#   endif
  } /* thread != my_thread */

# ifndef DARWIN_DONT_PARSE_STACK
    /* TODO: Determine p and handle altstack if !DARWIN_DONT_PARSE_STACK */
    UNUSED_ARG(paltstack_hi);
# else
    /* p is guaranteed to be non-NULL regardless of GC_query_task_threads. */
#   ifdef CPPCHECK
      if (NULL == p) ABORT("Bad GC_stack_range_for call");
#   endif
    crtn = p -> crtn;
    *phi = crtn -> stack_end;
    if (crtn -> altstack != NULL && ADDR_GE(lo, crtn -> altstack)
        && ADDR_GE(crtn -> altstack + crtn -> altstack_size, lo)) {
      *paltstack_lo = lo;
      *paltstack_hi = crtn -> altstack + crtn -> altstack_size;
      lo = crtn -> normstack;
      *phi = lo + crtn -> normstack_size;
    } else
# endif
  /* else */ {
    *paltstack_lo = NULL;
  }
# if defined(STACKPTR_CORRECTOR_AVAILABLE) && defined(DARWIN_DONT_PARSE_STACK)
    if (GC_sp_corrector != 0)
      GC_sp_corrector((void **)&lo, (void *)(p -> id));
# endif
# ifdef DEBUG_THREADS
    GC_log_printf("Darwin: Stack for thread %p is [%p,%p)\n",
                  (void *)(word)thread, (void *)lo, (void *)(*phi));
# endif
  return lo;
}

GC_INNER void GC_push_all_stacks(void)
{
  ptr_t hi, altstack_lo, altstack_hi;
  task_t my_task = current_task();
  mach_port_t my_thread = mach_thread_self();
  GC_bool found_me = FALSE;
  int nthreads = 0;
  word total_size = 0;

  GC_ASSERT(I_HOLD_LOCK());
  GC_ASSERT(GC_thr_initialized);

# ifndef DARWIN_DONT_PARSE_STACK
    if (GC_query_task_threads) {
      int i;
      kern_return_t kern_result;
      thread_act_array_t act_list;
      mach_msg_type_number_t listcount;

      /* Obtain the list of the threads from the kernel.  */
      kern_result = task_threads(my_task, &act_list, &listcount);
      if (kern_result != KERN_SUCCESS)
        ABORT("task_threads failed");

      for (i = 0; i < (int)listcount; i++) {
        thread_act_t thread = act_list[i];
        ptr_t lo = GC_stack_range_for(&hi, thread, NULL, my_thread,
                                      &altstack_lo, &altstack_hi, &found_me);

        if (lo) {
          GC_ASSERT(ADDR_GE(hi, lo));
          total_size += hi - lo;
          GC_push_all_stack(lo, hi);
        }
        /* TODO: Handle altstack */
        nthreads++;
        mach_port_deallocate(my_task, thread);
      } /* for (i=0; ...) */

      vm_deallocate(my_task, (vm_address_t)act_list,
                    sizeof(thread_t) * listcount);
    } else
# endif /* !DARWIN_DONT_PARSE_STACK */
  /* else */ {
    int i;

    for (i = 0; i < THREAD_TABLE_SZ; i++) {
      GC_thread p;

      for (p = GC_threads[i]; p != NULL; p = p -> tm.next) {
        GC_ASSERT(THREAD_TABLE_INDEX(p -> id) == i);
        if (!KNOWN_FINISHED(p)) {
          thread_act_t thread = (thread_act_t)(p -> mach_thread);
          ptr_t lo = GC_stack_range_for(&hi, thread, p, my_thread,
                                        &altstack_lo, &altstack_hi, &found_me);

          if (lo) {
            GC_ASSERT(ADDR_GE(hi, lo));
            total_size += hi - lo;
            GC_push_all_stack_sections(lo, hi, p -> crtn -> traced_stack_sect);
          }
          if (altstack_lo) {
            total_size += altstack_hi - altstack_lo;
            GC_push_all_stack(altstack_lo, altstack_hi);
          }
          nthreads++;
        }
      }
    } /* for (i=0; ...) */
  }

  mach_port_deallocate(my_task, my_thread);
  GC_VERBOSE_LOG_PRINTF("Pushed %d thread stacks\n", nthreads);
  if (!found_me && !GC_in_thread_creation)
    ABORT("Collecting from unknown thread");
  GC_total_stacksize = total_size;
}

#ifndef GC_NO_THREADS_DISCOVERY

# ifdef MPROTECT_VDB
    STATIC mach_port_t GC_mach_handler_thread = 0;
    STATIC GC_bool GC_use_mach_handler_thread = FALSE;

    GC_INNER void GC_darwin_register_self_mach_handler(void)
    {
      GC_mach_handler_thread = mach_thread_self();
      GC_use_mach_handler_thread = TRUE;
    }
# endif /* MPROTECT_VDB */

# ifndef GC_MAX_MACH_THREADS
#   define GC_MAX_MACH_THREADS THREAD_TABLE_SZ
# endif

  struct GC_mach_thread {
    thread_act_t thread;
    GC_bool suspended;
  };

  struct GC_mach_thread GC_mach_threads[GC_MAX_MACH_THREADS];
  STATIC int GC_mach_threads_count = 0;
  /* FIXME: it is better to implement GC_mach_threads as a hash set.  */

/* returns true if there's a thread in act_list that wasn't in old_list */
STATIC GC_bool GC_suspend_thread_list(thread_act_array_t act_list, int count,
                                      thread_act_array_t old_list,
                                      int old_count, task_t my_task,
                                      mach_port_t my_thread)
{
  int i;
  int j = -1;
  GC_bool changed = FALSE;

  GC_ASSERT(I_HOLD_LOCK());
  for (i = 0; i < count; i++) {
    thread_act_t thread = act_list[i];
    GC_bool found;
    kern_return_t kern_result;

    if (thread == my_thread
#       ifdef MPROTECT_VDB
          || (GC_mach_handler_thread == thread && GC_use_mach_handler_thread)
#       endif
#       ifdef PARALLEL_MARK
          || GC_is_mach_marker(thread) /* ignore the parallel markers */
#       endif
        ) {
      /* Do not add our one, parallel marker and the handler threads;   */
      /* consider it as found (e.g., it was processed earlier).         */
      mach_port_deallocate(my_task, thread);
      continue;
    }

    /* find the current thread in the old list */
    found = FALSE;
    {
      int last_found = j; /* remember the previous found thread index */

      /* Search for the thread starting from the last found one first.  */
      while (++j < old_count)
        if (old_list[j] == thread) {
          found = TRUE;
          break;
        }
      if (!found) {
        /* If not found, search in the rest (beginning) of the list.    */
        for (j = 0; j < last_found; j++)
          if (old_list[j] == thread) {
            found = TRUE;
            break;
          }
      }
    }

    if (found) {
      /* It is already in the list, skip processing, release mach port. */
      mach_port_deallocate(my_task, thread);
      continue;
    }

    /* add it to the GC_mach_threads list */
    if (GC_mach_threads_count == GC_MAX_MACH_THREADS)
      ABORT("Too many threads");
    GC_mach_threads[GC_mach_threads_count].thread = thread;
    /* default is not suspended */
    GC_mach_threads[GC_mach_threads_count].suspended = FALSE;
    changed = TRUE;

#   ifdef DEBUG_THREADS
      GC_log_printf("Suspending %p\n", (void *)(word)thread);
#   endif
    /* Unconditionally suspend the thread.  It will do no     */
    /* harm if it is already suspended by the client logic.   */
    GC_acquire_dirty_lock();
    do {
      kern_result = thread_suspend(thread);
    } while (kern_result == KERN_ABORTED);
    GC_release_dirty_lock();
    if (kern_result != KERN_SUCCESS) {
      /* The thread may have quit since the thread_threads() call we  */
      /* mark already suspended so it's not dealt with anymore later. */
      GC_mach_threads[GC_mach_threads_count].suspended = FALSE;
    } else {
      /* Mark the thread as suspended and require resume.     */
      GC_mach_threads[GC_mach_threads_count].suspended = TRUE;
      if (GC_on_thread_event)
        GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED, (void *)(word)thread);
    }
    GC_mach_threads_count++;
  }
  return changed;
}

#endif /* !GC_NO_THREADS_DISCOVERY */

GC_INNER void GC_stop_world(void)
{
  task_t my_task = current_task();
  mach_port_t my_thread = mach_thread_self();
  kern_return_t kern_result;

  GC_ASSERT(I_HOLD_LOCK());
  GC_ASSERT(GC_thr_initialized);
# ifdef DEBUG_THREADS
    GC_log_printf("Stopping the world from thread %p\n",
                  (void *)(word)my_thread);
# endif
# ifdef PARALLEL_MARK
    if (GC_parallel) {
      GC_acquire_mark_lock();
      GC_ASSERT(GC_fl_builder_count == 0);
      /* We should have previously waited for it to become zero. */
    }
# endif /* PARALLEL_MARK */

  if (GC_query_task_threads) {
#   ifndef GC_NO_THREADS_DISCOVERY
      GC_bool changed;
      thread_act_array_t act_list, prev_list;
      mach_msg_type_number_t listcount, prevcount;

      /* Clear out the mach threads list table.  We do not need to      */
      /* really clear GC_mach_threads[] as it is used only in the range */
      /* from 0 to GC_mach_threads_count-1, inclusive.                  */
      GC_mach_threads_count = 0;

      /* Loop stopping threads until you have gone over the whole list  */
      /* twice without a new one appearing.  thread_create() won't      */
      /* return (and thus the thread stop) until the new thread exists, */
      /* so there is no window whereby you could stop a thread,         */
      /* recognize it is stopped, but then have a new thread it created */
      /* before stopping show up later.                                 */
      changed = TRUE;
      prev_list = NULL;
      prevcount = 0;
      do {
        kern_result = task_threads(my_task, &act_list, &listcount);

        if (kern_result == KERN_SUCCESS) {
          changed = GC_suspend_thread_list(act_list, listcount, prev_list,
                                           prevcount, my_task, my_thread);

          if (prev_list != NULL) {
            /* Thread ports are not deallocated by list, unused ports   */
            /* deallocated in GC_suspend_thread_list, used - kept in    */
            /* GC_mach_threads till GC_start_world as otherwise thread  */
            /* object change can occur and GC_start_world will not      */
            /* find the thread to resume which will cause app to hang.  */
            vm_deallocate(my_task, (vm_address_t)prev_list,
                          sizeof(thread_t) * prevcount);
          }

          /* Repeat while having changes. */
          prev_list = act_list;
          prevcount = listcount;
        }
      } while (changed);

      GC_ASSERT(prev_list != 0);
      /* The thread ports are not deallocated by list, see above.       */
      vm_deallocate(my_task, (vm_address_t)act_list,
                    sizeof(thread_t) * listcount);
#   endif /* !GC_NO_THREADS_DISCOVERY */

  } else {
    unsigned i;

    for (i = 0; i < THREAD_TABLE_SZ; i++) {
      GC_thread p;

      for (p = GC_threads[i]; p != NULL; p = p -> tm.next) {
        if ((p -> flags & (FINISHED | DO_BLOCKING)) == 0
            && p -> mach_thread != my_thread) {
          GC_acquire_dirty_lock();
          do {
            kern_result = thread_suspend(p -> mach_thread);
          } while (kern_result == KERN_ABORTED);
          GC_release_dirty_lock();
          if (kern_result != KERN_SUCCESS)
            ABORT("thread_suspend failed");
          if (GC_on_thread_event)
            GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED,
                               (void *)(word)(p -> mach_thread));
        }
      }
    }
  }

# ifdef MPROTECT_VDB
    if (GC_auto_incremental) {
      GC_mprotect_stop();
    }
# endif
# ifdef PARALLEL_MARK
    if (GC_parallel)
      GC_release_mark_lock();
# endif

# ifdef DEBUG_THREADS
    GC_log_printf("World stopped from %p\n", (void *)(word)my_thread);
# endif
  mach_port_deallocate(my_task, my_thread);
}

GC_INLINE void GC_thread_resume(thread_act_t thread)
{
  kern_return_t kern_result;
# if defined(DEBUG_THREADS) || defined(GC_ASSERTIONS)
    struct thread_basic_info info;
    mach_msg_type_number_t outCount = THREAD_BASIC_INFO_COUNT;

#   ifdef CPPCHECK
      info.run_state = 0;
#   endif
    kern_result = thread_info(thread, THREAD_BASIC_INFO,
                              (thread_info_t)&info, &outCount);
    if (kern_result != KERN_SUCCESS)
      ABORT("thread_info failed");
# endif
  GC_ASSERT(I_HOLD_LOCK());
# ifdef DEBUG_THREADS
    GC_log_printf("Resuming thread %p with state %d\n", (void *)(word)thread,
                  info.run_state);
# endif
  /* Resume the thread */
  kern_result = thread_resume(thread);
  if (kern_result != KERN_SUCCESS) {
    WARN("thread_resume(%p) failed: mach port invalid\n", thread);
  } else if (GC_on_thread_event) {
    GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, (void *)(word)thread);
  }
}

GC_INNER void GC_start_world(void)
{
  task_t my_task = current_task();

  GC_ASSERT(I_HOLD_LOCK()); /* held continuously since the world stopped */
# ifdef DEBUG_THREADS
    GC_log_printf("World starting\n");
# endif
# ifdef MPROTECT_VDB
    if (GC_auto_incremental) {
      GC_mprotect_resume();
    }
# endif

  if (GC_query_task_threads) {
#   ifndef GC_NO_THREADS_DISCOVERY
      int i, j;
      kern_return_t kern_result;
      thread_act_array_t act_list;
      mach_msg_type_number_t listcount;

      kern_result = task_threads(my_task, &act_list, &listcount);
      if (kern_result != KERN_SUCCESS)
        ABORT("task_threads failed");

      j = (int)listcount;
      for (i = 0; i < GC_mach_threads_count; i++) {
        thread_act_t thread = GC_mach_threads[i].thread;

        if (GC_mach_threads[i].suspended) {
          int last_found = j;   /* The thread index found during the    */
                                /* previous iteration (count value      */
                                /* means no thread found yet).          */

          /* Search for the thread starting from the last found one first. */
          while (++j < (int)listcount) {
            if (act_list[j] == thread)
              break;
          }
          if (j >= (int)listcount) {
            /* If not found, search in the rest (beginning) of the list. */
            for (j = 0; j < last_found; j++) {
              if (act_list[j] == thread)
                break;
            }
          }
          if (j != last_found) {
            /* The thread is alive, resume it.  */
            GC_thread_resume(thread);
          }
        } else {
          /* This thread was failed to be suspended by GC_stop_world,   */
          /* no action needed.                                          */
#         ifdef DEBUG_THREADS
            GC_log_printf("Not resuming thread %p as it is not suspended\n",
                          (void *)(word)thread);
#         endif
        }
        mach_port_deallocate(my_task, thread);
      }

      for (i = 0; i < (int)listcount; i++)
        mach_port_deallocate(my_task, act_list[i]);
      vm_deallocate(my_task, (vm_address_t)act_list,
                    sizeof(thread_t) * listcount);
#   endif /* !GC_NO_THREADS_DISCOVERY */

  } else {
    int i;
    mach_port_t my_thread = mach_thread_self();

    for (i = 0; i < THREAD_TABLE_SZ; i++) {
      GC_thread p;

      for (p = GC_threads[i]; p != NULL; p = p -> tm.next) {
        if ((p -> flags & (FINISHED | DO_BLOCKING)) == 0
            && p -> mach_thread != my_thread)
          GC_thread_resume(p -> mach_thread);
      }
    }

    mach_port_deallocate(my_task, my_thread);
  }

# ifdef DEBUG_THREADS
    GC_log_printf("World started\n");
# endif
}

#endif /* GC_DARWIN_THREADS */

/*
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1997 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 2009-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


/*
 * This is incredibly OS specific code for tracking down data sections in
 * dynamic libraries.  There appears to be no way of doing this quickly
 * without groveling through undocumented data structures.  We would argue
 * that this is a bug in the design of the dlopen interface.  THIS CODE
 * MAY BREAK IN FUTURE OS RELEASES.  If this matters to you, don't hesitate
 * to let your vendor know ...
 *
 * None of this is safe with dlclose and incremental collection.
 * But then not much of anything is safe in the presence of dlclose.
 */

/* BTL: avoid circular redefinition of dlopen if GC_SOLARIS_THREADS defined */
#undef GC_MUST_RESTORE_REDEFINED_DLOPEN
#if defined(GC_PTHREADS) && !defined(GC_NO_DLOPEN) \
    && !defined(GC_NO_THREAD_REDIRECTS) && !defined(GC_USE_LD_WRAP)
  /* To support threads in Solaris, gc.h interposes on dlopen by        */
  /* defining "dlopen" to be "GC_dlopen", which is implemented below.   */
  /* However, both GC_FirstDLOpenedLinkMap() and GC_dlopen() use the    */
  /* real system dlopen() in their implementation. We first remove      */
  /* gc.h's dlopen definition and restore it later, after GC_dlopen().  */
# undef dlopen
# define GC_MUST_RESTORE_REDEFINED_DLOPEN
#endif /* !GC_NO_DLOPEN */

#if defined(SOLARISDL) && defined(THREADS) && !defined(PCR) \
    && !defined(GC_SOLARIS_THREADS) && !defined(CPPCHECK)
#  error Fix mutual exclusion with dlopen
#endif

/* A user-supplied routine (custom filter) that might be called to      */
/* determine whether a DSO really needs to be scanned by the GC.        */
/* 0 means no filter installed.  May be unused on some platforms.       */
/* FIXME: Add filter support for more platforms.                        */
STATIC GC_has_static_roots_func GC_has_static_roots = 0;

#if defined(DYNAMIC_LOADING) && !defined(PCR) || defined(ANY_MSWIN)

#if !(defined(CPPCHECK) || defined(AIX) || defined(ANY_MSWIN) \
      || defined(DARWIN) || defined(DGUX) || defined(IRIX5) \
      || defined(HAIKU) || defined(HPUX) || defined(HURD) \
      || defined(NACL) || defined(SCO_ELF) || defined(SOLARISDL) \
      || ((defined(ANY_BSD) || defined(LINUX)) && defined(__ELF__)) \
      || (defined(ALPHA) && defined(OSF1)) \
      || (defined(OPENBSD) && defined(M68K)))
# error We only know how to find data segments of dynamic libraries for above.
# error Additional SVR4 variants might not be too hard to add.
#endif

#if defined(DARWIN) && !defined(USE_DYLD_TO_BIND) \
    && !defined(NO_DYLD_BIND_FULLY_IMAGE)
# include <dlfcn.h>
#endif

#ifdef SOLARISDL
#   include <sys/elf.h>
#   include <dlfcn.h>
#   include <link.h>
#endif

#if defined(NETBSD)
#   include <sys/param.h>
#   include <dlfcn.h>
#   include <machine/elf_machdep.h>
#   define ELFSIZE ARCH_ELFSIZE
#endif

#if defined(OPENBSD)
# include <sys/param.h>
# if (OpenBSD >= 200519) && !defined(HAVE_DL_ITERATE_PHDR)
#   define HAVE_DL_ITERATE_PHDR
# endif
#endif /* OPENBSD */

#if defined(DGUX) || defined(HURD) || defined(NACL) || defined(SCO_ELF) \
    || ((defined(ANY_BSD) || defined(LINUX)) && defined(__ELF__))
# include <stddef.h>
# if !defined(OPENBSD) && !defined(HOST_ANDROID)
    /* OpenBSD does not have elf.h file; link.h below is sufficient.    */
    /* Exclude Android because linker.h below includes its own version. */
#   include <elf.h>
# endif
# ifdef HOST_ANDROID
    /* If you don't need the "dynamic loading" feature, you may build   */
    /* the collector with -D IGNORE_DYNAMIC_LOADING.                    */
#   ifdef BIONIC_ELFDATA_REDEF_BUG
      /* Workaround a problem in Bionic (as of Android 4.2) which has   */
      /* mismatching ELF_DATA definitions in sys/exec_elf.h and         */
      /* asm/elf.h included from linker.h file (similar to EM_ALPHA).   */
#     include <asm/elf.h>
#     include <linux/elf-em.h>
#     undef ELF_DATA
#     undef EM_ALPHA
#   endif
#   include <link.h>
#   if !defined(GC_DONT_DEFINE_LINK_MAP) && !(__ANDROID_API__ >= 21)
      /* link_map and r_debug are defined in link.h of NDK r10+.        */
      /* bionic/linker/linker.h defines them too but the header         */
      /* itself is a C++ one starting from Android 4.3.                 */
      struct link_map {
        uintptr_t l_addr;
        char* l_name;
        uintptr_t l_ld;
        struct link_map* l_next;
        struct link_map* l_prev;
      };
      struct r_debug {
        int32_t r_version;
        struct link_map* r_map;
        void (*r_brk)(void);
        int32_t r_state;
        uintptr_t r_ldbase;
      };
#   endif
# else
    EXTERN_C_BEGIN      /* Workaround missing extern "C" around _DYNAMIC */
                        /* symbol in link.h of some Linux hosts.         */
#   include <link.h>
    EXTERN_C_END
# endif
#endif

/* Newer versions of GNU/Linux define this macro.  We
 * define it similarly for any ELF systems that don't.  */
# ifndef ElfW
#   if defined(FREEBSD)
#     if __ELF_WORD_SIZE == 32
#       define ElfW(type) Elf32_##type
#     else
#       define ElfW(type) Elf64_##type
#     endif
#   elif defined(NETBSD) || defined(OPENBSD)
#     if ELFSIZE == 32
#       define ElfW(type) Elf32_##type
#     elif ELFSIZE == 64
#       define ElfW(type) Elf64_##type
#     else
#       error Missing ELFSIZE define
#     endif
#   else
#     if !defined(ELF_CLASS) || ELF_CLASS == ELFCLASS32
#       define ElfW(type) Elf32_##type
#     else
#       define ElfW(type) Elf64_##type
#     endif
#   endif
# endif

#if defined(SOLARISDL) && !defined(USE_PROC_FOR_LIBRARIES)

  EXTERN_C_BEGIN
  extern ElfW(Dyn) _DYNAMIC;
  EXTERN_C_END

  STATIC struct link_map *
  GC_FirstDLOpenedLinkMap(void)
  {
    ElfW(Dyn) *dp;
    static struct link_map * cachedResult = 0;
    static ElfW(Dyn) *dynStructureAddr = 0;
                /* BTL: added to avoid Solaris 5.3 ld.so _DYNAMIC bug   */

#   ifdef SUNOS53_SHARED_LIB
        /* BTL: Avoid the Solaris 5.3 bug that _DYNAMIC isn't being set */
        /* up properly in dynamically linked .so's. This means we have  */
        /* to use its value in the set of original object files loaded  */
        /* at program startup.                                          */
        if (0 == dynStructureAddr) {
          void *startupSyms = dlopen(0, RTLD_LAZY);

          dynStructureAddr = (ElfW(Dyn) *)dlsym(startupSyms, "_DYNAMIC");
          /* Note: dlclose() is not called intentionally. */
        }
#   else
        dynStructureAddr = &_DYNAMIC;
#   endif

    if (0 == COVERT_DATAFLOW(ADDR(dynStructureAddr))) {
        /* _DYNAMIC symbol not resolved. */
        return NULL;
    }
    if (cachedResult == 0) {
        int tag;

        for (dp = (ElfW(Dyn) *)&_DYNAMIC; (tag = dp->d_tag) != 0; dp++) {
            if (tag == DT_DEBUG) {
                const struct r_debug *rd = (struct r_debug *)dp->d_un.d_ptr;
                if (rd != NULL) {
                    const struct link_map *lm = rd->r_map;
                    if (lm != NULL)
                        cachedResult = lm->l_next; /* might be NULL */
                }
                break;
            }
        }
    }
    return cachedResult;
  }

  GC_INNER void GC_register_dynamic_libraries(void)
  {
    struct link_map *lm;

    GC_ASSERT(I_HOLD_LOCK());
    for (lm = GC_FirstDLOpenedLinkMap(); lm != 0; lm = lm->l_next) {
        ElfW(Ehdr) * e;
        ElfW(Phdr) * p;
        unsigned long offset;
        char * start;
        int i;

        e = (ElfW(Ehdr) *)lm->l_addr;
        p = ((ElfW(Phdr) *)(((char *)(e)) + e->e_phoff));
        offset = ((unsigned long)(lm->l_addr));
        for (i = 0; i < (int)e->e_phnum; i++, p++) {
          switch (p->p_type) {
            case PT_LOAD:
              if (!(p->p_flags & PF_W)) break;
              start = (char *)(p->p_vaddr) + offset;
              GC_add_roots_inner(start, start + p->p_memsz, TRUE);
              break;
            default:
              break;
          }
        }
    }
  }

#endif /* SOLARISDL && !USE_PROC_FOR_LIBRARIES */

#if defined(DGUX) || defined(HURD) || defined(NACL) || defined(SCO_ELF) \
    || ((defined(ANY_BSD) || defined(LINUX)) && defined(__ELF__))

#ifdef USE_PROC_FOR_LIBRARIES

#include <string.h>

#include <sys/stat.h>
#include <fcntl.h>

#define MAPS_BUF_SIZE (32*1024)

/* Sort an array of HeapSects by start address.                         */
/* Unfortunately at least some versions of                              */
/* Linux qsort end up calling malloc by way of sysconf, and hence can't */
/* be used in the collector.  Hence we roll our own.  Should be         */
/* reasonably fast if the array is already mostly sorted, as we expect  */
/* it to be.                                                            */
static void sort_heap_sects(struct HeapSect *base, size_t number_of_elements)
{
    signed_word n = (signed_word)number_of_elements;
    signed_word nsorted = 1;

    while (nsorted < n) {
      signed_word i;

      while (nsorted < n
             && ADDR_LT(base[nsorted-1].hs_start, base[nsorted].hs_start)) {
        ++nsorted;
      }
      if (nsorted == n) break;
      GC_ASSERT(ADDR_LT(base[nsorted].hs_start, base[nsorted-1].hs_start));
      for (i = nsorted - 1;
           i >= 0 && ADDR_LT(base[i+1].hs_start, base[i].hs_start); --i) {
        struct HeapSect tmp = base[i];

        base[i] = base[i+1];
        base[i+1] = tmp;
      }
      GC_ASSERT(ADDR_LT(base[nsorted-1].hs_start, base[nsorted].hs_start));
      ++nsorted;
    }
}

STATIC void GC_register_map_entries(const char *maps)
{
    const char *prot, *path;
    ptr_t start, end;
    ptr_t least_ha, greatest_ha;
    unsigned maj_dev;
    unsigned i;

    GC_ASSERT(I_HOLD_LOCK());
    sort_heap_sects(GC_our_memory, GC_n_memory);
    least_ha = GC_our_memory[0].hs_start;
    greatest_ha = GC_our_memory[GC_n_memory-1].hs_start
                  + GC_our_memory[GC_n_memory-1].hs_bytes;

    for (;;) {
        maps = GC_parse_map_entry(maps, &start, &end, &prot, &maj_dev, &path);
        if (NULL == maps) break;

        if (prot[1] == 'w') {
            /* This is a writable mapping.  Add it to           */
            /* the root set unless it is already otherwise      */
            /* accounted for.                                   */
#           ifndef THREADS
              if (ADDR_GE(GC_stackbottom, start)
                  && ADDR_GE(end, GC_stackbottom)) {
                /* Stack mapping; discard it.   */
                continue;
              }
#           endif
#           if defined(E2K) && defined(__ptr64__)
              /* TODO: avoid hard-coded addresses */
              if (ADDR(start) == 0xc2fffffff000UL
                  && ADDR(end) == 0xc30000000000UL && path[0] == '\n')
                continue; /* discard some special mapping */
#           endif
            if (path[0] == '[' && strncmp(path + 1, "heap]", 5) != 0)
              continue; /* discard if a pseudo-path unless "[heap]" */

#           ifdef THREADS
              /* This may fail, since a thread may already be           */
              /* unregistered, but its thread stack may still be there. */
              /* That can fail because the stack may disappear while    */
              /* we're marking.  Thus the marker is, and has to be      */
              /* prepared to recover from segmentation faults.          */

              if (GC_segment_is_thread_stack(start, end)) continue;

              /* FIXME: NPTL squirrels                                  */
              /* away pointers in pieces of the stack segment that we   */
              /* don't scan.  We work around this                       */
              /* by treating anything allocated by libpthread as        */
              /* uncollectible, as we do in some other cases.           */
              /* A specifically identified problem is that              */
              /* thread stacks contain pointers to dynamic thread       */
              /* vectors, which may be reused due to thread caching.    */
              /* They may not be marked if the thread is still live.    */
              /* This specific instance should be addressed by          */
              /* INCLUDE_LINUX_THREAD_DESCR, but that doesn't quite     */
              /* seem to suffice.                                       */
              /* We currently trace entire thread stacks, if they are   */
              /* are currently cached but unused.  This is              */
              /* very suboptimal for performance reasons.               */
#           endif
            /* We no longer exclude the main data segment.              */
            if (ADDR_GE(least_ha, end) || ADDR_GE(start, greatest_ha)) {
              /* The easy case; just trace the entire segment.  */
              GC_add_roots_inner(start, end, TRUE);
              continue;
            }
            /* Add sections that don't belong to us. */
              i = 0;
              while (ADDR_LT(GC_our_memory[i].hs_start
                             + GC_our_memory[i].hs_bytes, start)) {
                ++i;
              }
              GC_ASSERT(i < GC_n_memory);
              if (ADDR_GE(start, GC_our_memory[i].hs_start)) {
                start = GC_our_memory[i].hs_start + GC_our_memory[i].hs_bytes;
                ++i;
              }
              for (; i < GC_n_memory && ADDR_LT(start, end)
                     && ADDR_LT(GC_our_memory[i].hs_start, end); ++i) {
                if (ADDR_LT(start, GC_our_memory[i].hs_start))
                  GC_add_roots_inner(start,
                                     GC_our_memory[i].hs_start, TRUE);
                start = GC_our_memory[i].hs_start
                        + GC_our_memory[i].hs_bytes;
              }
              if (ADDR_LT(start, end))
                GC_add_roots_inner(start, end, TRUE);
        } else if (prot[0] == '-' && prot[1] == '-' && prot[2] == '-') {
            /* Even roots added statically might disappear partially    */
            /* (e.g. the roots added by INCLUDE_LINUX_THREAD_DESCR).    */
            GC_remove_roots_subregion(start, end);
        }
    }
}

GC_INNER void GC_register_dynamic_libraries(void)
{
    GC_register_map_entries(GC_get_maps());
}

/* We now take care of the main data segment ourselves: */
GC_INNER GC_bool GC_register_main_static_data(void)
{
    return FALSE;
}

# define HAVE_REGISTER_MAIN_STATIC_DATA

#else /* !USE_PROC_FOR_LIBRARIES */

/* The following is the preferred way to walk dynamic libraries */
/* for glibc 2.2.4+.  Unfortunately, it doesn't work for older  */
/* versions.  Thanks to Jakub Jelinek for most of the code.     */

#if GC_GLIBC_PREREQ(2, 3) || defined(HOST_ANDROID)
                        /* Are others OK here, too? */
# ifndef HAVE_DL_ITERATE_PHDR
#   define HAVE_DL_ITERATE_PHDR
# endif
# ifdef HOST_ANDROID
    /* Android headers might have no such definition for some targets.  */
    EXTERN_C_BEGIN
    extern int dl_iterate_phdr(int (*cb)(struct dl_phdr_info *,
                                         size_t, void *),
                               void *data);
    EXTERN_C_END
# endif
#endif /* __GLIBC__ >= 2 || HOST_ANDROID */

#if defined(__DragonFly__) || defined(__FreeBSD_kernel__) \
    || (defined(FREEBSD) && __FreeBSD__ >= 7)
  /* On the FreeBSD system, any target system at major version 7 shall   */
  /* have dl_iterate_phdr; therefore, we need not make it weak as below. */
# ifndef HAVE_DL_ITERATE_PHDR
#   define HAVE_DL_ITERATE_PHDR
# endif
# define DL_ITERATE_PHDR_STRONG
#elif defined(HAVE_DL_ITERATE_PHDR)
  /* We have the header files for a glibc that includes dl_iterate_phdr.*/
  /* It may still not be available in the library on the target system. */
  /* Thus we also treat it as a weak symbol.                            */
  EXTERN_C_BEGIN
# pragma weak dl_iterate_phdr
  EXTERN_C_END
#endif

#if defined(HAVE_DL_ITERATE_PHDR)

# ifdef PT_GNU_RELRO
/* Instead of registering PT_LOAD sections directly, we keep them       */
/* in a temporary list, and filter them by excluding PT_GNU_RELRO       */
/* segments.  Processing PT_GNU_RELRO sections with                     */
/* GC_exclude_static_roots instead would be superficially cleaner.  But */
/* it runs into trouble if a client registers an overlapping segment,   */
/* which unfortunately seems quite possible.                            */

#   define MAX_LOAD_SEGS MAX_ROOT_SETS

    static struct load_segment {
      ptr_t start;
      ptr_t end;
      /* Room for a second segment if we remove a RELRO segment */
      /* from the middle.                                       */
      ptr_t start2;
      ptr_t end2;
    } load_segs[MAX_LOAD_SEGS];

    static int n_load_segs;
    static GC_bool load_segs_overflow;
# endif /* PT_GNU_RELRO */

STATIC int GC_register_dynlib_callback(struct dl_phdr_info * info,
                                       size_t size, void * ptr)
{
  const ElfW(Phdr) * p;
  ptr_t start, end;
  int i;

  GC_ASSERT(I_HOLD_LOCK());
  /* Make sure struct dl_phdr_info is at least as big as we need.  */
  if (size < offsetof(struct dl_phdr_info, dlpi_phnum)
                + sizeof(info->dlpi_phnum))
    return -1;

  p = info->dlpi_phdr;
  for (i = 0; i < (int)info->dlpi_phnum; i++, p++) {
    if (p->p_type == PT_LOAD) {
      GC_has_static_roots_func callback = GC_has_static_roots;
      if ((p->p_flags & PF_W) == 0) continue;

      start = (ptr_t)p->p_vaddr + info->dlpi_addr;
      end = start + p->p_memsz;
      if (callback != 0 && !callback(info->dlpi_name, start, p->p_memsz))
        continue;
#     ifdef PT_GNU_RELRO
#       if CPP_PTRSZ >= 64
          /* TODO: GC_push_all eventually does the correct          */
          /* rounding to the next multiple of ALIGNMENT, so, most   */
          /* probably, we should remove the corresponding assertion */
          /* check in GC_add_roots_inner along with this code line. */
          /* start pointer value may require aligning.              */
          start = PTR_ALIGN_DOWN(start, sizeof(ptr_t));
#       endif
        if (n_load_segs >= MAX_LOAD_SEGS) {
          if (!load_segs_overflow) {
            WARN("Too many PT_LOAD segments;"
                 " registering as roots directly...\n", 0);
            load_segs_overflow = TRUE;
          }
          GC_add_roots_inner(start, end, TRUE);
        } else {
          load_segs[n_load_segs].start = start;
          load_segs[n_load_segs].end = end;
          load_segs[n_load_segs].start2 = NULL;
          load_segs[n_load_segs].end2 = NULL;
          ++n_load_segs;
        }
#     else
        GC_add_roots_inner(start, end, TRUE);
#     endif /* !PT_GNU_RELRO */
    }
  }

# ifdef PT_GNU_RELRO
    p = info->dlpi_phdr;
    for (i = 0; i < (int)info->dlpi_phnum; i++, p++) {
      if (p->p_type == PT_GNU_RELRO) {
        /* This entry is known to be constant and will eventually be    */
        /* remapped as read-only.  However, the address range covered   */
        /* by this entry is typically a subset of a previously          */
        /* encountered "LOAD" segment, so we need to exclude it.        */
        int j;

        start = (ptr_t)p->p_vaddr + info->dlpi_addr;
        end = start + p->p_memsz;
        for (j = n_load_segs; --j >= 0; ) {
          if (ADDR_INSIDE(start, load_segs[j].start, load_segs[j].end)) {
            if (load_segs[j].start2 != NULL) {
              WARN("More than one GNU_RELRO segment per load one\n", 0);
            } else {
              GC_ASSERT(ADDR_GE(PTR_ALIGN_UP(load_segs[j].end, GC_page_size),
                                end));
              /* Remove from the existing load segment. */
              load_segs[j].end2 = load_segs[j].end;
              load_segs[j].end = start;
              load_segs[j].start2 = end;
              /* Note that start2 may be greater than end2 because of   */
              /* p->p_memsz value multiple of page size.                */
            }
            break;
          }
          if (0 == j && 0 == GC_has_static_roots)
            WARN("Failed to find PT_GNU_RELRO segment"
                 " inside PT_LOAD region\n", 0);
            /* No warning reported in case of the callback is present   */
            /* because most likely the segment has been excluded.       */
        }
      }
    }
# endif

  *(int *)ptr = 1;     /* Signal that we were called */
  return 0;
}

/* Do we need to separately register the main static data segment? */
GC_INNER GC_bool GC_register_main_static_data(void)
{
# if defined(DL_ITERATE_PHDR_STRONG) && !defined(CPPCHECK)
    /* If dl_iterate_phdr is not a weak symbol then don't test against  */
    /* zero (otherwise a compiler might issue a warning).               */
    return FALSE;
# else
    return 0 == COVERT_DATAFLOW(ADDR(dl_iterate_phdr));
# endif
}

/* Return TRUE if we succeed, FALSE if dl_iterate_phdr wasn't there. */
STATIC GC_bool GC_register_dynamic_libraries_dl_iterate_phdr(void)
{
  int did_something;

  GC_ASSERT(I_HOLD_LOCK());
  if (GC_register_main_static_data())
    return FALSE;

# ifdef PT_GNU_RELRO
    {
      static GC_bool excluded_segs = FALSE;
      n_load_segs = 0;
      load_segs_overflow = FALSE;
      if (!EXPECT(excluded_segs, TRUE)) {
        GC_exclude_static_roots_inner((ptr_t)load_segs,
                                      (ptr_t)load_segs + sizeof(load_segs));
        excluded_segs = TRUE;
      }
    }
# endif

  did_something = 0;
  dl_iterate_phdr(GC_register_dynlib_callback, &did_something);
  if (did_something) {
#   ifdef PT_GNU_RELRO
      int i;

      for (i = 0; i < n_load_segs; ++i) {
        if (ADDR_LT(load_segs[i].start, load_segs[i].end))
          GC_add_roots_inner(load_segs[i].start, load_segs[i].end, TRUE);
        if (ADDR_LT(load_segs[i].start2, load_segs[i].end2))
          GC_add_roots_inner(load_segs[i].start2, load_segs[i].end2, TRUE);
      }
#   endif
  } else {
      ptr_t datastart, dataend;
#     ifdef DATASTART_IS_FUNC
        static ptr_t datastart_cached = MAKE_CPTR(GC_WORD_MAX);

        /* Evaluate DATASTART only once.  */
        if (ADDR(datastart_cached) == GC_WORD_MAX) {
          datastart_cached = DATASTART;
        }
        datastart = datastart_cached;
#     else
        datastart = DATASTART;
#     endif
#     ifdef DATAEND_IS_FUNC
        {
          static ptr_t dataend_cached = 0;
          /* Evaluate DATAEND only once. */
          if (dataend_cached == 0) {
            dataend_cached = DATAEND;
          }
          dataend = dataend_cached;
        }
#     else
        dataend = DATAEND;
#     endif
      if (NULL == *(char * volatile *)&datastart
          || ADDR_LT(dataend, datastart))
        ABORT_ARG2("Wrong DATASTART/END pair",
                   ": %p .. %p", (void *)datastart, (void *)dataend);

      /* dl_iterate_phdr may forget the static data segment in  */
      /* statically linked executables.                         */
      GC_add_roots_inner(datastart, dataend, TRUE);
#     ifdef GC_HAVE_DATAREGION2
        if (ADDR(DATASTART2) - 1U >= ADDR(DATAEND2)) {
                        /* Subtract one to check also for NULL  */
                        /* without a compiler warning.          */
          ABORT_ARG2("Wrong DATASTART/END2 pair",
                     ": %p .. %p", (void *)DATASTART2, (void *)DATAEND2);
        }
        GC_add_roots_inner(DATASTART2, DATAEND2, TRUE);
#     endif
  }
  return TRUE;
}

# define HAVE_REGISTER_MAIN_STATIC_DATA

#else /* !HAVE_DL_ITERATE_PHDR */

/* Dynamic loading code for Linux running ELF. Somewhat tested on
 * Linux/i686, untested but hopefully should work on Linux/Alpha.
 * This code was derived from the Solaris/ELF support. Thanks to
 * whatever kind soul wrote that.  - Patrick Bridges */

/* This doesn't necessarily work in all cases, e.g. with preloaded
 * dynamic libraries.                                           */

# if defined(NETBSD) || defined(OPENBSD)
#   include <sys/exec_elf.h>
   /* for compatibility with 1.4.x */
#   ifndef DT_DEBUG
#     define DT_DEBUG   21
#   endif
#   ifndef PT_LOAD
#     define PT_LOAD    1
#   endif
#   ifndef PF_W
#     define PF_W       2
#   endif
# elif !defined(HOST_ANDROID)
#   include <elf.h>
# endif

# ifndef HOST_ANDROID
#   include <link.h>
# endif

#endif /* !HAVE_DL_ITERATE_PHDR */

EXTERN_C_BEGIN
#ifdef __GNUC__
# pragma weak _DYNAMIC
#endif
extern ElfW(Dyn) _DYNAMIC[];
EXTERN_C_END

STATIC struct link_map *
GC_FirstDLOpenedLinkMap(void)
{
    static struct link_map *cachedResult = 0;

    if (0 == COVERT_DATAFLOW(ADDR(_DYNAMIC))) {
        /* _DYNAMIC symbol not resolved. */
        return NULL;
    }
    if (NULL == cachedResult) {
#     if defined(NETBSD) && defined(RTLD_DI_LINKMAP)
#       if defined(CPPCHECK)
#         define GC_RTLD_DI_LINKMAP 2
#       else
#         define GC_RTLD_DI_LINKMAP RTLD_DI_LINKMAP
#       endif
        struct link_map *lm = NULL;
        if (!dlinfo(RTLD_SELF, GC_RTLD_DI_LINKMAP, &lm) && lm != NULL) {
            /* Now lm points link_map object of libgc.  Since it    */
            /* might not be the first dynamically linked object,    */
            /* try to find it (object next to the main object).     */
            while (lm->l_prev != NULL) {
                lm = lm->l_prev;
            }
            cachedResult = lm->l_next;
        }
#     else
        ElfW(Dyn) *dp;
        int tag;

        for (dp = _DYNAMIC; (tag = dp->d_tag) != 0; dp++) {
            if (tag == DT_DEBUG) {
                const struct r_debug *rd = (struct r_debug *)dp->d_un.d_ptr;
                /* d_ptr could be null if libs are linked statically. */
                if (rd != NULL) {
                    const struct link_map *lm = rd->r_map;
                    if (lm != NULL)
                        cachedResult = lm->l_next; /* might be NULL */
                }
                break;
            }
        }
#     endif /* !NETBSD || !RTLD_DI_LINKMAP */
    }
    return cachedResult;
}

GC_INNER void GC_register_dynamic_libraries(void)
{
  struct link_map *lm;

  GC_ASSERT(I_HOLD_LOCK());
# ifdef HAVE_DL_ITERATE_PHDR
    if (GC_register_dynamic_libraries_dl_iterate_phdr()) {
        return;
    }
# endif
  for (lm = GC_FirstDLOpenedLinkMap(); lm != 0; lm = lm->l_next)
    {
        ElfW(Ehdr) * e;
        ElfW(Phdr) * p;
        unsigned long offset;
        char * start;
        int i;

        e = (ElfW(Ehdr) *)lm->l_addr;
#       ifdef HOST_ANDROID
          if (e == NULL)
            continue;
#       endif
        p = ((ElfW(Phdr) *)(((char *)(e)) + e->e_phoff));
        offset = ((unsigned long)(lm->l_addr));
        for (i = 0; i < (int)e->e_phnum; i++, p++) {
          switch (p->p_type) {
            case PT_LOAD:
              if (!(p->p_flags & PF_W)) break;
              start = (char *)(p->p_vaddr) + offset;
              GC_add_roots_inner(start, start + p->p_memsz, TRUE);
              break;
            default:
              break;
          }
        }
#       if defined(CPPCHECK) && defined(HOST_ANDROID) \
           && !defined(GC_DONT_DEFINE_LINK_MAP) && !(__ANDROID_API__ >= 21)
          GC_noop1_ptr(lm -> l_name);
          GC_noop1((word)(lm -> l_ld));
          GC_noop1_ptr(lm -> l_prev);
#       endif
    }
}

#endif /* !USE_PROC_FOR_LIBRARIES */

#endif /* LINUX */

#if defined(IRIX5) || (defined(USE_PROC_FOR_LIBRARIES) && !defined(LINUX))

#include <sys/procfs.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <elf.h>
#include <errno.h>
#include <signal.h>  /* Only for the following test. */
#ifndef _sigargs
# define IRIX6
#endif

/* We use /proc to track down all parts of the address space that are   */
/* mapped by the process, and throw out regions we know we shouldn't    */
/* worry about.  This may also work under other SVR4 variants.          */
GC_INNER void GC_register_dynamic_libraries(void)
{
    static int fd = -1;
    static prmap_t * addr_map = 0;
    static int current_sz = 0;  /* Number of records currently in addr_map */

    char buf[32];
    int needed_sz = 0;          /* Required size of addr_map            */
    int i;
    long flags;
    ptr_t start;
    ptr_t limit;
    word heap_start = ADDR(HEAP_START);
    word heap_end = heap_start;

#   ifdef SOLARISDL
#     define MA_PHYS 0
#   endif /* SOLARISDL */

    GC_ASSERT(I_HOLD_LOCK());
    if (fd < 0) {
      (void)snprintf(buf, sizeof(buf), "/proc/%ld", (long)getpid());
      buf[sizeof(buf) - 1] = '\0';
      fd = open(buf, O_RDONLY);
      if (fd < 0) {
        ABORT("/proc open failed");
      }
    }
    if (ioctl(fd, PIOCNMAP, &needed_sz) < 0) {
        ABORT_ARG2("/proc PIOCNMAP ioctl failed",
                   ": fd= %d, errno= %d", fd, errno);
    }
    if (needed_sz >= current_sz) {
        GC_scratch_recycle_no_gww(addr_map,
                                  (size_t)current_sz * sizeof(prmap_t));
        current_sz = needed_sz * 2 + 1;
                        /* Expansion, plus room for 0 record */
        addr_map = (prmap_t *)GC_scratch_alloc(
                                (size_t)current_sz * sizeof(prmap_t));
        if (addr_map == NULL)
          ABORT("Insufficient memory for address map");
    }
    if (ioctl(fd, PIOCMAP, addr_map) < 0) {
        ABORT_ARG3("/proc PIOCMAP ioctl failed",
                   ": errcode= %d, needed_sz= %d, addr_map= %p",
                   errno, needed_sz, (void *)addr_map);
    }
    if (GC_n_heap_sects > 0) {
        heap_end = ADDR(GC_heap_sects[GC_n_heap_sects-1].hs_start)
                        + GC_heap_sects[GC_n_heap_sects-1].hs_bytes;
        if (heap_end < GC_scratch_last_end_addr)
          heap_end = GC_scratch_last_end_addr;
    }
    for (i = 0; i < needed_sz; i++) {
        flags = addr_map[i].pr_mflags;
        if ((flags & (MA_BREAK | MA_STACK | MA_PHYS
                      | MA_FETCHOP | MA_NOTCACHED)) != 0) goto irrelevant;
        if ((flags & (MA_READ | MA_WRITE)) != (MA_READ | MA_WRITE))
            goto irrelevant;
          /* The latter test is empirically useless in very old Irix    */
          /* versions.  Other than the                                  */
          /* main data and stack segments, everything appears to be     */
          /* mapped readable, writable, executable, and shared(!!).     */
          /* This makes no sense to me. - HB                            */
        start = (ptr_t)(addr_map[i].pr_vaddr);
        if (GC_roots_present(start)
            || (ADDR(start) >= heap_start && ADDR(start) < heap_end))
          goto irrelevant;

        limit = start + addr_map[i].pr_size;
        /* The following seemed to be necessary for very old versions   */
        /* of Irix, but it has been reported to discard relevant        */
        /* segments under Irix 6.5.                                     */
#       ifndef IRIX6
          if (addr_map[i].pr_off == 0 && strncmp(start, ELFMAG, 4) == 0) {
            /* Discard text segments, i.e. 0-offset mappings against    */
            /* executable files which appear to have ELF headers.       */
            caddr_t arg;
            int obj;
#           define MAP_IRR_SZ 10
            static ptr_t map_irr[MAP_IRR_SZ];
                                        /* Known irrelevant map entries */
            static int n_irr = 0;
            struct stat buf;
            int j;

            for (j = 0; j < n_irr; j++) {
                if (map_irr[j] == start) goto irrelevant;
            }
            arg = (caddr_t)start;
            obj = ioctl(fd, PIOCOPENM, &arg);
            if (obj >= 0) {
                fstat(obj, &buf);
                close(obj);
                if ((buf.st_mode & 0111) != 0) {
                    if (n_irr < MAP_IRR_SZ) {
                        map_irr[n_irr++] = start;
                    }
                    goto irrelevant;
                }
            }
          }
#       endif /* !IRIX6 */
        GC_add_roots_inner(start, limit, TRUE);
      irrelevant: ;
    }
    /* Don't keep cached descriptor, for now.  Some kernels don't like us */
    /* to keep a /proc file descriptor around during kill -9.             */
    /* Otherwise, it should also require FD_CLOEXEC and proper handling   */
    /* at fork (i.e. close because of the pid change).                    */
        if (close(fd) < 0) ABORT("Couldn't close /proc file");
        fd = -1;
}

# endif /* USE_PROC_FOR_LIBRARIES || IRIX5 */

#ifdef ANY_MSWIN
  /* We traverse the entire address space and register all segments     */
  /* that could possibly have been written to.                          */
  STATIC void GC_cond_add_roots(ptr_t base, ptr_t limit)
  {
#   ifdef THREADS
      ptr_t curr_base = base;
      ptr_t next_stack_lo, next_stack_hi;
#   else
      ptr_t stack_top;
#   endif

    GC_ASSERT(I_HOLD_LOCK());
    if (base == limit) return;
#   ifdef THREADS
      for (;;) {
          GC_get_next_stack(curr_base, limit, &next_stack_lo, &next_stack_hi);
          if (ADDR_GE(next_stack_lo, limit)) break;
          if (ADDR_LT(curr_base, next_stack_lo))
            GC_add_roots_inner(curr_base, next_stack_lo, TRUE);
          curr_base = next_stack_hi;
      }
      if (ADDR_LT(curr_base, limit))
        GC_add_roots_inner(curr_base, limit, TRUE);
#   else
      stack_top = PTR_ALIGN_DOWN(GC_approx_sp(),
                                 GC_sysinfo.dwAllocationGranularity);
      if (ADDR_LT(stack_top, limit) && ADDR_LT(base, GC_stackbottom)) {
          /* Part of the stack; ignore it. */
          return;
      }
      GC_add_roots_inner(base, limit, TRUE);
#   endif
  }

#ifdef DYNAMIC_LOADING
  /* GC_register_main_static_data is not needed unless DYNAMIC_LOADING. */
  GC_INNER GC_bool GC_register_main_static_data(void)
  {
#   if defined(MSWINCE) || defined(CYGWIN32)
      /* Do we need to separately register the main static data segment? */
      return FALSE;
#   else
      return GC_no_win32_dlls;
#   endif
  }
# define HAVE_REGISTER_MAIN_STATIC_DATA
#endif /* DYNAMIC_LOADING */

# ifdef DEBUG_VIRTUALQUERY
  void GC_dump_meminfo(MEMORY_BASIC_INFORMATION *buf)
  {
    GC_printf("BaseAddress= 0x%lx, AllocationBase= 0x%lx,"
              " RegionSize= 0x%lx(%lu)\n",
              buf -> BaseAddress, buf -> AllocationBase,
              buf -> RegionSize, buf -> RegionSize);
    GC_printf("\tAllocationProtect= 0x%lx, State= 0x%lx, Protect= 0x%lx, "
              "Type= 0x%lx\n", buf -> AllocationProtect, buf -> State,
              buf -> Protect, buf -> Type);
  }
# endif /* DEBUG_VIRTUALQUERY */

# if defined(MSWINCE) || defined(CYGWIN32)
    /* FIXME: Should we really need to scan MEM_PRIVATE sections?       */
    /* For now, we don't add MEM_PRIVATE sections to the data roots for */
    /* WinCE because otherwise SEGV fault sometimes happens to occur in */
    /* GC_mark_from() (and, even if we use WRAP_MARK_SOME, WinCE prints */
    /* a "Data Abort" message to the debugging console).                */
    /* To workaround that, use -DGC_REGISTER_MEM_PRIVATE.               */
#   define GC_wnt TRUE
# endif

  GC_INNER void GC_register_dynamic_libraries(void)
  {
    ptr_t p, base, limit;

    GC_ASSERT(I_HOLD_LOCK());
#   ifdef MSWIN32
      if (GC_no_win32_dlls) return;
#   endif
    p = (ptr_t)GC_sysinfo.lpMinimumApplicationAddress;
    base = limit = p;
    while (ADDR_LT(p, (ptr_t)GC_sysinfo.lpMaximumApplicationAddress)) {
      MEMORY_BASIC_INFORMATION buf;
      size_t result = VirtualQuery((LPVOID)p, &buf, sizeof(buf));

#     ifdef MSWINCE
        if (0 == result) {
          if (ADDR(p) > GC_WORD_MAX - GC_sysinfo.dwAllocationGranularity)
            break; /* overflow */
          /* Page is free; advance to the next possible allocation base. */
          p = PTR_ALIGN_UP(p + 1, GC_sysinfo.dwAllocationGranularity);
        } else
#     endif
      /* else */ {
        DWORD protect;

        if (result != sizeof(buf))
          ABORT("Weird VirtualQuery result");
        if (ADDR(p) > GC_WORD_MAX - buf.RegionSize) break; /* overflow */

        protect = buf.Protect;
        if (buf.State == MEM_COMMIT
            && (protect == PAGE_EXECUTE_READWRITE
                || protect == PAGE_EXECUTE_WRITECOPY
                || protect == PAGE_READWRITE
                || protect == PAGE_WRITECOPY)
            && (buf.Type == MEM_IMAGE
#               ifdef GC_REGISTER_MEM_PRIVATE
                  || (protect == PAGE_READWRITE && buf.Type == MEM_PRIVATE)
#               else
                  /* There is some evidence that we cannot always   */
                  /* ignore MEM_PRIVATE sections under Windows ME   */
                  /* and predecessors.  Hence we now also check for */
                  /* that case.                                     */
                  || (!GC_wnt && buf.Type == MEM_PRIVATE)
#               endif
               )
            && !GC_is_heap_base(buf.AllocationBase)) {
#         ifdef DEBUG_VIRTUALQUERY
            GC_dump_meminfo(&buf);
#         endif
          if (p != limit) {
            GC_cond_add_roots(base, limit);
            base = p;
          }
          limit = p + buf.RegionSize;
        }
        p += buf.RegionSize;
      }
    }
    GC_cond_add_roots(base, limit);
  }
#endif /* ANY_MSWIN */

#if defined(ALPHA) && defined(OSF1)
# include <loader.h>

  EXTERN_C_BEGIN
  extern char *sys_errlist[];
  extern int sys_nerr;
  extern int errno;
  EXTERN_C_END

  GC_INNER void GC_register_dynamic_libraries(void)
  {
    ldr_module_t moduleid = LDR_NULL_MODULE;
    ldr_process_t mypid;

    GC_ASSERT(I_HOLD_LOCK());
    mypid = ldr_my_process(); /* obtain id of this process */

    /* For each module. */
    for (;;) {
      ldr_module_info_t moduleinfo;
      size_t modulereturnsize;
      ldr_region_t region;
      ldr_region_info_t regioninfo;
      size_t regionreturnsize;
      int status = ldr_next_module(mypid, &moduleid);
                                /* Get the next (first) module */

      if (moduleid == LDR_NULL_MODULE)
        break;  /* no more modules */

      /* Check status AFTER checking moduleid because       */
      /* of a bug in the non-shared ldr_next_module stub.   */
        if (status != 0) {
          ABORT_ARG3("ldr_next_module failed",
                     ": status= %d, errcode= %d (%s)", status, errno,
                     errno < sys_nerr ? sys_errlist[errno] : "");
        }

      /* Get the module information */
        status = ldr_inq_module(mypid, moduleid, &moduleinfo,
                                sizeof(moduleinfo), &modulereturnsize);
        if (status != 0 )
            ABORT("ldr_inq_module failed");

      /* is module for the main program (i.e. nonshared portion)? */
          if (moduleinfo.lmi_flags & LDR_MAIN)
              continue;    /* skip the main module */

#     ifdef DL_VERBOSE
        GC_log_printf("---Module---\n");
        GC_log_printf("Module ID: %ld\n", moduleinfo.lmi_modid);
        GC_log_printf("Count of regions: %d\n", moduleinfo.lmi_nregion);
        GC_log_printf("Flags for module: %016lx\n", moduleinfo.lmi_flags);
        GC_log_printf("Module pathname: \"%s\"\n", moduleinfo.lmi_name);
#     endif

      /* For each region in this module. */
        for (region = 0; region < moduleinfo.lmi_nregion; region++) {
          /* Get the region information */
            status = ldr_inq_region(mypid, moduleid, region, &regioninfo,
                                    sizeof(regioninfo), &regionreturnsize);
            if (status != 0 )
                ABORT("ldr_inq_region failed");

          /* only process writable (data) regions */
            if (! (regioninfo.lri_prot & LDR_W))
                continue;

#         ifdef DL_VERBOSE
            GC_log_printf("--- Region ---\n");
            GC_log_printf("Region number: %ld\n", regioninfo.lri_region_no);
            GC_log_printf("Protection flags: %016x\n", regioninfo.lri_prot);
            GC_log_printf("Virtual address: %p\n", regioninfo.lri_vaddr);
            GC_log_printf("Mapped address: %p\n", regioninfo.lri_mapaddr);
            GC_log_printf("Region size: %ld\n", regioninfo.lri_size);
            GC_log_printf("Region name: \"%s\"\n", regioninfo.lri_name);
#         endif

          /* register region as a garbage collection root */
          GC_add_roots_inner((char *)regioninfo.lri_mapaddr,
                        (char *)regioninfo.lri_mapaddr + regioninfo.lri_size,
                        TRUE);

        }
    }
  }
#endif /* ALPHA && OSF1 */

#if defined(HPUX)
#include <errno.h>
#include <dl.h>

EXTERN_C_BEGIN
extern char *sys_errlist[];
extern int sys_nerr;
EXTERN_C_END

GC_INNER void GC_register_dynamic_libraries(void)
{
  int index = 1; /* Ordinal position in shared library search list */

  GC_ASSERT(I_HOLD_LOCK());
  /* For each dynamic library loaded. */
  for (;;) {
      struct shl_descriptor *shl_desc; /* Shared library info, see dl.h */
      int status = shl_get(index, &shl_desc);
                                /* Get info about next shared library   */

      /* Check if this is the end of the list or if some error occurred */
        if (status != 0) {
#        ifdef GC_HPUX_THREADS
           /* I've seen errno values of 0.  The man page is not clear   */
           /* as to whether errno should get set on a -1 return.        */
           break;
#        else
          if (errno == EINVAL) {
            break; /* Moved past end of shared library list --> finished */
          } else {
            ABORT_ARG3("shl_get failed",
                       ": status= %d, errcode= %d (%s)", status, errno,
                       errno < sys_nerr ? sys_errlist[errno] : "");
          }
#        endif
        }

#     ifdef DL_VERBOSE
        GC_log_printf("---Shared library---\n");
        GC_log_printf("filename= \"%s\"\n", shl_desc->filename);
        GC_log_printf("index= %d\n", index);
        GC_log_printf("handle= %08x\n", (unsigned long) shl_desc->handle);
        GC_log_printf("text seg.start= %08x\n", shl_desc->tstart);
        GC_log_printf("text seg.end= %08x\n", shl_desc->tend);
        GC_log_printf("data seg.start= %08x\n", shl_desc->dstart);
        GC_log_printf("data seg.end= %08x\n", shl_desc->dend);
        GC_log_printf("ref.count= %lu\n", shl_desc->ref_count);
#     endif

      /* register shared library's data segment as a garbage collection root */
        GC_add_roots_inner((char *) shl_desc->dstart,
                           (char *) shl_desc->dend, TRUE);

        index++;
  }
}
#endif /* HPUX */

#ifdef AIX
# include <alloca.h>
# include <sys/ldr.h>
# include <sys/errno.h>

  GC_INNER void GC_register_dynamic_libraries(void)
  {
      int ldibuflen = 8192;

      GC_ASSERT(I_HOLD_LOCK());
      for (;;) {
        int len;
        struct ld_info *ldi;
#       if defined(CPPCHECK)
          char ldibuf[ldibuflen];
#       else
          char *ldibuf = alloca(ldibuflen);
#       endif

        len = loadquery(L_GETINFO, ldibuf, ldibuflen);
        if (len < 0) {
                if (errno != ENOMEM) {
                        ABORT("loadquery failed");
                }
                ldibuflen *= 2;
                continue;
        }

        ldi = (struct ld_info *)ldibuf;
        while (ldi) {
                len = ldi->ldinfo_next;
                GC_add_roots_inner(
                                ldi->ldinfo_dataorg,
                                (ptr_t)(unsigned long)ldi->ldinfo_dataorg
                                + ldi->ldinfo_datasize,
                                TRUE);
                ldi = len ? (struct ld_info *)((char *)ldi + len) : 0;
        }
        break;
      }
  }
#endif /* AIX */

#ifdef DARWIN

/* __private_extern__ hack required for pre-3.4 gcc versions.   */
#ifndef __private_extern__
# define __private_extern__ extern
# include <mach-o/dyld.h>
# undef __private_extern__
#else
# include <mach-o/dyld.h>
#endif

#if CPP_WORDSZ == 64
# define GC_MACH_HEADER mach_header_64
#else
# define GC_MACH_HEADER mach_header
#endif

#ifdef MISSING_MACH_O_GETSECT_H
  EXTERN_C_BEGIN
  extern uint8_t *getsectiondata(const struct GC_MACH_HEADER *,
                    const char *seg, const char *sect, unsigned long *psz);
  EXTERN_C_END
#else
# include <mach-o/getsect.h>
#endif

/*#define DARWIN_DEBUG*/

/* Writable sections generally available on Darwin.     */
STATIC const struct dyld_sections_s {
    const char *seg;
    const char *sect;
} GC_dyld_sections[] = {
    { SEG_DATA, SECT_DATA },
    /* Used by FSF GCC, but not by OS X system tools, so far.   */
    { SEG_DATA, "__static_data" },
    { SEG_DATA, SECT_BSS },
    { SEG_DATA, SECT_COMMON },
    /* FSF GCC - zero-sized object sections for targets         */
    /*supporting section anchors.                               */
    { SEG_DATA, "__zobj_data" },
    { SEG_DATA, "__zobj_bss" }
};

/* Additional writable sections:                                */
/* GCC on Darwin constructs aligned sections "on demand", where */
/* the alignment size is embedded in the section name.          */
/* Furthermore, there are distinctions between sections         */
/* containing private vs. public symbols.  It also constructs   */
/* sections specifically for zero-sized objects, when the       */
/* target supports section anchors.                             */
STATIC const char * const GC_dyld_bss_prefixes[] = {
  "__bss",
  "__pu_bss",
  "__zo_bss",
  "__zo_pu_bss"
};

/* Currently, mach-o will allow up to the max of 2^15 alignment */
/* in an object file.                                           */
#ifndef L2_MAX_OFILE_ALIGNMENT
# define L2_MAX_OFILE_ALIGNMENT 15
#endif

STATIC const char *GC_dyld_name_for_hdr(const struct GC_MACH_HEADER *phdr)
{
    unsigned long i, count = _dyld_image_count();

    for (i = 0; i < count; i++) {
      if ((const struct GC_MACH_HEADER *)_dyld_get_image_header(i) == phdr)
        return _dyld_get_image_name(i);
    }
    /* TODO: probably ABORT in this case? */
    return NULL; /* not found */
}

/* getsectbynamefromheader is deprecated (first time in macOS 13.0),    */
/* getsectiondata (introduced in macOS 10.7) is used instead if exists. */
/* Define USE_GETSECTBYNAME to use the deprecated symbol, if needed.    */
#if !defined(USE_GETSECTBYNAME) \
    && (MAC_OS_X_VERSION_MIN_REQUIRED < 1070 /*MAC_OS_X_VERSION_10_7*/)
# define USE_GETSECTBYNAME
#endif

static void dyld_section_add_del(const struct GC_MACH_HEADER *phdr,
                                 intptr_t slide, const char *dlpi_name,
                                 GC_has_static_roots_func callback,
                                 const char *seg, const char *secnam,
                                 GC_bool is_add)
{
  unsigned long start, end, sec_size;
# ifdef USE_GETSECTBYNAME
#   if CPP_WORDSZ == 64
      const struct section_64 *sec = getsectbynamefromheader_64(phdr, seg,
                                                                secnam);
#   else
      const struct section *sec = getsectbynamefromheader(phdr, seg, secnam);
#   endif

    if (NULL == sec) return;
    sec_size = sec -> size;
    start = slide + sec -> addr;
# else

    UNUSED_ARG(slide);
    sec_size = 0;
    start = (unsigned long)getsectiondata(phdr, seg, secnam, &sec_size);
    if (0 == start) return;
# endif
  if (sec_size < sizeof(ptr_t)) return;
  end = start + sec_size;
  if (is_add) {
    LOCK();
    /* The user callback is invoked holding the allocator lock.   */
    if (EXPECT(callback != 0, FALSE)
        && !callback(dlpi_name, (void *)start, (size_t)sec_size)) {
      UNLOCK();
      return; /* skip section */
    }
    GC_add_roots_inner((ptr_t)start, (ptr_t)end, FALSE);
    UNLOCK();
  } else {
    GC_remove_roots((void *)start, (void *)end);
  }
# ifdef DARWIN_DEBUG
    GC_log_printf("%s section __DATA,%s at %p-%p (%lu bytes) from image %s\n",
                  is_add ? "Added" : "Removed",
                  secnam, (void *)start, (void *)end, sec_size, dlpi_name);
# endif
}

static void dyld_image_add_del(const struct GC_MACH_HEADER *phdr,
                               intptr_t slide,
                               GC_has_static_roots_func callback,
                               GC_bool is_add)
{
  unsigned i, j;
  const char *dlpi_name;

  GC_ASSERT(I_DONT_HOLD_LOCK());
# ifndef DARWIN_DEBUG
    if (0 == callback) {
      dlpi_name = NULL;
    } else
# endif
  /* else */ {
    dlpi_name = GC_dyld_name_for_hdr(phdr);
  }
  for (i = 0; i < sizeof(GC_dyld_sections)/sizeof(GC_dyld_sections[0]); i++) {
    dyld_section_add_del(phdr, slide, dlpi_name, callback,
                         GC_dyld_sections[i].seg, GC_dyld_sections[i].sect,
                         is_add);
  }

  /* Sections constructed on demand.    */
  for (j = 0; j < sizeof(GC_dyld_bss_prefixes) / sizeof(char *); j++) {
    /* Our manufactured aligned BSS sections.   */
    for (i = 0; i <= L2_MAX_OFILE_ALIGNMENT; i++) {
      char secnam[16];

      (void)snprintf(secnam, sizeof(secnam), "%s%u",
                     GC_dyld_bss_prefixes[j], i);
      secnam[sizeof(secnam) - 1] = '\0';
      dyld_section_add_del(phdr, slide, dlpi_name, 0 /* callback */, SEG_DATA,
                           secnam, is_add);
    }
  }

# if defined(DARWIN_DEBUG) && !defined(NO_DEBUGGING)
    READER_LOCK();
    GC_print_static_roots();
    READER_UNLOCK();
# endif
}

STATIC void GC_dyld_image_add(const struct GC_MACH_HEADER *phdr,
                              intptr_t slide)
{
  if (!GC_no_dls)
    dyld_image_add_del(phdr, slide, GC_has_static_roots, TRUE);
}

STATIC void GC_dyld_image_remove(const struct GC_MACH_HEADER *phdr,
                                 intptr_t slide)
{
  dyld_image_add_del(phdr, slide, 0 /* callback */, FALSE);
}

GC_INNER void GC_register_dynamic_libraries(void)
{
    /* Currently does nothing. The callbacks are setup by GC_init_dyld()
    The dyld library takes it from there. */
}

/* The _dyld_* functions have an internal lock, so none of them can be  */
/* called while the world is stopped without the risk of a deadlock.    */
/* Because of this we MUST setup callbacks BEFORE we ever stop the      */
/* world.  This should be called BEFORE any thread is created and       */
/* WITHOUT the allocator lock held.                                     */

/* _dyld_bind_fully_image_containing_address is deprecated, so use      */
/* dlopen(0,RTLD_NOW) instead; define USE_DYLD_TO_BIND to override this */
/* if needed.                                                           */

GC_INNER void GC_init_dyld(void)
{
  static GC_bool initialized = FALSE;

  GC_ASSERT(I_DONT_HOLD_LOCK());
  if (initialized) return;

# ifdef DARWIN_DEBUG
    GC_log_printf("Registering dyld callbacks...\n");
# endif

  /* Apple's Documentation:
     When you call _dyld_register_func_for_add_image, the dynamic linker
     runtime calls the specified callback (func) once for each of the images
     that is currently loaded into the program. When a new image is added to
     the program, your callback is called again with the mach_header for the
     new image, and the virtual memory slide amount of the new image.

     This WILL properly register already linked libraries and libraries
     linked in the future.
  */
  _dyld_register_func_for_add_image(
        (void (*)(const struct mach_header*, intptr_t))GC_dyld_image_add);
  _dyld_register_func_for_remove_image(
        (void (*)(const struct mach_header*, intptr_t))GC_dyld_image_remove);
                        /* Structure mach_header_64 has the same fields */
                        /* as mach_header except for the reserved one   */
                        /* at the end, so these casts are OK.           */

  /* Set this early to avoid reentrancy issues. */
  initialized = TRUE;

# ifndef NO_DYLD_BIND_FULLY_IMAGE
    if (GC_no_dls) return; /* skip main data segment registration */

    /* When the environment variable is set, the dynamic linker binds   */
    /* all undefined symbols the application needs at launch time.      */
    /* This includes function symbols that are normally bound lazily at */
    /* the time of their first invocation.                              */
    if (GETENV("DYLD_BIND_AT_LAUNCH") != NULL) return;

    /* The environment variable is unset, so we should bind manually.   */
#   ifdef DARWIN_DEBUG
      GC_log_printf("Forcing full bind of GC code...\n");
#   endif
#   ifndef USE_DYLD_TO_BIND
      {
        void *dl_handle = dlopen(NULL, RTLD_NOW);

        if (!dl_handle)
          ABORT("dlopen failed (to bind fully image)");
        /* Note that the handle is never closed.        */
#       if defined(CPPCHECK) || defined(LINT2)
          GC_noop1_ptr(dl_handle);
#       endif
      }
#   else
      /* Note: '_dyld_bind_fully_image_containing_address' is deprecated. */
      if (!_dyld_bind_fully_image_containing_address(
                                                  (unsigned long *)GC_malloc))
        ABORT("_dyld_bind_fully_image_containing_address failed");
#   endif
# endif
}

#define HAVE_REGISTER_MAIN_STATIC_DATA
GC_INNER GC_bool GC_register_main_static_data(void)
{
  /* Already done through dyld callbacks */
  return FALSE;
}

#endif /* DARWIN */

#if defined(HAIKU)
# include <kernel/image.h>

  GC_INNER void GC_register_dynamic_libraries(void)
  {
    image_info info;
    int32 cookie = 0;

    GC_ASSERT(I_HOLD_LOCK());
    while (get_next_image_info(0, &cookie, &info) == B_OK) {
      ptr_t data = (ptr_t)info.data;
      GC_add_roots_inner(data, data + info.data_size, TRUE);
    }
  }
#endif /* HAIKU */

#elif defined(PCR)








  GC_INNER void GC_register_dynamic_libraries(void)
  {
    /* Add new static data areas of dynamically loaded modules. */
    const PCR_IL_LoadedFile * p = PCR_IL_GetLastLoadedFile();
    PCR_IL_LoadedSegment * q;

    /* Skip uncommitted files */
    while (p != NIL && !(p -> lf_commitPoint)) {
        /* The loading of this file has not yet been committed    */
        /* Hence its description could be inconsistent.           */
        /* Furthermore, it hasn't yet been run.  Hence its data   */
        /* segments can't possibly reference heap allocated       */
        /* objects.                                               */
        p = p -> lf_prev;
    }
    for (; p != NIL; p = p -> lf_prev) {
      for (q = p -> lf_ls; q != NIL; q = q -> ls_next) {
        if ((q -> ls_flags & PCR_IL_SegFlags_Traced_MASK)
            == PCR_IL_SegFlags_Traced_on) {
          GC_add_roots_inner((ptr_t)q->ls_addr,
                             (ptr_t)q->ls_addr + q->ls_bytes, TRUE);
        }
      }
    }
  }

#endif /* PCR && !ANY_MSWIN */

#ifdef GC_MUST_RESTORE_REDEFINED_DLOPEN
# define dlopen GC_dlopen
#endif

#if !defined(HAVE_REGISTER_MAIN_STATIC_DATA) && defined(DYNAMIC_LOADING)
  /* Do we need to separately register the main static data segment? */
  GC_INNER GC_bool GC_register_main_static_data(void)
  {
    return TRUE;
  }
#endif /* HAVE_REGISTER_MAIN_STATIC_DATA */

/* Register a routine to filter dynamic library registration.  */
GC_API void GC_CALL GC_register_has_static_roots_callback(
                                        GC_has_static_roots_func callback)
{
    GC_has_static_roots = callback;
}

/*
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1997 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 2000 by Hewlett-Packard Company.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


/* This used to be in dyn_load.c.  It was extracted into a separate     */
/* file to avoid having to link against libdl.{a,so} if the client      */
/* doesn't call dlopen.  Of course this fails if the collector is in    */
/* a dynamic library. -HB                                               */
#if defined(GC_PTHREADS) && !defined(GC_NO_DLOPEN)

#undef GC_MUST_RESTORE_REDEFINED_DLOPEN
#if defined(dlopen) && !defined(GC_USE_LD_WRAP)
  /* To support various threads pkgs, gc.h interposes on dlopen by      */
  /* defining "dlopen" to be "GC_dlopen", which is implemented below.   */
  /* However, both GC_FirstDLOpenedLinkMap() and GC_dlopen() use the    */
  /* real system dlopen() in their implementation. We first remove      */
  /* gc.h's dlopen definition and restore it later, after GC_dlopen().  */
# undef dlopen
# define GC_MUST_RESTORE_REDEFINED_DLOPEN
#endif

/* Make sure we're not in the middle of a collection, and make sure we  */
/* don't start any.  This is invoked prior to a dlopen call to avoid    */
/* synchronization issues.  We cannot just acquire the allocator lock,  */
/* since startup code in dlopen may try to allocate.  This solution     */
/* risks heap growth (or, even, heap overflow) in the presence of many  */
/* dlopen calls in either a multi-threaded environment, or if the       */
/* library initialization code allocates substantial amounts of GC'ed   */
/* memory.                                                              */
#ifndef USE_PROC_FOR_LIBRARIES
  static void disable_gc_for_dlopen(void)
  {
    LOCK();
    while (GC_incremental && GC_collection_in_progress()) {
      ENTER_GC();
      GC_collect_a_little_inner(1000);
      EXIT_GC();
    }
    ++GC_dont_gc;
    UNLOCK();
  }
#endif

/* Redefine dlopen to guarantee mutual exclusion with           */
/* GC_register_dynamic_libraries.  Should probably happen for   */
/* other operating systems, too.                                */

/* This is similar to WRAP/REAL_FUNC() in pthread_support.c.    */
#ifdef GC_USE_LD_WRAP
# define WRAP_DLFUNC(f) __wrap_##f
# define REAL_DLFUNC(f) __real_##f
  void * REAL_DLFUNC(dlopen)(const char *, int);
#else
# define WRAP_DLFUNC(f) GC_##f
# define REAL_DLFUNC(f) f
#endif

GC_API void * WRAP_DLFUNC(dlopen)(const char *path, int mode)
{
  void * result;

# ifndef USE_PROC_FOR_LIBRARIES
    /* Disable collections.  This solution risks heap growth (or,       */
    /* even, heap overflow) but there seems no better solutions.        */
    disable_gc_for_dlopen();
# endif
  result = REAL_DLFUNC(dlopen)(path, mode);
# ifndef USE_PROC_FOR_LIBRARIES
    GC_enable(); /* undoes disable_gc_for_dlopen */
# endif
  return result;
}

#ifdef GC_USE_LD_WRAP
  /* Define GC_ function as an alias for the plain one, which will be   */
  /* intercepted.  This allows files which include gc.h, and hence      */
  /* generate references to the GC_ symbol, to see the right symbol.    */
  GC_API void *GC_dlopen(const char *path, int mode)
  {
    return dlopen(path, mode);
  }
#endif /* GC_USE_LD_WRAP */

#ifdef GC_MUST_RESTORE_REDEFINED_DLOPEN
# define dlopen GC_dlopen
#endif

#endif  /* GC_PTHREADS && !GC_NO_DLOPEN */

#if !defined(PLATFORM_MACH_DEP)
/*
 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
 * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#if !defined(PLATFORM_MACH_DEP) && !defined(SN_TARGET_PSP2)

#ifdef AMIGA
# ifndef __GNUC__
#   include <dos.h>
# else
#   include <machine/reg.h>
# endif
#endif

#if defined(MACOS) && defined(__MWERKS__)

#if defined(POWERPC)

# define NONVOLATILE_GPR_COUNT 19
  struct ppc_registers {
        unsigned long gprs[NONVOLATILE_GPR_COUNT];      /* R13-R31 */
  };
  typedef struct ppc_registers ppc_registers;

# if defined(CPPCHECK)
    void getRegisters(ppc_registers* regs);
# else
    asm static void getRegisters(register ppc_registers* regs)
    {
        stmw    r13,regs->gprs                          /* save R13-R31 */
        blr
    }
# endif

  static void PushMacRegisters(void)
  {
        ppc_registers regs;
        int i;
        getRegisters(&regs);
        for (i = 0; i < NONVOLATILE_GPR_COUNT; i++)
                GC_push_one(regs.gprs[i]);
  }

#else /* M68K */

  asm static void PushMacRegisters(void)
  {
    sub.w   #4,sp                   /* reserve space for one parameter */
    move.l  a2,(sp)
    jsr         GC_push_one
    move.l  a3,(sp)
    jsr         GC_push_one
    move.l  a4,(sp)
    jsr         GC_push_one
#   if !__option(a6frames)
      /* <pcb> perhaps a6 should be pushed if stack frames are not being used */
        move.l  a6,(sp)
        jsr             GC_push_one
#   endif
        /* skip a5 (globals), a6 (frame pointer), and a7 (stack pointer) */
    move.l  d2,(sp)
    jsr         GC_push_one
    move.l  d3,(sp)
    jsr         GC_push_one
    move.l  d4,(sp)
    jsr         GC_push_one
    move.l  d5,(sp)
    jsr         GC_push_one
    move.l  d6,(sp)
    jsr         GC_push_one
    move.l  d7,(sp)
    jsr         GC_push_one
    add.w   #4,sp                   /* fix stack */
    rts
  }

#endif /* M68K */

#endif /* MACOS && __MWERKS__ */

# if defined(IA64) && !defined(THREADS)
    /* Value returned from register flushing routine (ar.bsp).  */
    GC_INNER ptr_t GC_save_regs_ret_val = NULL;
# endif

/* Routine to mark from registers that are preserved by the C compiler. */
/* This must be ported to every new architecture.  It is not optional,  */
/* and should not be used on platforms that are either UNIX-like, or    */
/* require thread support.                                              */

#undef HAVE_PUSH_REGS

#if defined(USE_ASM_PUSH_REGS)
# define HAVE_PUSH_REGS
#else  /* No asm implementation */

# ifdef STACK_NOT_SCANNED
    void GC_push_regs(void)
    {
      /* empty */
    }
#   define HAVE_PUSH_REGS

# elif defined(M68K) && defined(AMIGA)
    /* This function is not static because it could also be             */
    /* erroneously defined in .S file, so this error would be caught    */
    /* by the linker.                                                   */
    void GC_push_regs(void)
    {
         /*  AMIGA - could be replaced by generic code                  */
         /* a0, a1, d0 and d1 are caller save */

#       ifdef __GNUC__
          asm("subq.w &0x4,%sp");       /* allocate word on top of stack */

          asm("mov.l %a2,(%sp)"); asm("jsr _GC_push_one");
          asm("mov.l %a3,(%sp)"); asm("jsr _GC_push_one");
          asm("mov.l %a4,(%sp)"); asm("jsr _GC_push_one");
          asm("mov.l %a5,(%sp)"); asm("jsr _GC_push_one");
          asm("mov.l %a6,(%sp)"); asm("jsr _GC_push_one");
          /* Skip frame pointer and stack pointer */
          asm("mov.l %d2,(%sp)"); asm("jsr _GC_push_one");
          asm("mov.l %d3,(%sp)"); asm("jsr _GC_push_one");
          asm("mov.l %d4,(%sp)"); asm("jsr _GC_push_one");
          asm("mov.l %d5,(%sp)"); asm("jsr _GC_push_one");
          asm("mov.l %d6,(%sp)"); asm("jsr _GC_push_one");
          asm("mov.l %d7,(%sp)"); asm("jsr _GC_push_one");

          asm("addq.w &0x4,%sp");       /* put stack back where it was  */
#       else /* !__GNUC__ */
          GC_push_one(getreg(REG_A2));
          GC_push_one(getreg(REG_A3));
#         ifndef __SASC
            /* Can probably be changed to #if 0 -Kjetil M. (a4=globals) */
            GC_push_one(getreg(REG_A4));
#         endif
          GC_push_one(getreg(REG_A5));
          GC_push_one(getreg(REG_A6));
          /* Skip stack pointer */
          GC_push_one(getreg(REG_D2));
          GC_push_one(getreg(REG_D3));
          GC_push_one(getreg(REG_D4));
          GC_push_one(getreg(REG_D5));
          GC_push_one(getreg(REG_D6));
          GC_push_one(getreg(REG_D7));
#       endif /* !__GNUC__ */
    }
#   define HAVE_PUSH_REGS

# elif defined(MACOS)

#   if defined(M68K) && defined(THINK_C) && !defined(CPPCHECK)
#     define PushMacReg(reg) \
              move.l  reg,(sp) \
              jsr             GC_push_one
      void GC_push_regs(void)
      {
          asm {
              sub.w   #4,sp          ; reserve space for one parameter.
              PushMacReg(a2);
              PushMacReg(a3);
              PushMacReg(a4);
              ; skip a5 (globals), a6 (frame pointer), and a7 (stack pointer)
              PushMacReg(d2);
              PushMacReg(d3);
              PushMacReg(d4);
              PushMacReg(d5);
              PushMacReg(d6);
              PushMacReg(d7);
              add.w   #4,sp          ; fix stack.
          }
      }
#     define HAVE_PUSH_REGS
#     undef PushMacReg
#   elif defined(__MWERKS__)
      void GC_push_regs(void)
      {
          PushMacRegisters();
      }
#     define HAVE_PUSH_REGS
#   endif /* __MWERKS__ */
# endif /* MACOS */

#endif /* !USE_ASM_PUSH_REGS */

#if defined(HAVE_PUSH_REGS) && defined(THREADS)
# error GC_push_regs cannot be used with threads
 /* Would fail for GC_do_blocking.  There are probably other safety     */
 /* issues.                                                             */
# undef HAVE_PUSH_REGS
#endif

#if !defined(HAVE_PUSH_REGS) && defined(UNIX_LIKE)
# include <signal.h>
# ifndef NO_GETCONTEXT
#   if defined(DARWIN) \
       && (MAC_OS_X_VERSION_MAX_ALLOWED >= 1060 /*MAC_OS_X_VERSION_10_6*/)
#     include <sys/ucontext.h>
#   else
#     include <ucontext.h>
#   endif /* !DARWIN */
#   ifdef GETCONTEXT_FPU_EXCMASK_BUG
#     include <fenv.h>
#   endif
# endif
#endif /* !HAVE_PUSH_REGS */

/* Ensure that either registers are pushed, or callee-save registers    */
/* are somewhere on the stack, and then call fn(arg, ctxt).             */
/* ctxt is either a pointer to a ucontext_t we generated, or NULL.      */
/* Could be called with or w/o the allocator lock held; could be called */
/* from a signal handler as well.                                       */
GC_ATTR_NO_SANITIZE_ADDR
GC_INNER void GC_with_callee_saves_pushed(GC_with_callee_saves_func fn,
                                          ptr_t arg)
{
  volatile int dummy;
  volatile ptr_t context = 0;
# if defined(EMSCRIPTEN) || defined(HAVE_BUILTIN_UNWIND_INIT) \
     || defined(HAVE_PUSH_REGS) || (defined(NO_CRT) && defined(MSWIN32)) \
     || !defined(NO_UNDERSCORE_SETJMP)
#   define volatile_arg arg
# else
    volatile ptr_t volatile_arg = arg;
                        /* To avoid 'arg might be clobbered by setjmp'  */
                        /* warning produced by some compilers.          */
# endif

# if defined(HAVE_PUSH_REGS)
    GC_push_regs();
# elif defined(EMSCRIPTEN)
    /* No-op, "registers" are pushed in GC_push_other_roots().  */
# else
#   if defined(UNIX_LIKE) && !defined(NO_GETCONTEXT)
      /* Older versions of Darwin seem to lack getcontext().    */
      /* ARM and MIPS Linux often doesn't support a real        */
      /* getcontext().                                          */
      static signed char getcontext_works = 0; /* (-1) - broken, 1 - works */
      ucontext_t ctxt;
#     ifdef GETCONTEXT_FPU_EXCMASK_BUG
        /* Workaround a bug (clearing the FPU exception mask) in        */
        /* getcontext on Linux/x86_64.                                  */
#       ifdef X86_64
          /* We manipulate FPU control word here just not to force the  */
          /* client application to use -lm linker option.               */
          unsigned short old_fcw;

#         if defined(CPPCHECK)
            GC_noop1_ptr(&old_fcw);
#         endif
          __asm__ __volatile__ ("fstcw %0" : "=m" (*&old_fcw));
#       else
          int except_mask = fegetexcept();
#       endif
#     endif

      if (getcontext_works >= 0) {
        if (getcontext(&ctxt) < 0) {
          WARN("getcontext failed:"
               " using another register retrieval method...\n", 0);
          /* getcontext() is broken, do not try again.          */
          /* E.g., to workaround a bug in Docker ubuntu_32bit.  */
        } else {
          context = (ptr_t)&ctxt;
        }
        if (EXPECT(0 == getcontext_works, FALSE))
          getcontext_works = context != NULL ? 1 : -1;
      }
#     ifdef GETCONTEXT_FPU_EXCMASK_BUG
#       ifdef X86_64
          __asm__ __volatile__ ("fldcw %0" : : "m" (*&old_fcw));
          {
            unsigned mxcsr;
            /* And now correct the exception mask in SSE MXCSR. */
            __asm__ __volatile__ ("stmxcsr %0" : "=m" (*&mxcsr));
            mxcsr = (mxcsr & ~(FE_ALL_EXCEPT << 7)) |
                        ((old_fcw & FE_ALL_EXCEPT) << 7);
            __asm__ __volatile__ ("ldmxcsr %0" : : "m" (*&mxcsr));
          }
#       else /* !X86_64 */
          if (feenableexcept(except_mask) < 0)
            ABORT("feenableexcept failed");
#       endif
#     endif /* GETCONTEXT_FPU_EXCMASK_BUG */
#     if defined(IA64) || defined(SPARC)
        /* On a register window machine, we need to save register       */
        /* contents on the stack for this to work.  This may already be */
        /* subsumed by the getcontext() call.                           */
#       if defined(IA64) && !defined(THREADS)
          GC_save_regs_ret_val =
#       endif
            GC_save_regs_in_stack();
#     endif
      if (NULL == context) /* getcontext failed */
#   endif /* !NO_GETCONTEXT */
    {
#     if defined(HAVE_BUILTIN_UNWIND_INIT)
        /* This was suggested by Richard Henderson as the way to        */
        /* force callee-save registers and register windows onto        */
        /* the stack.                                                   */
        __builtin_unwind_init();
#     elif defined(NO_CRT) && defined(MSWIN32)
        CONTEXT ctx;

        RtlCaptureContext(&ctx);
#     else
        /* Generic code.                         */
        /* The idea is due to Parag Patel at HP. */
        /* We're not sure whether he would like  */
        /* to be acknowledged for it or not.     */
        jmp_buf regs;

        /* setjmp doesn't always clear all of the buffer.               */
        /* That tends to preserve garbage.  Clear it.                   */
        BZERO(regs, sizeof(regs));
#       ifdef NO_UNDERSCORE_SETJMP
          (void)setjmp(regs);
#       else
          (void) _setjmp(regs);
          /* We don't want to mess with signals. According to   */
          /* SUSV3, setjmp() may or may not save signal mask.   */
          /* _setjmp won't, but is less portable.               */
#       endif
#     endif /* !HAVE_BUILTIN_UNWIND_INIT */
    }
# endif /* !HAVE_PUSH_REGS */
  /* TODO: context here is sometimes just zero.  At the moment, the     */
  /* callees don't really need it.                                      */
  /* Cast fn to a volatile type to prevent call inlining.               */
  (*(GC_with_callee_saves_func volatile *)&fn)(volatile_arg,
                        CAST_AWAY_VOLATILE_PVOID(context));
  /* Strongly discourage the compiler from treating the above   */
  /* as a tail-call, since that would pop the register          */
  /* contents before we get a chance to look at them.           */
  GC_noop1(COVERT_DATAFLOW(ADDR(&dummy)));
# undef volatile_arg
}

#endif /* !PLATFORM_MACH_DEP && !SN_TARGET_PSP2 */

#endif
#if !defined(PLATFORM_STOP_WORLD)
/*
 * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
 * Copyright (c) 2000-2009 by Hewlett-Packard Development Company.
 * All rights reserved.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#ifdef PTHREAD_STOP_WORLD_IMPL

#ifdef NACL
# include <sys/time.h>
#else
# include <signal.h>
# include <semaphore.h>
# include <errno.h>
# include <time.h> /* for nanosleep() */
#endif /* !NACL */

#ifdef E2K
# include <alloca.h>
#endif

GC_INLINE void GC_usleep(unsigned us)
{
#   if defined(LINT2) || defined(THREAD_SANITIZER)
      /* Workaround "waiting while holding a lock" static analyzer warning. */
      /* Workaround a rare hang in usleep() trying to acquire TSan Lock.    */
      while (us-- > 0)
        sched_yield(); /* pretending it takes 1us */
#   elif defined(CPPCHECK) /* || _POSIX_C_SOURCE >= 199309L */
      struct timespec ts;

      ts.tv_sec = 0;
      ts.tv_nsec = (unsigned32)us * 1000;
      /* This requires _POSIX_TIMERS feature. */
      (void)nanosleep(&ts, NULL);
#   else
      usleep(us);
#   endif
}

#ifdef NACL

  STATIC int GC_nacl_num_gc_threads = 0;
  STATIC volatile int GC_nacl_park_threads_now = 0;
  STATIC volatile pthread_t GC_nacl_thread_parker = -1;

  STATIC __thread int GC_nacl_thread_idx = -1;

  STATIC __thread GC_thread GC_nacl_gc_thread_self = NULL;
                                /* TODO: Use GC_get_tlfs() instead. */

  volatile int GC_nacl_thread_parked[MAX_NACL_GC_THREADS];
  int GC_nacl_thread_used[MAX_NACL_GC_THREADS];

#else

#if (!defined(AO_HAVE_load_acquire) || !defined(AO_HAVE_store_release)) \
    && !defined(CPPCHECK)
# error AO_load_acquire and/or AO_store_release are missing;
# error please define AO_REQUIRE_CAS manually
#endif

#ifdef DEBUG_THREADS
  /* It's safe to call original pthread_sigmask() here. */
# undef pthread_sigmask

# ifndef NSIG
#   ifdef CPPCHECK
#     define NSIG 32
#   elif defined(MAXSIG)
#     define NSIG (MAXSIG+1)
#   elif defined(_NSIG)
#     define NSIG _NSIG
#   elif defined(__SIGRTMAX)
#     define NSIG (__SIGRTMAX+1)
#   else
#     error define NSIG
#   endif
# endif /* !NSIG */

  void GC_print_sig_mask(void)
  {
    sigset_t blocked;
    int i;

    if (pthread_sigmask(SIG_BLOCK, NULL, &blocked) != 0)
      ABORT("pthread_sigmask failed");
    for (i = 1; i < NSIG; i++) {
      if (sigismember(&blocked, i))
        GC_printf("Signal blocked: %d\n", i);
    }
  }
#endif /* DEBUG_THREADS */

/* Remove the signals that we want to allow in thread stopping  */
/* handler from a set.                                          */
STATIC void GC_remove_allowed_signals(sigset_t *set)
{
    if (sigdelset(set, SIGINT) != 0
          || sigdelset(set, SIGQUIT) != 0
          || sigdelset(set, SIGABRT) != 0
          || sigdelset(set, SIGTERM) != 0) {
        ABORT("sigdelset failed");
    }

#   ifdef MPROTECT_VDB
      /* Handlers write to the thread structure, which is in the heap,  */
      /* and hence can trigger a protection fault.                      */
      if (sigdelset(set, SIGSEGV) != 0
#         ifdef HAVE_SIGBUS
            || sigdelset(set, SIGBUS) != 0
#         endif
          ) {
        ABORT("sigdelset failed");
      }
#   endif
}

static sigset_t suspend_handler_mask;

#define THREAD_RESTARTED 0x1

STATIC volatile AO_t GC_stop_count;
                        /* Incremented (to the nearest even value) at   */
                        /* the beginning of GC_stop_world() (or when    */
                        /* a thread is requested to be suspended by     */
                        /* GC_suspend_thread) and once more (to an odd  */
                        /* value) at the beginning of GC_start_world(). */
                        /* The lowest bit is THREAD_RESTARTED one       */
                        /* which, if set, means it is safe for threads  */
                        /* to restart, i.e. they will see another       */
                        /* suspend signal before they are expected to   */
                        /* stop (unless they have stopped voluntarily). */

STATIC GC_bool GC_retry_signals = FALSE;

/*
 * We use signals to stop threads during GC.
 *
 * Suspended threads wait in signal handler for SIG_THR_RESTART.
 * That's more portable than semaphores or condition variables.
 * (We do use sem_post from a signal handler, but that should be portable.)
 *
 * The thread suspension signal SIG_SUSPEND is now defined in gc_priv.h.
 * Note that we can't just stop a thread; we need it to save its stack
 * pointer(s) and acknowledge.
 */
#ifndef SIG_THR_RESTART
# ifdef SUSPEND_HANDLER_NO_CONTEXT
    /* Reuse the suspend signal. */
#   define SIG_THR_RESTART SIG_SUSPEND
# elif defined(GC_HPUX_THREADS) || defined(GC_OSF1_THREADS) \
     || defined(GC_NETBSD_THREADS) || defined(GC_USESIGRT_SIGNALS)
#   if defined(_SIGRTMIN) && !defined(CPPCHECK)
#     define SIG_THR_RESTART _SIGRTMIN + 5
#   else
#     define SIG_THR_RESTART SIGRTMIN + 5
#   endif
# elif defined(GC_FREEBSD_THREADS) && defined(__GLIBC__)
#   define SIG_THR_RESTART (32+5)
# elif defined(GC_FREEBSD_THREADS) || defined(HURD) || defined(RTEMS)
#   define SIG_THR_RESTART SIGUSR2
# else
#   define SIG_THR_RESTART SIGXCPU
# endif
#endif /* !SIG_THR_RESTART */

#define SIGNAL_UNSET (-1)
    /* Since SIG_SUSPEND and/or SIG_THR_RESTART could represent */
    /* a non-constant expression (e.g., in case of SIGRTMIN),   */
    /* actual signal numbers are determined by GC_stop_init()   */
    /* unless manually set (before GC initialization).          */
    /* Might be set to the same signal number.                  */
STATIC int GC_sig_suspend = SIGNAL_UNSET;
STATIC int GC_sig_thr_restart = SIGNAL_UNSET;

GC_API void GC_CALL GC_set_suspend_signal(int sig)
{
  if (GC_is_initialized) return;

  GC_sig_suspend = sig;
}

GC_API void GC_CALL GC_set_thr_restart_signal(int sig)
{
  if (GC_is_initialized) return;

  GC_sig_thr_restart = sig;
}

GC_API int GC_CALL GC_get_suspend_signal(void)
{
  return GC_sig_suspend != SIGNAL_UNSET ? GC_sig_suspend : SIG_SUSPEND;
}

GC_API int GC_CALL GC_get_thr_restart_signal(void)
{
  return GC_sig_thr_restart != SIGNAL_UNSET
            ? GC_sig_thr_restart : SIG_THR_RESTART;
}

#ifdef BASE_ATOMIC_OPS_EMULATED
 /* The AO primitives emulated with locks cannot be used inside signal  */
 /* handlers as this could cause a deadlock or a double lock.           */
 /* The following "async" macro definitions are correct only for        */
 /* an uniprocessor case and are provided for a test purpose.           */
# define ao_load_acquire_async(p) (*(p))
# define ao_load_async(p) ao_load_acquire_async(p)
# define ao_store_release_async(p, v) (void)(*(p) = (v))
# define ao_cptr_store_async(p, v) (void)(*(p) = (v))
#else
# define ao_load_acquire_async(p) AO_load_acquire(p)
# define ao_load_async(p) AO_load(p)
# define ao_store_release_async(p, v) AO_store_release(p, v)
# define ao_cptr_store_async(p, v) GC_cptr_store(p, v)
#endif /* !BASE_ATOMIC_OPS_EMULATED */

STATIC sem_t GC_suspend_ack_sem; /* also used to acknowledge restart */

STATIC void GC_suspend_handler_inner(ptr_t dummy, void *context);

#ifdef SUSPEND_HANDLER_NO_CONTEXT
  STATIC void GC_suspend_handler(int sig)
#else
  STATIC void GC_suspend_sigaction(int sig, siginfo_t *info, void *context)
#endif
{
  int old_errno = errno;

  if (sig != GC_sig_suspend) {
#   if defined(GC_FREEBSD_THREADS)
      /* Workaround "deferred signal handling" bug in FreeBSD 9.2.      */
      if (0 == sig) return;
#   endif
    ABORT("Bad signal in suspend_handler");
  }

# ifdef SUSPEND_HANDLER_NO_CONTEXT
    /* A quick check if the signal is called to restart the world.      */
    if ((ao_load_async(&GC_stop_count) & THREAD_RESTARTED) != 0)
      return;
    GC_with_callee_saves_pushed(GC_suspend_handler_inner, NULL);
# else
    UNUSED_ARG(info);
    /* We believe that in this case the full context is already         */
    /* in the signal handler frame.                                     */
    GC_suspend_handler_inner(NULL, context);
# endif
  errno = old_errno;
}

/* The lookup here is safe, since this is done on behalf        */
/* of a thread which holds the allocator lock in order          */
/* to stop the world.  Thus concurrent modification of the      */
/* data structure is impossible.  Unfortunately, we have to     */
/* instruct TSan that the lookup is safe.                       */
#ifdef THREAD_SANITIZER
  /* Almost same as GC_self_thread_inner() except for the       */
  /* no-sanitize attribute added and the result is never NULL.  */
  GC_ATTR_NO_SANITIZE_THREAD
  static GC_thread GC_lookup_self_thread_async(void)
  {
    thread_id_t self_id = thread_id_self();
    GC_thread p = GC_threads[THREAD_TABLE_INDEX(self_id)];

    for (;; p = p -> tm.next) {
      if (THREAD_EQUAL(p -> id, self_id)) break;
    }
    return p;
  }
#else
# define GC_lookup_self_thread_async() GC_self_thread_inner()
#endif

GC_INLINE void GC_store_stack_ptr(GC_stack_context_t crtn)
{
  /* There is no data race between the suspend handler (storing         */
  /* stack_ptr) and GC_push_all_stacks (fetching stack_ptr) because     */
  /* GC_push_all_stacks is executed after GC_stop_world exits and the   */
  /* latter runs sem_wait repeatedly waiting for all the suspended      */
  /* threads to call sem_post.  Nonetheless, stack_ptr is stored (here) */
  /* and fetched (by GC_push_all_stacks) using the atomic primitives to */
  /* avoid the related TSan warning.                                    */
# ifdef SPARC
    ao_cptr_store_async(&(crtn -> stack_ptr), GC_save_regs_in_stack());
    /* TODO: regs saving already done by GC_with_callee_saves_pushed */
# else
#   ifdef IA64
      crtn -> backing_store_ptr = GC_save_regs_in_stack();
#   endif
    ao_cptr_store_async(&(crtn -> stack_ptr), GC_approx_sp());
# endif
}

STATIC void GC_suspend_handler_inner(ptr_t dummy, void *context)
{
  GC_thread me;
  GC_stack_context_t crtn;
# ifdef E2K
    ptr_t bs_lo;
    size_t stack_size;
# endif
  IF_CANCEL(int cancel_state;)
# ifdef GC_ENABLE_SUSPEND_THREAD
    AO_t suspend_cnt;
# endif
  AO_t my_stop_count = ao_load_acquire_async(&GC_stop_count);
                        /* After the barrier, this thread should see    */
                        /* the actual content of GC_threads.            */

  UNUSED_ARG(dummy);
  UNUSED_ARG(context);
  if ((my_stop_count & THREAD_RESTARTED) != 0)
    return; /* Restarting the world. */

  DISABLE_CANCEL(cancel_state);
      /* pthread_setcancelstate is not defined to be async-signal-safe. */
      /* But the glibc version appears to be in the absence of          */
      /* asynchronous cancellation.  And since this signal handler      */
      /* to block on sigsuspend, which is both async-signal-safe        */
      /* and a cancellation point, there seems to be no obvious way     */
      /* out of it.  In fact, it looks to me like an async-signal-safe  */
      /* cancellation point is inherently a problem, unless there is    */
      /* some way to disable cancellation in the handler.               */

# ifdef DEBUG_THREADS
    GC_log_printf("Suspending %p\n", (void *)pthread_self());
# endif
  me = GC_lookup_self_thread_async();
  if (NULL == me) return; // __v_, make the sanitizers and -cstrict happy
  if ((me -> last_stop_count & ~(word)THREAD_RESTARTED) == my_stop_count) {
      /* Duplicate signal.  OK if we are retrying.      */
      if (!GC_retry_signals) {
          WARN("Duplicate suspend signal in thread %p\n", pthread_self());
      }
      RESTORE_CANCEL(cancel_state);
      return;
  }
  crtn = me -> crtn;
  GC_store_stack_ptr(crtn);
# ifdef E2K
    GC_ASSERT(NULL == crtn -> backing_store_end);
    GET_PROCEDURE_STACK_LOCAL(crtn -> ps_ofs, &bs_lo, &stack_size);
    crtn -> backing_store_end = bs_lo;
    crtn -> backing_store_ptr = bs_lo + stack_size;
# endif
# ifdef GC_ENABLE_SUSPEND_THREAD
    suspend_cnt = ao_load_async(&(me -> ext_suspend_cnt));
# endif

  /* Tell the thread that wants to stop the world that this     */
  /* thread has been stopped.  Note that sem_post() is          */
  /* the only async-signal-safe primitive in LinuxThreads.      */
  sem_post(&GC_suspend_ack_sem);
  ao_store_release_async(&(me -> last_stop_count), my_stop_count);

  /* Wait until that thread tells us to restart by sending      */
  /* this thread a GC_sig_thr_restart signal (should be masked  */
  /* at this point thus there is no race).                      */
  /* We do not continue until we receive that signal,           */
  /* but we do not take that as authoritative.  (We may be      */
  /* accidentally restarted by one of the user signals we       */
  /* don't block.)  After we receive the signal, we use a       */
  /* primitive and expensive mechanism to wait until it's       */
  /* really safe to proceed.  Under normal circumstances,       */
  /* this code should not be executed.                          */
  do {
      sigsuspend(&suspend_handler_mask);
      /* Iterate while not restarting the world or thread is suspended. */
  } while (ao_load_acquire_async(&GC_stop_count) == my_stop_count
#          ifdef GC_ENABLE_SUSPEND_THREAD
             || ((suspend_cnt & 1) != 0
                 && ao_load_async(&(me -> ext_suspend_cnt)) == suspend_cnt)
#          endif
          );

# ifdef DEBUG_THREADS
    GC_log_printf("Resuming %p\n", (void *)pthread_self());
# endif
# ifdef E2K
    GC_ASSERT(crtn -> backing_store_end == bs_lo);
    crtn -> backing_store_ptr = NULL;
    crtn -> backing_store_end = NULL;
# endif

# ifndef GC_NETBSD_THREADS_WORKAROUND
    if (GC_retry_signals || GC_sig_suspend == GC_sig_thr_restart)
# endif
  {
    /* If the RESTART signal loss is possible (though it should be      */
    /* less likely than losing the SUSPEND signal as we do not do       */
    /* much between the first sem_post and sigsuspend calls), more      */
    /* handshaking is provided to work around it.                       */
    sem_post(&GC_suspend_ack_sem);
    /* Set the flag that the thread has been restarted. */
    if (GC_retry_signals)
      ao_store_release_async(&(me -> last_stop_count),
                             my_stop_count | THREAD_RESTARTED);
  }
  RESTORE_CANCEL(cancel_state);
}

static void suspend_restart_barrier(int n_live_threads)
{
    int i;

    for (i = 0; i < n_live_threads; i++) {
      while (0 != sem_wait(&GC_suspend_ack_sem)) {
        /* On Linux, sem_wait is documented to always return zero.      */
        /* But the documentation appears to be incorrect.               */
        /* EINTR seems to happen with some versions of gdb.             */
        if (errno != EINTR)
          ABORT("sem_wait failed");
      }
    }
#   ifdef GC_ASSERTIONS
      sem_getvalue(&GC_suspend_ack_sem, &i);
      GC_ASSERT(0 == i);
#   endif
}

# define WAIT_UNIT 3000 /* us */

static int resend_lost_signals(int n_live_threads,
                               int (*suspend_restart_all)(void))
{
#   define RETRY_INTERVAL 100000 /* us */
#   define RESEND_SIGNALS_LIMIT 150

    if (n_live_threads > 0) {
      unsigned long wait_usecs = 0;  /* Total wait since retry. */
      int retry = 0;
      int prev_sent = 0;

      for (;;) {
        int ack_count;

        sem_getvalue(&GC_suspend_ack_sem, &ack_count);
        if (ack_count == n_live_threads)
          break;
        if (wait_usecs > RETRY_INTERVAL) {
          int newly_sent = suspend_restart_all();

          if (newly_sent != prev_sent) {
            retry = 0; /* restart the counter */
          } else if (++retry >= RESEND_SIGNALS_LIMIT) /* no progress */
            ABORT_ARG1("Signals delivery fails constantly",
                       " at GC #%lu", (unsigned long)GC_gc_no);

          GC_COND_LOG_PRINTF("Resent %d signals after timeout, retry: %d\n",
                             newly_sent, retry);
          sem_getvalue(&GC_suspend_ack_sem, &ack_count);
          if (newly_sent < n_live_threads - ack_count) {
            WARN("Lost some threads while stopping or starting world?!\n", 0);
            n_live_threads = ack_count + newly_sent;
          }
          prev_sent = newly_sent;
          wait_usecs = 0;
        }
        GC_usleep(WAIT_UNIT);
        wait_usecs += WAIT_UNIT;
      }
    }
    return n_live_threads;
}

#ifdef HAVE_CLOCK_GETTIME
# define TS_NSEC_ADD(ts, ns) \
                (ts.tv_nsec += (ns), \
                 (void)(ts.tv_nsec >= 1000000L*1000 ? \
                       (ts.tv_nsec -= 1000000L*1000, ts.tv_sec++, 0) : 0))
#endif

static void resend_lost_signals_retry(int n_live_threads,
                                      int (*suspend_restart_all)(void))
{
# if defined(HAVE_CLOCK_GETTIME) && !defined(DONT_TIMEDWAIT_ACK_SEM)
#   define TIMEOUT_BEFORE_RESEND 10000 /* us */
    struct timespec ts;

    if (n_live_threads > 0 && clock_gettime(CLOCK_REALTIME, &ts) == 0) {
      int i;

      TS_NSEC_ADD(ts, TIMEOUT_BEFORE_RESEND * (unsigned32)1000);
      /* First, try to wait for the semaphore with some timeout.            */
      /* On failure, fallback to WAIT_UNIT pause and resend of the signal.  */
      for (i = 0; i < n_live_threads; i++) {
        if (0 != sem_timedwait(&GC_suspend_ack_sem, &ts))
          break; /* Wait timed out or any other error.  */
      }
      /* Update the count of threads to wait the ack from.      */
      n_live_threads -= i;
    }
# endif
  n_live_threads = resend_lost_signals(n_live_threads, suspend_restart_all);
  suspend_restart_barrier(n_live_threads);
}

STATIC void GC_restart_handler(int sig)
{
# if defined(DEBUG_THREADS)
    int old_errno = errno;      /* Preserve errno value.        */
# endif

  if (sig != GC_sig_thr_restart)
    ABORT("Bad signal in restart handler");

  /* Note: even if we do not do anything useful here, it would still    */
  /* be necessary to have a signal handler, rather than ignoring the    */
  /* signals, otherwise the signals will not be delivered at all,       */
  /* and will thus not interrupt the sigsuspend() above.                */
# ifdef DEBUG_THREADS
    GC_log_printf("In GC_restart_handler for %p\n", (void *)pthread_self());
    errno = old_errno;
# endif
}

# ifdef USE_TKILL_ON_ANDROID
    EXTERN_C_BEGIN
    extern int tkill(pid_t tid, int sig); /* from sys/linux-unistd.h */
    EXTERN_C_END
#   define THREAD_SYSTEM_ID(t) (t)->kernel_id
# else
#   define THREAD_SYSTEM_ID(t) (t)->id
# endif

# ifndef RETRY_TKILL_EAGAIN_LIMIT
#   define RETRY_TKILL_EAGAIN_LIMIT 16
# endif

  static int raise_signal(GC_thread p, int sig)
  {
    int res;
#   ifdef RETRY_TKILL_ON_EAGAIN
      int retry;
#   endif
#   if defined(SIMULATE_LOST_SIGNALS) && !defined(GC_ENABLE_SUSPEND_THREAD)
#     ifndef LOST_SIGNALS_RATIO
#       define LOST_SIGNALS_RATIO 25
#     endif
      static int signal_cnt; /* race is OK, it is for test purpose only */

      if (GC_retry_signals && (++signal_cnt) % LOST_SIGNALS_RATIO == 0)
        return 0; /* simulate the signal is sent but lost */
#   endif
#   ifdef RETRY_TKILL_ON_EAGAIN
      for (retry = 0;; retry++)
#   endif
    {
#     ifdef USE_TKILL_ON_ANDROID
        int old_errno = errno;

        res = tkill(THREAD_SYSTEM_ID(p), sig);
        if (res < 0) {
          res = errno;
          errno = old_errno;
        }
#     else
        res = pthread_kill(THREAD_SYSTEM_ID(p), sig);
#     endif
#     ifdef RETRY_TKILL_ON_EAGAIN
        if (res != EAGAIN || retry >= RETRY_TKILL_EAGAIN_LIMIT) break;
        /* A temporal overflow of the real-time signal queue.   */
        GC_usleep(WAIT_UNIT);
#     endif
    }
    return res;
  }

# ifdef GC_ENABLE_SUSPEND_THREAD
#   include <sys/time.h>



    STATIC void GC_brief_async_signal_safe_sleep(void)
    {
      struct timeval tv;
      tv.tv_sec = 0;
#     if defined(GC_TIME_LIMIT) && !defined(CPPCHECK)
        tv.tv_usec = 1000 * GC_TIME_LIMIT / 2;
#     else
        tv.tv_usec = 1000 * 15 / 2;
#     endif
      (void)select(0, 0, 0, 0, &tv);
    }

    GC_INNER void GC_suspend_self_inner(GC_thread me, size_t suspend_cnt) {
      IF_CANCEL(int cancel_state;)

      GC_ASSERT((suspend_cnt & 1) != 0);
      DISABLE_CANCEL(cancel_state);
#     ifdef DEBUG_THREADS
        GC_log_printf("Suspend self: %p\n", (void *)(me -> id));
#     endif
      while (ao_load_acquire_async(&(me -> ext_suspend_cnt)) == suspend_cnt) {
        /* TODO: Use sigsuspend() even for self-suspended threads. */
        GC_brief_async_signal_safe_sleep();
      }
#     ifdef DEBUG_THREADS
        GC_log_printf("Resume self: %p\n", (void *)(me -> id));
#     endif
      RESTORE_CANCEL(cancel_state);
    }

    GC_API void GC_CALL GC_suspend_thread(GC_SUSPEND_THREAD_ID thread) {
      GC_thread t;
      AO_t next_stop_count;
      AO_t suspend_cnt;
      IF_CANCEL(int cancel_state;)

      LOCK();
      t = GC_lookup_by_pthread((pthread_t)thread);
      if (NULL == t) {
        UNLOCK();
        return;
      }
      suspend_cnt = t -> ext_suspend_cnt;
      if ((suspend_cnt & 1) != 0) /* already suspended? */ {
        GC_ASSERT(!THREAD_EQUAL((pthread_t)thread, pthread_self()));
        UNLOCK();
        return;
      }
      if ((t -> flags & (FINISHED | DO_BLOCKING)) != 0) {
        t -> ext_suspend_cnt = suspend_cnt | 1; /* suspend */
        /* Terminated but not joined yet, or in do-blocking state.  */
        UNLOCK();
        return;
      }

      if (THREAD_EQUAL((pthread_t)thread, pthread_self())) {
        t -> ext_suspend_cnt = suspend_cnt | 1;
        GC_with_callee_saves_pushed(GC_suspend_self_blocked, (ptr_t)t);
        UNLOCK();
        return;
      }

      DISABLE_CANCEL(cancel_state);
                /* GC_suspend_thread is not a cancellation point.   */
#     ifdef PARALLEL_MARK
        /* Ensure we do not suspend a thread while it is rebuilding */
        /* a free list, otherwise such a deadlock is possible:      */
        /* thread 1 is blocked in GC_wait_for_reclaim holding       */
        /* the allocator lock, thread 2 is suspended in             */
        /* GC_reclaim_generic invoked from GC_generic_malloc_many   */
        /* (with GC_fl_builder_count > 0), and thread 3 is blocked  */
        /* acquiring the allocator lock in GC_resume_thread.        */
        if (GC_parallel)
          GC_wait_for_reclaim();
#     endif

      if (GC_manual_vdb) {
        /* See the relevant comment in GC_stop_world.   */
        GC_acquire_dirty_lock();
      }
      /* Else do not acquire the dirty lock as the write fault handler  */
      /* might be trying to acquire it too, and the suspend handler     */
      /* execution is deferred until the write fault handler completes. */

      next_stop_count = GC_stop_count + THREAD_RESTARTED;
      GC_ASSERT((next_stop_count & THREAD_RESTARTED) == 0);
      AO_store(&GC_stop_count, next_stop_count);

      /* Set the flag making the change visible to the signal handler.  */
      AO_store_release(&(t -> ext_suspend_cnt), suspend_cnt | 1);

      /* TODO: Support GC_retry_signals (not needed for TSan) */
      switch (raise_signal(t, GC_sig_suspend)) {
      /* ESRCH cannot happen as terminated threads are handled above.   */
      case 0:
        break;
      default:
        ABORT("pthread_kill failed");
      }

      /* Wait for the thread to complete threads table lookup and   */
      /* stack_ptr assignment.                                      */
      GC_ASSERT(GC_thr_initialized);
      suspend_restart_barrier(1);
      if (GC_manual_vdb)
        GC_release_dirty_lock();
      AO_store(&GC_stop_count, next_stop_count | THREAD_RESTARTED);

      RESTORE_CANCEL(cancel_state);
      UNLOCK();
    }

    GC_API void GC_CALL GC_resume_thread(GC_SUSPEND_THREAD_ID thread) {
      GC_thread t;

      LOCK();
      t = GC_lookup_by_pthread((pthread_t)thread);
      if (t != NULL) {
        AO_t suspend_cnt = t -> ext_suspend_cnt;

        if ((suspend_cnt & 1) != 0) /* is suspended? */ {
          GC_ASSERT((GC_stop_count & THREAD_RESTARTED) != 0);
          /* Mark the thread as not suspended - it will be resumed shortly. */
          AO_store(&(t -> ext_suspend_cnt), suspend_cnt + 1);

          if ((t -> flags & (FINISHED | DO_BLOCKING)) == 0) {
            int result = raise_signal(t, GC_sig_thr_restart);

            /* TODO: Support signal resending on GC_retry_signals */
            if (result != 0)
              ABORT_ARG1("pthread_kill failed in GC_resume_thread",
                         ": errcode= %d", result);
#           ifndef GC_NETBSD_THREADS_WORKAROUND
              if (GC_retry_signals || GC_sig_suspend == GC_sig_thr_restart)
#           endif
            {
              IF_CANCEL(int cancel_state;)

              DISABLE_CANCEL(cancel_state);
              suspend_restart_barrier(1);
              RESTORE_CANCEL(cancel_state);
            }
          }
        }
      }
      UNLOCK();
    }

    GC_API int GC_CALL GC_is_thread_suspended(GC_SUSPEND_THREAD_ID thread) {
      GC_thread t;
      int is_suspended = 0;

      READER_LOCK();
      t = GC_lookup_by_pthread((pthread_t)thread);
      if (t != NULL && (t -> ext_suspend_cnt & 1) != 0)
        is_suspended = (int)TRUE;
      READER_UNLOCK();
      return is_suspended;
    }
# endif /* GC_ENABLE_SUSPEND_THREAD */

# undef ao_cptr_store_async
# undef ao_load_acquire_async
# undef ao_load_async
# undef ao_store_release_async
#endif /* !NACL */

/* Should do exactly the right thing if the world is stopped; should    */
/* not fail if it is not.                                               */
GC_INNER void GC_push_all_stacks(void)
{
    GC_bool found_me = FALSE;
    size_t nthreads = 0;
    int i;
    GC_thread p;
    ptr_t lo; /* stack top (sp) */
    ptr_t hi; /* bottom */
#   if defined(E2K) || defined(IA64)
      /* We also need to scan the register backing store.   */
      ptr_t bs_lo, bs_hi;
#   endif
    struct GC_traced_stack_sect_s *traced_stack_sect;
    pthread_t self = pthread_self();
    word total_size = 0;

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(GC_thr_initialized);
#   ifdef DEBUG_THREADS
      GC_log_printf("Pushing stacks from thread %p\n", (void *)self);
#   endif
    for (i = 0; i < THREAD_TABLE_SZ; i++) {
      for (p = GC_threads[i]; p != NULL; p = p -> tm.next) {
#       if defined(E2K) || defined(IA64)
          GC_bool is_self = FALSE;
#       endif
        GC_stack_context_t crtn = p -> crtn;

        GC_ASSERT(THREAD_TABLE_INDEX(p -> id) == i);
        if (KNOWN_FINISHED(p)) continue;
        ++nthreads;
        traced_stack_sect = crtn -> traced_stack_sect;
        if (THREAD_EQUAL(p -> id, self)) {
            GC_ASSERT((p -> flags & DO_BLOCKING) == 0);
#           ifdef SPARC
              lo = GC_save_regs_in_stack();
#           else
              lo = GC_approx_sp();
#             ifdef IA64
                bs_hi = GC_save_regs_in_stack();
#             elif defined(E2K)
                {
                  size_t stack_size;

                  GC_ASSERT(NULL == crtn -> backing_store_end);
                  GET_PROCEDURE_STACK_LOCAL(crtn -> ps_ofs,
                                            &bs_lo, &stack_size);
                  bs_hi = bs_lo + stack_size;
                }
#             endif
#           endif
            found_me = TRUE;
#           if defined(E2K) || defined(IA64)
              is_self = TRUE;
#           endif
        } else {
            lo = GC_cptr_load(&(crtn -> stack_ptr));
#           ifdef IA64
              bs_hi = crtn -> backing_store_ptr;
#           elif defined(E2K)
              bs_lo = crtn -> backing_store_end;
              bs_hi = crtn -> backing_store_ptr;
#           endif
            if (traced_stack_sect != NULL
                    && traced_stack_sect -> saved_stack_ptr == lo) {
              /* If the thread has never been stopped since the recent  */
              /* GC_call_with_gc_active invocation then skip the top    */
              /* "stack section" as stack_ptr already points to.        */
              traced_stack_sect = traced_stack_sect -> prev;
            }
        }
        hi = crtn -> stack_end;
#       ifdef IA64
          bs_lo = crtn -> backing_store_end;
#       endif
#       ifdef DEBUG_THREADS
#         ifdef STACK_GROWS_UP
            GC_log_printf("Stack for thread %p is (%p,%p]\n",
                          (void *)(p -> id), (void *)hi, (void *)lo);
#         else
            GC_log_printf("Stack for thread %p is [%p,%p)\n",
                          (void *)(p -> id), (void *)lo, (void *)hi);
#         endif
#       endif
        if (NULL == lo) ABORT("GC_push_all_stacks: sp not set!");
        if (crtn -> altstack != NULL && ADDR_GE(lo, crtn -> altstack)
            && ADDR_GE(crtn -> altstack + crtn -> altstack_size, lo)) {
#         ifdef STACK_GROWS_UP
            hi = crtn -> altstack;
#         else
            hi = crtn -> altstack + crtn -> altstack_size;
#         endif
          /* FIXME: Need to scan the normal stack too, but how ? */
        }
#       ifdef STACKPTR_CORRECTOR_AVAILABLE
          if (GC_sp_corrector != 0)
            GC_sp_corrector((void **)&lo, (void *)(p -> id));
#       endif
        GC_push_all_stack_sections(lo, hi, traced_stack_sect);
#       ifdef STACK_GROWS_UP
          total_size += lo - hi;
#       else
          total_size += hi - lo; /* lo <= hi */
#       endif
#       ifdef NACL
          /* Push reg_storage as roots, this will cover the reg context. */
          GC_push_all_stack((ptr_t)p -> reg_storage,
                        (ptr_t)(p -> reg_storage + NACL_GC_REG_STORAGE_SIZE));
          total_size += NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t);
#       endif
#       ifdef E2K
          if ((GC_stop_count & THREAD_RESTARTED) != 0
#             ifdef GC_ENABLE_SUSPEND_THREAD
                && (p -> ext_suspend_cnt & 1) == 0
#             endif
              && !is_self && (p -> flags & DO_BLOCKING) == 0)
            continue; /* procedure stack buffer has already been freed */
#       endif
#       if defined(E2K) || defined(IA64)
#         ifdef DEBUG_THREADS
            GC_log_printf("Reg stack for thread %p is [%p,%p)\n",
                          (void *)(p -> id), (void *)bs_lo, (void *)bs_hi);
#         endif
          GC_ASSERT(bs_lo != NULL && bs_hi != NULL);
          /* FIXME: This (if is_self) may add an unbounded number of    */
          /* entries, and hence overflow the mark stack, which is bad.  */
#         ifdef IA64
            GC_push_all_register_sections(bs_lo, bs_hi, is_self,
                                          traced_stack_sect);
#         else
            if (is_self) {
              GC_push_all_eager(bs_lo, bs_hi);
            } else {
              GC_push_all_stack(bs_lo, bs_hi);
            }
#         endif
          total_size += bs_hi - bs_lo; /* bs_lo <= bs_hi */
#       endif
      }
    }
    GC_VERBOSE_LOG_PRINTF("Pushed %d thread stacks\n", (int)nthreads);
    if (!found_me && !GC_in_thread_creation)
      ABORT("Collecting from unknown thread");
    GC_total_stacksize = total_size;
}

#ifdef DEBUG_THREADS
  /* There seems to be a very rare thread stopping problem.  To help us */
  /* debug that, we save the ids of the stopping thread.                */
  pthread_t GC_stopping_thread;
  int GC_stopping_pid = 0;
#endif

/* Suspend all threads that might still be running.  Return the number  */
/* of suspend signals that were sent.                                   */
STATIC int GC_suspend_all(void)
{
  int n_live_threads = 0;
  int i;
# ifndef NACL
    GC_thread p;
    pthread_t self = pthread_self();
    int result;

    GC_ASSERT((GC_stop_count & THREAD_RESTARTED) == 0);
    GC_ASSERT(I_HOLD_LOCK());
    for (i = 0; i < THREAD_TABLE_SZ; i++) {
      for (p = GC_threads[i]; p != NULL; p = p -> tm.next) {
        if (!THREAD_EQUAL(p -> id, self)) {
            if ((p -> flags & (FINISHED | DO_BLOCKING)) != 0) continue;
#           ifdef GC_ENABLE_SUSPEND_THREAD
                if ((p -> ext_suspend_cnt & 1) != 0) continue;
#           endif
            if (AO_load(&(p -> last_stop_count)) == GC_stop_count)
              continue; /* matters only if GC_retry_signals */
            n_live_threads++;
#           ifdef DEBUG_THREADS
              GC_log_printf("Sending suspend signal to %p\n", (void *)p->id);
#           endif

              /* The synchronization between GC_dirty (based on         */
              /* test-and-set) and the signal-based thread suspension   */
              /* is performed in GC_stop_world because                  */
              /* GC_release_dirty_lock cannot be called before          */
              /* acknowledging the thread is really suspended.          */
            result = raise_signal(p, GC_sig_suspend);
            switch (result) {
                case ESRCH:
                    /* Not really there anymore.  Possible? */
                    n_live_threads--;
                    break;
                case 0:
                    if (GC_on_thread_event)
                      GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED,
                                         (void *)(word)THREAD_SYSTEM_ID(p));
                                /* Note: thread id might be truncated.  */
                    break;
                default:
                    ABORT_ARG1("pthread_kill failed at suspend",
                               ": errcode= %d", result);
            }
        }
      }
    }

# else /* NACL */
#   ifndef NACL_PARK_WAIT_USEC
#     define NACL_PARK_WAIT_USEC 100 /* us */
#   endif
    unsigned long num_sleeps = 0;

    GC_ASSERT(I_HOLD_LOCK());
#   ifdef DEBUG_THREADS
      GC_log_printf("pthread_stop_world: number of threads: %d\n",
                    GC_nacl_num_gc_threads - 1);
#   endif
    GC_nacl_thread_parker = pthread_self();
    GC_nacl_park_threads_now = 1;

    if (GC_manual_vdb)
      GC_acquire_dirty_lock();
    for (;;) {
      int num_threads_parked = 0;
      int num_used = 0;

      /* Check the 'parked' flag for each thread the GC knows about.    */
      for (i = 0; i < MAX_NACL_GC_THREADS
                  && num_used < GC_nacl_num_gc_threads; i++) {
        if (GC_nacl_thread_used[i] == 1) {
          num_used++;
          if (GC_nacl_thread_parked[i] == 1) {
            num_threads_parked++;
            if (GC_on_thread_event)
              GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED, (void *)(word)i);
          }
        }
      }
      /* -1 for the current thread.     */
      if (num_threads_parked >= GC_nacl_num_gc_threads - 1)
        break;
#     ifdef DEBUG_THREADS
        GC_log_printf("Sleep waiting for %d threads to park...\n",
                      GC_nacl_num_gc_threads - num_threads_parked - 1);
#     endif
      GC_usleep(NACL_PARK_WAIT_USEC);
      if (++num_sleeps > (1000 * 1000) / NACL_PARK_WAIT_USEC) {
        WARN("GC appears stalled waiting for %" WARN_PRIdPTR
             " threads to park...\n",
             GC_nacl_num_gc_threads - num_threads_parked - 1);
        num_sleeps = 0;
      }
    }
    if (GC_manual_vdb)
      GC_release_dirty_lock();
# endif /* NACL */
  return n_live_threads;
}

GC_INNER void GC_stop_world(void)
{
# if !defined(NACL)
    int n_live_threads;
# endif
  GC_ASSERT(I_HOLD_LOCK());
  /* Make sure all free list construction has stopped before we start.  */
  /* No new construction can start, since it is required to acquire and */
  /* release the allocator lock before start.                           */

  GC_ASSERT(GC_thr_initialized);
# ifdef DEBUG_THREADS
    GC_stopping_thread = pthread_self();
    GC_stopping_pid = getpid();
    GC_log_printf("Stopping the world from %p\n", (void *)GC_stopping_thread);
# endif
# ifdef PARALLEL_MARK
    if (GC_parallel) {
      GC_acquire_mark_lock();
      GC_ASSERT(GC_fl_builder_count == 0);
      /* We should have previously waited for it to become zero.        */
    }
# endif /* PARALLEL_MARK */

# if defined(NACL)
    (void)GC_suspend_all();
# else
    AO_store(&GC_stop_count, GC_stop_count + THREAD_RESTARTED);
        /* Only concurrent reads are possible. */
    if (GC_manual_vdb) {
      GC_acquire_dirty_lock();
      /* The write fault handler cannot be called if GC_manual_vdb      */
      /* (thus double-locking should not occur in                       */
      /* async_set_pht_entry_from_index based on test-and-set).         */
    }
    n_live_threads = GC_suspend_all();
    if (GC_retry_signals) {
      resend_lost_signals_retry(n_live_threads, GC_suspend_all);
    } else {
      suspend_restart_barrier(n_live_threads);
    }
    if (GC_manual_vdb)
      GC_release_dirty_lock(); /* cannot be done in GC_suspend_all */
# endif

# ifdef PARALLEL_MARK
    if (GC_parallel)
      GC_release_mark_lock();
# endif
# ifdef DEBUG_THREADS
    GC_log_printf("World stopped from %p\n", (void *)pthread_self());
    GC_stopping_thread = 0;
# endif
}

#ifdef NACL
# if defined(__x86_64__)
#   define NACL_STORE_REGS() \
        do { \
          __asm__ __volatile__ ("push %rbx"); \
          __asm__ __volatile__ ("push %rbp"); \
          __asm__ __volatile__ ("push %r12"); \
          __asm__ __volatile__ ("push %r13"); \
          __asm__ __volatile__ ("push %r14"); \
          __asm__ __volatile__ ("push %r15"); \
          __asm__ __volatile__ ("mov %%esp, %0" \
                    : "=m" (GC_nacl_gc_thread_self -> crtn -> stack_ptr)); \
          BCOPY(GC_nacl_gc_thread_self -> crtn -> stack_ptr, \
                GC_nacl_gc_thread_self -> reg_storage, \
                NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t)); \
          __asm__ __volatile__ ("naclasp $48, %r15"); \
        } while (0)
# elif defined(__i386__)
#   define NACL_STORE_REGS() \
        do { \
          __asm__ __volatile__ ("push %ebx"); \
          __asm__ __volatile__ ("push %ebp"); \
          __asm__ __volatile__ ("push %esi"); \
          __asm__ __volatile__ ("push %edi"); \
          __asm__ __volatile__ ("mov %%esp, %0" \
                    : "=m" (GC_nacl_gc_thread_self -> crtn -> stack_ptr)); \
          BCOPY(GC_nacl_gc_thread_self -> crtn -> stack_ptr, \
                GC_nacl_gc_thread_self -> reg_storage, \
                NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t));\
          __asm__ __volatile__ ("add $16, %esp"); \
        } while (0)
# elif defined(__arm__)
#   define NACL_STORE_REGS() \
        do { \
          __asm__ __volatile__ ("push {r4-r8,r10-r12,lr}"); \
          __asm__ __volatile__ ("mov r0, %0" \
                : : "r" (&GC_nacl_gc_thread_self -> crtn -> stack_ptr)); \
          __asm__ __volatile__ ("bic r0, r0, #0xc0000000"); \
          __asm__ __volatile__ ("str sp, [r0]"); \
          BCOPY(GC_nacl_gc_thread_self -> crtn -> stack_ptr, \
                GC_nacl_gc_thread_self -> reg_storage, \
                NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t)); \
          __asm__ __volatile__ ("add sp, sp, #40"); \
          __asm__ __volatile__ ("bic sp, sp, #0xc0000000"); \
        } while (0)
# else
#   error TODO Please port NACL_STORE_REGS
# endif

  GC_API_OSCALL void nacl_pre_syscall_hook(void)
  {
    if (GC_nacl_thread_idx != -1) {
      NACL_STORE_REGS();
      GC_nacl_gc_thread_self -> crtn -> stack_ptr = GC_approx_sp();
      GC_nacl_thread_parked[GC_nacl_thread_idx] = 1;
    }
  }

  GC_API_OSCALL void __nacl_suspend_thread_if_needed(void)
  {
      if (!GC_nacl_park_threads_now) return;

      /* Don't try to park the thread parker.   */
      if (GC_nacl_thread_parker == pthread_self()) return;

      /* This can happen when a thread is created outside of the GC     */
      /* system (wthread mostly).                                       */
      if (GC_nacl_thread_idx < 0) return;

      /* If it was already 'parked', we're returning from a syscall,    */
      /* so don't bother storing registers again, the GC has a set.     */
      if (!GC_nacl_thread_parked[GC_nacl_thread_idx]) {
        NACL_STORE_REGS();
        GC_nacl_gc_thread_self -> crtn -> stack_ptr = GC_approx_sp();
      }
      GC_nacl_thread_parked[GC_nacl_thread_idx] = 1;
      while (GC_nacl_park_threads_now) {
        /* Just spin.   */
      }
      GC_nacl_thread_parked[GC_nacl_thread_idx] = 0;

      /* Clear out the reg storage for next suspend.    */
      BZERO(GC_nacl_gc_thread_self -> reg_storage,
            NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t));
  }

  GC_API_OSCALL void nacl_post_syscall_hook(void)
  {
    /* Calling __nacl_suspend_thread_if_needed right away should        */
    /* guarantee we don't mutate the GC set.                            */
    __nacl_suspend_thread_if_needed();
    if (GC_nacl_thread_idx != -1) {
      GC_nacl_thread_parked[GC_nacl_thread_idx] = 0;
    }
  }

  STATIC GC_bool GC_nacl_thread_parking_inited = FALSE;
  STATIC pthread_mutex_t GC_nacl_thread_alloc_lock = PTHREAD_MUTEX_INITIALIZER;

  struct nacl_irt_blockhook {
    int (*register_block_hooks)(void (*pre)(void), void (*post)(void));
  };

  EXTERN_C_BEGIN
  extern size_t nacl_interface_query(const char *interface_ident,
                                     void *table, size_t tablesize);
  EXTERN_C_END

  GC_INNER void GC_nacl_initialize_gc_thread(GC_thread me)
  {
    int i;
    static struct nacl_irt_blockhook gc_hook;

    GC_ASSERT(NULL == GC_nacl_gc_thread_self);
    GC_nacl_gc_thread_self = me;
    pthread_mutex_lock(&GC_nacl_thread_alloc_lock);
    if (!EXPECT(GC_nacl_thread_parking_inited, TRUE)) {
      BZERO(GC_nacl_thread_parked, sizeof(GC_nacl_thread_parked));
      BZERO(GC_nacl_thread_used, sizeof(GC_nacl_thread_used));
      /* TODO: replace with public 'register hook' function when        */
      /* available from glibc.                                          */
      nacl_interface_query("nacl-irt-blockhook-0.1",
                           &gc_hook, sizeof(gc_hook));
      gc_hook.register_block_hooks(nacl_pre_syscall_hook,
                                   nacl_post_syscall_hook);
      GC_nacl_thread_parking_inited = TRUE;
    }
    GC_ASSERT(GC_nacl_num_gc_threads <= MAX_NACL_GC_THREADS);
    for (i = 0; i < MAX_NACL_GC_THREADS; i++) {
      if (GC_nacl_thread_used[i] == 0) {
        GC_nacl_thread_used[i] = 1;
        GC_nacl_thread_idx = i;
        GC_nacl_num_gc_threads++;
        break;
      }
    }
    pthread_mutex_unlock(&GC_nacl_thread_alloc_lock);
  }

  GC_INNER void GC_nacl_shutdown_gc_thread(void)
  {
    GC_ASSERT(GC_nacl_gc_thread_self != NULL);
    pthread_mutex_lock(&GC_nacl_thread_alloc_lock);
    GC_ASSERT(GC_nacl_thread_idx >= 0);
    GC_ASSERT(GC_nacl_thread_idx < MAX_NACL_GC_THREADS);
    GC_ASSERT(GC_nacl_thread_used[GC_nacl_thread_idx] != 0);
    GC_nacl_thread_used[GC_nacl_thread_idx] = 0;
    GC_nacl_thread_idx = -1;
    GC_nacl_num_gc_threads--;
    pthread_mutex_unlock(&GC_nacl_thread_alloc_lock);
    GC_nacl_gc_thread_self = NULL;
  }

#else /* !NACL */

  /* Restart all threads that were suspended by the collector.  */
  /* Return the number of restart signals that were sent.       */
  STATIC int GC_restart_all(void)
  {
    int n_live_threads = 0;
    int i;
    pthread_t self = pthread_self();
    GC_thread p;
    int result;

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT((GC_stop_count & THREAD_RESTARTED) != 0);
    for (i = 0; i < THREAD_TABLE_SZ; i++) {
      for (p = GC_threads[i]; p != NULL; p = p -> tm.next) {
        if (!THREAD_EQUAL(p -> id, self)) {
          if ((p -> flags & (FINISHED | DO_BLOCKING)) != 0) continue;
#         ifdef GC_ENABLE_SUSPEND_THREAD
              if ((p -> ext_suspend_cnt & 1) != 0) continue;
#         endif
          if (GC_retry_signals
                && AO_load(&(p -> last_stop_count)) == GC_stop_count)
              continue; /* The thread has been restarted. */
          n_live_threads++;
#         ifdef DEBUG_THREADS
            GC_log_printf("Sending restart signal to %p\n", (void *)p->id);
#         endif
          result = raise_signal(p, GC_sig_thr_restart);
          switch (result) {
            case ESRCH:
              /* Not really there anymore.  Possible?   */
              n_live_threads--;
              break;
            case 0:
              if (GC_on_thread_event)
                GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED,
                                   (void *)(word)THREAD_SYSTEM_ID(p));
              break;
            default:
              ABORT_ARG1("pthread_kill failed at resume",
                         ": errcode= %d", result);
          }
        }
      }
    }
    return n_live_threads;
  }
#endif /* !NACL */

GC_INNER void GC_start_world(void)
{
# ifndef NACL
    int n_live_threads;

    GC_ASSERT(I_HOLD_LOCK()); /* held continuously since the world stopped */
#   ifdef DEBUG_THREADS
      GC_log_printf("World starting\n");
#   endif
    AO_store_release(&GC_stop_count, GC_stop_count + THREAD_RESTARTED);
                    /* The updated value should now be visible to the   */
                    /* signal handler (note that pthread_kill is not on */
                    /* the list of functions which synchronize memory). */
    n_live_threads = GC_restart_all();
    if (GC_retry_signals) {
        resend_lost_signals_retry(n_live_threads, GC_restart_all);
    } else {
#       ifndef GC_NETBSD_THREADS_WORKAROUND
          if (GC_sig_suspend == GC_sig_thr_restart)
#       endif
        {
          suspend_restart_barrier(n_live_threads);
        }
    }
#   ifdef DEBUG_THREADS
      GC_log_printf("World started\n");
#   endif
# else /* NACL */
#   ifdef DEBUG_THREADS
      GC_log_printf("World starting...\n");
#   endif
    GC_nacl_park_threads_now = 0;
    if (GC_on_thread_event)
      GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, NULL);
      /* TODO: Send event for every unsuspended thread. */
# endif
}

GC_INNER void GC_stop_init(void)
{
# if !defined(NACL)
    struct sigaction act;
    char *str;

    if (SIGNAL_UNSET == GC_sig_suspend)
        GC_sig_suspend = SIG_SUSPEND;
    if (SIGNAL_UNSET == GC_sig_thr_restart)
        GC_sig_thr_restart = SIG_THR_RESTART;

    if (sem_init(&GC_suspend_ack_sem, GC_SEM_INIT_PSHARED, 0) != 0)
        ABORT("sem_init failed");
    GC_stop_count = THREAD_RESTARTED; /* i.e. the world is not stopped */

    if (sigfillset(&act.sa_mask) != 0) {
        ABORT("sigfillset failed");
    }
#   ifdef GC_RTEMS_PTHREADS
      if(sigprocmask(SIG_UNBLOCK, &act.sa_mask, NULL) != 0) {
        ABORT("sigprocmask failed");
      }
#   endif
    GC_remove_allowed_signals(&act.sa_mask);
    /* GC_sig_thr_restart is set in the resulting mask. */
    /* It is unmasked by the handler when necessary.    */

#   ifdef SA_RESTART
      act.sa_flags = SA_RESTART;
#   else
      act.sa_flags = 0;
#   endif
#   ifdef SUSPEND_HANDLER_NO_CONTEXT
      act.sa_handler = GC_suspend_handler;
#   else
      act.sa_flags |= SA_SIGINFO;
      act.sa_sigaction = GC_suspend_sigaction;
#   endif
    /* act.sa_restorer is deprecated and should not be initialized. */
    if (sigaction(GC_sig_suspend, &act, NULL) != 0) {
        ABORT("Cannot set SIG_SUSPEND handler");
    }

    if (GC_sig_suspend != GC_sig_thr_restart) {
#     ifndef SUSPEND_HANDLER_NO_CONTEXT
        act.sa_flags &= ~SA_SIGINFO;
#     endif
      act.sa_handler = GC_restart_handler;
      if (sigaction(GC_sig_thr_restart, &act, NULL) != 0)
        ABORT("Cannot set SIG_THR_RESTART handler");
    } else {
      GC_COND_LOG_PRINTF("Using same signal for suspend and restart\n");
    }

    /* Initialize suspend_handler_mask (excluding GC_sig_thr_restart).  */
    if (sigfillset(&suspend_handler_mask) != 0) ABORT("sigfillset failed");
    GC_remove_allowed_signals(&suspend_handler_mask);
    if (sigdelset(&suspend_handler_mask, GC_sig_thr_restart) != 0)
        ABORT("sigdelset failed");

#   ifndef NO_RETRY_SIGNALS
      /* Any platform could lose signals, so let's be conservative and  */
      /* always enable signals retry logic.                             */
      GC_retry_signals = TRUE;
#   endif
    /* Override the default value of GC_retry_signals.  */
    str = GETENV("GC_RETRY_SIGNALS");
    if (str != NULL) {
        GC_retry_signals = *str != '0' || *(str + 1) != '\0';
            /* Do not retry if the environment variable is set to "0". */
    }
    if (GC_retry_signals) {
      GC_COND_LOG_PRINTF(
                "Will retry suspend and restart signals if necessary\n");
    }

#   ifndef NO_SIGNALS_UNBLOCK_IN_MAIN
      /* Explicitly unblock the signals once before new threads creation. */
      GC_unblock_gc_signals();
#   endif
# endif /* !NACL */
}

#endif /* PTHREAD_STOP_WORLD_IMPL */

#endif
/*
 * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
 * Copyright (c) 2000-2008 by Hewlett-Packard Company.  All rights reserved.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


/*
 * Support code originally for LinuxThreads, the clone()-based kernel
 * thread package for Linux which is included in libc6.
 *
 * This code no doubt makes some assumptions beyond what is
 * guaranteed by the pthread standard, though it now does
 * very little of that.  It now also supports NPTL, and many
 * other Posix thread implementations.  We are trying to merge
 * all flavors of pthread support code into this file.
 */

#ifdef THREADS

#ifdef GC_PTHREADS
# if defined(GC_DARWIN_THREADS) \
     || (defined(GC_WIN32_THREADS) && defined(EMULATE_PTHREAD_SEMAPHORE))
/*
 * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
 * Copyright (c) 2000-2009 by Hewlett-Packard Development Company.
 * All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

#ifndef GC_DARWIN_SEMAPHORE_H
#define GC_DARWIN_SEMAPHORE_H


#if !defined(GC_DARWIN_THREADS) && !defined(GC_WIN32_THREADS)
# error darwin_semaphore.h included for improper target
#endif

#include <errno.h>

#ifdef __cplusplus
  extern "C" {
#endif

/* This is a very simple semaphore implementation based on pthreads.    */
/* It is not async-signal safe.  But this is not a problem because      */
/* signals are not used to suspend threads on the target.               */

typedef struct {
    pthread_mutex_t mutex;
    pthread_cond_t cond;
    int value;
} sem_t;

GC_INLINE int sem_init(sem_t *sem, int pshared, int value) {
    if (pshared != 0) {
        errno = EPERM; /* unsupported */
        return -1;
    }
    sem->value = value;
    if (pthread_mutex_init(&sem->mutex, NULL) != 0)
      return -1;
    if (pthread_cond_init(&sem->cond, NULL) != 0) {
      (void)pthread_mutex_destroy(&sem->mutex);
      return -1;
    }
    return 0;
}

GC_INLINE int sem_post(sem_t *sem) {
    if (pthread_mutex_lock(&sem->mutex) != 0)
      return -1;
    sem->value++;
    if (pthread_cond_signal(&sem->cond) != 0) {
      (void)pthread_mutex_unlock(&sem->mutex);
      return -1;
    }
    return pthread_mutex_unlock(&sem->mutex) != 0 ? -1 : 0;
}

GC_INLINE int sem_wait(sem_t *sem) {
    if (pthread_mutex_lock(&sem->mutex) != 0)
      return -1;
    while (sem->value == 0) {
        if (pthread_cond_wait(&sem->cond, &sem->mutex) != 0) {
            (void)pthread_mutex_unlock(&sem->mutex);
            return -1;
        }
    }
    sem->value--;
    return pthread_mutex_unlock(&sem->mutex) != 0 ? -1 : 0;
}

GC_INLINE int sem_destroy(sem_t *sem) {
    return pthread_cond_destroy(&sem->cond) != 0
           || pthread_mutex_destroy(&sem->mutex) != 0 ? -1 : 0;
}

#ifdef __cplusplus
  } /* extern "C" */
#endif

#endif

# elif !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
#   include <semaphore.h>
# endif
# include <errno.h>
#endif /* GC_PTHREADS */

#ifndef GC_WIN32_THREADS
# include <sched.h>
# include <time.h>
# if !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
#   if !defined(GC_RTEMS_PTHREADS)
#     include <sys/mman.h>
#   endif
#   include <sys/time.h>
#   include <sys/stat.h>
#   include <fcntl.h>
# endif
# if defined(GC_EXPLICIT_SIGNALS_UNBLOCK) || !defined(GC_NO_PTHREAD_SIGMASK) \
     || (defined(GC_PTHREADS_PARAMARK) && !defined(NO_MARKER_SPECIAL_SIGMASK))
#   include <signal.h>
# endif
#endif /* !GC_WIN32_THREADS */

#ifdef E2K
# include <alloca.h>
#endif

#if defined(GC_DARWIN_THREADS) || defined(GC_FREEBSD_THREADS)
# include <sys/sysctl.h>
#endif

#if defined(GC_NETBSD_THREADS) || defined(GC_OPENBSD_THREADS)
# include <sys/param.h>
# include <sys/sysctl.h>
#endif

#if defined(GC_DGUX386_THREADS)
# include <sys/dg_sys_info.h>
# include <sys/_int_psem.h>
  /* sem_t is an uint in DG/UX */
  typedef unsigned int sem_t;
#endif /* GC_DGUX386_THREADS */

#if defined(GC_PTHREADS) \
    && !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
  /* Undefine macros used to redirect pthread primitives.       */
# undef pthread_create
# ifndef GC_NO_PTHREAD_SIGMASK
#   undef pthread_sigmask
# endif
# ifndef GC_NO_PTHREAD_CANCEL
#   undef pthread_cancel
# endif
# ifdef GC_HAVE_PTHREAD_EXIT
#   undef pthread_exit
# endif
# undef pthread_join
# undef pthread_detach
# if defined(GC_OSF1_THREADS) && defined(_PTHREAD_USE_MANGLED_NAMES_) \
     && !defined(_PTHREAD_USE_PTDNAM_)
    /* Restore the original mangled names on Tru64 UNIX.        */
#   define pthread_create __pthread_create
#   define pthread_join   __pthread_join
#   define pthread_detach __pthread_detach
#   ifndef GC_NO_PTHREAD_CANCEL
#     define pthread_cancel __pthread_cancel
#   endif
#   ifdef GC_HAVE_PTHREAD_EXIT
#     define pthread_exit __pthread_exit
#   endif
# endif /* GC_OSF1_THREADS */
#endif /* GC_PTHREADS */

#if !defined(GC_WIN32_THREADS) \
    && !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)
  /* TODO: Enable GC_USE_DLOPEN_WRAP for Cygwin? */

# ifdef GC_USE_LD_WRAP
#   define WRAP_FUNC(f) __wrap_##f
#   define REAL_FUNC(f) __real_##f
    int REAL_FUNC(pthread_create)(pthread_t *,
                                  GC_PTHREAD_CREATE_CONST pthread_attr_t *,
                                  void * (*start_routine)(void *), void *);
    int REAL_FUNC(pthread_join)(pthread_t, void **);
    int REAL_FUNC(pthread_detach)(pthread_t);
#   ifndef GC_NO_PTHREAD_SIGMASK
      int REAL_FUNC(pthread_sigmask)(int, const sigset_t *, sigset_t *);
#   endif
#   ifndef GC_NO_PTHREAD_CANCEL
      int REAL_FUNC(pthread_cancel)(pthread_t);
#   endif
#   ifdef GC_HAVE_PTHREAD_EXIT
      void REAL_FUNC(pthread_exit)(void *) GC_PTHREAD_EXIT_ATTRIBUTE;
#   endif
# elif defined(GC_USE_DLOPEN_WRAP)
#   include <dlfcn.h>
#   define WRAP_FUNC(f) f
#   define REAL_FUNC(f) GC_real_##f
    /* We define both GC_f and plain f to be the wrapped function.  */
    /* In that way plain calls work, as do calls from files that    */
    /* included gc.h, which redefined f to GC_f.                    */
    /* FIXME: Needs work for DARWIN and True64 (OSF1) */
    typedef int (*GC_pthread_create_t)(pthread_t *,
                                GC_PTHREAD_CREATE_CONST pthread_attr_t *,
                                void * (*)(void *), void *);
    static GC_pthread_create_t REAL_FUNC(pthread_create);
#   ifndef GC_NO_PTHREAD_SIGMASK
      typedef int (*GC_pthread_sigmask_t)(int, const sigset_t *, sigset_t *);
      static GC_pthread_sigmask_t REAL_FUNC(pthread_sigmask);
#   endif
    typedef int (*GC_pthread_join_t)(pthread_t, void **);
    static GC_pthread_join_t REAL_FUNC(pthread_join);
    typedef int (*GC_pthread_detach_t)(pthread_t);
    static GC_pthread_detach_t REAL_FUNC(pthread_detach);
#   ifndef GC_NO_PTHREAD_CANCEL
      typedef int (*GC_pthread_cancel_t)(pthread_t);
      static GC_pthread_cancel_t REAL_FUNC(pthread_cancel);
#   endif
#   ifdef GC_HAVE_PTHREAD_EXIT
      typedef void (*GC_pthread_exit_t)(void *) GC_PTHREAD_EXIT_ATTRIBUTE;
      static GC_pthread_exit_t REAL_FUNC(pthread_exit);
#   endif
# else
#   define WRAP_FUNC(f) GC_##f
#   ifdef GC_DGUX386_THREADS
#     define REAL_FUNC(f) __d10_##f
#   else
#     define REAL_FUNC(f) f
#   endif
# endif /* !GC_USE_LD_WRAP && !GC_USE_DLOPEN_WRAP */

# if defined(GC_USE_LD_WRAP) || defined(GC_USE_DLOPEN_WRAP)
    /* Define GC_ functions as aliases for the plain ones, which will   */
    /* be intercepted.  This allows files which include gc.h, and hence */
    /* generate references to the GC_ symbols, to see the right ones.   */
    GC_API int GC_pthread_create(pthread_t *t,
                                 GC_PTHREAD_CREATE_CONST pthread_attr_t *a,
                                 void * (*fn)(void *), void *arg)
    {
      return pthread_create(t, a, fn, arg);
    }

#   ifndef GC_NO_PTHREAD_SIGMASK
      GC_API int GC_pthread_sigmask(int how, const sigset_t *mask,
                                    sigset_t *old)
      {
        return pthread_sigmask(how, mask, old);
      }
#   endif /* !GC_NO_PTHREAD_SIGMASK */

    GC_API int GC_pthread_join(pthread_t t, void **res)
    {
      return pthread_join(t, res);
    }

    GC_API int GC_pthread_detach(pthread_t t)
    {
      return pthread_detach(t);
    }

#   ifndef GC_NO_PTHREAD_CANCEL
      GC_API int GC_pthread_cancel(pthread_t t)
      {
        return pthread_cancel(t);
      }
#   endif /* !GC_NO_PTHREAD_CANCEL */

#   ifdef GC_HAVE_PTHREAD_EXIT
      GC_API GC_PTHREAD_EXIT_ATTRIBUTE void GC_pthread_exit(void *retval)
      {
        pthread_exit(retval);
      }
#   endif
# endif /* GC_USE_LD_WRAP || GC_USE_DLOPEN_WRAP */

# ifdef GC_USE_DLOPEN_WRAP
    STATIC GC_bool GC_syms_initialized = FALSE;

    /* Resolve a symbol from the dynamic library (given by a handle)    */
    /* and cast it to the given functional type.                        */
#   define TYPED_DLSYM(fn, h, name) CAST_THRU_UINTPTR(fn, dlsym(h, name))

    STATIC void GC_init_real_syms(void)
    {
      void *dl_handle;

      GC_ASSERT(!GC_syms_initialized);
#     ifdef RTLD_NEXT
        dl_handle = RTLD_NEXT;
#     else
        dl_handle = dlopen("libpthread.so.0", RTLD_LAZY);
        if (NULL == dl_handle) {
          dl_handle = dlopen("libpthread.so", RTLD_LAZY); /* without ".0" */
          if (NULL == dl_handle) ABORT("Couldn't open libpthread");
        }
#     endif
      REAL_FUNC(pthread_create) = TYPED_DLSYM(GC_pthread_create_t, dl_handle,
                                              "pthread_create");
#     ifdef RTLD_NEXT
        if (REAL_FUNC(pthread_create) == 0)
          ABORT("pthread_create not found"
                " (probably -lgc is specified after -lpthread)");
#     endif
#     ifndef GC_NO_PTHREAD_SIGMASK
        REAL_FUNC(pthread_sigmask) = TYPED_DLSYM(GC_pthread_sigmask_t,
                                                 dl_handle, "pthread_sigmask");
#     endif
      REAL_FUNC(pthread_join) = TYPED_DLSYM(GC_pthread_join_t, dl_handle,
                                            "pthread_join");
      REAL_FUNC(pthread_detach) = TYPED_DLSYM(GC_pthread_detach_t, dl_handle,
                                              "pthread_detach");
#     ifndef GC_NO_PTHREAD_CANCEL
        REAL_FUNC(pthread_cancel) = TYPED_DLSYM(GC_pthread_cancel_t,
                                                dl_handle, "pthread_cancel");
#     endif
#     ifdef GC_HAVE_PTHREAD_EXIT
        REAL_FUNC(pthread_exit) = TYPED_DLSYM(GC_pthread_exit_t, dl_handle,
                                              "pthread_exit");
#     endif
      GC_syms_initialized = TRUE;
    }

#   define INIT_REAL_SYMS() if (EXPECT(GC_syms_initialized, TRUE)) {} \
                            else GC_init_real_syms()
# else
#   define INIT_REAL_SYMS() (void)0
# endif /* !GC_USE_DLOPEN_WRAP */

#else
# define WRAP_FUNC(f) GC_##f
# define REAL_FUNC(f) f
# define INIT_REAL_SYMS() (void)0
#endif /* GC_WIN32_THREADS */

#if defined(MPROTECT_VDB) && defined(DARWIN)
  GC_INNER int GC_inner_pthread_create(pthread_t *t,
                                GC_PTHREAD_CREATE_CONST pthread_attr_t *a,
                                void *(*fn)(void *), void *arg)
  {
    INIT_REAL_SYMS();
    return REAL_FUNC(pthread_create)(t, a, fn, arg);
  }
#endif

#ifndef GC_ALWAYS_MULTITHREADED
  GC_INNER GC_bool GC_need_to_lock = FALSE;
#endif

#ifdef THREAD_LOCAL_ALLOC
  /* We must explicitly mark ptrfree and gcj free lists, since the free */
  /* list links wouldn't otherwise be found.  We also set them in the   */
  /* normal free lists, since that involves touching less memory than   */
  /* if we scanned them normally.                                       */
  GC_INNER void GC_mark_thread_local_free_lists(void)
  {
    int i;
    GC_thread p;

    for (i = 0; i < THREAD_TABLE_SZ; ++i) {
      for (p = GC_threads[i]; p != NULL; p = p -> tm.next) {
        if (!KNOWN_FINISHED(p))
          GC_mark_thread_local_fls_for(&p->tlfs);
      }
    }
  }

# if defined(GC_ASSERTIONS)
    /* Check that all thread-local free-lists are completely marked.    */
    /* Also check that thread-specific-data structures are marked.      */
    void GC_check_tls(void)
    {
        int i;
        GC_thread p;

        for (i = 0; i < THREAD_TABLE_SZ; ++i) {
          for (p = GC_threads[i]; p != NULL; p = p -> tm.next) {
            if (!KNOWN_FINISHED(p))
              GC_check_tls_for(&p->tlfs);
          }
        }
#       if defined(USE_CUSTOM_SPECIFIC)
          if (GC_thread_key != 0)
            GC_check_tsd_marks(GC_thread_key);
#       endif
    }
# endif /* GC_ASSERTIONS */
#endif /* THREAD_LOCAL_ALLOC */

#ifdef GC_WIN32_THREADS
  /* A macro for functions and variables which should be accessible     */
  /* from win32_threads.c but otherwise could be static.                */
# define GC_INNER_WIN32THREAD GC_INNER
#else
# define GC_INNER_WIN32THREAD STATIC
#endif

#ifdef PARALLEL_MARK

# if defined(GC_WIN32_THREADS) || defined(USE_PROC_FOR_LIBRARIES) \
     || (defined(IA64) && (defined(HAVE_PTHREAD_ATTR_GET_NP) \
                           || defined(HAVE_PTHREAD_GETATTR_NP)))
    GC_INNER_WIN32THREAD ptr_t GC_marker_sp[MAX_MARKERS - 1] = {0};
                                        /* The cold end of the stack    */
                                        /* for markers.                 */
# endif /* GC_WIN32_THREADS || USE_PROC_FOR_LIBRARIES */

# if defined(IA64) && defined(USE_PROC_FOR_LIBRARIES)
    static ptr_t marker_bsp[MAX_MARKERS - 1] = {0};
# endif

# if defined(GC_DARWIN_THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
    static mach_port_t marker_mach_threads[MAX_MARKERS - 1] = {0};

    /* Used only by GC_suspend_thread_list().   */
    GC_INNER GC_bool GC_is_mach_marker(thread_act_t thread)
    {
      int i;
      for (i = 0; i < GC_markers_m1; i++) {
        if (marker_mach_threads[i] == thread)
          return TRUE;
      }
      return FALSE;
    }
# endif /* GC_DARWIN_THREADS */

# ifdef HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG /* NetBSD */
    static void set_marker_thread_name(unsigned id)
    {
      int err = pthread_setname_np(pthread_self(), "GC-marker-%zu",
                                   (void*)(size_t)id);
      if (EXPECT(err != 0, FALSE))
        WARN("pthread_setname_np failed, errno= %" WARN_PRIdPTR "\n",
             (signed_word)err);
    }

# elif defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) \
       || defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID) \
       || defined(HAVE_PTHREAD_SET_NAME_NP)
#   ifdef HAVE_PTHREAD_SET_NAME_NP
#     include <pthread_np.h>
#   endif
    static void set_marker_thread_name(unsigned id)
    {
      char name_buf[16]; /* pthread_setname_np may fail for longer names */
      int len = sizeof("GC-marker-") - 1;

      /* Compose the name manually as snprintf may be unavailable or    */
      /* "%u directive output may be truncated" warning may occur.      */
      BCOPY("GC-marker-", name_buf, len);
      if (id >= 10)
        name_buf[len++] = (char)('0' + (id / 10) % 10);
      name_buf[len] = (char)('0' + id % 10);
      name_buf[len + 1] = '\0';

#     ifdef HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID /* iOS, OS X */
        (void)pthread_setname_np(name_buf);
#     elif defined(HAVE_PTHREAD_SET_NAME_NP) /* OpenBSD */
        pthread_set_name_np(pthread_self(), name_buf);
#     else /* Linux, Solaris, etc. */
        if (EXPECT(pthread_setname_np(pthread_self(), name_buf) != 0, FALSE))
          WARN("pthread_setname_np failed\n", 0);
#     endif
    }

# elif defined(GC_WIN32_THREADS) && !defined(MSWINCE)
    /* A pointer to SetThreadDescription() which is available since     */
    /* Windows 10.  The function prototype is in processthreadsapi.h.   */
    static FARPROC setThreadDescription_fn;

    GC_INNER void GC_init_win32_thread_naming(HMODULE hK32)
    {
      if (hK32)
        setThreadDescription_fn = GetProcAddress(hK32, "SetThreadDescription");
    }

    static void set_marker_thread_name(unsigned id)
    {
      WCHAR name_buf[16];
      int len = sizeof(L"GC-marker-") / sizeof(WCHAR) - 1;
      HRESULT hr;

      if (!setThreadDescription_fn) return; /* missing SetThreadDescription */

      /* Compose the name manually as swprintf may be unavailable.      */
      BCOPY(L"GC-marker-", name_buf, len * sizeof(WCHAR));
      if (id >= 10)
        name_buf[len++] = (WCHAR)('0' + (id / 10) % 10);
      name_buf[len] = (WCHAR)('0' + id % 10);
      name_buf[len + 1] = 0;

      /* Invoke SetThreadDescription().  Cast the function pointer to word  */
      /* first to avoid "incompatible function types" compiler warning.     */
      hr = (*(HRESULT (WINAPI *)(HANDLE, const WCHAR *))
            (GC_funcptr_uint)setThreadDescription_fn)(GetCurrentThread(),
                                                      name_buf);
      if (hr < 0)
        WARN("SetThreadDescription failed\n", 0);
    }
# else
#   define set_marker_thread_name(id) (void)(id)
# endif

  GC_INNER_WIN32THREAD
# ifdef GC_PTHREADS_PARAMARK
    void *GC_mark_thread(void *id)
# elif defined(MSWINCE)
    DWORD WINAPI GC_mark_thread(LPVOID id)
# else
    unsigned __stdcall GC_mark_thread(void *id)
# endif
  {
    word my_mark_no = 0;
    word id_n = (word)id;
    IF_CANCEL(int cancel_state;)

    if (id_n == GC_WORD_MAX) return 0; /* to prevent a compiler warning */
    DISABLE_CANCEL(cancel_state);
                         /* Mark threads are not cancellable; they      */
                         /* should be invisible to client.              */
    set_marker_thread_name((unsigned)id_n);
#   if defined(GC_WIN32_THREADS) || defined(USE_PROC_FOR_LIBRARIES) \
       || (defined(IA64) && (defined(HAVE_PTHREAD_ATTR_GET_NP) \
                             || defined(HAVE_PTHREAD_GETATTR_NP)))
      GC_marker_sp[id_n] = GC_approx_sp();
#   endif
#   if defined(IA64) && defined(USE_PROC_FOR_LIBRARIES)
      marker_bsp[id_n] = GC_save_regs_in_stack();
#   endif
#   if defined(GC_DARWIN_THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
      marker_mach_threads[id_n] = mach_thread_self();
#   endif
#   if !defined(GC_PTHREADS_PARAMARK)
      GC_marker_Id[id_n] = thread_id_self();
#   endif

    /* Inform GC_start_mark_threads about completion of marker data init. */
    GC_acquire_mark_lock();
    if (0 == --GC_fl_builder_count) /* count may have a negative value */
      GC_notify_all_builder();

    /* GC_mark_no is passed only to allow GC_help_marker to terminate   */
    /* promptly.  This is important if it were called from the signal   */
    /* handler or from the allocator lock acquisition code.  Under      */
    /* Linux, it is not safe to call it from a signal handler, since it */
    /* uses mutexes and condition variables.  Since it is called only   */
    /* here, the argument is unnecessary.                               */
    for (;; ++my_mark_no) {
      if (my_mark_no - GC_mark_no > (word)2) {
        /* Resynchronize if we get far off, e.g. because GC_mark_no     */
        /* wrapped.                                                     */
        my_mark_no = GC_mark_no;
      }
#     ifdef DEBUG_THREADS
        GC_log_printf("Starting helper for mark number %lu (thread %u)\n",
                      (unsigned long)my_mark_no, (unsigned)id_n);
#     endif
      GC_help_marker(my_mark_no);
    }
  }

  GC_INNER_WIN32THREAD int GC_available_markers_m1 = 0;

#endif /* PARALLEL_MARK */

#ifdef GC_PTHREADS_PARAMARK

# ifdef GLIBC_2_1_MUTEX_HACK
    /* Ugly workaround for a linux threads bug in the final versions    */
    /* of glibc 2.1.  Pthread_mutex_trylock sets the mutex owner        */
    /* field even when it fails to acquire the mutex.  This causes      */
    /* pthread_cond_wait to die.  Should not be needed for glibc 2.2.   */
    /* According to the man page, we should use                         */
    /* PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP, but that isn't actually */
    /* defined.                                                         */
    static pthread_mutex_t mark_mutex =
        {0, 0, 0, PTHREAD_MUTEX_ERRORCHECK_NP, {0, 0}};
# else
    static pthread_mutex_t mark_mutex = PTHREAD_MUTEX_INITIALIZER;
# endif

# ifdef CAN_HANDLE_FORK
    static pthread_cond_t mark_cv;
                        /* initialized by GC_start_mark_threads_inner   */
# else
    static pthread_cond_t mark_cv = PTHREAD_COND_INITIALIZER;
# endif

  GC_INNER void GC_start_mark_threads_inner(void)
  {
    int i;
    pthread_attr_t attr;
#   ifndef NO_MARKER_SPECIAL_SIGMASK
      sigset_t set, oldset;
#   endif

    GC_ASSERT(I_HOLD_LOCK());
    ASSERT_CANCEL_DISABLED();
    if (GC_available_markers_m1 <= 0 || GC_parallel) return;
                /* Skip if parallel markers disabled or already started. */
    GC_wait_for_gc_completion(TRUE);

#   ifdef CAN_HANDLE_FORK
      /* Initialize mark_cv (for the first time), or cleanup its value  */
      /* after forking in the child process.  All the marker threads in */
      /* the parent process were blocked on this variable at fork, so   */
      /* pthread_cond_wait() malfunction (hang) is possible in the      */
      /* child process without such a cleanup.                          */
      /* TODO: This is not portable, it is better to shortly unblock    */
      /* all marker threads in the parent process at fork.              */
      {
        pthread_cond_t mark_cv_local = PTHREAD_COND_INITIALIZER;
        BCOPY(&mark_cv_local, &mark_cv, sizeof(mark_cv));
      }
#   endif

    GC_ASSERT(GC_fl_builder_count == 0);
    INIT_REAL_SYMS(); /* for pthread_create */
    if (0 != pthread_attr_init(&attr)) ABORT("pthread_attr_init failed");
    if (0 != pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED))
        ABORT("pthread_attr_setdetachstate failed");

#   ifdef DEFAULT_STACK_MAYBE_SMALL
      /* Default stack size is usually too small: increase it.  */
      /* Otherwise marker threads or GC may run out of space.   */
      {
        size_t old_size;

        if (pthread_attr_getstacksize(&attr, &old_size) != 0)
          ABORT("pthread_attr_getstacksize failed");
        if (old_size < MIN_STACK_SIZE
            && old_size != 0 /* stack size is known */) {
          if (pthread_attr_setstacksize(&attr, MIN_STACK_SIZE) != 0)
            ABORT("pthread_attr_setstacksize failed");
        }
      }
#   endif /* DEFAULT_STACK_MAYBE_SMALL */

#   ifndef NO_MARKER_SPECIAL_SIGMASK
      /* Apply special signal mask to GC marker threads, and don't drop */
      /* user defined signals by GC marker threads.                     */
      if (sigfillset(&set) != 0)
        ABORT("sigfillset failed");

#     ifdef SIGNAL_BASED_STOP_WORLD
        /* These are used by GC to stop and restart the world.  */
        if (sigdelset(&set, GC_get_suspend_signal()) != 0
            || sigdelset(&set, GC_get_thr_restart_signal()) != 0)
          ABORT("sigdelset failed");
#     endif

      if (EXPECT(REAL_FUNC(pthread_sigmask)(SIG_BLOCK,
                                            &set, &oldset) < 0, FALSE)) {
        WARN("pthread_sigmask set failed, no markers started\n", 0);
        GC_markers_m1 = 0;
        (void)pthread_attr_destroy(&attr);
        return;
      }
#   endif /* !NO_MARKER_SPECIAL_SIGMASK */

    /* To have proper GC_parallel value in GC_help_marker.      */
    GC_markers_m1 = GC_available_markers_m1;

    for (i = 0; i < GC_available_markers_m1; ++i) {
      pthread_t new_thread;

#     ifdef GC_WIN32_THREADS
        GC_marker_last_stack_min[i] = ADDR_LIMIT;
#     endif
      if (EXPECT(REAL_FUNC(pthread_create)(&new_thread, &attr, GC_mark_thread,
                                           (void *)(word)i) != 0, FALSE)) {
        WARN("Marker thread %" WARN_PRIdPTR " creation failed\n",
             (signed_word)i);
        /* Don't try to create other marker threads.    */
        GC_markers_m1 = i;
        break;
      }
    }

#   ifndef NO_MARKER_SPECIAL_SIGMASK
      /* Restore previous signal mask.  */
      if (EXPECT(REAL_FUNC(pthread_sigmask)(SIG_SETMASK,
                                            &oldset, NULL) < 0, FALSE)) {
        WARN("pthread_sigmask restore failed\n", 0);
      }
#   endif

    (void)pthread_attr_destroy(&attr);
    GC_wait_for_markers_init();
    GC_COND_LOG_PRINTF("Started %d mark helper threads\n", GC_markers_m1);
  }

#endif /* GC_PTHREADS_PARAMARK */

/* A hash table to keep information about the registered threads.       */
/* Not used if GC_win32_dll_threads is set.                             */
GC_INNER GC_thread GC_threads[THREAD_TABLE_SZ] = {0};

/* It may not be safe to allocate when we register the first thread.    */
/* Note that next and status fields are unused, but there might be some */
/* other fields (crtn) to be pushed.                                    */
static struct GC_StackContext_Rep first_crtn;
static struct GC_Thread_Rep first_thread;

/* A place to retain a pointer to an allocated object while a thread    */
/* registration is ongoing.  Protected by the allocator lock.           */
static GC_stack_context_t saved_crtn = NULL;

#ifdef GC_ASSERTIONS
  GC_INNER GC_bool GC_thr_initialized = FALSE;
#endif

void GC_push_thread_structures(void)
{
  GC_ASSERT(I_HOLD_LOCK());
# if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
    if (GC_win32_dll_threads) {
      /* Unlike the other threads implementations, the thread table     */
      /* here contains no pointers to the collectible heap (note also   */
      /* that GC_PTHREADS is incompatible with DllMain-based thread     */
      /* registration).  Thus we have no private structures we need     */
      /* to preserve.                                                   */
    } else
# endif
  /* else */ {
    GC_push_all(&GC_threads, (ptr_t)(&GC_threads) + sizeof(GC_threads));
    GC_ASSERT(NULL == first_thread.tm.next);
#   ifdef GC_PTHREADS
      GC_ASSERT(NULL == first_thread.status);
#   endif
    GC_PUSH_ALL_SYM(first_thread.crtn);
    GC_PUSH_ALL_SYM(saved_crtn);
  }
# if defined(THREAD_LOCAL_ALLOC) && defined(USE_CUSTOM_SPECIFIC)
    GC_PUSH_ALL_SYM(GC_thread_key);
# endif
}

#if defined(MPROTECT_VDB) && defined(GC_WIN32_THREADS)
  GC_INNER void GC_win32_unprotect_thread(GC_thread t)
  {
    GC_ASSERT(I_HOLD_LOCK());
    if (!GC_win32_dll_threads && GC_auto_incremental) {
      GC_stack_context_t crtn = t -> crtn;

      if (crtn != &first_crtn) {
        GC_ASSERT(SMALL_OBJ(GC_size(crtn)));
        GC_remove_protection(HBLKPTR(crtn), 1, FALSE);
      }
      if (t != &first_thread) {
        GC_ASSERT(SMALL_OBJ(GC_size(t)));
        GC_remove_protection(HBLKPTR(t), 1, FALSE);
      }
    }
  }
#endif /* MPROTECT_VDB && GC_WIN32_THREADS */

#ifdef DEBUG_THREADS
  STATIC int GC_count_threads(void)
  {
    int i;
    int count = 0;

#   if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
      if (GC_win32_dll_threads) return -1; /* not implemented */
#   endif
    GC_ASSERT(I_HOLD_READER_LOCK());
    for (i = 0; i < THREAD_TABLE_SZ; ++i) {
        GC_thread p;

        for (p = GC_threads[i]; p != NULL; p = p -> tm.next) {
            if (!KNOWN_FINISHED(p))
                ++count;
        }
    }
    return count;
  }
#endif /* DEBUG_THREADS */

/* Add a thread to GC_threads.  We assume it wasn't already there.      */
/* The id field is set by the caller.                                   */
GC_INNER_WIN32THREAD GC_thread GC_new_thread(thread_id_t self_id)
{
    int hv = THREAD_TABLE_INDEX(self_id);
    GC_thread result;

    GC_ASSERT(I_HOLD_LOCK());
#   ifdef DEBUG_THREADS
        GC_log_printf("Creating thread %p\n", (void *)(signed_word)self_id);
        for (result = GC_threads[hv];
             result != NULL; result = result -> tm.next)
          if (!THREAD_ID_EQUAL(result -> id, self_id)) {
            GC_log_printf("Hash collision at GC_threads[%d]\n", hv);
            break;
          }
#   endif
    if (EXPECT(NULL == first_thread.crtn, FALSE)) {
        result = &first_thread;
        first_thread.crtn = &first_crtn;
        GC_ASSERT(NULL == GC_threads[hv]);
#       ifdef CPPCHECK
          GC_noop1((unsigned char)first_thread.flags_pad[0]);
#         if defined(THREAD_SANITIZER) && defined(SIGNAL_BASED_STOP_WORLD)
            GC_noop1((unsigned char)first_crtn.dummy[0]);
#         endif
#         ifndef GC_NO_FINALIZATION
            GC_noop1((unsigned char)first_crtn.fnlz_pad[0]);
#         endif
#       endif
    } else {
        GC_stack_context_t crtn;

        GC_ASSERT(!GC_win32_dll_threads);
        GC_ASSERT(!GC_in_thread_creation);
        GC_in_thread_creation = TRUE; /* OK to collect from unknown thread */
        crtn = (GC_stack_context_t)GC_INTERNAL_MALLOC(
                        sizeof(struct GC_StackContext_Rep), NORMAL);

        /* The current stack is not scanned until the thread is         */
        /* registered, thus crtn pointer is to be retained in the       */
        /* global data roots for a while (and pushed explicitly if      */
        /* a collection occurs here).                                   */
        GC_ASSERT(NULL == saved_crtn);
        saved_crtn = crtn;
        result = (GC_thread)GC_INTERNAL_MALLOC(sizeof(struct GC_Thread_Rep),
                                               NORMAL);
        saved_crtn = NULL; /* no more collections till thread is registered */
        GC_in_thread_creation = FALSE;
        if (NULL == crtn || NULL == result)
          ABORT("Failed to allocate memory for thread registering");
        result -> crtn = crtn;
    }
    /* The id field is not set here. */
#   ifdef USE_TKILL_ON_ANDROID
      result -> kernel_id = gettid();
#   endif
    result -> tm.next = GC_threads[hv];
    GC_threads[hv] = result;
#   ifdef NACL
      GC_nacl_initialize_gc_thread(result);
#   endif
    GC_ASSERT(0 == result -> flags);
    if (EXPECT(result != &first_thread, TRUE))
      GC_dirty(result);
    return result;
}

/* Delete a thread from GC_threads.  We assume it is there.  (The code  */
/* intentionally traps if it was not.)  It is also safe to delete the   */
/* main thread.  If GC_win32_dll_threads is set, it should be called    */
/* only from the thread being deleted.  If a thread has been joined,    */
/* but we have not yet been notified, then there may be more than one   */
/* thread in the table with the same thread id - this is OK because we  */
/* delete a specific one.                                               */
GC_INNER_WIN32THREAD void GC_delete_thread(GC_thread t)
{
# if defined(GC_WIN32_THREADS) && !defined(MSWINCE)
    CloseHandle(t -> handle);
# endif
# if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
    if (GC_win32_dll_threads) {
      /* This is intended to be lock-free.  It is either called         */
      /* synchronously from the thread being deleted, or by the joining */
      /* thread.  In this branch asynchronous changes to (*t) are       */
      /* possible.  Note that it is not allowed to call GC_printf (and  */
      /* the friends) here, see GC_stop_world() in win32_threads.c for  */
      /* the information.                                               */
      t -> crtn -> stack_end = NULL;
      t -> id = 0;
      t -> flags = 0; /* !IS_SUSPENDED */
#     ifdef RETRY_GET_THREAD_CONTEXT
        t -> context_sp = NULL;
#     endif
      AO_store_release(&(t -> tm.in_use), FALSE);
    } else
# endif
  /* else */ {
    thread_id_t id = t -> id;
    int hv = THREAD_TABLE_INDEX(id);
    GC_thread p;
    GC_thread prev = NULL;

    GC_ASSERT(I_HOLD_LOCK());
#   if defined(DEBUG_THREADS) && !defined(MSWINCE) \
       && (!defined(MSWIN32) || defined(CONSOLE_LOG))
      GC_log_printf("Deleting thread %p, n_threads= %d\n",
                    (void *)(signed_word)id, GC_count_threads());
#   endif
    for (p = GC_threads[hv]; p != t; p = p -> tm.next) {
      prev = p;
    }
    if (NULL == prev) {
        GC_threads[hv] = p -> tm.next;
    } else {
        GC_ASSERT(prev != &first_thread);
        prev -> tm.next = p -> tm.next;
        GC_dirty(prev);
    }
    if (EXPECT(p != &first_thread, TRUE)) {
#     ifdef GC_DARWIN_THREADS
        mach_port_deallocate(mach_task_self(), p -> mach_thread);
#     endif
      GC_ASSERT(p -> crtn != &first_crtn);
      GC_INTERNAL_FREE(p -> crtn);
      GC_INTERNAL_FREE(p);
    }
  }
}

/* Return a GC_thread corresponding to a given thread id, or    */
/* NULL if it is not there.  Caller holds the allocator lock    */
/* at least in the reader mode or otherwise inhibits updates.   */
/* If there is more than one thread with the given id, we       */
/* return the most recent one.                                  */
GC_INNER GC_thread GC_lookup_thread(thread_id_t id)
{
  GC_thread p;

# if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
    if (GC_win32_dll_threads)
      return GC_win32_dll_lookup_thread(id);
# endif
  for (p = GC_threads[THREAD_TABLE_INDEX(id)];
       p != NULL; p = p -> tm.next) {
    if (EXPECT(THREAD_ID_EQUAL(p -> id, id), TRUE)) break;
  }
  return p;
}

/* Same as GC_self_thread_inner() but acquires the allocator lock (in   */
/* the reader mode).                                                    */
STATIC GC_thread GC_self_thread(void) {
  GC_thread p;

  READER_LOCK();
  p = GC_self_thread_inner();
  READER_UNLOCK();
  return p;
}

#ifndef GC_NO_FINALIZATION
  /* Called by GC_finalize() (in case of an allocation failure observed). */
  GC_INNER void GC_reset_finalizer_nested(void)
  {
    GC_ASSERT(I_HOLD_LOCK());
    GC_thread me;
    me = GC_self_thread_inner();
    if (NULL == me) return; // __v_, make the sanitizers and -cstrict happy
    me -> crtn -> finalizer_nested = 0;
  }

  /* Checks and updates the thread-local level of finalizers recursion. */
  /* Returns NULL if GC_invoke_finalizers() should not be called by the */
  /* collector (to minimize the risk of a deep finalizers recursion),   */
  /* otherwise returns a pointer to the thread-local finalizer_nested.  */
  /* Called by GC_notify_or_invoke_finalizers() only.                   */
  GC_INNER unsigned char *GC_check_finalizer_nested(void)
  {
    GC_thread me;
    GC_stack_context_t crtn;
    unsigned nesting_level;

    GC_ASSERT(I_HOLD_LOCK());
    me = GC_self_thread_inner();
#   if defined(INCLUDE_LINUX_THREAD_DESCR) && defined(REDIRECT_MALLOC)
      /* As noted in GC_pthread_start, an allocation may happen in  */
      /* GC_get_stack_base, causing GC_notify_or_invoke_finalizers  */
      /* to be called before the thread gets registered.            */
      if (EXPECT(NULL == me, FALSE)) return NULL;
#   endif
    if (NULL == me) return NULL; // __v_, make the sanitizers and -cstrict happy
    crtn = me -> crtn;
    nesting_level = crtn -> finalizer_nested;
    if (nesting_level) {
      /* We are inside another GC_invoke_finalizers().          */
      /* Skip some implicitly-called GC_invoke_finalizers()     */
      /* depending on the nesting (recursion) level.            */
      if (++(crtn -> finalizer_skipped) < (1U << nesting_level))
        return NULL;
      crtn -> finalizer_skipped = 0;
    }
    crtn -> finalizer_nested = (unsigned char)(nesting_level + 1);
    return &(crtn -> finalizer_nested);
  }
#endif /* !GC_NO_FINALIZATION */

#define ADDR_INSIDE_OBJ(p, obj) \
            ADDR_INSIDE(p, (ptr_t)(&(obj)), (ptr_t)(&(obj)) + sizeof(obj))

#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
  /* This is called from thread-local GC_malloc(). */
  GC_bool GC_is_thread_tsd_valid(void *tsd)
  {
    GC_thread me = GC_self_thread();

    return ADDR_INSIDE_OBJ((ptr_t)tsd, me -> tlfs);
  }
#endif /* GC_ASSERTIONS && THREAD_LOCAL_ALLOC */

GC_API int GC_CALL GC_thread_is_registered(void)
{
  /* TODO: Use GC_get_tlfs() instead. */
  GC_thread me = GC_self_thread();

  return me != NULL && !KNOWN_FINISHED(me);
}

GC_API void GC_CALL GC_register_altstack(void *normstack,
                                         size_t normstack_size,
                                         void *altstack, size_t altstack_size)
{
#ifdef GC_WIN32_THREADS
  /* TODO: Implement */
  UNUSED_ARG(normstack);
  UNUSED_ARG(normstack_size);
  UNUSED_ARG(altstack);
  UNUSED_ARG(altstack_size);
#else
  GC_thread me;
  GC_stack_context_t crtn;

  READER_LOCK();
  me = GC_self_thread_inner();
  if (EXPECT(NULL == me, FALSE)) {
    /* We are called before GC_thr_init. */
    me = &first_thread;
  }
  crtn = me -> crtn;
  crtn -> normstack = (ptr_t)normstack;
  crtn -> normstack_size = normstack_size;
  crtn -> altstack = (ptr_t)altstack;
  crtn -> altstack_size = altstack_size;
  READER_UNLOCK_RELEASE();
#endif
}

#ifdef USE_PROC_FOR_LIBRARIES
  GC_INNER GC_bool GC_segment_is_thread_stack(ptr_t lo, ptr_t hi)
  {
    int i;
    GC_thread p;

    GC_ASSERT(I_HOLD_READER_LOCK());
#   ifdef PARALLEL_MARK
      for (i = 0; i < GC_markers_m1; ++i) {
        if (ADDR_LT(lo, GC_marker_sp[i]) && ADDR_LT(GC_marker_sp[i], hi))
          return TRUE;
#       ifdef IA64
          if (ADDR_LT(lo, marker_bsp[i]) && ADDR_LT(marker_bsp[i], hi))
            return TRUE;
#       endif
      }
#   endif
    for (i = 0; i < THREAD_TABLE_SZ; i++) {
      for (p = GC_threads[i]; p != NULL; p = p -> tm.next) {
        ptr_t stack_end = p -> crtn -> stack_end;

        if (stack_end != NULL) {
#         ifdef STACK_GROWS_UP
            if (ADDR_INSIDE(stack_end, lo, hi))
              return TRUE;
#         else
            if (ADDR_LT(lo, stack_end) && ADDR_GE(hi, stack_end))
              return TRUE;
#         endif
        }
      }
    }
    return FALSE;
  }
#endif /* USE_PROC_FOR_LIBRARIES */

#if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
    && defined(IA64)
  /* Find the largest stack base smaller than bound.  May be used       */
  /* to find the boundary between a register stack and adjacent         */
  /* immediately preceding memory stack.                                */
  GC_INNER ptr_t GC_greatest_stack_base_below(ptr_t bound)
  {
    int i;
    GC_thread p;
    ptr_t result = NULL;

    GC_ASSERT(I_HOLD_READER_LOCK());
#   ifdef PARALLEL_MARK
      for (i = 0; i < GC_markers_m1; ++i) {
        if (ADDR_LT(result, GC_marker_sp[i])
            && ADDR_LT(GC_marker_sp[i], bound))
          result = GC_marker_sp[i];
      }
#   endif
    for (i = 0; i < THREAD_TABLE_SZ; i++) {
      for (p = GC_threads[i]; p != NULL; p = p -> tm.next) {
        ptr_t stack_end = p -> crtn -> stack_end;

        if (ADDR_LT(result, stack_end) && ADDR_LT(stack_end, bound))
          result = stack_end;
      }
    }
    return result;
  }
#endif /* IA64 */

#ifndef STAT_READ
# define STAT_READ read
        /* If read is wrapped, this may need to be redefined to call    */
        /* the real one.                                                */
#endif

#ifdef GC_HPUX_THREADS
# define GC_get_nprocs() pthread_num_processors_np()

#elif defined(GC_OSF1_THREADS) || defined(GC_AIX_THREADS) \
      || defined(GC_HAIKU_THREADS) || defined(GC_SOLARIS_THREADS) \
      || defined(HURD) || defined(HOST_ANDROID) || defined(NACL)
  GC_INLINE int GC_get_nprocs(void)
  {
    int nprocs = (int)sysconf(_SC_NPROCESSORS_ONLN);
    return nprocs > 0 ? nprocs : 1; /* ignore error silently */
  }

#elif defined(GC_IRIX_THREADS)
  GC_INLINE int GC_get_nprocs(void)
  {
    int nprocs = (int)sysconf(_SC_NPROC_ONLN);
    return nprocs > 0 ? nprocs : 1; /* ignore error silently */
  }

#elif defined(GC_LINUX_THREADS) /* && !HOST_ANDROID && !NACL */
  /* Return the number of processors. */
  STATIC int GC_get_nprocs(void)
  {
    /* Should be "return sysconf(_SC_NPROCESSORS_ONLN);" but that     */
    /* appears to be buggy in many cases.                             */
    /* We look for lines "cpu<n>" in /proc/stat.                      */
#   define PROC_STAT_BUF_SZ ((1 + MAX_MARKERS) * 100) /* should be enough */
    /* No need to read the entire /proc/stat to get maximum cpu<N> as   */
    /* - the requested lines are located at the beginning of the file;  */
    /* - the lines with cpu<N> where N > MAX_MARKERS are not needed.    */
    char stat_buf[PROC_STAT_BUF_SZ+1];
    int f;
    int result, i, len;

    f = open("/proc/stat", O_RDONLY);
    if (f < 0) {
      WARN("Could not open /proc/stat\n", 0);
      return 1; /* assume an uniprocessor */
    }
    len = STAT_READ(f, stat_buf, sizeof(stat_buf)-1);
    /* Unlikely that we need to retry because of an incomplete read here. */
    if (len < 0) {
      WARN("Failed to read /proc/stat, errno= %" WARN_PRIdPTR "\n",
           (signed_word)errno);
      close(f);
      return 1;
    }
    stat_buf[len] = '\0'; /* to avoid potential buffer overrun by atoi() */
    close(f);

    result = 1;
        /* Some old kernels only have a single "cpu nnnn ..."   */
        /* entry in /proc/stat.  We identify those as           */
        /* uniprocessors.                                       */

    for (i = 0; i < len - 4; ++i) {
      if (stat_buf[i] == '\n' && stat_buf[i+1] == 'c'
          && stat_buf[i+2] == 'p' && stat_buf[i+3] == 'u') {
        int cpu_no = atoi(&stat_buf[i + 4]);
        if (cpu_no >= result)
          result = cpu_no + 1;
      }
    }
    return result;
  }

#elif defined(GC_DGUX386_THREADS)
  /* Return the number of processors, or i <= 0 if it can't be determined. */
  STATIC int GC_get_nprocs(void)
  {
    int numCpus;
    struct dg_sys_info_pm_info pm_sysinfo;
    int status = 0;

    status = dg_sys_info((long int *) &pm_sysinfo,
        DG_SYS_INFO_PM_INFO_TYPE, DG_SYS_INFO_PM_CURRENT_VERSION);
    if (status < 0) {
       /* set -1 for error */
       numCpus = -1;
    } else {
      /* Active CPUs */
      numCpus = pm_sysinfo.idle_vp_count;
    }
    return numCpus;
  }

#elif defined(GC_DARWIN_THREADS) || defined(GC_FREEBSD_THREADS) \
      || defined(GC_NETBSD_THREADS) || defined(GC_OPENBSD_THREADS)
  STATIC int GC_get_nprocs(void)
  {
    int mib[] = {CTL_HW,HW_NCPU};
    int res;
    size_t len = sizeof(res);

    sysctl(mib, sizeof(mib)/sizeof(int), &res, &len, NULL, 0);
    return res;
  }

#else
  /* E.g., GC_RTEMS_PTHREADS */
# define GC_get_nprocs() 1 /* not implemented */
#endif /* !GC_LINUX_THREADS && !GC_DARWIN_THREADS && ... */

#if defined(ARM32) && defined(GC_LINUX_THREADS) && !defined(NACL)
  /* Some buggy Linux/arm kernels show only non-sleeping CPUs in        */
  /* /proc/stat (and /proc/cpuinfo), so another data system source is   */
  /* tried first.  Result <= 0 on error.                                */
  STATIC int GC_get_nprocs_present(void)
  {
    char stat_buf[16];
    int f;
    int len;

    f = open("/sys/devices/system/cpu/present", O_RDONLY);
    if (f < 0)
      return -1; /* cannot open the file */

    len = STAT_READ(f, stat_buf, sizeof(stat_buf));
    close(f);

    /* Recognized file format: "0\n" or "0-<max_cpu_id>\n"      */
    /* The file might probably contain a comma-separated list   */
    /* but we do not need to handle it (just silently ignore).  */
    if (len < 2 || stat_buf[0] != '0' || stat_buf[len - 1] != '\n') {
      return 0; /* read error or unrecognized content */
    } else if (len == 2) {
      return 1; /* an uniprocessor */
    } else if (stat_buf[1] != '-') {
      return 0; /* unrecognized content */
    }

    stat_buf[len - 1] = '\0'; /* terminate the string */
    return atoi(&stat_buf[2]) + 1; /* skip "0-" and parse max_cpu_num */
  }
#endif /* ARM32 && GC_LINUX_THREADS && !NACL */

#if defined(CAN_HANDLE_FORK) && defined(THREAD_SANITIZER)

  /* Workaround for TSan which does not notice that the allocator lock  */
  /* is acquired in fork_prepare_proc().                                */
  GC_ATTR_NO_SANITIZE_THREAD
  static GC_bool collection_in_progress(void)
  {
    return GC_mark_state != MS_NONE;
  }
#else
# define collection_in_progress() GC_collection_in_progress()
#endif

/* We hold the allocator lock.  Wait until an in-progress GC has        */
/* finished.  Repeatedly releases the allocator lock in order to wait.  */
/* If wait_for_all is true, then we exit with the allocator lock held   */
/* and no collection is in progress; otherwise we just wait for the     */
/* current collection to finish.                                        */
GC_INNER void GC_wait_for_gc_completion(GC_bool wait_for_all)
{
# if !defined(THREAD_SANITIZER) || !defined(CAN_CALL_ATFORK)
    /* GC_lock_holder is accessed with the allocator lock held, so      */
    /* there is no data race actually (unlike what's reported by TSan). */
    GC_ASSERT(I_HOLD_LOCK());
# endif
  ASSERT_CANCEL_DISABLED();
# ifdef GC_DISABLE_INCREMENTAL
    (void)wait_for_all;
# else
    if (GC_incremental && collection_in_progress()) {
        word old_gc_no = GC_gc_no;

        /* Make sure that no part of our stack is still on the mark     */
        /* stack, since it's about to be unmapped.                      */
#       ifdef LINT2
          /* Note: do not transform this if-do-while construction into  */
          /* a single while statement because it might cause some       */
          /* static code analyzers to report a false positive (FP)      */
          /* code defect about missing unlock after lock.               */
#       endif
        do {
            ENTER_GC();
            GC_ASSERT(!GC_in_thread_creation);
            GC_in_thread_creation = TRUE;
            GC_collect_a_little_inner(1);
            GC_in_thread_creation = FALSE;
            EXIT_GC();

            UNLOCK();
#           ifdef GC_WIN32_THREADS
              Sleep(0);
#           else
              sched_yield();
#           endif
            LOCK();
        } while (GC_incremental && collection_in_progress()
                 && (wait_for_all || old_gc_no == GC_gc_no));
    }
# endif
}

#ifdef CAN_HANDLE_FORK

  /* Procedures called before and after a fork.  The goal here is to    */
  /* make it safe to call GC_malloc() in a forked child.  It is unclear */
  /* that is attainable, since the single UNIX spec seems to imply that */
  /* one should only call async-signal-safe functions, and we probably  */
  /* cannot quite guarantee that.  But we give it our best shot.  (That */
  /* same spec also implies that it is not safe to call the system      */
  /* malloc between fork and exec.  Thus we're doing no worse than it.) */

  IF_CANCEL(static int fork_cancel_state;)
                                /* protected by the allocator lock */

# ifdef PARALLEL_MARK
#   ifdef THREAD_SANITIZER
#     if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
        STATIC void GC_generic_lock(pthread_mutex_t *);
#     endif
      GC_ATTR_NO_SANITIZE_THREAD
      static void wait_for_reclaim_atfork(void);
#   else
#     define wait_for_reclaim_atfork() GC_wait_for_reclaim()
#   endif
# endif /* PARALLEL_MARK */

  /* Prevent TSan false positive about the race during items removal    */
  /* from GC_threads.  (The race cannot happen since only one thread    */
  /* survives in the child.)                                            */
# ifdef CAN_CALL_ATFORK
    GC_ATTR_NO_SANITIZE_THREAD
# endif
  static void store_to_threads_table(int hv, GC_thread me)
  {
    GC_threads[hv] = me;
  }

  /* Remove all entries from the GC_threads table, except the one for   */
  /* the current thread.  We need to do this in the child process after */
  /* a fork(), since only the current thread survives in the child.     */
  STATIC void GC_remove_all_threads_but_me(void)
  {
    int hv;
    GC_thread me = NULL;
    pthread_t self = pthread_self(); /* same as in parent */
#   ifndef GC_WIN32_THREADS
#     define pthread_id id
#   endif

    for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
      GC_thread p, next;

      for (p = GC_threads[hv]; p != NULL; p = next) {
        next = p -> tm.next;
        if (THREAD_EQUAL(p -> pthread_id, self)
            && me == NULL) { /* ignore dead threads with the same id */
          me = p;
          p -> tm.next = NULL;
        } else {
#         ifdef THREAD_LOCAL_ALLOC
            if (!KNOWN_FINISHED(p)) {
              /* Cannot call GC_destroy_thread_local here.  The free    */
              /* lists may be in an inconsistent state (as thread p may */
              /* be updating one of the lists by GC_generic_malloc_many */
              /* or GC_FAST_MALLOC_GRANS when fork is invoked).         */
              /* This should not be a problem because the lost elements */
              /* of the free lists will be collected during GC.         */
              GC_remove_specific_after_fork(GC_thread_key, p -> pthread_id);
            }
#         endif
          /* TODO: To avoid TSan hang (when updating GC_bytes_freed),   */
          /* we just skip explicit freeing of GC_threads entries.       */
#         if !defined(THREAD_SANITIZER) || !defined(CAN_CALL_ATFORK)
            if (p != &first_thread) {
              /* TODO: Should call mach_port_deallocate? */
              GC_ASSERT(p -> crtn != &first_crtn);
              GC_INTERNAL_FREE(p -> crtn);
              GC_INTERNAL_FREE(p);
            }
#         endif
        }
      }
      store_to_threads_table(hv, NULL);
    }

#   if defined(CPPCHECK) || defined(LINT2)
      if (NULL == me) ABORT("Current thread is not found after fork");
#   else
      GC_ASSERT(me != NULL);
#   endif
#   ifdef GC_WIN32_THREADS
      /* Update Win32 thread id and handle.     */
      me -> id = thread_id_self(); /* differs from that in parent */
#     ifndef MSWINCE
        if (!DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
                        GetCurrentProcess(), (HANDLE *)&(me -> handle),
                        0 /* dwDesiredAccess */, FALSE /* bInheritHandle */,
                        DUPLICATE_SAME_ACCESS))
          ABORT("DuplicateHandle failed");
#     endif
#   endif
#   ifdef GC_DARWIN_THREADS
      /* Update thread Id after fork (it is OK to call  */
      /* GC_destroy_thread_local and GC_free_inner      */
      /* before update).                                */
      me -> mach_thread = mach_thread_self();
#   endif
#   ifdef USE_TKILL_ON_ANDROID
      me -> kernel_id = gettid();
#   endif

    /* Put "me" back to GC_threads.     */
    store_to_threads_table(THREAD_TABLE_INDEX(me -> id), me);

#   if defined(THREAD_LOCAL_ALLOC) && !defined(USE_CUSTOM_SPECIFIC)
      /* Some TLS implementations (e.g., on Cygwin) might be not        */
      /* fork-friendly, so we re-assign thread-local pointer to 'tlfs'  */
      /* for safety instead of the assertion check (again, it is OK to  */
      /* call GC_destroy_thread_local and GC_free_inner before).        */
      {
        int res = GC_setspecific(GC_thread_key, &me->tlfs);

        if (COVERT_DATAFLOW(res) != 0)
          ABORT("GC_setspecific failed (in child)");
      }
#   endif
#   undef pthread_id
  }

  /* Called before a fork().    */
# if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
    /* GC_lock_holder is updated safely (no data race actually).        */
    GC_ATTR_NO_SANITIZE_THREAD
# endif
  static void fork_prepare_proc(void)
  {
    /* Acquire all relevant locks, so that after releasing the locks    */
    /* the child will see a consistent state in which monitor           */
    /* invariants hold.  Unfortunately, we can't acquire libc locks     */
    /* we might need, and there seems to be no guarantee that libc      */
    /* must install a suitable fork handler.                            */
    /* Wait for an ongoing GC to finish, since we can't finish it in    */
    /* the (one remaining thread in) the child.                         */

      LOCK();
      DISABLE_CANCEL(fork_cancel_state);
                /* Following waits may include cancellation points. */
#     ifdef PARALLEL_MARK
        if (GC_parallel)
          wait_for_reclaim_atfork();
#     endif
      GC_wait_for_gc_completion(TRUE);
#     ifdef PARALLEL_MARK
        if (GC_parallel) {
#         if defined(THREAD_SANITIZER) && defined(GC_ASSERTIONS) \
             && defined(CAN_CALL_ATFORK)
            /* Prevent TSan false positive about the data race  */
            /* when updating GC_mark_lock_holder.               */
            GC_generic_lock(&mark_mutex);
#         else
            GC_acquire_mark_lock();
#         endif
        }
#     endif
      GC_acquire_dirty_lock();
  }

  /* Called in parent after a fork() (even if the latter failed).       */
# if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
    GC_ATTR_NO_SANITIZE_THREAD
# endif
  static void fork_parent_proc(void)
  {
    GC_release_dirty_lock();
#   ifdef PARALLEL_MARK
      if (GC_parallel) {
#       if defined(THREAD_SANITIZER) && defined(GC_ASSERTIONS) \
           && defined(CAN_CALL_ATFORK)
          /* To match that in fork_prepare_proc. */
          (void)pthread_mutex_unlock(&mark_mutex);
#       else
          GC_release_mark_lock();
#       endif
      }
#   endif
    RESTORE_CANCEL(fork_cancel_state);
    UNLOCK();
  }

  /* Called in child after a fork().    */
# if defined(GC_ASSERTIONS) && defined(CAN_CALL_ATFORK)
    GC_ATTR_NO_SANITIZE_THREAD
# endif
  static void fork_child_proc(void)
  {
    GC_release_dirty_lock();
#   ifndef GC_DISABLE_INCREMENTAL
      GC_dirty_update_child();
#   endif
#   ifdef PARALLEL_MARK
      if (GC_parallel) {
#       if defined(THREAD_SANITIZER) && defined(GC_ASSERTIONS) \
           && defined(CAN_CALL_ATFORK)
          (void)pthread_mutex_unlock(&mark_mutex);
#       else
          GC_release_mark_lock();
#       endif
        /* Turn off parallel marking in the child, since we are probably  */
        /* just going to exec, and we would have to restart mark threads. */
        GC_parallel = FALSE;
      }
#     ifdef THREAD_SANITIZER
        /* TSan does not support threads creation in the child process. */
        GC_available_markers_m1 = 0;
#     endif
#   endif
    /* Clean up the thread table, so that just our thread is left.      */
    GC_remove_all_threads_but_me();
    RESTORE_CANCEL(fork_cancel_state);
    UNLOCK();
    /* Even though after a fork the child only inherits the single      */
    /* thread that called the fork(), if another thread in the parent   */
    /* was attempting to lock the mutex while being held in             */
    /* fork_child_prepare(), the mutex will be left in an inconsistent  */
    /* state in the child after the UNLOCK.  This is the case, at       */
    /* least, in Mac OS X and leads to an unusable GC in the child      */
    /* which will block when attempting to perform any GC operation     */
    /* that acquires the allocation mutex.                              */
#   if defined(USE_PTHREAD_LOCKS) && !defined(GC_WIN32_THREADS)
      GC_ASSERT(I_DONT_HOLD_LOCK());
      /* Reinitialize the mutex.  It should be safe since we are        */
      /* running this in the child which only inherits a single thread. */
      /* pthread_mutex_destroy() and pthread_rwlock_destroy() may       */
      /* return EBUSY, which makes no sense, but that is the reason for */
      /* the need of the reinitialization.                              */
      /* Note: excluded for Cygwin as does not seem to be needed.       */
#     ifdef USE_RWLOCK
        (void)pthread_rwlock_destroy(&GC_allocate_ml);
#       ifdef DARWIN
          /* A workaround for pthread_rwlock_init() fail with EBUSY.    */
          {
            pthread_rwlock_t rwlock_local = PTHREAD_RWLOCK_INITIALIZER;
            BCOPY(&rwlock_local, &GC_allocate_ml, sizeof(GC_allocate_ml));
          }
#       else
          if (pthread_rwlock_init(&GC_allocate_ml, NULL) != 0)
            ABORT("pthread_rwlock_init failed (in child)");
#       endif
#     else
        (void)pthread_mutex_destroy(&GC_allocate_ml);
        /* TODO: Probably some targets might need the default mutex     */
        /* attribute to be passed instead of NULL.                      */
        if (0 != pthread_mutex_init(&GC_allocate_ml, NULL))
          ABORT("pthread_mutex_init failed (in child)");
#     endif
#   endif
  }

  /* Routines for fork handling by client (no-op if pthread_atfork works). */
  GC_API void GC_CALL GC_atfork_prepare(void)
  {
    if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
    if (GC_handle_fork <= 0)
      fork_prepare_proc();
  }

  GC_API void GC_CALL GC_atfork_parent(void)
  {
    if (GC_handle_fork <= 0)
      fork_parent_proc();
  }

  GC_API void GC_CALL GC_atfork_child(void)
  {
    if (GC_handle_fork <= 0)
      fork_child_proc();
  }

  /* Prepare for forks if requested.    */
  GC_INNER_WIN32THREAD void GC_setup_atfork(void)
  {
    if (GC_handle_fork) {
#     ifdef CAN_CALL_ATFORK
        if (pthread_atfork(fork_prepare_proc, fork_parent_proc,
                           fork_child_proc) == 0) {
          /* Handlers successfully registered.  */
          GC_handle_fork = 1;
        } else
#     endif
      /* else */ if (GC_handle_fork != -1)
        ABORT("pthread_atfork failed");
    }
  }

#endif /* CAN_HANDLE_FORK */

#ifdef INCLUDE_LINUX_THREAD_DESCR
  __thread int GC_dummy_thread_local;
#endif

#ifdef PARALLEL_MARK
# ifndef GC_WIN32_THREADS
    static void setup_mark_lock(void);
# endif

  GC_INNER_WIN32THREAD unsigned GC_required_markers_cnt = 0;
                        /* The default value (0) means the number of    */
                        /* markers should be selected automatically.    */

  GC_API void GC_CALL GC_set_markers_count(unsigned markers)
  {
    GC_required_markers_cnt = markers < MAX_MARKERS ? markers : MAX_MARKERS;
  }
#endif /* PARALLEL_MARK */

GC_INNER GC_bool GC_in_thread_creation = FALSE;
                                /* protected by the allocator lock */

GC_INNER_WIN32THREAD void GC_record_stack_base(GC_stack_context_t crtn,
                                               const struct GC_stack_base *sb)
{
# if !defined(GC_DARWIN_THREADS) && !defined(GC_WIN32_THREADS)
    crtn -> stack_ptr = (ptr_t)(sb -> mem_base);
# endif
  if ((crtn -> stack_end = (ptr_t)(sb -> mem_base)) == NULL)
    ABORT("Bad stack base in GC_register_my_thread");
# ifdef E2K
    crtn -> ps_ofs = (size_t)(GC_uintptr_t)(sb -> reg_base);
# elif defined(IA64)
    crtn -> backing_store_end = (ptr_t)(sb -> reg_base);
# elif defined(I386) && defined(GC_WIN32_THREADS)
    crtn -> initial_stack_base = (ptr_t)(sb -> mem_base);
# endif
}

#if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS) \
    || !defined(DONT_USE_ATEXIT)
  GC_INNER_WIN32THREAD thread_id_t GC_main_thread_id;
#endif

#ifndef DONT_USE_ATEXIT
  GC_INNER GC_bool GC_is_main_thread(void)
  {
    GC_ASSERT(GC_thr_initialized);
    return THREAD_ID_EQUAL(GC_main_thread_id, thread_id_self());
  }
#endif /* !DONT_USE_ATEXIT */

#ifndef GC_WIN32_THREADS

STATIC GC_thread GC_register_my_thread_inner(const struct GC_stack_base *sb,
                                             thread_id_t self_id)
{
  GC_thread me;

  GC_ASSERT(I_HOLD_LOCK());
  me = GC_new_thread(self_id);
  me -> id = self_id;
# ifdef GC_DARWIN_THREADS
    me -> mach_thread = mach_thread_self();
# endif
  GC_record_stack_base(me -> crtn, sb);
  return me;
}

  STATIC int GC_nprocs = 1;
                        /* Number of processors.  We may not have       */
                        /* access to all of them, but this is as good   */
                        /* a guess as any ...                           */

GC_INNER void GC_thr_init(void)
{
  GC_ASSERT(I_HOLD_LOCK());
  GC_ASSERT(!GC_thr_initialized);
  GC_ASSERT(ADDR(&GC_threads) % sizeof(ptr_t) == 0);
# ifdef GC_ASSERTIONS
    GC_thr_initialized = TRUE;
# endif
# ifdef CAN_HANDLE_FORK
    GC_setup_atfork();
# endif

# ifdef INCLUDE_LINUX_THREAD_DESCR
    /* Explicitly register the region including the address     */
    /* of a thread-local variable.  This should include thread  */
    /* locals for the main thread, except for those allocated   */
    /* in response to dlopen calls.                             */
    {
      ptr_t thread_local_addr = (ptr_t)(&GC_dummy_thread_local);
      ptr_t main_thread_start, main_thread_end;
      if (!GC_enclosing_writable_mapping(thread_local_addr,
                                &main_thread_start, &main_thread_end)) {
        ABORT("Failed to find TLS mapping for the primordial thread");
      } else {
        /* main_thread_start and main_thread_end are initialized.       */
        GC_add_roots_inner(main_thread_start, main_thread_end, FALSE);
      }
    }
# endif

  /* Set GC_nprocs and GC_available_markers_m1. */
  {
    const char *nprocs_string = GETENV("GC_NPROCS");
    GC_nprocs = -1;
    if (nprocs_string != NULL) GC_nprocs = atoi(nprocs_string);
  }
  if (GC_nprocs <= 0
#     if defined(ARM32) && defined(GC_LINUX_THREADS) && !defined(NACL)
        && (GC_nprocs = GC_get_nprocs_present()) <= 1
                                /* Workaround for some Linux/arm kernels */
#     endif
      )
  {
    GC_nprocs = GC_get_nprocs();
  }
  if (GC_nprocs <= 0) {
    WARN("GC_get_nprocs() returned %" WARN_PRIdPTR "\n",
         (signed_word)GC_nprocs);
    GC_nprocs = 2; /* assume dual-core */
#   ifdef PARALLEL_MARK
      GC_available_markers_m1 = 0; /* but use only one marker */
#   endif
  } else {
#   ifdef PARALLEL_MARK
      {
        const char *markers_string = GETENV("GC_MARKERS");
        int markers = GC_required_markers_cnt;

        if (markers_string != NULL) {
          markers = atoi(markers_string);
          if (markers <= 0 || markers > MAX_MARKERS) {
            WARN("Too big or invalid number of mark threads: %" WARN_PRIdPTR
                 "; using maximum threads\n", (signed_word)markers);
            markers = MAX_MARKERS;
          }
        } else if (0 == markers) {
          /* Unless the client sets the desired number of       */
          /* parallel markers, it is determined based on the    */
          /* number of CPU cores.                               */
          markers = GC_nprocs;
#         if defined(GC_MIN_MARKERS) && !defined(CPPCHECK)
            /* This is primarily for targets without getenv().  */
            if (markers < GC_MIN_MARKERS)
              markers = GC_MIN_MARKERS;
#         endif
          if (markers > MAX_MARKERS)
            markers = MAX_MARKERS; /* silently limit the value */
        }
        GC_available_markers_m1 = markers - 1;
      }
#   endif
  }
  GC_COND_LOG_PRINTF("Number of processors: %d\n", GC_nprocs);

# if defined(BASE_ATOMIC_OPS_EMULATED) && defined(SIGNAL_BASED_STOP_WORLD)
    /* Ensure the process is running on just one CPU core.      */
    /* This is needed because the AO primitives emulated with   */
    /* locks cannot be used inside signal handlers.             */
    {
      cpu_set_t mask;
      int cpu_set_cnt = 0;
      int cpu_lowest_set = 0;
#     ifdef RANDOM_ONE_CPU_CORE
        int cpu_highest_set = 0;
#     endif
      int i = GC_nprocs > 1 ? GC_nprocs : 2; /* check at least 2 cores */

      if (sched_getaffinity(0 /* current process */,
                            sizeof(mask), &mask) == -1)
        ABORT_ARG1("sched_getaffinity failed", ": errno= %d", errno);
      while (i-- > 0)
        if (CPU_ISSET(i, &mask)) {
#         ifdef RANDOM_ONE_CPU_CORE
            if (i + 1 != cpu_lowest_set) cpu_highest_set = i;
#         endif
          cpu_lowest_set = i;
          cpu_set_cnt++;
        }
      if (0 == cpu_set_cnt)
        ABORT("sched_getaffinity returned empty mask");
      if (cpu_set_cnt > 1) {
#       ifdef RANDOM_ONE_CPU_CORE
          if (cpu_lowest_set < cpu_highest_set) {
            /* Pseudo-randomly adjust the bit to set among valid ones.  */
            cpu_lowest_set += (unsigned)getpid() %
                                (cpu_highest_set - cpu_lowest_set + 1);
          }
#       endif
        CPU_ZERO(&mask);
        CPU_SET(cpu_lowest_set, &mask); /* select just one CPU */
        if (sched_setaffinity(0, sizeof(mask), &mask) == -1)
          ABORT_ARG1("sched_setaffinity failed", ": errno= %d", errno);
        WARN("CPU affinity mask is set to %p\n", (word)1 << cpu_lowest_set);
      }
    }
# endif /* BASE_ATOMIC_OPS_EMULATED */

# ifndef GC_DARWIN_THREADS
    GC_stop_init();
# endif

# ifdef PARALLEL_MARK
    if (GC_available_markers_m1 <= 0) {
      /* Disable parallel marking.      */
      GC_parallel = FALSE;
      GC_COND_LOG_PRINTF(
                "Single marker thread, turning off parallel marking\n");
    } else {
      setup_mark_lock();
    }
# endif

  /* Add the initial thread, so we can stop it. */
  {
    struct GC_stack_base sb;
    GC_thread me;
    thread_id_t self_id = thread_id_self();

    sb.mem_base = GC_stackbottom;
    GC_ASSERT(sb.mem_base != NULL);
#   if defined(E2K) || defined(IA64)
      sb.reg_base = GC_register_stackbottom;
#   endif
    GC_ASSERT(NULL == GC_self_thread_inner());
    me = GC_register_my_thread_inner(&sb, self_id);
#   ifndef DONT_USE_ATEXIT
      GC_main_thread_id = self_id;
#   endif
    me -> flags = DETACHED;
  }
}

#endif /* !GC_WIN32_THREADS */

/* Perform all initializations, including those that may require        */
/* allocation, e.g. initialize thread-local free lists if used.         */
/* Must be called before a thread is created.                           */
GC_INNER void GC_init_parallel(void)
{
# ifdef THREAD_LOCAL_ALLOC
    GC_thread me;

    GC_ASSERT(GC_is_initialized);
    LOCK();
    me = GC_self_thread_inner();
    GC_init_thread_local(&me->tlfs);
    UNLOCK();
# endif
# if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
    if (GC_win32_dll_threads) {
      set_need_to_lock();
        /* Cannot intercept thread creation.  Hence we don't know if    */
        /* other threads exist.  However, client is not allowed to      */
        /* create other threads before collector initialization.        */
        /* Thus it's OK not to lock before this.                        */
    }
# endif
}

#if !defined(GC_NO_PTHREAD_SIGMASK) && defined(GC_PTHREADS)
  GC_API int WRAP_FUNC(pthread_sigmask)(int how, const sigset_t *set,
                                        sigset_t *oset)
  {
#   ifdef GC_WIN32_THREADS
      /* pthreads-win32 does not support sigmask.       */
      /* So, nothing required here...                   */
#   else
      sigset_t fudged_set;

      INIT_REAL_SYMS();
      if (EXPECT(set != NULL, TRUE)
          && (how == SIG_BLOCK || how == SIG_SETMASK)) {
        int sig_suspend = GC_get_suspend_signal();

        fudged_set = *set;
        GC_ASSERT(sig_suspend >= 0);
        if (sigdelset(&fudged_set, sig_suspend) != 0)
          ABORT("sigdelset failed");
        set = &fudged_set;
      }
#   endif
    return REAL_FUNC(pthread_sigmask)(how, set, oset);
  }
#endif /* !GC_NO_PTHREAD_SIGMASK */

/* Wrapper for functions that are likely to block for an appreciable    */
/* length of time.                                                      */

#ifdef E2K
  /* Cannot be defined as a function because the stack-allocated buffer */
  /* (pointed to by bs_lo) should be preserved till completion of       */
  /* GC_do_blocking_inner (or GC_suspend_self_blocked).                 */
# define do_blocking_enter(pTopOfStackUnset, me)            \
        do {                                                \
          ptr_t bs_lo;                                      \
          size_t stack_size;                                \
          GC_stack_context_t crtn = (me) -> crtn;           \
                                                            \
          *(pTopOfStackUnset) = FALSE;                      \
          crtn -> stack_ptr = GC_approx_sp();               \
          GC_ASSERT(NULL == crtn -> backing_store_end);     \
          GET_PROCEDURE_STACK_LOCAL(crtn -> ps_ofs,         \
                                    &bs_lo, &stack_size);   \
          crtn -> backing_store_end = bs_lo;                \
          crtn -> backing_store_ptr = bs_lo + stack_size;   \
          (me) -> flags |= DO_BLOCKING;                     \
        } while (0)

#else /* !E2K */
  static void do_blocking_enter(GC_bool *pTopOfStackUnset, GC_thread me)
  {
#   if defined(SPARC) || defined(IA64)
        ptr_t bs_hi = GC_save_regs_in_stack();
        /* TODO: regs saving already done by GC_with_callee_saves_pushed */
#   endif
	if(NULL == me) { exit(12); }
    GC_stack_context_t crtn = me -> crtn;

    GC_ASSERT(I_HOLD_READER_LOCK());
    GC_ASSERT((me -> flags & DO_BLOCKING) == 0);
    *pTopOfStackUnset = FALSE;
#   ifdef SPARC
        crtn -> stack_ptr = bs_hi;
#   else
        crtn -> stack_ptr = GC_approx_sp();
#   endif
#   if defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
        if (NULL == crtn -> topOfStack) {
            /* GC_do_blocking_inner is not called recursively,  */
            /* so topOfStack should be computed now.            */
            *pTopOfStackUnset = TRUE;
            crtn -> topOfStack = GC_FindTopOfStack(0);
        }
#   endif
#   ifdef IA64
        crtn -> backing_store_ptr = bs_hi;
#   endif
    me -> flags |= DO_BLOCKING;
    /* Save context here if we want to support precise stack marking.   */
  }
#endif /* !E2K */

static void do_blocking_leave(GC_thread me, GC_bool topOfStackUnset)
{
    GC_ASSERT(I_HOLD_READER_LOCK());
    me -> flags &= (unsigned char)~DO_BLOCKING;
#   ifdef E2K
      {
        GC_stack_context_t crtn = me -> crtn;

        GC_ASSERT(crtn -> backing_store_end != NULL);
        crtn -> backing_store_ptr = NULL;
        crtn -> backing_store_end = NULL;
      }
#   endif
#   if defined(GC_DARWIN_THREADS) && !defined(DARWIN_DONT_PARSE_STACK)
        if (topOfStackUnset)
          me -> crtn -> topOfStack = NULL; /* make it unset again */
#   else
        (void)topOfStackUnset;
#   endif
}

GC_INNER void GC_do_blocking_inner(ptr_t data, void *context)
{
    struct blocking_data *d = (struct blocking_data *)data;
    GC_thread me;
    GC_bool topOfStackUnset;

    UNUSED_ARG(context);
    READER_LOCK();
    me = GC_self_thread_inner();
    do_blocking_enter(&topOfStackUnset, me);
    READER_UNLOCK_RELEASE();

    d -> client_data = (d -> fn)(d -> client_data);

    READER_LOCK();  /* This will block if the world is stopped. */
#   ifdef LINT2
      {
#        ifdef GC_ASSERTIONS
           GC_thread saved_me = me;
#        endif

         /* The pointer to the GC thread descriptor should not be   */
         /* changed while the thread is registered but a static     */
         /* analysis tool might complain that this pointer value    */
         /* (obtained in the first locked section) is unreliable in */
         /* the second locked section.                              */
         me = GC_self_thread_inner();
         GC_ASSERT(me == saved_me);
      }
#   endif
#   if defined(GC_ENABLE_SUSPEND_THREAD) && defined(SIGNAL_BASED_STOP_WORLD)
      /* Note: this code cannot be moved into do_blocking_leave()   */
      /* otherwise there could be a static analysis tool warning    */
      /* (false positive) about unlock without a matching lock.     */
      while (EXPECT((me -> ext_suspend_cnt & 1) != 0, FALSE)) {
        size_t suspend_cnt = me -> ext_suspend_cnt;
                        /* read suspend counter (number) before unlocking */

        READER_UNLOCK_RELEASE();
        GC_suspend_self_inner(me, suspend_cnt);
        READER_LOCK();
      }
#   endif
    do_blocking_leave(me, topOfStackUnset);
    READER_UNLOCK_RELEASE();
}

#if defined(GC_ENABLE_SUSPEND_THREAD) && defined(SIGNAL_BASED_STOP_WORLD)
  /* Similar to GC_do_blocking_inner() but assuming the allocator lock  */
  /* is held and fn is GC_suspend_self_inner.                           */
  GC_INNER void GC_suspend_self_blocked(ptr_t thread_me, void *context)
  {
    GC_thread me = (GC_thread)thread_me;
    GC_bool topOfStackUnset;

    UNUSED_ARG(context);
    GC_ASSERT(I_HOLD_LOCK());
                        /* The caller holds the allocator lock in the   */
                        /* exclusive mode, thus we require and restore  */
                        /* it to the same mode upon return from the     */
                        /* function.                                    */
    do_blocking_enter(&topOfStackUnset, me);
    while ((me -> ext_suspend_cnt & 1) != 0) {
      size_t suspend_cnt = me -> ext_suspend_cnt;

      UNLOCK();
      GC_suspend_self_inner(me, suspend_cnt);
      LOCK();
    }
    do_blocking_leave(me, topOfStackUnset);
  }
#endif /* GC_ENABLE_SUSPEND_THREAD */

GC_API void GC_CALL GC_set_stackbottom(void *gc_thread_handle,
                                       const struct GC_stack_base *sb)
{
    GC_thread t = (GC_thread)gc_thread_handle;
    GC_stack_context_t crtn;

    GC_ASSERT(sb -> mem_base != NULL);
    if (!EXPECT(GC_is_initialized, TRUE)) {
      GC_ASSERT(NULL == t);
      /* Alter the stack bottom of the primordial thread.       */
      GC_stackbottom = (char*)(sb -> mem_base);
#     if defined(E2K) || defined(IA64)
        GC_register_stackbottom = (ptr_t)(sb -> reg_base);
#     endif
      return;
    }

    if (NULL == t) /* current thread? */
    GC_ASSERT(I_HOLD_READER_LOCK());
      t = GC_self_thread_inner();
    GC_ASSERT(!KNOWN_FINISHED(t));
    if(NULL == t) exit(11);
	crtn = t -> crtn;
    GC_ASSERT((t -> flags & DO_BLOCKING) == 0
              && NULL == crtn -> traced_stack_sect); /* for now */

    crtn -> stack_end = (ptr_t)(sb -> mem_base);
#   ifdef E2K
      crtn -> ps_ofs = (size_t)(GC_uintptr_t)(sb -> reg_base);
#   elif defined(IA64)
      crtn -> backing_store_end = (ptr_t)(sb -> reg_base);
#   endif
#   ifdef GC_WIN32_THREADS
      /* Reset the known minimum (hottest address in the stack). */
      crtn -> last_stack_min = ADDR_LIMIT;
#   endif
}

GC_API void * GC_CALL GC_get_my_stackbottom(struct GC_stack_base *sb)
{
    GC_thread me;
    GC_stack_context_t crtn;

    READER_LOCK();
    me = GC_self_thread_inner();
    if(NULL == me) exit(13);
    /* The thread is assumed to be registered.  */

    crtn = me -> crtn;
    sb -> mem_base = crtn -> stack_end;
#   ifdef E2K
      /* Store the offset in the procedure stack, not address.  */
      sb -> reg_base = (void *)(GC_uintptr_t)(crtn -> ps_ofs);
#   elif defined(IA64)
      sb -> reg_base = crtn -> backing_store_end;
#   endif
    READER_UNLOCK();
    return me; /* gc_thread_handle */
}

/* GC_call_with_gc_active() has the opposite to GC_do_blocking()        */
/* functionality.  It might be called from a user function invoked by   */
/* GC_do_blocking() to temporarily back allow calling any GC function   */
/* and/or manipulating pointers to the garbage collected heap.          */
GC_API void * GC_CALL GC_call_with_gc_active(GC_fn_type fn, void *client_data)
{
    struct GC_traced_stack_sect_s stacksect;
    GC_thread me;
    GC_stack_context_t crtn;
    ptr_t stack_end;
#   ifdef E2K
      ptr_t saved_bs_ptr, saved_bs_end;
      size_t saved_ps_ofs;
#   endif

    READER_LOCK();  /* This will block if the world is stopped. */
    me = GC_self_thread_inner();
    crtn = me -> crtn;

    /* Adjust our stack bottom value (this could happen unless  */
    /* GC_get_stack_base() was used which returned GC_SUCCESS). */
    stack_end = crtn -> stack_end; /* read of a volatile field */
    GC_ASSERT(stack_end != NULL);
    if (HOTTER_THAN(stack_end, (ptr_t)(&stacksect))) {
      crtn -> stack_end = (ptr_t)(&stacksect);
#     if defined(I386) && defined(GC_WIN32_THREADS)
        crtn -> initial_stack_base = (ptr_t)(&stacksect);
#     endif
    }

    if ((me -> flags & DO_BLOCKING) == 0) {
      /* We are not inside GC_do_blocking() - do nothing more.  */
      READER_UNLOCK_RELEASE();
      /* Cast fn to a volatile type to prevent its call inlining.   */
      client_data = (*(GC_fn_type volatile *)&fn)(client_data);
      /* Prevent treating the above as a tail call.     */
      GC_noop1(COVERT_DATAFLOW(ADDR(&stacksect)));
      return client_data; /* result */
    }

#   if defined(GC_ENABLE_SUSPEND_THREAD) && defined(SIGNAL_BASED_STOP_WORLD)
      while (EXPECT((me -> ext_suspend_cnt & 1) != 0, FALSE)) {
        size_t suspend_cnt = me -> ext_suspend_cnt;

        READER_UNLOCK_RELEASE();
        GC_suspend_self_inner(me, suspend_cnt);
        READER_LOCK();
        GC_ASSERT(me -> crtn == crtn);
      }
#   endif

    /* Setup new "stack section".       */
    stacksect.saved_stack_ptr = crtn -> stack_ptr;
#   ifdef E2K
      GC_ASSERT(crtn -> backing_store_end != NULL);
      {
        unsigned long long sz_ull;

        GET_PROCEDURE_STACK_SIZE_INNER(&sz_ull);
        saved_ps_ofs = crtn -> ps_ofs;
        GC_ASSERT(saved_ps_ofs <= (size_t)sz_ull);
        crtn -> ps_ofs = (size_t)sz_ull;
      }
      saved_bs_end = crtn -> backing_store_end;
      saved_bs_ptr = crtn -> backing_store_ptr;
      crtn -> backing_store_ptr = NULL;
      crtn -> backing_store_end = NULL;
#   elif defined(IA64)
      /* This is the same as in GC_call_with_stack_base().      */
      stacksect.backing_store_end = GC_save_regs_in_stack();
      /* Unnecessarily flushes register stack,          */
      /* but that probably doesn't hurt.                */
      stacksect.saved_backing_store_ptr = crtn -> backing_store_ptr;
#   endif
    stacksect.prev = crtn -> traced_stack_sect;
    me -> flags &= (unsigned char)~DO_BLOCKING;
    crtn -> traced_stack_sect = &stacksect;

    READER_UNLOCK_RELEASE();
    client_data = (*(GC_fn_type volatile *)&fn)(client_data);
    GC_ASSERT((me -> flags & DO_BLOCKING) == 0);

    /* Restore original "stack section".        */
    READER_LOCK();
    GC_ASSERT(me -> crtn == crtn);
    GC_ASSERT(crtn -> traced_stack_sect == &stacksect);
#   ifdef CPPCHECK
      GC_noop1_ptr(crtn -> traced_stack_sect);
#   endif
    crtn -> traced_stack_sect = stacksect.prev;
#   ifdef E2K
      GC_ASSERT(NULL == crtn -> backing_store_end);
      crtn -> backing_store_end = saved_bs_end;
      crtn -> backing_store_ptr = saved_bs_ptr;
      crtn -> ps_ofs = saved_ps_ofs;
#   elif defined(IA64)
      crtn -> backing_store_ptr = stacksect.saved_backing_store_ptr;
#   endif
    me -> flags |= DO_BLOCKING;
    crtn -> stack_ptr = stacksect.saved_stack_ptr;
    READER_UNLOCK_RELEASE();
    return client_data; /* result */
}

STATIC void GC_unregister_my_thread_inner(GC_thread me)
{
    GC_ASSERT(I_HOLD_LOCK());
#   ifdef DEBUG_THREADS
      GC_log_printf("Unregistering thread %p, gc_thread= %p, n_threads= %d\n",
                    (void *)(signed_word)(me -> id), (void *)me,
                    GC_count_threads());
#   endif
    GC_ASSERT(!KNOWN_FINISHED(me));
#   if defined(THREAD_LOCAL_ALLOC)
      GC_destroy_thread_local(&me->tlfs);
#   endif
#   ifdef NACL
      GC_nacl_shutdown_gc_thread();
#   endif
#   ifdef GC_PTHREADS
#     if defined(GC_HAVE_PTHREAD_EXIT) || !defined(GC_NO_PTHREAD_CANCEL)
        /* Handle DISABLED_GC flag which is set by the  */
        /* intercepted pthread_cancel or pthread_exit.  */
   	    if(NULL == me) exit(14);
        if ((me -> flags & DISABLED_GC) != 0) {
          GC_dont_gc--;
        }
#     endif
      if ((me -> flags & DETACHED) == 0) {
          me -> flags |= FINISHED;
      } else
#   endif
    /* else */ {
      GC_delete_thread(me);
    }
#   if defined(THREAD_LOCAL_ALLOC)
      /* It is required to call remove_specific defined in specific.c. */
      GC_remove_specific(GC_thread_key);
#   endif
}

GC_API int GC_CALL GC_unregister_my_thread(void)
{
    GC_thread me;
    IF_CANCEL(int cancel_state;)

    /* Client should not unregister the thread explicitly if it */
    /* is registered by DllMain, except for the main thread.    */
#   if !defined(GC_NO_THREADS_DISCOVERY) && defined(GC_WIN32_THREADS)
      GC_ASSERT(!GC_win32_dll_threads
                || THREAD_ID_EQUAL(GC_main_thread_id, thread_id_self()));
#   endif

    LOCK();
    DISABLE_CANCEL(cancel_state);
    /* Wait for any GC that may be marking from our stack to    */
    /* complete before we remove this thread.                   */
    GC_wait_for_gc_completion(FALSE);
    me = GC_self_thread_inner();
#   ifdef DEBUG_THREADS
        GC_log_printf(
                "Called GC_unregister_my_thread on %p, gc_thread= %p\n",
                (void *)(signed_word)thread_id_self(), (void *)me);
#   endif
    GC_ASSERT(THREAD_ID_EQUAL(me -> id, thread_id_self()));
    GC_unregister_my_thread_inner(me);
    RESTORE_CANCEL(cancel_state);
    UNLOCK();
    return GC_SUCCESS;
}

#if !defined(GC_NO_PTHREAD_CANCEL) && defined(GC_PTHREADS)
  /* We should deal with the fact that apparently on Solaris and,       */
  /* probably, on some Linux we can't collect while a thread is         */
  /* exiting, since signals aren't handled properly.  This currently    */
  /* gives rise to deadlocks.  The only workaround seen is to intercept */
  /* pthread_cancel() and pthread_exit(), and disable the collections   */
  /* until the thread exit handler is called.  That's ugly, because we  */
  /* risk growing the heap unnecessarily. But it seems that we don't    */
  /* really have an option in that the process is not in a fully        */
  /* functional state while a thread is exiting.                        */
  GC_API int WRAP_FUNC(pthread_cancel)(pthread_t thread)
  {
#   ifdef CANCEL_SAFE
      GC_thread t;
#   endif

    INIT_REAL_SYMS();
#   ifdef CANCEL_SAFE
      LOCK();
      t = GC_lookup_by_pthread(thread);
      /* We test DISABLED_GC because pthread_exit could be called at    */
      /* the same time.  (If t is NULL then pthread_cancel should       */
      /* return ESRCH.)                                                 */
      if (t != NULL && (t -> flags & DISABLED_GC) == 0) {
        t -> flags |= DISABLED_GC;
        GC_dont_gc++;
      }
      UNLOCK();
#   endif
    return REAL_FUNC(pthread_cancel)(thread);
  }
#endif /* !GC_NO_PTHREAD_CANCEL */

#ifdef GC_HAVE_PTHREAD_EXIT
  GC_API GC_PTHREAD_EXIT_ATTRIBUTE void WRAP_FUNC(pthread_exit)(void *retval)
  {
    GC_thread me;

    INIT_REAL_SYMS();
    LOCK();
    me = GC_self_thread_inner();
    /* We test DISABLED_GC because someone else could call    */
    /* pthread_cancel at the same time.                       */
    if (me != NULL && (me -> flags & DISABLED_GC) == 0) {
      me -> flags |= DISABLED_GC;
      GC_dont_gc++;
    }
    UNLOCK();

    REAL_FUNC(pthread_exit)(retval);
  }
#endif /* GC_HAVE_PTHREAD_EXIT */

GC_API void GC_CALL GC_allow_register_threads(void)
{
  /* Check GC is initialized and the current thread is registered.  */
  GC_ASSERT(GC_self_thread() != NULL);

  INIT_REAL_SYMS(); /* to initialize symbols while single-threaded */
  GC_init_lib_bounds();
  GC_start_mark_threads();
  set_need_to_lock();
}

#if defined(PTHREAD_STOP_WORLD_IMPL) && !defined(NO_SIGNALS_UNBLOCK_IN_MAIN) \
    || defined(GC_EXPLICIT_SIGNALS_UNBLOCK)
  /* Some targets (e.g., Solaris) might require this to be called when  */
  /* doing thread registering from the thread destructor.               */
  GC_INNER void GC_unblock_gc_signals(void)
  {
    sigset_t set;

    INIT_REAL_SYMS(); /* for pthread_sigmask */
    sigemptyset(&set);
    sigaddset(&set, GC_get_suspend_signal());
    sigaddset(&set, GC_get_thr_restart_signal());
    if (REAL_FUNC(pthread_sigmask)(SIG_UNBLOCK, &set, NULL) != 0)
      ABORT("pthread_sigmask failed");
  }
#endif /* PTHREAD_STOP_WORLD_IMPL || GC_EXPLICIT_SIGNALS_UNBLOCK */

GC_API int GC_CALL GC_register_my_thread(const struct GC_stack_base *sb)
{
    GC_thread me;

    if (GC_need_to_lock == FALSE)
        ABORT("Threads explicit registering is not previously enabled");

    /* We lock here, since we want to wait for an ongoing GC.   */
    LOCK();
    me = GC_self_thread_inner();
    if (EXPECT(NULL == me, TRUE)) {
      me = GC_register_my_thread_inner(sb, thread_id_self());
#     ifdef GC_PTHREADS
#       ifdef CPPCHECK
          GC_noop1(me -> flags);
#       endif
        /* Treat as detached, since we do not need to worry about       */
        /* pointer results.                                             */
        me -> flags |= DETACHED;
#     else
        (void)me;
#     endif
    } else
#   ifdef GC_PTHREADS
      /* else */ if (KNOWN_FINISHED(me)) {
        /* This code is executed when a thread is registered from the   */
        /* client thread key destructor.                                */
#       ifdef NACL
          GC_nacl_initialize_gc_thread(me);
#       endif
#       ifdef GC_DARWIN_THREADS
          /* Reinitialize mach_thread to avoid thread_suspend fail      */
          /* with MACH_SEND_INVALID_DEST error.                         */
          me -> mach_thread = mach_thread_self();
#       endif
        GC_record_stack_base(me -> crtn, sb);
        me -> flags &= (unsigned char)~FINISHED; /* but not DETACHED */
      } else
#   endif
    /* else */ {
        UNLOCK();
        return GC_DUPLICATE;
    }

#   ifdef THREAD_LOCAL_ALLOC
      GC_init_thread_local(&me->tlfs);
#   endif
#   ifdef GC_EXPLICIT_SIGNALS_UNBLOCK
      /* Since this could be executed from a thread destructor, */
      /* our signals might already be blocked.                  */
      GC_unblock_gc_signals();
#   endif
#   if defined(GC_ENABLE_SUSPEND_THREAD) && defined(SIGNAL_BASED_STOP_WORLD)
      if (EXPECT((me -> ext_suspend_cnt & 1) != 0, FALSE)) {
        GC_with_callee_saves_pushed(GC_suspend_self_blocked, (ptr_t)me);
      }
#   endif
    UNLOCK();
    return GC_SUCCESS;
}

#if defined(GC_PTHREADS) \
    && !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)

  /* Called at thread exit.  Never called for main thread.      */
  /* That is OK, since it results in at most a tiny one-time    */
  /* leak.  And linuxthreads implementation does not reclaim    */
  /* the primordial (main) thread resources or id anyway.       */
  GC_INNER_PTHRSTART void GC_thread_exit_proc(void *arg)
  {
    GC_thread me = (GC_thread)arg;
    IF_CANCEL(int cancel_state;)

#   ifdef DEBUG_THREADS
        GC_log_printf("Called GC_thread_exit_proc on %p, gc_thread= %p\n",
                      (void *)(signed_word)(me -> id), (void *)me);
#   endif
    LOCK();
    DISABLE_CANCEL(cancel_state);
    GC_wait_for_gc_completion(FALSE);
    GC_unregister_my_thread_inner(me);
    RESTORE_CANCEL(cancel_state);
    UNLOCK();
  }

  GC_API int WRAP_FUNC(pthread_join)(pthread_t thread, void **retval)
  {
    int result;
    GC_thread t;

    INIT_REAL_SYMS();
#   ifdef DEBUG_THREADS
      GC_log_printf("thread %p is joining thread %p\n",
                    (void *)GC_PTHREAD_PTRVAL(pthread_self()),
                    (void *)GC_PTHREAD_PTRVAL(thread));
#   endif

    /* After the join, thread id may have been recycled.                */
    READER_LOCK();
    t = (GC_thread)COVERT_DATAFLOW_P(GC_lookup_by_pthread(thread));
      /* This is guaranteed to be the intended one, since the thread id */
      /* cannot have been recycled by pthreads.                         */
    READER_UNLOCK();

    result = REAL_FUNC(pthread_join)(thread, retval);
#   if defined(GC_FREEBSD_THREADS)
      /* On FreeBSD, the wrapped pthread_join() sometimes returns       */
      /* (what appears to be) a spurious EINTR which caused the test    */
      /* and real code to fail gratuitously.  Having looked at system   */
      /* pthread library source code, I see how such return code value  */
      /* may be generated.  In one path of the code, pthread_join just  */
      /* returns the errno setting of the thread being joined - this    */
      /* does not match the POSIX specification or the local man pages. */
      /* Thus, I have taken the liberty to catch this one spurious      */
      /* return value.                                                  */
      if (EXPECT(result == EINTR, FALSE)) result = 0;
#   endif

    if (EXPECT(0 == result, TRUE)) {
      LOCK();
      /* Here the pthread id may have been recycled.  Delete the thread */
      /* from GC_threads (unless it has been registered again from the  */
      /* client thread key destructor).                                 */
      if (KNOWN_FINISHED(t)) {
        GC_delete_thread(t);
      }
      UNLOCK();
    }

#   ifdef DEBUG_THREADS
      GC_log_printf("thread %p join with thread %p %s\n",
                    (void *)GC_PTHREAD_PTRVAL(pthread_self()),
                    (void *)GC_PTHREAD_PTRVAL(thread),
                    result != 0 ? "failed" : "succeeded");
#   endif
    return result;
  }

  GC_API int WRAP_FUNC(pthread_detach)(pthread_t thread)
  {
    int result;
    GC_thread t;

    INIT_REAL_SYMS();
    READER_LOCK();
    t = (GC_thread)COVERT_DATAFLOW_P(GC_lookup_by_pthread(thread));
    READER_UNLOCK();
    result = REAL_FUNC(pthread_detach)(thread);
    if (EXPECT(0 == result, TRUE)) {
      LOCK();
      /* Here the pthread id may have been recycled.    */
      if (KNOWN_FINISHED(t)) {
        GC_delete_thread(t);
      } else {
        t -> flags |= DETACHED;
      }
      UNLOCK();
    }
    return result;
  }

  struct start_info {
    void *(*start_routine)(void *);
    void *arg;
    sem_t registered;           /* 1 ==> in our thread table, but       */
                                /* parent hasn't yet noticed.           */
    unsigned char flags;
  };

  /* Called from GC_pthread_start_inner().  Defined in this file to     */
  /* minimize the number of include files in pthread_start.c (because   */
  /* sem_t and sem_post() are not used in that file directly).          */
  GC_INNER_PTHRSTART GC_thread GC_start_rtn_prepare_thread(
                                        void *(**pstart)(void *),
                                        void **pstart_arg,
                                        struct GC_stack_base *sb, void *arg)
  {
    struct start_info *psi = (struct start_info *)arg;
    thread_id_t self_id = thread_id_self();
    GC_thread me;

#   ifdef DEBUG_THREADS
      GC_log_printf("Starting thread %p, sp= %p\n",
                    (void *)GC_PTHREAD_PTRVAL(pthread_self()), (void *)&arg);
#   endif
    /* If a GC occurs before the thread is registered, that GC will     */
    /* ignore this thread.  That's fine, since it will block trying to  */
    /* acquire the allocator lock, and won't yet hold interesting       */
    /* pointers.                                                        */
    LOCK();
    /* We register the thread here instead of in the parent, so that    */
    /* we don't need to hold the allocator lock during pthread_create.  */
    me = GC_register_my_thread_inner(sb, self_id);
    GC_ASSERT(me != &first_thread);
    me -> flags = psi -> flags;
#   ifdef GC_WIN32_THREADS
      GC_win32_cache_self_pthread(self_id);
#   endif
#   ifdef THREAD_LOCAL_ALLOC
      GC_init_thread_local(&me->tlfs);
#   endif
    UNLOCK();

    *pstart = psi -> start_routine;
    *pstart_arg = psi -> arg;
#   if defined(DEBUG_THREADS) && defined(FUNCPTR_IS_DATAPTR)
      GC_log_printf("start_routine= %p\n", CAST_THRU_UINTPTR(void*, *pstart));
#   endif
    sem_post(&(psi -> registered));     /* Last action on *psi; */
                                        /* OK to deallocate.    */
    return me;
  }

  STATIC void * GC_pthread_start(void * arg)
  {
#   ifdef INCLUDE_LINUX_THREAD_DESCR
      struct GC_stack_base sb;

#     ifdef REDIRECT_MALLOC
        /* GC_get_stack_base may call pthread_getattr_np, which can     */
        /* unfortunately call realloc, which may allocate from an       */
        /* unregistered thread.  This is unpleasant, since it might     */
        /* force heap growth (or, even, heap overflow).                 */
        GC_disable();
#     endif
      if (GC_get_stack_base(&sb) != GC_SUCCESS)
        ABORT("Failed to get thread stack base");
#     ifdef REDIRECT_MALLOC
        GC_enable();
#     endif
      return GC_pthread_start_inner(&sb, arg);
#   else
      return GC_call_with_stack_base(GC_pthread_start_inner, arg);
#   endif
  }

  GC_API int WRAP_FUNC(pthread_create)(pthread_t *new_thread,
                       GC_PTHREAD_CREATE_CONST pthread_attr_t *attr,
                       void *(*start_routine)(void *), void *arg)
  {
    int result;
    struct start_info si;

    GC_ASSERT(I_DONT_HOLD_LOCK());
    INIT_REAL_SYMS();
    if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
    GC_ASSERT(GC_thr_initialized);

    GC_init_lib_bounds();
    if (sem_init(&si.registered, GC_SEM_INIT_PSHARED, 0) != 0)
        ABORT("sem_init failed");
    si.flags = 0;
    si.start_routine = start_routine;
    si.arg = arg;

    /* We resist the temptation to muck with the stack size here,       */
    /* even if the default is unreasonably small.  That is the client's */
    /* responsibility.                                                  */
#   ifdef GC_ASSERTIONS
      {
        size_t stack_size = 0;
        if (NULL != attr) {
          if (pthread_attr_getstacksize(attr, &stack_size) != 0)
            ABORT("pthread_attr_getstacksize failed");
        }
        if (0 == stack_size) {
          pthread_attr_t my_attr;

          if (pthread_attr_init(&my_attr) != 0)
            ABORT("pthread_attr_init failed");
          if (pthread_attr_getstacksize(&my_attr, &stack_size) != 0)
            ABORT("pthread_attr_getstacksize failed");
          (void)pthread_attr_destroy(&my_attr);
        }
        /* On Solaris 10 and on Win32 with winpthreads, with the        */
        /* default attr initialization, stack_size remains 0; fudge it. */
        if (EXPECT(0 == stack_size, FALSE)) {
#           if !defined(SOLARIS) && !defined(GC_WIN32_PTHREADS)
              WARN("Failed to get stack size for assertion checking\n", 0);
#           endif
            stack_size = 1000000;
        }
        GC_ASSERT(stack_size >= 65536);
        /* Our threads may need to do some work for the GC.     */
        /* Ridiculously small threads won't work, and they      */
        /* probably wouldn't work anyway.                       */
      }
#   endif

    if (attr != NULL) {
        int detachstate;

        if (pthread_attr_getdetachstate(attr, &detachstate) != 0)
            ABORT("pthread_attr_getdetachstate failed");
        if (PTHREAD_CREATE_DETACHED == detachstate)
          si.flags |= DETACHED;
    }

#   ifdef PARALLEL_MARK
      if (EXPECT(!GC_parallel && GC_available_markers_m1 > 0, FALSE))
        GC_start_mark_threads();
#   endif
#   ifdef DEBUG_THREADS
      GC_log_printf("About to start new thread from thread %p\n",
                    (void *)GC_PTHREAD_PTRVAL(pthread_self()));
#   endif
    set_need_to_lock();
    result = REAL_FUNC(pthread_create)(new_thread, attr,
                                       GC_pthread_start, &si);

    /* Wait until child has been added to the thread table.             */
    /* This also ensures that we hold onto the stack-allocated si       */
    /* until the child is done with it.                                 */
    if (EXPECT(0 == result, TRUE)) {
        IF_CANCEL(int cancel_state;)

        DISABLE_CANCEL(cancel_state);
                /* pthread_create is not a cancellation point.  */
        while (0 != sem_wait(&si.registered)) {
#           if defined(GC_HAIKU_THREADS)
              /* To workaround some bug in Haiku semaphores.    */
              if (EACCES == errno) continue;
#           endif
            if (EINTR != errno) ABORT("sem_wait failed");
        }
        RESTORE_CANCEL(cancel_state);
    }
    sem_destroy(&si.registered);
    return result;
  }

#endif /* GC_PTHREADS && !SN_TARGET_ORBIS && !SN_TARGET_PSP2 */

#if ((defined(GC_PTHREADS_PARAMARK) || defined(USE_PTHREAD_LOCKS)) \
     && !defined(NO_PTHREAD_TRYLOCK)) || defined(USE_SPIN_LOCK)
  /* Spend a few cycles in a way that can't introduce contention with   */
  /* other threads.                                                     */
# define GC_PAUSE_SPIN_CYCLES 10
  STATIC void GC_pause(void)
  {
    int i;

    for (i = 0; i < GC_PAUSE_SPIN_CYCLES; ++i) {
        /* Something that's unlikely to be optimized away. */
#     if defined(AO_HAVE_compiler_barrier) \
         && !defined(BASE_ATOMIC_OPS_EMULATED)
        AO_compiler_barrier();
#     else
        GC_noop1(i);
#     endif
    }
  }
#endif /* USE_SPIN_LOCK || !NO_PTHREAD_TRYLOCK */

#ifndef SPIN_MAX
# define SPIN_MAX 128   /* Maximum number of calls to GC_pause before   */
                        /* give up.                                     */
#endif

#if (!defined(USE_SPIN_LOCK) && !defined(NO_PTHREAD_TRYLOCK) \
     && defined(USE_PTHREAD_LOCKS)) || defined(GC_PTHREADS_PARAMARK)
  /* If we do not want to use the below spinlock implementation, either */
  /* because we don't have a GC_test_and_set implementation, or because */
  /* we don't want to risk sleeping, we can still try spinning on       */
  /* pthread_mutex_trylock for a while.  This appears to be very        */
  /* beneficial in many cases.                                          */
  /* I suspect that under high contention this is nearly always better  */
  /* than the spin lock.  But it is a bit slower on a uniprocessor.     */
  /* Hence we still default to the spin lock.                           */
  /* This is also used to acquire the mark lock for the parallel        */
  /* marker.                                                            */

  /* Here we use a strict exponential backoff scheme.  I don't know     */
  /* whether that's better or worse than the above.  We eventually      */
  /* yield by calling pthread_mutex_lock(); it never makes sense to     */
  /* explicitly sleep.                                                  */

# ifdef LOCK_STATS
    /* Note that LOCK_STATS requires AO_HAVE_test_and_set.      */
    volatile AO_t GC_spin_count = 0;
    volatile AO_t GC_block_count = 0;
    volatile AO_t GC_unlocked_count = 0;
# endif

  STATIC void GC_generic_lock(pthread_mutex_t * lock)
  {
#   ifndef NO_PTHREAD_TRYLOCK
      unsigned pause_length = 1;
      unsigned i;

      if (EXPECT(0 == pthread_mutex_trylock(lock), TRUE)) {
#       ifdef LOCK_STATS
            (void)AO_fetch_and_add1(&GC_unlocked_count);
#       endif
        return;
      }
      for (; pause_length <= (unsigned)SPIN_MAX; pause_length <<= 1) {
         for (i = 0; i < pause_length; ++i) {
            GC_pause();
        }
        switch (pthread_mutex_trylock(lock)) {
            case 0:
#               ifdef LOCK_STATS
                    (void)AO_fetch_and_add1(&GC_spin_count);
#               endif
                return;
            case EBUSY:
                break;
            default:
                ABORT("Unexpected error from pthread_mutex_trylock");
        }
      }
#   endif /* !NO_PTHREAD_TRYLOCK */
#   ifdef LOCK_STATS
        (void)AO_fetch_and_add1(&GC_block_count);
#   endif
    pthread_mutex_lock(lock);
  }
#endif /* !USE_SPIN_LOCK || ... */

#if defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS)
  GC_INNER volatile unsigned char GC_collecting = FALSE;
                        /* A hint that we are in the collector and      */
                        /* holding the allocator lock for an extended   */
                        /* period.                                      */

# if defined(AO_HAVE_char_load) && !defined(BASE_ATOMIC_OPS_EMULATED)
#   define is_collecting() ((GC_bool)AO_char_load(&GC_collecting))
# else
    /* GC_collecting is a hint, a potential data race between   */
    /* GC_lock() and ENTER/EXIT_GC() is OK to ignore.           */
#   define is_collecting() ((GC_bool)GC_collecting)
# endif
#endif /* GC_PTHREADS && !GC_WIN32_THREADS */

#ifdef GC_ASSERTIONS
  GC_INNER unsigned long GC_lock_holder = NO_THREAD;
#endif

#if defined(USE_SPIN_LOCK)
  /* Reasonably fast spin locks.  Basically the same implementation     */
  /* as STL alloc.h.  This isn't really the right way to do this.       */
  /* but until the POSIX scheduling mess gets straightened out ...      */

  GC_INNER volatile AO_TS_t GC_allocate_lock = AO_TS_INITIALIZER;

# define low_spin_max 30 /* spin cycles if we suspect uniprocessor  */
# define high_spin_max SPIN_MAX /* spin cycles for multiprocessor   */

  static volatile AO_t spin_max = low_spin_max;
  static volatile AO_t last_spins = 0;
                                /* A potential data race between        */
                                /* threads invoking GC_lock which reads */
                                /* and updates spin_max and last_spins  */
                                /* could be ignored because these       */
                                /* variables are hints only.            */

  GC_INNER void GC_lock(void)
  {
    AO_t my_spin_max, my_last_spins_half;
    size_t i;

    if (EXPECT(AO_test_and_set_acquire(&GC_allocate_lock)
                == AO_TS_CLEAR, TRUE)) {
        return;
    }
    my_spin_max = AO_load(&spin_max);
    my_last_spins_half = AO_load(&last_spins) / 2;
    for (i = 0; i < my_spin_max; i++) {
        if (is_collecting() || GC_nprocs == 1)
          goto yield;
        if (i < my_last_spins_half) {
            GC_pause();
            continue;
        }
        if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
            /*
             * got it!
             * Spinning worked.  Thus we're probably not being scheduled
             * against the other process with which we were contending.
             * Thus it makes sense to spin longer the next time.
             */
            AO_store(&last_spins, i);
            AO_store(&spin_max, high_spin_max);
            return;
        }
    }
    /* We are probably being scheduled against the other process.  Sleep. */
    AO_store(&spin_max, low_spin_max);
  yield:
    for (i = 0;; ++i) {
        if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_CLEAR) {
            return;
        }
#       define SLEEP_THRESHOLD 12
                /* Under Linux very short sleeps tend to wait until     */
                /* the current time quantum expires.  On old Linux      */
                /* kernels nanosleep (<= 2 ms) just spins.              */
                /* (Under 2.4, this happens only for real-time          */
                /* processes.)  We want to minimize both behaviors      */
                /* here.                                                */
        if (i < SLEEP_THRESHOLD) {
            sched_yield();
        } else {
            struct timespec ts;

            if (i > 24) i = 24;
                        /* Don't wait for more than about 15 ms,        */
                        /* even under extreme contention.               */

            ts.tv_sec = 0;
            ts.tv_nsec = (unsigned32)1 << i;
            nanosleep(&ts, 0);
        }
    }
  }

#elif defined(USE_PTHREAD_LOCKS)
# ifdef USE_RWLOCK
    GC_INNER pthread_rwlock_t GC_allocate_ml = PTHREAD_RWLOCK_INITIALIZER;
# else
    GC_INNER pthread_mutex_t GC_allocate_ml = PTHREAD_MUTEX_INITIALIZER;
# endif

# ifndef NO_PTHREAD_TRYLOCK
    GC_INNER void GC_lock(void)
    {
      if (1 == GC_nprocs || is_collecting()) {
        pthread_mutex_lock(&GC_allocate_ml);
      } else {
        GC_generic_lock(&GC_allocate_ml);
      }
    }
# elif defined(GC_ASSERTIONS)
    GC_INNER void GC_lock(void)
    {
#     ifdef USE_RWLOCK
        (void)pthread_rwlock_wrlock(&GC_allocate_ml); /* exclusive */
#     else
        pthread_mutex_lock(&GC_allocate_ml);
#     endif
    }
# endif /* NO_PTHREAD_TRYLOCK && GC_ASSERTIONS */

#endif /* !USE_SPIN_LOCK && USE_PTHREAD_LOCKS */

#ifdef GC_PTHREADS_PARAMARK

# if defined(GC_ASSERTIONS) && defined(GC_WIN32_THREADS) \
     && !defined(USE_PTHREAD_LOCKS)
#   define NUMERIC_THREAD_ID(id) (unsigned long)(word)GC_PTHREAD_PTRVAL(id)
    /* Id not guaranteed to be unique. */
# endif

# ifdef GC_ASSERTIONS
    STATIC unsigned long GC_mark_lock_holder = NO_THREAD;
#   define SET_MARK_LOCK_HOLDER \
                (void)(GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self()))
#   define UNSET_MARK_LOCK_HOLDER \
                do { \
                  GC_ASSERT(GC_mark_lock_holder \
                                == NUMERIC_THREAD_ID(pthread_self())); \
                  GC_mark_lock_holder = NO_THREAD; \
                } while (0)
# else
#   define SET_MARK_LOCK_HOLDER (void)0
#   define UNSET_MARK_LOCK_HOLDER (void)0
# endif /* !GC_ASSERTIONS */

  static pthread_cond_t builder_cv = PTHREAD_COND_INITIALIZER;

# ifndef GC_WIN32_THREADS
    static void setup_mark_lock(void)
    {
#     ifdef GLIBC_2_19_TSX_BUG
        pthread_mutexattr_t mattr;
        int glibc_minor = -1;
        int glibc_major = GC_parse_version(&glibc_minor,
                                           gnu_get_libc_version());

        if (glibc_major > 2 || (glibc_major == 2 && glibc_minor >= 19)) {
          /* TODO: disable this workaround for glibc with fixed TSX */
          /* This disables lock elision to workaround a bug in glibc 2.19+ */
          if (0 != pthread_mutexattr_init(&mattr)) {
            ABORT("pthread_mutexattr_init failed");
          }
          if (0 != pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_NORMAL)) {
            ABORT("pthread_mutexattr_settype failed");
          }
          if (0 != pthread_mutex_init(&mark_mutex, &mattr)) {
            ABORT("pthread_mutex_init failed");
          }
          (void)pthread_mutexattr_destroy(&mattr);
        }
#     endif
    }
# endif /* !GC_WIN32_THREADS */

  GC_INNER void GC_acquire_mark_lock(void)
  {
#   if defined(NUMERIC_THREAD_ID_UNIQUE) && !defined(THREAD_SANITIZER)
      GC_ASSERT(GC_mark_lock_holder != NUMERIC_THREAD_ID(pthread_self()));
#   endif
    GC_generic_lock(&mark_mutex);
    SET_MARK_LOCK_HOLDER;
  }

  GC_INNER void GC_release_mark_lock(void)
  {
    UNSET_MARK_LOCK_HOLDER;
    if (pthread_mutex_unlock(&mark_mutex) != 0) {
        ABORT("pthread_mutex_unlock failed");
    }
  }

  /* Collector must wait for free-list builders for 2 reasons:          */
  /* 1) Mark bits may still be getting examined without lock.           */
  /* 2) Partial free lists referenced only by locals may not be scanned */
  /*    correctly, e.g. if they contain "pointer-free" objects, since   */
  /*    the free-list link may be ignored.                              */
  STATIC void GC_wait_builder(void)
  {
    ASSERT_CANCEL_DISABLED();
    UNSET_MARK_LOCK_HOLDER;
    if (pthread_cond_wait(&builder_cv, &mark_mutex) != 0) {
        ABORT("pthread_cond_wait failed");
    }
    GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
    SET_MARK_LOCK_HOLDER;
  }

  GC_INNER void GC_wait_for_reclaim(void)
  {
    GC_acquire_mark_lock();
    while (GC_fl_builder_count > 0) {
        GC_wait_builder();
    }
    GC_release_mark_lock();
  }

# if defined(CAN_HANDLE_FORK) && defined(THREAD_SANITIZER)
    /* Identical to GC_wait_for_reclaim() but with the no_sanitize      */
    /* attribute as a workaround for TSan which does not notice that    */
    /* the allocator lock is acquired in fork_prepare_proc().           */
    GC_ATTR_NO_SANITIZE_THREAD
    static void wait_for_reclaim_atfork(void)
    {
      GC_acquire_mark_lock();
      while (GC_fl_builder_count > 0)
        GC_wait_builder();
      GC_release_mark_lock();
    }
# endif /* CAN_HANDLE_FORK && THREAD_SANITIZER */

  GC_INNER void GC_notify_all_builder(void)
  {
    GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
    if (pthread_cond_broadcast(&builder_cv) != 0) {
        ABORT("pthread_cond_broadcast failed");
    }
  }

  GC_INNER void GC_wait_marker(void)
  {
    ASSERT_CANCEL_DISABLED();
    GC_ASSERT(GC_parallel);
    UNSET_MARK_LOCK_HOLDER;
    if (pthread_cond_wait(&mark_cv, &mark_mutex) != 0) {
        ABORT("pthread_cond_wait failed");
    }
    GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
    SET_MARK_LOCK_HOLDER;
  }

  GC_INNER void GC_notify_all_marker(void)
  {
    GC_ASSERT(GC_parallel);
    if (pthread_cond_broadcast(&mark_cv) != 0) {
        ABORT("pthread_cond_broadcast failed");
    }
  }

#endif /* GC_PTHREADS_PARAMARK */

GC_INNER GC_on_thread_event_proc GC_on_thread_event = 0;

GC_API void GC_CALL GC_set_on_thread_event(GC_on_thread_event_proc fn)
{
  /* fn may be 0 (means no event notifier). */
  LOCK();
  GC_on_thread_event = fn;
  UNLOCK();
}

GC_API GC_on_thread_event_proc GC_CALL GC_get_on_thread_event(void)
{
  GC_on_thread_event_proc fn;

  READER_LOCK();
  fn = GC_on_thread_event;
  READER_UNLOCK();
  return fn;
}

#ifdef STACKPTR_CORRECTOR_AVAILABLE
  GC_INNER GC_sp_corrector_proc GC_sp_corrector = 0;
#endif

GC_API void GC_CALL GC_set_sp_corrector(GC_sp_corrector_proc fn)
{
# ifdef STACKPTR_CORRECTOR_AVAILABLE
    LOCK();
    GC_sp_corrector = fn;
    UNLOCK();
# else
    UNUSED_ARG(fn);
# endif
}

GC_API GC_sp_corrector_proc GC_CALL GC_get_sp_corrector(void)
{
# ifdef STACKPTR_CORRECTOR_AVAILABLE
    GC_sp_corrector_proc fn;

    READER_LOCK();
    fn = GC_sp_corrector;
    READER_UNLOCK();
    return fn;
# else
    return 0; /* unsupported */
# endif
}

#ifdef PTHREAD_REGISTER_CANCEL_WEAK_STUBS
  /* Workaround "undefined reference" linkage errors on some targets. */
  EXTERN_C_BEGIN
  extern void __pthread_register_cancel(void) __attribute__((__weak__));
  extern void __pthread_unregister_cancel(void) __attribute__((__weak__));
  EXTERN_C_END

  void __pthread_register_cancel(void) {}
  void __pthread_unregister_cancel(void) {}
#endif

#undef do_blocking_enter

#endif /* THREADS */

/*
 * Copyright (c) 2000 by Hewlett-Packard Company.  All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

                /* To determine type of tsd impl.       */
                /* Includes private/specific.h          */
                /* if needed.                           */

#if defined(USE_CUSTOM_SPECIFIC)

static const tse invalid_tse = {INVALID_QTID, 0, 0, INVALID_THREADID};
            /* A thread-specific data entry which will never    */
            /* appear valid to a reader.  Used to fill in empty */
            /* cache entries to avoid a check for 0.            */

GC_INNER int GC_key_create_inner(tsd ** key_ptr)
{
    int i;
    int ret;
    tsd * result;

    GC_ASSERT(I_HOLD_LOCK());
    /* A quick alignment check, since we need atomic stores */
    GC_ASSERT(ADDR(&invalid_tse.next) % sizeof(tse *) == 0);
    result = (tsd *)MALLOC_CLEAR(sizeof(tsd));
    if (NULL == result) return ENOMEM;
    ret = pthread_mutex_init(&(result -> lock), NULL);
    if (ret != 0) return ret;
    for (i = 0; i < TS_CACHE_SIZE; ++i) {
      result -> cache[i] = (tse *)GC_CAST_AWAY_CONST_PVOID(&invalid_tse);
    }
#   ifdef GC_ASSERTIONS
      for (i = 0; i < TS_HASH_SIZE; ++i) {
        GC_ASSERT(NULL == result -> hash[i]);
      }
#   endif
    *key_ptr = result;
    return 0;
}

/* Set the thread-local value associated with the key.  Should not  */
/* be used to overwrite a previously set value.                     */
GC_INNER int GC_setspecific(tsd * key, void * value)
{
    pthread_t self = pthread_self();
    unsigned hash_val = TS_HASH(self);
    volatile tse * entry;

    GC_ASSERT(I_HOLD_LOCK());
    GC_ASSERT(self != INVALID_THREADID);
    GC_dont_gc++; /* disable GC */
    entry = (volatile tse *)MALLOC_CLEAR(sizeof(tse));
    GC_dont_gc--;
    if (EXPECT(NULL == entry, FALSE)) return ENOMEM;

    pthread_mutex_lock(&(key -> lock));
    entry -> next = key -> hash[hash_val];
#   ifdef GC_ASSERTIONS
      {
        tse *p;

        /* Ensure no existing entry.    */
        for (p = entry -> next; p != NULL; p = p -> next) {
          GC_ASSERT(!THREAD_EQUAL(p -> thread, self));
        }
      }
#   endif
    entry -> thread = self;
    entry -> value = TS_HIDE_VALUE(value);
    GC_ASSERT(entry -> qtid == INVALID_QTID);
    /* There can only be one writer at a time, but this needs to be     */
    /* atomic with respect to concurrent readers.                       */
    GC_cptr_store_release((volatile ptr_t *)&(key -> hash[hash_val]),
                          (ptr_t)CAST_AWAY_VOLATILE_PVOID(entry));
    GC_dirty(CAST_AWAY_VOLATILE_PVOID(entry));
    GC_dirty(key -> hash + hash_val);
    if (pthread_mutex_unlock(&(key -> lock)) != 0)
      ABORT("pthread_mutex_unlock failed (setspecific)");
    return 0;
}

/* Remove thread-specific data for a given thread.  This function is    */
/* called at fork from the child process for all threads except for the */
/* survived one.  GC_remove_specific() should be called on thread exit. */
GC_INNER void GC_remove_specific_after_fork(tsd * key, pthread_t t)
{
    unsigned hash_val = TS_HASH(t);
    tse *entry;
    tse *prev = NULL;

#   ifdef CAN_HANDLE_FORK
      /* Both GC_setspecific and GC_remove_specific should be called    */
      /* with the allocator lock held to ensure the consistency of      */
      /* the hash table in the forked child.                            */
      GC_ASSERT(I_HOLD_LOCK());
#   endif
    pthread_mutex_lock(&(key -> lock));
    for (entry = key -> hash[hash_val];
         entry != NULL && !THREAD_EQUAL(entry -> thread, t);
         entry = entry -> next) {
      prev = entry;
    }
    /* Invalidate qtid field, since qtids may be reused, and a later    */
    /* cache lookup could otherwise find this entry.                    */
    if (entry != NULL) {
      entry -> qtid = INVALID_QTID;
      if (NULL == prev) {
        key -> hash[hash_val] = entry -> next;
        GC_dirty(key -> hash + hash_val);
      } else {
        prev -> next = entry -> next;
        GC_dirty(prev);
      }
      /* Atomic! concurrent accesses still work.        */
      /* They must, since readers don't lock.           */
      /* We shouldn't need a volatile access here,      */
      /* since both this and the preceding write        */
      /* should become visible no later than            */
      /* the pthread_mutex_unlock() call.               */
    }
    /* If we wanted to deallocate the entry, we'd first have to clear   */
    /* any cache entries pointing to it.  That probably requires        */
    /* additional synchronization, since we can't prevent a concurrent  */
    /* cache lookup, which should still be examining deallocated memory.*/
    /* This can only happen if the concurrent access is from another    */
    /* thread, and hence has missed the cache, but still...             */
#   ifdef LINT2
      GC_noop1_ptr(entry);
#   endif

    /* With GC, we're done, since the pointers from the cache will      */
    /* be overwritten, all local pointers to the entries will be        */
    /* dropped, and the entry will then be reclaimed.                   */
    if (pthread_mutex_unlock(&(key -> lock)) != 0)
      ABORT("pthread_mutex_unlock failed (remove_specific after fork)");
}

/* Note that even the slow path doesn't lock.   */
GC_INNER void * GC_slow_getspecific(tsd * key, size_t qtid,
                                    tse * volatile * cache_ptr)
{
    pthread_t self = pthread_self();
    tse *entry = key -> hash[TS_HASH(self)];

    GC_ASSERT(qtid != INVALID_QTID);
    while (entry != NULL && !THREAD_EQUAL(entry -> thread, self)) {
      entry = entry -> next;
    }
    if (entry == NULL) return NULL;
    /* Set cache_entry. */
    AO_store(&(entry -> qtid), qtid);
        /* It's safe to do this asynchronously.  Either value   */
        /* is safe, though may produce spurious misses.         */
        /* We're replacing one qtid with another one for the    */
        /* same thread.                                         */
    GC_cptr_store((volatile ptr_t *)cache_ptr, (ptr_t)entry);
    return TS_REVEAL_PTR(entry -> value);
}

#ifdef GC_ASSERTIONS
  /* Check that that all elements of the data structure associated  */
  /* with key are marked.                                           */
  void GC_check_tsd_marks(tsd *key)
  {
    int i;
    tse *p;

    if (!GC_is_marked(GC_base(key))) {
      ABORT("Unmarked thread-specific-data table");
    }
    for (i = 0; i < TS_HASH_SIZE; ++i) {
      for (p = key -> hash[i]; p != NULL; p = p -> next) {
        if (!GC_is_marked(GC_base(p))) {
          ABORT_ARG1("Unmarked thread-specific-data entry",
                     " at %p", (void *)p);
        }
      }
    }
    for (i = 0; i < TS_CACHE_SIZE; ++i) {
      p = key -> cache[i];
      if (p != &invalid_tse && !GC_is_marked(GC_base(p))) {
        ABORT_ARG1("Unmarked cached thread-specific-data entry",
                   " at %p", (void *)p);
      }
    }
  }
#endif /* GC_ASSERTIONS */

#endif /* USE_CUSTOM_SPECIFIC */

/*
 * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
 * Copyright (c) 2000-2008 by Hewlett-Packard Development Company.
 * All rights reserved.
 * Copyright (c) 2008-2022 Ivan Maidanski
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */


#if defined(GC_WIN32_THREADS)

/* The allocator lock definition.       */
#ifndef USE_PTHREAD_LOCKS
# ifdef USE_RWLOCK
    GC_INNER SRWLOCK GC_allocate_ml;
# else
    GC_INNER CRITICAL_SECTION GC_allocate_ml;
# endif
#endif /* !USE_PTHREAD_LOCKS */

#undef CreateThread
#undef ExitThread
#undef _beginthreadex
#undef _endthreadex

#if !defined(GC_PTHREADS) && !defined(MSWINCE)
# include <errno.h> /* for errno, EAGAIN */
# include <process.h>  /* For _beginthreadex, _endthreadex */
#endif

static ptr_t copy_ptr_regs(word *regs, const CONTEXT *pcontext);

#ifndef GC_NO_THREADS_DISCOVERY
  /* This code operates in two distinct modes, depending on     */
  /* the setting of GC_win32_dll_threads.                       */
  /* If GC_win32_dll_threads is set, all threads in the process */
  /* are implicitly registered with the GC by DllMain.          */
  /* No explicit registration is required, and attempts at      */
  /* explicit registration are ignored.  This mode is           */
  /* very different from the Posix operation of the collector.  */
  /* In this mode access to the thread table is lock-free.      */
  /* Hence there is a static limit on the number of threads.    */

  /* GC_DISCOVER_TASK_THREADS should be used if DllMain-based   */
  /* thread registration is required but it is impossible to    */
  /* call GC_use_threads_discovery before other GC routines.    */

# ifndef GC_DISCOVER_TASK_THREADS
    /* GC_win32_dll_threads must be set (if needed) at the      */
    /* application initialization time, i.e. before any         */
    /* collector or thread calls.  We make it a "dynamic"       */
    /* option only to avoid multiple library versions.          */
    GC_INNER GC_bool GC_win32_dll_threads = FALSE;
# endif
#else
  /* If GC_win32_dll_threads is FALSE (or the collector is      */
  /* built without GC_DLL defined), things operate in a way     */
  /* that is very similar to Posix platforms, and new threads   */
  /* must be registered with the collector, e.g. by using       */
  /* preprocessor-based interception of the thread primitives.  */
  /* In this case, we use a real data structure for the thread  */
  /* table.  Note that there is no equivalent of linker-based   */
  /* call interception, since we don't have ELF-like            */
  /* facilities.  The Windows analog appears to be "API         */
  /* hooking", which really seems to be a standard way to       */
  /* do minor binary rewriting (?).  I'd prefer not to have     */
  /* the basic collector rely on such facilities, but an        */
  /* optional package that intercepts thread calls this way     */
  /* would probably be nice.                                    */
# undef MAX_THREADS
# define MAX_THREADS 1 /* dll_thread_table[] is always empty.   */
#endif /* GC_NO_THREADS_DISCOVERY */

/* We have two versions of the thread table.  Which one */
/* we use depends on whether GC_win32_dll_threads       */
/* is set.  Note that before initialization, we don't   */
/* add any entries to either table, even if DllMain is  */
/* called.  The main thread will be added on            */
/* initialization.                                      */

/* GC_use_threads_discovery() is currently incompatible with pthreads   */
/* and WinCE.  It might be possible to get DllMain-based thread         */
/* registration to work with Cygwin, but if you try it then you are on  */
/* your own.                                                            */
GC_API void GC_CALL GC_use_threads_discovery(void)
{
# ifdef GC_NO_THREADS_DISCOVERY
    ABORT("GC DllMain-based thread registration unsupported");
# else
    /* Turn on GC_win32_dll_threads. */
    GC_ASSERT(!GC_is_initialized);
    /* Note that GC_use_threads_discovery is expected to be called by   */
    /* the client application (not from DllMain) at start-up.           */
#   ifndef GC_DISCOVER_TASK_THREADS
      GC_win32_dll_threads = TRUE;
#   endif
    GC_init();
#   ifdef CPPCHECK
      GC_noop1((word)(GC_funcptr_uint)&GC_DllMain);
#   endif
# endif
}

#ifndef GC_NO_THREADS_DISCOVERY
  /* We track thread attachments while the world is supposed to be      */
  /* stopped.  Unfortunately, we cannot stop them from starting, since  */
  /* blocking in DllMain seems to cause the world to deadlock.  Thus,   */
  /* we have to recover if we notice this in the middle of marking.     */
  STATIC volatile AO_t GC_attached_thread = FALSE;

  /* We assumed that volatile ==> memory ordering, at least among       */
  /* volatiles.  This code should consistently use atomic_ops.          */
  STATIC volatile GC_bool GC_please_stop = FALSE;
#elif defined(GC_ASSERTIONS)
  STATIC GC_bool GC_please_stop = FALSE;
#endif /* GC_NO_THREADS_DISCOVERY && GC_ASSERTIONS */

#if defined(WRAP_MARK_SOME) && !defined(GC_PTHREADS)
  /* Return TRUE if an thread was attached since we last asked or */
  /* since GC_attached_thread was explicitly reset.               */
  GC_INNER GC_bool GC_started_thread_while_stopped(void)
  {
#   ifndef GC_NO_THREADS_DISCOVERY
      if (GC_win32_dll_threads) {
#       ifdef AO_HAVE_compare_and_swap_release
          if (AO_compare_and_swap_release(&GC_attached_thread, TRUE,
                                          FALSE /* stored */))
            return TRUE;
#       else
          AO_nop_full(); /* Prior heap reads need to complete earlier. */
          if (AO_load(&GC_attached_thread)) {
            AO_store(&GC_attached_thread, FALSE);
            return TRUE;
          }
#       endif
      }
#   endif
    return FALSE;
  }
#endif /* WRAP_MARK_SOME */

/* Thread table used if GC_win32_dll_threads is set.    */
/* This is a fixed size array.                          */
/* Since we use runtime conditionals, both versions     */
/* are always defined.                                  */
# ifndef MAX_THREADS
#   define MAX_THREADS 512
# endif

/* Things may get quite slow for large numbers of threads,      */
/* since we look them up with sequential search.                */
static volatile struct GC_Thread_Rep dll_thread_table[MAX_THREADS];
#ifndef GC_NO_THREADS_DISCOVERY
  static struct GC_StackContext_Rep dll_crtn_table[MAX_THREADS];
#endif

STATIC volatile LONG GC_max_thread_index = 0;
                        /* Largest index in dll_thread_table    */
                        /* that was ever used.                  */

/* This may be called from DllMain, and hence operates under unusual    */
/* constraints.  In particular, it must be lock-free if                 */
/* GC_win32_dll_threads is set.  Always called from the thread being    */
/* added.  If GC_win32_dll_threads is not set, we already hold the      */
/* allocator lock except possibly during single-threaded startup code.  */
/* Does not initialize thread-local free lists.                         */
GC_INNER GC_thread GC_register_my_thread_inner(const struct GC_stack_base *sb,
                                               thread_id_t self_id)
{
  GC_thread me;

# ifdef GC_NO_THREADS_DISCOVERY
    GC_ASSERT(I_HOLD_LOCK());
# endif
  /* The following should be a no-op according to the Win32     */
  /* documentation.  There is empirical evidence that it        */
  /* isn't.             - HB                                    */
# if defined(MPROTECT_VDB) && !defined(CYGWIN32)
    if (GC_auto_incremental
#       ifdef GWW_VDB
          && !GC_gww_dirty_init()
#       endif
        )
      GC_set_write_fault_handler();
# endif

# ifndef GC_NO_THREADS_DISCOVERY
    if (GC_win32_dll_threads) {
      int i;
      /* It appears to be unsafe to acquire a lock here, since this     */
      /* code is apparently not preemptible on some systems.            */
      /* (This is based on complaints, not on Microsoft's official      */
      /* documentation, which says this should perform "only simple     */
      /* initialization tasks".)                                        */
      /* Hence we make do with nonblocking synchronization.             */
      /* It has been claimed that DllMain is really only executed with  */
      /* a particular system lock held, and thus careful use of locking */
      /* around code that doesn't call back into the system libraries   */
      /* might be OK.  But this has not been tested across all Win32    */
      /* variants.                                                      */
      for (i = 0;
           InterlockedExchange(&dll_thread_table[i].tm.long_in_use, 1) != 0;
           i++) {
        /* Compare-and-swap would make this cleaner, but that's not     */
        /* supported before Windows 98 and NT 4.0.  In Windows 2000,    */
        /* InterlockedExchange is supposed to be replaced by            */
        /* InterlockedExchangePointer, but that's not really what I     */
        /* want here.                                                   */
        /* FIXME: We should eventually declare Windows 95 dead and use  */
        /* AO_ primitives here.                                         */
        if (i == MAX_THREADS - 1)
          ABORT("Too many threads");
      }
      /* Update GC_max_thread_index if necessary.  The following is     */
      /* safe, and unlike CompareExchange-based solutions seems to work */
      /* on all Windows 95 and later platforms.  Unfortunately,         */
      /* GC_max_thread_index may be temporarily out of bounds, so       */
      /* readers have to compensate.                                    */
      while (i > GC_max_thread_index) {
        InterlockedIncrement((LONG *)&GC_max_thread_index);
                /* Cast away volatile for older versions of Win32 headers. */
      }
      if (EXPECT(GC_max_thread_index >= MAX_THREADS, FALSE)) {
        /* We overshot due to simultaneous increments.  */
        /* Setting it to MAX_THREADS-1 is always safe.  */
        GC_max_thread_index = MAX_THREADS - 1;
      }
      me = (GC_thread)(dll_thread_table + i);
      me -> crtn = &dll_crtn_table[i];
    } else
# endif
  /* else */ /* Not using DllMain */ {
    me = GC_new_thread(self_id);
  }
# ifdef GC_PTHREADS
    me -> pthread_id = pthread_self();
# endif
# ifndef MSWINCE
    /* GetCurrentThread() returns a pseudohandle (a const value).       */
    if (!DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
                        GetCurrentProcess(),
                        (HANDLE*)&(me -> handle),
                        0 /* dwDesiredAccess */, FALSE /* bInheritHandle */,
                        DUPLICATE_SAME_ACCESS)) {
        ABORT_ARG1("DuplicateHandle failed",
                   ": errcode= 0x%X", (unsigned)GetLastError());
    }
# endif
# if defined(WOW64_THREAD_CONTEXT_WORKAROUND) && defined(MSWINRT_FLAVOR)
    /* Lookup TIB value via a call to NtCurrentTeb() on thread          */
    /* registration rather than calling GetThreadSelectorEntry() which  */
    /* is not available on UWP.                                         */
    me -> tib = (PNT_TIB)NtCurrentTeb();
# endif
  me -> crtn -> last_stack_min = ADDR_LIMIT;
  GC_record_stack_base(me -> crtn, sb);
  /* Up until this point, GC_push_all_stacks considers this thread      */
  /* invalid.                                                           */
  /* Up until this point, this entry is viewed as reserved but invalid  */
  /* by GC_win32_dll_lookup_thread.                                     */
  ((volatile struct GC_Thread_Rep *)me) -> id = self_id;
# ifndef GC_NO_THREADS_DISCOVERY
    if (GC_win32_dll_threads) {
      if (GC_please_stop) {
        AO_store(&GC_attached_thread, TRUE);
        AO_nop_full(); /* Later updates must become visible after this. */
      }
      /* We'd like to wait here, but can't, since waiting in DllMain    */
      /* provokes deadlocks.                                            */
      /* Thus we force marking to be restarted instead.                 */
    } else
# endif
  /* else */ {
    GC_ASSERT(!GC_please_stop);
        /* Otherwise both we and the thread stopping code would be      */
        /* holding the allocator lock.                                  */
  }
  return me;
}

/*
 * GC_max_thread_index may temporarily be larger than MAX_THREADS.
 * To avoid subscript errors, we check on access.
 */
GC_INLINE LONG GC_get_max_thread_index(void)
{
  LONG my_max = GC_max_thread_index;
  if (EXPECT(my_max >= MAX_THREADS, FALSE)) return MAX_THREADS - 1;
  return my_max;
}

#ifndef GC_NO_THREADS_DISCOVERY
  /* Search in dll_thread_table and return the GC_thread entity         */
  /* corresponding to the given thread id.                              */
  /* May be called without a lock, but should be called in contexts in  */
  /* which the requested thread cannot be asynchronously deleted, e.g.  */
  /* from the thread itself.                                            */
  GC_INNER GC_thread GC_win32_dll_lookup_thread(thread_id_t id)
  {
      int i;
      LONG my_max = GC_get_max_thread_index();

      GC_ASSERT(GC_win32_dll_threads);
      for (i = 0; i <= my_max; i++) {
        if (AO_load_acquire(&dll_thread_table[i].tm.in_use)
            && dll_thread_table[i].id == id)
          break;    /* Must still be in use, since nobody else can      */
                    /* store our thread id.                             */
      }
      return i <= my_max ? (GC_thread)(dll_thread_table + i) : NULL;
  }
#endif /* !GC_NO_THREADS_DISCOVERY */

#ifdef GC_PTHREADS
  /* A quick-and-dirty cache of the mapping between pthread_t   */
  /* and Win32 thread id.                                       */
# define PTHREAD_MAP_SIZE 512
  thread_id_t GC_pthread_map_cache[PTHREAD_MAP_SIZE] = {0};
# define PTHREAD_MAP_INDEX(pthread_id) \
                ((NUMERIC_THREAD_ID(pthread_id) >> 5) % PTHREAD_MAP_SIZE)
        /* It appears pthread_t is really a pointer type ... */
# define SET_PTHREAD_MAP_CACHE(pthread_id, win32_id) \
      (void)(GC_pthread_map_cache[PTHREAD_MAP_INDEX(pthread_id)] = (win32_id))
# define GET_PTHREAD_MAP_CACHE(pthread_id) \
          GC_pthread_map_cache[PTHREAD_MAP_INDEX(pthread_id)]

  GC_INNER void GC_win32_cache_self_pthread(thread_id_t self_id)
  {
    pthread_t self = pthread_self();

    GC_ASSERT(I_HOLD_LOCK());
    SET_PTHREAD_MAP_CACHE(self, self_id);
  }

  /* Return a GC_thread corresponding to a given pthread_t, or  */
  /* NULL if it is not there.  We assume that this is only      */
  /* called for pthread ids that have not yet terminated or are */
  /* still joinable, and cannot be terminated concurrently.     */
  GC_INNER GC_thread GC_lookup_by_pthread(pthread_t thread)
  {
      /* TODO: search in dll_thread_table instead when DllMain-based    */
      /* thread registration is made compatible with pthreads (and      */
      /* turned on).                                                    */

      thread_id_t id;
      GC_thread p;
      int hv;

      GC_ASSERT(I_HOLD_READER_LOCK());
      id = GET_PTHREAD_MAP_CACHE(thread);
      /* We first try the cache.        */
      for (p = GC_threads[THREAD_TABLE_INDEX(id)];
           p != NULL; p = p -> tm.next) {
        if (EXPECT(THREAD_EQUAL(p -> pthread_id, thread), TRUE))
          return p;
      }

      /* If that fails, we use a very slow approach.    */
      for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
        for (p = GC_threads[hv]; p != NULL; p = p -> tm.next) {
          if (THREAD_EQUAL(p -> pthread_id, thread))
            return p;
        }
      }
      return NULL;
  }
#endif /* GC_PTHREADS */

#ifdef WOW64_THREAD_CONTEXT_WORKAROUND
# ifndef CONTEXT_EXCEPTION_ACTIVE
#   define CONTEXT_EXCEPTION_ACTIVE    0x08000000
#   define CONTEXT_EXCEPTION_REQUEST   0x40000000
#   define CONTEXT_EXCEPTION_REPORTING 0x80000000
# endif
  static GC_bool isWow64; /* Is running 32-bit code on Win64?   */
# define GET_THREAD_CONTEXT_FLAGS (isWow64 \
                        ? CONTEXT_INTEGER | CONTEXT_CONTROL \
                          | CONTEXT_EXCEPTION_REQUEST | CONTEXT_SEGMENTS \
                        : CONTEXT_INTEGER | CONTEXT_CONTROL)
#elif defined(I386) || defined(XMM_CANT_STORE_PTRS)
# define GET_THREAD_CONTEXT_FLAGS (CONTEXT_INTEGER | CONTEXT_CONTROL)
#else
# define GET_THREAD_CONTEXT_FLAGS (CONTEXT_INTEGER | CONTEXT_CONTROL \
                                   | CONTEXT_FLOATING_POINT)
#endif /* !WOW64_THREAD_CONTEXT_WORKAROUND && !I386 */

/* Suspend the given thread, if it's still active.      */
STATIC void GC_suspend(GC_thread t)
{
# ifndef MSWINCE
    DWORD exitCode;
#   ifdef RETRY_GET_THREAD_CONTEXT
      int retry_cnt;
#     define MAX_SUSPEND_THREAD_RETRIES (1000 * 1000)
#   endif
# endif

  GC_ASSERT(I_HOLD_LOCK());
# if defined(DEBUG_THREADS) && !defined(MSWINCE) \
     && (!defined(MSWIN32) || defined(CONSOLE_LOG))
    GC_log_printf("Suspending 0x%x\n", (int)t->id);
# endif
  GC_win32_unprotect_thread(t);
  GC_acquire_dirty_lock();

# ifdef MSWINCE
    /* SuspendThread() will fail if thread is running kernel code.      */
    while (SuspendThread(THREAD_HANDLE(t)) == (DWORD)-1) {
      GC_release_dirty_lock();
      Sleep(10); /* in millis */
      GC_acquire_dirty_lock();
    }
# elif defined(RETRY_GET_THREAD_CONTEXT)
    for (retry_cnt = 0;;) {
      /* Apparently the Windows 95 GetOpenFileName call creates         */
      /* a thread that does not properly get cleaned up, and            */
      /* SuspendThread on its descriptor may provoke a crash.           */
      /* This reduces the probability of that event, though it still    */
      /* appears there is a race here.                                  */
      if (GetExitCodeThread(t -> handle, &exitCode)
          && exitCode != STILL_ACTIVE) {
        GC_release_dirty_lock();
#       ifdef GC_PTHREADS
          t -> crtn -> stack_end = NULL; /* prevent stack from being pushed */
#       else
          /* This breaks pthread_join on Cygwin, which is guaranteed to */
          /* only see user threads.                                     */
          GC_delete_thread(t);
#       endif
        return;
      }

      if (SuspendThread(t -> handle) != (DWORD)-1) {
        CONTEXT context;

        context.ContextFlags = GET_THREAD_CONTEXT_FLAGS;
        if (GetThreadContext(t -> handle, &context)) {
          /* TODO: WoW64 extra workaround: if CONTEXT_EXCEPTION_ACTIVE  */
          /* then Sleep(1) and retry.                                   */
          t->context_sp = copy_ptr_regs(t->context_regs, &context);
          break; /* success; the context pointer registers are saved */
        }

        /* Resume the thread, try to suspend it in a better location.   */
        if (ResumeThread(t -> handle) == (DWORD)-1)
          ABORT("ResumeThread failed in suspend loop");
      }
      if (retry_cnt > 1) {
        GC_release_dirty_lock();
        Sleep(0); /* yield */
        GC_acquire_dirty_lock();
      }
      if (++retry_cnt >= MAX_SUSPEND_THREAD_RETRIES)
        ABORT("SuspendThread loop failed"); /* something must be wrong */
    }
# else
    if (GetExitCodeThread(t -> handle, &exitCode)
        && exitCode != STILL_ACTIVE) {
      GC_release_dirty_lock();
#     ifdef GC_PTHREADS
        t -> crtn -> stack_end = NULL; /* prevent stack from being pushed */
#     else
        GC_delete_thread(t);
#     endif
      return;
    }
    if (SuspendThread(t -> handle) == (DWORD)-1)
      ABORT("SuspendThread failed");
# endif
  t -> flags |= IS_SUSPENDED;
  GC_release_dirty_lock();
  if (GC_on_thread_event)
    GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED, THREAD_HANDLE(t));
}

#if defined(GC_ASSERTIONS) \
    && ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE))
  GC_INNER GC_bool GC_write_disabled = FALSE;
                /* TRUE only if GC_stop_world() acquired GC_write_cs.   */
#endif

GC_INNER void GC_stop_world(void)
{
  thread_id_t self_id = GetCurrentThreadId();

  GC_ASSERT(I_HOLD_LOCK());
  GC_ASSERT(GC_thr_initialized);

  /* This code is the same as in pthread_stop_world.c */
# ifdef PARALLEL_MARK
    if (GC_parallel) {
      GC_acquire_mark_lock();
      GC_ASSERT(GC_fl_builder_count == 0);
      /* We should have previously waited for it to become zero. */
    }
# endif /* PARALLEL_MARK */

# if !defined(GC_NO_THREADS_DISCOVERY) || defined(GC_ASSERTIONS)
    GC_please_stop = TRUE;
# endif
# if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)
    GC_ASSERT(!GC_write_disabled);
    EnterCriticalSection(&GC_write_cs);
    /* It's not allowed to call GC_printf() (and friends) here down to  */
    /* LeaveCriticalSection (same applies recursively to GC_suspend,    */
    /* GC_delete_thread, GC_get_max_thread_index, GC_size and           */
    /* GC_remove_protection).                                           */
#   ifdef GC_ASSERTIONS
      GC_write_disabled = TRUE;
#   endif
# endif
# ifndef GC_NO_THREADS_DISCOVERY
    if (GC_win32_dll_threads) {
      int i;
      int my_max;
      /* Any threads being created during this loop will end up setting */
      /* GC_attached_thread when they start.  This will force marking   */
      /* to restart.  This is not ideal, but hopefully correct.         */
      AO_store(&GC_attached_thread, FALSE);
      my_max = (int)GC_get_max_thread_index();
      for (i = 0; i <= my_max; i++) {
        GC_thread p = (GC_thread)(dll_thread_table + i);

        if (p -> crtn -> stack_end != NULL && (p -> flags & DO_BLOCKING) == 0
            && p -> id != self_id) {
          GC_suspend(p);
        }
      }
    } else
# endif
  /* else */ {
    GC_thread p;
    int i;

    for (i = 0; i < THREAD_TABLE_SZ; i++) {
      for (p = GC_threads[i]; p != NULL; p = p -> tm.next)
        if (p -> crtn -> stack_end != NULL && p -> id != self_id
            && (p -> flags & (FINISHED | DO_BLOCKING)) == 0)
          GC_suspend(p);
    }
  }
# if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)
#   ifdef GC_ASSERTIONS
      GC_write_disabled = FALSE;
#   endif
    LeaveCriticalSection(&GC_write_cs);
# endif
# ifdef PARALLEL_MARK
    if (GC_parallel)
      GC_release_mark_lock();
# endif
}

GC_INNER void GC_start_world(void)
{
# ifdef GC_ASSERTIONS
    thread_id_t self_id = GetCurrentThreadId();
# endif

  GC_ASSERT(I_HOLD_LOCK());
  if (GC_win32_dll_threads) {
    LONG my_max = GC_get_max_thread_index();
    int i;

    for (i = 0; i <= my_max; i++) {
      GC_thread p = (GC_thread)(dll_thread_table + i);

      if ((p -> flags & IS_SUSPENDED) != 0) {
#       ifdef DEBUG_THREADS
          GC_log_printf("Resuming 0x%x\n", (int)p->id);
#       endif
        GC_ASSERT(p -> id != self_id);
        GC_ASSERT(*(ptr_t *)CAST_AWAY_VOLATILE_PVOID(
                                &(p -> crtn -> stack_end)) != NULL);
        if (ResumeThread(THREAD_HANDLE(p)) == (DWORD)-1)
          ABORT("ResumeThread failed");
        p -> flags &= (unsigned char)~IS_SUSPENDED;
        if (GC_on_thread_event)
          GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, THREAD_HANDLE(p));
      }
      /* Else thread is unregistered or not suspended. */
    }
  } else {
    GC_thread p;
    int i;

    for (i = 0; i < THREAD_TABLE_SZ; i++) {
      for (p = GC_threads[i]; p != NULL; p = p -> tm.next) {
        if ((p -> flags & IS_SUSPENDED) != 0) {
#         ifdef DEBUG_THREADS
            GC_log_printf("Resuming 0x%x\n", (int)p->id);
#         endif
          GC_ASSERT(p -> id != self_id
                    && *(ptr_t *)&(p -> crtn -> stack_end) != NULL);
          if (ResumeThread(THREAD_HANDLE(p)) == (DWORD)-1)
            ABORT("ResumeThread failed");
          GC_win32_unprotect_thread(p);
          p -> flags &= (unsigned char)~IS_SUSPENDED;
          if (GC_on_thread_event)
            GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, THREAD_HANDLE(p));
        } else {
#         ifdef DEBUG_THREADS
            GC_log_printf("Not resuming thread 0x%x as it is not suspended\n",
                          (int)p->id);
#         endif
        }
      }
    }
  }
# if !defined(GC_NO_THREADS_DISCOVERY) || defined(GC_ASSERTIONS)
    GC_please_stop = FALSE;
# endif
}

#ifdef MSWINCE
  /* The VirtualQuery calls below won't work properly on some old WinCE */
  /* versions, but since each stack is restricted to an aligned 64 KiB  */
  /* region of virtual memory we can just take the next lowest multiple */
  /* of 64 KiB.  The result of this macro must not be used as its       */
  /* argument later and must not be used as the lower bound for sp      */
  /* check (since the stack may be bigger than 64 KiB).                 */
# define GC_wince_evaluate_stack_min(s) \
                        (ptr_t)(((word)(s) - 1) & ~(word)0xFFFF)
#elif defined(GC_ASSERTIONS)
# define GC_dont_query_stack_min FALSE
#endif

/* A cache holding the results of the recent VirtualQuery call. */
/* Protected by the allocator lock.                             */
static ptr_t last_address = 0;
static MEMORY_BASIC_INFORMATION last_info;

/* Probe stack memory region (starting at "s") to find out its  */
/* lowest address (i.e. stack top).                             */
/* S must be a mapped address inside the region, NOT the first  */
/* unmapped address.                                            */
STATIC ptr_t GC_get_stack_min(ptr_t s)
{
  ptr_t bottom;

  GC_ASSERT(I_HOLD_LOCK());
  if (s != last_address) {
    VirtualQuery(s, &last_info, sizeof(last_info));
    last_address = s;
  }
  do {
    bottom = (ptr_t)last_info.BaseAddress;
    VirtualQuery(bottom - 1, &last_info, sizeof(last_info));
    last_address = bottom - 1;
  } while ((last_info.Protect & PAGE_READWRITE)
           && !(last_info.Protect & PAGE_GUARD));
  return bottom;
}

/* Return true if the page at s has protections appropriate     */
/* for a stack page.                                            */
static GC_bool may_be_in_stack(ptr_t s)
{
  GC_ASSERT(I_HOLD_LOCK());
  if (s != last_address) {
    VirtualQuery(s, &last_info, sizeof(last_info));
    last_address = s;
  }
  return (last_info.Protect & PAGE_READWRITE)
          && !(last_info.Protect & PAGE_GUARD);
}

/* Copy all registers that might point into the heap.  Frame    */
/* pointer registers are included in case client code was       */
/* compiled with the 'omit frame pointer' optimization.         */
/* The context register values are stored to regs argument      */
/* which is expected to be of PUSHED_REGS_COUNT length exactly. */
/* The functions returns the context stack pointer value.       */
static ptr_t copy_ptr_regs(word *regs, const CONTEXT *pcontext) {
    ptr_t sp;
    int cnt = 0;
#   define context (*pcontext)
#   define PUSH1(reg) (regs[cnt++] = (word)pcontext->reg)
#   define PUSH2(r1,r2) (PUSH1(r1), PUSH1(r2))
#   define PUSH4(r1,r2,r3,r4) (PUSH2(r1,r2), PUSH2(r3,r4))
#   define PUSH8_LH(r1,r2,r3,r4) (PUSH4(r1.Low,r1.High,r2.Low,r2.High), \
                                  PUSH4(r3.Low,r3.High,r4.Low,r4.High))
#   if defined(I386)
#     ifdef WOW64_THREAD_CONTEXT_WORKAROUND
        PUSH2(ContextFlags, SegFs); /* cannot contain pointers */
#     endif
      PUSH4(Edi,Esi,Ebx,Edx), PUSH2(Ecx,Eax), PUSH1(Ebp);
      sp = (ptr_t)context.Esp;
#   elif defined(X86_64)
      PUSH4(Rax,Rcx,Rdx,Rbx); PUSH2(Rbp, Rsi); PUSH1(Rdi);
      PUSH4(R8, R9, R10, R11); PUSH4(R12, R13, R14, R15);
#     ifndef XMM_CANT_STORE_PTRS
        PUSH8_LH(Xmm0,  Xmm1,  Xmm2,  Xmm3);
        PUSH8_LH(Xmm4,  Xmm5,  Xmm6,  Xmm7);
        PUSH8_LH(Xmm8,  Xmm9,  Xmm10, Xmm11);
        PUSH8_LH(Xmm12, Xmm13, Xmm14, Xmm15);
#     endif
      sp = (ptr_t)context.Rsp;
#   elif defined(ARM32)
      PUSH4(R0,R1,R2,R3),PUSH4(R4,R5,R6,R7),PUSH4(R8,R9,R10,R11);
      PUSH1(R12);
      sp = (ptr_t)context.Sp;
#   elif defined(AARCH64)
      PUSH4(X0,X1,X2,X3),PUSH4(X4,X5,X6,X7),PUSH4(X8,X9,X10,X11);
      PUSH4(X12,X13,X14,X15),PUSH4(X16,X17,X18,X19),PUSH4(X20,X21,X22,X23);
      PUSH4(X24,X25,X26,X27),PUSH1(X28);
      PUSH1(Lr);
      sp = (ptr_t)context.Sp;
#   elif defined(SHx)
      PUSH4(R0,R1,R2,R3), PUSH4(R4,R5,R6,R7), PUSH4(R8,R9,R10,R11);
      PUSH2(R12,R13), PUSH1(R14);
      sp = (ptr_t)context.R15;
#   elif defined(MIPS)
      PUSH4(IntAt,IntV0,IntV1,IntA0), PUSH4(IntA1,IntA2,IntA3,IntT0);
      PUSH4(IntT1,IntT2,IntT3,IntT4), PUSH4(IntT5,IntT6,IntT7,IntS0);
      PUSH4(IntS1,IntS2,IntS3,IntS4), PUSH4(IntS5,IntS6,IntS7,IntT8);
      PUSH4(IntT9,IntK0,IntK1,IntS8);
      sp = (ptr_t)context.IntSp;
#   elif defined(PPC)
      PUSH4(Gpr0, Gpr3, Gpr4, Gpr5),  PUSH4(Gpr6, Gpr7, Gpr8, Gpr9);
      PUSH4(Gpr10,Gpr11,Gpr12,Gpr14), PUSH4(Gpr15,Gpr16,Gpr17,Gpr18);
      PUSH4(Gpr19,Gpr20,Gpr21,Gpr22), PUSH4(Gpr23,Gpr24,Gpr25,Gpr26);
      PUSH4(Gpr27,Gpr28,Gpr29,Gpr30), PUSH1(Gpr31);
      sp = (ptr_t)context.Gpr1;
#   elif defined(ALPHA)
      PUSH4(IntV0,IntT0,IntT1,IntT2), PUSH4(IntT3,IntT4,IntT5,IntT6);
      PUSH4(IntT7,IntS0,IntS1,IntS2), PUSH4(IntS3,IntS4,IntS5,IntFp);
      PUSH4(IntA0,IntA1,IntA2,IntA3), PUSH4(IntA4,IntA5,IntT8,IntT9);
      PUSH4(IntT10,IntT11,IntT12,IntAt);
      sp = (ptr_t)context.IntSp;
#   elif defined(CPPCHECK)
      GC_noop1_ptr(regs);
      sp = (ptr_t)(word)cnt; /* to workaround "cnt not used" false positive */
#   else
#     error Architecture is not supported
#   endif
#   undef context
#   undef PUSH1
#   undef PUSH2
#   undef PUSH4
#   undef PUSH8_LH
    GC_ASSERT(cnt == PUSHED_REGS_COUNT);
    return sp;
}

STATIC word GC_push_stack_for(GC_thread thread, thread_id_t self_id,
                              GC_bool *pfound_me)
{
  GC_bool is_self = FALSE;
  ptr_t sp, stack_min;
  GC_stack_context_t crtn = thread -> crtn;
  ptr_t stack_end = crtn -> stack_end;
  struct GC_traced_stack_sect_s *traced_stack_sect = crtn -> traced_stack_sect;

  GC_ASSERT(I_HOLD_LOCK());
  if (EXPECT(NULL == stack_end, FALSE)) return 0;

  if (thread -> id == self_id) {
    GC_ASSERT((thread -> flags & DO_BLOCKING) == 0);
    sp = GC_approx_sp();
    is_self = TRUE;
    *pfound_me = TRUE;
  } else if ((thread -> flags & DO_BLOCKING) != 0) {
    /* Use saved sp value for blocked threads.  */
    sp = crtn -> stack_ptr;
  } else {
#   ifdef RETRY_GET_THREAD_CONTEXT
      /* We cache context when suspending the thread since it may       */
      /* require looping.                                               */
      word *regs = thread -> context_regs;

      if ((thread -> flags & IS_SUSPENDED) != 0) {
        sp = thread -> context_sp;
      } else
#   else
      word regs[PUSHED_REGS_COUNT];
#   endif

      /* else */ {
        CONTEXT context;

        /* For unblocked threads call GetThreadContext().       */
        context.ContextFlags = GET_THREAD_CONTEXT_FLAGS;
        if (GetThreadContext(THREAD_HANDLE(thread), &context)) {
          sp = copy_ptr_regs(regs, &context);
        } else {
#         ifdef RETRY_GET_THREAD_CONTEXT
            /* At least, try to use the stale context if saved. */
            sp = thread -> context_sp;
            if (NULL == sp) {
              /* Skip the current thread, anyway its stack will */
              /* be pushed when the world is stopped.           */
              return 0;
            }
#         else
            *(volatile ptr_t *)&sp = NULL;
                    /* to avoid "might be uninitialized" compiler warning */
            ABORT("GetThreadContext failed");
#         endif
        }
      }
#   ifdef THREAD_LOCAL_ALLOC
      GC_ASSERT((thread -> flags & IS_SUSPENDED) != 0 || !GC_world_stopped);
#   endif

#   ifndef WOW64_THREAD_CONTEXT_WORKAROUND
      GC_push_many_regs(regs, PUSHED_REGS_COUNT);
#   else
      GC_push_many_regs(regs + 2, PUSHED_REGS_COUNT - 2);
                                        /* skip ContextFlags and SegFs */

      /* WoW64 workaround. */
      if (isWow64) {
        DWORD ContextFlags = (DWORD)regs[0];

        if ((ContextFlags & CONTEXT_EXCEPTION_REPORTING) != 0
            && (ContextFlags & (CONTEXT_EXCEPTION_ACTIVE
                                /* | CONTEXT_SERVICE_ACTIVE */)) != 0) {
          PNT_TIB tib;

#         ifdef MSWINRT_FLAVOR
            tib = thread -> tib;
#         else
            WORD SegFs = (WORD)regs[1];
            LDT_ENTRY selector;

            if (!GetThreadSelectorEntry(THREAD_HANDLE(thread), SegFs,
                                        &selector))
              ABORT("GetThreadSelectorEntry failed");
            tib = (PNT_TIB)(selector.BaseLow
                            | (selector.HighWord.Bits.BaseMid << 16)
                            | (selector.HighWord.Bits.BaseHi << 24));
#         endif
#         ifdef DEBUG_THREADS
            GC_log_printf("TIB stack limit/base: %p .. %p\n",
                          (void *)(tib -> StackLimit),
                          (void *)(tib -> StackBase));
#         endif
          GC_ASSERT(!HOTTER_THAN((ptr_t)(tib -> StackBase), stack_end));
          if (stack_end != crtn -> initial_stack_base
              /* We are in a coroutine (old-style way of the support).  */
              && (ADDR(stack_end) <= (word)(tib -> StackLimit)
                  || (word)(tib -> StackBase) < ADDR(stack_end))) {
            /* The coroutine stack is not within TIB stack.   */
            WARN("GetThreadContext might return stale register values"
                 " including ESP= %p\n", sp);
            /* TODO: Because of WoW64 bug, there is no guarantee that   */
            /* sp really points to the stack top but, for now, we do    */
            /* our best as the TIB stack limit/base cannot be used      */
            /* while we are inside a coroutine.                         */
          } else {
            /* GetThreadContext() might return stale register values,   */
            /* so we scan the entire stack region (down to the stack    */
            /* limit).  There is no 100% guarantee that all the         */
            /* registers are pushed but we do our best (the proper      */
            /* solution would be to fix it inside Windows).             */
            sp = (ptr_t)(tib -> StackLimit);
          }
        } /* else */
#       ifdef DEBUG_THREADS
          else {
            static GC_bool logged;
            if (!logged
                && (ContextFlags & CONTEXT_EXCEPTION_REPORTING) == 0) {
              GC_log_printf("CONTEXT_EXCEPTION_REQUEST not supported\n");
              logged = TRUE;
            }
          }
#       endif
      }
#   endif /* WOW64_THREAD_CONTEXT_WORKAROUND */
  } /* not current thread */
# ifdef STACKPTR_CORRECTOR_AVAILABLE
    if (GC_sp_corrector != 0)
      GC_sp_corrector((void **)&sp, (void *)(thread -> pthread_id));
# endif

  /* Set stack_min to the lowest address in the thread stack,   */
  /* or to an address in the thread stack no larger than sp,    */
  /* taking advantage of the old value to avoid slow traversals */
  /* of large stacks.                                           */
  if (crtn -> last_stack_min == ADDR_LIMIT) {
#   ifdef MSWINCE
      if (GC_dont_query_stack_min) {
        stack_min = GC_wince_evaluate_stack_min(traced_stack_sect != NULL ?
                                        (ptr_t)traced_stack_sect : stack_end);
        /* Keep last_stack_min value unmodified. */
      } else
#   endif
    /* else */ {
      stack_min = GC_get_stack_min(traced_stack_sect != NULL ?
                                        (ptr_t)traced_stack_sect : stack_end);
      GC_win32_unprotect_thread(thread);
      crtn -> last_stack_min = stack_min;
    }
  } else {
    /* First, adjust the latest known minimum stack address if we       */
    /* are inside GC_call_with_gc_active().                             */
    if (traced_stack_sect != NULL
        && ADDR_LT((ptr_t)traced_stack_sect, crtn -> last_stack_min)) {
      GC_win32_unprotect_thread(thread);
      crtn -> last_stack_min = (ptr_t)traced_stack_sect;
    }

    if (ADDR_INSIDE(sp, crtn -> last_stack_min, stack_end)) {
      stack_min = sp;
    } else {
      /* In the current thread it is always safe to use sp value.       */
      if (may_be_in_stack(is_self && ADDR_LT(sp, crtn -> last_stack_min)
                            ? sp : crtn -> last_stack_min)) {
        stack_min = (ptr_t)last_info.BaseAddress;
        /* Do not probe rest of the stack if sp is correct. */
        if (!ADDR_INSIDE(sp, stack_min, stack_end))
          stack_min = GC_get_stack_min(crtn -> last_stack_min);
      } else {
        /* Stack shrunk?  Is this possible? */
        stack_min = GC_get_stack_min(stack_end);
      }
      GC_win32_unprotect_thread(thread);
      crtn -> last_stack_min = stack_min;
    }
  }

  GC_ASSERT(GC_dont_query_stack_min
            || stack_min == GC_get_stack_min(stack_end)
            || (ADDR_GE(sp, stack_min) && ADDR_LT(stack_min, stack_end)
                && ADDR_LT(GC_get_stack_min(stack_end), stack_min)));

  if (ADDR_INSIDE(sp, stack_min, stack_end)) {
#   ifdef DEBUG_THREADS
      GC_log_printf("Pushing stack for 0x%x from sp %p to %p from 0x%x\n",
                    (int)(thread -> id), (void *)sp, (void *)stack_end,
                    (int)self_id);
#   endif
    GC_push_all_stack_sections(sp, stack_end, traced_stack_sect);
  } else {
    /* If not current thread then it is possible for sp to point to     */
    /* the guarded (untouched yet) page just below the current          */
    /* stack_min of the thread.                                         */
    if (is_self || ADDR_GE(sp, stack_end)
        || ADDR_LT(sp + GC_page_size, stack_min))
      WARN("Thread stack pointer %p out of range, pushing everything\n", sp);
#   ifdef DEBUG_THREADS
      GC_log_printf("Pushing stack for 0x%x from (min) %p to %p from 0x%x\n",
                    (int)(thread -> id), (void *)stack_min, (void *)stack_end,
                    (int)self_id);
#   endif
    /* Push everything - ignore "traced stack section" data.            */
    GC_push_all_stack(stack_min, stack_end);
  }
  return stack_end - sp; /* stack grows down */
}

/* Should do exactly the right thing if the world is stopped; should    */
/* not fail if it is not.                                               */
GC_INNER void GC_push_all_stacks(void)
{
  thread_id_t self_id = GetCurrentThreadId();
  GC_bool found_me = FALSE;
# ifndef SMALL_CONFIG
    unsigned nthreads = 0;
# endif
  word total_size = 0;

  GC_ASSERT(I_HOLD_LOCK());
  GC_ASSERT(GC_thr_initialized);
# ifndef GC_NO_THREADS_DISCOVERY
    if (GC_win32_dll_threads) {
      int i;
      LONG my_max = GC_get_max_thread_index();

      for (i = 0; i <= my_max; i++) {
        GC_thread p = (GC_thread)(dll_thread_table + i);

        if (p -> tm.in_use) {
#         ifndef SMALL_CONFIG
            ++nthreads;
#         endif
          total_size += GC_push_stack_for(p, self_id, &found_me);
        }
      }
    } else
# endif
  /* else */ {
    int i;
    for (i = 0; i < THREAD_TABLE_SZ; i++) {
      GC_thread p;

      for (p = GC_threads[i]; p != NULL; p = p -> tm.next) {
        GC_ASSERT(THREAD_TABLE_INDEX(p -> id) == i);
        if (!KNOWN_FINISHED(p)) {
#         ifndef SMALL_CONFIG
            ++nthreads;
#         endif
          total_size += GC_push_stack_for(p, self_id, &found_me);
        }
      }
    }
  }
# ifndef SMALL_CONFIG
    GC_VERBOSE_LOG_PRINTF("Pushed %d thread stacks%s\n", nthreads,
                          GC_win32_dll_threads ?
                                " based on DllMain thread tracking" : "");
# endif
  if (!found_me && !GC_in_thread_creation)
    ABORT("Collecting from unknown thread");
  GC_total_stacksize = total_size;
}

#ifdef PARALLEL_MARK
  GC_INNER ptr_t GC_marker_last_stack_min[MAX_MARKERS - 1] = {0};
                                /* Last known minimum (hottest) address */
                                /* in stack (or ADDR_LIMIT if unset)    */
                                /* for markers.                         */

#endif /* PARALLEL_MARK */

GC_INNER void GC_get_next_stack(ptr_t start, ptr_t limit,
                                ptr_t *plo, ptr_t *phi)
{
  int i;
  ptr_t current_min = ADDR_LIMIT;   /* Least in-range stack base      */
  ptr_t *plast_stack_min = NULL;    /* Address of last_stack_min      */
                                    /* field for thread corresponding */
                                    /* to current_min.                */
  GC_thread thread = NULL;          /* Either NULL or points to the   */
                                    /* thread's hash table entry      */
                                    /* containing *plast_stack_min.   */

  GC_ASSERT(I_HOLD_LOCK());
  /* First set current_min, ignoring limit. */
  if (GC_win32_dll_threads) {
    LONG my_max = GC_get_max_thread_index();

    for (i = 0; i <= my_max; i++) {
      ptr_t stack_end = (ptr_t)dll_thread_table[i].crtn -> stack_end;

      if (ADDR_LT(start, stack_end) && ADDR_LT(stack_end, current_min)) {
        /* Update address of last_stack_min. */
        plast_stack_min = &(dll_thread_table[i].crtn -> last_stack_min);
        current_min = stack_end;
#       ifdef CPPCHECK
          /* To avoid a warning that thread is always null.     */
          thread = (GC_thread)&dll_thread_table[i];
#       endif
      }
    }
  } else {
    for (i = 0; i < THREAD_TABLE_SZ; i++) {
      GC_thread p;

      for (p = GC_threads[i]; p != NULL; p = p -> tm.next) {
        GC_stack_context_t crtn = p -> crtn;
        ptr_t stack_end = crtn -> stack_end; /* read of a volatile field */

        if (ADDR_LT(start, stack_end) && ADDR_LT(stack_end, current_min)) {
          /* Update address of last_stack_min. */
          plast_stack_min = &(crtn -> last_stack_min);
          thread = p; /* Remember current thread to unprotect. */
          current_min = stack_end;
        }
      }
    }
#   ifdef PARALLEL_MARK
      for (i = 0; i < GC_markers_m1; ++i) {
        ptr_t s = GC_marker_sp[i];

#       ifdef IA64
          /* FIXME: not implemented */
#       endif
        if (ADDR_LT(start, s) && ADDR_LT(s, current_min)) {
          GC_ASSERT(GC_marker_last_stack_min[i] != NULL);
          plast_stack_min = &GC_marker_last_stack_min[i];
          current_min = s;
          thread = NULL; /* Not a thread's hash table entry. */
        }
      }
#   endif
  }

  *phi = current_min;
  if (current_min == ADDR_LIMIT) {
      *plo = ADDR_LIMIT;
      return;
  }

  GC_ASSERT(ADDR_LT(start, current_min) && plast_stack_min != NULL);
# ifdef MSWINCE
    if (GC_dont_query_stack_min) {
      *plo = GC_wince_evaluate_stack_min(current_min);
      /* Keep last_stack_min value unmodified. */
      return;
    }
# endif

  if (ADDR_LT(limit, current_min) && !may_be_in_stack(limit)) {
    /* Skip the rest since the memory region at limit address is        */
    /* not a stack (so the lowest address of the found stack would      */
    /* be above the limit value anyway).                                */
    *plo = ADDR_LIMIT;
    return;
  }

  /* Get the minimum address of the found stack by probing its memory   */
  /* region starting from the recent known minimum (if set).            */
  if (*plast_stack_min == ADDR_LIMIT
      || !may_be_in_stack(*plast_stack_min)) {
    /* Unsafe to start from last_stack_min value. */
    *plo = GC_get_stack_min(current_min);
  } else {
    /* Use the recent value to optimize search for min address. */
    *plo = GC_get_stack_min(*plast_stack_min);
  }

  /* Remember current stack_min value. */
  if (thread != NULL)
    GC_win32_unprotect_thread(thread);
  *plast_stack_min = *plo;
}

#if defined(PARALLEL_MARK) && !defined(GC_PTHREADS_PARAMARK)

#   ifndef MARK_THREAD_STACK_SIZE
#     define MARK_THREAD_STACK_SIZE 0   /* default value */
#   endif

    STATIC HANDLE GC_marker_cv[MAX_MARKERS - 1] = {0};
                        /* Events with manual reset (one for each       */
                        /* mark helper).                                */

    GC_INNER thread_id_t GC_marker_Id[MAX_MARKERS - 1] = {0};
                        /* This table is used for mapping helper        */
                        /* threads ID to mark helper index (linear      */
                        /* search is used since the mapping contains    */
                        /* only a few entries).                         */

    /* mark_mutex_event, builder_cv, mark_cv are initialized in GC_thr_init */
    static HANDLE mark_mutex_event = (HANDLE)0; /* Event with auto-reset.   */
    static HANDLE builder_cv = (HANDLE)0; /* Event with manual reset.       */
    static HANDLE mark_cv = (HANDLE)0; /* Event with manual reset.          */

    GC_INNER void GC_start_mark_threads_inner(void)
    {
      int i;

      GC_ASSERT(I_HOLD_LOCK());
      ASSERT_CANCEL_DISABLED();
      if (GC_available_markers_m1 <= 0 || GC_parallel) return;
      GC_wait_for_gc_completion(TRUE);

      GC_ASSERT(GC_fl_builder_count == 0);
      /* Initialize GC_marker_cv[] fully before starting the    */
      /* first helper thread.                                   */
      GC_markers_m1 = GC_available_markers_m1;
      for (i = 0; i < GC_markers_m1; ++i) {
        if ((GC_marker_cv[i] = CreateEvent(NULL /* attrs */,
                                        TRUE /* isManualReset */,
                                        FALSE /* initialState */,
                                        NULL /* name (A/W) */)) == (HANDLE)0)
          ABORT("CreateEvent failed");
      }

      for (i = 0; i < GC_markers_m1; ++i) {
#       if defined(MSWINCE) || defined(MSWIN_XBOX1)
          HANDLE handle;
          DWORD thread_id;

          GC_marker_last_stack_min[i] = ADDR_LIMIT;
          /* There is no _beginthreadex() in WinCE. */
          handle = CreateThread(NULL /* lpsa */,
                                MARK_THREAD_STACK_SIZE /* ignored */,
                                GC_mark_thread, (LPVOID)(word)i,
                                0 /* fdwCreate */, &thread_id);
          if (EXPECT(NULL == handle, FALSE)) {
            WARN("Marker thread %" WARN_PRIdPTR " creation failed\n",
                 (signed_word)i);
            /* The most probable failure reason is "not enough memory". */
            /* Don't try to create other marker threads.                */
            break;
          }
          /* It is safe to detach the thread.   */
          CloseHandle(handle);
#       else
          GC_uintptr_t handle;
          unsigned thread_id;

          GC_marker_last_stack_min[i] = ADDR_LIMIT;
          handle = _beginthreadex(NULL /* security_attr */,
                                MARK_THREAD_STACK_SIZE, GC_mark_thread,
                                (void *)(word)i, 0 /* flags */, &thread_id);
          if (EXPECT(!handle || handle == (GC_uintptr_t)-1L, FALSE)) {
            WARN("Marker thread %" WARN_PRIdPTR " creation failed\n",
                 (signed_word)i);
            /* Don't try to create other marker threads.                */
            break;
          } else {/* We may detach the thread (if handle is of HANDLE type) */
            /* CloseHandle((HANDLE)handle); */
          }
#       endif
      }

      /* Adjust GC_markers_m1 (and free unused resources) if failed.    */
      while (GC_markers_m1 > i) {
        GC_markers_m1--;
        CloseHandle(GC_marker_cv[GC_markers_m1]);
      }
      GC_wait_for_markers_init();
      GC_COND_LOG_PRINTF("Started %d mark helper threads\n", GC_markers_m1);
      if (EXPECT(0 == i, FALSE)) {
        CloseHandle(mark_cv);
        CloseHandle(builder_cv);
        CloseHandle(mark_mutex_event);
      }
    }

#   ifdef GC_ASSERTIONS
      STATIC unsigned long GC_mark_lock_holder = NO_THREAD;
#     define SET_MARK_LOCK_HOLDER \
                (void)(GC_mark_lock_holder = GetCurrentThreadId())
#     define UNSET_MARK_LOCK_HOLDER \
                do { \
                  GC_ASSERT(GC_mark_lock_holder == GetCurrentThreadId()); \
                  GC_mark_lock_holder = NO_THREAD; \
                } while (0)
#   else
#     define SET_MARK_LOCK_HOLDER (void)0
#     define UNSET_MARK_LOCK_HOLDER (void)0
#   endif /* !GC_ASSERTIONS */

    STATIC /* volatile */ LONG GC_mark_mutex_state = 0;
                                /* Mutex state: 0 - unlocked,           */
                                /* 1 - locked and no other waiters,     */
                                /* -1 - locked and waiters may exist.   */
                                /* Accessed by InterlockedExchange().   */

#   ifdef LOCK_STATS
      volatile AO_t GC_block_count = 0;
      volatile AO_t GC_unlocked_count = 0;
#   endif

    GC_INNER void GC_acquire_mark_lock(void)
    {
      GC_ASSERT(GC_mark_lock_holder != GetCurrentThreadId());
      if (EXPECT(InterlockedExchange(&GC_mark_mutex_state,
                                     1 /* locked */) != 0, FALSE)) {
#       ifdef LOCK_STATS
          (void)AO_fetch_and_add1(&GC_block_count);
#       endif
        /* Repeatedly reset the state and wait until we acquire the */
        /* mark lock.                                               */
        while (InterlockedExchange(&GC_mark_mutex_state,
                                   -1 /* locked_and_has_waiters */) != 0) {
          if (WaitForSingleObject(mark_mutex_event, INFINITE) == WAIT_FAILED)
            ABORT("WaitForSingleObject failed");
        }
      }
#     ifdef LOCK_STATS
        else {
          (void)AO_fetch_and_add1(&GC_unlocked_count);
        }
#     endif

      GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
      SET_MARK_LOCK_HOLDER;
    }

    GC_INNER void GC_release_mark_lock(void)
    {
      UNSET_MARK_LOCK_HOLDER;
      if (EXPECT(InterlockedExchange(&GC_mark_mutex_state,
                                     0 /* unlocked */) < 0, FALSE)) {
        /* wake a waiter */
        if (SetEvent(mark_mutex_event) == FALSE)
          ABORT("SetEvent failed");
      }
    }

    /* In GC_wait_for_reclaim/GC_notify_all_builder() we emulate POSIX  */
    /* cond_wait/cond_broadcast() primitives with WinAPI Event object   */
    /* (working in "manual reset" mode).  This works here because       */
    /* GC_notify_all_builder() is always called holding the mark lock   */
    /* and the checked condition (GC_fl_builder_count == 0) is the only */
    /* one for which broadcasting on builder_cv is performed.           */

    GC_INNER void GC_wait_for_reclaim(void)
    {
      GC_ASSERT(builder_cv != 0);
      for (;;) {
        GC_acquire_mark_lock();
        if (GC_fl_builder_count == 0)
          break;
        if (ResetEvent(builder_cv) == FALSE)
          ABORT("ResetEvent failed");
        GC_release_mark_lock();
        if (WaitForSingleObject(builder_cv, INFINITE) == WAIT_FAILED)
          ABORT("WaitForSingleObject failed");
      }
      GC_release_mark_lock();
    }

    GC_INNER void GC_notify_all_builder(void)
    {
      GC_ASSERT(GC_mark_lock_holder == GetCurrentThreadId());
      GC_ASSERT(builder_cv != 0);
      GC_ASSERT(GC_fl_builder_count == 0);
      if (SetEvent(builder_cv) == FALSE)
        ABORT("SetEvent failed");
    }

    /* mark_cv is used (for waiting) by a non-helper thread.    */

    GC_INNER void GC_wait_marker(void)
    {
      HANDLE event = mark_cv;
      thread_id_t self_id = GetCurrentThreadId();
      int i = GC_markers_m1;

      while (i-- > 0) {
        if (GC_marker_Id[i] == self_id) {
          event = GC_marker_cv[i];
          break;
        }
      }

      if (ResetEvent(event) == FALSE)
        ABORT("ResetEvent failed");
      GC_release_mark_lock();
      if (WaitForSingleObject(event, INFINITE) == WAIT_FAILED)
        ABORT("WaitForSingleObject failed");
      GC_acquire_mark_lock();
    }

    GC_INNER void GC_notify_all_marker(void)
    {
      thread_id_t self_id = GetCurrentThreadId();
      int i = GC_markers_m1;

      while (i-- > 0) {
        /* Notify every marker ignoring self (for efficiency).  */
        if (SetEvent(GC_marker_Id[i] != self_id ? GC_marker_cv[i] :
                     mark_cv) == FALSE)
          ABORT("SetEvent failed");
      }
    }

#endif /* PARALLEL_MARK && !GC_PTHREADS_PARAMARK */

/* We have no DllMain to take care of new threads.  Thus, we    */
/* must properly intercept thread creation.                     */

struct win32_start_info {
  LPTHREAD_START_ROUTINE start_routine;
  LPVOID arg;
};

STATIC void *GC_CALLBACK GC_win32_start_inner(struct GC_stack_base *sb,
                                              void *arg)
{
    void * ret;
    LPTHREAD_START_ROUTINE start_routine =
                        ((struct win32_start_info *)arg) -> start_routine;
    LPVOID start_arg = ((struct win32_start_info *)arg) -> arg;

    GC_ASSERT(!GC_win32_dll_threads);
    GC_register_my_thread(sb); /* This waits for an in-progress GC.     */
#   ifdef DEBUG_THREADS
      GC_log_printf("thread 0x%lx starting...\n", (long)GetCurrentThreadId());
#   endif
    GC_free(arg);

    /* Clear the thread entry even if we exit with an exception.        */
    /* This is probably pointless, since an uncaught exception is       */
    /* supposed to result in the process being killed.                  */
#   ifndef NO_SEH_AVAILABLE
      ret = NULL; /* to avoid "might be uninitialized" compiler warning */
      __try
#   endif
    {
      ret = (void *)(word)((*start_routine)(start_arg));
    }
#   ifndef NO_SEH_AVAILABLE
      __finally
#   endif
    {
      (void)GC_unregister_my_thread();
    }

#   ifdef DEBUG_THREADS
      GC_log_printf("thread 0x%lx returned from start routine\n",
                    (long)GetCurrentThreadId());
#   endif
#   if defined(CPPCHECK)
      GC_noop1_ptr(sb);
#   endif
    return ret;
}

STATIC DWORD WINAPI GC_win32_start(LPVOID arg)
{
    return (DWORD)(word)GC_call_with_stack_base(GC_win32_start_inner, arg);
}

GC_API HANDLE WINAPI GC_CreateThread(
                        LPSECURITY_ATTRIBUTES lpThreadAttributes,
                        GC_WIN32_SIZE_T dwStackSize,
                        LPTHREAD_START_ROUTINE lpStartAddress,
                        LPVOID lpParameter, DWORD dwCreationFlags,
                        LPDWORD lpThreadId)
{
    if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
    GC_ASSERT(GC_thr_initialized);
        /* Make sure GC is initialized (i.e. main thread is attached,   */
        /* tls is initialized).  This is redundant when                 */
        /* GC_win32_dll_threads is set by GC_use_threads_discovery().   */

#   ifdef DEBUG_THREADS
      GC_log_printf("About to create a thread from 0x%lx\n",
                    (long)GetCurrentThreadId());
#   endif
    if (GC_win32_dll_threads) {
      return CreateThread(lpThreadAttributes, dwStackSize, lpStartAddress,
                          lpParameter, dwCreationFlags, lpThreadId);
    } else {
      struct win32_start_info *psi =
                (struct win32_start_info *)GC_malloc_uncollectable(
                                        sizeof(struct win32_start_info));
                /* Handed off to and deallocated by child thread.       */
      HANDLE thread_h;

      if (EXPECT(NULL == psi, FALSE)) {
        SetLastError(ERROR_NOT_ENOUGH_MEMORY);
        return NULL;
      }

      /* set up thread arguments */
      psi -> start_routine = lpStartAddress;
      psi -> arg = lpParameter;
      GC_dirty(psi);
      REACHABLE_AFTER_DIRTY(lpParameter);

#     ifdef PARALLEL_MARK
        if (EXPECT(!GC_parallel && GC_available_markers_m1 > 0, FALSE))
          GC_start_mark_threads();
#     endif
      set_need_to_lock();
      thread_h = CreateThread(lpThreadAttributes, dwStackSize, GC_win32_start,
                              psi, dwCreationFlags, lpThreadId);
      if (EXPECT(0 == thread_h, FALSE)) GC_free(psi);
      return thread_h;
    }
}

GC_API DECLSPEC_NORETURN void WINAPI GC_ExitThread(DWORD dwExitCode)
{
    if (!GC_win32_dll_threads) (void)GC_unregister_my_thread();
    ExitThread(dwExitCode);
}

#if !defined(CYGWIN32) && !defined(MSWINCE) && !defined(MSWIN_XBOX1) \
    && !defined(NO_CRT)
    GC_API GC_uintptr_t GC_CALL GC_beginthreadex(
                                  void *security, unsigned stack_size,
                                  unsigned (__stdcall *start_address)(void *),
                                  void *arglist, unsigned initflag,
                                  unsigned *thrdaddr)
    {
      if (!EXPECT(GC_is_initialized, TRUE)) GC_init();
      GC_ASSERT(GC_thr_initialized);
#     ifdef DEBUG_THREADS
        GC_log_printf("About to create a thread from 0x%lx\n",
                      (long)GetCurrentThreadId());
#     endif

      if (GC_win32_dll_threads) {
        return _beginthreadex(security, stack_size, start_address,
                              arglist, initflag, thrdaddr);
      } else {
        GC_uintptr_t thread_h;
        struct win32_start_info *psi =
                (struct win32_start_info *)GC_malloc_uncollectable(
                                        sizeof(struct win32_start_info));
                /* Handed off to and deallocated by child thread.       */

        if (EXPECT(NULL == psi, FALSE)) {
          /* MSDN docs say _beginthreadex() returns 0 on error and sets */
          /* errno to either EAGAIN (too many threads) or EINVAL (the   */
          /* argument is invalid or the stack size is incorrect), so we */
          /* set errno to EAGAIN on "not enough memory".                */
          errno = EAGAIN;
          return 0;
        }

        /* set up thread arguments */
        psi -> start_routine = (LPTHREAD_START_ROUTINE)start_address;
        psi -> arg = arglist;
        GC_dirty(psi);
        REACHABLE_AFTER_DIRTY(arglist);

#       ifdef PARALLEL_MARK
          if (EXPECT(!GC_parallel && GC_available_markers_m1 > 0, FALSE))
            GC_start_mark_threads();
#       endif
        set_need_to_lock();
        thread_h = _beginthreadex(security, stack_size,
                        (unsigned (__stdcall *)(void *))GC_win32_start,
                        psi, initflag, thrdaddr);
        if (EXPECT(0 == thread_h, FALSE)) GC_free(psi);
        return thread_h;
      }
    }

    GC_API void GC_CALL GC_endthreadex(unsigned retval)
    {
      if (!GC_win32_dll_threads) (void)GC_unregister_my_thread();
      _endthreadex(retval);
    }
#endif /* !CYGWIN32 && !MSWINCE && !MSWIN_XBOX1 && !NO_CRT */

#ifdef GC_WINMAIN_REDIRECT
  /* This might be useful on WinCE.  Shouldn't be used with GC_DLL.     */

# if defined(MSWINCE) && defined(UNDER_CE)
#   define WINMAIN_LPTSTR LPWSTR
# else
#   define WINMAIN_LPTSTR LPSTR
# endif

  /* This is defined in gc.h.   */
# undef WinMain

  /* Defined outside GC by an application.      */
  int WINAPI GC_WinMain(HINSTANCE, HINSTANCE, WINMAIN_LPTSTR, int);

  typedef struct {
    HINSTANCE hInstance;
    HINSTANCE hPrevInstance;
    WINMAIN_LPTSTR lpCmdLine;
    int nShowCmd;
  } main_thread_args;

  static DWORD WINAPI main_thread_start(LPVOID arg)
  {
    main_thread_args *main_args = (main_thread_args *)arg;
    return (DWORD)GC_WinMain(main_args -> hInstance,
                             main_args -> hPrevInstance,
                             main_args -> lpCmdLine, main_args -> nShowCmd);
  }

  STATIC void *GC_CALLBACK GC_waitForSingleObjectInfinite(void *handle)
  {
    return (void *)(word)WaitForSingleObject((HANDLE)handle, INFINITE);
  }

# ifndef WINMAIN_THREAD_STACK_SIZE
#   define WINMAIN_THREAD_STACK_SIZE 0  /* default value */
# endif

  int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
                     WINMAIN_LPTSTR lpCmdLine, int nShowCmd)
  {
    DWORD exit_code = 1;

    main_thread_args args = {
                hInstance, hPrevInstance, lpCmdLine, nShowCmd
    };
    HANDLE thread_h;
    DWORD thread_id;

    /* initialize everything */
    GC_INIT();

    /* start the main thread */
    thread_h = GC_CreateThread(NULL /* lpsa */,
                        WINMAIN_THREAD_STACK_SIZE /* ignored on WinCE */,
                        main_thread_start, &args, 0 /* fdwCreate */,
                        &thread_id);
    if (NULL == thread_h)
      ABORT("GC_CreateThread(main_thread) failed");

    if ((DWORD)(word)GC_do_blocking(GC_waitForSingleObjectInfinite,
                                    (void *)thread_h) == WAIT_FAILED)
      ABORT("WaitForSingleObject(main_thread) failed");
    GetExitCodeThread(thread_h, &exit_code);
    CloseHandle(thread_h);

#   ifdef MSWINCE
      GC_deinit();
#   endif
    return (int)exit_code;
  }

#endif /* GC_WINMAIN_REDIRECT */

#ifdef WOW64_THREAD_CONTEXT_WORKAROUND
# ifdef MSWINRT_FLAVOR
    /* Available on WinRT but we have to declare it manually.   */
    __declspec(dllimport) HMODULE WINAPI GetModuleHandleW(LPCWSTR);
# endif

  static GC_bool is_wow64_process(HMODULE hK32)
  {
    BOOL is_wow64;
#   ifdef MSWINRT_FLAVOR
      /* Try to use IsWow64Process2 as it handles different WoW64 cases. */
      HMODULE hWow64 = GetModuleHandleW(L"api-ms-win-core-wow64-l1-1-1.dll");

      UNUSED_ARG(hK32);
      if (hWow64) {
        FARPROC pfn2 = GetProcAddress(hWow64, "IsWow64Process2");
        USHORT process_machine, native_machine;

        if (pfn2
            && (*(BOOL (WINAPI *)(HANDLE, USHORT *, USHORT *))
                (GC_funcptr_uint)pfn2)(GetCurrentProcess(), &process_machine,
                                       &native_machine))
          return process_machine != native_machine;
      }
      if (IsWow64Process(GetCurrentProcess(), &is_wow64))
        return (GC_bool)is_wow64;
#   else
      if (hK32) {
        FARPROC pfn = GetProcAddress(hK32, "IsWow64Process");

        if (pfn
            && (*(BOOL (WINAPI *)(HANDLE, BOOL *))
                (GC_funcptr_uint)pfn)(GetCurrentProcess(), &is_wow64))
          return (GC_bool)is_wow64;
      }
#   endif
    return FALSE; /* IsWow64Process failed */
  }
#endif /* WOW64_THREAD_CONTEXT_WORKAROUND */

GC_INNER void GC_thr_init(void)
{
  struct GC_stack_base sb;
  thread_id_t self_id = GetCurrentThreadId();
# if (!defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) && !defined(MSWINCE) \
      && defined(PARALLEL_MARK)) || defined(WOW64_THREAD_CONTEXT_WORKAROUND)
    HMODULE hK32;
#   if defined(MSWINRT_FLAVOR) && defined(FUNCPTR_IS_DATAPTR)
      MEMORY_BASIC_INFORMATION memInfo;

      if (VirtualQuery(CAST_THRU_UINTPTR(void*, GetProcAddress),
                       &memInfo, sizeof(memInfo)) != sizeof(memInfo))
        ABORT("Weird VirtualQuery result");
      hK32 = (HMODULE)memInfo.AllocationBase;
#   else
      hK32 = GetModuleHandle(TEXT("kernel32.dll"));
#   endif
# endif

  GC_ASSERT(I_HOLD_LOCK());
  GC_ASSERT(!GC_thr_initialized);
  GC_ASSERT(ADDR(&GC_threads) % sizeof(ptr_t) == 0);
# ifdef GC_ASSERTIONS
    GC_thr_initialized = TRUE;
# endif
# if !defined(DONT_USE_ATEXIT) || !defined(GC_NO_THREADS_DISCOVERY)
    GC_main_thread_id = self_id;
# endif
# ifdef CAN_HANDLE_FORK
    GC_setup_atfork();
# endif
# ifdef WOW64_THREAD_CONTEXT_WORKAROUND
    /* Set isWow64 flag. */
    isWow64 = is_wow64_process(hK32);
# endif
  /* Add the initial thread, so we can stop it. */
  sb.mem_base = GC_stackbottom;
  GC_ASSERT(sb.mem_base != NULL);
# ifdef IA64
    sb.reg_base = GC_register_stackbottom;
# endif

# if defined(PARALLEL_MARK)
    {
      const char *markers_string = GETENV("GC_MARKERS");
      int markers = GC_required_markers_cnt;

      if (markers_string != NULL) {
        markers = atoi(markers_string);
        if (markers <= 0 || markers > MAX_MARKERS) {
          WARN("Too big or invalid number of mark threads: %" WARN_PRIdPTR
               "; using maximum threads\n", (signed_word)markers);
          markers = MAX_MARKERS;
        }
      } else if (0 == markers) {
        /* Unless the client sets the desired number of         */
        /* parallel markers, it is determined based on the      */
        /* number of CPU cores.                                 */
#       ifdef MSWINCE
          /* There is no GetProcessAffinityMask() in WinCE.     */
          /* GC_sysinfo is already initialized.                 */
          markers = (int)GC_sysinfo.dwNumberOfProcessors;
#       else
#         ifdef _WIN64
            DWORD_PTR procMask = 0;
            DWORD_PTR sysMask;
#         else
            DWORD procMask = 0;
            DWORD sysMask;
#         endif
          int ncpu = 0;
          if (
#           ifdef __cplusplus
              GetProcessAffinityMask(GetCurrentProcess(), &procMask, &sysMask)
#           else
              /* Cast args to void* for compatibility with some old SDKs. */
              GetProcessAffinityMask(GetCurrentProcess(),
                                     (void *)&procMask, (void *)&sysMask)
#           endif
              && procMask) {
            do {
              ncpu++;
            } while ((procMask &= procMask - 1) != 0);
          }
          markers = ncpu;
#       endif
#       if defined(GC_MIN_MARKERS) && !defined(CPPCHECK)
          /* This is primarily for testing on systems without getenv(). */
          if (markers < GC_MIN_MARKERS)
            markers = GC_MIN_MARKERS;
#       endif
        if (markers > MAX_MARKERS)
          markers = MAX_MARKERS; /* silently limit the value */
      }
      GC_available_markers_m1 = markers - 1;
    }

    /* Check whether parallel mode could be enabled.    */
      if (GC_win32_dll_threads || GC_available_markers_m1 <= 0) {
        /* Disable parallel marking. */
        GC_parallel = FALSE;
        GC_COND_LOG_PRINTF(
                "Single marker thread, turning off parallel marking\n");
      } else {
#       ifndef GC_PTHREADS_PARAMARK
          /* Initialize Win32 event objects for parallel marking.       */
          mark_mutex_event = CreateEvent(NULL /* attrs */,
                                FALSE /* isManualReset */,
                                FALSE /* initialState */, NULL /* name */);
          builder_cv = CreateEvent(NULL /* attrs */,
                                TRUE /* isManualReset */,
                                FALSE /* initialState */, NULL /* name */);
          mark_cv = CreateEvent(NULL /* attrs */, TRUE /* isManualReset */,
                                FALSE /* initialState */, NULL /* name */);
          if (mark_mutex_event == (HANDLE)0 || builder_cv == (HANDLE)0
              || mark_cv == (HANDLE)0)
            ABORT("CreateEvent failed");
#       endif
#       if !defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID) && !defined(MSWINCE)
          GC_init_win32_thread_naming(hK32);
#       endif
      }
# endif /* PARALLEL_MARK */

  GC_register_my_thread_inner(&sb, self_id);
}

#ifndef GC_NO_THREADS_DISCOVERY
    /* We avoid acquiring locks here, since this doesn't seem to be     */
    /* preemptible.  This may run with an uninitialized collector, in   */
    /* which case we don't do much.  This implies that no threads other */
    /* than the main one should be created with an uninitialized        */
    /* collector.  (The alternative of initializing the collector here  */
    /* seems dangerous, since DllMain is limited in what it can do.)    */

# ifdef GC_INSIDE_DLL
    /* Export only if needed by client. */
    GC_API
# else
#   define GC_DllMain DllMain
# endif
  BOOL WINAPI GC_DllMain(HINSTANCE inst, ULONG reason, LPVOID reserved)
  {
      thread_id_t self_id;

      UNUSED_ARG(inst);
      UNUSED_ARG(reserved);
      /* Note that GC_use_threads_discovery should be called by the     */
      /* client application at start-up to activate automatic thread    */
      /* registration (it is the default GC behavior);                  */
      /* to always have automatic thread registration turned on, the GC */
      /* should be compiled with -D GC_DISCOVER_TASK_THREADS.           */
      if (!GC_win32_dll_threads && GC_is_initialized) return TRUE;

      switch (reason) {
       case DLL_THREAD_ATTACH: /* invoked for threads other than main */
#       ifdef PARALLEL_MARK
          /* Don't register marker threads. */
          if (GC_parallel) {
            /* We could reach here only if GC is not initialized.       */
            /* Because GC_thr_init() sets GC_parallel to off.           */
            break;
          }
#       endif
        /* FALLTHRU */
       case DLL_PROCESS_ATTACH:
        /* This may run with the collector uninitialized. */
        self_id = GetCurrentThreadId();
        if (GC_is_initialized && GC_main_thread_id != self_id) {
            struct GC_stack_base sb;
            /* Don't lock here. */
#           ifdef GC_ASSERTIONS
              int sb_result =
#           endif
                        GC_get_stack_base(&sb);
            GC_ASSERT(sb_result == GC_SUCCESS);
            GC_register_my_thread_inner(&sb, self_id);
        } /* else we already did it during GC_thr_init, called by GC_init */
        break;

       case DLL_THREAD_DETACH:
        /* We are hopefully running in the context of the exiting thread. */
        if (GC_win32_dll_threads) {
          GC_thread t = GC_win32_dll_lookup_thread(GetCurrentThreadId());

          if (EXPECT(t != NULL, TRUE)) GC_delete_thread(t);
        }
        break;

       case DLL_PROCESS_DETACH:
        if (GC_win32_dll_threads) {
          int i;
          int my_max = (int)GC_get_max_thread_index();

          for (i = 0; i <= my_max; ++i) {
           if (AO_load(&(dll_thread_table[i].tm.in_use)))
             GC_delete_thread((GC_thread)&dll_thread_table[i]);
          }
          GC_deinit();
        }
        break;
      }
      return TRUE;
  }
#endif /* !GC_NO_THREADS_DISCOVERY */

# ifndef GC_NO_THREAD_REDIRECTS
    /* Restore thread calls redirection.        */
#   define CreateThread GC_CreateThread
#   define ExitThread GC_ExitThread
#   undef _beginthreadex
#   define _beginthreadex GC_beginthreadex
#   undef _endthreadex
#   define _endthreadex GC_endthreadex
# endif /* !GC_NO_THREAD_REDIRECTS */

#endif /* GC_WIN32_THREADS */


#ifndef GC_PTHREAD_START_STANDALONE
/*
 * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
 * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
 * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
 * Copyright (c) 2000-2010 by Hewlett-Packard Development Company.
 * All rights reserved.
 *
 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
 * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
 *
 * Permission is hereby granted to use or copy this program
 * for any purpose, provided the above notices are retained on all copies.
 * Permission to modify the code and to distribute modified code is granted,
 * provided the above notices are retained, and a notice that the code was
 * modified is included with the above copyright notice.
 */

/* We want to make sure that GC_thread_exit_proc() is unconditionally   */
/* invoked, even if the client is not compiled with -fexceptions, but   */
/* the GC is.  The workaround is to put GC_pthread_start_inner() in its */
/* own file (pthread_start.c), and undefine __EXCEPTIONS in the GCC     */
/* case at the top of the file.  FIXME: it's still unclear whether this */
/* will actually cause the exit handler to be invoked last when         */
/* thread_exit is called (and if -fexceptions is used).                 */
#if !defined(DONT_UNDEF_EXCEPTIONS) && defined(__GNUC__) && defined(__linux__)
  /* We undefine __EXCEPTIONS to avoid using GCC __cleanup__ attribute. */
  /* The current NPTL implementation of pthread_cleanup_push uses       */
  /* __cleanup__ attribute when __EXCEPTIONS is defined (-fexceptions). */
  /* The stack unwinding and cleanup with __cleanup__ attributes work   */
  /* correctly when everything is compiled with -fexceptions, but it is */
  /* not the requirement for this library clients to use -fexceptions   */
  /* everywhere.  With __EXCEPTIONS undefined, the cleanup routines are */
  /* registered with __pthread_register_cancel thus should work anyway. */
# undef __EXCEPTIONS
#endif


#if defined(GC_PTHREADS) \
    && !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2)

/* Invoked from GC_pthread_start. */
GC_INNER_PTHRSTART void *GC_CALLBACK GC_pthread_start_inner(
                                        struct GC_stack_base *sb, void *arg)
{
  void * (*start)(void *);
  void * start_arg;
  void * result;
  volatile GC_thread me =
                GC_start_rtn_prepare_thread(&start, &start_arg, sb, arg);

# ifndef NACL
    pthread_cleanup_push(GC_thread_exit_proc, (/* no volatile */ void *)me);
# endif
  result = (*start)(start_arg);
# if defined(DEBUG_THREADS) && !defined(GC_PTHREAD_START_STANDALONE)
    GC_log_printf("Finishing thread %p\n",
                  (void *)GC_PTHREAD_PTRVAL(pthread_self()));
# endif
  me -> status = result;
  GC_end_stubborn_change(me); /* cannot use GC_dirty */
  /* Cleanup acquires the allocator lock, ensuring that we cannot exit  */
  /* while a collection that thinks we are alive is trying to stop us.  */
# ifdef NACL
    GC_thread_exit_proc((/* no volatile */ void *)me);
# else
    pthread_cleanup_pop(1);
# endif
  return result;
}

#endif /* GC_PTHREADS */

#endif

/* Restore pthread calls redirection (if altered in             */
/* pthread_stop_world.c, pthread_support.c or win32_threads.c). */
/* This is only useful if directly included from application    */
/* (instead of linking gc).                                     */
#ifndef GC_NO_THREAD_REDIRECTS
# define GC_PTHREAD_REDIRECTS_ONLY
# include "gc/gc_pthread_redirects.h"
#endif

/* The files from "extra" folder are not included. */
