diff --git a/src/core.c b/src/core.c index ce297d8a..4aa34e7d 100644 --- a/src/core.c +++ b/src/core.c @@ -823,6 +823,70 @@ size_t __hot mdbx_e2k_strnlen_bug_workaround(const char *s, size_t maxlen) { /*------------------------------------------------------------------------------ * safe read/write volatile 64-bit fields on 32-bit architectures. */ +MDBX_MAYBE_UNUSED static __always_inline uint64_t +atomic_store64(MDBX_atomic_uint64_t *p, const uint64_t value, + enum MDBX_memory_order order) { + STATIC_ASSERT(sizeof(MDBX_atomic_uint64_t) == 8); +#if MDBX_64BIT_ATOMIC +#ifdef MDBX_HAVE_C11ATOMICS + assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p))); + atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value, mo_c11_store(order)); +#else /* MDBX_HAVE_C11ATOMICS */ + if (order != mo_Relaxed) + mdbx_compiler_barrier(); + p->weak = value; + mdbx_memory_fence(order, true); +#endif /* MDBX_HAVE_C11ATOMICS */ +#else /* !MDBX_64BIT_ATOMIC */ + mdbx_compiler_barrier(); + atomic_store32(&p->low, (uint32_t)value, mo_Relaxed); + mdbx_jitter4testing(true); + atomic_store32(&p->high, (uint32_t)(value >> 32), order); + mdbx_jitter4testing(true); +#endif /* !MDBX_64BIT_ATOMIC */ + return value; +} + +MDBX_MAYBE_UNUSED static +#if MDBX_64BIT_ATOMIC + __always_inline +#endif /* MDBX_64BIT_ATOMIC */ + uint64_t + atomic_load64(const MDBX_atomic_uint64_t *p, + enum MDBX_memory_order order) { + STATIC_ASSERT(sizeof(MDBX_atomic_uint64_t) == 8); +#if MDBX_64BIT_ATOMIC +#ifdef MDBX_HAVE_C11ATOMICS + assert(atomic_is_lock_free(MDBX_c11a_ro(uint64_t, p))); + return atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)); +#else /* MDBX_HAVE_C11ATOMICS */ + mdbx_memory_fence(order, false); + const uint64_t value = p->weak; + if (order != mo_Relaxed) + mdbx_compiler_barrier(); + return value; +#endif /* MDBX_HAVE_C11ATOMICS */ +#else /* !MDBX_64BIT_ATOMIC */ + mdbx_compiler_barrier(); + uint64_t value = (uint64_t)atomic_load32(&p->high, order) << 32; + mdbx_jitter4testing(true); + value |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed + : mo_AcquireRelease); + mdbx_jitter4testing(true); + for (;;) { + mdbx_compiler_barrier(); + uint64_t again = (uint64_t)atomic_load32(&p->high, order) << 32; + mdbx_jitter4testing(true); + again |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed + : mo_AcquireRelease); + mdbx_jitter4testing(true); + if (likely(value == again)) + return value; + value = again; + } +#endif /* !MDBX_64BIT_ATOMIC */ +} + static __always_inline void atomic_yield(void) { #if defined(_WIN32) || defined(_WIN64) YieldProcessor(); diff --git a/src/internals.h b/src/internals.h index dbe7aeac..a3401200 100644 --- a/src/internals.h +++ b/src/internals.h @@ -264,8 +264,6 @@ static __always_inline memory_order mo_c11_load(enum MDBX_memory_order fence) { #ifndef __cplusplus -static __inline void mdbx_jitter4testing(bool tiny); - MDBX_MAYBE_UNUSED static __always_inline void mdbx_memory_fence(enum MDBX_memory_order order, bool write) { #ifdef MDBX_HAVE_C11ATOMICS @@ -309,70 +307,6 @@ atomic_load32(const MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) { #endif /* MDBX_HAVE_C11ATOMICS */ } -MDBX_MAYBE_UNUSED static __always_inline uint64_t -atomic_store64(MDBX_atomic_uint64_t *p, const uint64_t value, - enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint64_t) == 8); -#if MDBX_64BIT_ATOMIC -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p))); - atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value, mo_c11_store(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - if (order != mo_Relaxed) - mdbx_compiler_barrier(); - p->weak = value; - mdbx_memory_fence(order, true); -#endif /* MDBX_HAVE_C11ATOMICS */ -#else /* !MDBX_64BIT_ATOMIC */ - mdbx_compiler_barrier(); - atomic_store32(&p->low, (uint32_t)value, mo_Relaxed); - mdbx_jitter4testing(true); - atomic_store32(&p->high, (uint32_t)(value >> 32), order); - mdbx_jitter4testing(true); -#endif /* !MDBX_64BIT_ATOMIC */ - return value; -} - -MDBX_MAYBE_UNUSED static -#if MDBX_64BIT_ATOMIC - __always_inline -#endif /* MDBX_64BIT_ATOMIC */ - uint64_t - atomic_load64(const MDBX_atomic_uint64_t *p, - enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint64_t) == 8); -#if MDBX_64BIT_ATOMIC -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_ro(uint64_t, p))); - return atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - mdbx_memory_fence(order, false); - const uint64_t value = p->weak; - if (order != mo_Relaxed) - mdbx_compiler_barrier(); - return value; -#endif /* MDBX_HAVE_C11ATOMICS */ -#else /* !MDBX_64BIT_ATOMIC */ - mdbx_compiler_barrier(); - uint64_t value = (uint64_t)atomic_load32(&p->high, order) << 32; - mdbx_jitter4testing(true); - value |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed - : mo_AcquireRelease); - mdbx_jitter4testing(true); - for (;;) { - mdbx_compiler_barrier(); - uint64_t again = (uint64_t)atomic_load32(&p->high, order) << 32; - mdbx_jitter4testing(true); - again |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed - : mo_AcquireRelease); - mdbx_jitter4testing(true); - if (likely(value == again)) - return value; - value = again; - } -#endif /* !MDBX_64BIT_ATOMIC */ -} - #endif /* !__cplusplus */ /*----------------------------------------------------------------------------*/ @@ -1426,15 +1360,6 @@ MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void); MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void); MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr); -MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny) { -#if MDBX_DEBUG - if (MDBX_DBG_JITTER & mdbx_runtime_flags) - mdbx_osal_jitter(tiny); -#else - (void)tiny; -#endif -} - #endif /* !__cplusplus */ #define MDBX_IS_ERROR(rc) \ diff --git a/src/osal.h b/src/osal.h index 2e8359b5..3c6864ec 100644 --- a/src/osal.h +++ b/src/osal.h @@ -481,9 +481,18 @@ typedef union MDBX_srwlock { } MDBX_srwlock; #endif /* Windows */ -#ifdef __cplusplus -extern void mdbx_osal_jitter(bool tiny); +#ifndef __cplusplus + +MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny); + +MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (MDBX_DBG_JITTER & mdbx_runtime_flags) + mdbx_osal_jitter(tiny); #else + (void)tiny; +#endif +} /*----------------------------------------------------------------------------*/ /* Atomics */ @@ -735,7 +744,6 @@ MDBX_MAYBE_UNUSED static __inline uintptr_t mdbx_thread_self(void) { return (uintptr_t)thunk; } -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny); MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void); MDBX_INTERNAL_FUNC uint64_t mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16);