mirror of
https://github.com/isar/libmdbx.git
synced 2025-02-03 04:00:51 +08:00
mdbx: refine visibility of internal mdbx_osal_jitter().
This commit is contained in:
Leonid Yuriev
parent
c456625ef2
commit
0c3a5da3cb
64
src/core.c
64
src/core.c
@ -823,6 +823,70 @@ size_t __hot mdbx_e2k_strnlen_bug_workaround(const char *s, size_t maxlen) {
|
||||
/*------------------------------------------------------------------------------
|
||||
* safe read/write volatile 64-bit fields on 32-bit architectures. */
|
||||
|
||||
MDBX_MAYBE_UNUSED static __always_inline uint64_t
|
||||
atomic_store64(MDBX_atomic_uint64_t *p, const uint64_t value,
|
||||
enum MDBX_memory_order order) {
|
||||
STATIC_ASSERT(sizeof(MDBX_atomic_uint64_t) == 8);
|
||||
#if MDBX_64BIT_ATOMIC
|
||||
#ifdef MDBX_HAVE_C11ATOMICS
|
||||
assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p)));
|
||||
atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value, mo_c11_store(order));
|
||||
#else /* MDBX_HAVE_C11ATOMICS */
|
||||
if (order != mo_Relaxed)
|
||||
mdbx_compiler_barrier();
|
||||
p->weak = value;
|
||||
mdbx_memory_fence(order, true);
|
||||
#endif /* MDBX_HAVE_C11ATOMICS */
|
||||
#else /* !MDBX_64BIT_ATOMIC */
|
||||
mdbx_compiler_barrier();
|
||||
atomic_store32(&p->low, (uint32_t)value, mo_Relaxed);
|
||||
mdbx_jitter4testing(true);
|
||||
atomic_store32(&p->high, (uint32_t)(value >> 32), order);
|
||||
mdbx_jitter4testing(true);
|
||||
#endif /* !MDBX_64BIT_ATOMIC */
|
||||
return value;
|
||||
}
|
||||
|
||||
MDBX_MAYBE_UNUSED static
|
||||
#if MDBX_64BIT_ATOMIC
|
||||
__always_inline
|
||||
#endif /* MDBX_64BIT_ATOMIC */
|
||||
uint64_t
|
||||
atomic_load64(const MDBX_atomic_uint64_t *p,
|
||||
enum MDBX_memory_order order) {
|
||||
STATIC_ASSERT(sizeof(MDBX_atomic_uint64_t) == 8);
|
||||
#if MDBX_64BIT_ATOMIC
|
||||
#ifdef MDBX_HAVE_C11ATOMICS
|
||||
assert(atomic_is_lock_free(MDBX_c11a_ro(uint64_t, p)));
|
||||
return atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order));
|
||||
#else /* MDBX_HAVE_C11ATOMICS */
|
||||
mdbx_memory_fence(order, false);
|
||||
const uint64_t value = p->weak;
|
||||
if (order != mo_Relaxed)
|
||||
mdbx_compiler_barrier();
|
||||
return value;
|
||||
#endif /* MDBX_HAVE_C11ATOMICS */
|
||||
#else /* !MDBX_64BIT_ATOMIC */
|
||||
mdbx_compiler_barrier();
|
||||
uint64_t value = (uint64_t)atomic_load32(&p->high, order) << 32;
|
||||
mdbx_jitter4testing(true);
|
||||
value |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed
|
||||
: mo_AcquireRelease);
|
||||
mdbx_jitter4testing(true);
|
||||
for (;;) {
|
||||
mdbx_compiler_barrier();
|
||||
uint64_t again = (uint64_t)atomic_load32(&p->high, order) << 32;
|
||||
mdbx_jitter4testing(true);
|
||||
again |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed
|
||||
: mo_AcquireRelease);
|
||||
mdbx_jitter4testing(true);
|
||||
if (likely(value == again))
|
||||
return value;
|
||||
value = again;
|
||||
}
|
||||
#endif /* !MDBX_64BIT_ATOMIC */
|
||||
}
|
||||
|
||||
static __always_inline void atomic_yield(void) {
|
||||
#if defined(_WIN32) || defined(_WIN64)
|
||||
YieldProcessor();
|
||||
|
||||
@ -264,8 +264,6 @@ static __always_inline memory_order mo_c11_load(enum MDBX_memory_order fence) {
|
||||
|
||||
#ifndef __cplusplus
|
||||
|
||||
static __inline void mdbx_jitter4testing(bool tiny);
|
||||
|
||||
MDBX_MAYBE_UNUSED static __always_inline void
|
||||
mdbx_memory_fence(enum MDBX_memory_order order, bool write) {
|
||||
#ifdef MDBX_HAVE_C11ATOMICS
|
||||
@ -309,70 +307,6 @@ atomic_load32(const MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) {
|
||||
#endif /* MDBX_HAVE_C11ATOMICS */
|
||||
}
|
||||
|
||||
MDBX_MAYBE_UNUSED static __always_inline uint64_t
|
||||
atomic_store64(MDBX_atomic_uint64_t *p, const uint64_t value,
|
||||
enum MDBX_memory_order order) {
|
||||
STATIC_ASSERT(sizeof(MDBX_atomic_uint64_t) == 8);
|
||||
#if MDBX_64BIT_ATOMIC
|
||||
#ifdef MDBX_HAVE_C11ATOMICS
|
||||
assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p)));
|
||||
atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value, mo_c11_store(order));
|
||||
#else /* MDBX_HAVE_C11ATOMICS */
|
||||
if (order != mo_Relaxed)
|
||||
mdbx_compiler_barrier();
|
||||
p->weak = value;
|
||||
mdbx_memory_fence(order, true);
|
||||
#endif /* MDBX_HAVE_C11ATOMICS */
|
||||
#else /* !MDBX_64BIT_ATOMIC */
|
||||
mdbx_compiler_barrier();
|
||||
atomic_store32(&p->low, (uint32_t)value, mo_Relaxed);
|
||||
mdbx_jitter4testing(true);
|
||||
atomic_store32(&p->high, (uint32_t)(value >> 32), order);
|
||||
mdbx_jitter4testing(true);
|
||||
#endif /* !MDBX_64BIT_ATOMIC */
|
||||
return value;
|
||||
}
|
||||
|
||||
MDBX_MAYBE_UNUSED static
|
||||
#if MDBX_64BIT_ATOMIC
|
||||
__always_inline
|
||||
#endif /* MDBX_64BIT_ATOMIC */
|
||||
uint64_t
|
||||
atomic_load64(const MDBX_atomic_uint64_t *p,
|
||||
enum MDBX_memory_order order) {
|
||||
STATIC_ASSERT(sizeof(MDBX_atomic_uint64_t) == 8);
|
||||
#if MDBX_64BIT_ATOMIC
|
||||
#ifdef MDBX_HAVE_C11ATOMICS
|
||||
assert(atomic_is_lock_free(MDBX_c11a_ro(uint64_t, p)));
|
||||
return atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order));
|
||||
#else /* MDBX_HAVE_C11ATOMICS */
|
||||
mdbx_memory_fence(order, false);
|
||||
const uint64_t value = p->weak;
|
||||
if (order != mo_Relaxed)
|
||||
mdbx_compiler_barrier();
|
||||
return value;
|
||||
#endif /* MDBX_HAVE_C11ATOMICS */
|
||||
#else /* !MDBX_64BIT_ATOMIC */
|
||||
mdbx_compiler_barrier();
|
||||
uint64_t value = (uint64_t)atomic_load32(&p->high, order) << 32;
|
||||
mdbx_jitter4testing(true);
|
||||
value |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed
|
||||
: mo_AcquireRelease);
|
||||
mdbx_jitter4testing(true);
|
||||
for (;;) {
|
||||
mdbx_compiler_barrier();
|
||||
uint64_t again = (uint64_t)atomic_load32(&p->high, order) << 32;
|
||||
mdbx_jitter4testing(true);
|
||||
again |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed
|
||||
: mo_AcquireRelease);
|
||||
mdbx_jitter4testing(true);
|
||||
if (likely(value == again))
|
||||
return value;
|
||||
value = again;
|
||||
}
|
||||
#endif /* !MDBX_64BIT_ATOMIC */
|
||||
}
|
||||
|
||||
#endif /* !__cplusplus */
|
||||
|
||||
/*----------------------------------------------------------------------------*/
|
||||
@ -1426,15 +1360,6 @@ MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void);
|
||||
MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void);
|
||||
MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr);
|
||||
|
||||
MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny) {
|
||||
#if MDBX_DEBUG
|
||||
if (MDBX_DBG_JITTER & mdbx_runtime_flags)
|
||||
mdbx_osal_jitter(tiny);
|
||||
#else
|
||||
(void)tiny;
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif /* !__cplusplus */
|
||||
|
||||
#define MDBX_IS_ERROR(rc) \
|
||||
|
||||
14
src/osal.h
14
src/osal.h
@ -481,9 +481,18 @@ typedef union MDBX_srwlock {
|
||||
} MDBX_srwlock;
|
||||
#endif /* Windows */
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern void mdbx_osal_jitter(bool tiny);
|
||||
#ifndef __cplusplus
|
||||
|
||||
MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny);
|
||||
|
||||
MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny) {
|
||||
#if MDBX_DEBUG
|
||||
if (MDBX_DBG_JITTER & mdbx_runtime_flags)
|
||||
mdbx_osal_jitter(tiny);
|
||||
#else
|
||||
(void)tiny;
|
||||
#endif
|
||||
}
|
||||
|
||||
/*----------------------------------------------------------------------------*/
|
||||
/* Atomics */
|
||||
@ -735,7 +744,6 @@ MDBX_MAYBE_UNUSED static __inline uintptr_t mdbx_thread_self(void) {
|
||||
return (uintptr_t)thunk;
|
||||
}
|
||||
|
||||
MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void mdbx_osal_jitter(bool tiny);
|
||||
MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void);
|
||||
MDBX_INTERNAL_FUNC uint64_t
|
||||
mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user