mirror of
https://github.com/isar/libmdbx.git
synced 2024-10-29 23:19:20 +08:00
lmdb: ITS#7969 properly compiler/memory/coherent barriers.
Change-Id: I875d0908a2c828626df544fed48b3144e95e3982
This commit is contained in:
Leo Yuriev
parent
e7a0c74c09
commit
1f58fb66ae
170
mdb.c
170
mdb.c
@ -70,15 +70,152 @@
|
||||
#include <fcntl.h>
|
||||
#endif
|
||||
|
||||
/*****************************************************************************
|
||||
* Properly compiler/memory/coherence barriers
|
||||
* in the most portable way for ReOpenLDAP project.
|
||||
*
|
||||
* Feedback and comments are welcome.
|
||||
* https://gist.github.com/leo-yuriev/ba186a6bf5cf3a27bae7 */
|
||||
|
||||
#if defined(__mips) && defined(__linux)
|
||||
/* MIPS has cache coherency issues, requires explicit cache control */
|
||||
#include <asm/cachectl.h>
|
||||
extern int cacheflush(char *addr, int nbytes, int cache);
|
||||
#define CACHEFLUSH(addr, bytes, cache) cacheflush(addr, bytes, cache)
|
||||
#else
|
||||
#define CACHEFLUSH(addr, bytes, cache)
|
||||
/* Only MIPS has explicit cache control */
|
||||
# include <asm/cachectl.h>
|
||||
#endif
|
||||
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
/* LY: noting needed */
|
||||
#elif defined(_MSC_VER) || defined(_WIN32) || defined(_WIN64)
|
||||
# include <winnt.h>
|
||||
# include <intrin.h>
|
||||
# pragma intrinsic(_ReadWriteBarrier)
|
||||
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
|
||||
# include <intrin.h>
|
||||
# if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
|
||||
# pragma intrinsic(__mf)
|
||||
# elif defined(__i386__) || defined(__x86_64__)
|
||||
# pragma intrinsic(_mm_mfence)
|
||||
# endif
|
||||
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
|
||||
# include <mbarrier.h>
|
||||
# define __inline inline
|
||||
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) \
|
||||
&& (defined(HP_IA64) || defined(__ia64))
|
||||
# include <machine/sys/inline.h>
|
||||
# define __inline
|
||||
#elif defined(__IBMC__) && defined(__powerpc)
|
||||
# include <atomic.h>
|
||||
# define __inline
|
||||
#elif defined(_AIX)
|
||||
# include <builtins.h>
|
||||
# include <sys/atomic_op.h>
|
||||
# define __inline
|
||||
#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
|
||||
# include <machine/builtins.h>
|
||||
# include <c_asm.h>
|
||||
# define __inline
|
||||
#elif defined(__MWERKS__)
|
||||
/* CodeWarrior - troubles ? */
|
||||
# pragma gcc_extensions
|
||||
# define __inline
|
||||
#elif defined(__SNC__)
|
||||
/* Sony PS3 - troubles ? */
|
||||
# define __inline
|
||||
#else
|
||||
# define __inline
|
||||
#endif
|
||||
|
||||
#if defined(__i386__) || defined(__x86_64__) \
|
||||
|| defined(_M_AMD64) || defined(_M_IX86) \
|
||||
|| defined(__i386) || defined(__amd64) \
|
||||
|| defined(i386) || defined(__x86_64) \
|
||||
|| defined(_AMD64_) || defined(_M_X64)
|
||||
# define MDB_CACHE_IS_COHERENT 1
|
||||
#elif defined(__hppa) || defined(__hppa__)
|
||||
# define MDB_CACHE_IS_COHERENT 1
|
||||
#endif
|
||||
|
||||
#ifndef MDB_CACHE_IS_COHERENT
|
||||
# define MDB_CACHE_IS_COHERENT 0
|
||||
#endif
|
||||
|
||||
#define MDB_BARRIER_COMPILER 0
|
||||
#define MDB_BARRIER_MEMORY 1
|
||||
|
||||
static __inline void mdb_barrier(int type) {
|
||||
#if defined(__clang__)
|
||||
__asm__ __volatile__ ("" ::: "memory");
|
||||
if (type > MDB_BARRIER_COMPILER)
|
||||
# if __has_extension(c_atomic) || __has_extension(cxx_atomic)
|
||||
__c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
|
||||
# else
|
||||
__sync_synchronize();
|
||||
# endif
|
||||
#elif defined(__GNUC__)
|
||||
__asm__ __volatile__ ("" ::: "memory");
|
||||
if (type > MDB_BARRIER_COMPILER)
|
||||
# if defined(__ATOMIC_SEQ_CST)
|
||||
__atomic_thread_fence(__ATOMIC_SEQ_CST);
|
||||
# else
|
||||
__sync_synchronize();
|
||||
# endif
|
||||
#elif defined(_MSC_VER) || defined(_WIN32) || defined(_WIN64)
|
||||
_ReadWriteBarrier();
|
||||
if (type > MDB_BARRIER_COMPILER)
|
||||
MemoryBarrier();
|
||||
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
|
||||
__memory_barrier();
|
||||
if (type > MDB_BARRIER_COMPILER)
|
||||
# if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
|
||||
__mf();
|
||||
# elif defined(__i386__) || defined(__x86_64__)
|
||||
_mm_mfence();
|
||||
# else
|
||||
# error "Unknown target for Intel Compiler, please report to us."
|
||||
# endif
|
||||
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
|
||||
__compiler_barrier();
|
||||
if (type > MDB_BARRIER_COMPILER)
|
||||
__machine_rw_barrier();
|
||||
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) \
|
||||
&& (defined(HP_IA64) || defined(__ia64))
|
||||
_Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
|
||||
if (type > MDB_BARRIER_COMPILER)
|
||||
_Asm_mf();
|
||||
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) \
|
||||
|| defined(__ppc64__) || defined(__powerpc64__)
|
||||
__fence();
|
||||
if (type > MDB_BARRIER_COMPILER)
|
||||
__lwsync();
|
||||
#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
|
||||
__PAL_DRAINA(); /* LY: excessive ? */
|
||||
__MB();
|
||||
#else
|
||||
# error "Could not guess the kind of compiler, please report to us."
|
||||
#endif
|
||||
}
|
||||
|
||||
#define mdb_compiler_barrier() \
|
||||
mdb_barrier(MDB_BARRIER_COMPILER)
|
||||
#define mdb_memory_barrier() \
|
||||
mdb_barrier(MDB_BARRIER_MEMORY)
|
||||
#define mdb_coherent_barrier() \
|
||||
mdb_barrier(MDB_CACHE_IS_COHERENT ? MDB_BARRIER_COMPILER : MDB_BARRIER_MEMORY)
|
||||
|
||||
static __inline void mdb_invalidate_cache(void *addr, int nbytes) {
|
||||
mdb_coherent_barrier();
|
||||
#if defined(__mips) && defined(__linux)
|
||||
/* MIPS has cache coherency issues.
|
||||
* Note: for any nbytes >= on-chip cache size, entire is flushed. */
|
||||
cacheflush(addr, nbytes, DCACHE);
|
||||
#elif defined(_M_MRX000) || defined(_MIPS_)
|
||||
# error "Sorry, cacheflush() for MIPS not implemented"
|
||||
#else
|
||||
/* LY: assume no mmap/dcache issues. */
|
||||
#endif
|
||||
}
|
||||
|
||||
/*****************************************************************************/
|
||||
|
||||
#if defined(__linux) && !defined(MDB_FDATASYNC_WORKS)
|
||||
/** fdatasync is broken on ext3/ext4fs on older kernels, see
|
||||
* description in #mdb_env_open2 comments. You can safely
|
||||
@ -2834,6 +2971,7 @@ mdb_txn_renew0(MDB_txn *txn)
|
||||
r->mr_txnid = (txnid_t)-1;
|
||||
r->mr_tid = tid;
|
||||
r->mr_pid = pid; /* should be written last, see ITS#7971. */
|
||||
mdb_coherent_barrier();
|
||||
if (i == nr)
|
||||
ti->mti_numreaders = ++nr;
|
||||
/* Save numreaders for un-mutexed mdb_env_close() */
|
||||
@ -2846,9 +2984,10 @@ mdb_txn_renew0(MDB_txn *txn)
|
||||
return rc;
|
||||
}
|
||||
}
|
||||
do /* LY: Retry on a race, ITS#7970. */
|
||||
do { /* LY: Retry on a race, ITS#7970. */
|
||||
r->mr_txnid = ti->mti_txnid;
|
||||
while(r->mr_txnid != ti->mti_txnid);
|
||||
mdb_coherent_barrier();
|
||||
} while(r->mr_txnid != ti->mti_txnid);
|
||||
txn->mt_txnid = r->mr_txnid;
|
||||
txn->mt_u.reader = r;
|
||||
meta = env->me_metas[txn->mt_txnid & 1];
|
||||
@ -3654,11 +3793,7 @@ mdb_page_flush(MDB_txn *txn, int keep)
|
||||
#endif /* _WIN32 */
|
||||
}
|
||||
|
||||
/* MIPS has cache coherency issues, this is a no-op everywhere else
|
||||
* Note: for any size >= on-chip cache size, entire on-chip cache is
|
||||
* flushed.
|
||||
*/
|
||||
CACHEFLUSH(env->me_map, txn->mt_next_pgno * env->me_psize, DCACHE);
|
||||
mdb_invalidate_cache(env->me_map, txn->mt_next_pgno * env->me_psize);
|
||||
|
||||
for (i = keep; ++i <= pagecount; ) {
|
||||
dp = dl[i].mptr;
|
||||
@ -4083,10 +4218,8 @@ mdb_env_write_meta(MDB_txn *txn, int force)
|
||||
mp->mm_dbs[0] = txn->mt_dbs[0];
|
||||
mp->mm_dbs[1] = txn->mt_dbs[1];
|
||||
mp->mm_last_pg = txn->mt_next_pgno - 1;
|
||||
#if !(defined(_MSC_VER) || defined(__i386__) || defined(__x86_64__))
|
||||
/* LY: issue a memory barrier, if not x86. ITS#7969 */
|
||||
__sync_synchronize();
|
||||
#endif
|
||||
/* (LY) ITS#7969: issue a memory barrier, it is noop for x86. */
|
||||
mdb_coherent_barrier();
|
||||
mp->mm_txnid = txn->mt_txnid;
|
||||
if (force || !(env->me_flags & (MDB_NOMETASYNC|MDB_NOSYNC))) {
|
||||
unsigned meta_size = env->me_psize;
|
||||
@ -4163,8 +4296,7 @@ fail:
|
||||
env->me_flags |= MDB_FATAL_ERROR;
|
||||
return rc;
|
||||
}
|
||||
/* MIPS has cache coherency issues, this is a no-op everywhere else */
|
||||
CACHEFLUSH(env->me_map + off, len, DCACHE);
|
||||
mdb_invalidate_cache(env->me_map + off, len);
|
||||
done:
|
||||
/* Memory ordering issues are irrelevant; since the entire writer
|
||||
* is wrapped by wmutex, all of these changes will become visible
|
||||
|
||||
Loading…
Reference in New Issue
Block a user