From 61a2c567842465190812e20f04a0fae119149c64 Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Thu, 22 Mar 2018 20:34:09 +0300 Subject: [PATCH] mdbx: fix/rework cache-line alignment. --- src/bits.h | 59 +++++++++++++++++++++++++++++++----------------------- src/defs.h | 10 --------- src/mdbx.c | 9 ++++++--- src/osal.h | 12 ++++++----- 4 files changed, 47 insertions(+), 43 deletions(-) diff --git a/src/bits.h b/src/bits.h index 9d6f2f29..001289f3 100644 --- a/src/bits.h +++ b/src/bits.h @@ -55,6 +55,7 @@ #pragma warning(disable : 4310) /* cast truncates constant value */ #pragma warning(disable : 4820) /* bytes padding added after data member for aligment */ #pragma warning(disable : 4548) /* expression before comma has no effect; expected expression with side - effect */ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be unaligned */ #endif /* _MSC_VER (warnings) */ #include "../mdbx.h" @@ -252,7 +253,7 @@ typedef struct MDBX_reader { uint8_t pad[MDBX_CACHELINE_SIZE - (sizeof(txnid_t) + sizeof(mdbx_pid_t) + sizeof(mdbx_tid_t)) % MDBX_CACHELINE_SIZE]; -} __cache_aligned MDBX_reader; +} MDBX_reader; /* Information about a single database in the environment. */ typedef struct MDBX_db { @@ -410,42 +411,50 @@ typedef struct MDBX_lockinfo { /* Flags which environment was opened. */ volatile uint32_t mti_envmode; - union { #ifdef MDBX_OSAL_LOCK + /* Mutex protecting write access to this table. */ + union { MDBX_OSAL_LOCK mti_wmutex; + uint8_t pad_mti_wmutex[MDBX_OSAL_LOCK_SIZE % sizeof(size_t)]; + }; #endif - uint64_t align_wmutex; - }; +#define MDBX_lockinfo_SIZE_A \ + (8 /* mti_magic_and_version */ + 4 /* mti_os_and_format */ + \ + 4 /* mti_envmode */ + MDBX_OSAL_LOCK_SIZE /* mti_wmutex */ + \ + MDBX_OSAL_LOCK_SIZE % sizeof(size_t) /* pad_mti_wmutex */) - union { - /* The number of slots that have been used in the reader table. - * This always records the maximum count, it is not decremented - * when readers release their slots. */ - volatile unsigned __cache_aligned mti_numreaders; - uint64_t align_numreaders; - }; + /* cache-line alignment */ + uint8_t + pad_a[MDBX_CACHELINE_SIZE - MDBX_lockinfo_SIZE_A % MDBX_CACHELINE_SIZE]; + + /* The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. */ + volatile unsigned mti_numreaders; - union { #ifdef MDBX_OSAL_LOCK - /* Mutex protecting access to this table. */ + /* Mutex protecting readers registration access to this table. */ + union { MDBX_OSAL_LOCK mti_rmutex; + uint8_t pad_mti_rmutex[MDBX_OSAL_LOCK_SIZE % sizeof(size_t)]; + }; #endif - uint64_t align_rmutex; - }; - union { - volatile txnid_t mti_oldest; - uint64_t align_oldest; - }; + volatile txnid_t mti_oldest; + volatile uint32_t mti_readers_refresh_flag; - union { - volatile uint32_t mti_readers_refresh_flag; - uint64_t align_reader_finished_flag; - }; +#define MDBX_lockinfo_SIZE_B \ + (sizeof(unsigned) /* mti_numreaders */ + \ + MDBX_OSAL_LOCK_SIZE /* mti_rmutex */ + sizeof(txnid_t) /* mti_oldest */ + \ + sizeof(uint32_t) /* mti_readers_refresh_flag */ + \ + MDBX_OSAL_LOCK_SIZE % sizeof(size_t) /* pad_mti_rmutex */) - uint8_t pad_align[MDBX_CACHELINE_SIZE - sizeof(uint64_t) * 7]; + /* cache-line alignment */ + uint8_t + pad_b[MDBX_CACHELINE_SIZE - MDBX_lockinfo_SIZE_B % MDBX_CACHELINE_SIZE]; + + MDBX_reader mti_readers[1]; - MDBX_reader __cache_aligned mti_readers[1]; } MDBX_lockinfo; #pragma pack(pop) diff --git a/src/defs.h b/src/defs.h index 7c3bc1d5..6da5a963 100644 --- a/src/defs.h +++ b/src/defs.h @@ -192,16 +192,6 @@ # endif #endif /* __prefetch */ -#ifndef __aligned -# if defined(__GNUC__) || __has_attribute(aligned) -# define __aligned(N) __attribute__((aligned(N))) -# elif defined(_MSC_VER) -# define __aligned(N) __declspec(align(N)) -# else -# define __aligned(N) -# endif -#endif /* __aligned */ - #ifndef __noreturn # if defined(__GNUC__) || __has_attribute(noreturn) # define __noreturn __attribute__((noreturn)) diff --git a/src/mdbx.c b/src/mdbx.c index 91fccbad..c61f41d7 100644 --- a/src/mdbx.c +++ b/src/mdbx.c @@ -2618,6 +2618,12 @@ static int mdbx_txn_renew0(MDBX_txn *txn, unsigned flags) { return MDBX_PANIC; } + STATIC_ASSERT(sizeof(MDBX_reader) == MDBX_CACHELINE_SIZE); + STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_numreaders) % MDBX_CACHELINE_SIZE == + 0); + STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_readers) % MDBX_CACHELINE_SIZE == + 0); + pgno_t upper_pgno = 0; if (flags & MDBX_TXN_RDONLY) { txn->mt_flags = MDBX_TXN_RDONLY; @@ -2673,9 +2679,6 @@ static int mdbx_txn_renew0(MDBX_txn *txn, unsigned flags) { } } - STATIC_ASSERT(sizeof(MDBX_reader) == MDBX_CACHELINE_SIZE); - STATIC_ASSERT( - offsetof(MDBX_lockinfo, mti_readers) % MDBX_CACHELINE_SIZE == 0); r = &env->me_lck->mti_readers[slot]; /* Claim the reader slot, carefully since other code * uses the reader table un-mutexed: First reset the diff --git a/src/osal.h b/src/osal.h index 5706b77a..1f20b10f 100644 --- a/src/osal.h +++ b/src/osal.h @@ -353,10 +353,6 @@ static __inline void mdbx_memory_barrier(void) { #endif #endif /* MDBX_CACHELINE_SIZE */ -#ifndef __cache_aligned -#define __cache_aligned __aligned(MDBX_CACHELINE_SIZE) -#endif - #if MDBX_CACHE_IS_COHERENT #define mdbx_coherent_barrier() mdbx_compiler_barrier() #else @@ -553,7 +549,13 @@ void mdbx_osal_jitter(bool tiny); #else #define MDBX_OSAL_LOCK pthread_mutex_t #define MDBX_OSAL_LOCK_SIGN UINT32_C(0x8017) -#endif +#endif /* MDBX_OSAL_LOCK */ + +#ifdef MDBX_OSAL_LOCK +#define MDBX_OSAL_LOCK_SIZE sizeof(MDBX_OSAL_LOCK) +#else +#define MDBX_OSAL_LOCK_SIZE 0 +#endif /* MDBX_OSAL_LOCK_SIZE */ int mdbx_lck_init(MDBX_env *env);