From 2d2a3ebcfb81c29e62eb18d28dac4bb332306017 Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Fri, 4 Dec 2015 23:16:25 +0300 Subject: [PATCH 01/10] mdbx: add missing h-deps into Makefile. Change-Id: Ia3af98e4194b7bfb43973069a634c4c0fb011d5c --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f9a11225..8577a80b 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ IPROGS := mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_chk IDOCS := mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1 PROGS := $(IPROGS) mtest0 mtest1 mtest2 mtest3 mtest4 mtest5 mtest6 wbench -SRC_LMDB := mdb.c midl.c lmdb.h midl.h +SRC_LMDB := mdb.c midl.c lmdb.h midl.h reopen.h barriers.h SRC_MDBX := $(SRC_LMDB) mdbx.h .PHONY: mdbx lmdb all install clean check tests coverage From dec88f8c358423f308c15fc018e5a5b31fc9ad72 Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Thu, 10 Dec 2015 00:22:01 +0300 Subject: [PATCH 02/10] mdbx: fix double-free on thread-destroy. Seems this is fixes the https://github.com/ReOpen/ReOpenLDAP/issues/60 Change-Id: I29830a1b9cec0285edc976e6892678cd38e4445c --- mdb.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mdb.c b/mdb.c index 7919b88d..7a13dc88 100644 --- a/mdb.c +++ b/mdb.c @@ -4966,7 +4966,6 @@ mdb_env_close0(MDB_env *env) mdb_ensure(env, rthc->rc_reader == reader); rthc->rc_reader = NULL; reader->mr_rthc = NULL; - free(rthc); } reader->mr_pid = 0; } From 6069149b055717de861385437435256039be5aff Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Fri, 4 Dec 2015 22:38:56 +0300 Subject: [PATCH 03/10] mdbx: basic support for ThreadSanitizer (tsan). Change-Id: Ia23751ac569b64e94cd6f5e6910f77a203259fa3 --- reopen.h | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/reopen.h b/reopen.h index 4c16d067..943ca503 100644 --- a/reopen.h +++ b/reopen.h @@ -67,6 +67,14 @@ # endif #endif /* __forceinline */ +#ifndef __noinline +# if defined(__GNUC__) || defined(__clang__) +# define __noinline __attribute__((noinline)) +# elif defined(_MSC_VER) +# define __noinline __declspec(noinline) +# endif +#endif /* __noinline */ + #ifndef __must_check_result # if defined(__GNUC__) || defined(__clang__) # define __must_check_result __attribute__((warn_unused_result)) @@ -115,6 +123,8 @@ #ifndef __noreturn # if defined(__GNUC__) || defined(__clang__) # define __noreturn __attribute__((noreturn)) +# elif defined(__MSC_VER) +# define __noreturn __declspec(noreturn) # else # define __noreturn # endif @@ -123,6 +133,8 @@ #ifndef __nothrow # if defined(__GNUC__) || defined(__clang__) # define __nothrow __attribute__((nothrow)) +# elif defined(__MSC_VER) +# define __nothrow __declspec(nothrow) # else # define __nothrow # endif @@ -217,4 +229,33 @@ __extern_C void __assert_fail( # define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0) #endif /* ! USE_VALGRIND */ +#if defined(__has_feature) +# if __has_feature(thread_sanitizer) +# define __SANITIZE_THREAD__ 1 +# endif +#endif + +#ifdef __SANITIZE_THREAD__ +# define ATTRIBUTE_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread, noinline)) +#else +# define ATTRIBUTE_NO_SANITIZE_THREAD +#endif + +#if defined(__has_feature) +# if __has_feature(address_sanitizer) +# define __SANITIZE_ADDRESS__ 1 +# endif +#endif + +#ifdef __SANITIZE_ADDRESS__ +# include +# define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address, noinline)) +#else +# define ASAN_POISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +# define ATTRIBUTE_NO_SANITIZE_ADDRESS +#endif /* __SANITIZE_ADDRESS__ */ + #endif /* _REOPEN_H */ From 50c480e2de7c5ee183c01068b1d08d61c2d20267 Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Fri, 4 Dec 2015 22:06:30 +0300 Subject: [PATCH 04/10] mdbx: avoid tran-trap in mdb_txn_renew0() by mt_flags. Avoid read txn-flags from shared write TXN (e.g. env->me_tnx0->mt_flags) without holding a write-mutex. Change-Id: I3a3a64597f69b7df205043c567a51fe509247826 --- mdb.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/mdb.c b/mdb.c index 7a13dc88..96ba2566 100644 --- a/mdb.c +++ b/mdb.c @@ -2700,11 +2700,11 @@ mdb_reader_pid(MDB_env *env, int op, pid_t pid) * @return 0 on success, non-zero on failure. */ static int -mdb_txn_renew0(MDB_txn *txn) +mdb_txn_renew0(MDB_txn *txn, unsigned flags) { MDB_env *env = txn->mt_env; MDB_meta *meta; - unsigned i, nr, flags = txn->mt_flags; + unsigned i, nr; uint16_t x; int rc, new_notls = 0; @@ -2713,9 +2713,11 @@ mdb_txn_renew0(MDB_txn *txn) return MDB_PANIC; } - if ((flags &= MDB_TXN_RDONLY) != 0) { + if (flags & MDB_TXN_RDONLY) { struct MDB_rthc *rthc = NULL; MDB_reader *r = NULL; + + txn->mt_flags = MDB_TXN_RDONLY; if (likely(env->me_flags & MDB_ENV_TXKEY)) { mdb_assert(env, !(env->me_flags & MDB_NOTLS)); rthc = pthread_getspecific(env->me_txkey); @@ -2811,9 +2813,9 @@ mdb_txn_renew0(MDB_txn *txn) return rc; meta = mdb_meta_head_w(env); - txn->mt_txnid = meta->mm_txnid; + txn->mt_txnid = meta->mm_txnid + 1; + txn->mt_flags = flags; - txn->mt_txnid++; #if MDB_DEBUG if (unlikely(txn->mt_txnid == mdb_debug_edge)) { if (! mdb_debug_logger) @@ -2842,8 +2844,6 @@ mdb_txn_renew0(MDB_txn *txn) txn->mt_next_pgno = meta->mm_last_pg+1; } - txn->mt_flags = flags; - /* Setup db info */ txn->mt_numdbs = env->me_numdbs; for (i=CORE_DBS; imt_numdbs; i++) { @@ -2880,7 +2880,7 @@ mdb_txn_renew(MDB_txn *txn) if (unlikely(!F_ISSET(txn->mt_flags, MDB_TXN_RDONLY|MDB_TXN_FINISHED))) return EINVAL; - rc = mdb_txn_renew0(txn); + rc = mdb_txn_renew0(txn, MDB_TXN_RDONLY); if (rc == MDB_SUCCESS) { mdb_debug("renew txn %zu%c %p on mdbenv %p, root page %zu", txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', @@ -2988,13 +2988,12 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned flags, MDB_txn **ret) } else { /* MDB_RDONLY */ txn->mt_dbiseqs = env->me_dbiseqs; renew: - rc = mdb_txn_renew0(txn); + rc = mdb_txn_renew0(txn, flags); } if (unlikely(rc)) { if (txn != env->me_txn0) free(txn); } else { - txn->mt_flags |= flags; /* could not change txn=me_txn0 earlier */ txn->mt_signature = MDBX_MT_SIGNATURE; *ret = txn; mdb_debug("begin txn %zu%c %p on mdbenv %p, root page %zu", @@ -9306,7 +9305,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) if (unlikely(rc)) goto leave; - rc = mdb_txn_renew0(txn); + rc = mdb_txn_renew0(txn, MDB_RDONLY); if (rc) { mdb_mutex_unlock(env, wmutex); goto leave; From 5bc0a768990e39bcaf3fc1f323367fd8ec911b67 Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Sat, 5 Dec 2015 12:19:29 +0300 Subject: [PATCH 05/10] mdbx: avoid tsan-trap in mdb_txn_renew0() by mti_txnid. Change-Id: Ia91bb913fc99880ec34fd0bf6fc9396555c6a56f --- mdb.c | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/mdb.c b/mdb.c index 96ba2566..8f908b2c 100644 --- a/mdb.c +++ b/mdb.c @@ -2695,6 +2695,25 @@ mdb_reader_pid(MDB_env *env, int op, pid_t pid) } } +static ATTRIBUTE_NO_SANITIZE_THREAD +txnid_t lead_txnid__tsan_workaround(MDB_txn *txn, MDB_reader *r) +{ + while(1) { /* LY: Retry on a race, ITS#7970. */ + MDB_meta *meta = mdb_meta_head_r(txn->mt_env); + txnid_t lead = meta->mm_txnid; + r->mr_txnid = lead; + mdb_coherent_barrier(); + /* Copy the DB info and flags */ + memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDB_db)); + txn->mt_next_pgno = meta->mm_last_pg+1; + if (likely(lead == txn->mt_env->me_txns->mti_txnid)) + return lead; +#if defined(__i386__) || defined(__x86_64__) + __asm__ __volatile__("pause"); +#endif + } +} + /** Common code for #mdb_txn_begin() and #mdb_txn_renew(). * @param[in] txn the transaction handle to initialize * @return 0 on success, non-zero on failure. @@ -2703,7 +2722,6 @@ static int mdb_txn_renew0(MDB_txn *txn, unsigned flags) { MDB_env *env = txn->mt_env; - MDB_meta *meta; unsigned i, nr; uint16_t x; int rc, new_notls = 0; @@ -2794,16 +2812,7 @@ mdb_txn_renew0(MDB_txn *txn, unsigned flags) } } - do { /* LY: Retry on a race, ITS#7970. */ - meta = mdb_meta_head_r(env); - r->mr_txnid = meta->mm_txnid; - mdb_coherent_barrier(); - /* Copy the DB info and flags */ - memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDB_db)); - txn->mt_next_pgno = meta->mm_last_pg+1; - } while(unlikely(r->mr_txnid != env->me_txns->mti_txnid)); - - txn->mt_txnid = r->mr_txnid; + txn->mt_txnid = lead_txnid__tsan_workaround(txn, r); txn->mt_u.reader = r; txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ } else { @@ -2812,7 +2821,7 @@ mdb_txn_renew0(MDB_txn *txn, unsigned flags) if (unlikely(rc)) return rc; - meta = mdb_meta_head_w(env); + MDB_meta *meta = mdb_meta_head_w(env); txn->mt_txnid = meta->mm_txnid + 1; txn->mt_flags = flags; From 4e5d5ef9354906dcc627aa35f41e83aee73930e6 Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Sun, 6 Dec 2015 13:37:45 +0300 Subject: [PATCH 06/10] mdbx: avoid tsan-trap in mdb_meta_head_r(). Change-Id: I594c13e1fe908f8afdc663d349cfc4cf634bba77 --- mdb.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/mdb.c b/mdb.c index 8f908b2c..c54f380c 100644 --- a/mdb.c +++ b/mdb.c @@ -1857,17 +1857,18 @@ static MDB_meta* mdb_meta_head_w(MDB_env *env) { return a; } -static MDB_meta* mdb_meta_head_r(MDB_env *env) { +static ATTRIBUTE_NO_SANITIZE_THREAD /* LY: avoid tsan-trap by meta->mm_txnid */ +MDB_meta* mdb_meta_head_r(MDB_env *env) { MDB_meta* a = METAPAGE_1(env); MDB_meta* b = METAPAGE_2(env), *h; txnid_t head_txnid; int loop = 0, rc; - do { + while(1) { head_txnid = env->me_txns->mti_txnid; mdb_assert(env, a->mm_txnid != b->mm_txnid || head_txnid == 0); - if (a->mm_txnid == head_txnid) + if (likely(a->mm_txnid == head_txnid)) return a; if (likely(b->mm_txnid == head_txnid)) return b; @@ -1877,9 +1878,13 @@ static MDB_meta* mdb_meta_head_r(MDB_env *env) { __asm__ __volatile__("pause"); #endif mdb_coherent_barrier(); - if (loop > 2) - pthread_yield(); - } while (++loop < 5); + loop += 1; + if(likely(loop < 3)) + continue; + if(unlikely(loop > 5)) + break; + pthread_yield(); + } rc = mdb_mutex_lock(env, MDB_MUTEX(env, w)); h = mdb_meta_head_w(env); From 75ab3716fb7cc9ffec0dd4336666bc0d2b25c60c Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Wed, 9 Dec 2015 20:07:51 +0300 Subject: [PATCH 07/10] mdbx: avoid tsan-trap in mdbx_txn_straggler(). Change-Id: Ib62cdd5592e6001a1734c321f3801722e4dcf7e4 --- mdb.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mdb.c b/mdb.c index c54f380c..d59c959e 100644 --- a/mdb.c +++ b/mdb.c @@ -3063,8 +3063,8 @@ mdb_dbis_update(MDB_txn *txn, int keep) env->me_numdbs = n; } -int -mdbx_txn_straggler(MDB_txn *txn, int *percent) +ATTRIBUTE_NO_SANITIZE_THREAD /* LY: avoid tsan-trap by me_txn, mm_last_pg and mt_next_pgno */ +int mdbx_txn_straggler(MDB_txn *txn, int *percent) { MDB_env *env; MDB_meta *meta; @@ -3082,9 +3082,11 @@ mdbx_txn_straggler(MDB_txn *txn, int *percent) env = txn->mt_env; meta = mdb_meta_head_r(env); if (percent) { - long cent = env->me_maxpg / 100; - long last = env->me_txn ? env->me_txn0->mt_next_pgno : meta->mm_last_pg; - *percent = (last + cent / 2) / (cent ? cent : 1); + size_t maxpg = env->me_maxpg; + size_t last = meta->mm_last_pg + 1; + if (env->me_txn) + last = env->me_txn0->mt_next_pgno; + *percent = (last + maxpg / 2) * 100u / maxpg; } lag = meta->mm_txnid - txn->mt_u.reader->mr_txnid; return (0 > (long) lag) ? ~0u >> 1: lag; From f38bb9621f55a6c2ffd8761caa3380a0a4999d39 Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Fri, 4 Dec 2015 22:52:44 +0300 Subject: [PATCH 08/10] mdbx: avoid tsan-trap in mdb_find_oldest(). Change-Id: Icaaf032fd3600b2ee61feb39d32af60c4e3bb0d5 --- mdb.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mdb.c b/mdb.c index d59c959e..09eabe50 100644 --- a/mdb.c +++ b/mdb.c @@ -1903,8 +1903,8 @@ static int mdb_meta_lt(MDB_meta* a, MDB_meta* b) { } /** Find oldest txnid still referenced. */ -static txnid_t -mdb_find_oldest(MDB_env *env, int *laggard) +static ATTRIBUTE_NO_SANITIZE_THREAD /* LY: avoid tran-trap by reader[].mr_txnid */ +txnid_t mdb_find_oldest(MDB_env *env, int *laggard) { int i, reader; MDB_reader *r = env->me_txns->mti_readers; @@ -1973,7 +1973,7 @@ mdb_oomkick(MDB_env *env, txnid_t oldest) break; if (rc) { - r->mr_txnid = (txnid_t)-1L; + r->mr_txnid = ~(txnid_t)0; if (rc > 1) { r->mr_tid = 0; r->mr_pid = 0; @@ -2766,7 +2766,7 @@ mdb_txn_renew0(MDB_txn *txn, unsigned flags) } if (likely(r)) { - if (unlikely(r->mr_pid != env->me_pid || r->mr_txnid != (txnid_t)-1)) + if (unlikely(r->mr_pid != env->me_pid || r->mr_txnid != ~(txnid_t)0)) return MDB_BAD_RSLOT; } else { pid_t pid = env->me_pid; @@ -2799,7 +2799,7 @@ mdb_txn_renew0(MDB_txn *txn, unsigned flags) * When it will be closed, we can finally claim it. */ r->mr_pid = 0; - r->mr_txnid = (txnid_t)-1; + r->mr_txnid = ~(txnid_t)0; r->mr_tid = tid; mdb_coherent_barrier(); if (i == nr) @@ -3118,7 +3118,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode) if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { if (txn->mt_u.reader) { - txn->mt_u.reader->mr_txnid = (txnid_t)-1; + txn->mt_u.reader->mr_txnid = ~(txnid_t)0; if (!(env->me_flags & MDB_NOTLS)) { txn->mt_u.reader = NULL; /* txn does not own reader */ } else if (mode & MDB_END_SLOT) { @@ -10075,7 +10075,7 @@ mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) for (i=0; i Date: Mon, 7 Dec 2015 19:37:00 +0300 Subject: [PATCH 09/10] mdbx: avoid tsan-trap in mdb_txn_renew0() by me_live_reader. Change-Id: I70f173d3e77c9cc84881c86e8fc64ae2a4906f0b --- mdb.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/mdb.c b/mdb.c index 09eabe50..1cea45f1 100644 --- a/mdb.c +++ b/mdb.c @@ -2773,16 +2773,19 @@ mdb_txn_renew0(MDB_txn *txn, unsigned flags) pthread_t tid = pthread_self(); pthread_mutex_t *rmutex = MDB_MUTEX(env, r); - if (unlikely(!env->me_live_reader)) { - rc = mdb_reader_pid(env, F_SETLK, pid); - if (unlikely(rc != MDB_SUCCESS)) - return rc; - env->me_live_reader = 1; - } - rc = mdb_mutex_lock(env, rmutex); if (unlikely(rc != MDB_SUCCESS)) return rc; + + if (unlikely(!env->me_live_reader)) { + rc = mdb_reader_pid(env, F_SETLK, pid); + if (unlikely(rc != MDB_SUCCESS)) { + mdb_mutex_unlock(env, rmutex); + return rc; + } + env->me_live_reader = 1; + } + nr = env->me_txns->mti_numreaders; for (i=0; ime_txns->mti_readers[i].mr_pid == 0) From 886fa96a5d54a62c07f9d9ae601ce496a3baa24c Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Tue, 8 Dec 2015 12:59:46 +0300 Subject: [PATCH 10/10] mdbx: tsan 'edge-mutex'. Change-Id: If6fe58d9e1e334fbdaec529a14a78f5ebc6d8103 --- mdb.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/mdb.c b/mdb.c index 1cea45f1..69d4c5d1 100644 --- a/mdb.c +++ b/mdb.c @@ -1092,6 +1092,10 @@ static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead); static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int_ai, mdb_cmp_int_a2, mdb_cmp_int_ua; /** @endcond */ +#ifdef __SANITIZE_THREAD__ +static pthread_mutex_t tsan_mutex = PTHREAD_MUTEX_INITIALIZER; +#endif + /** Return the library version info. */ char * __cold mdb_version(int *major, int *minor, int *patch) @@ -1865,6 +1869,10 @@ MDB_meta* mdb_meta_head_r(MDB_env *env) { int loop = 0, rc; while(1) { +#ifdef __SANITIZE_THREAD__ + pthread_mutex_lock(&tsan_mutex); + pthread_mutex_unlock(&tsan_mutex); +#endif head_txnid = env->me_txns->mti_txnid; mdb_assert(env, a->mm_txnid != b->mm_txnid || head_txnid == 0); @@ -1903,7 +1911,7 @@ static int mdb_meta_lt(MDB_meta* a, MDB_meta* b) { } /** Find oldest txnid still referenced. */ -static ATTRIBUTE_NO_SANITIZE_THREAD /* LY: avoid tran-trap by reader[].mr_txnid */ +static ATTRIBUTE_NO_SANITIZE_THREAD /* LY: avoid tsan-trap by reader[].mr_txnid */ txnid_t mdb_find_oldest(MDB_env *env, int *laggard) { int i, reader; @@ -2711,8 +2719,13 @@ txnid_t lead_txnid__tsan_workaround(MDB_txn *txn, MDB_reader *r) /* Copy the DB info and flags */ memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDB_db)); txn->mt_next_pgno = meta->mm_last_pg+1; - if (likely(lead == txn->mt_env->me_txns->mti_txnid)) + if (likely(lead == txn->mt_env->me_txns->mti_txnid)) { +#ifdef __SANITIZE_THREAD__ + pthread_mutex_lock(&tsan_mutex); + pthread_mutex_unlock(&tsan_mutex); +#endif return lead; + } #if defined(__i386__) || defined(__x86_64__) __asm__ __volatile__("pause"); #endif @@ -2829,6 +2842,10 @@ mdb_txn_renew0(MDB_txn *txn, unsigned flags) if (unlikely(rc)) return rc; +#ifdef __SANITIZE_THREAD__ + pthread_mutex_lock(&tsan_mutex); + pthread_mutex_unlock(&tsan_mutex); +#endif MDB_meta *meta = mdb_meta_head_w(env); txn->mt_txnid = meta->mm_txnid + 1; txn->mt_flags = flags;