From b9835389f448147ca45dd144c0e5a9067b2a287a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9B=D0=B5=D0=BE=D0=BD=D0=B8=D0=B4=20=D0=AE=D1=80=D1=8C?= =?UTF-8?q?=D0=B5=D0=B2=20=28Leonid=20Yuriev=29?= Date: Sat, 2 Jul 2022 18:35:13 +0300 Subject: [PATCH] mdbx: add cache for pointers to last/steady meta-pages (off by default). --- src/core.c | 57 +++++++++++++++++++++++++++++++++++++++++++------ src/internals.h | 4 ++++ src/options.h | 6 ++++++ 3 files changed, 61 insertions(+), 6 deletions(-) diff --git a/src/core.c b/src/core.c index 1af2703d..580106c6 100644 --- a/src/core.c +++ b/src/core.c @@ -5720,6 +5720,15 @@ constmeta_txnid(const MDBX_env *env, const MDBX_meta *meta) { return (a == b) ? a : 0; } +static __inline void meta_cache_clear(MDBX_env *env) { +#if MDBX_CACHE_METAS + env->cache_last_meta = nullptr; + env->cache_steady_meta = nullptr; +#else + (void)env; +#endif /* MDBX_CACHE_METAS */ +} + static __inline txnid_t meta_txnid(const MDBX_env *env, volatile const MDBX_meta *meta) { (void)env; @@ -5856,28 +5865,52 @@ meta_mostrecent(const enum meta_choise_mode mode, const MDBX_env *env) { } static volatile const MDBX_meta *meta_prefer_steady(const MDBX_env *env) { - return meta_mostrecent(prefer_steady, env); + return +#if MDBX_CACHE_METAS + ((MDBX_env *)env)->cache_steady_meta = +#endif /* MDBX_CACHE_METAS */ + meta_mostrecent(prefer_steady, env); } MDBX_NOTHROW_PURE_FUNCTION static const MDBX_meta * constmeta_prefer_steady(const MDBX_env *env) { - return (const MDBX_meta *)meta_mostrecent(prefer_steady, env); +#if MDBX_CACHE_METAS + mdbx_assert(env, !env->cache_steady_meta || + env->cache_steady_meta == + meta_mostrecent(prefer_steady, env)); + return (const MDBX_meta *)(env->cache_steady_meta ? env->cache_steady_meta : +#else + return (const MDBX_meta *)( +#endif /* MDBX_CACHE_METAS */ + meta_prefer_steady(env)); } static volatile const MDBX_meta *meta_prefer_last(const MDBX_env *env) { - return meta_mostrecent(prefer_last, env); + return +#if MDBX_CACHE_METAS + ((MDBX_env *)env)->cache_last_meta = +#endif /* MDBX_CACHE_METAS */ + meta_mostrecent(prefer_last, env); } MDBX_NOTHROW_PURE_FUNCTION static const MDBX_meta * constmeta_prefer_last(const MDBX_env *env) { - return (const MDBX_meta *)meta_mostrecent(prefer_last, env); +#if MDBX_CACHE_METAS + mdbx_assert(env, + !env->cache_last_meta || + env->cache_last_meta == meta_mostrecent(prefer_last, env)); + return (const MDBX_meta *)(env->cache_last_meta ? env->cache_last_meta : +#else + return (const MDBX_meta *)( +#endif /* MDBX_CACHE_METAS */ + meta_prefer_last(env)); } static txnid_t mdbx_recent_committed_txnid(const MDBX_env *env) { while (true) { volatile const MDBX_meta *head = meta_prefer_last(env); const txnid_t recent = meta_txnid(env, head); - mdbx_compiler_barrier(); + mdbx_memory_barrier(); if (likely(head == meta_prefer_last(env) && recent == meta_txnid(env, head))) return recent; @@ -6328,6 +6361,7 @@ __cold static int mdbx_mapresize(MDBX_env *env, const pgno_t used_pgno, } #endif /* MDBX_ENABLE_MADVISE */ + meta_cache_clear(env); rc = mdbx_mresize(mresize_flags, &env->me_dxb_mmap, size_bytes, limit_bytes); #if MDBX_ENABLE_MADVISE @@ -6480,6 +6514,7 @@ __cold static int mdbx_wipe_steady(MDBX_env *env, const txnid_t last_steady) { /* force oldest refresh */ atomic_store32(&env->me_lck->mti_readers_refresh_flag, true, mo_Relaxed); + meta_cache_clear(env); return MDBX_SUCCESS; } @@ -7409,6 +7444,7 @@ retry:; #if MDBX_ENABLE_PGOP_STAT env->me_lck->mti_pgop_stat.wops.weak += wops; #endif /* MDBX_ENABLE_PGOP_STAT */ + meta_cache_clear(env); goto retry; } env->me_txn0->mt_txnid = head_txnid; @@ -7653,6 +7689,7 @@ static void mdbx_txn_valgrind(MDBX_env *env, MDBX_txn *txn) { /* no write-txn */ last = NUM_METAS; should_unlock = true; + meta_cache_clear(env); } else { /* write txn is running, therefore shouldn't poison any memory range */ return; @@ -7994,6 +8031,7 @@ static int mdbx_txn_renew0(MDBX_txn *txn, const unsigned flags) { if (likely(/* not recovery mode */ env->me_stuck_meta < 0)) { uint64_t timestamp = 0; while (1) { + meta_cache_clear(env); volatile const MDBX_meta *const meta = meta_prefer_last(env); mdbx_jitter4testing(false); const txnid_t snap = meta_txnid(env, meta); @@ -8120,6 +8158,7 @@ static int mdbx_txn_renew0(MDBX_txn *txn, const unsigned flags) { } #endif /* Windows */ + meta_cache_clear(env); mdbx_jitter4testing(false); const MDBX_meta *meta = constmeta_prefer_last(env); uint64_t timestamp = 0; @@ -11644,6 +11683,7 @@ static int mdbx_sync_locked(MDBX_env *env, unsigned flags, } } + meta_cache_clear(env); uint64_t timestamp = 0; while ("workaround for todo4recovery://erased_by_github/libmdbx/issues/269") { rc = meta_waittxnid(env, target, ×tamp); @@ -11903,6 +11943,7 @@ mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t size_now, if (unlikely(err != MDBX_SUCCESS)) return err; need_unlock = true; + meta_cache_clear(env); } const MDBX_meta *head = constmeta_prefer_last(env); if (!inside_txn) { @@ -13127,6 +13168,7 @@ __cold static int __must_check_result mdbx_override_meta( } mdbx_flush_incoherent_mmap(env->me_map, pgno2bytes(env, NUM_METAS), env->me_os_psize); + meta_cache_clear(env); return rc; } @@ -13600,7 +13642,7 @@ __cold int mdbx_env_open(MDBX_env *env, const char *pathname, #if MDBX_DEBUG if (rc == MDBX_SUCCESS) { - const MDBX_meta *meta = constmeta_prefer_last(env); + const MDBX_meta *meta = (const MDBX_meta *)meta_prefer_last(env); const MDBX_db *db = &meta->mm_dbs[MAIN_DBI]; mdbx_debug("opened database version %u, pagesize %u", @@ -20136,6 +20178,7 @@ __cold int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags, if (unlikely(rc)) return rc; should_unlock = true; + meta_cache_clear(env); } if (onoff) @@ -22968,6 +23011,7 @@ __cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, if (unlikely(err != MDBX_SUCCESS)) return err; should_unlock = true; + meta_cache_clear(env); } env->me_options.dp_reserve_limit = (unsigned)value; while (env->me_dp_reserve_len > env->me_options.dp_reserve_limit) { @@ -23004,6 +23048,7 @@ __cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, if (unlikely(err != MDBX_SUCCESS)) return err; should_unlock = true; + meta_cache_clear(env); } if (env->me_txn) err = MDBX_EPERM /* unable change during transaction */; diff --git a/src/internals.h b/src/internals.h index 309d3113..cda28c2b 100644 --- a/src/internals.h +++ b/src/internals.h @@ -1169,6 +1169,10 @@ struct MDBX_env { MDBX_txn *me_txn; /* current write transaction */ mdbx_fastmutex_t me_dbi_lock; +#if MDBX_CACHE_METAS + volatile const MDBX_meta *cache_last_meta; + volatile const MDBX_meta *cache_steady_meta; +#endif /* MDBX_CACHE_METAS */ MDBX_dbi me_numdbs; /* number of DBs opened */ MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */ diff --git a/src/options.h b/src/options.h index bf4b71f3..283eec9e 100644 --- a/src/options.h +++ b/src/options.h @@ -92,6 +92,12 @@ #error MDBX_ENABLE_BIGFOOT must be defined as 0 or 1 #endif /* MDBX_ENABLE_BIGFOOT */ +#ifndef MDBX_CACHE_METAS +#define MDBX_CACHE_METAS 0 +#elif !(MDBX_CACHE_METAS == 0 || MDBX_CACHE_METAS == 1) +#error MDBX_CACHE_METAS must be defined as 0 or 1 +#endif /* MDBX_CACHE_METAS */ + /** Controls use of POSIX madvise() hints and friends. */ #ifndef MDBX_ENABLE_MADVISE #define MDBX_ENABLE_MADVISE 1