From 314b8ce1f092251c245444311d9207c1284a143b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9B=D0=B5=D0=BE=D0=BD=D0=B8=D0=B4=20=D0=AE=D1=80=D1=8C?= =?UTF-8?q?=D0=B5=D0=B2=20=28Leonid=20Yuriev=29?= Date: Tue, 14 Jan 2025 22:40:56 +0300 Subject: [PATCH] =?UTF-8?q?mdbx:=20=D0=BF=D0=B5=D1=80=D0=B5=D0=B8=D0=BC?= =?UTF-8?q?=D0=B5=D0=BD=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D0=B5=20(=D0=BA?= =?UTF-8?q?=D0=BE=D1=81=D0=BC=D0=B5=D1=82=D0=B8=D0=BA=D0=B0).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api-env.c | 12 +-- src/api-txn.c | 22 ++-- src/audit.c | 10 +- src/cursor.c | 6 +- src/dpl.c | 68 ++++++------ src/dpl.h | 18 ++-- src/dxb.c | 6 +- src/env.c | 6 +- src/gc-get.c | 152 +++++++++++++------------- src/gc-put.c | 264 ++++++++++++++++++++++----------------------- src/internals.h | 10 +- src/meta.c | 4 +- src/mvcc-readers.c | 4 +- src/page-get.c | 4 +- src/page-ops.c | 82 +++++++------- src/page-ops.h | 14 +-- src/refund.c | 54 +++++----- src/spill.c | 114 ++++++++++---------- src/spill.h | 10 +- src/tree-ops.c | 4 +- src/txn-basal.c | 60 +++++------ src/txn-nested.c | 198 +++++++++++++++++----------------- src/txn-ro.c | 28 ++--- src/txn.c | 51 +++++---- 24 files changed, 600 insertions(+), 601 deletions(-) diff --git a/src/api-env.c b/src/api-env.c index cc747b80..94d3e1a3 100644 --- a/src/api-env.c +++ b/src/api-env.c @@ -683,7 +683,7 @@ static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo #endif } - *troika = (txn && !(txn->flags & MDBX_TXN_RDONLY)) ? txn->tw.troika : meta_tap(env); + *troika = (txn && !(txn->flags & MDBX_TXN_RDONLY)) ? txn->wr.troika : meta_tap(env); const meta_ptr_t head = meta_recent(env, troika); const meta_t *const meta0 = METAPAGE(env, 0); const meta_t *const meta1 = METAPAGE(env, 1); @@ -947,9 +947,9 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t si if (unlikely(err != MDBX_SUCCESS)) return LOG_IFERR(err); should_unlock = true; - env->basal_txn->tw.troika = meta_tap(env); + env->basal_txn->wr.troika = meta_tap(env); eASSERT(env, !env->txn && !env->basal_txn->nested); - env->basal_txn->txnid = env->basal_txn->tw.troika.txnid[env->basal_txn->tw.troika.recent]; + env->basal_txn->txnid = env->basal_txn->wr.troika.txnid[env->basal_txn->wr.troika.recent]; txn_snapshot_oldest(env->basal_txn); } @@ -957,7 +957,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t si if (pagesize <= 0 || pagesize >= INT_MAX) pagesize = env->ps; const geo_t *const geo = - inside_txn ? &env->txn->geo : &meta_recent(env, &env->basal_txn->tw.troika).ptr_c->geometry; + inside_txn ? &env->txn->geo : &meta_recent(env, &env->basal_txn->wr.troika).ptr_c->geometry; if (size_lower < 0) size_lower = pgno2bytes(env, geo->lower); if (size_now < 0) @@ -1173,7 +1173,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t si memset(&meta, 0, sizeof(meta)); if (!inside_txn) { eASSERT(env, should_unlock); - const meta_ptr_t head = meta_recent(env, &env->basal_txn->tw.troika); + const meta_ptr_t head = meta_recent(env, &env->basal_txn->wr.troika); uint64_t timestamp = 0; while ("workaround for " @@ -1267,7 +1267,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t si env->txn->flags |= MDBX_TXN_DIRTY; } else { meta.geometry = new_geo; - rc = dxb_sync_locked(env, env->flags, &meta, &env->basal_txn->tw.troika); + rc = dxb_sync_locked(env, env->flags, &meta, &env->basal_txn->wr.troika); if (likely(rc == MDBX_SUCCESS)) { env->geo_in_bytes.now = pgno2bytes(env, new_geo.now = meta.geometry.now); env->geo_in_bytes.upper = pgno2bytes(env, new_geo.upper = meta.geometry.upper); diff --git a/src/api-txn.c b/src/api-txn.c index 6a5319e1..c303c7c9 100644 --- a/src/api-txn.c +++ b/src/api-txn.c @@ -56,8 +56,8 @@ MDBX_txn_flags_t mdbx_txn_flags(const MDBX_txn *txn) { assert(0 == (int)(txn->flags & MDBX_TXN_INVALID)); MDBX_txn_flags_t flags = txn->flags; - if (F_ISSET(flags, MDBX_TXN_PARKED | MDBX_TXN_RDONLY) && txn->to.reader && - safe64_read(&txn->to.reader->tid) == MDBX_TID_TXN_OUSTED) + if (F_ISSET(flags, MDBX_TXN_PARKED | MDBX_TXN_RDONLY) && txn->ro.slot && + safe64_read(&txn->ro.slot->tid) == MDBX_TID_TXN_OUSTED) flags |= MDBX_TXN_OUSTED; return flags; } @@ -257,7 +257,7 @@ int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, M else { eASSERT(env, (txn->flags & ~(MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP | txn_shrink_allowed | txn_may_have_cursors | MDBX_NOMETASYNC | MDBX_SAFE_NOSYNC | MDBX_TXN_SPILLS)) == 0); - assert(!txn->tw.spilled.list && !txn->tw.spilled.least_removed); + assert(!txn->wr.spilled.list && !txn->wr.spilled.least_removed); } txn->signature = txn_signature; txn->userctx = context; @@ -456,10 +456,10 @@ int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) { info->txn_reader_lag = head.txnid - info->txn_id; info->txn_space_dirty = info->txn_space_retired = 0; uint64_t reader_snapshot_pages_retired = 0; - if (txn->to.reader && - ((txn->flags & MDBX_TXN_PARKED) == 0 || safe64_read(&txn->to.reader->tid) != MDBX_TID_TXN_OUSTED) && + if (txn->ro.slot && + ((txn->flags & MDBX_TXN_PARKED) == 0 || safe64_read(&txn->ro.slot->tid) != MDBX_TID_TXN_OUSTED) && head_retired > - (reader_snapshot_pages_retired = atomic_load64(&txn->to.reader->snapshot_pages_retired, mo_Relaxed))) { + (reader_snapshot_pages_retired = atomic_load64(&txn->ro.slot->snapshot_pages_retired, mo_Relaxed))) { info->txn_space_dirty = info->txn_space_retired = pgno2bytes(env, (pgno_t)(head_retired - reader_snapshot_pages_retired)); @@ -486,7 +486,7 @@ int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) { if (snap_txnid < next_reader && snap_tid >= MDBX_TID_TXN_OUSTED) { next_reader = snap_txnid; retired_next_reader = pgno2bytes( - env, (pgno_t)(snap_retired - atomic_load64(&txn->to.reader->snapshot_pages_retired, mo_Relaxed))); + env, (pgno_t)(snap_retired - atomic_load64(&txn->ro.slot->snapshot_pages_retired, mo_Relaxed))); } } } @@ -497,11 +497,11 @@ int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) { info->txn_space_limit_soft = pgno2bytes(env, txn->geo.now); info->txn_space_limit_hard = pgno2bytes(env, txn->geo.upper); info->txn_space_retired = - pgno2bytes(env, txn->nested ? (size_t)txn->tw.retired_pages : MDBX_PNL_GETSIZE(txn->tw.retired_pages)); - info->txn_space_leftover = pgno2bytes(env, txn->tw.dirtyroom); + pgno2bytes(env, txn->nested ? (size_t)txn->wr.retired_pages : MDBX_PNL_GETSIZE(txn->wr.retired_pages)); + info->txn_space_leftover = pgno2bytes(env, txn->wr.dirtyroom); info->txn_space_dirty = - pgno2bytes(env, txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose - : (txn->tw.writemap_dirty_npages + txn->tw.writemap_spilled_npages)); + pgno2bytes(env, txn->wr.dirtylist ? txn->wr.dirtylist->pages_including_loose + : (txn->wr.writemap_dirty_npages + txn->wr.writemap_spilled_npages)); info->txn_reader_lag = INT64_MAX; lck_t *const lck = env->lck_mmap.lck; if (scan_rlt && lck) { diff --git a/src/audit.c b/src/audit.c index 91e18873..e46c0dff 100644 --- a/src/audit.c +++ b/src/audit.c @@ -28,8 +28,8 @@ __cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored, bool don const MDBX_env *const env = txn->env; size_t pending = 0; if ((txn->flags & MDBX_TXN_RDONLY) == 0) - pending = txn->tw.loose_count + MDBX_PNL_GETSIZE(txn->tw.repnl) + - (MDBX_PNL_GETSIZE(txn->tw.retired_pages) - retired_stored); + pending = txn->wr.loose_count + MDBX_PNL_GETSIZE(txn->wr.repnl) + + (MDBX_PNL_GETSIZE(txn->wr.retired_pages) - retired_stored); cursor_couple_t cx; int rc = cursor_init(&cx.outer, txn, FREE_DBI); @@ -46,7 +46,7 @@ __cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored, bool don return MDBX_CORRUPTED; } txnid_t id = unaligned_peek_u64(4, key.iov_base); - if (txn->tw.gc.retxl ? txl_contain(txn->tw.gc.retxl, id) : (id <= txn->tw.gc.last_reclaimed)) + if (txn->wr.gc.retxl ? txl_contain(txn->wr.gc.retxl, id) : (id <= txn->wr.gc.last_reclaimed)) goto skip; } gc += *(pgno_t *)data.iov_base; @@ -89,8 +89,8 @@ __cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored, bool don if ((txn->flags & MDBX_TXN_RDONLY) == 0) ERROR("audit @%" PRIaTXN ": %zu(pending) = %zu(loose) + " "%zu(reclaimed) + %zu(retired-pending) - %zu(retired-stored)", - txn->txnid, pending, txn->tw.loose_count, MDBX_PNL_GETSIZE(txn->tw.repnl), - txn->tw.retired_pages ? MDBX_PNL_GETSIZE(txn->tw.retired_pages) : 0, retired_stored); + txn->txnid, pending, txn->wr.loose_count, MDBX_PNL_GETSIZE(txn->wr.repnl), + txn->wr.retired_pages ? MDBX_PNL_GETSIZE(txn->wr.retired_pages) : 0, retired_stored); ERROR("audit @%" PRIaTXN ": %zu(pending) + %zu" "(gc) + %zu(count) = %zu(total) <> %zu" "(allocated)", diff --git a/src/cursor.c b/src/cursor.c index 8f1bba85..dd1bc6f4 100644 --- a/src/cursor.c +++ b/src/cursor.c @@ -6,12 +6,12 @@ #include "internals.h" __cold int cursor_check(const MDBX_cursor *mc) { - if (!mc->txn->tw.dirtylist) { + if (!mc->txn->wr.dirtylist) { cASSERT(mc, (mc->txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); } else { cASSERT(mc, (mc->txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - cASSERT(mc, mc->txn->tw.dirtyroom + mc->txn->tw.dirtylist->length == - (mc->txn->parent ? mc->txn->parent->tw.dirtyroom : mc->txn->env->options.dp_limit)); + cASSERT(mc, mc->txn->wr.dirtyroom + mc->txn->wr.dirtylist->length == + (mc->txn->parent ? mc->txn->parent->wr.dirtyroom : mc->txn->env->options.dp_limit)); } cASSERT(mc, (mc->checking & z_updating) ? mc->top + 1 <= mc->tree->height : mc->top + 1 == mc->tree->height); diff --git a/src/dpl.c b/src/dpl.c index 1d2bef79..0e46db8f 100644 --- a/src/dpl.c +++ b/src/dpl.c @@ -28,9 +28,9 @@ static inline size_t dpl_bytes2size(const ptrdiff_t bytes) { } void dpl_free(MDBX_txn *txn) { - if (likely(txn->tw.dirtylist)) { - osal_free(txn->tw.dirtylist); - txn->tw.dirtylist = nullptr; + if (likely(txn->wr.dirtylist)) { + osal_free(txn->wr.dirtylist); + txn->wr.dirtylist = nullptr; } } @@ -39,14 +39,14 @@ dpl_t *dpl_reserve(MDBX_txn *txn, size_t size) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); size_t bytes = dpl_size2bytes((size < PAGELIST_LIMIT) ? size : PAGELIST_LIMIT); - dpl_t *const dl = osal_realloc(txn->tw.dirtylist, bytes); + dpl_t *const dl = osal_realloc(txn->wr.dirtylist, bytes); if (likely(dl)) { #ifdef osal_malloc_usable_size bytes = osal_malloc_usable_size(dl); #endif /* osal_malloc_usable_size */ dl->detent = dpl_bytes2size(bytes); - tASSERT(txn, txn->tw.dirtylist == nullptr || dl->length <= dl->detent); - txn->tw.dirtylist = dl; + tASSERT(txn, txn->wr.dirtylist == nullptr || dl->length <= dl->detent); + txn->wr.dirtylist = dl; } return dl; } @@ -57,15 +57,15 @@ int dpl_alloc(MDBX_txn *txn) { const size_t wanna = (txn->env->options.dp_initial < txn->geo.upper) ? txn->env->options.dp_initial : txn->geo.upper; #if MDBX_FORCE_ASSERTIONS || MDBX_DEBUG - if (txn->tw.dirtylist) + if (txn->wr.dirtylist) /* обнуляем чтобы не сработал ассерт внутри dpl_reserve() */ - txn->tw.dirtylist->sorted = txn->tw.dirtylist->length = 0; + txn->wr.dirtylist->sorted = txn->wr.dirtylist->length = 0; #endif /* asertions enabled */ - if (unlikely(!txn->tw.dirtylist || txn->tw.dirtylist->detent < wanna || txn->tw.dirtylist->detent > wanna + wanna) && + if (unlikely(!txn->wr.dirtylist || txn->wr.dirtylist->detent < wanna || txn->wr.dirtylist->detent > wanna + wanna) && unlikely(!dpl_reserve(txn, wanna))) return MDBX_ENOMEM; - dpl_clear(txn->tw.dirtylist); + dpl_clear(txn->wr.dirtylist); return MDBX_SUCCESS; } @@ -79,7 +79,7 @@ __hot __noinline dpl_t *dpl_sort_slowpath(const MDBX_txn *txn) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - dpl_t *dl = txn->tw.dirtylist; + dpl_t *dl = txn->wr.dirtylist; assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); const size_t unsorted = dl->length - dl->sorted; if (likely(unsorted < MDBX_RADIXSORT_THRESHOLD) || unlikely(!dp_radixsort(dl->items + 1, dl->length))) { @@ -133,7 +133,7 @@ __hot __noinline size_t dpl_search(const MDBX_txn *txn, pgno_t pgno) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - dpl_t *dl = txn->tw.dirtylist; + dpl_t *dl = txn->wr.dirtylist; assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); if (AUDIT_ENABLED()) { for (const dp_t *ptr = dl->items + dl->sorted; --ptr > dl->items;) { @@ -175,7 +175,7 @@ __hot __noinline size_t dpl_search(const MDBX_txn *txn, pgno_t pgno) { const page_t *debug_dpl_find(const MDBX_txn *txn, const pgno_t pgno) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); - const dpl_t *dl = txn->tw.dirtylist; + const dpl_t *dl = txn->wr.dirtylist; if (dl) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); @@ -198,7 +198,7 @@ void dpl_remove_ex(const MDBX_txn *txn, size_t i, size_t npages) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - dpl_t *dl = txn->tw.dirtylist; + dpl_t *dl = txn->wr.dirtylist; assert((intptr_t)i > 0 && i <= dl->length); assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); dl->pages_including_loose -= npages; @@ -214,10 +214,10 @@ int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, siz const dp_t dp = {page, pgno, (pgno_t)npages}; if ((txn->flags & MDBX_WRITEMAP) == 0) { size_t *const ptr = ptr_disp(page, -(ptrdiff_t)sizeof(size_t)); - *ptr = txn->tw.dirtylru; + *ptr = txn->wr.dirtylru; } - dpl_t *dl = txn->tw.dirtylist; + dpl_t *dl = txn->wr.dirtylist; tASSERT(txn, dl->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE); tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); if (AUDIT_ENABLED()) { @@ -313,7 +313,7 @@ int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, siz __cold bool dpl_check(MDBX_txn *txn) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); - const dpl_t *const dl = txn->tw.dirtylist; + const dpl_t *const dl = txn->wr.dirtylist; if (!dl) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); return true; @@ -322,7 +322,7 @@ __cold bool dpl_check(MDBX_txn *txn) { assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); tASSERT(txn, - txn->tw.dirtyroom + dl->length == (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); + txn->wr.dirtyroom + dl->length == (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit)); if (!AUDIT_ENABLED()) return true; @@ -362,28 +362,28 @@ __cold bool dpl_check(MDBX_txn *txn) { return false; } - const size_t rpa = pnl_search(txn->tw.repnl, dp->pgno, txn->geo.first_unallocated); - tASSERT(txn, rpa > MDBX_PNL_GETSIZE(txn->tw.repnl) || txn->tw.repnl[rpa] != dp->pgno); - if (rpa <= MDBX_PNL_GETSIZE(txn->tw.repnl) && unlikely(txn->tw.repnl[rpa] == dp->pgno)) + const size_t rpa = pnl_search(txn->wr.repnl, dp->pgno, txn->geo.first_unallocated); + tASSERT(txn, rpa > MDBX_PNL_GETSIZE(txn->wr.repnl) || txn->wr.repnl[rpa] != dp->pgno); + if (rpa <= MDBX_PNL_GETSIZE(txn->wr.repnl) && unlikely(txn->wr.repnl[rpa] == dp->pgno)) return false; if (num > 1) { - const size_t rpb = pnl_search(txn->tw.repnl, dp->pgno + num - 1, txn->geo.first_unallocated); + const size_t rpb = pnl_search(txn->wr.repnl, dp->pgno + num - 1, txn->geo.first_unallocated); tASSERT(txn, rpa == rpb); if (unlikely(rpa != rpb)) return false; } } - tASSERT(txn, loose == txn->tw.loose_count); - if (unlikely(loose != txn->tw.loose_count)) + tASSERT(txn, loose == txn->wr.loose_count); + if (unlikely(loose != txn->wr.loose_count)) return false; tASSERT(txn, pages == dl->pages_including_loose); if (unlikely(pages != dl->pages_including_loose)) return false; - for (size_t i = 1; i <= MDBX_PNL_GETSIZE(txn->tw.retired_pages); ++i) { - const page_t *const dp = debug_dpl_find(txn, txn->tw.retired_pages[i]); + for (size_t i = 1; i <= MDBX_PNL_GETSIZE(txn->wr.retired_pages); ++i) { + const page_t *const dp = debug_dpl_find(txn, txn->wr.retired_pages[i]); tASSERT(txn, !dp); if (unlikely(dp)) return false; @@ -395,11 +395,11 @@ __cold bool dpl_check(MDBX_txn *txn) { /*----------------------------------------------------------------------------*/ __noinline void dpl_lru_reduce(MDBX_txn *txn) { - NOTICE("lru-reduce %u -> %u", txn->tw.dirtylru, txn->tw.dirtylru >> 1); + NOTICE("lru-reduce %u -> %u", txn->wr.dirtylru, txn->wr.dirtylru >> 1); tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); do { - txn->tw.dirtylru >>= 1; - dpl_t *dl = txn->tw.dirtylist; + txn->wr.dirtylru >>= 1; + dpl_t *dl = txn->wr.dirtylist; for (size_t i = 1; i <= dl->length; ++i) { size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t)); *ptr >>= 1; @@ -411,7 +411,7 @@ __noinline void dpl_lru_reduce(MDBX_txn *txn) { void dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - if (MDBX_PNL_GETSIZE(pl) && txn->tw.dirtylist->length) { + if (MDBX_PNL_GETSIZE(pl) && txn->wr.dirtylist->length) { tASSERT(txn, pnl_check_allocated(pl, (size_t)txn->geo.first_unallocated << spilled)); dpl_t *dl = dpl_sort(txn); @@ -466,9 +466,9 @@ void dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled) { } } dl->sorted = dpl_setlen(dl, w - 1); - txn->tw.dirtyroom += r - w; - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); + txn->wr.dirtyroom += r - w; + tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length == + (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit)); return; } } @@ -477,7 +477,7 @@ void dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled) { void dpl_release_shadows(MDBX_txn *txn) { tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); MDBX_env *env = txn->env; - dpl_t *const dl = txn->tw.dirtylist; + dpl_t *const dl = txn->wr.dirtylist; for (size_t i = 1; i <= dl->length; i++) page_shadow_release(env, dl->items[i].ptr, dpl_npages(dl, i)); diff --git a/src/dpl.h b/src/dpl.h index 0059dc58..39003e24 100644 --- a/src/dpl.h +++ b/src/dpl.h @@ -46,7 +46,7 @@ static inline dpl_t *dpl_sort(const MDBX_txn *txn) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - dpl_t *dl = txn->tw.dirtylist; + dpl_t *dl = txn->wr.dirtylist; tASSERT(txn, dl->length <= PAGELIST_LIMIT); tASSERT(txn, dl->sorted <= dl->length); tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); @@ -72,7 +72,7 @@ static inline bool dpl_intersect(const MDBX_txn *txn, pgno_t pgno, size_t npages tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - dpl_t *dl = txn->tw.dirtylist; + dpl_t *dl = txn->wr.dirtylist; tASSERT(txn, dl->sorted == dl->length); tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); size_t const n = dpl_search(txn, pgno); @@ -96,7 +96,7 @@ static inline bool dpl_intersect(const MDBX_txn *txn, pgno_t pgno, size_t npages MDBX_NOTHROW_PURE_FUNCTION static inline size_t dpl_exist(const MDBX_txn *txn, pgno_t pgno) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - dpl_t *dl = txn->tw.dirtylist; + dpl_t *dl = txn->wr.dirtylist; size_t i = dpl_search(txn, pgno); tASSERT(txn, (int)i > 0); return (dl->items[i].pgno == pgno) ? i : 0; @@ -105,7 +105,7 @@ MDBX_NOTHROW_PURE_FUNCTION static inline size_t dpl_exist(const MDBX_txn *txn, p MDBX_INTERNAL void dpl_remove_ex(const MDBX_txn *txn, size_t i, size_t npages); static inline void dpl_remove(const MDBX_txn *txn, size_t i) { - dpl_remove_ex(txn, i, dpl_npages(txn->tw.dirtylist, i)); + dpl_remove_ex(txn, i, dpl_npages(txn->wr.dirtylist, i)); } MDBX_INTERNAL int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, size_t npages); @@ -114,19 +114,19 @@ MDBX_MAYBE_UNUSED MDBX_INTERNAL bool dpl_check(MDBX_txn *txn); MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t dpl_age(const MDBX_txn *txn, size_t i) { tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); - const dpl_t *dl = txn->tw.dirtylist; + const dpl_t *dl = txn->wr.dirtylist; assert((intptr_t)i > 0 && i <= dl->length); size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t)); - return txn->tw.dirtylru - (uint32_t)*ptr; + return txn->wr.dirtylru - (uint32_t)*ptr; } MDBX_INTERNAL void dpl_lru_reduce(MDBX_txn *txn); static inline uint32_t dpl_lru_turn(MDBX_txn *txn) { - txn->tw.dirtylru += 1; - if (unlikely(txn->tw.dirtylru > UINT32_MAX / 3) && (txn->flags & MDBX_WRITEMAP) == 0) + txn->wr.dirtylru += 1; + if (unlikely(txn->wr.dirtylru > UINT32_MAX / 3) && (txn->flags & MDBX_WRITEMAP) == 0) dpl_lru_reduce(txn); - return txn->tw.dirtylru; + return txn->wr.dirtylru; } MDBX_INTERNAL void dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled); diff --git a/src/dxb.c b/src/dxb.c index b5117c4b..3699d597 100644 --- a/src/dxb.c +++ b/src/dxb.c @@ -370,7 +370,7 @@ void dxb_sanitize_tail(MDBX_env *env, MDBX_txn *txn) { return; } else if (env->txn && env_txn0_owned(env)) { /* inside write-txn */ - last = meta_recent(env, &env->basal_txn->tw.troika).ptr_v->geometry.first_unallocated; + last = meta_recent(env, &env->basal_txn->wr.troika).ptr_v->geometry.first_unallocated; } else if (env->flags & MDBX_RDONLY) { /* read-only mode, no write-txn, no wlock mutex */ last = NUM_METAS; @@ -1300,8 +1300,8 @@ int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, troika *troika = meta_tap(env); for (MDBX_txn *txn = env->basal_txn; txn; txn = txn->nested) - if (troika != &txn->tw.troika) - txn->tw.troika = *troika; + if (troika != &txn->wr.troika) + txn->wr.troika = *troika; /* LY: shrink datafile if needed */ if (unlikely(shrink)) { diff --git a/src/env.c b/src/env.c index 99b5d895..158ec030 100644 --- a/src/env.c +++ b/src/env.c @@ -71,7 +71,7 @@ retry:; goto bailout; } - const troika_t troika = (txn0_owned | should_unlock) ? env->basal_txn->tw.troika : meta_tap(env); + const troika_t troika = (txn0_owned | should_unlock) ? env->basal_txn->wr.troika : meta_tap(env); const meta_ptr_t head = meta_recent(env, &troika); const uint64_t unsynced_pages = atomic_load64(&env->lck->unsynced_pages, mo_Relaxed); if (unsynced_pages == 0) { @@ -153,7 +153,7 @@ retry:; #if MDBX_ENABLE_PGOP_STAT env->lck->pgops.wops.weak += wops; #endif /* MDBX_ENABLE_PGOP_STAT */ - env->basal_txn->tw.troika = meta_tap(env); + env->basal_txn->wr.troika = meta_tap(env); eASSERT(env, !env->txn && !env->basal_txn->nested); goto retry; } @@ -177,7 +177,7 @@ retry:; DEBUG("meta-head %" PRIaPGNO ", %s, sync_pending %" PRIu64, data_page(head.ptr_c)->pgno, durable_caption(head.ptr_c), unsynced_pages); meta_t meta = *head.ptr_c; - rc = dxb_sync_locked(env, flags, &meta, &env->basal_txn->tw.troika); + rc = dxb_sync_locked(env, flags, &meta, &env->basal_txn->wr.troika); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; } diff --git a/src/gc-get.c b/src/gc-get.c index c6d2c62b..adcb5053 100644 --- a/src/gc-get.c +++ b/src/gc-get.c @@ -590,12 +590,12 @@ static inline bool is_gc_usable(MDBX_txn *txn, const MDBX_cursor *mc, const uint return true; } -static inline bool is_already_reclaimed(const MDBX_txn *txn, txnid_t id) { return txl_contain(txn->tw.gc.retxl, id); } +static inline bool is_already_reclaimed(const MDBX_txn *txn, txnid_t id) { return txl_contain(txn->wr.gc.retxl, id); } __hot static pgno_t repnl_get_single(MDBX_txn *txn) { - const size_t len = MDBX_PNL_GETSIZE(txn->tw.repnl); + const size_t len = MDBX_PNL_GETSIZE(txn->wr.repnl); assert(len > 0); - pgno_t *target = MDBX_PNL_EDGE(txn->tw.repnl); + pgno_t *target = MDBX_PNL_EDGE(txn->wr.repnl); const ptrdiff_t dir = MDBX_PNL_ASCENDING ? 1 : -1; /* Есть ТРИ потенциально выигрышные, но противо-направленные тактики: @@ -663,7 +663,7 @@ __hot static pgno_t repnl_get_single(MDBX_txn *txn) { #else /* вырезаем элемент с перемещением хвоста */ const pgno_t pgno = *scan; - MDBX_PNL_SETSIZE(txn->tw.repnl, len - 1); + MDBX_PNL_SETSIZE(txn->wr.repnl, len - 1); while (++scan <= target) scan[-1] = *scan; return pgno; @@ -676,44 +676,44 @@ __hot static pgno_t repnl_get_single(MDBX_txn *txn) { const pgno_t pgno = *target; #if MDBX_PNL_ASCENDING /* вырезаем элемент с перемещением хвоста */ - MDBX_PNL_SETSIZE(txn->tw.repnl, len - 1); - for (const pgno_t *const end = txn->tw.repnl + len - 1; target <= end; ++target) + MDBX_PNL_SETSIZE(txn->wr.repnl, len - 1); + for (const pgno_t *const end = txn->wr.repnl + len - 1; target <= end; ++target) *target = target[1]; #else /* перемещать хвост не нужно, просто усекам список */ - MDBX_PNL_SETSIZE(txn->tw.repnl, len - 1); + MDBX_PNL_SETSIZE(txn->wr.repnl, len - 1); #endif return pgno; } __hot static pgno_t repnl_get_sequence(MDBX_txn *txn, const size_t num, uint8_t flags) { - const size_t len = MDBX_PNL_GETSIZE(txn->tw.repnl); - pgno_t *edge = MDBX_PNL_EDGE(txn->tw.repnl); + const size_t len = MDBX_PNL_GETSIZE(txn->wr.repnl); + pgno_t *edge = MDBX_PNL_EDGE(txn->wr.repnl); assert(len >= num && num > 1); const size_t seq = num - 1; #if !MDBX_PNL_ASCENDING if (edge[-(ptrdiff_t)seq] - *edge == seq) { if (unlikely(flags & ALLOC_RESERVE)) return P_INVALID; - assert(edge == scan4range_checker(txn->tw.repnl, seq)); + assert(edge == scan4range_checker(txn->wr.repnl, seq)); /* перемещать хвост не нужно, просто усекам список */ - MDBX_PNL_SETSIZE(txn->tw.repnl, len - num); + MDBX_PNL_SETSIZE(txn->wr.repnl, len - num); return *edge; } #endif pgno_t *target = scan4seq_impl(edge, len, seq); - assert(target == scan4range_checker(txn->tw.repnl, seq)); + assert(target == scan4range_checker(txn->wr.repnl, seq)); if (target) { if (unlikely(flags & ALLOC_RESERVE)) return P_INVALID; const pgno_t pgno = *target; /* вырезаем найденную последовательность с перемещением хвоста */ - MDBX_PNL_SETSIZE(txn->tw.repnl, len - num); + MDBX_PNL_SETSIZE(txn->wr.repnl, len - num); #if MDBX_PNL_ASCENDING - for (const pgno_t *const end = txn->tw.repnl + len - num; target <= end; ++target) + for (const pgno_t *const end = txn->wr.repnl + len - num; target <= end; ++target) *target = target[num]; #else - for (const pgno_t *const end = txn->tw.repnl + len; ++target <= end;) + for (const pgno_t *const end = txn->wr.repnl + len; ++target <= end;) target[-(ptrdiff_t)num] = *target; #endif return pgno; @@ -762,7 +762,7 @@ static inline pgr_t page_alloc_finalize(MDBX_env *const env, MDBX_txn *const txn * обновляться PTE с последующей генерацией page-fault и чтением данных из * грязной I/O очереди. Из-за этого штраф за лишнюю запись может быть * сравним с избегаемым ненужным чтением. */ - if (txn->tw.prefault_write_activated) { + if (txn->wr.prefault_write_activated) { void *const pattern = ptr_disp(env->page_auxbuf, need_clean ? env->ps : env->ps * 2); size_t file_offset = pgno2bytes(env, pgno); if (likely(num == 1)) { @@ -823,7 +823,7 @@ static inline pgr_t page_alloc_finalize(MDBX_env *const env, MDBX_txn *const txn ret.err = page_dirty(txn, ret.page, (pgno_t)num); bailout: - tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); #if MDBX_ENABLE_PROFGC size_t majflt_after; prof->xtime_cpu += osal_cputime(&majflt_after) - cputime_before; @@ -843,7 +843,7 @@ pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags) #endif /* MDBX_ENABLE_PROFGC */ eASSERT(env, num > 0 || (flags & ALLOC_RESERVE)); - eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + eASSERT(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); size_t newnext; const uint64_t monotime_begin = (MDBX_ENABLE_PROFGC || (num > 1 && env->options.gc_time_limit)) ? osal_monotime() : 0; @@ -858,15 +858,15 @@ pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags) #if MDBX_ENABLE_PROFGC prof->xpages += 1; #endif /* MDBX_ENABLE_PROFGC */ - if (MDBX_PNL_GETSIZE(txn->tw.repnl) >= num) { - eASSERT(env, MDBX_PNL_LAST(txn->tw.repnl) < txn->geo.first_unallocated && - MDBX_PNL_FIRST(txn->tw.repnl) < txn->geo.first_unallocated); + if (MDBX_PNL_GETSIZE(txn->wr.repnl) >= num) { + eASSERT(env, MDBX_PNL_LAST(txn->wr.repnl) < txn->geo.first_unallocated && + MDBX_PNL_FIRST(txn->wr.repnl) < txn->geo.first_unallocated); pgno = repnl_get_sequence(txn, num, flags); if (likely(pgno)) goto done; } } else { - eASSERT(env, num == 0 || MDBX_PNL_GETSIZE(txn->tw.repnl) == 0); + eASSERT(env, num == 0 || MDBX_PNL_GETSIZE(txn->wr.repnl) == 0); eASSERT(env, !(flags & ALLOC_RESERVE) || num == 0); } @@ -884,7 +884,7 @@ pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags) * Иначе попытка увеличить резерв может приводить к необходимости ещё * большего резерва из-за увеличения списка переработанных страниц. */ (flags & ALLOC_RESERVE) == 0) { - if (txn->dbs[FREE_DBI].branch_pages && MDBX_PNL_GETSIZE(txn->tw.repnl) < env->maxgc_large1page / 2) + if (txn->dbs[FREE_DBI].branch_pages && MDBX_PNL_GETSIZE(txn->wr.repnl) < env->maxgc_large1page / 2) flags += ALLOC_COALESCE; } @@ -894,8 +894,8 @@ pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags) gc->dbi_state = txn->dbi_state; gc->top_and_flags = z_fresh_mark; - txn->tw.prefault_write_activated = env->options.prefault_write; - if (txn->tw.prefault_write_activated) { + txn->wr.prefault_write_activated = env->options.prefault_write; + if (txn->wr.prefault_write_activated) { /* Проверка посредством minicore() существенно снижает затраты, но в * простейших случаях (тривиальный бенчмарк) интегральная производительность * становится вдвое меньше. А на платформах без mincore() и с проблемной @@ -908,7 +908,7 @@ pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags) (txn->dbs[FREE_DBI].branch_pages == 0 && txn->geo.now < 1234) || /* Не суетимся если страница в зоне включенного упреждающего чтения */ (readahead_enabled && pgno + num < readahead_edge)) - txn->tw.prefault_write_activated = false; + txn->wr.prefault_write_activated = false; } retry_gc_refresh_oldest:; @@ -924,9 +924,9 @@ retry_gc_have_oldest: txnid_t id = 0; MDBX_cursor_op op = MDBX_FIRST; if (flags & ALLOC_LIFO) { - if (!txn->tw.gc.retxl) { - txn->tw.gc.retxl = txl_alloc(); - if (unlikely(!txn->tw.gc.retxl)) { + if (!txn->wr.gc.retxl) { + txn->wr.gc.retxl = txl_alloc(); + if (unlikely(!txn->wr.gc.retxl)) { ret.err = MDBX_ENOMEM; goto fail; } @@ -934,9 +934,9 @@ retry_gc_have_oldest: /* Begin lookup backward from oldest reader */ id = detent - 1; op = MDBX_SET_RANGE; - } else if (txn->tw.gc.last_reclaimed) { + } else if (txn->wr.gc.last_reclaimed) { /* Continue lookup forward from last-reclaimed */ - id = txn->tw.gc.last_reclaimed + 1; + id = txn->wr.gc.last_reclaimed + 1; if (id >= detent) goto depleted_gc; op = MDBX_SET_RANGE; @@ -994,9 +994,9 @@ next_gc:; } const size_t gc_len = MDBX_PNL_GETSIZE(gc_pnl); - TRACE("gc-read: id #%" PRIaTXN " len %zu, re-list will %zu ", id, gc_len, gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl)); + TRACE("gc-read: id #%" PRIaTXN " len %zu, re-list will %zu ", id, gc_len, gc_len + MDBX_PNL_GETSIZE(txn->wr.repnl)); - if (unlikely(gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl) >= env->maxgc_large1page)) { + if (unlikely(gc_len + MDBX_PNL_GETSIZE(txn->wr.repnl) >= env->maxgc_large1page)) { /* Don't try to coalesce too much. */ if (flags & ALLOC_SHOULD_SCAN) { eASSERT(env, flags & ALLOC_COALESCE); @@ -1006,9 +1006,9 @@ next_gc:; env->lck->pgops.gc_prof.coalescences += 1; #endif /* MDBX_ENABLE_PROFGC */ TRACE("clear %s %s", "ALLOC_COALESCE", "since got threshold"); - if (MDBX_PNL_GETSIZE(txn->tw.repnl) >= num) { - eASSERT(env, MDBX_PNL_LAST(txn->tw.repnl) < txn->geo.first_unallocated && - MDBX_PNL_FIRST(txn->tw.repnl) < txn->geo.first_unallocated); + if (MDBX_PNL_GETSIZE(txn->wr.repnl) >= num) { + eASSERT(env, MDBX_PNL_LAST(txn->wr.repnl) < txn->geo.first_unallocated && + MDBX_PNL_FIRST(txn->wr.repnl) < txn->geo.first_unallocated); if (likely(num == 1)) { pgno = repnl_get_single(txn); goto done; @@ -1019,34 +1019,34 @@ next_gc:; } flags -= ALLOC_COALESCE | ALLOC_SHOULD_SCAN; } - if (unlikely(/* list is too long already */ MDBX_PNL_GETSIZE(txn->tw.repnl) >= env->options.rp_augment_limit) && + if (unlikely(/* list is too long already */ MDBX_PNL_GETSIZE(txn->wr.repnl) >= env->options.rp_augment_limit) && ((/* not a slot-request from gc-update */ num && /* have enough unallocated space */ txn->geo.upper >= txn->geo.first_unallocated + num && - monotime_since_cached(monotime_begin, &now_cache) + txn->tw.gc.time_acc >= env->options.gc_time_limit) || - gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl) >= PAGELIST_LIMIT)) { + monotime_since_cached(monotime_begin, &now_cache) + txn->wr.gc.time_acc >= env->options.gc_time_limit) || + gc_len + MDBX_PNL_GETSIZE(txn->wr.repnl) >= PAGELIST_LIMIT)) { /* Stop reclaiming to avoid large/overflow the page list. This is a rare * case while search for a continuously multi-page region in a * large database, see https://libmdbx.dqdkfa.ru/dead-github/issues/123 */ NOTICE("stop reclaiming %s: %zu (current) + %zu " "(chunk) -> %zu, rp_augment_limit %u", - likely(gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl) < PAGELIST_LIMIT) ? "since rp_augment_limit was reached" + likely(gc_len + MDBX_PNL_GETSIZE(txn->wr.repnl) < PAGELIST_LIMIT) ? "since rp_augment_limit was reached" : "to avoid PNL overflow", - MDBX_PNL_GETSIZE(txn->tw.repnl), gc_len, gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl), + MDBX_PNL_GETSIZE(txn->wr.repnl), gc_len, gc_len + MDBX_PNL_GETSIZE(txn->wr.repnl), env->options.rp_augment_limit); goto depleted_gc; } } /* Remember ID of readed GC record */ - txn->tw.gc.last_reclaimed = id; + txn->wr.gc.last_reclaimed = id; if (flags & ALLOC_LIFO) { - ret.err = txl_append(&txn->tw.gc.retxl, id); + ret.err = txl_append(&txn->wr.gc.retxl, id); if (unlikely(ret.err != MDBX_SUCCESS)) goto fail; } - /* Append PNL from GC record to tw.repnl */ - ret.err = pnl_need(&txn->tw.repnl, gc_len); + /* Append PNL from GC record to wr.repnl */ + ret.err = pnl_need(&txn->wr.repnl, gc_len); if (unlikely(ret.err != MDBX_SUCCESS)) goto fail; @@ -1061,36 +1061,36 @@ next_gc:; #if MDBX_ENABLE_PROFGC const uint64_t merge_begin = osal_monotime(); #endif /* MDBX_ENABLE_PROFGC */ - pnl_merge(txn->tw.repnl, gc_pnl); + pnl_merge(txn->wr.repnl, gc_pnl); #if MDBX_ENABLE_PROFGC prof->pnl_merge.calls += 1; - prof->pnl_merge.volume += MDBX_PNL_GETSIZE(txn->tw.repnl); + prof->pnl_merge.volume += MDBX_PNL_GETSIZE(txn->wr.repnl); prof->pnl_merge.time += osal_monotime() - merge_begin; #endif /* MDBX_ENABLE_PROFGC */ flags |= ALLOC_SHOULD_SCAN; if (AUDIT_ENABLED()) { - if (unlikely(!pnl_check(txn->tw.repnl, txn->geo.first_unallocated))) { + if (unlikely(!pnl_check(txn->wr.repnl, txn->geo.first_unallocated))) { ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid txn retired-list"); ret.err = MDBX_CORRUPTED; goto fail; } } else { - eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated)); + eASSERT(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated)); } eASSERT(env, dpl_check(txn)); - eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.repnl) == 0 || MDBX_PNL_MOST(txn->tw.repnl) < txn->geo.first_unallocated); - if (MDBX_ENABLE_REFUND && MDBX_PNL_GETSIZE(txn->tw.repnl) && - unlikely(MDBX_PNL_MOST(txn->tw.repnl) == txn->geo.first_unallocated - 1)) { + eASSERT(env, MDBX_PNL_GETSIZE(txn->wr.repnl) == 0 || MDBX_PNL_MOST(txn->wr.repnl) < txn->geo.first_unallocated); + if (MDBX_ENABLE_REFUND && MDBX_PNL_GETSIZE(txn->wr.repnl) && + unlikely(MDBX_PNL_MOST(txn->wr.repnl) == txn->geo.first_unallocated - 1)) { /* Refund suitable pages into "unallocated" space */ txn_refund(txn); } - eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + eASSERT(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); /* Done for a kick-reclaim mode, actually no page needed */ if (unlikely(num == 0)) { eASSERT(env, ret.err == MDBX_SUCCESS); - TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "early-exit for slot", id, MDBX_PNL_GETSIZE(txn->tw.repnl)); + TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "early-exit for slot", id, MDBX_PNL_GETSIZE(txn->wr.repnl)); goto early_exit; } @@ -1098,16 +1098,16 @@ next_gc:; eASSERT(env, op == MDBX_PREV || op == MDBX_NEXT); if (flags & ALLOC_COALESCE) { - TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "coalesce-continue", id, MDBX_PNL_GETSIZE(txn->tw.repnl)); + TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "coalesce-continue", id, MDBX_PNL_GETSIZE(txn->wr.repnl)); goto next_gc; } scan: eASSERT(env, flags & ALLOC_SHOULD_SCAN); eASSERT(env, num > 0); - if (MDBX_PNL_GETSIZE(txn->tw.repnl) >= num) { - eASSERT(env, MDBX_PNL_LAST(txn->tw.repnl) < txn->geo.first_unallocated && - MDBX_PNL_FIRST(txn->tw.repnl) < txn->geo.first_unallocated); + if (MDBX_PNL_GETSIZE(txn->wr.repnl) >= num) { + eASSERT(env, MDBX_PNL_LAST(txn->wr.repnl) < txn->geo.first_unallocated && + MDBX_PNL_FIRST(txn->wr.repnl) < txn->geo.first_unallocated); if (likely(num == 1)) { eASSERT(env, !(flags & ALLOC_RESERVE)); pgno = repnl_get_single(txn); @@ -1119,12 +1119,12 @@ scan: } flags -= ALLOC_SHOULD_SCAN; if (ret.err == MDBX_SUCCESS) { - TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "continue-search", id, MDBX_PNL_GETSIZE(txn->tw.repnl)); + TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "continue-search", id, MDBX_PNL_GETSIZE(txn->wr.repnl)); goto next_gc; } depleted_gc: - TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "gc-depleted", id, MDBX_PNL_GETSIZE(txn->tw.repnl)); + TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "gc-depleted", id, MDBX_PNL_GETSIZE(txn->wr.repnl)); ret.err = MDBX_NOTFOUND; if (flags & ALLOC_SHOULD_SCAN) goto scan; @@ -1143,8 +1143,8 @@ depleted_gc: newnext = txn->geo.first_unallocated + num; /* Does reclaiming stopped at the last steady point? */ - const meta_ptr_t recent = meta_recent(env, &txn->tw.troika); - const meta_ptr_t prefer_steady = meta_prefer_steady(env, &txn->tw.troika); + const meta_ptr_t recent = meta_recent(env, &txn->wr.troika); + const meta_ptr_t prefer_steady = meta_prefer_steady(env, &txn->wr.troika); if (recent.ptr_c != prefer_steady.ptr_c && prefer_steady.is_steady && detent == prefer_steady.txnid + 1) { DEBUG("gc-kick-steady: recent %" PRIaTXN "-%s, steady %" PRIaTXN "-%s, detent %" PRIaTXN, recent.txnid, durable_caption(recent.ptr_c), prefer_steady.txnid, durable_caption(prefer_steady.ptr_c), detent); @@ -1170,7 +1170,7 @@ depleted_gc: DEBUG("gc-wipe-steady, rc %d", ret.err); if (unlikely(ret.err != MDBX_SUCCESS)) goto fail; - eASSERT(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->tw.troika).ptr_c); + eASSERT(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->wr.troika).ptr_c); goto retry_gc_refresh_oldest; } if ((autosync_threshold && atomic_load64(&env->lck->unsynced_pages, mo_Relaxed) >= autosync_threshold) || @@ -1183,12 +1183,12 @@ depleted_gc: env->lck->pgops.gc_prof.flushes += 1; #endif /* MDBX_ENABLE_PROFGC */ meta_t meta = *recent.ptr_c; - ret.err = dxb_sync_locked(env, env->flags & MDBX_WRITEMAP, &meta, &txn->tw.troika); + ret.err = dxb_sync_locked(env, env->flags & MDBX_WRITEMAP, &meta, &txn->wr.troika); DEBUG("gc-make-steady, rc %d", ret.err); eASSERT(env, ret.err != MDBX_RESULT_TRUE); if (unlikely(ret.err != MDBX_SUCCESS)) goto fail; - eASSERT(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->tw.troika).ptr_c); + eASSERT(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->wr.troika).ptr_c); goto retry_gc_refresh_oldest; } } @@ -1263,7 +1263,7 @@ done: if (likely((flags & ALLOC_RESERVE) == 0)) { if (pgno) { eASSERT(env, pgno + num <= txn->geo.first_unallocated && pgno >= NUM_METAS); - eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + eASSERT(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); } else { pgno = txn->geo.first_unallocated; txn->geo.first_unallocated += (pgno_t)num; @@ -1275,7 +1275,7 @@ done: if (unlikely(ret.err != MDBX_SUCCESS)) { fail: eASSERT(env, ret.err != MDBX_SUCCESS); - eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + eASSERT(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); int level; const char *what; if (flags & ALLOC_RESERVE) { @@ -1291,14 +1291,14 @@ done: "unable alloc %zu %s, alloc-flags 0x%x, err %d, txn-flags " "0x%x, re-list-len %zu, loose-count %zu, gc: height %u, " "branch %zu, leaf %zu, large %zu, entries %zu\n", - num, what, flags, ret.err, txn->flags, MDBX_PNL_GETSIZE(txn->tw.repnl), txn->tw.loose_count, + num, what, flags, ret.err, txn->flags, MDBX_PNL_GETSIZE(txn->wr.repnl), txn->wr.loose_count, txn->dbs[FREE_DBI].height, (size_t)txn->dbs[FREE_DBI].branch_pages, (size_t)txn->dbs[FREE_DBI].leaf_pages, (size_t)txn->dbs[FREE_DBI].large_pages, (size_t)txn->dbs[FREE_DBI].items); ret.page = nullptr; } if (num > 1) - txn->tw.gc.time_acc += monotime_since_cached(monotime_begin, &now_cache); + txn->wr.gc.time_acc += monotime_since_cached(monotime_begin, &now_cache); } else { early_exit: DEBUG("return nullptr for %zu pages for ALLOC_%s, rc %d", num, num ? "RESERVE" : "SLOT", ret.err); @@ -1317,20 +1317,20 @@ __hot pgr_t gc_alloc_single(const MDBX_cursor *const mc) { tASSERT(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY)); /* If there are any loose pages, just use them */ - while (likely(txn->tw.loose_pages)) { + while (likely(txn->wr.loose_pages)) { #if MDBX_ENABLE_REFUND - if (unlikely(txn->tw.loose_refund_wl > txn->geo.first_unallocated)) { + if (unlikely(txn->wr.loose_refund_wl > txn->geo.first_unallocated)) { txn_refund(txn); - if (!txn->tw.loose_pages) + if (!txn->wr.loose_pages) break; } #endif /* MDBX_ENABLE_REFUND */ - page_t *lp = txn->tw.loose_pages; + page_t *lp = txn->wr.loose_pages; MDBX_ASAN_UNPOISON_MEMORY_REGION(lp, txn->env->ps); VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); - txn->tw.loose_pages = page_next(lp); - txn->tw.loose_count--; + txn->wr.loose_pages = page_next(lp); + txn->wr.loose_count--; DEBUG_EXTRA("db %d use loose page %" PRIaPGNO, cursor_dbi_dbg(mc), lp->pgno); tASSERT(txn, lp->pgno < txn->geo.first_unallocated); tASSERT(txn, lp->pgno >= NUM_METAS); @@ -1340,7 +1340,7 @@ __hot pgr_t gc_alloc_single(const MDBX_cursor *const mc) { return ret; } - if (likely(MDBX_PNL_GETSIZE(txn->tw.repnl) > 0)) + if (likely(MDBX_PNL_GETSIZE(txn->wr.repnl) > 0)) return page_alloc_finalize(txn->env, txn, mc, repnl_get_single(txn), 1); return gc_alloc_ex(mc, 1, ALLOC_DEFAULT); diff --git a/src/gc-put.c b/src/gc-put.c index 4c6fd070..51c23f36 100644 --- a/src/gc-put.c +++ b/src/gc-put.c @@ -11,7 +11,7 @@ MDBX_MAYBE_UNUSED static inline const char *dbg_prefix(const gcu_t *ctx) { return is_lifo(ctx->cursor.txn) ? " lifo" : " fifo"; } -static inline size_t backlog_size(MDBX_txn *txn) { return MDBX_PNL_GETSIZE(txn->tw.repnl) + txn->tw.loose_count; } +static inline size_t backlog_size(MDBX_txn *txn) { return MDBX_PNL_GETSIZE(txn->wr.repnl) + txn->wr.loose_count; } static int clean_stored_retired(MDBX_txn *txn, gcu_t *ctx) { int err = MDBX_SUCCESS; @@ -55,7 +55,7 @@ static int touch_gc(gcu_t *ctx) { MDBX_val key, val; key.iov_base = val.iov_base = nullptr; key.iov_len = sizeof(txnid_t); - val.iov_len = MDBX_PNL_SIZEOF(ctx->cursor.txn->tw.retired_pages); + val.iov_len = MDBX_PNL_SIZEOF(ctx->cursor.txn->wr.retired_pages); ctx->cursor.flags |= z_gcu_preparation; int err = cursor_touch(&ctx->cursor, &key, &val); ctx->cursor.flags -= z_gcu_preparation; @@ -71,7 +71,7 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) { size_t for_split = ctx->retired_stored == 0; tASSERT(txn, is_pointed(&ctx->cursor) || txn->dbs[FREE_DBI].leaf_pages == 0); - const intptr_t retired_left = MDBX_PNL_SIZEOF(txn->tw.retired_pages) - ctx->retired_stored; + const intptr_t retired_left = MDBX_PNL_SIZEOF(txn->wr.retired_pages) - ctx->retired_stored; size_t for_repnl = 0; if (MDBX_ENABLE_BIGFOOT && retired_left > 0) { for_repnl = (retired_left + txn->env->maxgc_large1page - 1) / txn->env->maxgc_large1page; @@ -79,7 +79,7 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) { for (size_t entries = for_repnl; entries > 1; for_split += entries) entries = (entries + per_branch_page - 1) / per_branch_page; } else if (!MDBX_ENABLE_BIGFOOT && retired_left != 0) { - for_repnl = largechunk_npages(txn->env, MDBX_PNL_SIZEOF(txn->tw.retired_pages)); + for_repnl = largechunk_npages(txn->env, MDBX_PNL_SIZEOF(txn->wr.retired_pages)); } const size_t for_tree_before_touch = for_cow + for_rebalance + for_split; @@ -101,7 +101,7 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) { TRACE("== after-touch, backlog %zu, err %d", backlog_size(txn), err); if (!MDBX_ENABLE_BIGFOOT && unlikely(for_repnl > 1) && - MDBX_PNL_GETSIZE(txn->tw.retired_pages) != ctx->retired_stored && err == MDBX_SUCCESS) { + MDBX_PNL_GETSIZE(txn->wr.retired_pages) != ctx->retired_stored && err == MDBX_SUCCESS) { if (unlikely(ctx->retired_stored)) { err = clean_stored_retired(txn, ctx); if (unlikely(err != MDBX_SUCCESS)) @@ -145,51 +145,51 @@ static inline void zeroize_reserved(const MDBX_env *env, MDBX_val pnl) { } static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) { - tASSERT(txn, txn->tw.loose_count > 0); - /* Return loose page numbers to tw.repnl, + tASSERT(txn, txn->wr.loose_count > 0); + /* Return loose page numbers to wr.repnl, * though usually none are left at this point. * The pages themselves remain in dirtylist. */ - if (unlikely(!txn->tw.gc.retxl && txn->tw.gc.last_reclaimed < 1)) { - TRACE("%s: try allocate gc-slot for %zu loose-pages", dbg_prefix(ctx), txn->tw.loose_count); + if (unlikely(!txn->wr.gc.retxl && txn->wr.gc.last_reclaimed < 1)) { + TRACE("%s: try allocate gc-slot for %zu loose-pages", dbg_prefix(ctx), txn->wr.loose_count); int err = gc_alloc_ex(&ctx->cursor, 0, ALLOC_RESERVE).err; if (err == MDBX_SUCCESS) { - TRACE("%s: retry since gc-slot for %zu loose-pages available", dbg_prefix(ctx), txn->tw.loose_count); + TRACE("%s: retry since gc-slot for %zu loose-pages available", dbg_prefix(ctx), txn->wr.loose_count); return MDBX_RESULT_TRUE; } - /* Put loose page numbers in tw.retired_pages, - * since unable to return ones to tw.repnl. */ - err = pnl_need(&txn->tw.retired_pages, txn->tw.loose_count); + /* Put loose page numbers in wr.retired_pages, + * since unable to return ones to wr.repnl. */ + err = pnl_need(&txn->wr.retired_pages, txn->wr.loose_count); if (unlikely(err != MDBX_SUCCESS)) return err; - for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) { - pnl_append_prereserved(txn->tw.retired_pages, lp->pgno); + for (page_t *lp = txn->wr.loose_pages; lp; lp = page_next(lp)) { + pnl_append_prereserved(txn->wr.retired_pages, lp->pgno); MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *)); VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); } - TRACE("%s: append %zu loose-pages to retired-pages", dbg_prefix(ctx), txn->tw.loose_count); + TRACE("%s: append %zu loose-pages to retired-pages", dbg_prefix(ctx), txn->wr.loose_count); } else { /* Room for loose pages + temp PNL with same */ - int err = pnl_need(&txn->tw.repnl, 2 * txn->tw.loose_count + 2); + int err = pnl_need(&txn->wr.repnl, 2 * txn->wr.loose_count + 2); if (unlikely(err != MDBX_SUCCESS)) return err; - pnl_t loose = txn->tw.repnl + MDBX_PNL_ALLOCLEN(txn->tw.repnl) - txn->tw.loose_count - 1; + pnl_t loose = txn->wr.repnl + MDBX_PNL_ALLOCLEN(txn->wr.repnl) - txn->wr.loose_count - 1; size_t count = 0; - for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) { + for (page_t *lp = txn->wr.loose_pages; lp; lp = page_next(lp)) { tASSERT(txn, lp->flags == P_LOOSE); loose[++count] = lp->pgno; MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *)); VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); } - tASSERT(txn, count == txn->tw.loose_count); + tASSERT(txn, count == txn->wr.loose_count); MDBX_PNL_SETSIZE(loose, count); pnl_sort(loose, txn->geo.first_unallocated); - pnl_merge(txn->tw.repnl, loose); - TRACE("%s: append %zu loose-pages to reclaimed-pages", dbg_prefix(ctx), txn->tw.loose_count); + pnl_merge(txn->wr.repnl, loose); + TRACE("%s: append %zu loose-pages to reclaimed-pages", dbg_prefix(ctx), txn->wr.loose_count); } /* filter-out list of dirty-pages from loose-pages */ - dpl_t *const dl = txn->tw.dirtylist; + dpl_t *const dl = txn->wr.dirtylist; if (dl) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); tASSERT(txn, dl->sorted <= dl->length); @@ -209,21 +209,21 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) { } } TRACE("%s: filtered-out loose-pages from %zu -> %zu dirty-pages", dbg_prefix(ctx), dl->length, w); - tASSERT(txn, txn->tw.loose_count == dl->length - w); + tASSERT(txn, txn->wr.loose_count == dl->length - w); dl->sorted -= sorted_out; tASSERT(txn, dl->sorted <= w); dpl_setlen(dl, w); - dl->pages_including_loose -= txn->tw.loose_count; - txn->tw.dirtyroom += txn->tw.loose_count; - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); + dl->pages_including_loose -= txn->wr.loose_count; + txn->wr.dirtyroom += txn->wr.loose_count; + tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length == + (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit)); } else { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); } - txn->tw.loose_pages = nullptr; - txn->tw.loose_count = 0; + txn->wr.loose_pages = nullptr; + txn->wr.loose_count = 0; #if MDBX_ENABLE_REFUND - txn->tw.loose_refund_wl = 0; + txn->wr.loose_refund_wl = 0; #endif /* MDBX_ENABLE_REFUND */ return MDBX_SUCCESS; } @@ -250,17 +250,17 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) { tASSERT(txn, ctx->bigfoot <= txn->txnid); } - retired_pages_before = MDBX_PNL_GETSIZE(txn->tw.retired_pages); + retired_pages_before = MDBX_PNL_GETSIZE(txn->wr.retired_pages); err = prepare_backlog(txn, ctx); if (unlikely(err != MDBX_SUCCESS)) return err; - if (retired_pages_before != MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { + if (retired_pages_before != MDBX_PNL_GETSIZE(txn->wr.retired_pages)) { TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx), retired_pages_before, - MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + MDBX_PNL_GETSIZE(txn->wr.retired_pages)); break; } - pnl_sort(txn->tw.retired_pages, txn->geo.first_unallocated); + pnl_sort(txn->wr.retired_pages, txn->geo.first_unallocated); ctx->retired_stored = 0; ctx->bigfoot = txn->txnid; do { @@ -268,15 +268,15 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) { err = prepare_backlog(txn, ctx); if (unlikely(err != MDBX_SUCCESS)) return err; - if (ctx->retired_stored >= MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { + if (ctx->retired_stored >= MDBX_PNL_GETSIZE(txn->wr.retired_pages)) { TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx), retired_pages_before, - MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + MDBX_PNL_GETSIZE(txn->wr.retired_pages)); break; } } key.iov_len = sizeof(txnid_t); key.iov_base = &ctx->bigfoot; - const size_t left = MDBX_PNL_GETSIZE(txn->tw.retired_pages) - ctx->retired_stored; + const size_t left = MDBX_PNL_GETSIZE(txn->wr.retired_pages) - ctx->retired_stored; const size_t chunk = (left > txn->env->maxgc_large1page && ctx->bigfoot < MAX_TXNID) ? txn->env->maxgc_large1page : left; data.iov_len = (chunk + 1) * sizeof(pgno_t); @@ -293,9 +293,9 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) { memset(data.iov_base, 0xBB, data.iov_len); #endif /* MDBX_DEBUG && (ENABLE_MEMCHECK || __SANITIZE_ADDRESS__) */ - if (retired_pages_before == MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { + if (retired_pages_before == MDBX_PNL_GETSIZE(txn->wr.retired_pages)) { const size_t at = (is_lifo(txn) == MDBX_PNL_ASCENDING) ? left - chunk : ctx->retired_stored; - pgno_t *const begin = txn->tw.retired_pages + at; + pgno_t *const begin = txn->wr.retired_pages + at; /* MDBX_PNL_ASCENDING == false && LIFO == false: * - the larger pgno is at the beginning of retired list * and should be placed with the larger txnid. @@ -310,15 +310,15 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) { ctx->bigfoot, (unsigned)(ctx->bigfoot - txn->txnid), chunk, at, at + chunk, retired_pages_before); } ctx->retired_stored += chunk; - } while (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages) && (++ctx->bigfoot, true)); - } while (retired_pages_before != MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + } while (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->wr.retired_pages) && (++ctx->bigfoot, true)); + } while (retired_pages_before != MDBX_PNL_GETSIZE(txn->wr.retired_pages)); #else /* Write to last page of GC */ key.iov_len = sizeof(txnid_t); key.iov_base = &txn->txnid; do { prepare_backlog(txn, ctx); - data.iov_len = MDBX_PNL_SIZEOF(txn->tw.retired_pages); + data.iov_len = MDBX_PNL_SIZEOF(txn->wr.retired_pages); err = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE); if (unlikely(err != MDBX_SUCCESS)) return err; @@ -331,13 +331,13 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) { memset(data.iov_base, 0xBB, data.iov_len); #endif /* MDBX_DEBUG && (ENABLE_MEMCHECK || __SANITIZE_ADDRESS__) */ - /* Retry if tw.retired_pages[] grew during the Put() */ - } while (data.iov_len < MDBX_PNL_SIZEOF(txn->tw.retired_pages)); + /* Retry if wr.retired_pages[] grew during the Put() */ + } while (data.iov_len < MDBX_PNL_SIZEOF(txn->wr.retired_pages)); - ctx->retired_stored = MDBX_PNL_GETSIZE(txn->tw.retired_pages); - pnl_sort(txn->tw.retired_pages, txn->geo.first_unallocated); - tASSERT(txn, data.iov_len == MDBX_PNL_SIZEOF(txn->tw.retired_pages)); - memcpy(data.iov_base, txn->tw.retired_pages, data.iov_len); + ctx->retired_stored = MDBX_PNL_GETSIZE(txn->wr.retired_pages); + pnl_sort(txn->wr.retired_pages, txn->geo.first_unallocated); + tASSERT(txn, data.iov_len == MDBX_PNL_SIZEOF(txn->wr.retired_pages)); + memcpy(data.iov_base, txn->wr.retired_pages, data.iov_len); TRACE("%s: put-retired #%zu @ %" PRIaTXN, dbg_prefix(ctx), ctx->retired_stored, txn->txnid); #endif /* MDBX_ENABLE_BIGFOOT */ @@ -345,7 +345,7 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) { size_t i = ctx->retired_stored; DEBUG_EXTRA("txn %" PRIaTXN " root %" PRIaPGNO " num %zu, retired-PNL", txn->txnid, txn->dbs[FREE_DBI].root, i); for (; i; i--) - DEBUG_EXTRA_PRINT(" %" PRIaPGNO, txn->tw.retired_pages[i]); + DEBUG_EXTRA_PRINT(" %" PRIaPGNO, txn->wr.retired_pages[i]); DEBUG_EXTRA_PRINT("%s\n", "."); } return MDBX_SUCCESS; @@ -359,15 +359,15 @@ typedef struct gcu_rid_result { static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left) { rid_t r; if (is_lifo(txn)) { - if (txn->tw.gc.retxl == nullptr) { - txn->tw.gc.retxl = txl_alloc(); - if (unlikely(!txn->tw.gc.retxl)) { + if (txn->wr.gc.retxl == nullptr) { + txn->wr.gc.retxl = txl_alloc(); + if (unlikely(!txn->wr.gc.retxl)) { r.err = MDBX_ENOMEM; goto return_error; } } - if (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) < txl_max && - left > (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page && !ctx->dense) { + if (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) < txl_max && + left > (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page && !ctx->dense) { /* Hужен свободный для для сохранения списка страниц. */ bool need_cleanup = false; txnid_t snap_oldest = 0; @@ -376,11 +376,11 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left) r.err = gc_alloc_ex(&ctx->cursor, 0, ALLOC_RESERVE).err; snap_oldest = txn->env->lck->cached_oldest.weak; if (likely(r.err == MDBX_SUCCESS)) { - TRACE("%s: took @%" PRIaTXN " from GC", dbg_prefix(ctx), MDBX_PNL_LAST(txn->tw.gc.retxl)); + TRACE("%s: took @%" PRIaTXN " from GC", dbg_prefix(ctx), MDBX_PNL_LAST(txn->wr.gc.retxl)); need_cleanup = true; } - } while (r.err == MDBX_SUCCESS && MDBX_PNL_GETSIZE(txn->tw.gc.retxl) < txl_max && - left > (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page); + } while (r.err == MDBX_SUCCESS && MDBX_PNL_GETSIZE(txn->wr.gc.retxl) < txl_max && + left > (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page); if (likely(r.err == MDBX_SUCCESS)) { TRACE("%s: got enough from GC.", dbg_prefix(ctx)); @@ -389,34 +389,34 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left) /* LY: some troubles... */ goto return_error; - if (MDBX_PNL_GETSIZE(txn->tw.gc.retxl)) { + if (MDBX_PNL_GETSIZE(txn->wr.gc.retxl)) { if (need_cleanup) { - txl_sort(txn->tw.gc.retxl); + txl_sort(txn->wr.gc.retxl); ctx->cleaned_slot = 0; } - ctx->rid = MDBX_PNL_LAST(txn->tw.gc.retxl); + ctx->rid = MDBX_PNL_LAST(txn->wr.gc.retxl); } else { - tASSERT(txn, txn->tw.gc.last_reclaimed == 0); + tASSERT(txn, txn->wr.gc.last_reclaimed == 0); if (unlikely(txn_snapshot_oldest(txn) != snap_oldest)) /* should retry gc_alloc_ex() * if the oldest reader changes since the last attempt */ goto retry_rid; /* no reclaimable GC entries, * therefore no entries with ID < mdbx_find_oldest(txn) */ - txn->tw.gc.last_reclaimed = ctx->rid = snap_oldest; + txn->wr.gc.last_reclaimed = ctx->rid = snap_oldest; TRACE("%s: none recycled yet, set rid to @%" PRIaTXN, dbg_prefix(ctx), ctx->rid); } /* В GC нет годных к переработке записей, * будем использовать свободные id в обратном порядке. */ - while (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) < txl_max && - left > (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page) { + while (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) < txl_max && + left > (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page) { if (unlikely(ctx->rid <= MIN_TXNID)) { ctx->dense = true; - if (unlikely(MDBX_PNL_GETSIZE(txn->tw.gc.retxl) <= ctx->reused_slot)) { + if (unlikely(MDBX_PNL_GETSIZE(txn->wr.gc.retxl) <= ctx->reused_slot)) { VERBOSE("** restart: reserve depleted (reused_gc_slot %zu >= " "gc.reclaimed %zu)", - ctx->reused_slot, MDBX_PNL_GETSIZE(txn->tw.gc.retxl)); + ctx->reused_slot, MDBX_PNL_GETSIZE(txn->wr.gc.retxl)); goto return_restart; } break; @@ -444,7 +444,7 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left) } tASSERT(txn, !ctx->dense); - r.err = txl_append(&txn->tw.gc.retxl, ctx->rid); + r.err = txl_append(&txn->wr.gc.retxl, ctx->rid); if (unlikely(r.err != MDBX_SUCCESS)) goto return_error; @@ -468,12 +468,12 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left) } } - const size_t i = MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot; - tASSERT(txn, i > 0 && i <= MDBX_PNL_GETSIZE(txn->tw.gc.retxl)); - r.rid = txn->tw.gc.retxl[i]; + const size_t i = MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot; + tASSERT(txn, i > 0 && i <= MDBX_PNL_GETSIZE(txn->wr.gc.retxl)); + r.rid = txn->wr.gc.retxl[i]; TRACE("%s: take @%" PRIaTXN " from lifo-reclaimed[%zu]", dbg_prefix(ctx), r.rid, i); } else { - tASSERT(txn, txn->tw.gc.retxl == nullptr); + tASSERT(txn, txn->wr.gc.retxl == nullptr); if (unlikely(ctx->rid == 0)) { ctx->rid = txn_snapshot_oldest(txn); MDBX_val key; @@ -496,7 +496,7 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left) r.rid = 0; return r; } - txn->tw.gc.last_reclaimed = ctx->rid; + txn->wr.gc.last_reclaimed = ctx->rid; ctx->cleaned_id = ctx->rid + 1; } r.rid = ctx->rid--; @@ -538,16 +538,16 @@ int gc_update(MDBX_txn *txn, gcu_t *ctx) { txn->cursors[FREE_DBI] = &ctx->cursor; int rc; - /* txn->tw.repnl[] can grow and shrink during this call. - * txn->tw.gc.last_reclaimed and txn->tw.retired_pages[] can only grow. - * But page numbers cannot disappear from txn->tw.retired_pages[]. */ + /* txn->wr.repnl[] can grow and shrink during this call. + * txn->wr.gc.last_reclaimed and txn->wr.retired_pages[] can only grow. + * But page numbers cannot disappear from txn->wr.retired_pages[]. */ retry_clean_adj: ctx->reserve_adj = 0; retry: ctx->loop += !(ctx->prev_first_unallocated > txn->geo.first_unallocated); TRACE(">> restart, loop %u", ctx->loop); - tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); tASSERT(txn, dpl_check(txn)); if (unlikely(/* paranoia */ ctx->loop > ((MDBX_DEBUG > 0) ? 12 : 42))) { ERROR("txn #%" PRIaTXN " too more loops %u, bailout", txn->txnid, ctx->loop); @@ -569,22 +569,22 @@ retry: ctx->amount = 0; ctx->fill_idx = ~0u; ctx->cleaned_id = 0; - ctx->rid = txn->tw.gc.last_reclaimed; + ctx->rid = txn->wr.gc.last_reclaimed; while (true) { /* Come back here after each Put() in case retired-list changed */ TRACE("%s", " >> continue"); - tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); MDBX_val key, data; if (is_lifo(txn)) { - if (ctx->cleaned_slot < (txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0)) { + if (ctx->cleaned_slot < (txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) : 0)) { ctx->reserved = 0; ctx->cleaned_slot = 0; ctx->reused_slot = 0; ctx->fill_idx = ~0u; /* LY: cleanup reclaimed records. */ do { - ctx->cleaned_id = txn->tw.gc.retxl[++ctx->cleaned_slot]; + ctx->cleaned_id = txn->wr.gc.retxl[++ctx->cleaned_slot]; tASSERT(txn, ctx->cleaned_slot > 0 && ctx->cleaned_id <= env->lck->cached_oldest.weak); key.iov_base = &ctx->cleaned_id; key.iov_len = sizeof(ctx->cleaned_id); @@ -602,16 +602,16 @@ retry: rc = cursor_del(&ctx->cursor, 0); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; - } while (ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->tw.gc.retxl)); - txl_sort(txn->tw.gc.retxl); + } while (ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->wr.gc.retxl)); + txl_sort(txn->wr.gc.retxl); } } else { /* Удаляем оставшиеся вынутые из GC записи. */ - while (txn->tw.gc.last_reclaimed && ctx->cleaned_id <= txn->tw.gc.last_reclaimed) { + while (txn->wr.gc.last_reclaimed && ctx->cleaned_id <= txn->wr.gc.last_reclaimed) { rc = outer_first(&ctx->cursor, &key, nullptr); if (rc == MDBX_NOTFOUND) { - ctx->cleaned_id = txn->tw.gc.last_reclaimed + 1; - ctx->rid = txn->tw.gc.last_reclaimed; + ctx->cleaned_id = txn->wr.gc.last_reclaimed + 1; + ctx->rid = txn->wr.gc.last_reclaimed; ctx->reserved = 0; ctx->reused_slot = 0; break; @@ -629,12 +629,12 @@ retry: ctx->reused_slot = 0; } ctx->cleaned_id = unaligned_peek_u64(4, key.iov_base); - if (ctx->cleaned_id > txn->tw.gc.last_reclaimed) + if (ctx->cleaned_id > txn->wr.gc.last_reclaimed) break; rc = prepare_backlog(txn, ctx); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; - tASSERT(txn, ctx->cleaned_id <= txn->tw.gc.last_reclaimed); + tASSERT(txn, ctx->cleaned_id <= txn->wr.gc.last_reclaimed); tASSERT(txn, ctx->cleaned_id <= env->lck->cached_oldest.weak); TRACE("%s: cleanup-reclaimed-id %" PRIaTXN, dbg_prefix(ctx), ctx->cleaned_id); tASSERT(txn, *txn->cursors == &ctx->cursor); @@ -644,7 +644,7 @@ retry: } } - tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); tASSERT(txn, dpl_check(txn)); if (AUDIT_ENABLED()) { rc = audit_ex(txn, ctx->retired_stored, false); @@ -654,7 +654,7 @@ retry: /* return suitable into unallocated space */ if (txn_refund(txn)) { - tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); if (AUDIT_ENABLED()) { rc = audit_ex(txn, ctx->retired_stored, false); if (unlikely(rc != MDBX_SUCCESS)) @@ -662,7 +662,7 @@ retry: } } - if (txn->tw.loose_pages) { + if (txn->wr.loose_pages) { /* put loose pages into the reclaimed- or retired-list */ rc = gcu_loose(txn, ctx); if (unlikely(rc != MDBX_SUCCESS)) { @@ -670,19 +670,19 @@ retry: continue; goto bailout; } - tASSERT(txn, txn->tw.loose_pages == 0); + tASSERT(txn, txn->wr.loose_pages == 0); } - if (unlikely(ctx->reserved > MDBX_PNL_GETSIZE(txn->tw.repnl)) && - (ctx->loop < 5 || ctx->reserved - MDBX_PNL_GETSIZE(txn->tw.repnl) > env->maxgc_large1page / 2)) { + if (unlikely(ctx->reserved > MDBX_PNL_GETSIZE(txn->wr.repnl)) && + (ctx->loop < 5 || ctx->reserved - MDBX_PNL_GETSIZE(txn->wr.repnl) > env->maxgc_large1page / 2)) { TRACE("%s: reclaimed-list changed %zu -> %zu, retry", dbg_prefix(ctx), ctx->amount, - MDBX_PNL_GETSIZE(txn->tw.repnl)); - ctx->reserve_adj += ctx->reserved - MDBX_PNL_GETSIZE(txn->tw.repnl); + MDBX_PNL_GETSIZE(txn->wr.repnl)); + ctx->reserve_adj += ctx->reserved - MDBX_PNL_GETSIZE(txn->wr.repnl); goto retry; } - ctx->amount = MDBX_PNL_GETSIZE(txn->tw.repnl); + ctx->amount = MDBX_PNL_GETSIZE(txn->wr.repnl); - if (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { + if (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->wr.retired_pages)) { /* store retired-list into GC */ rc = gcu_retired(txn, ctx); if (unlikely(rc != MDBX_SUCCESS)) @@ -690,8 +690,8 @@ retry: continue; } - tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); - tASSERT(txn, txn->tw.loose_count == 0); + tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + tASSERT(txn, txn->wr.loose_count == 0); TRACE("%s", " >> reserving"); if (AUDIT_ENABLED()) { @@ -704,7 +704,7 @@ retry: "lifo-reclaimed-slots %zu, " "reused-gc-slots %zu", dbg_prefix(ctx), ctx->amount, ctx->reserved, ctx->reserve_adj, left, - txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0, ctx->reused_slot); + txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) : 0, ctx->reused_slot); if (0 >= (intptr_t)left) break; @@ -722,7 +722,7 @@ retry: size_t chunk = left; if (unlikely(left > env->maxgc_large1page)) { - const size_t avail_gc_slots = txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot + 1 + const size_t avail_gc_slots = txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot + 1 : (ctx->rid < INT16_MAX) ? (size_t)ctx->rid : INT16_MAX; if (likely(avail_gc_slots > 1)) { @@ -749,8 +749,8 @@ retry: size_t avail = ((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) /* - 1 + span */; if (tail > avail) { for (size_t i = ctx->amount - span; i > 0; --i) { - if (MDBX_PNL_ASCENDING ? (txn->tw.repnl[i] + span) - : (txn->tw.repnl[i] - span) == txn->tw.repnl[i + span]) { + if (MDBX_PNL_ASCENDING ? (txn->wr.repnl[i] + span) + : (txn->wr.repnl[i] - span) == txn->wr.repnl[i + span]) { span += 1; avail = ((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) - 1 + span; if (avail >= tail) @@ -791,7 +791,7 @@ retry: ctx->reserved + chunk + 1, reservation_gc_id); prepare_backlog(txn, ctx); rc = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE | MDBX_NOOVERWRITE); - tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; @@ -802,14 +802,14 @@ retry: continue; } - tASSERT(txn, ctx->cleaned_slot == (txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0)); + tASSERT(txn, ctx->cleaned_slot == (txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) : 0)); TRACE("%s", " >> filling"); /* Fill in the reserved records */ size_t excess_slots = 0; - ctx->fill_idx = txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot : ctx->reused_slot; + ctx->fill_idx = txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot : ctx->reused_slot; rc = MDBX_SUCCESS; - tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); tASSERT(txn, dpl_check(txn)); if (ctx->amount) { MDBX_val key, data; @@ -817,7 +817,7 @@ retry: key.iov_base = data.iov_base = nullptr; size_t left = ctx->amount, excess = 0; - if (txn->tw.gc.retxl == nullptr) { + if (txn->wr.gc.retxl == nullptr) { tASSERT(txn, is_lifo(txn) == 0); rc = outer_first(&ctx->cursor, &key, &data); if (unlikely(rc != MDBX_SUCCESS)) { @@ -830,33 +830,33 @@ retry: while (true) { txnid_t fill_gc_id; - TRACE("%s: left %zu of %zu", dbg_prefix(ctx), left, MDBX_PNL_GETSIZE(txn->tw.repnl)); - if (txn->tw.gc.retxl == nullptr) { + TRACE("%s: left %zu of %zu", dbg_prefix(ctx), left, MDBX_PNL_GETSIZE(txn->wr.repnl)); + if (txn->wr.gc.retxl == nullptr) { tASSERT(txn, is_lifo(txn) == 0); fill_gc_id = key.iov_base ? unaligned_peek_u64(4, key.iov_base) : MIN_TXNID; - if (ctx->fill_idx == 0 || fill_gc_id > txn->tw.gc.last_reclaimed) { + if (ctx->fill_idx == 0 || fill_gc_id > txn->wr.gc.last_reclaimed) { if (!left) break; VERBOSE("** restart: reserve depleted (fill_idx %zu, fill_id %" PRIaTXN " > last_reclaimed %" PRIaTXN ", left %zu", - ctx->fill_idx, fill_gc_id, txn->tw.gc.last_reclaimed, left); + ctx->fill_idx, fill_gc_id, txn->wr.gc.last_reclaimed, left); ctx->reserve_adj = (ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0; goto retry; } ctx->fill_idx -= 1; } else { tASSERT(txn, is_lifo(txn) != 0); - if (ctx->fill_idx >= MDBX_PNL_GETSIZE(txn->tw.gc.retxl)) { + if (ctx->fill_idx >= MDBX_PNL_GETSIZE(txn->wr.gc.retxl)) { if (!left) break; VERBOSE("** restart: reserve depleted (fill_idx %zu >= " "gc.reclaimed %zu, left %zu", - ctx->fill_idx, MDBX_PNL_GETSIZE(txn->tw.gc.retxl), left); + ctx->fill_idx, MDBX_PNL_GETSIZE(txn->wr.gc.retxl), left); ctx->reserve_adj = (ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0; goto retry; } ctx->fill_idx += 1; - fill_gc_id = txn->tw.gc.retxl[ctx->fill_idx]; + fill_gc_id = txn->wr.gc.retxl[ctx->fill_idx]; TRACE("%s: seek-reservation @%" PRIaTXN " at gc.reclaimed[%zu]", dbg_prefix(ctx), fill_gc_id, ctx->fill_idx); key.iov_base = &fill_gc_id; key.iov_len = sizeof(fill_gc_id); @@ -864,7 +864,7 @@ retry: if (unlikely(rc != MDBX_SUCCESS)) goto bailout; } - tASSERT(txn, ctx->cleaned_slot == (txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0)); + tASSERT(txn, ctx->cleaned_slot == (txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) : 0)); tASSERT(txn, fill_gc_id > 0 && fill_gc_id <= env->lck->cached_oldest.weak); key.iov_base = &fill_gc_id; key.iov_len = sizeof(fill_gc_id); @@ -888,33 +888,33 @@ retry: goto bailout; zeroize_reserved(env, data); - if (unlikely(txn->tw.loose_count || ctx->amount != MDBX_PNL_GETSIZE(txn->tw.repnl))) { + if (unlikely(txn->wr.loose_count || ctx->amount != MDBX_PNL_GETSIZE(txn->wr.repnl))) { NOTICE("** restart: reclaimed-list changed (%zu -> %zu, loose +%zu)", ctx->amount, - MDBX_PNL_GETSIZE(txn->tw.repnl), txn->tw.loose_count); + MDBX_PNL_GETSIZE(txn->wr.repnl), txn->wr.loose_count); if (ctx->loop < 5 || (ctx->loop > 10 && (ctx->loop & 1))) goto retry_clean_adj; goto retry; } - if (unlikely(txn->tw.gc.retxl ? ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - : ctx->cleaned_id < txn->tw.gc.last_reclaimed)) { + if (unlikely(txn->wr.gc.retxl ? ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->wr.gc.retxl) + : ctx->cleaned_id < txn->wr.gc.last_reclaimed)) { NOTICE("%s", "** restart: reclaimed-slots changed"); goto retry; } - if (unlikely(ctx->retired_stored != MDBX_PNL_GETSIZE(txn->tw.retired_pages))) { - tASSERT(txn, ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + if (unlikely(ctx->retired_stored != MDBX_PNL_GETSIZE(txn->wr.retired_pages))) { + tASSERT(txn, ctx->retired_stored < MDBX_PNL_GETSIZE(txn->wr.retired_pages)); NOTICE("** restart: retired-list growth (%zu -> %zu)", ctx->retired_stored, - MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + MDBX_PNL_GETSIZE(txn->wr.retired_pages)); goto retry; } pgno_t *dst = data.iov_base; *dst++ = (pgno_t)chunk; - pgno_t *src = MDBX_PNL_BEGIN(txn->tw.repnl) + left - chunk; + pgno_t *src = MDBX_PNL_BEGIN(txn->wr.repnl) + left - chunk; memcpy(dst, src, chunk * sizeof(pgno_t)); pgno_t *from = src, *to = src + chunk; TRACE("%s: fill %zu [ %zu:%" PRIaPGNO "...%zu:%" PRIaPGNO "] @%" PRIaTXN, dbg_prefix(ctx), chunk, - from - txn->tw.repnl, from[0], to - txn->tw.repnl, to[-1], fill_gc_id); + from - txn->wr.repnl, from[0], to - txn->wr.repnl, to[-1], fill_gc_id); left -= chunk; if (AUDIT_ENABLED()) { @@ -925,7 +925,7 @@ retry: next: - if (txn->tw.gc.retxl == nullptr) { + if (txn->wr.gc.retxl == nullptr) { tASSERT(txn, is_lifo(txn) == 0); rc = outer_next(&ctx->cursor, &key, &data, MDBX_NEXT); if (unlikely(rc != MDBX_SUCCESS)) { @@ -950,9 +950,9 @@ retry: } tASSERT(txn, rc == MDBX_SUCCESS); - if (unlikely(txn->tw.loose_count != 0 || ctx->amount != MDBX_PNL_GETSIZE(txn->tw.repnl))) { - NOTICE("** restart: got %zu loose pages (reclaimed-list %zu -> %zu)", txn->tw.loose_count, ctx->amount, - MDBX_PNL_GETSIZE(txn->tw.repnl)); + if (unlikely(txn->wr.loose_count != 0 || ctx->amount != MDBX_PNL_GETSIZE(txn->wr.repnl))) { + NOTICE("** restart: got %zu loose pages (reclaimed-list %zu -> %zu)", txn->wr.loose_count, ctx->amount, + MDBX_PNL_GETSIZE(txn->wr.repnl)); goto retry; } @@ -965,12 +965,12 @@ retry: goto retry; } - tASSERT(txn, txn->tw.gc.retxl == nullptr || ctx->cleaned_slot == MDBX_PNL_GETSIZE(txn->tw.gc.retxl)); + tASSERT(txn, txn->wr.gc.retxl == nullptr || ctx->cleaned_slot == MDBX_PNL_GETSIZE(txn->wr.gc.retxl)); bailout: txn->cursors[FREE_DBI] = ctx->cursor.next; - MDBX_PNL_SETSIZE(txn->tw.repnl, 0); + MDBX_PNL_SETSIZE(txn->wr.repnl, 0); #if MDBX_ENABLE_PROFGC env->lck->pgops.gc_prof.wloops += (uint32_t)ctx->loop; #endif /* MDBX_ENABLE_PROFGC */ diff --git a/src/internals.h b/src/internals.h index a6a15800..dda95539 100644 --- a/src/internals.h +++ b/src/internals.h @@ -206,9 +206,9 @@ struct MDBX_txn { union { struct { - /* For read txns: This thread/txn's reader table slot, or nullptr. */ - reader_slot_t *reader; - } to; + /* For read txns: This thread/txn's slot table slot, or nullptr. */ + reader_slot_t *slot; + } ro; struct { troika_t troika; pnl_t __restrict repnl; /* Reclaimed GC pages */ @@ -236,7 +236,7 @@ struct MDBX_txn { /* The list of loose pages that became unused and may be reused * in this transaction, linked through `page_next()`. */ page_t *__restrict loose_pages; - /* Number of loose pages (tw.loose_pages) */ + /* Number of loose pages (wr.loose_pages) */ size_t loose_count; union { struct { @@ -250,7 +250,7 @@ struct MDBX_txn { size_t writemap_spilled_npages; }; /* In write txns, next is located the array of cursors for each DB */ - } tw; + } wr; }; }; diff --git a/src/meta.c b/src/meta.c index cbdea2d0..9a5a4ff2 100644 --- a/src/meta.c +++ b/src/meta.c @@ -252,9 +252,9 @@ __cold int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto) { /* force oldest refresh */ atomic_store32(&env->lck->rdt_refresh_flag, true, mo_Relaxed); - env->basal_txn->tw.troika = meta_tap(env); + env->basal_txn->wr.troika = meta_tap(env); for (MDBX_txn *scan = env->basal_txn->nested; scan; scan = scan->nested) - scan->tw.troika = env->basal_txn->tw.troika; + scan->wr.troika = env->basal_txn->wr.troika; return err; } diff --git a/src/mvcc-readers.c b/src/mvcc-readers.c index 6db3b8d2..39d09f45 100644 --- a/src/mvcc-readers.c +++ b/src/mvcc-readers.c @@ -308,7 +308,7 @@ __cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) { bool notify_eof_of_loop = false; int retry = 0; do { - const txnid_t steady = env->txn->tw.troika.txnid[env->txn->tw.troika.prefer_steady]; + const txnid_t steady = env->txn->wr.troika.txnid[env->txn->wr.troika.prefer_steady]; env->lck->rdt_refresh_flag.weak = /* force refresh */ true; oldest = mvcc_shapshot_oldest(env, steady); eASSERT(env, oldest < env->basal_txn->txnid); @@ -374,7 +374,7 @@ __cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) { if (safe64_read(&stucked->txnid) != straggler || !pid) continue; - const meta_ptr_t head = meta_recent(env, &env->txn->tw.troika); + const meta_ptr_t head = meta_recent(env, &env->txn->wr.troika); const txnid_t gap = (head.txnid - straggler) / xMDBX_TXNID_STEP; const uint64_t head_retired = unaligned_peek_u64(4, head.ptr_c->pages_retired); const size_t space = (head_retired > hold_retired) ? pgno2bytes(env, (pgno_t)(head_retired - hold_retired)) : 0; diff --git a/src/page-get.c b/src/page-get.c index a07768a7..b3f5e081 100644 --- a/src/page-get.c +++ b/src/page-get.c @@ -443,8 +443,8 @@ static __always_inline pgr_t page_get_inline(const uint16_t ILL, const MDBX_curs const size_t i = dpl_search(spiller, pgno); tASSERT(txn, (intptr_t)i > 0); - if (spiller->tw.dirtylist->items[i].pgno == pgno) { - r.page = spiller->tw.dirtylist->items[i].ptr; + if (spiller->wr.dirtylist->items[i].pgno == pgno) { + r.page = spiller->wr.dirtylist->items[i].ptr; break; } diff --git a/src/page-ops.c b/src/page-ops.c index 6cecd119..84478f28 100644 --- a/src/page-ops.c +++ b/src/page-ops.c @@ -144,14 +144,14 @@ __cold pgr_t __must_check_result page_unspill(MDBX_txn *const txn, const page_t } __hot int page_touch_modifable(MDBX_txn *txn, const page_t *const mp) { - tASSERT(txn, is_modifable(txn, mp) && txn->tw.dirtylist); + tASSERT(txn, is_modifable(txn, mp) && txn->wr.dirtylist); tASSERT(txn, !is_largepage(mp) && !is_subpage(mp)); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); const size_t n = dpl_search(txn, mp->pgno); - if (MDBX_AVOID_MSYNC && unlikely(txn->tw.dirtylist->items[n].pgno != mp->pgno)) { + if (MDBX_AVOID_MSYNC && unlikely(txn->wr.dirtylist->items[n].pgno != mp->pgno)) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP)); - tASSERT(txn, n > 0 && n <= txn->tw.dirtylist->length + 1); + tASSERT(txn, n > 0 && n <= txn->wr.dirtylist->length + 1); VERBOSE("unspill page %" PRIaPGNO, mp->pgno); #if MDBX_ENABLE_PGOP_STAT txn->env->lck->pgops.unspill.weak += 1; @@ -159,11 +159,11 @@ __hot int page_touch_modifable(MDBX_txn *txn, const page_t *const mp) { return page_dirty(txn, (page_t *)mp, 1); } - tASSERT(txn, n > 0 && n <= txn->tw.dirtylist->length); - tASSERT(txn, txn->tw.dirtylist->items[n].pgno == mp->pgno && txn->tw.dirtylist->items[n].ptr == mp); + tASSERT(txn, n > 0 && n <= txn->wr.dirtylist->length); + tASSERT(txn, txn->wr.dirtylist->items[n].pgno == mp->pgno && txn->wr.dirtylist->items[n].ptr == mp); if (!MDBX_AVOID_MSYNC || (txn->flags & MDBX_WRITEMAP) == 0) { - size_t *const ptr = ptr_disp(txn->tw.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t)); - *ptr = txn->tw.dirtylru; + size_t *const ptr = ptr_disp(txn->wr.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t)); + *ptr = txn->wr.dirtylru; } return MDBX_SUCCESS; } @@ -179,7 +179,7 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *c page_t *np; if (is_frozen(txn, mp)) { /* CoW the page */ - rc = pnl_need(&txn->tw.retired_pages, 1); + rc = pnl_need(&txn->wr.retired_pages, 1); if (unlikely(rc != MDBX_SUCCESS)) goto fail; const pgr_t par = gc_alloc_single(mc); @@ -191,7 +191,7 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *c const pgno_t pgno = np->pgno; DEBUG("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno, pgno); tASSERT(txn, mp->pgno != pgno); - pnl_append_prereserved(txn->tw.retired_pages, mp->pgno); + pnl_append_prereserved(txn->wr.retired_pages, mp->pgno); /* Update the parent page, if any, to point to the new page */ if (likely(mc->top)) { page_t *parent = mc->pg[mc->top - 1]; @@ -227,7 +227,7 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *c } DEBUG("clone db %d page %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno); - tASSERT(txn, txn->tw.dirtylist->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE); + tASSERT(txn, txn->wr.dirtylist->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE); /* No - copy it */ np = page_shadow_alloc(txn, 1); if (unlikely(!np)) { @@ -369,7 +369,7 @@ static inline bool suitable4loose(const MDBX_txn *txn, pgno_t pgno) { * страница не примыкает к какой-либо из уже находящийся в reclaimed. * 2) стоит подумать над тем, чтобы при большом loose-списке отбрасывать половину в reclaimed. */ - return txn->tw.loose_count < txn->env->options.dp_loose_limit && + return txn->wr.loose_count < txn->env->options.dp_loose_limit && (!MDBX_ENABLE_REFUND || /* skip pages near to the end in favor of compactification */ txn->geo.first_unallocated > pgno + txn->env->options.dp_loose_limit || @@ -417,14 +417,14 @@ int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null status = frozen; if (ASSERT_ENABLED()) { for (MDBX_txn *scan = txn; scan; scan = scan->parent) { - tASSERT(txn, !txn->tw.spilled.list || !spill_search(scan, pgno)); - tASSERT(txn, !scan->tw.dirtylist || !debug_dpl_find(scan, pgno)); + tASSERT(txn, !txn->wr.spilled.list || !spill_search(scan, pgno)); + tASSERT(txn, !scan->wr.dirtylist || !debug_dpl_find(scan, pgno)); } } goto status_done; - } else if (pageflags && txn->tw.dirtylist) { + } else if (pageflags && txn->wr.dirtylist) { if ((di = dpl_exist(txn, pgno)) != 0) { - mp = txn->tw.dirtylist->items[di].ptr; + mp = txn->wr.dirtylist->items[di].ptr; tASSERT(txn, is_modifable(txn, mp)); status = modifable; goto status_done; @@ -461,16 +461,16 @@ int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null tASSERT(txn, !is_spilled(txn, mp)); tASSERT(txn, !is_shadowed(txn, mp)); tASSERT(txn, !debug_dpl_find(txn, pgno)); - tASSERT(txn, !txn->tw.spilled.list || !spill_search(txn, pgno)); + tASSERT(txn, !txn->wr.spilled.list || !spill_search(txn, pgno)); } else if (is_modifable(txn, mp)) { status = modifable; - if (txn->tw.dirtylist) + if (txn->wr.dirtylist) di = dpl_exist(txn, pgno); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) || !is_spilled(txn, mp)); - tASSERT(txn, !txn->tw.spilled.list || !spill_search(txn, pgno)); + tASSERT(txn, !txn->wr.spilled.list || !spill_search(txn, pgno)); } else if (is_shadowed(txn, mp)) { status = shadowed; - tASSERT(txn, !txn->tw.spilled.list || !spill_search(txn, pgno)); + tASSERT(txn, !txn->wr.spilled.list || !spill_search(txn, pgno)); tASSERT(txn, !debug_dpl_find(txn, pgno)); } else { tASSERT(txn, is_spilled(txn, mp)); @@ -504,7 +504,7 @@ status_done: if (status == frozen) { retire: DEBUG("retire %zu page %" PRIaPGNO, npages, pgno); - rc = pnl_append_span(&txn->tw.retired_pages, pgno, npages); + rc = pnl_append_span(&txn->wr.retired_pages, pgno, npages); tASSERT(txn, dpl_check(txn)); return rc; } @@ -560,17 +560,17 @@ status_done: if (status == modifable) { /* Dirty page from this transaction */ /* If suitable we can reuse it through loose list */ - if (likely(npages == 1 && suitable4loose(txn, pgno)) && (di || !txn->tw.dirtylist)) { + if (likely(npages == 1 && suitable4loose(txn, pgno)) && (di || !txn->wr.dirtylist)) { DEBUG("loosen dirty page %" PRIaPGNO, pgno); if (MDBX_DEBUG != 0 || unlikely(txn->env->flags & MDBX_PAGEPERTURB)) memset(page_data(mp), -1, txn->env->ps - PAGEHDRSZ); mp->txnid = INVALID_TXNID; mp->flags = P_LOOSE; - page_next(mp) = txn->tw.loose_pages; - txn->tw.loose_pages = mp; - txn->tw.loose_count++; + page_next(mp) = txn->wr.loose_pages; + txn->wr.loose_pages = mp; + txn->wr.loose_count++; #if MDBX_ENABLE_REFUND - txn->tw.loose_refund_wl = (pgno + 2 > txn->tw.loose_refund_wl) ? pgno + 2 : txn->tw.loose_refund_wl; + txn->wr.loose_refund_wl = (pgno + 2 > txn->wr.loose_refund_wl) ? pgno + 2 : txn->wr.loose_refund_wl; #endif /* MDBX_ENABLE_REFUND */ VALGRIND_MAKE_MEM_NOACCESS(page_data(mp), txn->env->ps - PAGEHDRSZ); MDBX_ASAN_POISON_MEMORY_REGION(page_data(mp), txn->env->ps - PAGEHDRSZ); @@ -608,8 +608,8 @@ status_done: reclaim: DEBUG("reclaim %zu %s page %" PRIaPGNO, npages, "dirty", pgno); - rc = pnl_insert_span(&txn->tw.repnl, pgno, npages); - tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + rc = pnl_insert_span(&txn->wr.repnl, pgno, npages); + tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); tASSERT(txn, dpl_check(txn)); return rc; } @@ -660,10 +660,10 @@ status_done: __hot int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, size_t npages) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); mp->txnid = txn->front_txnid; - if (!txn->tw.dirtylist) { + if (!txn->wr.dirtylist) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); - txn->tw.writemap_dirty_npages += npages; - tASSERT(txn, txn->tw.spilled.list == nullptr); + txn->wr.writemap_dirty_npages += npages; + tASSERT(txn, txn->wr.spilled.list == nullptr); return MDBX_SUCCESS; } tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); @@ -671,29 +671,29 @@ __hot int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, size_t npage #if xMDBX_DEBUG_SPILLING == 2 txn->env->debug_dirtied_act += 1; ENSURE(txn->env, txn->env->debug_dirtied_act < txn->env->debug_dirtied_est); - ENSURE(txn->env, txn->tw.dirtyroom + txn->tw.loose_count > 0); + ENSURE(txn->env, txn->wr.dirtyroom + txn->wr.loose_count > 0); #endif /* xMDBX_DEBUG_SPILLING == 2 */ int rc; - if (unlikely(txn->tw.dirtyroom == 0)) { - if (txn->tw.loose_count) { - page_t *lp = txn->tw.loose_pages; + if (unlikely(txn->wr.dirtyroom == 0)) { + if (txn->wr.loose_count) { + page_t *lp = txn->wr.loose_pages; DEBUG("purge-and-reclaim loose page %" PRIaPGNO, lp->pgno); - rc = pnl_insert_span(&txn->tw.repnl, lp->pgno, 1); + rc = pnl_insert_span(&txn->wr.repnl, lp->pgno, 1); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; size_t di = dpl_search(txn, lp->pgno); - tASSERT(txn, txn->tw.dirtylist->items[di].ptr == lp); + tASSERT(txn, txn->wr.dirtylist->items[di].ptr == lp); dpl_remove(txn, di); MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *)); VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); - txn->tw.loose_pages = page_next(lp); - txn->tw.loose_count--; - txn->tw.dirtyroom++; + txn->wr.loose_pages = page_next(lp); + txn->wr.loose_count--; + txn->wr.dirtyroom++; if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP)) page_shadow_release(txn->env, lp, 1); } else { - ERROR("Dirtyroom is depleted, DPL length %zu", txn->tw.dirtylist->length); + ERROR("Dirtyroom is depleted, DPL length %zu", txn->wr.dirtylist->length); if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP)) page_shadow_release(txn->env, mp, npages); return MDBX_TXN_FULL; @@ -706,7 +706,7 @@ __hot int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, size_t npage txn->flags |= MDBX_TXN_ERROR; return rc; } - txn->tw.dirtyroom--; + txn->wr.dirtyroom--; tASSERT(txn, dpl_check(txn)); return MDBX_SUCCESS; } diff --git a/src/page-ops.h b/src/page-ops.h index bb4ebd16..3f6aee95 100644 --- a/src/page-ops.h +++ b/src/page-ops.h @@ -88,7 +88,7 @@ static inline int page_touch(MDBX_cursor *mc) { } if (is_modifable(txn, mp)) { - if (!txn->tw.dirtylist) { + if (!txn->wr.dirtylist) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) && !MDBX_AVOID_MSYNC); return MDBX_SUCCESS; } @@ -114,14 +114,14 @@ static inline void page_wash(MDBX_txn *txn, size_t di, page_t *const mp, const s mp->txnid = INVALID_TXNID; mp->flags = P_BAD; - if (txn->tw.dirtylist) { + if (txn->wr.dirtylist) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - tASSERT(txn, MDBX_AVOID_MSYNC || (di && txn->tw.dirtylist->items[di].ptr == mp)); + tASSERT(txn, MDBX_AVOID_MSYNC || (di && txn->wr.dirtylist->items[di].ptr == mp)); if (!MDBX_AVOID_MSYNC || di) { dpl_remove_ex(txn, di, npages); - txn->tw.dirtyroom++; - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); + txn->wr.dirtyroom++; + tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length == + (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit)); if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP)) { page_shadow_release(txn->env, mp, npages); return; @@ -129,7 +129,7 @@ static inline void page_wash(MDBX_txn *txn, size_t di, page_t *const mp, const s } } else { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) && !MDBX_AVOID_MSYNC && !di); - txn->tw.writemap_dirty_npages -= (txn->tw.writemap_dirty_npages > npages) ? npages : txn->tw.writemap_dirty_npages; + txn->wr.writemap_dirty_npages -= (txn->wr.writemap_dirty_npages > npages) ? npages : txn->wr.writemap_dirty_npages; } VALGRIND_MAKE_MEM_UNDEFINED(mp, PAGEHDRSZ); VALGRIND_MAKE_MEM_NOACCESS(page_data(mp), pgno2bytes(txn->env, npages) - PAGEHDRSZ); diff --git a/src/refund.c b/src/refund.c index 3d2fc368..365c9e71 100644 --- a/src/refund.c +++ b/src/refund.c @@ -7,7 +7,7 @@ static void refund_reclaimed(MDBX_txn *txn) { /* Scanning in descend order */ pgno_t first_unallocated = txn->geo.first_unallocated; - const pnl_t pnl = txn->tw.repnl; + const pnl_t pnl = txn->wr.repnl; tASSERT(txn, MDBX_PNL_GETSIZE(pnl) && MDBX_PNL_MOST(pnl) == first_unallocated - 1); #if MDBX_PNL_ASCENDING size_t i = MDBX_PNL_GETSIZE(pnl); @@ -28,16 +28,16 @@ static void refund_reclaimed(MDBX_txn *txn) { VERBOSE("refunded %" PRIaPGNO " pages: %" PRIaPGNO " -> %" PRIaPGNO, txn->geo.first_unallocated - first_unallocated, txn->geo.first_unallocated, first_unallocated); txn->geo.first_unallocated = first_unallocated; - tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - 1)); + tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - 1)); } static void refund_loose(MDBX_txn *txn) { - tASSERT(txn, txn->tw.loose_pages != nullptr); - tASSERT(txn, txn->tw.loose_count > 0); + tASSERT(txn, txn->wr.loose_pages != nullptr); + tASSERT(txn, txn->wr.loose_count > 0); - dpl_t *const dl = txn->tw.dirtylist; + dpl_t *const dl = txn->wr.dirtylist; if (dl) { - tASSERT(txn, dl->length >= txn->tw.loose_count); + tASSERT(txn, dl->length >= txn->wr.loose_count); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); } else { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); @@ -46,22 +46,22 @@ static void refund_loose(MDBX_txn *txn) { pgno_t onstack[MDBX_CACHELINE_SIZE * 8 / sizeof(pgno_t)]; pnl_t suitable = onstack; - if (!dl || dl->length - dl->sorted > txn->tw.loose_count) { + if (!dl || dl->length - dl->sorted > txn->wr.loose_count) { /* Dirty list is useless since unsorted. */ - if (pnl_bytes2size(sizeof(onstack)) < txn->tw.loose_count) { - suitable = pnl_alloc(txn->tw.loose_count); + if (pnl_bytes2size(sizeof(onstack)) < txn->wr.loose_count) { + suitable = pnl_alloc(txn->wr.loose_count); if (unlikely(!suitable)) return /* this is not a reason for transaction fail */; } /* Collect loose-pages which may be refunded. */ - tASSERT(txn, txn->geo.first_unallocated >= MIN_PAGENO + txn->tw.loose_count); + tASSERT(txn, txn->geo.first_unallocated >= MIN_PAGENO + txn->wr.loose_count); pgno_t most = MIN_PAGENO; size_t w = 0; - for (const page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) { + for (const page_t *lp = txn->wr.loose_pages; lp; lp = page_next(lp)) { tASSERT(txn, lp->flags == P_LOOSE); tASSERT(txn, txn->geo.first_unallocated > lp->pgno); - if (likely(txn->geo.first_unallocated - txn->tw.loose_count <= lp->pgno)) { + if (likely(txn->geo.first_unallocated - txn->wr.loose_count <= lp->pgno)) { tASSERT(txn, w < ((suitable == onstack) ? pnl_bytes2size(sizeof(onstack)) : MDBX_PNL_ALLOCLEN(suitable))); suitable[++w] = lp->pgno; most = (lp->pgno > most) ? lp->pgno : most; @@ -90,11 +90,11 @@ static void refund_loose(MDBX_txn *txn) { const size_t refunded = txn->geo.first_unallocated - most; DEBUG("refund-suitable %zu pages %" PRIaPGNO " -> %" PRIaPGNO, refunded, most, txn->geo.first_unallocated); txn->geo.first_unallocated = most; - txn->tw.loose_count -= refunded; + txn->wr.loose_count -= refunded; if (dl) { - txn->tw.dirtyroom += refunded; + txn->wr.dirtyroom += refunded; dl->pages_including_loose -= refunded; - assert(txn->tw.dirtyroom <= txn->env->options.dp_limit); + assert(txn->wr.dirtyroom <= txn->env->options.dp_limit); /* Filter-out dirty list */ size_t r = 0; @@ -115,8 +115,8 @@ static void refund_loose(MDBX_txn *txn) { } } dpl_setlen(dl, w); - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); + tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length == + (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit)); } goto unlink_loose; } @@ -141,15 +141,15 @@ static void refund_loose(MDBX_txn *txn) { if (dl->sorted != dl->length) { const size_t refunded = dl->sorted - dl->length; dl->sorted = dl->length; - txn->tw.loose_count -= refunded; - txn->tw.dirtyroom += refunded; + txn->wr.loose_count -= refunded; + txn->wr.dirtyroom += refunded; dl->pages_including_loose -= refunded; - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); + tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length == + (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit)); /* Filter-out loose chain & dispose refunded pages. */ unlink_loose: - for (page_t *__restrict *__restrict link = &txn->tw.loose_pages; *link;) { + for (page_t *__restrict *__restrict link = &txn->wr.loose_pages; *link;) { page_t *dp = *link; tASSERT(txn, dp->flags == P_LOOSE); MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(dp), sizeof(page_t *)); @@ -168,21 +168,21 @@ static void refund_loose(MDBX_txn *txn) { tASSERT(txn, dpl_check(txn)); if (suitable != onstack) pnl_free(suitable); - txn->tw.loose_refund_wl = txn->geo.first_unallocated; + txn->wr.loose_refund_wl = txn->geo.first_unallocated; } bool txn_refund(MDBX_txn *txn) { const pgno_t before = txn->geo.first_unallocated; - if (txn->tw.loose_pages && txn->tw.loose_refund_wl > txn->geo.first_unallocated) + if (txn->wr.loose_pages && txn->wr.loose_refund_wl > txn->geo.first_unallocated) refund_loose(txn); while (true) { - if (MDBX_PNL_GETSIZE(txn->tw.repnl) == 0 || MDBX_PNL_MOST(txn->tw.repnl) != txn->geo.first_unallocated - 1) + if (MDBX_PNL_GETSIZE(txn->wr.repnl) == 0 || MDBX_PNL_MOST(txn->wr.repnl) != txn->geo.first_unallocated - 1) break; refund_reclaimed(txn); - if (!txn->tw.loose_pages || txn->tw.loose_refund_wl <= txn->geo.first_unallocated) + if (!txn->wr.loose_pages || txn->wr.loose_refund_wl <= txn->geo.first_unallocated) break; const pgno_t memo = txn->geo.first_unallocated; @@ -194,7 +194,7 @@ bool txn_refund(MDBX_txn *txn) { if (before == txn->geo.first_unallocated) return false; - if (txn->tw.spilled.list) + if (txn->wr.spilled.list) /* Squash deleted pagenums if we refunded any */ spill_purge(txn); diff --git a/src/spill.c b/src/spill.c index 261adb78..d42ee950 100644 --- a/src/spill.c +++ b/src/spill.c @@ -4,42 +4,42 @@ #include "internals.h" void spill_remove(MDBX_txn *txn, size_t idx, size_t npages) { - tASSERT(txn, idx > 0 && idx <= MDBX_PNL_GETSIZE(txn->tw.spilled.list) && txn->tw.spilled.least_removed > 0); - txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed) ? idx : txn->tw.spilled.least_removed; - txn->tw.spilled.list[idx] |= 1; - MDBX_PNL_SETSIZE(txn->tw.spilled.list, - MDBX_PNL_GETSIZE(txn->tw.spilled.list) - (idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list))); + tASSERT(txn, idx > 0 && idx <= MDBX_PNL_GETSIZE(txn->wr.spilled.list) && txn->wr.spilled.least_removed > 0); + txn->wr.spilled.least_removed = (idx < txn->wr.spilled.least_removed) ? idx : txn->wr.spilled.least_removed; + txn->wr.spilled.list[idx] |= 1; + MDBX_PNL_SETSIZE(txn->wr.spilled.list, + MDBX_PNL_GETSIZE(txn->wr.spilled.list) - (idx == MDBX_PNL_GETSIZE(txn->wr.spilled.list))); while (unlikely(npages > 1)) { - const pgno_t pgno = (txn->tw.spilled.list[idx] >> 1) + 1; + const pgno_t pgno = (txn->wr.spilled.list[idx] >> 1) + 1; if (MDBX_PNL_ASCENDING) { - if (++idx > MDBX_PNL_GETSIZE(txn->tw.spilled.list) || (txn->tw.spilled.list[idx] >> 1) != pgno) + if (++idx > MDBX_PNL_GETSIZE(txn->wr.spilled.list) || (txn->wr.spilled.list[idx] >> 1) != pgno) return; } else { - if (--idx < 1 || (txn->tw.spilled.list[idx] >> 1) != pgno) + if (--idx < 1 || (txn->wr.spilled.list[idx] >> 1) != pgno) return; - txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed) ? idx : txn->tw.spilled.least_removed; + txn->wr.spilled.least_removed = (idx < txn->wr.spilled.least_removed) ? idx : txn->wr.spilled.least_removed; } - txn->tw.spilled.list[idx] |= 1; - MDBX_PNL_SETSIZE(txn->tw.spilled.list, - MDBX_PNL_GETSIZE(txn->tw.spilled.list) - (idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list))); + txn->wr.spilled.list[idx] |= 1; + MDBX_PNL_SETSIZE(txn->wr.spilled.list, + MDBX_PNL_GETSIZE(txn->wr.spilled.list) - (idx == MDBX_PNL_GETSIZE(txn->wr.spilled.list))); --npages; } } pnl_t spill_purge(MDBX_txn *txn) { - tASSERT(txn, txn->tw.spilled.least_removed > 0); - const pnl_t sl = txn->tw.spilled.list; - if (txn->tw.spilled.least_removed != INT_MAX) { + tASSERT(txn, txn->wr.spilled.least_removed > 0); + const pnl_t sl = txn->wr.spilled.list; + if (txn->wr.spilled.least_removed != INT_MAX) { size_t len = MDBX_PNL_GETSIZE(sl), r, w; - for (w = r = txn->tw.spilled.least_removed; r <= len; ++r) { + for (w = r = txn->wr.spilled.least_removed; r <= len; ++r) { sl[w] = sl[r]; w += 1 - (sl[r] & 1); } for (size_t i = 1; i < w; ++i) tASSERT(txn, (sl[i] & 1) == 0); MDBX_PNL_SETSIZE(sl, w - 1); - txn->tw.spilled.least_removed = INT_MAX; + txn->wr.spilled.least_removed = INT_MAX; } else { for (size_t i = 1; i <= MDBX_PNL_GETSIZE(sl); ++i) tASSERT(txn, (sl[i] & 1) == 0); @@ -57,7 +57,7 @@ static int spill_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, const size_t np const pgno_t pgno = dp->pgno; int err = iov_page(txn, ctx, dp, npages); if (likely(err == MDBX_SUCCESS)) - err = spill_append_span(&txn->tw.spilled.list, pgno, npages); + err = spill_append_span(&txn->wr.spilled.list, pgno, npages); return err; } @@ -75,10 +75,10 @@ static size_t spill_cursor_keep(const MDBX_txn *const txn, const MDBX_cursor *mc tASSERT(txn, !is_subpage(mp)); if (is_modifable(txn, mp)) { size_t const n = dpl_search(txn, mp->pgno); - if (txn->tw.dirtylist->items[n].pgno == mp->pgno && + if (txn->wr.dirtylist->items[n].pgno == mp->pgno && /* не считаем дважды */ dpl_age(txn, n)) { - size_t *const ptr = ptr_disp(txn->tw.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t)); - *ptr = txn->tw.dirtylru; + size_t *const ptr = ptr_disp(txn->wr.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t)); + *ptr = txn->wr.dirtylru; tASSERT(txn, dpl_age(txn, n) == 0); ++keep; } @@ -115,7 +115,7 @@ static size_t spill_txn_keep(MDBX_txn *txn, MDBX_cursor *m0) { * ... * > 255 = must not be spilled. */ MDBX_NOTHROW_PURE_FUNCTION static unsigned spill_prio(const MDBX_txn *txn, const size_t i, const uint32_t reciprocal) { - dpl_t *const dl = txn->tw.dirtylist; + dpl_t *const dl = txn->wr.dirtylist; const uint32_t age = dpl_age(txn, i); const size_t npages = dpl_npages(dl, i); const pgno_t pgno = dl->items[i].pgno; @@ -178,14 +178,14 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); int rc = MDBX_SUCCESS; - if (unlikely(txn->tw.loose_count >= - (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages))) + if (unlikely(txn->wr.loose_count >= + (txn->wr.dirtylist ? txn->wr.dirtylist->pages_including_loose : txn->wr.writemap_dirty_npages))) goto done; - const size_t dirty_entries = txn->tw.dirtylist ? (txn->tw.dirtylist->length - txn->tw.loose_count) : 1; + const size_t dirty_entries = txn->wr.dirtylist ? (txn->wr.dirtylist->length - txn->wr.loose_count) : 1; const size_t dirty_npages = - (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages) - - txn->tw.loose_count; + (txn->wr.dirtylist ? txn->wr.dirtylist->pages_including_loose : txn->wr.writemap_dirty_npages) - + txn->wr.loose_count; const size_t need_spill_entries = spill_gate(txn->env, wanna_spill_entries, dirty_entries); const size_t need_spill_npages = spill_gate(txn->env, wanna_spill_npages, dirty_npages); @@ -196,17 +196,17 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp if (txn->flags & MDBX_WRITEMAP) { NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "msync", dirty_entries, dirty_npages); const MDBX_env *env = txn->env; - tASSERT(txn, txn->tw.spilled.list == nullptr); + tASSERT(txn, txn->wr.spilled.list == nullptr); rc = osal_msync(&txn->env->dxb_mmap, 0, pgno_align2os_bytes(env, txn->geo.first_unallocated), MDBX_SYNC_KICK); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; #if MDBX_AVOID_MSYNC - MDBX_ANALYSIS_ASSUME(txn->tw.dirtylist != nullptr); + MDBX_ANALYSIS_ASSUME(txn->wr.dirtylist != nullptr); tASSERT(txn, dpl_check(txn)); - env->lck->unsynced_pages.weak += txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count; - dpl_clear(txn->tw.dirtylist); - txn->tw.dirtyroom = env->options.dp_limit - txn->tw.loose_count; - for (page_t *lp = txn->tw.loose_pages; lp != nullptr; lp = page_next(lp)) { + env->lck->unsynced_pages.weak += txn->wr.dirtylist->pages_including_loose - txn->wr.loose_count; + dpl_clear(txn->wr.dirtylist); + txn->wr.dirtyroom = env->options.dp_limit - txn->wr.loose_count; + for (page_t *lp = txn->wr.loose_pages; lp != nullptr; lp = page_next(lp)) { tASSERT(txn, lp->flags == P_LOOSE); rc = dpl_append(txn, lp->pgno, lp, 1); if (unlikely(rc != MDBX_SUCCESS)) @@ -216,22 +216,22 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp } tASSERT(txn, dpl_check(txn)); #else - tASSERT(txn, txn->tw.dirtylist == nullptr); - env->lck->unsynced_pages.weak += txn->tw.writemap_dirty_npages; - txn->tw.writemap_spilled_npages += txn->tw.writemap_dirty_npages; - txn->tw.writemap_dirty_npages = 0; + tASSERT(txn, txn->wr.dirtylist == nullptr); + env->lck->unsynced_pages.weak += txn->wr.writemap_dirty_npages; + txn->wr.writemap_spilled_npages += txn->wr.writemap_dirty_npages; + txn->wr.writemap_dirty_npages = 0; #endif /* MDBX_AVOID_MSYNC */ goto done; } NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "write", need_spill_entries, need_spill_npages); - MDBX_ANALYSIS_ASSUME(txn->tw.dirtylist != nullptr); - tASSERT(txn, txn->tw.dirtylist->length - txn->tw.loose_count >= 1); - tASSERT(txn, txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count >= need_spill_npages); - if (!txn->tw.spilled.list) { - txn->tw.spilled.least_removed = INT_MAX; - txn->tw.spilled.list = pnl_alloc(need_spill); - if (unlikely(!txn->tw.spilled.list)) { + MDBX_ANALYSIS_ASSUME(txn->wr.dirtylist != nullptr); + tASSERT(txn, txn->wr.dirtylist->length - txn->wr.loose_count >= 1); + tASSERT(txn, txn->wr.dirtylist->pages_including_loose - txn->wr.loose_count >= need_spill_npages); + if (!txn->wr.spilled.list) { + txn->wr.spilled.least_removed = INT_MAX; + txn->wr.spilled.list = pnl_alloc(need_spill); + if (unlikely(!txn->wr.spilled.list)) { rc = MDBX_ENOMEM; bailout: txn->flags |= MDBX_TXN_ERROR; @@ -240,7 +240,7 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp } else { /* purge deleted slots */ spill_purge(txn); - rc = pnl_reserve(&txn->tw.spilled.list, need_spill); + rc = pnl_reserve(&txn->wr.spilled.list, need_spill); (void)rc /* ignore since the resulting list may be shorter and pnl_append() will increase pnl on demand */ ; @@ -251,9 +251,9 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp /* Preserve pages which may soon be dirtied again */ const size_t unspillable = spill_txn_keep(txn, m0); - if (unspillable + txn->tw.loose_count >= dl->length) { + if (unspillable + txn->wr.loose_count >= dl->length) { #if xMDBX_DEBUG_SPILLING == 1 /* avoid false failure in debug mode */ - if (likely(txn->tw.dirtyroom + txn->tw.loose_count >= need)) + if (likely(txn->wr.dirtyroom + txn->wr.loose_count >= need)) return MDBX_SUCCESS; #endif /* xMDBX_DEBUG_SPILLING */ ERROR("all %zu dirty pages are unspillable since referenced " @@ -293,7 +293,7 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp age_max = (age_max >= age) ? age_max : age; } - VERBOSE("lru-head %u, age-max %u", txn->tw.dirtylru, age_max); + VERBOSE("lru-head %u, age-max %u", txn->wr.dirtylru, age_max); /* half of 8-bit radix-sort */ pgno_t radix_entries[256], radix_npages[256]; @@ -388,8 +388,8 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp tASSERT(txn, r - w == spilled_entries || rc != MDBX_SUCCESS); dl->sorted = dpl_setlen(dl, w); - txn->tw.dirtyroom += spilled_entries; - txn->tw.dirtylist->pages_including_loose -= spilled_npages; + txn->wr.dirtyroom += spilled_entries; + txn->wr.dirtylist->pages_including_loose -= spilled_npages; tASSERT(txn, dpl_check(txn)); if (!iov_empty(&ctx)) { @@ -400,10 +400,10 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp goto bailout; txn->env->lck->unsynced_pages.weak += spilled_npages; - pnl_sort(txn->tw.spilled.list, (size_t)txn->geo.first_unallocated << 1); + pnl_sort(txn->wr.spilled.list, (size_t)txn->geo.first_unallocated << 1); txn->flags |= MDBX_TXN_SPILLS; NOTICE("spilled %u dirty-entries, %u dirty-npages, now have %zu dirty-room", spilled_entries, spilled_npages, - txn->tw.dirtyroom); + txn->wr.dirtyroom); } else { tASSERT(txn, rc == MDBX_SUCCESS); for (size_t i = 1; i <= dl->length; ++i) { @@ -414,18 +414,18 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp } #if xMDBX_DEBUG_SPILLING == 2 - if (txn->tw.loose_count + txn->tw.dirtyroom <= need / 2 + 1) + if (txn->wr.loose_count + txn->wr.dirtyroom <= need / 2 + 1) ERROR("dirty-list length: before %zu, after %zu, parent %zi, loose %zu; " "needed %zu, spillable %zu; " "spilled %u dirty-entries, now have %zu dirty-room", dl->length + spilled_entries, dl->length, - (txn->parent && txn->parent->tw.dirtylist) ? (intptr_t)txn->parent->tw.dirtylist->length : -1, - txn->tw.loose_count, need, spillable_entries, spilled_entries, txn->tw.dirtyroom); - ENSURE(txn->env, txn->tw.loose_count + txn->tw.dirtyroom > need / 2); + (txn->parent && txn->parent->wr.dirtylist) ? (intptr_t)txn->parent->wr.dirtylist->length : -1, + txn->wr.loose_count, need, spillable_entries, spilled_entries, txn->wr.dirtyroom); + ENSURE(txn->env, txn->wr.loose_count + txn->wr.dirtyroom > need / 2); #endif /* xMDBX_DEBUG_SPILLING */ done: - return likely(txn->tw.dirtyroom + txn->tw.loose_count > ((need > CURSOR_STACK_SIZE) ? CURSOR_STACK_SIZE : need)) + return likely(txn->wr.dirtyroom + txn->wr.loose_count > ((need > CURSOR_STACK_SIZE) ? CURSOR_STACK_SIZE : need)) ? MDBX_SUCCESS : MDBX_TXN_FULL; } diff --git a/src/spill.h b/src/spill.h index 32a8c9b2..6db07a46 100644 --- a/src/spill.h +++ b/src/spill.h @@ -13,7 +13,7 @@ MDBX_INTERNAL int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, con static inline size_t spill_search(const MDBX_txn *txn, pgno_t pgno) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - const pnl_t pnl = txn->tw.spilled.list; + const pnl_t pnl = txn->wr.spilled.list; if (likely(!pnl)) return 0; pgno <<= 1; @@ -22,7 +22,7 @@ static inline size_t spill_search(const MDBX_txn *txn, pgno_t pgno) { } static inline bool spill_intersect(const MDBX_txn *txn, pgno_t pgno, size_t npages) { - const pnl_t pnl = txn->tw.spilled.list; + const pnl_t pnl = txn->wr.spilled.list; if (likely(!pnl)) return false; const size_t len = MDBX_PNL_GETSIZE(pnl); @@ -56,10 +56,10 @@ static inline int txn_spill(MDBX_txn *const txn, MDBX_cursor *const m0, const si tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); tASSERT(txn, !m0 || cursor_is_tracked(m0)); - const intptr_t wanna_spill_entries = txn->tw.dirtylist ? (need - txn->tw.dirtyroom - txn->tw.loose_count) : 0; + const intptr_t wanna_spill_entries = txn->wr.dirtylist ? (need - txn->wr.dirtyroom - txn->wr.loose_count) : 0; const intptr_t wanna_spill_npages = - need + (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages) - - txn->tw.loose_count - txn->env->options.dp_limit; + need + (txn->wr.dirtylist ? txn->wr.dirtylist->pages_including_loose : txn->wr.writemap_dirty_npages) - + txn->wr.loose_count - txn->env->options.dp_limit; /* production mode */ if (likely(wanna_spill_npages < 1 && wanna_spill_entries < 1) diff --git a/src/tree-ops.c b/src/tree-ops.c index 3c51a65c..68cef245 100644 --- a/src/tree-ops.c +++ b/src/tree-ops.c @@ -56,7 +56,7 @@ int tree_drop(MDBX_cursor *mc, const bool may_have_tables) { if (!(may_have_tables | mc->tree->large_pages)) cursor_pop(mc); - rc = pnl_need(&txn->tw.retired_pages, + rc = pnl_need(&txn->wr.retired_pages, (size_t)mc->tree->branch_pages + (size_t)mc->tree->leaf_pages + (size_t)mc->tree->large_pages); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; @@ -889,7 +889,7 @@ retry: goto retry; } if (likely(!involve) && - (likely(mc->tree != &mc->txn->dbs[FREE_DBI]) || mc->txn->tw.loose_pages || MDBX_PNL_GETSIZE(mc->txn->tw.repnl) || + (likely(mc->tree != &mc->txn->dbs[FREE_DBI]) || mc->txn->wr.loose_pages || MDBX_PNL_GETSIZE(mc->txn->wr.repnl) || (mc->flags & z_gcu_preparation) || (mc->txn->flags & txn_gc_drained) || room_threshold)) { involve = true; goto retry; diff --git a/src/txn-basal.c b/src/txn-basal.c index 91a6d8c4..733fcf22 100644 --- a/src/txn-basal.c +++ b/src/txn-basal.c @@ -32,16 +32,16 @@ static int txn_write(MDBX_txn *txn, iov_ctx_t *ctx) { txn->env->lck->eoos_timestamp.weak = osal_monotime(); } - txn->tw.dirtylist->pages_including_loose -= total_npages; + txn->wr.dirtylist->pages_including_loose -= total_npages; while (r <= dl->length) dl->items[++w] = dl->items[r++]; dl->sorted = dpl_setlen(dl, w); - txn->tw.dirtyroom += r - 1 - w; - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); - tASSERT(txn, txn->tw.dirtylist->length == txn->tw.loose_count); - tASSERT(txn, txn->tw.dirtylist->pages_including_loose == txn->tw.loose_count); + txn->wr.dirtyroom += r - 1 - w; + tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length == + (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit)); + tASSERT(txn, txn->wr.dirtylist->length == txn->wr.loose_count); + tASSERT(txn, txn->wr.dirtylist->pages_including_loose == txn->wr.loose_count); return rc; } @@ -70,9 +70,9 @@ __cold MDBX_txn *txn_basal_create(const size_t max_dbi) { txn->dbi_sparse = ptr_disp(txn->dbi_state, -bitmap_bytes); #endif /* MDBX_ENABLE_DBI_SPARSE */ txn->flags = MDBX_TXN_FINISHED; - txn->tw.retired_pages = pnl_alloc(MDBX_PNL_INITIAL); - txn->tw.repnl = pnl_alloc(MDBX_PNL_INITIAL); - if (unlikely(!txn->tw.retired_pages || !txn->tw.repnl)) { + txn->wr.retired_pages = pnl_alloc(MDBX_PNL_INITIAL); + txn->wr.repnl = pnl_alloc(MDBX_PNL_INITIAL); + if (unlikely(!txn->wr.retired_pages || !txn->wr.repnl)) { txn_basal_destroy(txn); txn = nullptr; } @@ -82,10 +82,10 @@ __cold MDBX_txn *txn_basal_create(const size_t max_dbi) { __cold void txn_basal_destroy(MDBX_txn *txn) { dpl_free(txn); - txl_free(txn->tw.gc.retxl); - pnl_free(txn->tw.retired_pages); - pnl_free(txn->tw.spilled.list); - pnl_free(txn->tw.repnl); + txl_free(txn->wr.gc.retxl); + pnl_free(txn->wr.retired_pages); + pnl_free(txn->wr.spilled.list); + pnl_free(txn->wr.repnl); osal_free(txn); } @@ -97,12 +97,12 @@ int txn_basal_end(MDBX_txn *txn, unsigned mode) { txn->flags = MDBX_TXN_FINISHED; env->txn = nullptr; - pnl_free(txn->tw.spilled.list); - txn->tw.spilled.list = nullptr; + pnl_free(txn->wr.spilled.list); + txn->wr.spilled.list = nullptr; eASSERT(env, txn->parent == nullptr); - pnl_shrink(&txn->tw.retired_pages); - pnl_shrink(&txn->tw.repnl); + pnl_shrink(&txn->wr.retired_pages); + pnl_shrink(&txn->wr.repnl); if (!(env->flags & MDBX_WRITEMAP)) dpl_release_shadows(txn); @@ -121,18 +121,18 @@ int txn_basal_end(MDBX_txn *txn, unsigned mode) { int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) { MDBX_env *const env = txn->env; tASSERT(txn, txn == env->basal_txn && !txn->parent && !txn->nested); - if (!txn->tw.dirtylist) { + if (!txn->wr.dirtylist) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); } else { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == env->options.dp_limit); + tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length == env->options.dp_limit); } if (txn->flags & txn_may_have_cursors) txn_done_cursors(txn); bool need_flush_for_nometasync = false; - const meta_ptr_t head = meta_recent(env, &txn->tw.troika); + const meta_ptr_t head = meta_recent(env, &txn->wr.troika); const uint32_t meta_sync_txnid = atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed); /* sync prev meta */ if (head.is_steady && meta_sync_txnid != (uint32_t)head.txnid) { @@ -175,7 +175,7 @@ int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) { } } - if ((!txn->tw.dirtylist || txn->tw.dirtylist->length == 0) && + if ((!txn->wr.dirtylist || txn->wr.dirtylist->length == 0) && (txn->flags & (MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | MDBX_TXN_NOSYNC | MDBX_TXN_NOMETASYNC)) == 0 && !need_flush_for_nometasync && !head.is_steady && !AUDIT_ENABLED()) { TXN_FOREACH_DBI_ALL(txn, i) { tASSERT(txn, !(txn->dbi_state[i] & DBI_DIRTY)); } @@ -225,7 +225,7 @@ int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) { if (unlikely(rc != MDBX_SUCCESS)) return rc; - tASSERT(txn, txn->tw.loose_count == 0); + tASSERT(txn, txn->wr.loose_count == 0); txn->dbs[FREE_DBI].mod_txnid = (txn->dbi_state[FREE_DBI] & DBI_DIRTY) ? txn->txnid : txn->dbs[FREE_DBI].mod_txnid; txn->dbs[MAIN_DBI].mod_txnid = (txn->dbi_state[MAIN_DBI] & DBI_DIRTY) ? txn->txnid : txn->dbs[MAIN_DBI].mod_txnid; @@ -234,16 +234,16 @@ int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) { ts->audit = ts->gc; } if (AUDIT_ENABLED()) { - rc = audit_ex(txn, MDBX_PNL_GETSIZE(txn->tw.retired_pages), true); + rc = audit_ex(txn, MDBX_PNL_GETSIZE(txn->wr.retired_pages), true); if (ts) ts->audit = osal_monotime(); if (unlikely(rc != MDBX_SUCCESS)) return rc; } - if (txn->tw.dirtylist) { + if (txn->wr.dirtylist) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - tASSERT(txn, txn->tw.loose_count == 0); + tASSERT(txn, txn->wr.loose_count == 0); mdbx_filehandle_t fd = #if defined(_WIN32) || defined(_WIN64) @@ -251,14 +251,14 @@ int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) { (void)need_flush_for_nometasync; #else (need_flush_for_nometasync || env->dsync_fd == INVALID_HANDLE_VALUE || - txn->tw.dirtylist->length > env->options.writethrough_threshold || + txn->wr.dirtylist->length > env->options.writethrough_threshold || atomic_load64(&env->lck->unsynced_pages, mo_Relaxed)) ? env->lazy_fd : env->dsync_fd; #endif /* Windows */ iov_ctx_t write_ctx; - rc = iov_init(txn, &write_ctx, txn->tw.dirtylist->length, txn->tw.dirtylist->pages_including_loose, fd, false); + rc = iov_init(txn, &write_ctx, txn->wr.dirtylist->length, txn->wr.dirtylist->pages_including_loose, fd, false); if (unlikely(rc != MDBX_SUCCESS)) { ERROR("txn-%s: error %d", "iov-init", rc); return rc; @@ -271,7 +271,7 @@ int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) { } } else { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); - env->lck->unsynced_pages.weak += txn->tw.writemap_dirty_npages; + env->lck->unsynced_pages.weak += txn->wr.writemap_dirty_npages; if (!env->lck->eoos_timestamp.weak) env->lck->eoos_timestamp.weak = osal_monotime(); } @@ -286,7 +286,7 @@ int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) { meta.validator_id = head.ptr_c->validator_id; meta.extra_pagehdr = head.ptr_c->extra_pagehdr; unaligned_poke_u64(4, meta.pages_retired, - unaligned_peek_u64(4, head.ptr_c->pages_retired) + MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + unaligned_peek_u64(4, head.ptr_c->pages_retired) + MDBX_PNL_GETSIZE(txn->wr.retired_pages)); meta.geometry = txn->geo; meta.trees.gc = txn->dbs[FREE_DBI]; meta.trees.main = txn->dbs[MAIN_DBI]; @@ -303,7 +303,7 @@ int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) { meta.unsafe_sign = DATASIGN_NONE; meta_set_txnid(env, &meta, commit_txnid); - rc = dxb_sync_locked(env, env->flags | txn->flags | txn_shrink_allowed, &meta, &txn->tw.troika); + rc = dxb_sync_locked(env, env->flags | txn->flags | txn_shrink_allowed, &meta, &txn->wr.troika); if (ts) ts->sync = osal_monotime(); diff --git a/src/txn-nested.c b/src/txn-nested.c index aeb2f4f1..cdc8f8b2 100644 --- a/src/txn-nested.c +++ b/src/txn-nested.c @@ -17,20 +17,20 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t page_shadow_release(txn->env, dst->items[n].ptr, npages); --n; } - parent->tw.dirtyroom += dst->sorted - n; + parent->wr.dirtyroom += dst->sorted - n; dst->sorted = dpl_setlen(dst, n); - tASSERT(parent, parent->tw.dirtyroom + parent->tw.dirtylist->length == - (parent->parent ? parent->parent->tw.dirtyroom : parent->env->options.dp_limit)); + tASSERT(parent, parent->wr.dirtyroom + parent->wr.dirtylist->length == + (parent->parent ? parent->parent->wr.dirtyroom : parent->env->options.dp_limit)); } /* Remove reclaimed pages from parent's dirty list */ - const pnl_t reclaimed_list = parent->tw.repnl; + const pnl_t reclaimed_list = parent->wr.repnl; dpl_sift(parent, reclaimed_list, false); /* Move retired pages from parent's dirty & spilled list to reclaimed */ size_t r, w, d, s, l; - for (r = w = parent_retired_len; ++r <= MDBX_PNL_GETSIZE(parent->tw.retired_pages);) { - const pgno_t pgno = parent->tw.retired_pages[r]; + for (r = w = parent_retired_len; ++r <= MDBX_PNL_GETSIZE(parent->wr.retired_pages);) { + const pgno_t pgno = parent->wr.retired_pages[r]; const size_t di = dpl_exist(parent, pgno); const size_t si = !di ? spill_search(parent, pgno) : 0; unsigned npages; @@ -54,14 +54,14 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t /* Список retired страниц не сортирован, но для ускорения сортировки * дополняется в соответствии с MDBX_PNL_ASCENDING */ #if MDBX_PNL_ASCENDING - const size_t len = MDBX_PNL_GETSIZE(parent->tw.retired_pages); - while (r < len && parent->tw.retired_pages[r + 1] == pgno + l) { + const size_t len = MDBX_PNL_GETSIZE(parent->wr.retired_pages); + while (r < len && parent->wr.retired_pages[r + 1] == pgno + l) { ++r; if (++l == npages) break; } #else - while (w > parent_retired_len && parent->tw.retired_pages[w - 1] == pgno + l) { + while (w > parent_retired_len && parent->wr.retired_pages[w - 1] == pgno + l) { --w; if (++l == npages) break; @@ -73,18 +73,18 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t spill_remove(parent, si, 1); kind = "spilled"; } else { - parent->tw.retired_pages[++w] = pgno; + parent->wr.retired_pages[++w] = pgno; continue; } DEBUG("reclaim retired parent's %u -> %zu %s page %" PRIaPGNO, npages, l, kind, pgno); - int err = pnl_insert_span(&parent->tw.repnl, pgno, l); + int err = pnl_insert_span(&parent->wr.repnl, pgno, l); ENSURE(txn->env, err == MDBX_SUCCESS); } - MDBX_PNL_SETSIZE(parent->tw.retired_pages, w); + MDBX_PNL_SETSIZE(parent->wr.retired_pages, w); /* Filter-out parent spill list */ - if (parent->tw.spilled.list && MDBX_PNL_GETSIZE(parent->tw.spilled.list) > 0) { + if (parent->wr.spilled.list && MDBX_PNL_GETSIZE(parent->wr.spilled.list) > 0) { const pnl_t sl = spill_purge(parent); size_t len = MDBX_PNL_GETSIZE(sl); if (len) { @@ -169,11 +169,11 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t } /* Remove anything in our spill list from parent's dirty list */ - if (txn->tw.spilled.list) { - tASSERT(txn, pnl_check_allocated(txn->tw.spilled.list, (size_t)parent->geo.first_unallocated << 1)); - dpl_sift(parent, txn->tw.spilled.list, true); - tASSERT(parent, parent->tw.dirtyroom + parent->tw.dirtylist->length == - (parent->parent ? parent->parent->tw.dirtyroom : parent->env->options.dp_limit)); + if (txn->wr.spilled.list) { + tASSERT(txn, pnl_check_allocated(txn->wr.spilled.list, (size_t)parent->geo.first_unallocated << 1)); + dpl_sift(parent, txn->wr.spilled.list, true); + tASSERT(parent, parent->wr.dirtyroom + parent->wr.dirtylist->length == + (parent->parent ? parent->parent->wr.dirtyroom : parent->env->options.dp_limit)); } /* Find length of merging our dirty list with parent's and release @@ -300,10 +300,10 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t } } } - parent->tw.dirtyroom -= dst->sorted - dst->length; - assert(parent->tw.dirtyroom <= parent->env->options.dp_limit); + parent->wr.dirtyroom -= dst->sorted - dst->length; + assert(parent->wr.dirtyroom <= parent->env->options.dp_limit); dpl_setlen(dst, dst->sorted); - parent->tw.dirtylru = txn->tw.dirtylru; + parent->wr.dirtylru = txn->wr.dirtylru; /* В текущем понимании выгоднее пересчитать кол-во страниц, * чем подмешивать лишние ветвления и вычисления в циклы выше. */ @@ -314,22 +314,22 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t tASSERT(parent, dpl_check(parent)); dpl_free(txn); - if (txn->tw.spilled.list) { - if (parent->tw.spilled.list) { + if (txn->wr.spilled.list) { + if (parent->wr.spilled.list) { /* Must not fail since space was preserved above. */ - pnl_merge(parent->tw.spilled.list, txn->tw.spilled.list); - pnl_free(txn->tw.spilled.list); + pnl_merge(parent->wr.spilled.list, txn->wr.spilled.list); + pnl_free(txn->wr.spilled.list); } else { - parent->tw.spilled.list = txn->tw.spilled.list; - parent->tw.spilled.least_removed = txn->tw.spilled.least_removed; + parent->wr.spilled.list = txn->wr.spilled.list; + parent->wr.spilled.least_removed = txn->wr.spilled.least_removed; } tASSERT(parent, dpl_check(parent)); } parent->flags &= ~MDBX_TXN_HAS_CHILD; - if (parent->tw.spilled.list) { - assert(pnl_check_allocated(parent->tw.spilled.list, (size_t)parent->geo.first_unallocated << 1)); - if (MDBX_PNL_GETSIZE(parent->tw.spilled.list)) + if (parent->wr.spilled.list) { + assert(pnl_check_allocated(parent->wr.spilled.list, (size_t)parent->geo.first_unallocated << 1)); + if (MDBX_PNL_GETSIZE(parent->wr.spilled.list)) parent->flags |= MDBX_TXN_SPILLS; } } @@ -338,7 +338,7 @@ int txn_nested_create(MDBX_txn *parent, const MDBX_txn_flags_t flags) { if (parent->env->options.spill_parent4child_denominator) { /* Spill dirty-pages of parent to provide dirtyroom for child txn */ int err = - txn_spill(parent, nullptr, parent->tw.dirtylist->length / parent->env->options.spill_parent4child_denominator); + txn_spill(parent, nullptr, parent->wr.dirtylist->length / parent->env->options.spill_parent4child_denominator); if (unlikely(err != MDBX_SUCCESS)) return LOG_IFERR(err); } @@ -356,74 +356,74 @@ int txn_nested_create(MDBX_txn *parent, const MDBX_txn_flags_t flags) { txn->geo = parent->geo; int err = dpl_alloc(txn); if (likely(err == MDBX_SUCCESS)) { - const size_t len = MDBX_PNL_GETSIZE(parent->tw.repnl) + parent->tw.loose_count; - txn->tw.repnl = pnl_alloc((len > MDBX_PNL_INITIAL) ? len : MDBX_PNL_INITIAL); - if (unlikely(!txn->tw.repnl)) + const size_t len = MDBX_PNL_GETSIZE(parent->wr.repnl) + parent->wr.loose_count; + txn->wr.repnl = pnl_alloc((len > MDBX_PNL_INITIAL) ? len : MDBX_PNL_INITIAL); + if (unlikely(!txn->wr.repnl)) err = MDBX_ENOMEM; } if (unlikely(err != MDBX_SUCCESS)) { failed: - pnl_free(txn->tw.repnl); + pnl_free(txn->wr.repnl); dpl_free(txn); osal_free(txn); return LOG_IFERR(err); } /* Move loose pages to reclaimed list */ - if (parent->tw.loose_count) { + if (parent->wr.loose_count) { do { - page_t *lp = parent->tw.loose_pages; + page_t *lp = parent->wr.loose_pages; tASSERT(parent, lp->flags == P_LOOSE); - err = pnl_insert_span(&parent->tw.repnl, lp->pgno, 1); + err = pnl_insert_span(&parent->wr.repnl, lp->pgno, 1); if (unlikely(err != MDBX_SUCCESS)) goto failed; MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *)); VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); - parent->tw.loose_pages = page_next(lp); + parent->wr.loose_pages = page_next(lp); /* Remove from dirty list */ page_wash(parent, dpl_exist(parent, lp->pgno), lp, 1); - } while (parent->tw.loose_pages); - parent->tw.loose_count = 0; + } while (parent->wr.loose_pages); + parent->wr.loose_count = 0; #if MDBX_ENABLE_REFUND - parent->tw.loose_refund_wl = 0; + parent->wr.loose_refund_wl = 0; #endif /* MDBX_ENABLE_REFUND */ tASSERT(parent, dpl_check(parent)); } - txn->tw.dirtyroom = parent->tw.dirtyroom; - txn->tw.dirtylru = parent->tw.dirtylru; + txn->wr.dirtyroom = parent->wr.dirtyroom; + txn->wr.dirtylru = parent->wr.dirtylru; dpl_sort(parent); - if (parent->tw.spilled.list) + if (parent->wr.spilled.list) spill_purge(parent); - tASSERT(txn, MDBX_PNL_ALLOCLEN(txn->tw.repnl) >= MDBX_PNL_GETSIZE(parent->tw.repnl)); - memcpy(txn->tw.repnl, parent->tw.repnl, MDBX_PNL_SIZEOF(parent->tw.repnl)); - tASSERT(txn, pnl_check_allocated(txn->tw.repnl, (txn->geo.first_unallocated /* LY: intentional assignment + tASSERT(txn, MDBX_PNL_ALLOCLEN(txn->wr.repnl) >= MDBX_PNL_GETSIZE(parent->wr.repnl)); + memcpy(txn->wr.repnl, parent->wr.repnl, MDBX_PNL_SIZEOF(parent->wr.repnl)); + tASSERT(txn, pnl_check_allocated(txn->wr.repnl, (txn->geo.first_unallocated /* LY: intentional assignment here, only for assertion */ = parent->geo.first_unallocated) - MDBX_ENABLE_REFUND)); - txn->tw.gc.time_acc = parent->tw.gc.time_acc; - txn->tw.gc.last_reclaimed = parent->tw.gc.last_reclaimed; - if (parent->tw.gc.retxl) { - txn->tw.gc.retxl = parent->tw.gc.retxl; - parent->tw.gc.retxl = (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->tw.gc.retxl); + txn->wr.gc.time_acc = parent->wr.gc.time_acc; + txn->wr.gc.last_reclaimed = parent->wr.gc.last_reclaimed; + if (parent->wr.gc.retxl) { + txn->wr.gc.retxl = parent->wr.gc.retxl; + parent->wr.gc.retxl = (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->wr.gc.retxl); } - txn->tw.retired_pages = parent->tw.retired_pages; - parent->tw.retired_pages = (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->tw.retired_pages); + txn->wr.retired_pages = parent->wr.retired_pages; + parent->wr.retired_pages = (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->wr.retired_pages); txn->txnid = parent->txnid; txn->front_txnid = parent->front_txnid + 1; #if MDBX_ENABLE_REFUND - txn->tw.loose_refund_wl = 0; + txn->wr.loose_refund_wl = 0; #endif /* MDBX_ENABLE_REFUND */ txn->canary = parent->canary; parent->flags |= MDBX_TXN_HAS_CHILD; parent->nested = txn; txn->parent = parent; txn->owner = parent->owner; - txn->tw.troika = parent->tw.troika; + txn->wr.troika = parent->wr.troika; txn->cursors[FREE_DBI] = nullptr; txn->cursors[MAIN_DBI] = nullptr; @@ -432,10 +432,10 @@ int txn_nested_create(MDBX_txn *parent, const MDBX_txn_flags_t flags) { memset(txn->dbi_state + CORE_DBS, 0, (txn->n_dbi = parent->n_dbi) - CORE_DBS); memcpy(txn->dbs, parent->dbs, sizeof(txn->dbs[0]) * CORE_DBS); - tASSERT(parent, parent->tw.dirtyroom + parent->tw.dirtylist->length == - (parent->parent ? parent->parent->tw.dirtyroom : parent->env->options.dp_limit)); - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); + tASSERT(parent, parent->wr.dirtyroom + parent->wr.dirtylist->length == + (parent->parent ? parent->parent->wr.dirtyroom : parent->env->options.dp_limit)); + tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length == + (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit)); parent->env->txn = txn; tASSERT(parent, parent->cursors[FREE_DBI] == nullptr); return txn_shadow_cursors(parent, MAIN_DBI); @@ -447,26 +447,26 @@ void txn_nested_abort(MDBX_txn *nested) { nested->signature = 0; nested->owner = 0; - if (nested->tw.gc.retxl) { - tASSERT(parent, MDBX_PNL_GETSIZE(nested->tw.gc.retxl) >= (uintptr_t)parent->tw.gc.retxl); - MDBX_PNL_SETSIZE(nested->tw.gc.retxl, (uintptr_t)parent->tw.gc.retxl); - parent->tw.gc.retxl = nested->tw.gc.retxl; + if (nested->wr.gc.retxl) { + tASSERT(parent, MDBX_PNL_GETSIZE(nested->wr.gc.retxl) >= (uintptr_t)parent->wr.gc.retxl); + MDBX_PNL_SETSIZE(nested->wr.gc.retxl, (uintptr_t)parent->wr.gc.retxl); + parent->wr.gc.retxl = nested->wr.gc.retxl; } - if (nested->tw.retired_pages) { - tASSERT(parent, MDBX_PNL_GETSIZE(nested->tw.retired_pages) >= (uintptr_t)parent->tw.retired_pages); - MDBX_PNL_SETSIZE(nested->tw.retired_pages, (uintptr_t)parent->tw.retired_pages); - parent->tw.retired_pages = nested->tw.retired_pages; + if (nested->wr.retired_pages) { + tASSERT(parent, MDBX_PNL_GETSIZE(nested->wr.retired_pages) >= (uintptr_t)parent->wr.retired_pages); + MDBX_PNL_SETSIZE(nested->wr.retired_pages, (uintptr_t)parent->wr.retired_pages); + parent->wr.retired_pages = nested->wr.retired_pages; } - parent->tw.dirtylru = nested->tw.dirtylru; + parent->wr.dirtylru = nested->wr.dirtylru; parent->nested = nullptr; parent->flags &= ~MDBX_TXN_HAS_CHILD; tASSERT(parent, dpl_check(parent)); tASSERT(parent, audit_ex(parent, 0, false) == 0); dpl_release_shadows(nested); dpl_free(nested); - pnl_free(nested->tw.repnl); + pnl_free(nested->wr.repnl); osal_free(nested); } @@ -479,7 +479,7 @@ int txn_nested_join(MDBX_txn *txn, struct commit_timestamp *ts) { eASSERT(env, parent->nested == txn && (parent->flags & MDBX_TXN_HAS_CHILD) != 0); eASSERT(env, dpl_check(txn)); - if (txn->tw.dirtylist->length == 0 && !(txn->flags & MDBX_TXN_DIRTY) && parent->n_dbi == txn->n_dbi) { + if (txn->wr.dirtylist->length == 0 && !(txn->flags & MDBX_TXN_DIRTY) && parent->n_dbi == txn->n_dbi) { TXN_FOREACH_DBI_ALL(txn, i) { tASSERT(txn, (txn->dbi_state[i] & DBI_DIRTY) == 0); if ((txn->dbi_state[i] & DBI_STALE) && !(parent->dbi_state[i] & DBI_STALE)) @@ -488,8 +488,8 @@ int txn_nested_join(MDBX_txn *txn, struct commit_timestamp *ts) { tASSERT(txn, memcmp(&parent->geo, &txn->geo, sizeof(parent->geo)) == 0); tASSERT(txn, memcmp(&parent->canary, &txn->canary, sizeof(parent->canary)) == 0); - tASSERT(txn, !txn->tw.spilled.list || MDBX_PNL_GETSIZE(txn->tw.spilled.list) == 0); - tASSERT(txn, txn->tw.loose_count == 0); + tASSERT(txn, !txn->wr.spilled.list || MDBX_PNL_GETSIZE(txn->wr.spilled.list) == 0); + tASSERT(txn, txn->wr.loose_count == 0); VERBOSE("fast-complete pure nested txn %" PRIaTXN, txn->txnid); return txn_end(txn, TXN_END_PURE_COMMIT | TXN_END_SLOT | TXN_END_FREE); @@ -497,42 +497,42 @@ int txn_nested_join(MDBX_txn *txn, struct commit_timestamp *ts) { /* Preserve space for spill list to avoid parent's state corruption * if allocation fails. */ - const size_t parent_retired_len = (uintptr_t)parent->tw.retired_pages; - tASSERT(txn, parent_retired_len <= MDBX_PNL_GETSIZE(txn->tw.retired_pages)); - const size_t retired_delta = MDBX_PNL_GETSIZE(txn->tw.retired_pages) - parent_retired_len; + const size_t parent_retired_len = (uintptr_t)parent->wr.retired_pages; + tASSERT(txn, parent_retired_len <= MDBX_PNL_GETSIZE(txn->wr.retired_pages)); + const size_t retired_delta = MDBX_PNL_GETSIZE(txn->wr.retired_pages) - parent_retired_len; if (retired_delta) { - int err = pnl_need(&txn->tw.repnl, retired_delta); + int err = pnl_need(&txn->wr.repnl, retired_delta); if (unlikely(err != MDBX_SUCCESS)) return err; } - if (txn->tw.spilled.list) { - if (parent->tw.spilled.list) { - int err = pnl_need(&parent->tw.spilled.list, MDBX_PNL_GETSIZE(txn->tw.spilled.list)); + if (txn->wr.spilled.list) { + if (parent->wr.spilled.list) { + int err = pnl_need(&parent->wr.spilled.list, MDBX_PNL_GETSIZE(txn->wr.spilled.list)); if (unlikely(err != MDBX_SUCCESS)) return err; } spill_purge(txn); } - if (unlikely(txn->tw.dirtylist->length + parent->tw.dirtylist->length > parent->tw.dirtylist->detent && - !dpl_reserve(parent, txn->tw.dirtylist->length + parent->tw.dirtylist->length))) { + if (unlikely(txn->wr.dirtylist->length + parent->wr.dirtylist->length > parent->wr.dirtylist->detent && + !dpl_reserve(parent, txn->wr.dirtylist->length + parent->wr.dirtylist->length))) { return MDBX_ENOMEM; } //------------------------------------------------------------------------- - parent->tw.gc.retxl = txn->tw.gc.retxl; - txn->tw.gc.retxl = nullptr; + parent->wr.gc.retxl = txn->wr.gc.retxl; + txn->wr.gc.retxl = nullptr; - parent->tw.retired_pages = txn->tw.retired_pages; - txn->tw.retired_pages = nullptr; + parent->wr.retired_pages = txn->wr.retired_pages; + txn->wr.retired_pages = nullptr; - pnl_free(parent->tw.repnl); - parent->tw.repnl = txn->tw.repnl; - txn->tw.repnl = nullptr; - parent->tw.gc.time_acc = txn->tw.gc.time_acc; - parent->tw.gc.last_reclaimed = txn->tw.gc.last_reclaimed; + pnl_free(parent->wr.repnl); + parent->wr.repnl = txn->wr.repnl; + txn->wr.repnl = nullptr; + parent->wr.gc.time_acc = txn->wr.gc.time_acc; + parent->wr.gc.last_reclaimed = txn->wr.gc.last_reclaimed; parent->geo = txn->geo; parent->canary = txn->canary; @@ -540,10 +540,10 @@ int txn_nested_join(MDBX_txn *txn, struct commit_timestamp *ts) { /* Move loose pages to parent */ #if MDBX_ENABLE_REFUND - parent->tw.loose_refund_wl = txn->tw.loose_refund_wl; + parent->wr.loose_refund_wl = txn->wr.loose_refund_wl; #endif /* MDBX_ENABLE_REFUND */ - parent->tw.loose_count = txn->tw.loose_count; - parent->tw.loose_pages = txn->tw.loose_pages; + parent->wr.loose_count = txn->wr.loose_count; + parent->wr.loose_pages = txn->wr.loose_pages; if (txn->flags & txn_may_have_cursors) /* Merge our cursors into parent's and close them */ @@ -581,14 +581,14 @@ int txn_nested_join(MDBX_txn *txn, struct commit_timestamp *ts) { txn_refund(parent); if (ASSERT_ENABLED()) { /* Check parent's loose pages not suitable for refund */ - for (page_t *lp = parent->tw.loose_pages; lp; lp = page_next(lp)) { - tASSERT(parent, lp->pgno < parent->tw.loose_refund_wl && lp->pgno + 1 < parent->geo.first_unallocated); + for (page_t *lp = parent->wr.loose_pages; lp; lp = page_next(lp)) { + tASSERT(parent, lp->pgno < parent->wr.loose_refund_wl && lp->pgno + 1 < parent->geo.first_unallocated); MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *)); VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); } /* Check parent's reclaimed pages not suitable for refund */ - if (MDBX_PNL_GETSIZE(parent->tw.repnl)) - tASSERT(parent, MDBX_PNL_MOST(parent->tw.repnl) + 1 < parent->geo.first_unallocated); + if (MDBX_PNL_GETSIZE(parent->wr.repnl)) + tASSERT(parent, MDBX_PNL_MOST(parent->wr.repnl) + 1 < parent->geo.first_unallocated); } #endif /* MDBX_ENABLE_REFUND */ diff --git a/src/txn-ro.c b/src/txn-ro.c index 5be1d8fd..c8a50898 100644 --- a/src/txn-ro.c +++ b/src/txn-ro.c @@ -4,7 +4,7 @@ #include "internals.h" static inline int txn_ro_rslot(MDBX_txn *txn) { - reader_slot_t *slot = txn->to.reader; + reader_slot_t *slot = txn->ro.slot; STATIC_ASSERT(sizeof(uintptr_t) <= sizeof(slot->tid)); if (likely(slot)) { if (likely(slot->pid.weak == txn->env->pid && slot->txnid.weak >= SAFE64_INVALID_THRESHOLD)) { @@ -26,7 +26,7 @@ static inline int txn_ro_rslot(MDBX_txn *txn) { if (likely(slot->pid.weak == env->pid && slot->txnid.weak >= SAFE64_INVALID_THRESHOLD)) { tASSERT(txn, slot->pid.weak == osal_getpid()); tASSERT(txn, slot->tid.weak == ((env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self())); - txn->to.reader = slot; + txn->ro.slot = slot; return MDBX_SUCCESS; } if (unlikely(slot->pid.weak) || !(globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN)) @@ -42,7 +42,7 @@ static inline int txn_ro_rslot(MDBX_txn *txn) { tASSERT(txn, brs.slot->pid.weak == osal_getpid()); tASSERT(txn, brs.slot->tid.weak == ((env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self())); } - txn->to.reader = brs.slot; + txn->ro.slot = brs.slot; return brs.err; } @@ -55,7 +55,7 @@ static inline int txn_ro_seize(MDBX_txn *txn) { MDBX_env *const env = txn->env; const meta_ptr_t head = likely(env->stuck_meta < 0) ? /* regular */ meta_recent(env, &troika) : /* recovery mode */ meta_ptr(env, env->stuck_meta); - reader_slot_t *const r = txn->to.reader; + reader_slot_t *const r = txn->ro.slot; if (likely(r != nullptr)) { safe64_reset(&r->txnid, true); atomic_store32(&r->snapshot_pages_used, head.ptr_v->geometry.first_unallocated, mo_Relaxed); @@ -117,7 +117,7 @@ int txn_ro_start(MDBX_txn *txn, unsigned flags) { goto bailout; STATIC_ASSERT(MDBX_TXN_RDONLY_PREPARE > MDBX_TXN_RDONLY); - reader_slot_t *r = txn->to.reader; + reader_slot_t *r = txn->ro.slot; if (flags & (MDBX_TXN_RDONLY_PREPARE - MDBX_TXN_RDONLY)) { eASSERT(env, txn->txnid == 0); eASSERT(env, txn->owner == 0); @@ -153,8 +153,8 @@ int txn_ro_start(MDBX_txn *txn, unsigned flags) { bailout: tASSERT(txn, err != MDBX_SUCCESS); txn->txnid = INVALID_TXNID; - if (likely(txn->to.reader)) - safe64_reset(&txn->to.reader->txnid, true); + if (likely(txn->ro.slot)) + safe64_reset(&txn->ro.slot->txnid, true); return err; } @@ -162,10 +162,10 @@ int txn_ro_end(MDBX_txn *txn, unsigned mode) { MDBX_env *const env = txn->env; tASSERT(txn, (txn->flags & txn_may_have_cursors) == 0); txn->n_dbi = 0; /* prevent further DBI activity */ - if (txn->to.reader) { - reader_slot_t *slot = txn->to.reader; + if (txn->ro.slot) { + reader_slot_t *slot = txn->ro.slot; if (unlikely(!env->lck)) - txn->to.reader = nullptr; + txn->ro.slot = nullptr; else { eASSERT(env, slot->pid.weak == env->pid); if (likely((txn->flags & MDBX_TXN_FINISHED) == 0)) { @@ -194,7 +194,7 @@ int txn_ro_end(MDBX_txn *txn, unsigned mode) { if (mode & TXN_END_SLOT) { if ((env->flags & ENV_TXKEY) == 0) atomic_store32(&slot->pid, 0, mo_Relaxed); - txn->to.reader = nullptr; + txn->ro.slot = nullptr; } } } @@ -213,9 +213,9 @@ int txn_ro_end(MDBX_txn *txn, unsigned mode) { } int txn_ro_park(MDBX_txn *txn, bool autounpark) { - reader_slot_t *const rslot = txn->to.reader; + reader_slot_t *const rslot = txn->ro.slot; tASSERT(txn, (txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) == MDBX_TXN_RDONLY); - tASSERT(txn, txn->to.reader->tid.weak < MDBX_TID_TXN_OUSTED); + tASSERT(txn, txn->ro.slot->tid.weak < MDBX_TID_TXN_OUSTED); if (unlikely((txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) != MDBX_TXN_RDONLY)) return MDBX_BAD_TXN; @@ -244,7 +244,7 @@ int txn_ro_unpark(MDBX_txn *txn) { (MDBX_TXN_RDONLY | MDBX_TXN_PARKED))) return MDBX_BAD_TXN; - for (reader_slot_t *const rslot = txn->to.reader; rslot; atomic_yield()) { + for (reader_slot_t *const rslot = txn->ro.slot; rslot; atomic_yield()) { const uint32_t pid = atomic_load32(&rslot->pid, mo_Relaxed); uint64_t tid = safe64_read(&rslot->tid); uint64_t txnid = safe64_read(&rslot->txnid); diff --git a/src/txn.c b/src/txn.c index c764a9f7..9d0600a4 100644 --- a/src/txn.c +++ b/src/txn.c @@ -4,7 +4,7 @@ #include "internals.h" __hot txnid_t txn_snapshot_oldest(const MDBX_txn *const txn) { - return mvcc_shapshot_oldest(txn->env, txn->tw.troika.txnid[txn->tw.troika.prefer_steady]); + return mvcc_shapshot_oldest(txn->env, txn->wr.troika.txnid[txn->wr.troika.prefer_steady]); } void txn_done_cursors(MDBX_txn *txn) { @@ -79,7 +79,6 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { rc = txn_ro_start(txn, flags); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; - tASSERT(txn, rc == MDBX_SUCCESS); ENSURE(env, txn->txnid >= /* paranoia is appropriate here */ env->lck->cached_oldest.weak); tASSERT(txn, txn->dbs[FREE_DBI].flags == MDBX_INTEGERKEY); @@ -119,8 +118,8 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { } #endif /* Windows */ - txn->tw.troika = meta_tap(env); - const meta_ptr_t head = meta_recent(env, &txn->tw.troika); + txn->wr.troika = meta_tap(env); + const meta_ptr_t head = meta_recent(env, &txn->wr.troika); uint64_t timestamp = 0; while ("workaround for https://libmdbx.dqdkfa.ru/dead-github/issues/269") { rc = coherency_fetch_head(txn, head, ×tamp); @@ -141,18 +140,18 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { tASSERT(txn, check_table_flags(txn->dbs[MAIN_DBI].flags)); txn->flags = flags; txn->nested = nullptr; - txn->tw.loose_pages = nullptr; - txn->tw.loose_count = 0; + txn->wr.loose_pages = nullptr; + txn->wr.loose_count = 0; #if MDBX_ENABLE_REFUND - txn->tw.loose_refund_wl = 0; + txn->wr.loose_refund_wl = 0; #endif /* MDBX_ENABLE_REFUND */ - MDBX_PNL_SETSIZE(txn->tw.retired_pages, 0); - txn->tw.spilled.list = nullptr; - txn->tw.spilled.least_removed = 0; - txn->tw.gc.time_acc = 0; - txn->tw.gc.last_reclaimed = 0; - if (txn->tw.gc.retxl) - MDBX_PNL_SETSIZE(txn->tw.gc.retxl, 0); + MDBX_PNL_SETSIZE(txn->wr.retired_pages, 0); + txn->wr.spilled.list = nullptr; + txn->wr.spilled.least_removed = 0; + txn->wr.gc.time_acc = 0; + txn->wr.gc.last_reclaimed = 0; + if (txn->wr.gc.retxl) + MDBX_PNL_SETSIZE(txn->wr.gc.retxl, 0); env->txn = txn; } @@ -326,16 +325,16 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { rc = dpl_alloc(txn); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; - txn->tw.dirtyroom = txn->env->options.dp_limit; - txn->tw.dirtylru = MDBX_DEBUG ? UINT32_MAX / 3 - 42 : 0; + txn->wr.dirtyroom = txn->env->options.dp_limit; + txn->wr.dirtylru = MDBX_DEBUG ? UINT32_MAX / 3 - 42 : 0; } else { - tASSERT(txn, txn->tw.dirtylist == nullptr); - txn->tw.dirtylist = nullptr; - txn->tw.dirtyroom = MAX_PAGENO; - txn->tw.dirtylru = 0; + tASSERT(txn, txn->wr.dirtylist == nullptr); + txn->wr.dirtylist = nullptr; + txn->wr.dirtyroom = MAX_PAGENO; + txn->wr.dirtylru = 0; } - eASSERT(env, txn->tw.writemap_dirty_npages == 0); - eASSERT(env, txn->tw.writemap_spilled_npages == 0); + eASSERT(env, txn->wr.writemap_dirty_npages == 0); + eASSERT(env, txn->wr.writemap_spilled_npages == 0); MDBX_cursor *const gc = ptr_disp(txn, sizeof(MDBX_txn)); rc = cursor_init(gc, txn, FREE_DBI); @@ -378,8 +377,8 @@ int txn_end(MDBX_txn *txn, unsigned mode) { ERROR("parent txn %p is invalid or mismatch for nested txn %p", (void *)parent, (void *)txn); return MDBX_PROBLEM; } - tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); - tASSERT(txn, memcmp(&txn->tw.troika, &parent->tw.troika, sizeof(troika_t)) == 0); + tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + tASSERT(txn, memcmp(&txn->wr.troika, &parent->wr.troika, sizeof(troika_t)) == 0); tASSERT(txn, mode & TXN_END_FREE); env->txn = parent; const pgno_t nested_now = txn->geo.now, nested_upper = txn->geo.upper; @@ -434,9 +433,9 @@ MDBX_txn *txn_alloc(const MDBX_txn_flags_t flags, MDBX_env *env) { #else 0; #endif /* MDBX_ENABLE_DBI_SPARSE */ - STATIC_ASSERT(sizeof(txn->tw) > sizeof(txn->to)); + STATIC_ASSERT(sizeof(txn->wr) > sizeof(txn->ro)); const size_t base = - (flags & MDBX_TXN_RDONLY) ? sizeof(MDBX_txn) - sizeof(txn->tw) + sizeof(txn->to) : sizeof(MDBX_txn); + (flags & MDBX_TXN_RDONLY) ? sizeof(MDBX_txn) - sizeof(txn->wr) + sizeof(txn->ro) : sizeof(MDBX_txn); const size_t size = base + ((flags & MDBX_TXN_RDONLY) ? (size_t)bitmap_bytes + env->max_dbi * sizeof(txn->dbi_seqs[0]) : 0) + env->max_dbi * (sizeof(txn->dbs[0]) + sizeof(txn->cursors[0]) + sizeof(txn->dbi_state[0]));