ISAR/libmdbx
3
0
Fork 0
mirror of https://github.com/isar/libmdbx.git synced 2025-01-19 22:08:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

mdbx: переименование (косметика).

Browse Source
This commit is contained in:
Леонид Юрьев (Leonid Yuriev) 2025-01-14 22:40:56 +03:00
parent 7e772114bc
commit 314b8ce1f0
24 changed files with 600 additions and 601 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/api-env.c
View File

@ -683,7 +683,7 @@ static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo
#endif
}
*troika = (txn && !(txn->flags & MDBX_TXN_RDONLY)) ? txn->tw.troika : meta_tap(env);
*troika = (txn && !(txn->flags & MDBX_TXN_RDONLY)) ? txn->wr.troika : meta_tap(env);
const meta_ptr_t head = meta_recent(env, troika);
const meta_t *const meta0 = METAPAGE(env, 0);
const meta_t *const meta1 = METAPAGE(env, 1);
@ -947,9 +947,9 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t si
if (unlikely(err != MDBX_SUCCESS))
return LOG_IFERR(err);
should_unlock = true;
env->basal_txn->tw.troika = meta_tap(env);
env->basal_txn->wr.troika = meta_tap(env);
eASSERT(env, !env->txn && !env->basal_txn->nested);
env->basal_txn->txnid = env->basal_txn->tw.troika.txnid[env->basal_txn->tw.troika.recent];
env->basal_txn->txnid = env->basal_txn->wr.troika.txnid[env->basal_txn->wr.troika.recent];
txn_snapshot_oldest(env->basal_txn);
}
@ -957,7 +957,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t si
if (pagesize <= 0 || pagesize >= INT_MAX)
pagesize = env->ps;
const geo_t *const geo =
inside_txn ? &env->txn->geo : &meta_recent(env, &env->basal_txn->tw.troika).ptr_c->geometry;
inside_txn ? &env->txn->geo : &meta_recent(env, &env->basal_txn->wr.troika).ptr_c->geometry;
if (size_lower < 0)
size_lower = pgno2bytes(env, geo->lower);
if (size_now < 0)
@ -1173,7 +1173,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t si
memset(&meta, 0, sizeof(meta));
if (!inside_txn) {
eASSERT(env, should_unlock);
const meta_ptr_t head = meta_recent(env, &env->basal_txn->tw.troika);
const meta_ptr_t head = meta_recent(env, &env->basal_txn->wr.troika);
uint64_t timestamp = 0;
while ("workaround for "
@ -1267,7 +1267,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t si
env->txn->flags |= MDBX_TXN_DIRTY;
} else {
meta.geometry = new_geo;
rc = dxb_sync_locked(env, env->flags, &meta, &env->basal_txn->tw.troika);
rc = dxb_sync_locked(env, env->flags, &meta, &env->basal_txn->wr.troika);
if (likely(rc == MDBX_SUCCESS)) {
env->geo_in_bytes.now = pgno2bytes(env, new_geo.now = meta.geometry.now);
env->geo_in_bytes.upper = pgno2bytes(env, new_geo.upper = meta.geometry.upper);

22
src/api-txn.c
View File

@ -56,8 +56,8 @@ MDBX_txn_flags_t mdbx_txn_flags(const MDBX_txn *txn) {
assert(0 == (int)(txn->flags & MDBX_TXN_INVALID));
MDBX_txn_flags_t flags = txn->flags;
if (F_ISSET(flags, MDBX_TXN_PARKED | MDBX_TXN_RDONLY) && txn->to.reader &&
safe64_read(&txn->to.reader->tid) == MDBX_TID_TXN_OUSTED)
if (F_ISSET(flags, MDBX_TXN_PARKED | MDBX_TXN_RDONLY) && txn->ro.slot &&
safe64_read(&txn->ro.slot->tid) == MDBX_TID_TXN_OUSTED)
flags |= MDBX_TXN_OUSTED;
return flags;
}
@ -257,7 +257,7 @@ int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, M
else {
eASSERT(env, (txn->flags & ~(MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP | txn_shrink_allowed | txn_may_have_cursors |
MDBX_NOMETASYNC | MDBX_SAFE_NOSYNC | MDBX_TXN_SPILLS)) == 0);
assert(!txn->tw.spilled.list && !txn->tw.spilled.least_removed);
assert(!txn->wr.spilled.list && !txn->wr.spilled.least_removed);
}
txn->signature = txn_signature;
txn->userctx = context;
@ -456,10 +456,10 @@ int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) {
info->txn_reader_lag = head.txnid - info->txn_id;
info->txn_space_dirty = info->txn_space_retired = 0;
uint64_t reader_snapshot_pages_retired = 0;
if (txn->to.reader &&
((txn->flags & MDBX_TXN_PARKED) == 0 || safe64_read(&txn->to.reader->tid) != MDBX_TID_TXN_OUSTED) &&
if (txn->ro.slot &&
((txn->flags & MDBX_TXN_PARKED) == 0 || safe64_read(&txn->ro.slot->tid) != MDBX_TID_TXN_OUSTED) &&
head_retired >
(reader_snapshot_pages_retired = atomic_load64(&txn->to.reader->snapshot_pages_retired, mo_Relaxed))) {
(reader_snapshot_pages_retired = atomic_load64(&txn->ro.slot->snapshot_pages_retired, mo_Relaxed))) {
info->txn_space_dirty = info->txn_space_retired =
pgno2bytes(env, (pgno_t)(head_retired - reader_snapshot_pages_retired));
@ -486,7 +486,7 @@ int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) {
if (snap_txnid < next_reader && snap_tid >= MDBX_TID_TXN_OUSTED) {
next_reader = snap_txnid;
retired_next_reader = pgno2bytes(
env, (pgno_t)(snap_retired - atomic_load64(&txn->to.reader->snapshot_pages_retired, mo_Relaxed)));
env, (pgno_t)(snap_retired - atomic_load64(&txn->ro.slot->snapshot_pages_retired, mo_Relaxed)));
}
}
}
@ -497,11 +497,11 @@ int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) {
info->txn_space_limit_soft = pgno2bytes(env, txn->geo.now);
info->txn_space_limit_hard = pgno2bytes(env, txn->geo.upper);
info->txn_space_retired =
pgno2bytes(env, txn->nested ? (size_t)txn->tw.retired_pages : MDBX_PNL_GETSIZE(txn->tw.retired_pages));
info->txn_space_leftover = pgno2bytes(env, txn->tw.dirtyroom);
pgno2bytes(env, txn->nested ? (size_t)txn->wr.retired_pages : MDBX_PNL_GETSIZE(txn->wr.retired_pages));
info->txn_space_leftover = pgno2bytes(env, txn->wr.dirtyroom);
info->txn_space_dirty =
pgno2bytes(env, txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose
: (txn->tw.writemap_dirty_npages + txn->tw.writemap_spilled_npages));
pgno2bytes(env, txn->wr.dirtylist ? txn->wr.dirtylist->pages_including_loose
: (txn->wr.writemap_dirty_npages + txn->wr.writemap_spilled_npages));
info->txn_reader_lag = INT64_MAX;
lck_t *const lck = env->lck_mmap.lck;
if (scan_rlt && lck) {

10
src/audit.c
View File

@ -28,8 +28,8 @@ __cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored, bool don
const MDBX_env *const env = txn->env;
size_t pending = 0;
if ((txn->flags & MDBX_TXN_RDONLY) == 0)
pending = txn->tw.loose_count + MDBX_PNL_GETSIZE(txn->tw.repnl) +
(MDBX_PNL_GETSIZE(txn->tw.retired_pages) - retired_stored);
pending = txn->wr.loose_count + MDBX_PNL_GETSIZE(txn->wr.repnl) +
(MDBX_PNL_GETSIZE(txn->wr.retired_pages) - retired_stored);
cursor_couple_t cx;
int rc = cursor_init(&cx.outer, txn, FREE_DBI);
@ -46,7 +46,7 @@ __cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored, bool don
return MDBX_CORRUPTED;
}
txnid_t id = unaligned_peek_u64(4, key.iov_base);
if (txn->tw.gc.retxl ? txl_contain(txn->tw.gc.retxl, id) : (id <= txn->tw.gc.last_reclaimed))
if (txn->wr.gc.retxl ? txl_contain(txn->wr.gc.retxl, id) : (id <= txn->wr.gc.last_reclaimed))
goto skip;
}
gc += *(pgno_t *)data.iov_base;
@ -89,8 +89,8 @@ __cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored, bool don
if ((txn->flags & MDBX_TXN_RDONLY) == 0)
ERROR("audit @%" PRIaTXN ": %zu(pending) = %zu(loose) + "
"%zu(reclaimed) + %zu(retired-pending) - %zu(retired-stored)",
txn->txnid, pending, txn->tw.loose_count, MDBX_PNL_GETSIZE(txn->tw.repnl),
txn->tw.retired_pages ? MDBX_PNL_GETSIZE(txn->tw.retired_pages) : 0, retired_stored);
txn->txnid, pending, txn->wr.loose_count, MDBX_PNL_GETSIZE(txn->wr.repnl),
txn->wr.retired_pages ? MDBX_PNL_GETSIZE(txn->wr.retired_pages) : 0, retired_stored);
ERROR("audit @%" PRIaTXN ": %zu(pending) + %zu"
"(gc) + %zu(count) = %zu(total) <> %zu"
"(allocated)",

6
src/cursor.c
View File

@ -6,12 +6,12 @@
#include "internals.h"
__cold int cursor_check(const MDBX_cursor *mc) {
if (!mc->txn->tw.dirtylist) {
if (!mc->txn->wr.dirtylist) {
cASSERT(mc, (mc->txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
} else {
cASSERT(mc, (mc->txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
cASSERT(mc, mc->txn->tw.dirtyroom + mc->txn->tw.dirtylist->length ==
(mc->txn->parent ? mc->txn->parent->tw.dirtyroom : mc->txn->env->options.dp_limit));
cASSERT(mc, mc->txn->wr.dirtyroom + mc->txn->wr.dirtylist->length ==
(mc->txn->parent ? mc->txn->parent->wr.dirtyroom : mc->txn->env->options.dp_limit));
}
cASSERT(mc, (mc->checking & z_updating) ? mc->top + 1 <= mc->tree->height : mc->top + 1 == mc->tree->height);

68
src/dpl.c
View File

@ -28,9 +28,9 @@ static inline size_t dpl_bytes2size(const ptrdiff_t bytes) {
}
void dpl_free(MDBX_txn *txn) {
if (likely(txn->tw.dirtylist)) {
osal_free(txn->tw.dirtylist);
txn->tw.dirtylist = nullptr;
if (likely(txn->wr.dirtylist)) {
osal_free(txn->wr.dirtylist);
txn->wr.dirtylist = nullptr;
}
}
@ -39,14 +39,14 @@ dpl_t *dpl_reserve(MDBX_txn *txn, size_t size) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
size_t bytes = dpl_size2bytes((size < PAGELIST_LIMIT) ? size : PAGELIST_LIMIT);
dpl_t *const dl = osal_realloc(txn->tw.dirtylist, bytes);
dpl_t *const dl = osal_realloc(txn->wr.dirtylist, bytes);
if (likely(dl)) {
#ifdef osal_malloc_usable_size
bytes = osal_malloc_usable_size(dl);
#endif /* osal_malloc_usable_size */
dl->detent = dpl_bytes2size(bytes);
tASSERT(txn, txn->tw.dirtylist == nullptr || dl->length <= dl->detent);
txn->tw.dirtylist = dl;
tASSERT(txn, txn->wr.dirtylist == nullptr || dl->length <= dl->detent);
txn->wr.dirtylist = dl;
}
return dl;
}
@ -57,15 +57,15 @@ int dpl_alloc(MDBX_txn *txn) {
const size_t wanna = (txn->env->options.dp_initial < txn->geo.upper) ? txn->env->options.dp_initial : txn->geo.upper;
#if MDBX_FORCE_ASSERTIONS || MDBX_DEBUG
if (txn->tw.dirtylist)
if (txn->wr.dirtylist)
/* обнуляем чтобы не сработал ассерт внутри dpl_reserve() */
txn->tw.dirtylist->sorted = txn->tw.dirtylist->length = 0;
txn->wr.dirtylist->sorted = txn->wr.dirtylist->length = 0;
#endif /* asertions enabled */
if (unlikely(!txn->tw.dirtylist || txn->tw.dirtylist->detent < wanna || txn->tw.dirtylist->detent > wanna + wanna) &&
if (unlikely(!txn->wr.dirtylist || txn->wr.dirtylist->detent < wanna || txn->wr.dirtylist->detent > wanna + wanna) &&
unlikely(!dpl_reserve(txn, wanna)))
return MDBX_ENOMEM;
dpl_clear(txn->tw.dirtylist);
dpl_clear(txn->wr.dirtylist);
return MDBX_SUCCESS;
}
@ -79,7 +79,7 @@ __hot __noinline dpl_t *dpl_sort_slowpath(const MDBX_txn *txn) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
dpl_t *dl = txn->tw.dirtylist;
dpl_t *dl = txn->wr.dirtylist;
assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
const size_t unsorted = dl->length - dl->sorted;
if (likely(unsorted < MDBX_RADIXSORT_THRESHOLD) || unlikely(!dp_radixsort(dl->items + 1, dl->length))) {
@ -133,7 +133,7 @@ __hot __noinline size_t dpl_search(const MDBX_txn *txn, pgno_t pgno) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
dpl_t *dl = txn->tw.dirtylist;
dpl_t *dl = txn->wr.dirtylist;
assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
if (AUDIT_ENABLED()) {
for (const dp_t *ptr = dl->items + dl->sorted; --ptr > dl->items;) {
@ -175,7 +175,7 @@ __hot __noinline size_t dpl_search(const MDBX_txn *txn, pgno_t pgno) {
const page_t *debug_dpl_find(const MDBX_txn *txn, const pgno_t pgno) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
const dpl_t *dl = txn->tw.dirtylist;
const dpl_t *dl = txn->wr.dirtylist;
if (dl) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
@ -198,7 +198,7 @@ void dpl_remove_ex(const MDBX_txn *txn, size_t i, size_t npages) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
dpl_t *dl = txn->tw.dirtylist;
dpl_t *dl = txn->wr.dirtylist;
assert((intptr_t)i > 0 && i <= dl->length);
assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
dl->pages_including_loose -= npages;
@ -214,10 +214,10 @@ int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, siz
const dp_t dp = {page, pgno, (pgno_t)npages};
if ((txn->flags & MDBX_WRITEMAP) == 0) {
size_t *const ptr = ptr_disp(page, -(ptrdiff_t)sizeof(size_t));
*ptr = txn->tw.dirtylru;
*ptr = txn->wr.dirtylru;
}
dpl_t *dl = txn->tw.dirtylist;
dpl_t *dl = txn->wr.dirtylist;
tASSERT(txn, dl->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE);
tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
if (AUDIT_ENABLED()) {
@ -313,7 +313,7 @@ int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, siz
__cold bool dpl_check(MDBX_txn *txn) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
const dpl_t *const dl = txn->tw.dirtylist;
const dpl_t *const dl = txn->wr.dirtylist;
if (!dl) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
return true;
@ -322,7 +322,7 @@ __cold bool dpl_check(MDBX_txn *txn) {
assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
tASSERT(txn,
txn->tw.dirtyroom + dl->length == (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
txn->wr.dirtyroom + dl->length == (txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));
if (!AUDIT_ENABLED())
return true;
@ -362,28 +362,28 @@ __cold bool dpl_check(MDBX_txn *txn) {
return false;
}
const size_t rpa = pnl_search(txn->tw.repnl, dp->pgno, txn->geo.first_unallocated);
tASSERT(txn, rpa > MDBX_PNL_GETSIZE(txn->tw.repnl) || txn->tw.repnl[rpa] != dp->pgno);
if (rpa <= MDBX_PNL_GETSIZE(txn->tw.repnl) && unlikely(txn->tw.repnl[rpa] == dp->pgno))
const size_t rpa = pnl_search(txn->wr.repnl, dp->pgno, txn->geo.first_unallocated);
tASSERT(txn, rpa > MDBX_PNL_GETSIZE(txn->wr.repnl) || txn->wr.repnl[rpa] != dp->pgno);
if (rpa <= MDBX_PNL_GETSIZE(txn->wr.repnl) && unlikely(txn->wr.repnl[rpa] == dp->pgno))
return false;
if (num > 1) {
const size_t rpb = pnl_search(txn->tw.repnl, dp->pgno + num - 1, txn->geo.first_unallocated);
const size_t rpb = pnl_search(txn->wr.repnl, dp->pgno + num - 1, txn->geo.first_unallocated);
tASSERT(txn, rpa == rpb);
if (unlikely(rpa != rpb))
return false;
}
}
tASSERT(txn, loose == txn->tw.loose_count);
if (unlikely(loose != txn->tw.loose_count))
tASSERT(txn, loose == txn->wr.loose_count);
if (unlikely(loose != txn->wr.loose_count))
return false;
tASSERT(txn, pages == dl->pages_including_loose);
if (unlikely(pages != dl->pages_including_loose))
return false;
for (size_t i = 1; i <= MDBX_PNL_GETSIZE(txn->tw.retired_pages); ++i) {
const page_t *const dp = debug_dpl_find(txn, txn->tw.retired_pages[i]);
for (size_t i = 1; i <= MDBX_PNL_GETSIZE(txn->wr.retired_pages); ++i) {
const page_t *const dp = debug_dpl_find(txn, txn->wr.retired_pages[i]);
tASSERT(txn, !dp);
if (unlikely(dp))
return false;
@ -395,11 +395,11 @@ __cold bool dpl_check(MDBX_txn *txn) {
/*----------------------------------------------------------------------------*/
__noinline void dpl_lru_reduce(MDBX_txn *txn) {
NOTICE("lru-reduce %u -> %u", txn->tw.dirtylru, txn->tw.dirtylru >> 1);
NOTICE("lru-reduce %u -> %u", txn->wr.dirtylru, txn->wr.dirtylru >> 1);
tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0);
do {
txn->tw.dirtylru >>= 1;
dpl_t *dl = txn->tw.dirtylist;
txn->wr.dirtylru >>= 1;
dpl_t *dl = txn->wr.dirtylist;
for (size_t i = 1; i <= dl->length; ++i) {
size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t));
*ptr >>= 1;
@ -411,7 +411,7 @@ __noinline void dpl_lru_reduce(MDBX_txn *txn) {
void dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
if (MDBX_PNL_GETSIZE(pl) && txn->tw.dirtylist->length) {
if (MDBX_PNL_GETSIZE(pl) && txn->wr.dirtylist->length) {
tASSERT(txn, pnl_check_allocated(pl, (size_t)txn->geo.first_unallocated << spilled));
dpl_t *dl = dpl_sort(txn);
@ -466,9 +466,9 @@ void dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled) {
}
}
dl->sorted = dpl_setlen(dl, w - 1);
txn->tw.dirtyroom += r - w;
tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
(txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
txn->wr.dirtyroom += r - w;
tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length ==
(txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));
return;
}
}
@ -477,7 +477,7 @@ void dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled) {
void dpl_release_shadows(MDBX_txn *txn) {
tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0);
MDBX_env *env = txn->env;
dpl_t *const dl = txn->tw.dirtylist;
dpl_t *const dl = txn->wr.dirtylist;
for (size_t i = 1; i <= dl->length; i++)
page_shadow_release(env, dl->items[i].ptr, dpl_npages(dl, i));

18
src/dpl.h
View File

@ -46,7 +46,7 @@ static inline dpl_t *dpl_sort(const MDBX_txn *txn) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
dpl_t *dl = txn->tw.dirtylist;
dpl_t *dl = txn->wr.dirtylist;
tASSERT(txn, dl->length <= PAGELIST_LIMIT);
tASSERT(txn, dl->sorted <= dl->length);
tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
@ -72,7 +72,7 @@ static inline bool dpl_intersect(const MDBX_txn *txn, pgno_t pgno, size_t npages
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
dpl_t *dl = txn->tw.dirtylist;
dpl_t *dl = txn->wr.dirtylist;
tASSERT(txn, dl->sorted == dl->length);
tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
size_t const n = dpl_search(txn, pgno);
@ -96,7 +96,7 @@ static inline bool dpl_intersect(const MDBX_txn *txn, pgno_t pgno, size_t npages
MDBX_NOTHROW_PURE_FUNCTION static inline size_t dpl_exist(const MDBX_txn *txn, pgno_t pgno) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
dpl_t *dl = txn->tw.dirtylist;
dpl_t *dl = txn->wr.dirtylist;
size_t i = dpl_search(txn, pgno);
tASSERT(txn, (int)i > 0);
return (dl->items[i].pgno == pgno) ? i : 0;
@ -105,7 +105,7 @@ MDBX_NOTHROW_PURE_FUNCTION static inline size_t dpl_exist(const MDBX_txn *txn, p
MDBX_INTERNAL void dpl_remove_ex(const MDBX_txn *txn, size_t i, size_t npages);
static inline void dpl_remove(const MDBX_txn *txn, size_t i) {
dpl_remove_ex(txn, i, dpl_npages(txn->tw.dirtylist, i));
dpl_remove_ex(txn, i, dpl_npages(txn->wr.dirtylist, i));
}
MDBX_INTERNAL int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, size_t npages);
@ -114,19 +114,19 @@ MDBX_MAYBE_UNUSED MDBX_INTERNAL bool dpl_check(MDBX_txn *txn);
MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t dpl_age(const MDBX_txn *txn, size_t i) {
tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0);
const dpl_t *dl = txn->tw.dirtylist;
const dpl_t *dl = txn->wr.dirtylist;
assert((intptr_t)i > 0 && i <= dl->length);
size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t));
return txn->tw.dirtylru - (uint32_t)*ptr;
return txn->wr.dirtylru - (uint32_t)*ptr;
}
MDBX_INTERNAL void dpl_lru_reduce(MDBX_txn *txn);
static inline uint32_t dpl_lru_turn(MDBX_txn *txn) {
txn->tw.dirtylru += 1;
if (unlikely(txn->tw.dirtylru > UINT32_MAX / 3) && (txn->flags & MDBX_WRITEMAP) == 0)
txn->wr.dirtylru += 1;
if (unlikely(txn->wr.dirtylru > UINT32_MAX / 3) && (txn->flags & MDBX_WRITEMAP) == 0)
dpl_lru_reduce(txn);
return txn->tw.dirtylru;
return txn->wr.dirtylru;
}
MDBX_INTERNAL void dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled);

6
src/dxb.c
View File

@ -370,7 +370,7 @@ void dxb_sanitize_tail(MDBX_env *env, MDBX_txn *txn) {
return;
} else if (env->txn && env_txn0_owned(env)) {
/* inside write-txn */
last = meta_recent(env, &env->basal_txn->tw.troika).ptr_v->geometry.first_unallocated;
last = meta_recent(env, &env->basal_txn->wr.troika).ptr_v->geometry.first_unallocated;
} else if (env->flags & MDBX_RDONLY) {
/* read-only mode, no write-txn, no wlock mutex */
last = NUM_METAS;
@ -1300,8 +1300,8 @@ int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, troika
*troika = meta_tap(env);
for (MDBX_txn *txn = env->basal_txn; txn; txn = txn->nested)
if (troika != &txn->tw.troika)
txn->tw.troika = *troika;
if (troika != &txn->wr.troika)
txn->wr.troika = *troika;
/* LY: shrink datafile if needed */
if (unlikely(shrink)) {

6
src/env.c
View File

@ -71,7 +71,7 @@ retry:;
goto bailout;
}
const troika_t troika = (txn0_owned | should_unlock) ? env->basal_txn->tw.troika : meta_tap(env);
const troika_t troika = (txn0_owned | should_unlock) ? env->basal_txn->wr.troika : meta_tap(env);
const meta_ptr_t head = meta_recent(env, &troika);
const uint64_t unsynced_pages = atomic_load64(&env->lck->unsynced_pages, mo_Relaxed);
if (unsynced_pages == 0) {
@ -153,7 +153,7 @@ retry:;
#if MDBX_ENABLE_PGOP_STAT
env->lck->pgops.wops.weak += wops;
#endif /* MDBX_ENABLE_PGOP_STAT */
env->basal_txn->tw.troika = meta_tap(env);
env->basal_txn->wr.troika = meta_tap(env);
eASSERT(env, !env->txn && !env->basal_txn->nested);
goto retry;
}
@ -177,7 +177,7 @@ retry:;
DEBUG("meta-head %" PRIaPGNO ", %s, sync_pending %" PRIu64, data_page(head.ptr_c)->pgno,
durable_caption(head.ptr_c), unsynced_pages);
meta_t meta = *head.ptr_c;
rc = dxb_sync_locked(env, flags, &meta, &env->basal_txn->tw.troika);
rc = dxb_sync_locked(env, flags, &meta, &env->basal_txn->wr.troika);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
}

152
src/gc-get.c
View File

@ -590,12 +590,12 @@ static inline bool is_gc_usable(MDBX_txn *txn, const MDBX_cursor *mc, const uint
return true;
}
static inline bool is_already_reclaimed(const MDBX_txn *txn, txnid_t id) { return txl_contain(txn->tw.gc.retxl, id); }
static inline bool is_already_reclaimed(const MDBX_txn *txn, txnid_t id) { return txl_contain(txn->wr.gc.retxl, id); }
__hot static pgno_t repnl_get_single(MDBX_txn *txn) {
const size_t len = MDBX_PNL_GETSIZE(txn->tw.repnl);
const size_t len = MDBX_PNL_GETSIZE(txn->wr.repnl);
assert(len > 0);
pgno_t *target = MDBX_PNL_EDGE(txn->tw.repnl);
pgno_t *target = MDBX_PNL_EDGE(txn->wr.repnl);
const ptrdiff_t dir = MDBX_PNL_ASCENDING ? 1 : -1;
/* Есть ТРИ потенциально выигрышные, но противо-направленные тактики:
@ -663,7 +663,7 @@ __hot static pgno_t repnl_get_single(MDBX_txn *txn) {
#else
/* вырезаем элемент с перемещением хвоста */
const pgno_t pgno = *scan;
MDBX_PNL_SETSIZE(txn->tw.repnl, len - 1);
MDBX_PNL_SETSIZE(txn->wr.repnl, len - 1);
while (++scan <= target)
scan[-1] = *scan;
return pgno;
@ -676,44 +676,44 @@ __hot static pgno_t repnl_get_single(MDBX_txn *txn) {
const pgno_t pgno = *target;
#if MDBX_PNL_ASCENDING
/* вырезаем элемент с перемещением хвоста */
MDBX_PNL_SETSIZE(txn->tw.repnl, len - 1);
for (const pgno_t *const end = txn->tw.repnl + len - 1; target <= end; ++target)
MDBX_PNL_SETSIZE(txn->wr.repnl, len - 1);
for (const pgno_t *const end = txn->wr.repnl + len - 1; target <= end; ++target)
*target = target[1];
#else
/* перемещать хвост не нужно, просто усекам список */
MDBX_PNL_SETSIZE(txn->tw.repnl, len - 1);
MDBX_PNL_SETSIZE(txn->wr.repnl, len - 1);
#endif
return pgno;
}
__hot static pgno_t repnl_get_sequence(MDBX_txn *txn, const size_t num, uint8_t flags) {
const size_t len = MDBX_PNL_GETSIZE(txn->tw.repnl);
pgno_t *edge = MDBX_PNL_EDGE(txn->tw.repnl);
const size_t len = MDBX_PNL_GETSIZE(txn->wr.repnl);
pgno_t *edge = MDBX_PNL_EDGE(txn->wr.repnl);
assert(len >= num && num > 1);
const size_t seq = num - 1;
#if !MDBX_PNL_ASCENDING
if (edge[-(ptrdiff_t)seq] - *edge == seq) {
if (unlikely(flags & ALLOC_RESERVE))
return P_INVALID;
assert(edge == scan4range_checker(txn->tw.repnl, seq));
assert(edge == scan4range_checker(txn->wr.repnl, seq));
/* перемещать хвост не нужно, просто усекам список */
MDBX_PNL_SETSIZE(txn->tw.repnl, len - num);
MDBX_PNL_SETSIZE(txn->wr.repnl, len - num);
return *edge;
}
#endif
pgno_t *target = scan4seq_impl(edge, len, seq);
assert(target == scan4range_checker(txn->tw.repnl, seq));
assert(target == scan4range_checker(txn->wr.repnl, seq));
if (target) {
if (unlikely(flags & ALLOC_RESERVE))
return P_INVALID;
const pgno_t pgno = *target;
/* вырезаем найденную последовательность с перемещением хвоста */
MDBX_PNL_SETSIZE(txn->tw.repnl, len - num);
MDBX_PNL_SETSIZE(txn->wr.repnl, len - num);
#if MDBX_PNL_ASCENDING
for (const pgno_t *const end = txn->tw.repnl + len - num; target <= end; ++target)
for (const pgno_t *const end = txn->wr.repnl + len - num; target <= end; ++target)
*target = target[num];
#else
for (const pgno_t *const end = txn->tw.repnl + len; ++target <= end;)
for (const pgno_t *const end = txn->wr.repnl + len; ++target <= end;)
target[-(ptrdiff_t)num] = *target;
#endif
return pgno;
@ -762,7 +762,7 @@ static inline pgr_t page_alloc_finalize(MDBX_env *const env, MDBX_txn *const txn
* обновляться PTE с последующей генерацией page-fault и чтением данных из
* грязной I/O очереди. Из-за этого штраф за лишнюю запись может быть
* сравним с избегаемым ненужным чтением. */
if (txn->tw.prefault_write_activated) {
if (txn->wr.prefault_write_activated) {
void *const pattern = ptr_disp(env->page_auxbuf, need_clean ? env->ps : env->ps * 2);
size_t file_offset = pgno2bytes(env, pgno);
if (likely(num == 1)) {
@ -823,7 +823,7 @@ static inline pgr_t page_alloc_finalize(MDBX_env *const env, MDBX_txn *const txn
ret.err = page_dirty(txn, ret.page, (pgno_t)num);
bailout:
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
#if MDBX_ENABLE_PROFGC
size_t majflt_after;
prof->xtime_cpu += osal_cputime(&majflt_after) - cputime_before;
@ -843,7 +843,7 @@ pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags)
#endif /* MDBX_ENABLE_PROFGC */
eASSERT(env, num > 0 || (flags & ALLOC_RESERVE));
eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
eASSERT(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
size_t newnext;
const uint64_t monotime_begin = (MDBX_ENABLE_PROFGC || (num > 1 && env->options.gc_time_limit)) ? osal_monotime() : 0;
@ -858,15 +858,15 @@ pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags)
#if MDBX_ENABLE_PROFGC
prof->xpages += 1;
#endif /* MDBX_ENABLE_PROFGC */
if (MDBX_PNL_GETSIZE(txn->tw.repnl) >= num) {
eASSERT(env, MDBX_PNL_LAST(txn->tw.repnl) < txn->geo.first_unallocated &&
MDBX_PNL_FIRST(txn->tw.repnl) < txn->geo.first_unallocated);
if (MDBX_PNL_GETSIZE(txn->wr.repnl) >= num) {
eASSERT(env, MDBX_PNL_LAST(txn->wr.repnl) < txn->geo.first_unallocated &&
MDBX_PNL_FIRST(txn->wr.repnl) < txn->geo.first_unallocated);
pgno = repnl_get_sequence(txn, num, flags);
if (likely(pgno))
goto done;
}
} else {
eASSERT(env, num == 0 || MDBX_PNL_GETSIZE(txn->tw.repnl) == 0);
eASSERT(env, num == 0 || MDBX_PNL_GETSIZE(txn->wr.repnl) == 0);
eASSERT(env, !(flags & ALLOC_RESERVE) || num == 0);
}
@ -884,7 +884,7 @@ pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags)
* Иначе попытка увеличить резерв может приводить к необходимости ещё
* большего резерва из-за увеличения списка переработанных страниц. */
(flags & ALLOC_RESERVE) == 0) {
if (txn->dbs[FREE_DBI].branch_pages && MDBX_PNL_GETSIZE(txn->tw.repnl) < env->maxgc_large1page / 2)
if (txn->dbs[FREE_DBI].branch_pages && MDBX_PNL_GETSIZE(txn->wr.repnl) < env->maxgc_large1page / 2)
flags += ALLOC_COALESCE;
}
@ -894,8 +894,8 @@ pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags)
gc->dbi_state = txn->dbi_state;
gc->top_and_flags = z_fresh_mark;
txn->tw.prefault_write_activated = env->options.prefault_write;
if (txn->tw.prefault_write_activated) {
txn->wr.prefault_write_activated = env->options.prefault_write;
if (txn->wr.prefault_write_activated) {
/* Проверка посредством minicore() существенно снижает затраты, но в
* простейших случаях (тривиальный бенчмарк) интегральная производительность
* становится вдвое меньше. А на платформах без mincore() и с проблемной
@ -908,7 +908,7 @@ pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags)
(txn->dbs[FREE_DBI].branch_pages == 0 && txn->geo.now < 1234) ||
/* Не суетимся если страница в зоне включенного упреждающего чтения */
(readahead_enabled && pgno + num < readahead_edge))
txn->tw.prefault_write_activated = false;
txn->wr.prefault_write_activated = false;
}
retry_gc_refresh_oldest:;
@ -924,9 +924,9 @@ retry_gc_have_oldest:
txnid_t id = 0;
MDBX_cursor_op op = MDBX_FIRST;
if (flags & ALLOC_LIFO) {
if (!txn->tw.gc.retxl) {
txn->tw.gc.retxl = txl_alloc();
if (unlikely(!txn->tw.gc.retxl)) {
if (!txn->wr.gc.retxl) {
txn->wr.gc.retxl = txl_alloc();
if (unlikely(!txn->wr.gc.retxl)) {
ret.err = MDBX_ENOMEM;
goto fail;
}
@ -934,9 +934,9 @@ retry_gc_have_oldest:
/* Begin lookup backward from oldest reader */
id = detent - 1;
op = MDBX_SET_RANGE;
} else if (txn->tw.gc.last_reclaimed) {
} else if (txn->wr.gc.last_reclaimed) {
/* Continue lookup forward from last-reclaimed */
id = txn->tw.gc.last_reclaimed + 1;
id = txn->wr.gc.last_reclaimed + 1;
if (id >= detent)
goto depleted_gc;
op = MDBX_SET_RANGE;
@ -994,9 +994,9 @@ next_gc:;
}
const size_t gc_len = MDBX_PNL_GETSIZE(gc_pnl);
TRACE("gc-read: id #%" PRIaTXN " len %zu, re-list will %zu ", id, gc_len, gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl));
TRACE("gc-read: id #%" PRIaTXN " len %zu, re-list will %zu ", id, gc_len, gc_len + MDBX_PNL_GETSIZE(txn->wr.repnl));
if (unlikely(gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl) >= env->maxgc_large1page)) {
if (unlikely(gc_len + MDBX_PNL_GETSIZE(txn->wr.repnl) >= env->maxgc_large1page)) {
/* Don't try to coalesce too much. */
if (flags & ALLOC_SHOULD_SCAN) {
eASSERT(env, flags & ALLOC_COALESCE);
@ -1006,9 +1006,9 @@ next_gc:;
env->lck->pgops.gc_prof.coalescences += 1;
#endif /* MDBX_ENABLE_PROFGC */
TRACE("clear %s %s", "ALLOC_COALESCE", "since got threshold");
if (MDBX_PNL_GETSIZE(txn->tw.repnl) >= num) {
eASSERT(env, MDBX_PNL_LAST(txn->tw.repnl) < txn->geo.first_unallocated &&
MDBX_PNL_FIRST(txn->tw.repnl) < txn->geo.first_unallocated);
if (MDBX_PNL_GETSIZE(txn->wr.repnl) >= num) {
eASSERT(env, MDBX_PNL_LAST(txn->wr.repnl) < txn->geo.first_unallocated &&
MDBX_PNL_FIRST(txn->wr.repnl) < txn->geo.first_unallocated);
if (likely(num == 1)) {
pgno = repnl_get_single(txn);
goto done;
@ -1019,34 +1019,34 @@ next_gc:;
}
flags -= ALLOC_COALESCE | ALLOC_SHOULD_SCAN;
}
if (unlikely(/* list is too long already */ MDBX_PNL_GETSIZE(txn->tw.repnl) >= env->options.rp_augment_limit) &&
if (unlikely(/* list is too long already */ MDBX_PNL_GETSIZE(txn->wr.repnl) >= env->options.rp_augment_limit) &&
((/* not a slot-request from gc-update */ num &&
/* have enough unallocated space */ txn->geo.upper >= txn->geo.first_unallocated + num &&
monotime_since_cached(monotime_begin, &now_cache) + txn->tw.gc.time_acc >= env->options.gc_time_limit) ||
gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl) >= PAGELIST_LIMIT)) {
monotime_since_cached(monotime_begin, &now_cache) + txn->wr.gc.time_acc >= env->options.gc_time_limit) ||
gc_len + MDBX_PNL_GETSIZE(txn->wr.repnl) >= PAGELIST_LIMIT)) {
/* Stop reclaiming to avoid large/overflow the page list. This is a rare
* case while search for a continuously multi-page region in a
* large database, see https://libmdbx.dqdkfa.ru/dead-github/issues/123 */
NOTICE("stop reclaiming %s: %zu (current) + %zu "
"(chunk) -> %zu, rp_augment_limit %u",
likely(gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl) < PAGELIST_LIMIT) ? "since rp_augment_limit was reached"
likely(gc_len + MDBX_PNL_GETSIZE(txn->wr.repnl) < PAGELIST_LIMIT) ? "since rp_augment_limit was reached"
: "to avoid PNL overflow",
MDBX_PNL_GETSIZE(txn->tw.repnl), gc_len, gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl),
MDBX_PNL_GETSIZE(txn->wr.repnl), gc_len, gc_len + MDBX_PNL_GETSIZE(txn->wr.repnl),
env->options.rp_augment_limit);
goto depleted_gc;
}
}
/* Remember ID of readed GC record */
txn->tw.gc.last_reclaimed = id;
txn->wr.gc.last_reclaimed = id;
if (flags & ALLOC_LIFO) {
ret.err = txl_append(&txn->tw.gc.retxl, id);
ret.err = txl_append(&txn->wr.gc.retxl, id);
if (unlikely(ret.err != MDBX_SUCCESS))
goto fail;
}
/* Append PNL from GC record to tw.repnl */
ret.err = pnl_need(&txn->tw.repnl, gc_len);
/* Append PNL from GC record to wr.repnl */
ret.err = pnl_need(&txn->wr.repnl, gc_len);
if (unlikely(ret.err != MDBX_SUCCESS))
goto fail;
@ -1061,36 +1061,36 @@ next_gc:;
#if MDBX_ENABLE_PROFGC
const uint64_t merge_begin = osal_monotime();
#endif /* MDBX_ENABLE_PROFGC */
pnl_merge(txn->tw.repnl, gc_pnl);
pnl_merge(txn->wr.repnl, gc_pnl);
#if MDBX_ENABLE_PROFGC
prof->pnl_merge.calls += 1;
prof->pnl_merge.volume += MDBX_PNL_GETSIZE(txn->tw.repnl);
prof->pnl_merge.volume += MDBX_PNL_GETSIZE(txn->wr.repnl);
prof->pnl_merge.time += osal_monotime() - merge_begin;
#endif /* MDBX_ENABLE_PROFGC */
flags |= ALLOC_SHOULD_SCAN;
if (AUDIT_ENABLED()) {
if (unlikely(!pnl_check(txn->tw.repnl, txn->geo.first_unallocated))) {
if (unlikely(!pnl_check(txn->wr.repnl, txn->geo.first_unallocated))) {
ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid txn retired-list");
ret.err = MDBX_CORRUPTED;
goto fail;
}
} else {
eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated));
eASSERT(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated));
}
eASSERT(env, dpl_check(txn));
eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.repnl) == 0 || MDBX_PNL_MOST(txn->tw.repnl) < txn->geo.first_unallocated);
if (MDBX_ENABLE_REFUND && MDBX_PNL_GETSIZE(txn->tw.repnl) &&
unlikely(MDBX_PNL_MOST(txn->tw.repnl) == txn->geo.first_unallocated - 1)) {
eASSERT(env, MDBX_PNL_GETSIZE(txn->wr.repnl) == 0 || MDBX_PNL_MOST(txn->wr.repnl) < txn->geo.first_unallocated);
if (MDBX_ENABLE_REFUND && MDBX_PNL_GETSIZE(txn->wr.repnl) &&
unlikely(MDBX_PNL_MOST(txn->wr.repnl) == txn->geo.first_unallocated - 1)) {
/* Refund suitable pages into "unallocated" space */
txn_refund(txn);
}
eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
eASSERT(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
/* Done for a kick-reclaim mode, actually no page needed */
if (unlikely(num == 0)) {
eASSERT(env, ret.err == MDBX_SUCCESS);
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "early-exit for slot", id, MDBX_PNL_GETSIZE(txn->tw.repnl));
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "early-exit for slot", id, MDBX_PNL_GETSIZE(txn->wr.repnl));
goto early_exit;
}
@ -1098,16 +1098,16 @@ next_gc:;
eASSERT(env, op == MDBX_PREV || op == MDBX_NEXT);
if (flags & ALLOC_COALESCE) {
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "coalesce-continue", id, MDBX_PNL_GETSIZE(txn->tw.repnl));
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "coalesce-continue", id, MDBX_PNL_GETSIZE(txn->wr.repnl));
goto next_gc;
}
scan:
eASSERT(env, flags & ALLOC_SHOULD_SCAN);
eASSERT(env, num > 0);
if (MDBX_PNL_GETSIZE(txn->tw.repnl) >= num) {
eASSERT(env, MDBX_PNL_LAST(txn->tw.repnl) < txn->geo.first_unallocated &&
MDBX_PNL_FIRST(txn->tw.repnl) < txn->geo.first_unallocated);
if (MDBX_PNL_GETSIZE(txn->wr.repnl) >= num) {
eASSERT(env, MDBX_PNL_LAST(txn->wr.repnl) < txn->geo.first_unallocated &&
MDBX_PNL_FIRST(txn->wr.repnl) < txn->geo.first_unallocated);
if (likely(num == 1)) {
eASSERT(env, !(flags & ALLOC_RESERVE));
pgno = repnl_get_single(txn);
@ -1119,12 +1119,12 @@ scan:
}
flags -= ALLOC_SHOULD_SCAN;
if (ret.err == MDBX_SUCCESS) {
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "continue-search", id, MDBX_PNL_GETSIZE(txn->tw.repnl));
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "continue-search", id, MDBX_PNL_GETSIZE(txn->wr.repnl));
goto next_gc;
}
depleted_gc:
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "gc-depleted", id, MDBX_PNL_GETSIZE(txn->tw.repnl));
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "gc-depleted", id, MDBX_PNL_GETSIZE(txn->wr.repnl));
ret.err = MDBX_NOTFOUND;
if (flags & ALLOC_SHOULD_SCAN)
goto scan;
@ -1143,8 +1143,8 @@ depleted_gc:
newnext = txn->geo.first_unallocated + num;
/* Does reclaiming stopped at the last steady point? */
const meta_ptr_t recent = meta_recent(env, &txn->tw.troika);
const meta_ptr_t prefer_steady = meta_prefer_steady(env, &txn->tw.troika);
const meta_ptr_t recent = meta_recent(env, &txn->wr.troika);
const meta_ptr_t prefer_steady = meta_prefer_steady(env, &txn->wr.troika);
if (recent.ptr_c != prefer_steady.ptr_c && prefer_steady.is_steady && detent == prefer_steady.txnid + 1) {
DEBUG("gc-kick-steady: recent %" PRIaTXN "-%s, steady %" PRIaTXN "-%s, detent %" PRIaTXN, recent.txnid,
durable_caption(recent.ptr_c), prefer_steady.txnid, durable_caption(prefer_steady.ptr_c), detent);
@ -1170,7 +1170,7 @@ depleted_gc:
DEBUG("gc-wipe-steady, rc %d", ret.err);
if (unlikely(ret.err != MDBX_SUCCESS))
goto fail;
eASSERT(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->tw.troika).ptr_c);
eASSERT(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->wr.troika).ptr_c);
goto retry_gc_refresh_oldest;
}
if ((autosync_threshold && atomic_load64(&env->lck->unsynced_pages, mo_Relaxed) >= autosync_threshold) ||
@ -1183,12 +1183,12 @@ depleted_gc:
env->lck->pgops.gc_prof.flushes += 1;
#endif /* MDBX_ENABLE_PROFGC */
meta_t meta = *recent.ptr_c;
ret.err = dxb_sync_locked(env, env->flags & MDBX_WRITEMAP, &meta, &txn->tw.troika);
ret.err = dxb_sync_locked(env, env->flags & MDBX_WRITEMAP, &meta, &txn->wr.troika);
DEBUG("gc-make-steady, rc %d", ret.err);
eASSERT(env, ret.err != MDBX_RESULT_TRUE);
if (unlikely(ret.err != MDBX_SUCCESS))
goto fail;
eASSERT(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->tw.troika).ptr_c);
eASSERT(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->wr.troika).ptr_c);
goto retry_gc_refresh_oldest;
}
}
@ -1263,7 +1263,7 @@ done:
if (likely((flags & ALLOC_RESERVE) == 0)) {
if (pgno) {
eASSERT(env, pgno + num <= txn->geo.first_unallocated && pgno >= NUM_METAS);
eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
eASSERT(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
} else {
pgno = txn->geo.first_unallocated;
txn->geo.first_unallocated += (pgno_t)num;
@ -1275,7 +1275,7 @@ done:
if (unlikely(ret.err != MDBX_SUCCESS)) {
fail:
eASSERT(env, ret.err != MDBX_SUCCESS);
eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
eASSERT(env, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
int level;
const char *what;
if (flags & ALLOC_RESERVE) {
@ -1291,14 +1291,14 @@ done:
"unable alloc %zu %s, alloc-flags 0x%x, err %d, txn-flags "
"0x%x, re-list-len %zu, loose-count %zu, gc: height %u, "
"branch %zu, leaf %zu, large %zu, entries %zu\n",
num, what, flags, ret.err, txn->flags, MDBX_PNL_GETSIZE(txn->tw.repnl), txn->tw.loose_count,
num, what, flags, ret.err, txn->flags, MDBX_PNL_GETSIZE(txn->wr.repnl), txn->wr.loose_count,
txn->dbs[FREE_DBI].height, (size_t)txn->dbs[FREE_DBI].branch_pages,
(size_t)txn->dbs[FREE_DBI].leaf_pages, (size_t)txn->dbs[FREE_DBI].large_pages,
(size_t)txn->dbs[FREE_DBI].items);
ret.page = nullptr;
}
if (num > 1)
txn->tw.gc.time_acc += monotime_since_cached(monotime_begin, &now_cache);
txn->wr.gc.time_acc += monotime_since_cached(monotime_begin, &now_cache);
} else {
early_exit:
DEBUG("return nullptr for %zu pages for ALLOC_%s, rc %d", num, num ? "RESERVE" : "SLOT", ret.err);
@ -1317,20 +1317,20 @@ __hot pgr_t gc_alloc_single(const MDBX_cursor *const mc) {
tASSERT(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY));
/* If there are any loose pages, just use them */
while (likely(txn->tw.loose_pages)) {
while (likely(txn->wr.loose_pages)) {
#if MDBX_ENABLE_REFUND
if (unlikely(txn->tw.loose_refund_wl > txn->geo.first_unallocated)) {
if (unlikely(txn->wr.loose_refund_wl > txn->geo.first_unallocated)) {
txn_refund(txn);
if (!txn->tw.loose_pages)
if (!txn->wr.loose_pages)
break;
}
#endif /* MDBX_ENABLE_REFUND */
page_t *lp = txn->tw.loose_pages;
page_t *lp = txn->wr.loose_pages;
MDBX_ASAN_UNPOISON_MEMORY_REGION(lp, txn->env->ps);
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
txn->tw.loose_pages = page_next(lp);
txn->tw.loose_count--;
txn->wr.loose_pages = page_next(lp);
txn->wr.loose_count--;
DEBUG_EXTRA("db %d use loose page %" PRIaPGNO, cursor_dbi_dbg(mc), lp->pgno);
tASSERT(txn, lp->pgno < txn->geo.first_unallocated);
tASSERT(txn, lp->pgno >= NUM_METAS);
@ -1340,7 +1340,7 @@ __hot pgr_t gc_alloc_single(const MDBX_cursor *const mc) {
return ret;
}
if (likely(MDBX_PNL_GETSIZE(txn->tw.repnl) > 0))
if (likely(MDBX_PNL_GETSIZE(txn->wr.repnl) > 0))
return page_alloc_finalize(txn->env, txn, mc, repnl_get_single(txn), 1);
return gc_alloc_ex(mc, 1, ALLOC_DEFAULT);

264
src/gc-put.c
View File

@ -11,7 +11,7 @@ MDBX_MAYBE_UNUSED static inline const char *dbg_prefix(const gcu_t *ctx) {
return is_lifo(ctx->cursor.txn) ? " lifo" : " fifo";
}
static inline size_t backlog_size(MDBX_txn *txn) { return MDBX_PNL_GETSIZE(txn->tw.repnl) + txn->tw.loose_count; }
static inline size_t backlog_size(MDBX_txn *txn) { return MDBX_PNL_GETSIZE(txn->wr.repnl) + txn->wr.loose_count; }
static int clean_stored_retired(MDBX_txn *txn, gcu_t *ctx) {
int err = MDBX_SUCCESS;
@ -55,7 +55,7 @@ static int touch_gc(gcu_t *ctx) {
MDBX_val key, val;
key.iov_base = val.iov_base = nullptr;
key.iov_len = sizeof(txnid_t);
val.iov_len = MDBX_PNL_SIZEOF(ctx->cursor.txn->tw.retired_pages);
val.iov_len = MDBX_PNL_SIZEOF(ctx->cursor.txn->wr.retired_pages);
ctx->cursor.flags |= z_gcu_preparation;
int err = cursor_touch(&ctx->cursor, &key, &val);
ctx->cursor.flags -= z_gcu_preparation;
@ -71,7 +71,7 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) {
size_t for_split = ctx->retired_stored == 0;
tASSERT(txn, is_pointed(&ctx->cursor) || txn->dbs[FREE_DBI].leaf_pages == 0);
const intptr_t retired_left = MDBX_PNL_SIZEOF(txn->tw.retired_pages) - ctx->retired_stored;
const intptr_t retired_left = MDBX_PNL_SIZEOF(txn->wr.retired_pages) - ctx->retired_stored;
size_t for_repnl = 0;
if (MDBX_ENABLE_BIGFOOT && retired_left > 0) {
for_repnl = (retired_left + txn->env->maxgc_large1page - 1) / txn->env->maxgc_large1page;
@ -79,7 +79,7 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) {
for (size_t entries = for_repnl; entries > 1; for_split += entries)
entries = (entries + per_branch_page - 1) / per_branch_page;
} else if (!MDBX_ENABLE_BIGFOOT && retired_left != 0) {
for_repnl = largechunk_npages(txn->env, MDBX_PNL_SIZEOF(txn->tw.retired_pages));
for_repnl = largechunk_npages(txn->env, MDBX_PNL_SIZEOF(txn->wr.retired_pages));
}
const size_t for_tree_before_touch = for_cow + for_rebalance + for_split;
@ -101,7 +101,7 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) {
TRACE("== after-touch, backlog %zu, err %d", backlog_size(txn), err);
if (!MDBX_ENABLE_BIGFOOT && unlikely(for_repnl > 1) &&
MDBX_PNL_GETSIZE(txn->tw.retired_pages) != ctx->retired_stored && err == MDBX_SUCCESS) {
MDBX_PNL_GETSIZE(txn->wr.retired_pages) != ctx->retired_stored && err == MDBX_SUCCESS) {
if (unlikely(ctx->retired_stored)) {
err = clean_stored_retired(txn, ctx);
if (unlikely(err != MDBX_SUCCESS))
@ -145,51 +145,51 @@ static inline void zeroize_reserved(const MDBX_env *env, MDBX_val pnl) {
}
static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
tASSERT(txn, txn->tw.loose_count > 0);
/* Return loose page numbers to tw.repnl,
tASSERT(txn, txn->wr.loose_count > 0);
/* Return loose page numbers to wr.repnl,
* though usually none are left at this point.
* The pages themselves remain in dirtylist. */
if (unlikely(!txn->tw.gc.retxl && txn->tw.gc.last_reclaimed < 1)) {
TRACE("%s: try allocate gc-slot for %zu loose-pages", dbg_prefix(ctx), txn->tw.loose_count);
if (unlikely(!txn->wr.gc.retxl && txn->wr.gc.last_reclaimed < 1)) {
TRACE("%s: try allocate gc-slot for %zu loose-pages", dbg_prefix(ctx), txn->wr.loose_count);
int err = gc_alloc_ex(&ctx->cursor, 0, ALLOC_RESERVE).err;
if (err == MDBX_SUCCESS) {
TRACE("%s: retry since gc-slot for %zu loose-pages available", dbg_prefix(ctx), txn->tw.loose_count);
TRACE("%s: retry since gc-slot for %zu loose-pages available", dbg_prefix(ctx), txn->wr.loose_count);
return MDBX_RESULT_TRUE;
}
/* Put loose page numbers in tw.retired_pages,
* since unable to return ones to tw.repnl. */
err = pnl_need(&txn->tw.retired_pages, txn->tw.loose_count);
/* Put loose page numbers in wr.retired_pages,
* since unable to return ones to wr.repnl. */
err = pnl_need(&txn->wr.retired_pages, txn->wr.loose_count);
if (unlikely(err != MDBX_SUCCESS))
return err;
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
pnl_append_prereserved(txn->tw.retired_pages, lp->pgno);
for (page_t *lp = txn->wr.loose_pages; lp; lp = page_next(lp)) {
pnl_append_prereserved(txn->wr.retired_pages, lp->pgno);
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
}
TRACE("%s: append %zu loose-pages to retired-pages", dbg_prefix(ctx), txn->tw.loose_count);
TRACE("%s: append %zu loose-pages to retired-pages", dbg_prefix(ctx), txn->wr.loose_count);
} else {
/* Room for loose pages + temp PNL with same */
int err = pnl_need(&txn->tw.repnl, 2 * txn->tw.loose_count + 2);
int err = pnl_need(&txn->wr.repnl, 2 * txn->wr.loose_count + 2);
if (unlikely(err != MDBX_SUCCESS))
return err;
pnl_t loose = txn->tw.repnl + MDBX_PNL_ALLOCLEN(txn->tw.repnl) - txn->tw.loose_count - 1;
pnl_t loose = txn->wr.repnl + MDBX_PNL_ALLOCLEN(txn->wr.repnl) - txn->wr.loose_count - 1;
size_t count = 0;
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
for (page_t *lp = txn->wr.loose_pages; lp; lp = page_next(lp)) {
tASSERT(txn, lp->flags == P_LOOSE);
loose[++count] = lp->pgno;
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
}
tASSERT(txn, count == txn->tw.loose_count);
tASSERT(txn, count == txn->wr.loose_count);
MDBX_PNL_SETSIZE(loose, count);
pnl_sort(loose, txn->geo.first_unallocated);
pnl_merge(txn->tw.repnl, loose);
TRACE("%s: append %zu loose-pages to reclaimed-pages", dbg_prefix(ctx), txn->tw.loose_count);
pnl_merge(txn->wr.repnl, loose);
TRACE("%s: append %zu loose-pages to reclaimed-pages", dbg_prefix(ctx), txn->wr.loose_count);
}
/* filter-out list of dirty-pages from loose-pages */
dpl_t *const dl = txn->tw.dirtylist;
dpl_t *const dl = txn->wr.dirtylist;
if (dl) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
tASSERT(txn, dl->sorted <= dl->length);
@ -209,21 +209,21 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
}
}
TRACE("%s: filtered-out loose-pages from %zu -> %zu dirty-pages", dbg_prefix(ctx), dl->length, w);
tASSERT(txn, txn->tw.loose_count == dl->length - w);
tASSERT(txn, txn->wr.loose_count == dl->length - w);
dl->sorted -= sorted_out;
tASSERT(txn, dl->sorted <= w);
dpl_setlen(dl, w);
dl->pages_including_loose -= txn->tw.loose_count;
txn->tw.dirtyroom += txn->tw.loose_count;
tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
(txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
dl->pages_including_loose -= txn->wr.loose_count;
txn->wr.dirtyroom += txn->wr.loose_count;
tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length ==
(txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));
} else {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
}
txn->tw.loose_pages = nullptr;
txn->tw.loose_count = 0;
txn->wr.loose_pages = nullptr;
txn->wr.loose_count = 0;
#if MDBX_ENABLE_REFUND
txn->tw.loose_refund_wl = 0;
txn->wr.loose_refund_wl = 0;
#endif /* MDBX_ENABLE_REFUND */
return MDBX_SUCCESS;
}
@ -250,17 +250,17 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
tASSERT(txn, ctx->bigfoot <= txn->txnid);
}
retired_pages_before = MDBX_PNL_GETSIZE(txn->tw.retired_pages);
retired_pages_before = MDBX_PNL_GETSIZE(txn->wr.retired_pages);
err = prepare_backlog(txn, ctx);
if (unlikely(err != MDBX_SUCCESS))
return err;
if (retired_pages_before != MDBX_PNL_GETSIZE(txn->tw.retired_pages)) {
if (retired_pages_before != MDBX_PNL_GETSIZE(txn->wr.retired_pages)) {
TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx), retired_pages_before,
MDBX_PNL_GETSIZE(txn->tw.retired_pages));
MDBX_PNL_GETSIZE(txn->wr.retired_pages));
break;
}
pnl_sort(txn->tw.retired_pages, txn->geo.first_unallocated);
pnl_sort(txn->wr.retired_pages, txn->geo.first_unallocated);
ctx->retired_stored = 0;
ctx->bigfoot = txn->txnid;
do {
@ -268,15 +268,15 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
err = prepare_backlog(txn, ctx);
if (unlikely(err != MDBX_SUCCESS))
return err;
if (ctx->retired_stored >= MDBX_PNL_GETSIZE(txn->tw.retired_pages)) {
if (ctx->retired_stored >= MDBX_PNL_GETSIZE(txn->wr.retired_pages)) {
TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx), retired_pages_before,
MDBX_PNL_GETSIZE(txn->tw.retired_pages));
MDBX_PNL_GETSIZE(txn->wr.retired_pages));
break;
}
}
key.iov_len = sizeof(txnid_t);
key.iov_base = &ctx->bigfoot;
const size_t left = MDBX_PNL_GETSIZE(txn->tw.retired_pages) - ctx->retired_stored;
const size_t left = MDBX_PNL_GETSIZE(txn->wr.retired_pages) - ctx->retired_stored;
const size_t chunk =
(left > txn->env->maxgc_large1page && ctx->bigfoot < MAX_TXNID) ? txn->env->maxgc_large1page : left;
data.iov_len = (chunk + 1) * sizeof(pgno_t);
@ -293,9 +293,9 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
memset(data.iov_base, 0xBB, data.iov_len);
#endif /* MDBX_DEBUG && (ENABLE_MEMCHECK || __SANITIZE_ADDRESS__) */
if (retired_pages_before == MDBX_PNL_GETSIZE(txn->tw.retired_pages)) {
if (retired_pages_before == MDBX_PNL_GETSIZE(txn->wr.retired_pages)) {
const size_t at = (is_lifo(txn) == MDBX_PNL_ASCENDING) ? left - chunk : ctx->retired_stored;
pgno_t *const begin = txn->tw.retired_pages + at;
pgno_t *const begin = txn->wr.retired_pages + at;
/* MDBX_PNL_ASCENDING == false && LIFO == false:
* - the larger pgno is at the beginning of retired list
* and should be placed with the larger txnid.
@ -310,15 +310,15 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
ctx->bigfoot, (unsigned)(ctx->bigfoot - txn->txnid), chunk, at, at + chunk, retired_pages_before);
}
ctx->retired_stored += chunk;
} while (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages) && (++ctx->bigfoot, true));
} while (retired_pages_before != MDBX_PNL_GETSIZE(txn->tw.retired_pages));
} while (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->wr.retired_pages) && (++ctx->bigfoot, true));
} while (retired_pages_before != MDBX_PNL_GETSIZE(txn->wr.retired_pages));
#else
/* Write to last page of GC */
key.iov_len = sizeof(txnid_t);
key.iov_base = &txn->txnid;
do {
prepare_backlog(txn, ctx);
data.iov_len = MDBX_PNL_SIZEOF(txn->tw.retired_pages);
data.iov_len = MDBX_PNL_SIZEOF(txn->wr.retired_pages);
err = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE);
if (unlikely(err != MDBX_SUCCESS))
return err;
@ -331,13 +331,13 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
memset(data.iov_base, 0xBB, data.iov_len);
#endif /* MDBX_DEBUG && (ENABLE_MEMCHECK || __SANITIZE_ADDRESS__) */
/* Retry if tw.retired_pages[] grew during the Put() */
} while (data.iov_len < MDBX_PNL_SIZEOF(txn->tw.retired_pages));
/* Retry if wr.retired_pages[] grew during the Put() */
} while (data.iov_len < MDBX_PNL_SIZEOF(txn->wr.retired_pages));
ctx->retired_stored = MDBX_PNL_GETSIZE(txn->tw.retired_pages);
pnl_sort(txn->tw.retired_pages, txn->geo.first_unallocated);
tASSERT(txn, data.iov_len == MDBX_PNL_SIZEOF(txn->tw.retired_pages));
memcpy(data.iov_base, txn->tw.retired_pages, data.iov_len);
ctx->retired_stored = MDBX_PNL_GETSIZE(txn->wr.retired_pages);
pnl_sort(txn->wr.retired_pages, txn->geo.first_unallocated);
tASSERT(txn, data.iov_len == MDBX_PNL_SIZEOF(txn->wr.retired_pages));
memcpy(data.iov_base, txn->wr.retired_pages, data.iov_len);
TRACE("%s: put-retired #%zu @ %" PRIaTXN, dbg_prefix(ctx), ctx->retired_stored, txn->txnid);
#endif /* MDBX_ENABLE_BIGFOOT */
@ -345,7 +345,7 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
size_t i = ctx->retired_stored;
DEBUG_EXTRA("txn %" PRIaTXN " root %" PRIaPGNO " num %zu, retired-PNL", txn->txnid, txn->dbs[FREE_DBI].root, i);
for (; i; i--)
DEBUG_EXTRA_PRINT(" %" PRIaPGNO, txn->tw.retired_pages[i]);
DEBUG_EXTRA_PRINT(" %" PRIaPGNO, txn->wr.retired_pages[i]);
DEBUG_EXTRA_PRINT("%s\n", ".");
}
return MDBX_SUCCESS;
@ -359,15 +359,15 @@ typedef struct gcu_rid_result {
static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left) {
rid_t r;
if (is_lifo(txn)) {
if (txn->tw.gc.retxl == nullptr) {
txn->tw.gc.retxl = txl_alloc();
if (unlikely(!txn->tw.gc.retxl)) {
if (txn->wr.gc.retxl == nullptr) {
txn->wr.gc.retxl = txl_alloc();
if (unlikely(!txn->wr.gc.retxl)) {
r.err = MDBX_ENOMEM;
goto return_error;
}
}
if (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) < txl_max &&
left > (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page && !ctx->dense) {
if (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) < txl_max &&
left > (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page && !ctx->dense) {
/* Hужен свободный для для сохранения списка страниц. */
bool need_cleanup = false;
txnid_t snap_oldest = 0;
@ -376,11 +376,11 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left)
r.err = gc_alloc_ex(&ctx->cursor, 0, ALLOC_RESERVE).err;
snap_oldest = txn->env->lck->cached_oldest.weak;
if (likely(r.err == MDBX_SUCCESS)) {
TRACE("%s: took @%" PRIaTXN " from GC", dbg_prefix(ctx), MDBX_PNL_LAST(txn->tw.gc.retxl));
TRACE("%s: took @%" PRIaTXN " from GC", dbg_prefix(ctx), MDBX_PNL_LAST(txn->wr.gc.retxl));
need_cleanup = true;
}
} while (r.err == MDBX_SUCCESS && MDBX_PNL_GETSIZE(txn->tw.gc.retxl) < txl_max &&
left > (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page);
} while (r.err == MDBX_SUCCESS && MDBX_PNL_GETSIZE(txn->wr.gc.retxl) < txl_max &&
left > (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page);
if (likely(r.err == MDBX_SUCCESS)) {
TRACE("%s: got enough from GC.", dbg_prefix(ctx));
@ -389,34 +389,34 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left)
/* LY: some troubles... */
goto return_error;
if (MDBX_PNL_GETSIZE(txn->tw.gc.retxl)) {
if (MDBX_PNL_GETSIZE(txn->wr.gc.retxl)) {
if (need_cleanup) {
txl_sort(txn->tw.gc.retxl);
txl_sort(txn->wr.gc.retxl);
ctx->cleaned_slot = 0;
}
ctx->rid = MDBX_PNL_LAST(txn->tw.gc.retxl);
ctx->rid = MDBX_PNL_LAST(txn->wr.gc.retxl);
} else {
tASSERT(txn, txn->tw.gc.last_reclaimed == 0);
tASSERT(txn, txn->wr.gc.last_reclaimed == 0);
if (unlikely(txn_snapshot_oldest(txn) != snap_oldest))
/* should retry gc_alloc_ex()
* if the oldest reader changes since the last attempt */
goto retry_rid;
/* no reclaimable GC entries,
* therefore no entries with ID < mdbx_find_oldest(txn) */
txn->tw.gc.last_reclaimed = ctx->rid = snap_oldest;
txn->wr.gc.last_reclaimed = ctx->rid = snap_oldest;
TRACE("%s: none recycled yet, set rid to @%" PRIaTXN, dbg_prefix(ctx), ctx->rid);
}
/* В GC нет годных к переработке записей,
* будем использовать свободные id в обратном порядке. */
while (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) < txl_max &&
left > (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page) {
while (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) < txl_max &&
left > (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page) {
if (unlikely(ctx->rid <= MIN_TXNID)) {
ctx->dense = true;
if (unlikely(MDBX_PNL_GETSIZE(txn->tw.gc.retxl) <= ctx->reused_slot)) {
if (unlikely(MDBX_PNL_GETSIZE(txn->wr.gc.retxl) <= ctx->reused_slot)) {
VERBOSE("** restart: reserve depleted (reused_gc_slot %zu >= "
"gc.reclaimed %zu)",
ctx->reused_slot, MDBX_PNL_GETSIZE(txn->tw.gc.retxl));
ctx->reused_slot, MDBX_PNL_GETSIZE(txn->wr.gc.retxl));
goto return_restart;
}
break;
@ -444,7 +444,7 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left)
}
tASSERT(txn, !ctx->dense);
r.err = txl_append(&txn->tw.gc.retxl, ctx->rid);
r.err = txl_append(&txn->wr.gc.retxl, ctx->rid);
if (unlikely(r.err != MDBX_SUCCESS))
goto return_error;
@ -468,12 +468,12 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left)
}
}
const size_t i = MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot;
tASSERT(txn, i > 0 && i <= MDBX_PNL_GETSIZE(txn->tw.gc.retxl));
r.rid = txn->tw.gc.retxl[i];
const size_t i = MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot;
tASSERT(txn, i > 0 && i <= MDBX_PNL_GETSIZE(txn->wr.gc.retxl));
r.rid = txn->wr.gc.retxl[i];
TRACE("%s: take @%" PRIaTXN " from lifo-reclaimed[%zu]", dbg_prefix(ctx), r.rid, i);
} else {
tASSERT(txn, txn->tw.gc.retxl == nullptr);
tASSERT(txn, txn->wr.gc.retxl == nullptr);
if (unlikely(ctx->rid == 0)) {
ctx->rid = txn_snapshot_oldest(txn);
MDBX_val key;
@ -496,7 +496,7 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left)
r.rid = 0;
return r;
}
txn->tw.gc.last_reclaimed = ctx->rid;
txn->wr.gc.last_reclaimed = ctx->rid;
ctx->cleaned_id = ctx->rid + 1;
}
r.rid = ctx->rid--;
@ -538,16 +538,16 @@ int gc_update(MDBX_txn *txn, gcu_t *ctx) {
txn->cursors[FREE_DBI] = &ctx->cursor;
int rc;
/* txn->tw.repnl[] can grow and shrink during this call.
* txn->tw.gc.last_reclaimed and txn->tw.retired_pages[] can only grow.
* But page numbers cannot disappear from txn->tw.retired_pages[]. */
/* txn->wr.repnl[] can grow and shrink during this call.
* txn->wr.gc.last_reclaimed and txn->wr.retired_pages[] can only grow.
* But page numbers cannot disappear from txn->wr.retired_pages[]. */
retry_clean_adj:
ctx->reserve_adj = 0;
retry:
ctx->loop += !(ctx->prev_first_unallocated > txn->geo.first_unallocated);
TRACE(">> restart, loop %u", ctx->loop);
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, dpl_check(txn));
if (unlikely(/* paranoia */ ctx->loop > ((MDBX_DEBUG > 0) ? 12 : 42))) {
ERROR("txn #%" PRIaTXN " too more loops %u, bailout", txn->txnid, ctx->loop);
@ -569,22 +569,22 @@ retry:
ctx->amount = 0;
ctx->fill_idx = ~0u;
ctx->cleaned_id = 0;
ctx->rid = txn->tw.gc.last_reclaimed;
ctx->rid = txn->wr.gc.last_reclaimed;
while (true) {
/* Come back here after each Put() in case retired-list changed */
TRACE("%s", " >> continue");
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
MDBX_val key, data;
if (is_lifo(txn)) {
if (ctx->cleaned_slot < (txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0)) {
if (ctx->cleaned_slot < (txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) : 0)) {
ctx->reserved = 0;
ctx->cleaned_slot = 0;
ctx->reused_slot = 0;
ctx->fill_idx = ~0u;
/* LY: cleanup reclaimed records. */
do {
ctx->cleaned_id = txn->tw.gc.retxl[++ctx->cleaned_slot];
ctx->cleaned_id = txn->wr.gc.retxl[++ctx->cleaned_slot];
tASSERT(txn, ctx->cleaned_slot > 0 && ctx->cleaned_id <= env->lck->cached_oldest.weak);
key.iov_base = &ctx->cleaned_id;
key.iov_len = sizeof(ctx->cleaned_id);
@ -602,16 +602,16 @@ retry:
rc = cursor_del(&ctx->cursor, 0);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
} while (ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->tw.gc.retxl));
txl_sort(txn->tw.gc.retxl);
} while (ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->wr.gc.retxl));
txl_sort(txn->wr.gc.retxl);
}
} else {
/* Удаляем оставшиеся вынутые из GC записи. */
while (txn->tw.gc.last_reclaimed && ctx->cleaned_id <= txn->tw.gc.last_reclaimed) {
while (txn->wr.gc.last_reclaimed && ctx->cleaned_id <= txn->wr.gc.last_reclaimed) {
rc = outer_first(&ctx->cursor, &key, nullptr);
if (rc == MDBX_NOTFOUND) {
ctx->cleaned_id = txn->tw.gc.last_reclaimed + 1;
ctx->rid = txn->tw.gc.last_reclaimed;
ctx->cleaned_id = txn->wr.gc.last_reclaimed + 1;
ctx->rid = txn->wr.gc.last_reclaimed;
ctx->reserved = 0;
ctx->reused_slot = 0;
break;
@ -629,12 +629,12 @@ retry:
ctx->reused_slot = 0;
}
ctx->cleaned_id = unaligned_peek_u64(4, key.iov_base);
if (ctx->cleaned_id > txn->tw.gc.last_reclaimed)
if (ctx->cleaned_id > txn->wr.gc.last_reclaimed)
break;
rc = prepare_backlog(txn, ctx);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
tASSERT(txn, ctx->cleaned_id <= txn->tw.gc.last_reclaimed);
tASSERT(txn, ctx->cleaned_id <= txn->wr.gc.last_reclaimed);
tASSERT(txn, ctx->cleaned_id <= env->lck->cached_oldest.weak);
TRACE("%s: cleanup-reclaimed-id %" PRIaTXN, dbg_prefix(ctx), ctx->cleaned_id);
tASSERT(txn, *txn->cursors == &ctx->cursor);
@ -644,7 +644,7 @@ retry:
}
}
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, dpl_check(txn));
if (AUDIT_ENABLED()) {
rc = audit_ex(txn, ctx->retired_stored, false);
@ -654,7 +654,7 @@ retry:
/* return suitable into unallocated space */
if (txn_refund(txn)) {
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
if (AUDIT_ENABLED()) {
rc = audit_ex(txn, ctx->retired_stored, false);
if (unlikely(rc != MDBX_SUCCESS))
@ -662,7 +662,7 @@ retry:
}
}
if (txn->tw.loose_pages) {
if (txn->wr.loose_pages) {
/* put loose pages into the reclaimed- or retired-list */
rc = gcu_loose(txn, ctx);
if (unlikely(rc != MDBX_SUCCESS)) {
@ -670,19 +670,19 @@ retry:
continue;
goto bailout;
}
tASSERT(txn, txn->tw.loose_pages == 0);
tASSERT(txn, txn->wr.loose_pages == 0);
}
if (unlikely(ctx->reserved > MDBX_PNL_GETSIZE(txn->tw.repnl)) &&
(ctx->loop < 5 || ctx->reserved - MDBX_PNL_GETSIZE(txn->tw.repnl) > env->maxgc_large1page / 2)) {
if (unlikely(ctx->reserved > MDBX_PNL_GETSIZE(txn->wr.repnl)) &&
(ctx->loop < 5 || ctx->reserved - MDBX_PNL_GETSIZE(txn->wr.repnl) > env->maxgc_large1page / 2)) {
TRACE("%s: reclaimed-list changed %zu -> %zu, retry", dbg_prefix(ctx), ctx->amount,
MDBX_PNL_GETSIZE(txn->tw.repnl));
ctx->reserve_adj += ctx->reserved - MDBX_PNL_GETSIZE(txn->tw.repnl);
MDBX_PNL_GETSIZE(txn->wr.repnl));
ctx->reserve_adj += ctx->reserved - MDBX_PNL_GETSIZE(txn->wr.repnl);
goto retry;
}
ctx->amount = MDBX_PNL_GETSIZE(txn->tw.repnl);
ctx->amount = MDBX_PNL_GETSIZE(txn->wr.repnl);
if (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages)) {
if (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->wr.retired_pages)) {
/* store retired-list into GC */
rc = gcu_retired(txn, ctx);
if (unlikely(rc != MDBX_SUCCESS))
@ -690,8 +690,8 @@ retry:
continue;
}
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, txn->tw.loose_count == 0);
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, txn->wr.loose_count == 0);
TRACE("%s", " >> reserving");
if (AUDIT_ENABLED()) {
@ -704,7 +704,7 @@ retry:
"lifo-reclaimed-slots %zu, "
"reused-gc-slots %zu",
dbg_prefix(ctx), ctx->amount, ctx->reserved, ctx->reserve_adj, left,
txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0, ctx->reused_slot);
txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) : 0, ctx->reused_slot);
if (0 >= (intptr_t)left)
break;
@ -722,7 +722,7 @@ retry:
size_t chunk = left;
if (unlikely(left > env->maxgc_large1page)) {
const size_t avail_gc_slots = txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot + 1
const size_t avail_gc_slots = txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot + 1
: (ctx->rid < INT16_MAX) ? (size_t)ctx->rid
: INT16_MAX;
if (likely(avail_gc_slots > 1)) {
@ -749,8 +749,8 @@ retry:
size_t avail = ((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) /* - 1 + span */;
if (tail > avail) {
for (size_t i = ctx->amount - span; i > 0; --i) {
if (MDBX_PNL_ASCENDING ? (txn->tw.repnl[i] + span)
: (txn->tw.repnl[i] - span) == txn->tw.repnl[i + span]) {
if (MDBX_PNL_ASCENDING ? (txn->wr.repnl[i] + span)
: (txn->wr.repnl[i] - span) == txn->wr.repnl[i + span]) {
span += 1;
avail = ((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) - 1 + span;
if (avail >= tail)
@ -791,7 +791,7 @@ retry:
ctx->reserved + chunk + 1, reservation_gc_id);
prepare_backlog(txn, ctx);
rc = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE | MDBX_NOOVERWRITE);
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
@ -802,14 +802,14 @@ retry:
continue;
}
tASSERT(txn, ctx->cleaned_slot == (txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0));
tASSERT(txn, ctx->cleaned_slot == (txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) : 0));
TRACE("%s", " >> filling");
/* Fill in the reserved records */
size_t excess_slots = 0;
ctx->fill_idx = txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot : ctx->reused_slot;
ctx->fill_idx = txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot : ctx->reused_slot;
rc = MDBX_SUCCESS;
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, dpl_check(txn));
if (ctx->amount) {
MDBX_val key, data;
@ -817,7 +817,7 @@ retry:
key.iov_base = data.iov_base = nullptr;
size_t left = ctx->amount, excess = 0;
if (txn->tw.gc.retxl == nullptr) {
if (txn->wr.gc.retxl == nullptr) {
tASSERT(txn, is_lifo(txn) == 0);
rc = outer_first(&ctx->cursor, &key, &data);
if (unlikely(rc != MDBX_SUCCESS)) {
@ -830,33 +830,33 @@ retry:
while (true) {
txnid_t fill_gc_id;
TRACE("%s: left %zu of %zu", dbg_prefix(ctx), left, MDBX_PNL_GETSIZE(txn->tw.repnl));
if (txn->tw.gc.retxl == nullptr) {
TRACE("%s: left %zu of %zu", dbg_prefix(ctx), left, MDBX_PNL_GETSIZE(txn->wr.repnl));
if (txn->wr.gc.retxl == nullptr) {
tASSERT(txn, is_lifo(txn) == 0);
fill_gc_id = key.iov_base ? unaligned_peek_u64(4, key.iov_base) : MIN_TXNID;
if (ctx->fill_idx == 0 || fill_gc_id > txn->tw.gc.last_reclaimed) {
if (ctx->fill_idx == 0 || fill_gc_id > txn->wr.gc.last_reclaimed) {
if (!left)
break;
VERBOSE("** restart: reserve depleted (fill_idx %zu, fill_id %" PRIaTXN " > last_reclaimed %" PRIaTXN
", left %zu",
ctx->fill_idx, fill_gc_id, txn->tw.gc.last_reclaimed, left);
ctx->fill_idx, fill_gc_id, txn->wr.gc.last_reclaimed, left);
ctx->reserve_adj = (ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0;
goto retry;
}
ctx->fill_idx -= 1;
} else {
tASSERT(txn, is_lifo(txn) != 0);
if (ctx->fill_idx >= MDBX_PNL_GETSIZE(txn->tw.gc.retxl)) {
if (ctx->fill_idx >= MDBX_PNL_GETSIZE(txn->wr.gc.retxl)) {
if (!left)
break;
VERBOSE("** restart: reserve depleted (fill_idx %zu >= "
"gc.reclaimed %zu, left %zu",
ctx->fill_idx, MDBX_PNL_GETSIZE(txn->tw.gc.retxl), left);
ctx->fill_idx, MDBX_PNL_GETSIZE(txn->wr.gc.retxl), left);
ctx->reserve_adj = (ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0;
goto retry;
}
ctx->fill_idx += 1;
fill_gc_id = txn->tw.gc.retxl[ctx->fill_idx];
fill_gc_id = txn->wr.gc.retxl[ctx->fill_idx];
TRACE("%s: seek-reservation @%" PRIaTXN " at gc.reclaimed[%zu]", dbg_prefix(ctx), fill_gc_id, ctx->fill_idx);
key.iov_base = &fill_gc_id;
key.iov_len = sizeof(fill_gc_id);
@ -864,7 +864,7 @@ retry:
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
}
tASSERT(txn, ctx->cleaned_slot == (txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0));
tASSERT(txn, ctx->cleaned_slot == (txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) : 0));
tASSERT(txn, fill_gc_id > 0 && fill_gc_id <= env->lck->cached_oldest.weak);
key.iov_base = &fill_gc_id;
key.iov_len = sizeof(fill_gc_id);
@ -888,33 +888,33 @@ retry:
goto bailout;
zeroize_reserved(env, data);
if (unlikely(txn->tw.loose_count || ctx->amount != MDBX_PNL_GETSIZE(txn->tw.repnl))) {
if (unlikely(txn->wr.loose_count || ctx->amount != MDBX_PNL_GETSIZE(txn->wr.repnl))) {
NOTICE("** restart: reclaimed-list changed (%zu -> %zu, loose +%zu)", ctx->amount,
MDBX_PNL_GETSIZE(txn->tw.repnl), txn->tw.loose_count);
MDBX_PNL_GETSIZE(txn->wr.repnl), txn->wr.loose_count);
if (ctx->loop < 5 || (ctx->loop > 10 && (ctx->loop & 1)))
goto retry_clean_adj;
goto retry;
}
if (unlikely(txn->tw.gc.retxl ? ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->tw.gc.retxl)
: ctx->cleaned_id < txn->tw.gc.last_reclaimed)) {
if (unlikely(txn->wr.gc.retxl ? ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->wr.gc.retxl)
: ctx->cleaned_id < txn->wr.gc.last_reclaimed)) {
NOTICE("%s", "** restart: reclaimed-slots changed");
goto retry;
}
if (unlikely(ctx->retired_stored != MDBX_PNL_GETSIZE(txn->tw.retired_pages))) {
tASSERT(txn, ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages));
if (unlikely(ctx->retired_stored != MDBX_PNL_GETSIZE(txn->wr.retired_pages))) {
tASSERT(txn, ctx->retired_stored < MDBX_PNL_GETSIZE(txn->wr.retired_pages));
NOTICE("** restart: retired-list growth (%zu -> %zu)", ctx->retired_stored,
MDBX_PNL_GETSIZE(txn->tw.retired_pages));
MDBX_PNL_GETSIZE(txn->wr.retired_pages));
goto retry;
}
pgno_t *dst = data.iov_base;
*dst++ = (pgno_t)chunk;
pgno_t *src = MDBX_PNL_BEGIN(txn->tw.repnl) + left - chunk;
pgno_t *src = MDBX_PNL_BEGIN(txn->wr.repnl) + left - chunk;
memcpy(dst, src, chunk * sizeof(pgno_t));
pgno_t *from = src, *to = src + chunk;
TRACE("%s: fill %zu [ %zu:%" PRIaPGNO "...%zu:%" PRIaPGNO "] @%" PRIaTXN, dbg_prefix(ctx), chunk,
from - txn->tw.repnl, from[0], to - txn->tw.repnl, to[-1], fill_gc_id);
from - txn->wr.repnl, from[0], to - txn->wr.repnl, to[-1], fill_gc_id);
left -= chunk;
if (AUDIT_ENABLED()) {
@ -925,7 +925,7 @@ retry:
next:
if (txn->tw.gc.retxl == nullptr) {
if (txn->wr.gc.retxl == nullptr) {
tASSERT(txn, is_lifo(txn) == 0);
rc = outer_next(&ctx->cursor, &key, &data, MDBX_NEXT);
if (unlikely(rc != MDBX_SUCCESS)) {
@ -950,9 +950,9 @@ retry:
}
tASSERT(txn, rc == MDBX_SUCCESS);
if (unlikely(txn->tw.loose_count != 0 || ctx->amount != MDBX_PNL_GETSIZE(txn->tw.repnl))) {
NOTICE("** restart: got %zu loose pages (reclaimed-list %zu -> %zu)", txn->tw.loose_count, ctx->amount,
MDBX_PNL_GETSIZE(txn->tw.repnl));
if (unlikely(txn->wr.loose_count != 0 || ctx->amount != MDBX_PNL_GETSIZE(txn->wr.repnl))) {
NOTICE("** restart: got %zu loose pages (reclaimed-list %zu -> %zu)", txn->wr.loose_count, ctx->amount,
MDBX_PNL_GETSIZE(txn->wr.repnl));
goto retry;
}
@ -965,12 +965,12 @@ retry:
goto retry;
}
tASSERT(txn, txn->tw.gc.retxl == nullptr || ctx->cleaned_slot == MDBX_PNL_GETSIZE(txn->tw.gc.retxl));
tASSERT(txn, txn->wr.gc.retxl == nullptr || ctx->cleaned_slot == MDBX_PNL_GETSIZE(txn->wr.gc.retxl));
bailout:
txn->cursors[FREE_DBI] = ctx->cursor.next;
MDBX_PNL_SETSIZE(txn->tw.repnl, 0);
MDBX_PNL_SETSIZE(txn->wr.repnl, 0);
#if MDBX_ENABLE_PROFGC
env->lck->pgops.gc_prof.wloops += (uint32_t)ctx->loop;
#endif /* MDBX_ENABLE_PROFGC */

10
src/internals.h
View File

@ -206,9 +206,9 @@ struct MDBX_txn {
union {
struct {
/* For read txns: This thread/txn's reader table slot, or nullptr. */
reader_slot_t *reader;
} to;
/* For read txns: This thread/txn's slot table slot, or nullptr. */
reader_slot_t *slot;
} ro;
struct {
troika_t troika;
pnl_t __restrict repnl; /* Reclaimed GC pages */
@ -236,7 +236,7 @@ struct MDBX_txn {
/* The list of loose pages that became unused and may be reused
* in this transaction, linked through `page_next()`. */
page_t *__restrict loose_pages;
/* Number of loose pages (tw.loose_pages) */
/* Number of loose pages (wr.loose_pages) */
size_t loose_count;
union {
struct {
@ -250,7 +250,7 @@ struct MDBX_txn {
size_t writemap_spilled_npages;
};
/* In write txns, next is located the array of cursors for each DB */
} tw;
} wr;
};
};

4
src/meta.c
View File

@ -252,9 +252,9 @@ __cold int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto) {
/* force oldest refresh */
atomic_store32(&env->lck->rdt_refresh_flag, true, mo_Relaxed);
env->basal_txn->tw.troika = meta_tap(env);
env->basal_txn->wr.troika = meta_tap(env);
for (MDBX_txn *scan = env->basal_txn->nested; scan; scan = scan->nested)
scan->tw.troika = env->basal_txn->tw.troika;
scan->wr.troika = env->basal_txn->wr.troika;
return err;
}

4
src/mvcc-readers.c
View File

@ -308,7 +308,7 @@ __cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) {
bool notify_eof_of_loop = false;
int retry = 0;
do {
const txnid_t steady = env->txn->tw.troika.txnid[env->txn->tw.troika.prefer_steady];
const txnid_t steady = env->txn->wr.troika.txnid[env->txn->wr.troika.prefer_steady];
env->lck->rdt_refresh_flag.weak = /* force refresh */ true;
oldest = mvcc_shapshot_oldest(env, steady);
eASSERT(env, oldest < env->basal_txn->txnid);
@ -374,7 +374,7 @@ __cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) {
if (safe64_read(&stucked->txnid) != straggler || !pid)
continue;
const meta_ptr_t head = meta_recent(env, &env->txn->tw.troika);
const meta_ptr_t head = meta_recent(env, &env->txn->wr.troika);
const txnid_t gap = (head.txnid - straggler) / xMDBX_TXNID_STEP;
const uint64_t head_retired = unaligned_peek_u64(4, head.ptr_c->pages_retired);
const size_t space = (head_retired > hold_retired) ? pgno2bytes(env, (pgno_t)(head_retired - hold_retired)) : 0;

4
src/page-get.c
View File

@ -443,8 +443,8 @@ static __always_inline pgr_t page_get_inline(const uint16_t ILL, const MDBX_curs
const size_t i = dpl_search(spiller, pgno);
tASSERT(txn, (intptr_t)i > 0);
if (spiller->tw.dirtylist->items[i].pgno == pgno) {
r.page = spiller->tw.dirtylist->items[i].ptr;
if (spiller->wr.dirtylist->items[i].pgno == pgno) {
r.page = spiller->wr.dirtylist->items[i].ptr;
break;
}

82
src/page-ops.c
View File

@ -144,14 +144,14 @@ __cold pgr_t __must_check_result page_unspill(MDBX_txn *const txn, const page_t
}
__hot int page_touch_modifable(MDBX_txn *txn, const page_t *const mp) {
tASSERT(txn, is_modifable(txn, mp) && txn->tw.dirtylist);
tASSERT(txn, is_modifable(txn, mp) && txn->wr.dirtylist);
tASSERT(txn, !is_largepage(mp) && !is_subpage(mp));
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
const size_t n = dpl_search(txn, mp->pgno);
if (MDBX_AVOID_MSYNC && unlikely(txn->tw.dirtylist->items[n].pgno != mp->pgno)) {
if (MDBX_AVOID_MSYNC && unlikely(txn->wr.dirtylist->items[n].pgno != mp->pgno)) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP));
tASSERT(txn, n > 0 && n <= txn->tw.dirtylist->length + 1);
tASSERT(txn, n > 0 && n <= txn->wr.dirtylist->length + 1);
VERBOSE("unspill page %" PRIaPGNO, mp->pgno);
#if MDBX_ENABLE_PGOP_STAT
txn->env->lck->pgops.unspill.weak += 1;
@ -159,11 +159,11 @@ __hot int page_touch_modifable(MDBX_txn *txn, const page_t *const mp) {
return page_dirty(txn, (page_t *)mp, 1);
}
tASSERT(txn, n > 0 && n <= txn->tw.dirtylist->length);
tASSERT(txn, txn->tw.dirtylist->items[n].pgno == mp->pgno && txn->tw.dirtylist->items[n].ptr == mp);
tASSERT(txn, n > 0 && n <= txn->wr.dirtylist->length);
tASSERT(txn, txn->wr.dirtylist->items[n].pgno == mp->pgno && txn->wr.dirtylist->items[n].ptr == mp);
if (!MDBX_AVOID_MSYNC || (txn->flags & MDBX_WRITEMAP) == 0) {
size_t *const ptr = ptr_disp(txn->tw.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t));
*ptr = txn->tw.dirtylru;
size_t *const ptr = ptr_disp(txn->wr.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t));
*ptr = txn->wr.dirtylru;
}
return MDBX_SUCCESS;
}
@ -179,7 +179,7 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *c
page_t *np;
if (is_frozen(txn, mp)) {
/* CoW the page */
rc = pnl_need(&txn->tw.retired_pages, 1);
rc = pnl_need(&txn->wr.retired_pages, 1);
if (unlikely(rc != MDBX_SUCCESS))
goto fail;
const pgr_t par = gc_alloc_single(mc);
@ -191,7 +191,7 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *c
const pgno_t pgno = np->pgno;
DEBUG("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno, pgno);
tASSERT(txn, mp->pgno != pgno);
pnl_append_prereserved(txn->tw.retired_pages, mp->pgno);
pnl_append_prereserved(txn->wr.retired_pages, mp->pgno);
/* Update the parent page, if any, to point to the new page */
if (likely(mc->top)) {
page_t *parent = mc->pg[mc->top - 1];
@ -227,7 +227,7 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *c
}
DEBUG("clone db %d page %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno);
tASSERT(txn, txn->tw.dirtylist->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE);
tASSERT(txn, txn->wr.dirtylist->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE);
/* No - copy it */
np = page_shadow_alloc(txn, 1);
if (unlikely(!np)) {
@ -369,7 +369,7 @@ static inline bool suitable4loose(const MDBX_txn *txn, pgno_t pgno) {
* страница не примыкает к какой-либо из уже находящийся в reclaimed.
* 2) стоит подумать над тем, чтобы при большом loose-списке отбрасывать
половину в reclaimed. */
return txn->tw.loose_count < txn->env->options.dp_loose_limit &&
return txn->wr.loose_count < txn->env->options.dp_loose_limit &&
(!MDBX_ENABLE_REFUND ||
/* skip pages near to the end in favor of compactification */
txn->geo.first_unallocated > pgno + txn->env->options.dp_loose_limit ||
@ -417,14 +417,14 @@ int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null
status = frozen;
if (ASSERT_ENABLED()) {
for (MDBX_txn *scan = txn; scan; scan = scan->parent) {
tASSERT(txn, !txn->tw.spilled.list || !spill_search(scan, pgno));
tASSERT(txn, !scan->tw.dirtylist || !debug_dpl_find(scan, pgno));
tASSERT(txn, !txn->wr.spilled.list || !spill_search(scan, pgno));
tASSERT(txn, !scan->wr.dirtylist || !debug_dpl_find(scan, pgno));
}
}
goto status_done;
} else if (pageflags && txn->tw.dirtylist) {
} else if (pageflags && txn->wr.dirtylist) {
if ((di = dpl_exist(txn, pgno)) != 0) {
mp = txn->tw.dirtylist->items[di].ptr;
mp = txn->wr.dirtylist->items[di].ptr;
tASSERT(txn, is_modifable(txn, mp));
status = modifable;
goto status_done;
@ -461,16 +461,16 @@ int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null
tASSERT(txn, !is_spilled(txn, mp));
tASSERT(txn, !is_shadowed(txn, mp));
tASSERT(txn, !debug_dpl_find(txn, pgno));
tASSERT(txn, !txn->tw.spilled.list || !spill_search(txn, pgno));
tASSERT(txn, !txn->wr.spilled.list || !spill_search(txn, pgno));
} else if (is_modifable(txn, mp)) {
status = modifable;
if (txn->tw.dirtylist)
if (txn->wr.dirtylist)
di = dpl_exist(txn, pgno);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) || !is_spilled(txn, mp));
tASSERT(txn, !txn->tw.spilled.list || !spill_search(txn, pgno));
tASSERT(txn, !txn->wr.spilled.list || !spill_search(txn, pgno));
} else if (is_shadowed(txn, mp)) {
status = shadowed;
tASSERT(txn, !txn->tw.spilled.list || !spill_search(txn, pgno));
tASSERT(txn, !txn->wr.spilled.list || !spill_search(txn, pgno));
tASSERT(txn, !debug_dpl_find(txn, pgno));
} else {
tASSERT(txn, is_spilled(txn, mp));
@ -504,7 +504,7 @@ status_done:
if (status == frozen) {
retire:
DEBUG("retire %zu page %" PRIaPGNO, npages, pgno);
rc = pnl_append_span(&txn->tw.retired_pages, pgno, npages);
rc = pnl_append_span(&txn->wr.retired_pages, pgno, npages);
tASSERT(txn, dpl_check(txn));
return rc;
}
@ -560,17 +560,17 @@ status_done:
if (status == modifable) {
/* Dirty page from this transaction */
/* If suitable we can reuse it through loose list */
if (likely(npages == 1 && suitable4loose(txn, pgno)) && (di || !txn->tw.dirtylist)) {
if (likely(npages == 1 && suitable4loose(txn, pgno)) && (di || !txn->wr.dirtylist)) {
DEBUG("loosen dirty page %" PRIaPGNO, pgno);
if (MDBX_DEBUG != 0 || unlikely(txn->env->flags & MDBX_PAGEPERTURB))
memset(page_data(mp), -1, txn->env->ps - PAGEHDRSZ);
mp->txnid = INVALID_TXNID;
mp->flags = P_LOOSE;
page_next(mp) = txn->tw.loose_pages;
txn->tw.loose_pages = mp;
txn->tw.loose_count++;
page_next(mp) = txn->wr.loose_pages;
txn->wr.loose_pages = mp;
txn->wr.loose_count++;
#if MDBX_ENABLE_REFUND
txn->tw.loose_refund_wl = (pgno + 2 > txn->tw.loose_refund_wl) ? pgno + 2 : txn->tw.loose_refund_wl;
txn->wr.loose_refund_wl = (pgno + 2 > txn->wr.loose_refund_wl) ? pgno + 2 : txn->wr.loose_refund_wl;
#endif /* MDBX_ENABLE_REFUND */
VALGRIND_MAKE_MEM_NOACCESS(page_data(mp), txn->env->ps - PAGEHDRSZ);
MDBX_ASAN_POISON_MEMORY_REGION(page_data(mp), txn->env->ps - PAGEHDRSZ);
@ -608,8 +608,8 @@ status_done:
reclaim:
DEBUG("reclaim %zu %s page %" PRIaPGNO, npages, "dirty", pgno);
rc = pnl_insert_span(&txn->tw.repnl, pgno, npages);
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
rc = pnl_insert_span(&txn->wr.repnl, pgno, npages);
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, dpl_check(txn));
return rc;
}
@ -660,10 +660,10 @@ status_done:
__hot int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, size_t npages) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
mp->txnid = txn->front_txnid;
if (!txn->tw.dirtylist) {
if (!txn->wr.dirtylist) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
txn->tw.writemap_dirty_npages += npages;
tASSERT(txn, txn->tw.spilled.list == nullptr);
txn->wr.writemap_dirty_npages += npages;
tASSERT(txn, txn->wr.spilled.list == nullptr);
return MDBX_SUCCESS;
}
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
@ -671,29 +671,29 @@ __hot int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, size_t npage
#if xMDBX_DEBUG_SPILLING == 2
txn->env->debug_dirtied_act += 1;
ENSURE(txn->env, txn->env->debug_dirtied_act < txn->env->debug_dirtied_est);
ENSURE(txn->env, txn->tw.dirtyroom + txn->tw.loose_count > 0);
ENSURE(txn->env, txn->wr.dirtyroom + txn->wr.loose_count > 0);
#endif /* xMDBX_DEBUG_SPILLING == 2 */
int rc;
if (unlikely(txn->tw.dirtyroom == 0)) {
if (txn->tw.loose_count) {
page_t *lp = txn->tw.loose_pages;
if (unlikely(txn->wr.dirtyroom == 0)) {
if (txn->wr.loose_count) {
page_t *lp = txn->wr.loose_pages;
DEBUG("purge-and-reclaim loose page %" PRIaPGNO, lp->pgno);
rc = pnl_insert_span(&txn->tw.repnl, lp->pgno, 1);
rc = pnl_insert_span(&txn->wr.repnl, lp->pgno, 1);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
size_t di = dpl_search(txn, lp->pgno);
tASSERT(txn, txn->tw.dirtylist->items[di].ptr == lp);
tASSERT(txn, txn->wr.dirtylist->items[di].ptr == lp);
dpl_remove(txn, di);
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
txn->tw.loose_pages = page_next(lp);
txn->tw.loose_count--;
txn->tw.dirtyroom++;
txn->wr.loose_pages = page_next(lp);
txn->wr.loose_count--;
txn->wr.dirtyroom++;
if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP))
page_shadow_release(txn->env, lp, 1);
} else {
ERROR("Dirtyroom is depleted, DPL length %zu", txn->tw.dirtylist->length);
ERROR("Dirtyroom is depleted, DPL length %zu", txn->wr.dirtylist->length);
if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP))
page_shadow_release(txn->env, mp, npages);
return MDBX_TXN_FULL;
@ -706,7 +706,7 @@ __hot int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, size_t npage
txn->flags |= MDBX_TXN_ERROR;
return rc;
}
txn->tw.dirtyroom--;
txn->wr.dirtyroom--;
tASSERT(txn, dpl_check(txn));
return MDBX_SUCCESS;
}

14
src/page-ops.h
View File

@ -88,7 +88,7 @@ static inline int page_touch(MDBX_cursor *mc) {
}
if (is_modifable(txn, mp)) {
if (!txn->tw.dirtylist) {
if (!txn->wr.dirtylist) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) && !MDBX_AVOID_MSYNC);
return MDBX_SUCCESS;
}
@ -114,14 +114,14 @@ static inline void page_wash(MDBX_txn *txn, size_t di, page_t *const mp, const s
mp->txnid = INVALID_TXNID;
mp->flags = P_BAD;
if (txn->tw.dirtylist) {
if (txn->wr.dirtylist) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
tASSERT(txn, MDBX_AVOID_MSYNC || (di && txn->tw.dirtylist->items[di].ptr == mp));
tASSERT(txn, MDBX_AVOID_MSYNC || (di && txn->wr.dirtylist->items[di].ptr == mp));
if (!MDBX_AVOID_MSYNC || di) {
dpl_remove_ex(txn, di, npages);
txn->tw.dirtyroom++;
tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
(txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
txn->wr.dirtyroom++;
tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length ==
(txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));
if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP)) {
page_shadow_release(txn->env, mp, npages);
return;
@ -129,7 +129,7 @@ static inline void page_wash(MDBX_txn *txn, size_t di, page_t *const mp, const s
}
} else {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) && !MDBX_AVOID_MSYNC && !di);
txn->tw.writemap_dirty_npages -= (txn->tw.writemap_dirty_npages > npages) ? npages : txn->tw.writemap_dirty_npages;
txn->wr.writemap_dirty_npages -= (txn->wr.writemap_dirty_npages > npages) ? npages : txn->wr.writemap_dirty_npages;
}
VALGRIND_MAKE_MEM_UNDEFINED(mp, PAGEHDRSZ);
VALGRIND_MAKE_MEM_NOACCESS(page_data(mp), pgno2bytes(txn->env, npages) - PAGEHDRSZ);

54
src/refund.c
View File

@ -7,7 +7,7 @@
static void refund_reclaimed(MDBX_txn *txn) {
/* Scanning in descend order */
pgno_t first_unallocated = txn->geo.first_unallocated;
const pnl_t pnl = txn->tw.repnl;
const pnl_t pnl = txn->wr.repnl;
tASSERT(txn, MDBX_PNL_GETSIZE(pnl) && MDBX_PNL_MOST(pnl) == first_unallocated - 1);
#if MDBX_PNL_ASCENDING
size_t i = MDBX_PNL_GETSIZE(pnl);
@ -28,16 +28,16 @@ static void refund_reclaimed(MDBX_txn *txn) {
VERBOSE("refunded %" PRIaPGNO " pages: %" PRIaPGNO " -> %" PRIaPGNO, txn->geo.first_unallocated - first_unallocated,
txn->geo.first_unallocated, first_unallocated);
txn->geo.first_unallocated = first_unallocated;
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - 1));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - 1));
}
static void refund_loose(MDBX_txn *txn) {
tASSERT(txn, txn->tw.loose_pages != nullptr);
tASSERT(txn, txn->tw.loose_count > 0);
tASSERT(txn, txn->wr.loose_pages != nullptr);
tASSERT(txn, txn->wr.loose_count > 0);
dpl_t *const dl = txn->tw.dirtylist;
dpl_t *const dl = txn->wr.dirtylist;
if (dl) {
tASSERT(txn, dl->length >= txn->tw.loose_count);
tASSERT(txn, dl->length >= txn->wr.loose_count);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
} else {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
@ -46,22 +46,22 @@ static void refund_loose(MDBX_txn *txn) {
pgno_t onstack[MDBX_CACHELINE_SIZE * 8 / sizeof(pgno_t)];
pnl_t suitable = onstack;
if (!dl || dl->length - dl->sorted > txn->tw.loose_count) {
if (!dl || dl->length - dl->sorted > txn->wr.loose_count) {
/* Dirty list is useless since unsorted. */
if (pnl_bytes2size(sizeof(onstack)) < txn->tw.loose_count) {
suitable = pnl_alloc(txn->tw.loose_count);
if (pnl_bytes2size(sizeof(onstack)) < txn->wr.loose_count) {
suitable = pnl_alloc(txn->wr.loose_count);
if (unlikely(!suitable))
return /* this is not a reason for transaction fail */;
}
/* Collect loose-pages which may be refunded. */
tASSERT(txn, txn->geo.first_unallocated >= MIN_PAGENO + txn->tw.loose_count);
tASSERT(txn, txn->geo.first_unallocated >= MIN_PAGENO + txn->wr.loose_count);
pgno_t most = MIN_PAGENO;
size_t w = 0;
for (const page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
for (const page_t *lp = txn->wr.loose_pages; lp; lp = page_next(lp)) {
tASSERT(txn, lp->flags == P_LOOSE);
tASSERT(txn, txn->geo.first_unallocated > lp->pgno);
if (likely(txn->geo.first_unallocated - txn->tw.loose_count <= lp->pgno)) {
if (likely(txn->geo.first_unallocated - txn->wr.loose_count <= lp->pgno)) {
tASSERT(txn, w < ((suitable == onstack) ? pnl_bytes2size(sizeof(onstack)) : MDBX_PNL_ALLOCLEN(suitable)));
suitable[++w] = lp->pgno;
most = (lp->pgno > most) ? lp->pgno : most;
@ -90,11 +90,11 @@ static void refund_loose(MDBX_txn *txn) {
const size_t refunded = txn->geo.first_unallocated - most;
DEBUG("refund-suitable %zu pages %" PRIaPGNO " -> %" PRIaPGNO, refunded, most, txn->geo.first_unallocated);
txn->geo.first_unallocated = most;
txn->tw.loose_count -= refunded;
txn->wr.loose_count -= refunded;
if (dl) {
txn->tw.dirtyroom += refunded;
txn->wr.dirtyroom += refunded;
dl->pages_including_loose -= refunded;
assert(txn->tw.dirtyroom <= txn->env->options.dp_limit);
assert(txn->wr.dirtyroom <= txn->env->options.dp_limit);
/* Filter-out dirty list */
size_t r = 0;
@ -115,8 +115,8 @@ static void refund_loose(MDBX_txn *txn) {
}
}
dpl_setlen(dl, w);
tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
(txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length ==
(txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));
}
goto unlink_loose;
}
@ -141,15 +141,15 @@ static void refund_loose(MDBX_txn *txn) {
if (dl->sorted != dl->length) {
const size_t refunded = dl->sorted - dl->length;
dl->sorted = dl->length;
txn->tw.loose_count -= refunded;
txn->tw.dirtyroom += refunded;
txn->wr.loose_count -= refunded;
txn->wr.dirtyroom += refunded;
dl->pages_including_loose -= refunded;
tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
(txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length ==
(txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));
/* Filter-out loose chain & dispose refunded pages. */
unlink_loose:
for (page_t *__restrict *__restrict link = &txn->tw.loose_pages; *link;) {
for (page_t *__restrict *__restrict link = &txn->wr.loose_pages; *link;) {
page_t *dp = *link;
tASSERT(txn, dp->flags == P_LOOSE);
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(dp), sizeof(page_t *));
@ -168,21 +168,21 @@ static void refund_loose(MDBX_txn *txn) {
tASSERT(txn, dpl_check(txn));
if (suitable != onstack)
pnl_free(suitable);
txn->tw.loose_refund_wl = txn->geo.first_unallocated;
txn->wr.loose_refund_wl = txn->geo.first_unallocated;
}
bool txn_refund(MDBX_txn *txn) {
const pgno_t before = txn->geo.first_unallocated;
if (txn->tw.loose_pages && txn->tw.loose_refund_wl > txn->geo.first_unallocated)
if (txn->wr.loose_pages && txn->wr.loose_refund_wl > txn->geo.first_unallocated)
refund_loose(txn);
while (true) {
if (MDBX_PNL_GETSIZE(txn->tw.repnl) == 0 || MDBX_PNL_MOST(txn->tw.repnl) != txn->geo.first_unallocated - 1)
if (MDBX_PNL_GETSIZE(txn->wr.repnl) == 0 || MDBX_PNL_MOST(txn->wr.repnl) != txn->geo.first_unallocated - 1)
break;
refund_reclaimed(txn);
if (!txn->tw.loose_pages || txn->tw.loose_refund_wl <= txn->geo.first_unallocated)
if (!txn->wr.loose_pages || txn->wr.loose_refund_wl <= txn->geo.first_unallocated)
break;
const pgno_t memo = txn->geo.first_unallocated;
@ -194,7 +194,7 @@ bool txn_refund(MDBX_txn *txn) {
if (before == txn->geo.first_unallocated)
return false;
if (txn->tw.spilled.list)
if (txn->wr.spilled.list)
/* Squash deleted pagenums if we refunded any */
spill_purge(txn);

114
src/spill.c
View File

@ -4,42 +4,42 @@
#include "internals.h"
void spill_remove(MDBX_txn *txn, size_t idx, size_t npages) {
tASSERT(txn, idx > 0 && idx <= MDBX_PNL_GETSIZE(txn->tw.spilled.list) && txn->tw.spilled.least_removed > 0);
txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed) ? idx : txn->tw.spilled.least_removed;
txn->tw.spilled.list[idx] |= 1;
MDBX_PNL_SETSIZE(txn->tw.spilled.list,
MDBX_PNL_GETSIZE(txn->tw.spilled.list) - (idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list)));
tASSERT(txn, idx > 0 && idx <= MDBX_PNL_GETSIZE(txn->wr.spilled.list) && txn->wr.spilled.least_removed > 0);
txn->wr.spilled.least_removed = (idx < txn->wr.spilled.least_removed) ? idx : txn->wr.spilled.least_removed;
txn->wr.spilled.list[idx] |= 1;
MDBX_PNL_SETSIZE(txn->wr.spilled.list,
MDBX_PNL_GETSIZE(txn->wr.spilled.list) - (idx == MDBX_PNL_GETSIZE(txn->wr.spilled.list)));
while (unlikely(npages > 1)) {
const pgno_t pgno = (txn->tw.spilled.list[idx] >> 1) + 1;
const pgno_t pgno = (txn->wr.spilled.list[idx] >> 1) + 1;
if (MDBX_PNL_ASCENDING) {
if (++idx > MDBX_PNL_GETSIZE(txn->tw.spilled.list) || (txn->tw.spilled.list[idx] >> 1) != pgno)
if (++idx > MDBX_PNL_GETSIZE(txn->wr.spilled.list) || (txn->wr.spilled.list[idx] >> 1) != pgno)
return;
} else {
if (--idx < 1 || (txn->tw.spilled.list[idx] >> 1) != pgno)
if (--idx < 1 || (txn->wr.spilled.list[idx] >> 1) != pgno)
return;
txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed) ? idx : txn->tw.spilled.least_removed;
txn->wr.spilled.least_removed = (idx < txn->wr.spilled.least_removed) ? idx : txn->wr.spilled.least_removed;
}
txn->tw.spilled.list[idx] |= 1;
MDBX_PNL_SETSIZE(txn->tw.spilled.list,
MDBX_PNL_GETSIZE(txn->tw.spilled.list) - (idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list)));
txn->wr.spilled.list[idx] |= 1;
MDBX_PNL_SETSIZE(txn->wr.spilled.list,
MDBX_PNL_GETSIZE(txn->wr.spilled.list) - (idx == MDBX_PNL_GETSIZE(txn->wr.spilled.list)));
--npages;
}
}
pnl_t spill_purge(MDBX_txn *txn) {
tASSERT(txn, txn->tw.spilled.least_removed > 0);
const pnl_t sl = txn->tw.spilled.list;
if (txn->tw.spilled.least_removed != INT_MAX) {
tASSERT(txn, txn->wr.spilled.least_removed > 0);
const pnl_t sl = txn->wr.spilled.list;
if (txn->wr.spilled.least_removed != INT_MAX) {
size_t len = MDBX_PNL_GETSIZE(sl), r, w;
for (w = r = txn->tw.spilled.least_removed; r <= len; ++r) {
for (w = r = txn->wr.spilled.least_removed; r <= len; ++r) {
sl[w] = sl[r];
w += 1 - (sl[r] & 1);
}
for (size_t i = 1; i < w; ++i)
tASSERT(txn, (sl[i] & 1) == 0);
MDBX_PNL_SETSIZE(sl, w - 1);
txn->tw.spilled.least_removed = INT_MAX;
txn->wr.spilled.least_removed = INT_MAX;
} else {
for (size_t i = 1; i <= MDBX_PNL_GETSIZE(sl); ++i)
tASSERT(txn, (sl[i] & 1) == 0);
@ -57,7 +57,7 @@ static int spill_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, const size_t np
const pgno_t pgno = dp->pgno;
int err = iov_page(txn, ctx, dp, npages);
if (likely(err == MDBX_SUCCESS))
err = spill_append_span(&txn->tw.spilled.list, pgno, npages);
err = spill_append_span(&txn->wr.spilled.list, pgno, npages);
return err;
}
@ -75,10 +75,10 @@ static size_t spill_cursor_keep(const MDBX_txn *const txn, const MDBX_cursor *mc
tASSERT(txn, !is_subpage(mp));
if (is_modifable(txn, mp)) {
size_t const n = dpl_search(txn, mp->pgno);
if (txn->tw.dirtylist->items[n].pgno == mp->pgno &&
if (txn->wr.dirtylist->items[n].pgno == mp->pgno &&
/* не считаем дважды */ dpl_age(txn, n)) {
size_t *const ptr = ptr_disp(txn->tw.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t));
*ptr = txn->tw.dirtylru;
size_t *const ptr = ptr_disp(txn->wr.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t));
*ptr = txn->wr.dirtylru;
tASSERT(txn, dpl_age(txn, n) == 0);
++keep;
}
@ -115,7 +115,7 @@ static size_t spill_txn_keep(MDBX_txn *txn, MDBX_cursor *m0) {
* ...
* > 255 = must not be spilled. */
MDBX_NOTHROW_PURE_FUNCTION static unsigned spill_prio(const MDBX_txn *txn, const size_t i, const uint32_t reciprocal) {
dpl_t *const dl = txn->tw.dirtylist;
dpl_t *const dl = txn->wr.dirtylist;
const uint32_t age = dpl_age(txn, i);
const size_t npages = dpl_npages(dl, i);
const pgno_t pgno = dl->items[i].pgno;
@ -178,14 +178,14 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
int rc = MDBX_SUCCESS;
if (unlikely(txn->tw.loose_count >=
(txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages)))
if (unlikely(txn->wr.loose_count >=
(txn->wr.dirtylist ? txn->wr.dirtylist->pages_including_loose : txn->wr.writemap_dirty_npages)))
goto done;
const size_t dirty_entries = txn->tw.dirtylist ? (txn->tw.dirtylist->length - txn->tw.loose_count) : 1;
const size_t dirty_entries = txn->wr.dirtylist ? (txn->wr.dirtylist->length - txn->wr.loose_count) : 1;
const size_t dirty_npages =
(txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages) -
txn->tw.loose_count;
(txn->wr.dirtylist ? txn->wr.dirtylist->pages_including_loose : txn->wr.writemap_dirty_npages) -
txn->wr.loose_count;
const size_t need_spill_entries = spill_gate(txn->env, wanna_spill_entries, dirty_entries);
const size_t need_spill_npages = spill_gate(txn->env, wanna_spill_npages, dirty_npages);
@ -196,17 +196,17 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp
if (txn->flags & MDBX_WRITEMAP) {
NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "msync", dirty_entries, dirty_npages);
const MDBX_env *env = txn->env;
tASSERT(txn, txn->tw.spilled.list == nullptr);
tASSERT(txn, txn->wr.spilled.list == nullptr);
rc = osal_msync(&txn->env->dxb_mmap, 0, pgno_align2os_bytes(env, txn->geo.first_unallocated), MDBX_SYNC_KICK);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
#if MDBX_AVOID_MSYNC
MDBX_ANALYSIS_ASSUME(txn->tw.dirtylist != nullptr);
MDBX_ANALYSIS_ASSUME(txn->wr.dirtylist != nullptr);
tASSERT(txn, dpl_check(txn));
env->lck->unsynced_pages.weak += txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count;
dpl_clear(txn->tw.dirtylist);
txn->tw.dirtyroom = env->options.dp_limit - txn->tw.loose_count;
for (page_t *lp = txn->tw.loose_pages; lp != nullptr; lp = page_next(lp)) {
env->lck->unsynced_pages.weak += txn->wr.dirtylist->pages_including_loose - txn->wr.loose_count;
dpl_clear(txn->wr.dirtylist);
txn->wr.dirtyroom = env->options.dp_limit - txn->wr.loose_count;
for (page_t *lp = txn->wr.loose_pages; lp != nullptr; lp = page_next(lp)) {
tASSERT(txn, lp->flags == P_LOOSE);
rc = dpl_append(txn, lp->pgno, lp, 1);
if (unlikely(rc != MDBX_SUCCESS))
@ -216,22 +216,22 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp
}
tASSERT(txn, dpl_check(txn));
#else
tASSERT(txn, txn->tw.dirtylist == nullptr);
env->lck->unsynced_pages.weak += txn->tw.writemap_dirty_npages;
txn->tw.writemap_spilled_npages += txn->tw.writemap_dirty_npages;
txn->tw.writemap_dirty_npages = 0;
tASSERT(txn, txn->wr.dirtylist == nullptr);
env->lck->unsynced_pages.weak += txn->wr.writemap_dirty_npages;
txn->wr.writemap_spilled_npages += txn->wr.writemap_dirty_npages;
txn->wr.writemap_dirty_npages = 0;
#endif /* MDBX_AVOID_MSYNC */
goto done;
}
NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "write", need_spill_entries, need_spill_npages);
MDBX_ANALYSIS_ASSUME(txn->tw.dirtylist != nullptr);
tASSERT(txn, txn->tw.dirtylist->length - txn->tw.loose_count >= 1);
tASSERT(txn, txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count >= need_spill_npages);
if (!txn->tw.spilled.list) {
txn->tw.spilled.least_removed = INT_MAX;
txn->tw.spilled.list = pnl_alloc(need_spill);
if (unlikely(!txn->tw.spilled.list)) {
MDBX_ANALYSIS_ASSUME(txn->wr.dirtylist != nullptr);
tASSERT(txn, txn->wr.dirtylist->length - txn->wr.loose_count >= 1);
tASSERT(txn, txn->wr.dirtylist->pages_including_loose - txn->wr.loose_count >= need_spill_npages);
if (!txn->wr.spilled.list) {
txn->wr.spilled.least_removed = INT_MAX;
txn->wr.spilled.list = pnl_alloc(need_spill);
if (unlikely(!txn->wr.spilled.list)) {
rc = MDBX_ENOMEM;
bailout:
txn->flags |= MDBX_TXN_ERROR;
@ -240,7 +240,7 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp
} else {
/* purge deleted slots */
spill_purge(txn);
rc = pnl_reserve(&txn->tw.spilled.list, need_spill);
rc = pnl_reserve(&txn->wr.spilled.list, need_spill);
(void)rc /* ignore since the resulting list may be shorter
and pnl_append() will increase pnl on demand */
;
@ -251,9 +251,9 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp
/* Preserve pages which may soon be dirtied again */
const size_t unspillable = spill_txn_keep(txn, m0);
if (unspillable + txn->tw.loose_count >= dl->length) {
if (unspillable + txn->wr.loose_count >= dl->length) {
#if xMDBX_DEBUG_SPILLING == 1 /* avoid false failure in debug mode */
if (likely(txn->tw.dirtyroom + txn->tw.loose_count >= need))
if (likely(txn->wr.dirtyroom + txn->wr.loose_count >= need))
return MDBX_SUCCESS;
#endif /* xMDBX_DEBUG_SPILLING */
ERROR("all %zu dirty pages are unspillable since referenced "
@ -293,7 +293,7 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp
age_max = (age_max >= age) ? age_max : age;
}
VERBOSE("lru-head %u, age-max %u", txn->tw.dirtylru, age_max);
VERBOSE("lru-head %u, age-max %u", txn->wr.dirtylru, age_max);
/* half of 8-bit radix-sort */
pgno_t radix_entries[256], radix_npages[256];
@ -388,8 +388,8 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp
tASSERT(txn, r - w == spilled_entries || rc != MDBX_SUCCESS);
dl->sorted = dpl_setlen(dl, w);
txn->tw.dirtyroom += spilled_entries;
txn->tw.dirtylist->pages_including_loose -= spilled_npages;
txn->wr.dirtyroom += spilled_entries;
txn->wr.dirtylist->pages_including_loose -= spilled_npages;
tASSERT(txn, dpl_check(txn));
if (!iov_empty(&ctx)) {
@ -400,10 +400,10 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp
goto bailout;
txn->env->lck->unsynced_pages.weak += spilled_npages;
pnl_sort(txn->tw.spilled.list, (size_t)txn->geo.first_unallocated << 1);
pnl_sort(txn->wr.spilled.list, (size_t)txn->geo.first_unallocated << 1);
txn->flags |= MDBX_TXN_SPILLS;
NOTICE("spilled %u dirty-entries, %u dirty-npages, now have %zu dirty-room", spilled_entries, spilled_npages,
txn->tw.dirtyroom);
txn->wr.dirtyroom);
} else {
tASSERT(txn, rc == MDBX_SUCCESS);
for (size_t i = 1; i <= dl->length; ++i) {
@ -414,18 +414,18 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intp
}
#if xMDBX_DEBUG_SPILLING == 2
if (txn->tw.loose_count + txn->tw.dirtyroom <= need / 2 + 1)
if (txn->wr.loose_count + txn->wr.dirtyroom <= need / 2 + 1)
ERROR("dirty-list length: before %zu, after %zu, parent %zi, loose %zu; "
"needed %zu, spillable %zu; "
"spilled %u dirty-entries, now have %zu dirty-room",
dl->length + spilled_entries, dl->length,
(txn->parent && txn->parent->tw.dirtylist) ? (intptr_t)txn->parent->tw.dirtylist->length : -1,
txn->tw.loose_count, need, spillable_entries, spilled_entries, txn->tw.dirtyroom);
ENSURE(txn->env, txn->tw.loose_count + txn->tw.dirtyroom > need / 2);
(txn->parent && txn->parent->wr.dirtylist) ? (intptr_t)txn->parent->wr.dirtylist->length : -1,
txn->wr.loose_count, need, spillable_entries, spilled_entries, txn->wr.dirtyroom);
ENSURE(txn->env, txn->wr.loose_count + txn->wr.dirtyroom > need / 2);
#endif /* xMDBX_DEBUG_SPILLING */
done:
return likely(txn->tw.dirtyroom + txn->tw.loose_count > ((need > CURSOR_STACK_SIZE) ? CURSOR_STACK_SIZE : need))
return likely(txn->wr.dirtyroom + txn->wr.loose_count > ((need > CURSOR_STACK_SIZE) ? CURSOR_STACK_SIZE : need))
? MDBX_SUCCESS
: MDBX_TXN_FULL;
}

10
src/spill.h
View File

@ -13,7 +13,7 @@ MDBX_INTERNAL int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, con
static inline size_t spill_search(const MDBX_txn *txn, pgno_t pgno) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
const pnl_t pnl = txn->tw.spilled.list;
const pnl_t pnl = txn->wr.spilled.list;
if (likely(!pnl))
return 0;
pgno <<= 1;
@ -22,7 +22,7 @@ static inline size_t spill_search(const MDBX_txn *txn, pgno_t pgno) {
}
static inline bool spill_intersect(const MDBX_txn *txn, pgno_t pgno, size_t npages) {
const pnl_t pnl = txn->tw.spilled.list;
const pnl_t pnl = txn->wr.spilled.list;
if (likely(!pnl))
return false;
const size_t len = MDBX_PNL_GETSIZE(pnl);
@ -56,10 +56,10 @@ static inline int txn_spill(MDBX_txn *const txn, MDBX_cursor *const m0, const si
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
tASSERT(txn, !m0 || cursor_is_tracked(m0));
const intptr_t wanna_spill_entries = txn->tw.dirtylist ? (need - txn->tw.dirtyroom - txn->tw.loose_count) : 0;
const intptr_t wanna_spill_entries = txn->wr.dirtylist ? (need - txn->wr.dirtyroom - txn->wr.loose_count) : 0;
const intptr_t wanna_spill_npages =
need + (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages) -
txn->tw.loose_count - txn->env->options.dp_limit;
need + (txn->wr.dirtylist ? txn->wr.dirtylist->pages_including_loose : txn->wr.writemap_dirty_npages) -
txn->wr.loose_count - txn->env->options.dp_limit;
/* production mode */
if (likely(wanna_spill_npages < 1 && wanna_spill_entries < 1)

4
src/tree-ops.c
View File

@ -56,7 +56,7 @@ int tree_drop(MDBX_cursor *mc, const bool may_have_tables) {
if (!(may_have_tables | mc->tree->large_pages))
cursor_pop(mc);
rc = pnl_need(&txn->tw.retired_pages,
rc = pnl_need(&txn->wr.retired_pages,
(size_t)mc->tree->branch_pages + (size_t)mc->tree->leaf_pages + (size_t)mc->tree->large_pages);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
@ -889,7 +889,7 @@ retry:
goto retry;
}
if (likely(!involve) &&
(likely(mc->tree != &mc->txn->dbs[FREE_DBI]) || mc->txn->tw.loose_pages || MDBX_PNL_GETSIZE(mc->txn->tw.repnl) ||
(likely(mc->tree != &mc->txn->dbs[FREE_DBI]) || mc->txn->wr.loose_pages || MDBX_PNL_GETSIZE(mc->txn->wr.repnl) ||
(mc->flags & z_gcu_preparation) || (mc->txn->flags & txn_gc_drained) || room_threshold)) {
involve = true;
goto retry;

60
src/txn-basal.c
View File

@ -32,16 +32,16 @@ static int txn_write(MDBX_txn *txn, iov_ctx_t *ctx) {
txn->env->lck->eoos_timestamp.weak = osal_monotime();
}
txn->tw.dirtylist->pages_including_loose -= total_npages;
txn->wr.dirtylist->pages_including_loose -= total_npages;
while (r <= dl->length)
dl->items[++w] = dl->items[r++];
dl->sorted = dpl_setlen(dl, w);
txn->tw.dirtyroom += r - 1 - w;
tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
(txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
tASSERT(txn, txn->tw.dirtylist->length == txn->tw.loose_count);
tASSERT(txn, txn->tw.dirtylist->pages_including_loose == txn->tw.loose_count);
txn->wr.dirtyroom += r - 1 - w;
tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length ==
(txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));
tASSERT(txn, txn->wr.dirtylist->length == txn->wr.loose_count);
tASSERT(txn, txn->wr.dirtylist->pages_including_loose == txn->wr.loose_count);
return rc;
}
@ -70,9 +70,9 @@ __cold MDBX_txn *txn_basal_create(const size_t max_dbi) {
txn->dbi_sparse = ptr_disp(txn->dbi_state, -bitmap_bytes);
#endif /* MDBX_ENABLE_DBI_SPARSE */
txn->flags = MDBX_TXN_FINISHED;
txn->tw.retired_pages = pnl_alloc(MDBX_PNL_INITIAL);
txn->tw.repnl = pnl_alloc(MDBX_PNL_INITIAL);
if (unlikely(!txn->tw.retired_pages || !txn->tw.repnl)) {
txn->wr.retired_pages = pnl_alloc(MDBX_PNL_INITIAL);
txn->wr.repnl = pnl_alloc(MDBX_PNL_INITIAL);
if (unlikely(!txn->wr.retired_pages || !txn->wr.repnl)) {
txn_basal_destroy(txn);
txn = nullptr;
}
@ -82,10 +82,10 @@ __cold MDBX_txn *txn_basal_create(const size_t max_dbi) {
__cold void txn_basal_destroy(MDBX_txn *txn) {
dpl_free(txn);
txl_free(txn->tw.gc.retxl);
pnl_free(txn->tw.retired_pages);
pnl_free(txn->tw.spilled.list);
pnl_free(txn->tw.repnl);
txl_free(txn->wr.gc.retxl);
pnl_free(txn->wr.retired_pages);
pnl_free(txn->wr.spilled.list);
pnl_free(txn->wr.repnl);
osal_free(txn);
}
@ -97,12 +97,12 @@ int txn_basal_end(MDBX_txn *txn, unsigned mode) {
txn->flags = MDBX_TXN_FINISHED;
env->txn = nullptr;
pnl_free(txn->tw.spilled.list);
txn->tw.spilled.list = nullptr;
pnl_free(txn->wr.spilled.list);
txn->wr.spilled.list = nullptr;
eASSERT(env, txn->parent == nullptr);
pnl_shrink(&txn->tw.retired_pages);
pnl_shrink(&txn->tw.repnl);
pnl_shrink(&txn->wr.retired_pages);
pnl_shrink(&txn->wr.repnl);
if (!(env->flags & MDBX_WRITEMAP))
dpl_release_shadows(txn);
@ -121,18 +121,18 @@ int txn_basal_end(MDBX_txn *txn, unsigned mode) {
int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) {
MDBX_env *const env = txn->env;
tASSERT(txn, txn == env->basal_txn && !txn->parent && !txn->nested);
if (!txn->tw.dirtylist) {
if (!txn->wr.dirtylist) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
} else {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == env->options.dp_limit);
tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length == env->options.dp_limit);
}
if (txn->flags & txn_may_have_cursors)
txn_done_cursors(txn);
bool need_flush_for_nometasync = false;
const meta_ptr_t head = meta_recent(env, &txn->tw.troika);
const meta_ptr_t head = meta_recent(env, &txn->wr.troika);
const uint32_t meta_sync_txnid = atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed);
/* sync prev meta */
if (head.is_steady && meta_sync_txnid != (uint32_t)head.txnid) {
@ -175,7 +175,7 @@ int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) {
}
}
if ((!txn->tw.dirtylist || txn->tw.dirtylist->length == 0) &&
if ((!txn->wr.dirtylist || txn->wr.dirtylist->length == 0) &&
(txn->flags & (MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | MDBX_TXN_NOSYNC | MDBX_TXN_NOMETASYNC)) == 0 &&
!need_flush_for_nometasync && !head.is_steady && !AUDIT_ENABLED()) {
TXN_FOREACH_DBI_ALL(txn, i) { tASSERT(txn, !(txn->dbi_state[i] & DBI_DIRTY)); }
@ -225,7 +225,7 @@ int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) {
if (unlikely(rc != MDBX_SUCCESS))
return rc;
tASSERT(txn, txn->tw.loose_count == 0);
tASSERT(txn, txn->wr.loose_count == 0);
txn->dbs[FREE_DBI].mod_txnid = (txn->dbi_state[FREE_DBI] & DBI_DIRTY) ? txn->txnid : txn->dbs[FREE_DBI].mod_txnid;
txn->dbs[MAIN_DBI].mod_txnid = (txn->dbi_state[MAIN_DBI] & DBI_DIRTY) ? txn->txnid : txn->dbs[MAIN_DBI].mod_txnid;
@ -234,16 +234,16 @@ int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) {
ts->audit = ts->gc;
}
if (AUDIT_ENABLED()) {
rc = audit_ex(txn, MDBX_PNL_GETSIZE(txn->tw.retired_pages), true);
rc = audit_ex(txn, MDBX_PNL_GETSIZE(txn->wr.retired_pages), true);
if (ts)
ts->audit = osal_monotime();
if (unlikely(rc != MDBX_SUCCESS))
return rc;
}
if (txn->tw.dirtylist) {
if (txn->wr.dirtylist) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
tASSERT(txn, txn->tw.loose_count == 0);
tASSERT(txn, txn->wr.loose_count == 0);
mdbx_filehandle_t fd =
#if defined(_WIN32) || defined(_WIN64)
@ -251,14 +251,14 @@ int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) {
(void)need_flush_for_nometasync;
#else
(need_flush_for_nometasync || env->dsync_fd == INVALID_HANDLE_VALUE ||
txn->tw.dirtylist->length > env->options.writethrough_threshold ||
txn->wr.dirtylist->length > env->options.writethrough_threshold ||
atomic_load64(&env->lck->unsynced_pages, mo_Relaxed))
? env->lazy_fd
: env->dsync_fd;
#endif /* Windows */
iov_ctx_t write_ctx;
rc = iov_init(txn, &write_ctx, txn->tw.dirtylist->length, txn->tw.dirtylist->pages_including_loose, fd, false);
rc = iov_init(txn, &write_ctx, txn->wr.dirtylist->length, txn->wr.dirtylist->pages_including_loose, fd, false);
if (unlikely(rc != MDBX_SUCCESS)) {
ERROR("txn-%s: error %d", "iov-init", rc);
return rc;
@ -271,7 +271,7 @@ int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) {
}
} else {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
env->lck->unsynced_pages.weak += txn->tw.writemap_dirty_npages;
env->lck->unsynced_pages.weak += txn->wr.writemap_dirty_npages;
if (!env->lck->eoos_timestamp.weak)
env->lck->eoos_timestamp.weak = osal_monotime();
}
@ -286,7 +286,7 @@ int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) {
meta.validator_id = head.ptr_c->validator_id;
meta.extra_pagehdr = head.ptr_c->extra_pagehdr;
unaligned_poke_u64(4, meta.pages_retired,
unaligned_peek_u64(4, head.ptr_c->pages_retired) + MDBX_PNL_GETSIZE(txn->tw.retired_pages));
unaligned_peek_u64(4, head.ptr_c->pages_retired) + MDBX_PNL_GETSIZE(txn->wr.retired_pages));
meta.geometry = txn->geo;
meta.trees.gc = txn->dbs[FREE_DBI];
meta.trees.main = txn->dbs[MAIN_DBI];
@ -303,7 +303,7 @@ int txn_basal_commit(MDBX_txn *txn, struct commit_timestamp *ts) {
meta.unsafe_sign = DATASIGN_NONE;
meta_set_txnid(env, &meta, commit_txnid);
rc = dxb_sync_locked(env, env->flags | txn->flags | txn_shrink_allowed, &meta, &txn->tw.troika);
rc = dxb_sync_locked(env, env->flags | txn->flags | txn_shrink_allowed, &meta, &txn->wr.troika);
if (ts)
ts->sync = osal_monotime();

198
src/txn-nested.c
View File

@ -17,20 +17,20 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t
page_shadow_release(txn->env, dst->items[n].ptr, npages);
--n;
}
parent->tw.dirtyroom += dst->sorted - n;
parent->wr.dirtyroom += dst->sorted - n;
dst->sorted = dpl_setlen(dst, n);
tASSERT(parent, parent->tw.dirtyroom + parent->tw.dirtylist->length ==
(parent->parent ? parent->parent->tw.dirtyroom : parent->env->options.dp_limit));
tASSERT(parent, parent->wr.dirtyroom + parent->wr.dirtylist->length ==
(parent->parent ? parent->parent->wr.dirtyroom : parent->env->options.dp_limit));
}
/* Remove reclaimed pages from parent's dirty list */
const pnl_t reclaimed_list = parent->tw.repnl;
const pnl_t reclaimed_list = parent->wr.repnl;
dpl_sift(parent, reclaimed_list, false);
/* Move retired pages from parent's dirty & spilled list to reclaimed */
size_t r, w, d, s, l;
for (r = w = parent_retired_len; ++r <= MDBX_PNL_GETSIZE(parent->tw.retired_pages);) {
const pgno_t pgno = parent->tw.retired_pages[r];
for (r = w = parent_retired_len; ++r <= MDBX_PNL_GETSIZE(parent->wr.retired_pages);) {
const pgno_t pgno = parent->wr.retired_pages[r];
const size_t di = dpl_exist(parent, pgno);
const size_t si = !di ? spill_search(parent, pgno) : 0;
unsigned npages;
@ -54,14 +54,14 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t
/* Список retired страниц не сортирован, но для ускорения сортировки
* дополняется в соответствии с MDBX_PNL_ASCENDING */
#if MDBX_PNL_ASCENDING
const size_t len = MDBX_PNL_GETSIZE(parent->tw.retired_pages);
while (r < len && parent->tw.retired_pages[r + 1] == pgno + l) {
const size_t len = MDBX_PNL_GETSIZE(parent->wr.retired_pages);
while (r < len && parent->wr.retired_pages[r + 1] == pgno + l) {
++r;
if (++l == npages)
break;
}
#else
while (w > parent_retired_len && parent->tw.retired_pages[w - 1] == pgno + l) {
while (w > parent_retired_len && parent->wr.retired_pages[w - 1] == pgno + l) {
--w;
if (++l == npages)
break;
@ -73,18 +73,18 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t
spill_remove(parent, si, 1);
kind = "spilled";
} else {
parent->tw.retired_pages[++w] = pgno;
parent->wr.retired_pages[++w] = pgno;
continue;
}
DEBUG("reclaim retired parent's %u -> %zu %s page %" PRIaPGNO, npages, l, kind, pgno);
int err = pnl_insert_span(&parent->tw.repnl, pgno, l);
int err = pnl_insert_span(&parent->wr.repnl, pgno, l);
ENSURE(txn->env, err == MDBX_SUCCESS);
}
MDBX_PNL_SETSIZE(parent->tw.retired_pages, w);
MDBX_PNL_SETSIZE(parent->wr.retired_pages, w);
/* Filter-out parent spill list */
if (parent->tw.spilled.list && MDBX_PNL_GETSIZE(parent->tw.spilled.list) > 0) {
if (parent->wr.spilled.list && MDBX_PNL_GETSIZE(parent->wr.spilled.list) > 0) {
const pnl_t sl = spill_purge(parent);
size_t len = MDBX_PNL_GETSIZE(sl);
if (len) {
@ -169,11 +169,11 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t
}
/* Remove anything in our spill list from parent's dirty list */
if (txn->tw.spilled.list) {
tASSERT(txn, pnl_check_allocated(txn->tw.spilled.list, (size_t)parent->geo.first_unallocated << 1));
dpl_sift(parent, txn->tw.spilled.list, true);
tASSERT(parent, parent->tw.dirtyroom + parent->tw.dirtylist->length ==
(parent->parent ? parent->parent->tw.dirtyroom : parent->env->options.dp_limit));
if (txn->wr.spilled.list) {
tASSERT(txn, pnl_check_allocated(txn->wr.spilled.list, (size_t)parent->geo.first_unallocated << 1));
dpl_sift(parent, txn->wr.spilled.list, true);
tASSERT(parent, parent->wr.dirtyroom + parent->wr.dirtylist->length ==
(parent->parent ? parent->parent->wr.dirtyroom : parent->env->options.dp_limit));
}
/* Find length of merging our dirty list with parent's and release
@ -300,10 +300,10 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t
}
}
}
parent->tw.dirtyroom -= dst->sorted - dst->length;
assert(parent->tw.dirtyroom <= parent->env->options.dp_limit);
parent->wr.dirtyroom -= dst->sorted - dst->length;
assert(parent->wr.dirtyroom <= parent->env->options.dp_limit);
dpl_setlen(dst, dst->sorted);
parent->tw.dirtylru = txn->tw.dirtylru;
parent->wr.dirtylru = txn->wr.dirtylru;
/* В текущем понимании выгоднее пересчитать кол-во страниц,
* чем подмешивать лишние ветвления и вычисления в циклы выше. */
@ -314,22 +314,22 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t
tASSERT(parent, dpl_check(parent));
dpl_free(txn);
if (txn->tw.spilled.list) {
if (parent->tw.spilled.list) {
if (txn->wr.spilled.list) {
if (parent->wr.spilled.list) {
/* Must not fail since space was preserved above. */
pnl_merge(parent->tw.spilled.list, txn->tw.spilled.list);
pnl_free(txn->tw.spilled.list);
pnl_merge(parent->wr.spilled.list, txn->wr.spilled.list);
pnl_free(txn->wr.spilled.list);
} else {
parent->tw.spilled.list = txn->tw.spilled.list;
parent->tw.spilled.least_removed = txn->tw.spilled.least_removed;
parent->wr.spilled.list = txn->wr.spilled.list;
parent->wr.spilled.least_removed = txn->wr.spilled.least_removed;
}
tASSERT(parent, dpl_check(parent));
}
parent->flags &= ~MDBX_TXN_HAS_CHILD;
if (parent->tw.spilled.list) {
assert(pnl_check_allocated(parent->tw.spilled.list, (size_t)parent->geo.first_unallocated << 1));
if (MDBX_PNL_GETSIZE(parent->tw.spilled.list))
if (parent->wr.spilled.list) {
assert(pnl_check_allocated(parent->wr.spilled.list, (size_t)parent->geo.first_unallocated << 1));
if (MDBX_PNL_GETSIZE(parent->wr.spilled.list))
parent->flags |= MDBX_TXN_SPILLS;
}
}
@ -338,7 +338,7 @@ int txn_nested_create(MDBX_txn *parent, const MDBX_txn_flags_t flags) {
if (parent->env->options.spill_parent4child_denominator) {
/* Spill dirty-pages of parent to provide dirtyroom for child txn */
int err =
txn_spill(parent, nullptr, parent->tw.dirtylist->length / parent->env->options.spill_parent4child_denominator);
txn_spill(parent, nullptr, parent->wr.dirtylist->length / parent->env->options.spill_parent4child_denominator);
if (unlikely(err != MDBX_SUCCESS))
return LOG_IFERR(err);
}
@ -356,74 +356,74 @@ int txn_nested_create(MDBX_txn *parent, const MDBX_txn_flags_t flags) {
txn->geo = parent->geo;
int err = dpl_alloc(txn);
if (likely(err == MDBX_SUCCESS)) {
const size_t len = MDBX_PNL_GETSIZE(parent->tw.repnl) + parent->tw.loose_count;
txn->tw.repnl = pnl_alloc((len > MDBX_PNL_INITIAL) ? len : MDBX_PNL_INITIAL);
if (unlikely(!txn->tw.repnl))
const size_t len = MDBX_PNL_GETSIZE(parent->wr.repnl) + parent->wr.loose_count;
txn->wr.repnl = pnl_alloc((len > MDBX_PNL_INITIAL) ? len : MDBX_PNL_INITIAL);
if (unlikely(!txn->wr.repnl))
err = MDBX_ENOMEM;
}
if (unlikely(err != MDBX_SUCCESS)) {
failed:
pnl_free(txn->tw.repnl);
pnl_free(txn->wr.repnl);
dpl_free(txn);
osal_free(txn);
return LOG_IFERR(err);
}
/* Move loose pages to reclaimed list */
if (parent->tw.loose_count) {
if (parent->wr.loose_count) {
do {
page_t *lp = parent->tw.loose_pages;
page_t *lp = parent->wr.loose_pages;
tASSERT(parent, lp->flags == P_LOOSE);
err = pnl_insert_span(&parent->tw.repnl, lp->pgno, 1);
err = pnl_insert_span(&parent->wr.repnl, lp->pgno, 1);
if (unlikely(err != MDBX_SUCCESS))
goto failed;
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
parent->tw.loose_pages = page_next(lp);
parent->wr.loose_pages = page_next(lp);
/* Remove from dirty list */
page_wash(parent, dpl_exist(parent, lp->pgno), lp, 1);
} while (parent->tw.loose_pages);
parent->tw.loose_count = 0;
} while (parent->wr.loose_pages);
parent->wr.loose_count = 0;
#if MDBX_ENABLE_REFUND
parent->tw.loose_refund_wl = 0;
parent->wr.loose_refund_wl = 0;
#endif /* MDBX_ENABLE_REFUND */
tASSERT(parent, dpl_check(parent));
}
txn->tw.dirtyroom = parent->tw.dirtyroom;
txn->tw.dirtylru = parent->tw.dirtylru;
txn->wr.dirtyroom = parent->wr.dirtyroom;
txn->wr.dirtylru = parent->wr.dirtylru;
dpl_sort(parent);
if (parent->tw.spilled.list)
if (parent->wr.spilled.list)
spill_purge(parent);
tASSERT(txn, MDBX_PNL_ALLOCLEN(txn->tw.repnl) >= MDBX_PNL_GETSIZE(parent->tw.repnl));
memcpy(txn->tw.repnl, parent->tw.repnl, MDBX_PNL_SIZEOF(parent->tw.repnl));
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, (txn->geo.first_unallocated /* LY: intentional assignment
tASSERT(txn, MDBX_PNL_ALLOCLEN(txn->wr.repnl) >= MDBX_PNL_GETSIZE(parent->wr.repnl));
memcpy(txn->wr.repnl, parent->wr.repnl, MDBX_PNL_SIZEOF(parent->wr.repnl));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, (txn->geo.first_unallocated /* LY: intentional assignment
here, only for assertion */
= parent->geo.first_unallocated) -
MDBX_ENABLE_REFUND));
txn->tw.gc.time_acc = parent->tw.gc.time_acc;
txn->tw.gc.last_reclaimed = parent->tw.gc.last_reclaimed;
if (parent->tw.gc.retxl) {
txn->tw.gc.retxl = parent->tw.gc.retxl;
parent->tw.gc.retxl = (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->tw.gc.retxl);
txn->wr.gc.time_acc = parent->wr.gc.time_acc;
txn->wr.gc.last_reclaimed = parent->wr.gc.last_reclaimed;
if (parent->wr.gc.retxl) {
txn->wr.gc.retxl = parent->wr.gc.retxl;
parent->wr.gc.retxl = (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->wr.gc.retxl);
}
txn->tw.retired_pages = parent->tw.retired_pages;
parent->tw.retired_pages = (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->tw.retired_pages);
txn->wr.retired_pages = parent->wr.retired_pages;
parent->wr.retired_pages = (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->wr.retired_pages);
txn->txnid = parent->txnid;
txn->front_txnid = parent->front_txnid + 1;
#if MDBX_ENABLE_REFUND
txn->tw.loose_refund_wl = 0;
txn->wr.loose_refund_wl = 0;
#endif /* MDBX_ENABLE_REFUND */
txn->canary = parent->canary;
parent->flags |= MDBX_TXN_HAS_CHILD;
parent->nested = txn;
txn->parent = parent;
txn->owner = parent->owner;
txn->tw.troika = parent->tw.troika;
txn->wr.troika = parent->wr.troika;
txn->cursors[FREE_DBI] = nullptr;
txn->cursors[MAIN_DBI] = nullptr;
@ -432,10 +432,10 @@ int txn_nested_create(MDBX_txn *parent, const MDBX_txn_flags_t flags) {
memset(txn->dbi_state + CORE_DBS, 0, (txn->n_dbi = parent->n_dbi) - CORE_DBS);
memcpy(txn->dbs, parent->dbs, sizeof(txn->dbs[0]) * CORE_DBS);
tASSERT(parent, parent->tw.dirtyroom + parent->tw.dirtylist->length ==
(parent->parent ? parent->parent->tw.dirtyroom : parent->env->options.dp_limit));
tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
(txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
tASSERT(parent, parent->wr.dirtyroom + parent->wr.dirtylist->length ==
(parent->parent ? parent->parent->wr.dirtyroom : parent->env->options.dp_limit));
tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length ==
(txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));
parent->env->txn = txn;
tASSERT(parent, parent->cursors[FREE_DBI] == nullptr);
return txn_shadow_cursors(parent, MAIN_DBI);
@ -447,26 +447,26 @@ void txn_nested_abort(MDBX_txn *nested) {
nested->signature = 0;
nested->owner = 0;
if (nested->tw.gc.retxl) {
tASSERT(parent, MDBX_PNL_GETSIZE(nested->tw.gc.retxl) >= (uintptr_t)parent->tw.gc.retxl);
MDBX_PNL_SETSIZE(nested->tw.gc.retxl, (uintptr_t)parent->tw.gc.retxl);
parent->tw.gc.retxl = nested->tw.gc.retxl;
if (nested->wr.gc.retxl) {
tASSERT(parent, MDBX_PNL_GETSIZE(nested->wr.gc.retxl) >= (uintptr_t)parent->wr.gc.retxl);
MDBX_PNL_SETSIZE(nested->wr.gc.retxl, (uintptr_t)parent->wr.gc.retxl);
parent->wr.gc.retxl = nested->wr.gc.retxl;
}
if (nested->tw.retired_pages) {
tASSERT(parent, MDBX_PNL_GETSIZE(nested->tw.retired_pages) >= (uintptr_t)parent->tw.retired_pages);
MDBX_PNL_SETSIZE(nested->tw.retired_pages, (uintptr_t)parent->tw.retired_pages);
parent->tw.retired_pages = nested->tw.retired_pages;
if (nested->wr.retired_pages) {
tASSERT(parent, MDBX_PNL_GETSIZE(nested->wr.retired_pages) >= (uintptr_t)parent->wr.retired_pages);
MDBX_PNL_SETSIZE(nested->wr.retired_pages, (uintptr_t)parent->wr.retired_pages);
parent->wr.retired_pages = nested->wr.retired_pages;
}
parent->tw.dirtylru = nested->tw.dirtylru;
parent->wr.dirtylru = nested->wr.dirtylru;
parent->nested = nullptr;
parent->flags &= ~MDBX_TXN_HAS_CHILD;
tASSERT(parent, dpl_check(parent));
tASSERT(parent, audit_ex(parent, 0, false) == 0);
dpl_release_shadows(nested);
dpl_free(nested);
pnl_free(nested->tw.repnl);
pnl_free(nested->wr.repnl);
osal_free(nested);
}
@ -479,7 +479,7 @@ int txn_nested_join(MDBX_txn *txn, struct commit_timestamp *ts) {
eASSERT(env, parent->nested == txn && (parent->flags & MDBX_TXN_HAS_CHILD) != 0);
eASSERT(env, dpl_check(txn));
if (txn->tw.dirtylist->length == 0 && !(txn->flags & MDBX_TXN_DIRTY) && parent->n_dbi == txn->n_dbi) {
if (txn->wr.dirtylist->length == 0 && !(txn->flags & MDBX_TXN_DIRTY) && parent->n_dbi == txn->n_dbi) {
TXN_FOREACH_DBI_ALL(txn, i) {
tASSERT(txn, (txn->dbi_state[i] & DBI_DIRTY) == 0);
if ((txn->dbi_state[i] & DBI_STALE) && !(parent->dbi_state[i] & DBI_STALE))
@ -488,8 +488,8 @@ int txn_nested_join(MDBX_txn *txn, struct commit_timestamp *ts) {
tASSERT(txn, memcmp(&parent->geo, &txn->geo, sizeof(parent->geo)) == 0);
tASSERT(txn, memcmp(&parent->canary, &txn->canary, sizeof(parent->canary)) == 0);
tASSERT(txn, !txn->tw.spilled.list || MDBX_PNL_GETSIZE(txn->tw.spilled.list) == 0);
tASSERT(txn, txn->tw.loose_count == 0);
tASSERT(txn, !txn->wr.spilled.list || MDBX_PNL_GETSIZE(txn->wr.spilled.list) == 0);
tASSERT(txn, txn->wr.loose_count == 0);
VERBOSE("fast-complete pure nested txn %" PRIaTXN, txn->txnid);
return txn_end(txn, TXN_END_PURE_COMMIT | TXN_END_SLOT | TXN_END_FREE);
@ -497,42 +497,42 @@ int txn_nested_join(MDBX_txn *txn, struct commit_timestamp *ts) {
/* Preserve space for spill list to avoid parent's state corruption
* if allocation fails. */
const size_t parent_retired_len = (uintptr_t)parent->tw.retired_pages;
tASSERT(txn, parent_retired_len <= MDBX_PNL_GETSIZE(txn->tw.retired_pages));
const size_t retired_delta = MDBX_PNL_GETSIZE(txn->tw.retired_pages) - parent_retired_len;
const size_t parent_retired_len = (uintptr_t)parent->wr.retired_pages;
tASSERT(txn, parent_retired_len <= MDBX_PNL_GETSIZE(txn->wr.retired_pages));
const size_t retired_delta = MDBX_PNL_GETSIZE(txn->wr.retired_pages) - parent_retired_len;
if (retired_delta) {
int err = pnl_need(&txn->tw.repnl, retired_delta);
int err = pnl_need(&txn->wr.repnl, retired_delta);
if (unlikely(err != MDBX_SUCCESS))
return err;
}
if (txn->tw.spilled.list) {
if (parent->tw.spilled.list) {
int err = pnl_need(&parent->tw.spilled.list, MDBX_PNL_GETSIZE(txn->tw.spilled.list));
if (txn->wr.spilled.list) {
if (parent->wr.spilled.list) {
int err = pnl_need(&parent->wr.spilled.list, MDBX_PNL_GETSIZE(txn->wr.spilled.list));
if (unlikely(err != MDBX_SUCCESS))
return err;
}
spill_purge(txn);
}
if (unlikely(txn->tw.dirtylist->length + parent->tw.dirtylist->length > parent->tw.dirtylist->detent &&
!dpl_reserve(parent, txn->tw.dirtylist->length + parent->tw.dirtylist->length))) {
if (unlikely(txn->wr.dirtylist->length + parent->wr.dirtylist->length > parent->wr.dirtylist->detent &&
!dpl_reserve(parent, txn->wr.dirtylist->length + parent->wr.dirtylist->length))) {
return MDBX_ENOMEM;
}
//-------------------------------------------------------------------------
parent->tw.gc.retxl = txn->tw.gc.retxl;
txn->tw.gc.retxl = nullptr;
parent->wr.gc.retxl = txn->wr.gc.retxl;
txn->wr.gc.retxl = nullptr;
parent->tw.retired_pages = txn->tw.retired_pages;
txn->tw.retired_pages = nullptr;
parent->wr.retired_pages = txn->wr.retired_pages;
txn->wr.retired_pages = nullptr;
pnl_free(parent->tw.repnl);
parent->tw.repnl = txn->tw.repnl;
txn->tw.repnl = nullptr;
parent->tw.gc.time_acc = txn->tw.gc.time_acc;
parent->tw.gc.last_reclaimed = txn->tw.gc.last_reclaimed;
pnl_free(parent->wr.repnl);
parent->wr.repnl = txn->wr.repnl;
txn->wr.repnl = nullptr;
parent->wr.gc.time_acc = txn->wr.gc.time_acc;
parent->wr.gc.last_reclaimed = txn->wr.gc.last_reclaimed;
parent->geo = txn->geo;
parent->canary = txn->canary;
@ -540,10 +540,10 @@ int txn_nested_join(MDBX_txn *txn, struct commit_timestamp *ts) {
/* Move loose pages to parent */
#if MDBX_ENABLE_REFUND
parent->tw.loose_refund_wl = txn->tw.loose_refund_wl;
parent->wr.loose_refund_wl = txn->wr.loose_refund_wl;
#endif /* MDBX_ENABLE_REFUND */
parent->tw.loose_count = txn->tw.loose_count;
parent->tw.loose_pages = txn->tw.loose_pages;
parent->wr.loose_count = txn->wr.loose_count;
parent->wr.loose_pages = txn->wr.loose_pages;
if (txn->flags & txn_may_have_cursors)
/* Merge our cursors into parent's and close them */
@ -581,14 +581,14 @@ int txn_nested_join(MDBX_txn *txn, struct commit_timestamp *ts) {
txn_refund(parent);
if (ASSERT_ENABLED()) {
/* Check parent's loose pages not suitable for refund */
for (page_t *lp = parent->tw.loose_pages; lp; lp = page_next(lp)) {
tASSERT(parent, lp->pgno < parent->tw.loose_refund_wl && lp->pgno + 1 < parent->geo.first_unallocated);
for (page_t *lp = parent->wr.loose_pages; lp; lp = page_next(lp)) {
tASSERT(parent, lp->pgno < parent->wr.loose_refund_wl && lp->pgno + 1 < parent->geo.first_unallocated);
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
}
/* Check parent's reclaimed pages not suitable for refund */
if (MDBX_PNL_GETSIZE(parent->tw.repnl))
tASSERT(parent, MDBX_PNL_MOST(parent->tw.repnl) + 1 < parent->geo.first_unallocated);
if (MDBX_PNL_GETSIZE(parent->wr.repnl))
tASSERT(parent, MDBX_PNL_MOST(parent->wr.repnl) + 1 < parent->geo.first_unallocated);
}
#endif /* MDBX_ENABLE_REFUND */

28
src/txn-ro.c
View File

@ -4,7 +4,7 @@
#include "internals.h"
static inline int txn_ro_rslot(MDBX_txn *txn) {
reader_slot_t *slot = txn->to.reader;
reader_slot_t *slot = txn->ro.slot;
STATIC_ASSERT(sizeof(uintptr_t) <= sizeof(slot->tid));
if (likely(slot)) {
if (likely(slot->pid.weak == txn->env->pid && slot->txnid.weak >= SAFE64_INVALID_THRESHOLD)) {
@ -26,7 +26,7 @@ static inline int txn_ro_rslot(MDBX_txn *txn) {
if (likely(slot->pid.weak == env->pid && slot->txnid.weak >= SAFE64_INVALID_THRESHOLD)) {
tASSERT(txn, slot->pid.weak == osal_getpid());
tASSERT(txn, slot->tid.weak == ((env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self()));
txn->to.reader = slot;
txn->ro.slot = slot;
return MDBX_SUCCESS;
}
if (unlikely(slot->pid.weak) || !(globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN))
@ -42,7 +42,7 @@ static inline int txn_ro_rslot(MDBX_txn *txn) {
tASSERT(txn, brs.slot->pid.weak == osal_getpid());
tASSERT(txn, brs.slot->tid.weak == ((env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self()));
}
txn->to.reader = brs.slot;
txn->ro.slot = brs.slot;
return brs.err;
}
@ -55,7 +55,7 @@ static inline int txn_ro_seize(MDBX_txn *txn) {
MDBX_env *const env = txn->env;
const meta_ptr_t head = likely(env->stuck_meta < 0) ? /* regular */ meta_recent(env, &troika)
: /* recovery mode */ meta_ptr(env, env->stuck_meta);
reader_slot_t *const r = txn->to.reader;
reader_slot_t *const r = txn->ro.slot;
if (likely(r != nullptr)) {
safe64_reset(&r->txnid, true);
atomic_store32(&r->snapshot_pages_used, head.ptr_v->geometry.first_unallocated, mo_Relaxed);
@ -117,7 +117,7 @@ int txn_ro_start(MDBX_txn *txn, unsigned flags) {
goto bailout;
STATIC_ASSERT(MDBX_TXN_RDONLY_PREPARE > MDBX_TXN_RDONLY);
reader_slot_t *r = txn->to.reader;
reader_slot_t *r = txn->ro.slot;
if (flags & (MDBX_TXN_RDONLY_PREPARE - MDBX_TXN_RDONLY)) {
eASSERT(env, txn->txnid == 0);
eASSERT(env, txn->owner == 0);
@ -153,8 +153,8 @@ int txn_ro_start(MDBX_txn *txn, unsigned flags) {
bailout:
tASSERT(txn, err != MDBX_SUCCESS);
txn->txnid = INVALID_TXNID;
if (likely(txn->to.reader))
safe64_reset(&txn->to.reader->txnid, true);
if (likely(txn->ro.slot))
safe64_reset(&txn->ro.slot->txnid, true);
return err;
}
@ -162,10 +162,10 @@ int txn_ro_end(MDBX_txn *txn, unsigned mode) {
MDBX_env *const env = txn->env;
tASSERT(txn, (txn->flags & txn_may_have_cursors) == 0);
txn->n_dbi = 0; /* prevent further DBI activity */
if (txn->to.reader) {
reader_slot_t *slot = txn->to.reader;
if (txn->ro.slot) {
reader_slot_t *slot = txn->ro.slot;
if (unlikely(!env->lck))
txn->to.reader = nullptr;
txn->ro.slot = nullptr;
else {
eASSERT(env, slot->pid.weak == env->pid);
if (likely((txn->flags & MDBX_TXN_FINISHED) == 0)) {
@ -194,7 +194,7 @@ int txn_ro_end(MDBX_txn *txn, unsigned mode) {
if (mode & TXN_END_SLOT) {
if ((env->flags & ENV_TXKEY) == 0)
atomic_store32(&slot->pid, 0, mo_Relaxed);
txn->to.reader = nullptr;
txn->ro.slot = nullptr;
}
}
}
@ -213,9 +213,9 @@ int txn_ro_end(MDBX_txn *txn, unsigned mode) {
}
int txn_ro_park(MDBX_txn *txn, bool autounpark) {
reader_slot_t *const rslot = txn->to.reader;
reader_slot_t *const rslot = txn->ro.slot;
tASSERT(txn, (txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) == MDBX_TXN_RDONLY);
tASSERT(txn, txn->to.reader->tid.weak < MDBX_TID_TXN_OUSTED);
tASSERT(txn, txn->ro.slot->tid.weak < MDBX_TID_TXN_OUSTED);
if (unlikely((txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) != MDBX_TXN_RDONLY))
return MDBX_BAD_TXN;
@ -244,7 +244,7 @@ int txn_ro_unpark(MDBX_txn *txn) {
(MDBX_TXN_RDONLY | MDBX_TXN_PARKED)))
return MDBX_BAD_TXN;
for (reader_slot_t *const rslot = txn->to.reader; rslot; atomic_yield()) {
for (reader_slot_t *const rslot = txn->ro.slot; rslot; atomic_yield()) {
const uint32_t pid = atomic_load32(&rslot->pid, mo_Relaxed);
uint64_t tid = safe64_read(&rslot->tid);
uint64_t txnid = safe64_read(&rslot->txnid);

51
src/txn.c
View File

@ -4,7 +4,7 @@
#include "internals.h"
__hot txnid_t txn_snapshot_oldest(const MDBX_txn *const txn) {
return mvcc_shapshot_oldest(txn->env, txn->tw.troika.txnid[txn->tw.troika.prefer_steady]);
return mvcc_shapshot_oldest(txn->env, txn->wr.troika.txnid[txn->wr.troika.prefer_steady]);
}
void txn_done_cursors(MDBX_txn *txn) {
@ -79,7 +79,6 @@ int txn_renew(MDBX_txn *txn, unsigned flags) {
rc = txn_ro_start(txn, flags);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
tASSERT(txn, rc == MDBX_SUCCESS);
ENSURE(env, txn->txnid >=
/* paranoia is appropriate here */ env->lck->cached_oldest.weak);
tASSERT(txn, txn->dbs[FREE_DBI].flags == MDBX_INTEGERKEY);
@ -119,8 +118,8 @@ int txn_renew(MDBX_txn *txn, unsigned flags) {
}
#endif /* Windows */
txn->tw.troika = meta_tap(env);
const meta_ptr_t head = meta_recent(env, &txn->tw.troika);
txn->wr.troika = meta_tap(env);
const meta_ptr_t head = meta_recent(env, &txn->wr.troika);
uint64_t timestamp = 0;
while ("workaround for https://libmdbx.dqdkfa.ru/dead-github/issues/269") {
rc = coherency_fetch_head(txn, head, &timestamp);
@ -141,18 +140,18 @@ int txn_renew(MDBX_txn *txn, unsigned flags) {
tASSERT(txn, check_table_flags(txn->dbs[MAIN_DBI].flags));
txn->flags = flags;
txn->nested = nullptr;
txn->tw.loose_pages = nullptr;
txn->tw.loose_count = 0;
txn->wr.loose_pages = nullptr;
txn->wr.loose_count = 0;
#if MDBX_ENABLE_REFUND
txn->tw.loose_refund_wl = 0;
txn->wr.loose_refund_wl = 0;
#endif /* MDBX_ENABLE_REFUND */
MDBX_PNL_SETSIZE(txn->tw.retired_pages, 0);
txn->tw.spilled.list = nullptr;
txn->tw.spilled.least_removed = 0;
txn->tw.gc.time_acc = 0;
txn->tw.gc.last_reclaimed = 0;
if (txn->tw.gc.retxl)
MDBX_PNL_SETSIZE(txn->tw.gc.retxl, 0);
MDBX_PNL_SETSIZE(txn->wr.retired_pages, 0);
txn->wr.spilled.list = nullptr;
txn->wr.spilled.least_removed = 0;
txn->wr.gc.time_acc = 0;
txn->wr.gc.last_reclaimed = 0;
if (txn->wr.gc.retxl)
MDBX_PNL_SETSIZE(txn->wr.gc.retxl, 0);
env->txn = txn;
}
@ -326,16 +325,16 @@ int txn_renew(MDBX_txn *txn, unsigned flags) {
rc = dpl_alloc(txn);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
txn->tw.dirtyroom = txn->env->options.dp_limit;
txn->tw.dirtylru = MDBX_DEBUG ? UINT32_MAX / 3 - 42 : 0;
txn->wr.dirtyroom = txn->env->options.dp_limit;
txn->wr.dirtylru = MDBX_DEBUG ? UINT32_MAX / 3 - 42 : 0;
} else {
tASSERT(txn, txn->tw.dirtylist == nullptr);
txn->tw.dirtylist = nullptr;
txn->tw.dirtyroom = MAX_PAGENO;
txn->tw.dirtylru = 0;
tASSERT(txn, txn->wr.dirtylist == nullptr);
txn->wr.dirtylist = nullptr;
txn->wr.dirtyroom = MAX_PAGENO;
txn->wr.dirtylru = 0;
}
eASSERT(env, txn->tw.writemap_dirty_npages == 0);
eASSERT(env, txn->tw.writemap_spilled_npages == 0);
eASSERT(env, txn->wr.writemap_dirty_npages == 0);
eASSERT(env, txn->wr.writemap_spilled_npages == 0);
MDBX_cursor *const gc = ptr_disp(txn, sizeof(MDBX_txn));
rc = cursor_init(gc, txn, FREE_DBI);
@ -378,8 +377,8 @@ int txn_end(MDBX_txn *txn, unsigned mode) {
ERROR("parent txn %p is invalid or mismatch for nested txn %p", (void *)parent, (void *)txn);
return MDBX_PROBLEM;
}
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, memcmp(&txn->tw.troika, &parent->tw.troika, sizeof(troika_t)) == 0);
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, memcmp(&txn->wr.troika, &parent->wr.troika, sizeof(troika_t)) == 0);
tASSERT(txn, mode & TXN_END_FREE);
env->txn = parent;
const pgno_t nested_now = txn->geo.now, nested_upper = txn->geo.upper;
@ -434,9 +433,9 @@ MDBX_txn *txn_alloc(const MDBX_txn_flags_t flags, MDBX_env *env) {
#else
0;
#endif /* MDBX_ENABLE_DBI_SPARSE */
STATIC_ASSERT(sizeof(txn->tw) > sizeof(txn->to));
STATIC_ASSERT(sizeof(txn->wr) > sizeof(txn->ro));
const size_t base =
(flags & MDBX_TXN_RDONLY) ? sizeof(MDBX_txn) - sizeof(txn->tw) + sizeof(txn->to) : sizeof(MDBX_txn);
(flags & MDBX_TXN_RDONLY) ? sizeof(MDBX_txn) - sizeof(txn->wr) + sizeof(txn->ro) : sizeof(MDBX_txn);
const size_t size = base +
((flags & MDBX_TXN_RDONLY) ? (size_t)bitmap_bytes + env->max_dbi * sizeof(txn->dbi_seqs[0]) : 0) +
env->max_dbi * (sizeof(txn->dbs[0]) + sizeof(txn->cursors[0]) + sizeof(txn->dbi_state[0]));