ISAR/libmdbx
3
0
Fork 0
mirror of https://github.com/isar/libmdbx.git synced 2025-11-06 19:08:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

mdbx: переименование (косметика).

Browse Source
This commit is contained in:
Леонид Юрьев (Leonid Yuriev)
2025-01-14 22:40:56 +03:00
parent 7e772114bc
commit 314b8ce1f0
24 changed files with 600 additions and 601 deletions
Download Patch File Download Diff File Expand all files Collapse all files

264
src/gc-put.c
View File

@@ -11,7 +11,7 @@ MDBX_MAYBE_UNUSED static inline const char *dbg_prefix(const gcu_t *ctx) {
return is_lifo(ctx->cursor.txn) ? " lifo" : " fifo";
}
static inline size_t backlog_size(MDBX_txn *txn) { return MDBX_PNL_GETSIZE(txn->tw.repnl) + txn->tw.loose_count; }
static inline size_t backlog_size(MDBX_txn *txn) { return MDBX_PNL_GETSIZE(txn->wr.repnl) + txn->wr.loose_count; }
static int clean_stored_retired(MDBX_txn *txn, gcu_t *ctx) {
int err = MDBX_SUCCESS;
@@ -55,7 +55,7 @@ static int touch_gc(gcu_t *ctx) {
MDBX_val key, val;
key.iov_base = val.iov_base = nullptr;
key.iov_len = sizeof(txnid_t);
val.iov_len = MDBX_PNL_SIZEOF(ctx->cursor.txn->tw.retired_pages);
val.iov_len = MDBX_PNL_SIZEOF(ctx->cursor.txn->wr.retired_pages);
ctx->cursor.flags |= z_gcu_preparation;
int err = cursor_touch(&ctx->cursor, &key, &val);
ctx->cursor.flags -= z_gcu_preparation;
@@ -71,7 +71,7 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) {
size_t for_split = ctx->retired_stored == 0;
tASSERT(txn, is_pointed(&ctx->cursor) || txn->dbs[FREE_DBI].leaf_pages == 0);
const intptr_t retired_left = MDBX_PNL_SIZEOF(txn->tw.retired_pages) - ctx->retired_stored;
const intptr_t retired_left = MDBX_PNL_SIZEOF(txn->wr.retired_pages) - ctx->retired_stored;
size_t for_repnl = 0;
if (MDBX_ENABLE_BIGFOOT && retired_left > 0) {
for_repnl = (retired_left + txn->env->maxgc_large1page - 1) / txn->env->maxgc_large1page;
@@ -79,7 +79,7 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) {
for (size_t entries = for_repnl; entries > 1; for_split += entries)
entries = (entries + per_branch_page - 1) / per_branch_page;
} else if (!MDBX_ENABLE_BIGFOOT && retired_left != 0) {
for_repnl = largechunk_npages(txn->env, MDBX_PNL_SIZEOF(txn->tw.retired_pages));
for_repnl = largechunk_npages(txn->env, MDBX_PNL_SIZEOF(txn->wr.retired_pages));
}
const size_t for_tree_before_touch = for_cow + for_rebalance + for_split;
@@ -101,7 +101,7 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) {
TRACE("== after-touch, backlog %zu, err %d", backlog_size(txn), err);
if (!MDBX_ENABLE_BIGFOOT && unlikely(for_repnl > 1) &&
MDBX_PNL_GETSIZE(txn->tw.retired_pages) != ctx->retired_stored && err == MDBX_SUCCESS) {
MDBX_PNL_GETSIZE(txn->wr.retired_pages) != ctx->retired_stored && err == MDBX_SUCCESS) {
if (unlikely(ctx->retired_stored)) {
err = clean_stored_retired(txn, ctx);
if (unlikely(err != MDBX_SUCCESS))
@@ -145,51 +145,51 @@ static inline void zeroize_reserved(const MDBX_env *env, MDBX_val pnl) {
}
static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
tASSERT(txn, txn->tw.loose_count > 0);
/* Return loose page numbers to tw.repnl,
tASSERT(txn, txn->wr.loose_count > 0);
/* Return loose page numbers to wr.repnl,
* though usually none are left at this point.
* The pages themselves remain in dirtylist. */
if (unlikely(!txn->tw.gc.retxl && txn->tw.gc.last_reclaimed < 1)) {
TRACE("%s: try allocate gc-slot for %zu loose-pages", dbg_prefix(ctx), txn->tw.loose_count);
if (unlikely(!txn->wr.gc.retxl && txn->wr.gc.last_reclaimed < 1)) {
TRACE("%s: try allocate gc-slot for %zu loose-pages", dbg_prefix(ctx), txn->wr.loose_count);
int err = gc_alloc_ex(&ctx->cursor, 0, ALLOC_RESERVE).err;
if (err == MDBX_SUCCESS) {
TRACE("%s: retry since gc-slot for %zu loose-pages available", dbg_prefix(ctx), txn->tw.loose_count);
TRACE("%s: retry since gc-slot for %zu loose-pages available", dbg_prefix(ctx), txn->wr.loose_count);
return MDBX_RESULT_TRUE;
}
/* Put loose page numbers in tw.retired_pages,
* since unable to return ones to tw.repnl. */
err = pnl_need(&txn->tw.retired_pages, txn->tw.loose_count);
/* Put loose page numbers in wr.retired_pages,
* since unable to return ones to wr.repnl. */
err = pnl_need(&txn->wr.retired_pages, txn->wr.loose_count);
if (unlikely(err != MDBX_SUCCESS))
return err;
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
pnl_append_prereserved(txn->tw.retired_pages, lp->pgno);
for (page_t *lp = txn->wr.loose_pages; lp; lp = page_next(lp)) {
pnl_append_prereserved(txn->wr.retired_pages, lp->pgno);
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
}
TRACE("%s: append %zu loose-pages to retired-pages", dbg_prefix(ctx), txn->tw.loose_count);
TRACE("%s: append %zu loose-pages to retired-pages", dbg_prefix(ctx), txn->wr.loose_count);
} else {
/* Room for loose pages + temp PNL with same */
int err = pnl_need(&txn->tw.repnl, 2 * txn->tw.loose_count + 2);
int err = pnl_need(&txn->wr.repnl, 2 * txn->wr.loose_count + 2);
if (unlikely(err != MDBX_SUCCESS))
return err;
pnl_t loose = txn->tw.repnl + MDBX_PNL_ALLOCLEN(txn->tw.repnl) - txn->tw.loose_count - 1;
pnl_t loose = txn->wr.repnl + MDBX_PNL_ALLOCLEN(txn->wr.repnl) - txn->wr.loose_count - 1;
size_t count = 0;
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
for (page_t *lp = txn->wr.loose_pages; lp; lp = page_next(lp)) {
tASSERT(txn, lp->flags == P_LOOSE);
loose[++count] = lp->pgno;
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
}
tASSERT(txn, count == txn->tw.loose_count);
tASSERT(txn, count == txn->wr.loose_count);
MDBX_PNL_SETSIZE(loose, count);
pnl_sort(loose, txn->geo.first_unallocated);
pnl_merge(txn->tw.repnl, loose);
TRACE("%s: append %zu loose-pages to reclaimed-pages", dbg_prefix(ctx), txn->tw.loose_count);
pnl_merge(txn->wr.repnl, loose);
TRACE("%s: append %zu loose-pages to reclaimed-pages", dbg_prefix(ctx), txn->wr.loose_count);
}
/* filter-out list of dirty-pages from loose-pages */
dpl_t *const dl = txn->tw.dirtylist;
dpl_t *const dl = txn->wr.dirtylist;
if (dl) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
tASSERT(txn, dl->sorted <= dl->length);
@@ -209,21 +209,21 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
}
}
TRACE("%s: filtered-out loose-pages from %zu -> %zu dirty-pages", dbg_prefix(ctx), dl->length, w);
tASSERT(txn, txn->tw.loose_count == dl->length - w);
tASSERT(txn, txn->wr.loose_count == dl->length - w);
dl->sorted -= sorted_out;
tASSERT(txn, dl->sorted <= w);
dpl_setlen(dl, w);
dl->pages_including_loose -= txn->tw.loose_count;
txn->tw.dirtyroom += txn->tw.loose_count;
tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
(txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
dl->pages_including_loose -= txn->wr.loose_count;
txn->wr.dirtyroom += txn->wr.loose_count;
tASSERT(txn, txn->wr.dirtyroom + txn->wr.dirtylist->length ==
(txn->parent ? txn->parent->wr.dirtyroom : txn->env->options.dp_limit));
} else {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
}
txn->tw.loose_pages = nullptr;
txn->tw.loose_count = 0;
txn->wr.loose_pages = nullptr;
txn->wr.loose_count = 0;
#if MDBX_ENABLE_REFUND
txn->tw.loose_refund_wl = 0;
txn->wr.loose_refund_wl = 0;
#endif /* MDBX_ENABLE_REFUND */
return MDBX_SUCCESS;
}
@@ -250,17 +250,17 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
tASSERT(txn, ctx->bigfoot <= txn->txnid);
}
retired_pages_before = MDBX_PNL_GETSIZE(txn->tw.retired_pages);
retired_pages_before = MDBX_PNL_GETSIZE(txn->wr.retired_pages);
err = prepare_backlog(txn, ctx);
if (unlikely(err != MDBX_SUCCESS))
return err;
if (retired_pages_before != MDBX_PNL_GETSIZE(txn->tw.retired_pages)) {
if (retired_pages_before != MDBX_PNL_GETSIZE(txn->wr.retired_pages)) {
TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx), retired_pages_before,
MDBX_PNL_GETSIZE(txn->tw.retired_pages));
MDBX_PNL_GETSIZE(txn->wr.retired_pages));
break;
}
pnl_sort(txn->tw.retired_pages, txn->geo.first_unallocated);
pnl_sort(txn->wr.retired_pages, txn->geo.first_unallocated);
ctx->retired_stored = 0;
ctx->bigfoot = txn->txnid;
do {
@@ -268,15 +268,15 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
err = prepare_backlog(txn, ctx);
if (unlikely(err != MDBX_SUCCESS))
return err;
if (ctx->retired_stored >= MDBX_PNL_GETSIZE(txn->tw.retired_pages)) {
if (ctx->retired_stored >= MDBX_PNL_GETSIZE(txn->wr.retired_pages)) {
TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx), retired_pages_before,
MDBX_PNL_GETSIZE(txn->tw.retired_pages));
MDBX_PNL_GETSIZE(txn->wr.retired_pages));
break;
}
}
key.iov_len = sizeof(txnid_t);
key.iov_base = &ctx->bigfoot;
const size_t left = MDBX_PNL_GETSIZE(txn->tw.retired_pages) - ctx->retired_stored;
const size_t left = MDBX_PNL_GETSIZE(txn->wr.retired_pages) - ctx->retired_stored;
const size_t chunk =
(left > txn->env->maxgc_large1page && ctx->bigfoot < MAX_TXNID) ? txn->env->maxgc_large1page : left;
data.iov_len = (chunk + 1) * sizeof(pgno_t);
@@ -293,9 +293,9 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
memset(data.iov_base, 0xBB, data.iov_len);
#endif /* MDBX_DEBUG && (ENABLE_MEMCHECK || __SANITIZE_ADDRESS__) */
if (retired_pages_before == MDBX_PNL_GETSIZE(txn->tw.retired_pages)) {
if (retired_pages_before == MDBX_PNL_GETSIZE(txn->wr.retired_pages)) {
const size_t at = (is_lifo(txn) == MDBX_PNL_ASCENDING) ? left - chunk : ctx->retired_stored;
pgno_t *const begin = txn->tw.retired_pages + at;
pgno_t *const begin = txn->wr.retired_pages + at;
/* MDBX_PNL_ASCENDING == false && LIFO == false:
* - the larger pgno is at the beginning of retired list
* and should be placed with the larger txnid.
@@ -310,15 +310,15 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
ctx->bigfoot, (unsigned)(ctx->bigfoot - txn->txnid), chunk, at, at + chunk, retired_pages_before);
}
ctx->retired_stored += chunk;
} while (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages) && (++ctx->bigfoot, true));
} while (retired_pages_before != MDBX_PNL_GETSIZE(txn->tw.retired_pages));
} while (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->wr.retired_pages) && (++ctx->bigfoot, true));
} while (retired_pages_before != MDBX_PNL_GETSIZE(txn->wr.retired_pages));
#else
/* Write to last page of GC */
key.iov_len = sizeof(txnid_t);
key.iov_base = &txn->txnid;
do {
prepare_backlog(txn, ctx);
data.iov_len = MDBX_PNL_SIZEOF(txn->tw.retired_pages);
data.iov_len = MDBX_PNL_SIZEOF(txn->wr.retired_pages);
err = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE);
if (unlikely(err != MDBX_SUCCESS))
return err;
@@ -331,13 +331,13 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
memset(data.iov_base, 0xBB, data.iov_len);
#endif /* MDBX_DEBUG && (ENABLE_MEMCHECK || __SANITIZE_ADDRESS__) */
/* Retry if tw.retired_pages[] grew during the Put() */
} while (data.iov_len < MDBX_PNL_SIZEOF(txn->tw.retired_pages));
/* Retry if wr.retired_pages[] grew during the Put() */
} while (data.iov_len < MDBX_PNL_SIZEOF(txn->wr.retired_pages));
ctx->retired_stored = MDBX_PNL_GETSIZE(txn->tw.retired_pages);
pnl_sort(txn->tw.retired_pages, txn->geo.first_unallocated);
tASSERT(txn, data.iov_len == MDBX_PNL_SIZEOF(txn->tw.retired_pages));
memcpy(data.iov_base, txn->tw.retired_pages, data.iov_len);
ctx->retired_stored = MDBX_PNL_GETSIZE(txn->wr.retired_pages);
pnl_sort(txn->wr.retired_pages, txn->geo.first_unallocated);
tASSERT(txn, data.iov_len == MDBX_PNL_SIZEOF(txn->wr.retired_pages));
memcpy(data.iov_base, txn->wr.retired_pages, data.iov_len);
TRACE("%s: put-retired #%zu @ %" PRIaTXN, dbg_prefix(ctx), ctx->retired_stored, txn->txnid);
#endif /* MDBX_ENABLE_BIGFOOT */
@@ -345,7 +345,7 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
size_t i = ctx->retired_stored;
DEBUG_EXTRA("txn %" PRIaTXN " root %" PRIaPGNO " num %zu, retired-PNL", txn->txnid, txn->dbs[FREE_DBI].root, i);
for (; i; i--)
DEBUG_EXTRA_PRINT(" %" PRIaPGNO, txn->tw.retired_pages[i]);
DEBUG_EXTRA_PRINT(" %" PRIaPGNO, txn->wr.retired_pages[i]);
DEBUG_EXTRA_PRINT("%s\n", ".");
}
return MDBX_SUCCESS;
@@ -359,15 +359,15 @@ typedef struct gcu_rid_result {
static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left) {
rid_t r;
if (is_lifo(txn)) {
if (txn->tw.gc.retxl == nullptr) {
txn->tw.gc.retxl = txl_alloc();
if (unlikely(!txn->tw.gc.retxl)) {
if (txn->wr.gc.retxl == nullptr) {
txn->wr.gc.retxl = txl_alloc();
if (unlikely(!txn->wr.gc.retxl)) {
r.err = MDBX_ENOMEM;
goto return_error;
}
}
if (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) < txl_max &&
left > (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page && !ctx->dense) {
if (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) < txl_max &&
left > (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page && !ctx->dense) {
/* Hужен свободный для для сохранения списка страниц. */
bool need_cleanup = false;
txnid_t snap_oldest = 0;
@@ -376,11 +376,11 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left)
r.err = gc_alloc_ex(&ctx->cursor, 0, ALLOC_RESERVE).err;
snap_oldest = txn->env->lck->cached_oldest.weak;
if (likely(r.err == MDBX_SUCCESS)) {
TRACE("%s: took @%" PRIaTXN " from GC", dbg_prefix(ctx), MDBX_PNL_LAST(txn->tw.gc.retxl));
TRACE("%s: took @%" PRIaTXN " from GC", dbg_prefix(ctx), MDBX_PNL_LAST(txn->wr.gc.retxl));
need_cleanup = true;
}
} while (r.err == MDBX_SUCCESS && MDBX_PNL_GETSIZE(txn->tw.gc.retxl) < txl_max &&
left > (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page);
} while (r.err == MDBX_SUCCESS && MDBX_PNL_GETSIZE(txn->wr.gc.retxl) < txl_max &&
left > (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page);
if (likely(r.err == MDBX_SUCCESS)) {
TRACE("%s: got enough from GC.", dbg_prefix(ctx));
@@ -389,34 +389,34 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left)
/* LY: some troubles... */
goto return_error;
if (MDBX_PNL_GETSIZE(txn->tw.gc.retxl)) {
if (MDBX_PNL_GETSIZE(txn->wr.gc.retxl)) {
if (need_cleanup) {
txl_sort(txn->tw.gc.retxl);
txl_sort(txn->wr.gc.retxl);
ctx->cleaned_slot = 0;
}
ctx->rid = MDBX_PNL_LAST(txn->tw.gc.retxl);
ctx->rid = MDBX_PNL_LAST(txn->wr.gc.retxl);
} else {
tASSERT(txn, txn->tw.gc.last_reclaimed == 0);
tASSERT(txn, txn->wr.gc.last_reclaimed == 0);
if (unlikely(txn_snapshot_oldest(txn) != snap_oldest))
/* should retry gc_alloc_ex()
* if the oldest reader changes since the last attempt */
goto retry_rid;
/* no reclaimable GC entries,
* therefore no entries with ID < mdbx_find_oldest(txn) */
txn->tw.gc.last_reclaimed = ctx->rid = snap_oldest;
txn->wr.gc.last_reclaimed = ctx->rid = snap_oldest;
TRACE("%s: none recycled yet, set rid to @%" PRIaTXN, dbg_prefix(ctx), ctx->rid);
}
/* В GC нет годных к переработке записей,
* будем использовать свободные id в обратном порядке. */
while (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) < txl_max &&
left > (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page) {
while (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) < txl_max &&
left > (MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page) {
if (unlikely(ctx->rid <= MIN_TXNID)) {
ctx->dense = true;
if (unlikely(MDBX_PNL_GETSIZE(txn->tw.gc.retxl) <= ctx->reused_slot)) {
if (unlikely(MDBX_PNL_GETSIZE(txn->wr.gc.retxl) <= ctx->reused_slot)) {
VERBOSE("** restart: reserve depleted (reused_gc_slot %zu >= "
"gc.reclaimed %zu)",
ctx->reused_slot, MDBX_PNL_GETSIZE(txn->tw.gc.retxl));
ctx->reused_slot, MDBX_PNL_GETSIZE(txn->wr.gc.retxl));
goto return_restart;
}
break;
@@ -444,7 +444,7 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left)
}
tASSERT(txn, !ctx->dense);
r.err = txl_append(&txn->tw.gc.retxl, ctx->rid);
r.err = txl_append(&txn->wr.gc.retxl, ctx->rid);
if (unlikely(r.err != MDBX_SUCCESS))
goto return_error;
@@ -468,12 +468,12 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left)
}
}
const size_t i = MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot;
tASSERT(txn, i > 0 && i <= MDBX_PNL_GETSIZE(txn->tw.gc.retxl));
r.rid = txn->tw.gc.retxl[i];
const size_t i = MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot;
tASSERT(txn, i > 0 && i <= MDBX_PNL_GETSIZE(txn->wr.gc.retxl));
r.rid = txn->wr.gc.retxl[i];
TRACE("%s: take @%" PRIaTXN " from lifo-reclaimed[%zu]", dbg_prefix(ctx), r.rid, i);
} else {
tASSERT(txn, txn->tw.gc.retxl == nullptr);
tASSERT(txn, txn->wr.gc.retxl == nullptr);
if (unlikely(ctx->rid == 0)) {
ctx->rid = txn_snapshot_oldest(txn);
MDBX_val key;
@@ -496,7 +496,7 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left)
r.rid = 0;
return r;
}
txn->tw.gc.last_reclaimed = ctx->rid;
txn->wr.gc.last_reclaimed = ctx->rid;
ctx->cleaned_id = ctx->rid + 1;
}
r.rid = ctx->rid--;
@@ -538,16 +538,16 @@ int gc_update(MDBX_txn *txn, gcu_t *ctx) {
txn->cursors[FREE_DBI] = &ctx->cursor;
int rc;
/* txn->tw.repnl[] can grow and shrink during this call.
* txn->tw.gc.last_reclaimed and txn->tw.retired_pages[] can only grow.
* But page numbers cannot disappear from txn->tw.retired_pages[]. */
/* txn->wr.repnl[] can grow and shrink during this call.
* txn->wr.gc.last_reclaimed and txn->wr.retired_pages[] can only grow.
* But page numbers cannot disappear from txn->wr.retired_pages[]. */
retry_clean_adj:
ctx->reserve_adj = 0;
retry:
ctx->loop += !(ctx->prev_first_unallocated > txn->geo.first_unallocated);
TRACE(">> restart, loop %u", ctx->loop);
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, dpl_check(txn));
if (unlikely(/* paranoia */ ctx->loop > ((MDBX_DEBUG > 0) ? 12 : 42))) {
ERROR("txn #%" PRIaTXN " too more loops %u, bailout", txn->txnid, ctx->loop);
@@ -569,22 +569,22 @@ retry:
ctx->amount = 0;
ctx->fill_idx = ~0u;
ctx->cleaned_id = 0;
ctx->rid = txn->tw.gc.last_reclaimed;
ctx->rid = txn->wr.gc.last_reclaimed;
while (true) {
/* Come back here after each Put() in case retired-list changed */
TRACE("%s", " >> continue");
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
MDBX_val key, data;
if (is_lifo(txn)) {
if (ctx->cleaned_slot < (txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0)) {
if (ctx->cleaned_slot < (txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) : 0)) {
ctx->reserved = 0;
ctx->cleaned_slot = 0;
ctx->reused_slot = 0;
ctx->fill_idx = ~0u;
/* LY: cleanup reclaimed records. */
do {
ctx->cleaned_id = txn->tw.gc.retxl[++ctx->cleaned_slot];
ctx->cleaned_id = txn->wr.gc.retxl[++ctx->cleaned_slot];
tASSERT(txn, ctx->cleaned_slot > 0 && ctx->cleaned_id <= env->lck->cached_oldest.weak);
key.iov_base = &ctx->cleaned_id;
key.iov_len = sizeof(ctx->cleaned_id);
@@ -602,16 +602,16 @@ retry:
rc = cursor_del(&ctx->cursor, 0);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
} while (ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->tw.gc.retxl));
txl_sort(txn->tw.gc.retxl);
} while (ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->wr.gc.retxl));
txl_sort(txn->wr.gc.retxl);
}
} else {
/* Удаляем оставшиеся вынутые из GC записи. */
while (txn->tw.gc.last_reclaimed && ctx->cleaned_id <= txn->tw.gc.last_reclaimed) {
while (txn->wr.gc.last_reclaimed && ctx->cleaned_id <= txn->wr.gc.last_reclaimed) {
rc = outer_first(&ctx->cursor, &key, nullptr);
if (rc == MDBX_NOTFOUND) {
ctx->cleaned_id = txn->tw.gc.last_reclaimed + 1;
ctx->rid = txn->tw.gc.last_reclaimed;
ctx->cleaned_id = txn->wr.gc.last_reclaimed + 1;
ctx->rid = txn->wr.gc.last_reclaimed;
ctx->reserved = 0;
ctx->reused_slot = 0;
break;
@@ -629,12 +629,12 @@ retry:
ctx->reused_slot = 0;
}
ctx->cleaned_id = unaligned_peek_u64(4, key.iov_base);
if (ctx->cleaned_id > txn->tw.gc.last_reclaimed)
if (ctx->cleaned_id > txn->wr.gc.last_reclaimed)
break;
rc = prepare_backlog(txn, ctx);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
tASSERT(txn, ctx->cleaned_id <= txn->tw.gc.last_reclaimed);
tASSERT(txn, ctx->cleaned_id <= txn->wr.gc.last_reclaimed);
tASSERT(txn, ctx->cleaned_id <= env->lck->cached_oldest.weak);
TRACE("%s: cleanup-reclaimed-id %" PRIaTXN, dbg_prefix(ctx), ctx->cleaned_id);
tASSERT(txn, *txn->cursors == &ctx->cursor);
@@ -644,7 +644,7 @@ retry:
}
}
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, dpl_check(txn));
if (AUDIT_ENABLED()) {
rc = audit_ex(txn, ctx->retired_stored, false);
@@ -654,7 +654,7 @@ retry:
/* return suitable into unallocated space */
if (txn_refund(txn)) {
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
if (AUDIT_ENABLED()) {
rc = audit_ex(txn, ctx->retired_stored, false);
if (unlikely(rc != MDBX_SUCCESS))
@@ -662,7 +662,7 @@ retry:
}
}
if (txn->tw.loose_pages) {
if (txn->wr.loose_pages) {
/* put loose pages into the reclaimed- or retired-list */
rc = gcu_loose(txn, ctx);
if (unlikely(rc != MDBX_SUCCESS)) {
@@ -670,19 +670,19 @@ retry:
continue;
goto bailout;
}
tASSERT(txn, txn->tw.loose_pages == 0);
tASSERT(txn, txn->wr.loose_pages == 0);
}
if (unlikely(ctx->reserved > MDBX_PNL_GETSIZE(txn->tw.repnl)) &&
(ctx->loop < 5 || ctx->reserved - MDBX_PNL_GETSIZE(txn->tw.repnl) > env->maxgc_large1page / 2)) {
if (unlikely(ctx->reserved > MDBX_PNL_GETSIZE(txn->wr.repnl)) &&
(ctx->loop < 5 || ctx->reserved - MDBX_PNL_GETSIZE(txn->wr.repnl) > env->maxgc_large1page / 2)) {
TRACE("%s: reclaimed-list changed %zu -> %zu, retry", dbg_prefix(ctx), ctx->amount,
MDBX_PNL_GETSIZE(txn->tw.repnl));
ctx->reserve_adj += ctx->reserved - MDBX_PNL_GETSIZE(txn->tw.repnl);
MDBX_PNL_GETSIZE(txn->wr.repnl));
ctx->reserve_adj += ctx->reserved - MDBX_PNL_GETSIZE(txn->wr.repnl);
goto retry;
}
ctx->amount = MDBX_PNL_GETSIZE(txn->tw.repnl);
ctx->amount = MDBX_PNL_GETSIZE(txn->wr.repnl);
if (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages)) {
if (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->wr.retired_pages)) {
/* store retired-list into GC */
rc = gcu_retired(txn, ctx);
if (unlikely(rc != MDBX_SUCCESS))
@@ -690,8 +690,8 @@ retry:
continue;
}
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, txn->tw.loose_count == 0);
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, txn->wr.loose_count == 0);
TRACE("%s", " >> reserving");
if (AUDIT_ENABLED()) {
@@ -704,7 +704,7 @@ retry:
"lifo-reclaimed-slots %zu, "
"reused-gc-slots %zu",
dbg_prefix(ctx), ctx->amount, ctx->reserved, ctx->reserve_adj, left,
txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0, ctx->reused_slot);
txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) : 0, ctx->reused_slot);
if (0 >= (intptr_t)left)
break;
@@ -722,7 +722,7 @@ retry:
size_t chunk = left;
if (unlikely(left > env->maxgc_large1page)) {
const size_t avail_gc_slots = txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot + 1
const size_t avail_gc_slots = txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot + 1
: (ctx->rid < INT16_MAX) ? (size_t)ctx->rid
: INT16_MAX;
if (likely(avail_gc_slots > 1)) {
@@ -749,8 +749,8 @@ retry:
size_t avail = ((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) /* - 1 + span */;
if (tail > avail) {
for (size_t i = ctx->amount - span; i > 0; --i) {
if (MDBX_PNL_ASCENDING ? (txn->tw.repnl[i] + span)
: (txn->tw.repnl[i] - span) == txn->tw.repnl[i + span]) {
if (MDBX_PNL_ASCENDING ? (txn->wr.repnl[i] + span)
: (txn->wr.repnl[i] - span) == txn->wr.repnl[i + span]) {
span += 1;
avail = ((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) - 1 + span;
if (avail >= tail)
@@ -791,7 +791,7 @@ retry:
ctx->reserved + chunk + 1, reservation_gc_id);
prepare_backlog(txn, ctx);
rc = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE | MDBX_NOOVERWRITE);
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
@@ -802,14 +802,14 @@ retry:
continue;
}
tASSERT(txn, ctx->cleaned_slot == (txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0));
tASSERT(txn, ctx->cleaned_slot == (txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) : 0));
TRACE("%s", " >> filling");
/* Fill in the reserved records */
size_t excess_slots = 0;
ctx->fill_idx = txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot : ctx->reused_slot;
ctx->fill_idx = txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) - ctx->reused_slot : ctx->reused_slot;
rc = MDBX_SUCCESS;
tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->wr.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, dpl_check(txn));
if (ctx->amount) {
MDBX_val key, data;
@@ -817,7 +817,7 @@ retry:
key.iov_base = data.iov_base = nullptr;
size_t left = ctx->amount, excess = 0;
if (txn->tw.gc.retxl == nullptr) {
if (txn->wr.gc.retxl == nullptr) {
tASSERT(txn, is_lifo(txn) == 0);
rc = outer_first(&ctx->cursor, &key, &data);
if (unlikely(rc != MDBX_SUCCESS)) {
@@ -830,33 +830,33 @@ retry:
while (true) {
txnid_t fill_gc_id;
TRACE("%s: left %zu of %zu", dbg_prefix(ctx), left, MDBX_PNL_GETSIZE(txn->tw.repnl));
if (txn->tw.gc.retxl == nullptr) {
TRACE("%s: left %zu of %zu", dbg_prefix(ctx), left, MDBX_PNL_GETSIZE(txn->wr.repnl));
if (txn->wr.gc.retxl == nullptr) {
tASSERT(txn, is_lifo(txn) == 0);
fill_gc_id = key.iov_base ? unaligned_peek_u64(4, key.iov_base) : MIN_TXNID;
if (ctx->fill_idx == 0 || fill_gc_id > txn->tw.gc.last_reclaimed) {
if (ctx->fill_idx == 0 || fill_gc_id > txn->wr.gc.last_reclaimed) {
if (!left)
break;
VERBOSE("** restart: reserve depleted (fill_idx %zu, fill_id %" PRIaTXN " > last_reclaimed %" PRIaTXN
", left %zu",
ctx->fill_idx, fill_gc_id, txn->tw.gc.last_reclaimed, left);
ctx->fill_idx, fill_gc_id, txn->wr.gc.last_reclaimed, left);
ctx->reserve_adj = (ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0;
goto retry;
}
ctx->fill_idx -= 1;
} else {
tASSERT(txn, is_lifo(txn) != 0);
if (ctx->fill_idx >= MDBX_PNL_GETSIZE(txn->tw.gc.retxl)) {
if (ctx->fill_idx >= MDBX_PNL_GETSIZE(txn->wr.gc.retxl)) {
if (!left)
break;
VERBOSE("** restart: reserve depleted (fill_idx %zu >= "
"gc.reclaimed %zu, left %zu",
ctx->fill_idx, MDBX_PNL_GETSIZE(txn->tw.gc.retxl), left);
ctx->fill_idx, MDBX_PNL_GETSIZE(txn->wr.gc.retxl), left);
ctx->reserve_adj = (ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0;
goto retry;
}
ctx->fill_idx += 1;
fill_gc_id = txn->tw.gc.retxl[ctx->fill_idx];
fill_gc_id = txn->wr.gc.retxl[ctx->fill_idx];
TRACE("%s: seek-reservation @%" PRIaTXN " at gc.reclaimed[%zu]", dbg_prefix(ctx), fill_gc_id, ctx->fill_idx);
key.iov_base = &fill_gc_id;
key.iov_len = sizeof(fill_gc_id);
@@ -864,7 +864,7 @@ retry:
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
}
tASSERT(txn, ctx->cleaned_slot == (txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0));
tASSERT(txn, ctx->cleaned_slot == (txn->wr.gc.retxl ? MDBX_PNL_GETSIZE(txn->wr.gc.retxl) : 0));
tASSERT(txn, fill_gc_id > 0 && fill_gc_id <= env->lck->cached_oldest.weak);
key.iov_base = &fill_gc_id;
key.iov_len = sizeof(fill_gc_id);
@@ -888,33 +888,33 @@ retry:
goto bailout;
zeroize_reserved(env, data);
if (unlikely(txn->tw.loose_count || ctx->amount != MDBX_PNL_GETSIZE(txn->tw.repnl))) {
if (unlikely(txn->wr.loose_count || ctx->amount != MDBX_PNL_GETSIZE(txn->wr.repnl))) {
NOTICE("** restart: reclaimed-list changed (%zu -> %zu, loose +%zu)", ctx->amount,
MDBX_PNL_GETSIZE(txn->tw.repnl), txn->tw.loose_count);
MDBX_PNL_GETSIZE(txn->wr.repnl), txn->wr.loose_count);
if (ctx->loop < 5 || (ctx->loop > 10 && (ctx->loop & 1)))
goto retry_clean_adj;
goto retry;
}
if (unlikely(txn->tw.gc.retxl ? ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->tw.gc.retxl)
: ctx->cleaned_id < txn->tw.gc.last_reclaimed)) {
if (unlikely(txn->wr.gc.retxl ? ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->wr.gc.retxl)
: ctx->cleaned_id < txn->wr.gc.last_reclaimed)) {
NOTICE("%s", "** restart: reclaimed-slots changed");
goto retry;
}
if (unlikely(ctx->retired_stored != MDBX_PNL_GETSIZE(txn->tw.retired_pages))) {
tASSERT(txn, ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages));
if (unlikely(ctx->retired_stored != MDBX_PNL_GETSIZE(txn->wr.retired_pages))) {
tASSERT(txn, ctx->retired_stored < MDBX_PNL_GETSIZE(txn->wr.retired_pages));
NOTICE("** restart: retired-list growth (%zu -> %zu)", ctx->retired_stored,
MDBX_PNL_GETSIZE(txn->tw.retired_pages));
MDBX_PNL_GETSIZE(txn->wr.retired_pages));
goto retry;
}
pgno_t *dst = data.iov_base;
*dst++ = (pgno_t)chunk;
pgno_t *src = MDBX_PNL_BEGIN(txn->tw.repnl) + left - chunk;
pgno_t *src = MDBX_PNL_BEGIN(txn->wr.repnl) + left - chunk;
memcpy(dst, src, chunk * sizeof(pgno_t));
pgno_t *from = src, *to = src + chunk;
TRACE("%s: fill %zu [ %zu:%" PRIaPGNO "...%zu:%" PRIaPGNO "] @%" PRIaTXN, dbg_prefix(ctx), chunk,
from - txn->tw.repnl, from[0], to - txn->tw.repnl, to[-1], fill_gc_id);
from - txn->wr.repnl, from[0], to - txn->wr.repnl, to[-1], fill_gc_id);
left -= chunk;
if (AUDIT_ENABLED()) {
@@ -925,7 +925,7 @@ retry:
next:
if (txn->tw.gc.retxl == nullptr) {
if (txn->wr.gc.retxl == nullptr) {
tASSERT(txn, is_lifo(txn) == 0);
rc = outer_next(&ctx->cursor, &key, &data, MDBX_NEXT);
if (unlikely(rc != MDBX_SUCCESS)) {
@@ -950,9 +950,9 @@ retry:
}
tASSERT(txn, rc == MDBX_SUCCESS);
if (unlikely(txn->tw.loose_count != 0 || ctx->amount != MDBX_PNL_GETSIZE(txn->tw.repnl))) {
NOTICE("** restart: got %zu loose pages (reclaimed-list %zu -> %zu)", txn->tw.loose_count, ctx->amount,
MDBX_PNL_GETSIZE(txn->tw.repnl));
if (unlikely(txn->wr.loose_count != 0 || ctx->amount != MDBX_PNL_GETSIZE(txn->wr.repnl))) {
NOTICE("** restart: got %zu loose pages (reclaimed-list %zu -> %zu)", txn->wr.loose_count, ctx->amount,
MDBX_PNL_GETSIZE(txn->wr.repnl));
goto retry;
}
@@ -965,12 +965,12 @@ retry:
goto retry;
}
tASSERT(txn, txn->tw.gc.retxl == nullptr || ctx->cleaned_slot == MDBX_PNL_GETSIZE(txn->tw.gc.retxl));
tASSERT(txn, txn->wr.gc.retxl == nullptr || ctx->cleaned_slot == MDBX_PNL_GETSIZE(txn->wr.gc.retxl));
bailout:
txn->cursors[FREE_DBI] = ctx->cursor.next;
MDBX_PNL_SETSIZE(txn->tw.repnl, 0);
MDBX_PNL_SETSIZE(txn->wr.repnl, 0);
#if MDBX_ENABLE_PROFGC
env->lck->pgops.gc_prof.wloops += (uint32_t)ctx->loop;
#endif /* MDBX_ENABLE_PROFGC */