ISAR/libmdbx
3
0
Fork 0
mirror of https://github.com/isar/libmdbx.git synced 2025-01-02 00:44:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

mdbx: refine/optimize mdbx_page_alloc().

Browse Source 
Change-Id: Iebbb8a611a82a379cf23b683bb21c9b6626ea9a5
This commit is contained in:
Leonid Yuriev 2019-10-18 20:28:36 +03:00
parent 36477ef408
commit 15ce797863
2 changed files with 105 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

171
src/elements/core.c
View File

@ -3203,7 +3203,7 @@ bailout:
* *
* If there are free pages available from older transactions, they * If there are free pages available from older transactions, they
* are re-used first. Otherwise allocate a new page at mt_next_pgno. * are re-used first. Otherwise allocate a new page at mt_next_pgno.
* Do not modify the freedB, just merge GC records into mt_reclaimed_pglist * Do not modify the GC, just merge GC records into mt_reclaimed_pglist
* and move mt_last_reclaimed to say which records were consumed. Only this * and move mt_last_reclaimed to say which records were consumed. Only this
* function can create mt_reclaimed_pglist and move * function can create mt_reclaimed_pglist and move
* mt_last_reclaimed/mt_next_pgno. * mt_last_reclaimed/mt_next_pgno.
@ -3272,8 +3272,8 @@ skip_cache:
mdbx_tassert( mdbx_tassert(
txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, txn->mt_next_pgno)); txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, txn->mt_next_pgno));
pgno_t pgno, *repg_list = txn->tw.reclaimed_pglist; pgno_t pgno, *re_list = txn->tw.reclaimed_pglist;
unsigned repg_pos = 0, repg_len = MDBX_PNL_SIZE(repg_list); unsigned range_begin = 0, re_len = MDBX_PNL_SIZE(re_list);
txnid_t oldest = 0, last = 0; txnid_t oldest = 0, last = 0;
const unsigned wanna_range = num - 1; const unsigned wanna_range = num - 1;
@ -3293,24 +3293,33 @@ skip_cache:
* Prefer pages with lower pgno. */ * Prefer pages with lower pgno. */
mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
txn->mt_next_pgno)); txn->mt_next_pgno));
if (likely(flags & MDBX_ALLOC_CACHE) && repg_len > wanna_range && if (likely(flags & MDBX_ALLOC_CACHE) && re_len > wanna_range &&
(!(flags & MDBX_COALESCE) || op == MDBX_FIRST)) { (!(flags & MDBX_COALESCE) || op == MDBX_FIRST)) {
mdbx_tassert(txn, MDBX_PNL_LAST(repg_list) < txn->mt_next_pgno && mdbx_tassert(txn, MDBX_PNL_LAST(re_list) < txn->mt_next_pgno &&
MDBX_PNL_FIRST(repg_list) < txn->mt_next_pgno); MDBX_PNL_FIRST(re_list) < txn->mt_next_pgno);
range_begin = MDBX_PNL_ASCENDING ? 1 : re_len;
pgno = MDBX_PNL_LEAST(re_list);
if (likely(wanna_range == 0))
goto done;
#if MDBX_PNL_ASCENDING #if MDBX_PNL_ASCENDING
for (repg_pos = 1; repg_pos <= repg_len - wanna_range; ++repg_pos) { mdbx_tassert(txn, pgno == re_list[1] && range_begin == 1);
pgno = repg_list[repg_pos]; while (true) {
if (likely(repg_list[repg_pos + wanna_range - 1] == unsigned range_end = range_begin + wanna_range;
pgno + wanna_range - 1)) if (re_list[range_end] - pgno == wanna_range)
goto done; goto done;
if (range_end == re_len)
break;
pgno = re_list[++range_begin];
} }
#else #else
repg_pos = repg_len; mdbx_tassert(txn, pgno == re_list[re_len] && range_begin == re_len);
do { while (true) {
pgno = repg_list[repg_pos]; if (re_list[range_begin - wanna_range] - pgno == wanna_range)
if (likely(repg_list[repg_pos - wanna_range] == pgno + wanna_range))
goto done; goto done;
} while (--repg_pos > wanna_range); if (range_begin == wanna_range)
break;
pgno = re_list[--range_begin];
}
#endif /* MDBX_PNL sort-order */ #endif /* MDBX_PNL sort-order */
} }
@ -3369,7 +3378,15 @@ skip_cache:
goto fail; goto fail;
} }
last = *(txnid_t *)key.iov_base; if (unlikely(key.iov_len != sizeof(txnid_t))) {
rc = MDBX_CORRUPTED;
goto fail;
}
memcpy(&last, key.iov_base, sizeof(txnid_t));
if (unlikely(last < 1 || last >= SAFE64_INVALID_THRESHOLD)) {
rc = MDBX_CORRUPTED;
goto fail;
}
if (oldest <= last) { if (oldest <= last) {
oldest = mdbx_find_oldest(txn); oldest = mdbx_find_oldest(txn);
if (oldest <= last) { if (oldest <= last) {
@ -3407,20 +3424,20 @@ skip_cache:
/* Append PNL from GC record to me_reclaimed_pglist */ /* Append PNL from GC record to me_reclaimed_pglist */
mdbx_cassert(mc, (mc->mc_flags & C_GCFREEZE) == 0); mdbx_cassert(mc, (mc->mc_flags & C_GCFREEZE) == 0);
pgno_t *re_pnl = (pgno_t *)data.iov_base; pgno_t *gc_pnl = (pgno_t *)data.iov_base;
mdbx_tassert(txn, data.iov_len >= MDBX_PNL_SIZEOF(re_pnl)); mdbx_tassert(txn, data.iov_len >= MDBX_PNL_SIZEOF(gc_pnl));
if (unlikely(data.iov_len < MDBX_PNL_SIZEOF(re_pnl) || if (unlikely(data.iov_len < MDBX_PNL_SIZEOF(gc_pnl) ||
!mdbx_pnl_check(re_pnl, txn->mt_next_pgno))) { !mdbx_pnl_check(gc_pnl, txn->mt_next_pgno))) {
rc = MDBX_CORRUPTED; rc = MDBX_CORRUPTED;
goto fail; goto fail;
} }
repg_pos = MDBX_PNL_SIZE(re_pnl); const unsigned gc_len = MDBX_PNL_SIZE(gc_pnl);
rc = mdbx_pnl_need(&txn->tw.reclaimed_pglist, repg_pos); rc = mdbx_pnl_need(&txn->tw.reclaimed_pglist, gc_len);
if (unlikely(rc != MDBX_SUCCESS)) if (unlikely(rc != MDBX_SUCCESS))
goto fail; goto fail;
repg_list = txn->tw.reclaimed_pglist; re_list = txn->tw.reclaimed_pglist;
/* Remember ID of FreeDB record */ /* Remember ID of GC record */
if (flags & MDBX_LIFORECLAIM) { if (flags & MDBX_LIFORECLAIM) {
if ((rc = mdbx_txl_append(&txn->tw.lifo_reclaimed, last)) != 0) if ((rc = mdbx_txl_append(&txn->tw.lifo_reclaimed, last)) != 0)
goto fail; goto fail;
@ -3430,66 +3447,76 @@ skip_cache:
if (mdbx_log_enabled(MDBX_LOG_EXTRA)) { if (mdbx_log_enabled(MDBX_LOG_EXTRA)) {
mdbx_debug_extra("PNL read txn %" PRIaTXN " root %" PRIaPGNO mdbx_debug_extra("PNL read txn %" PRIaTXN " root %" PRIaPGNO
" num %u, PNL", " num %u, PNL",
last, txn->mt_dbs[FREE_DBI].md_root, repg_pos); last, txn->mt_dbs[FREE_DBI].md_root, gc_len);
unsigned i; unsigned i;
for (i = repg_pos; i; i--) for (i = gc_len; i; i--)
mdbx_debug_extra_print(" %" PRIaPGNO, re_pnl[i]); mdbx_debug_extra_print(" %" PRIaPGNO, gc_pnl[i]);
mdbx_debug_extra_print("\n"); mdbx_debug_extra_print("\n");
} }
/* Merge in descending sorted order */ /* Merge in descending sorted order */
mdbx_pnl_xmerge(repg_list, re_pnl); const unsigned prev_re_len = MDBX_PNL_SIZE(re_list);
mdbx_pnl_xmerge(re_list, gc_pnl);
/* re-check to avoid duplicates */ /* re-check to avoid duplicates */
if (unlikely(!mdbx_pnl_check(repg_list, txn->mt_next_pgno))) { if (unlikely(!mdbx_pnl_check(re_list, txn->mt_next_pgno))) {
rc = MDBX_CORRUPTED; rc = MDBX_CORRUPTED;
goto fail; goto fail;
} }
repg_len = MDBX_PNL_SIZE(repg_list);
re_len = MDBX_PNL_SIZE(re_list);
mdbx_tassert(txn, re_len == 0 || re_list[re_len] < txn->mt_next_pgno);
if (re_len && unlikely(MDBX_PNL_MOST(re_list) == txn->mt_next_pgno - 1)) {
/* Refund suitable pages into "unallocated" space */
mdbx_refund(txn);
re_list = txn->tw.reclaimed_pglist;
re_len = MDBX_PNL_SIZE(re_list);
}
if (unlikely((flags & MDBX_ALLOC_CACHE) == 0)) { if (unlikely((flags & MDBX_ALLOC_CACHE) == 0)) {
/* Done for a kick-reclaim mode, actually no page needed */ /* Done for a kick-reclaim mode, actually no page needed */
return MDBX_SUCCESS; return MDBX_SUCCESS;
} }
mdbx_tassert(txn,
repg_len == 0 || repg_list[repg_len] < txn->mt_next_pgno);
if (repg_len &&
unlikely(MDBX_PNL_MOST(repg_list) == txn->mt_next_pgno - 1)) {
/* Refund suitable pages into "unallocated" space */
mdbx_refund(txn);
repg_list = txn->tw.reclaimed_pglist;
repg_len = MDBX_PNL_SIZE(repg_list);
}
/* Don't try to coalesce too much. */ /* Don't try to coalesce too much. */
if (unlikely(repg_len > MDBX_DPL_TXNFULL / 4)) if (unlikely(re_len > MDBX_DPL_TXNFULL / 4))
break; break;
if (repg_len /* current size */ >= env->me_maxgc_ov1page || if (re_len /* current size */ >= env->me_maxgc_ov1page ||
repg_pos /* prev size */ >= env->me_maxgc_ov1page / 2) (re_len > prev_re_len && re_len - prev_re_len /* delta from prev */ >=
env->me_maxgc_ov1page / 2))
flags &= ~MDBX_COALESCE; flags &= ~MDBX_COALESCE;
} }
if ((flags & (MDBX_COALESCE | MDBX_ALLOC_CACHE)) == if ((flags & (MDBX_COALESCE | MDBX_ALLOC_CACHE)) ==
(MDBX_COALESCE | MDBX_ALLOC_CACHE) && (MDBX_COALESCE | MDBX_ALLOC_CACHE) &&
repg_len > wanna_range) { re_len > wanna_range) {
range_begin = MDBX_PNL_ASCENDING ? 1 : re_len;
pgno = MDBX_PNL_LEAST(re_list);
if (likely(wanna_range == 0))
goto done;
#if MDBX_PNL_ASCENDING #if MDBX_PNL_ASCENDING
for (repg_pos = 1; repg_pos <= repg_len - wanna_range; ++repg_pos) { mdbx_tassert(txn, pgno == re_list[1] && range_begin == 1);
pgno = repg_list[repg_pos]; while (true) {
if (likely(repg_list[repg_pos + wanna_range - 1] == unsigned range_end = range_begin + wanna_range;
pgno + wanna_range - 1)) if (re_list[range_end] - pgno == wanna_range)
goto done; goto done;
if (range_end == re_len)
break;
pgno = re_list[++range_begin];
} }
#else #else
repg_pos = repg_len; mdbx_tassert(txn, pgno == re_list[re_len] && range_begin == re_len);
do { while (true) {
pgno = repg_list[repg_pos]; if (re_list[range_begin - wanna_range] - pgno == wanna_range)
if (likely(repg_list[repg_pos - wanna_range] == pgno + wanna_range))
goto done; goto done;
} while (--repg_pos > wanna_range); if (range_begin == wanna_range)
break;
pgno = re_list[--range_begin];
}
#endif /* MDBX_PNL sort-order */ #endif /* MDBX_PNL sort-order */
} }
/* Use new pages from the map when nothing suitable in the GC */ /* Use new pages from the map when nothing suitable in the GC */
repg_pos = 0; range_begin = 0;
pgno = txn->mt_next_pgno; pgno = txn->mt_next_pgno;
rc = MDBX_MAP_FULL; rc = MDBX_MAP_FULL;
const pgno_t next = pgno_add(pgno, num); const pgno_t next = pgno_add(pgno, num);
@ -3590,14 +3617,20 @@ done:
} }
} }
if (repg_pos) { if (range_begin) {
mdbx_cassert(mc, (mc->mc_flags & C_GCFREEZE) == 0); mdbx_cassert(mc, (mc->mc_flags & C_GCFREEZE) == 0);
mdbx_tassert(txn, pgno < txn->mt_next_pgno); mdbx_tassert(txn, pgno < txn->mt_next_pgno);
mdbx_tassert(txn, pgno == repg_list[repg_pos]); mdbx_tassert(txn, pgno == re_list[range_begin]);
/* Cutoff allocated pages from me_reclaimed_pglist */ /* Cutoff allocated pages from me_reclaimed_pglist */
MDBX_PNL_SIZE(repg_list) = repg_len -= num; #if MDBX_PNL_ASCENDING
for (unsigned i = repg_pos - num; i < repg_len;) for (unsigned i = range_begin + num; i <= re_len;)
repg_list[++i] = repg_list[++repg_pos]; re_list[range_begin++] = re_list[i++];
MDBX_PNL_SIZE(re_list) = re_len = range_begin - 1;
#else
MDBX_PNL_SIZE(re_list) = re_len -= num;
for (unsigned i = range_begin - num; i < re_len;)
re_list[++i] = re_list[++range_begin];
#endif
mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist,
txn->mt_next_pgno)); txn->mt_next_pgno));
} else { } else {
@ -3606,7 +3639,7 @@ done:
} }
if (unlikely(env->me_flags & MDBX_PAGEPERTURB)) if (unlikely(env->me_flags & MDBX_PAGEPERTURB))
memset(np, 0x71 /* 'q', 113 */, pgno2bytes(env, num)); memset(np, -1, pgno2bytes(env, num));
VALGRIND_MAKE_MEM_UNDEFINED(np, pgno2bytes(env, num)); VALGRIND_MAKE_MEM_UNDEFINED(np, pgno2bytes(env, num));
np->mp_pgno = pgno; np->mp_pgno = pgno;
@ -5369,7 +5402,7 @@ retry:
rc = mdbx_page_alloc(&mc, 0, NULL, MDBX_ALLOC_GC | MDBX_ALLOC_KICK); rc = mdbx_page_alloc(&mc, 0, NULL, MDBX_ALLOC_GC | MDBX_ALLOC_KICK);
mc.mc_flags |= C_RECLAIMING; mc.mc_flags |= C_RECLAIMING;
if (likely(rc == MDBX_SUCCESS)) { if (likely(rc == MDBX_SUCCESS)) {
/* LY: ok, reclaimed from freedb. */ /* LY: ok, reclaimed from GC. */
mdbx_trace("%s: took @%" PRIaTXN " from GC, continue", mdbx_trace("%s: took @%" PRIaTXN " from GC, continue",
dbg_prefix_mode, MDBX_PNL_LAST(txn->tw.lifo_reclaimed)); dbg_prefix_mode, MDBX_PNL_LAST(txn->tw.lifo_reclaimed));
continue; continue;
@ -5378,7 +5411,7 @@ retry:
/* LY: other troubles... */ /* LY: other troubles... */
goto bailout; goto bailout;
/* LY: freedb is empty, will look any free txn-id in high2low order. */ /* LY: GC is empty, will look any free txn-id in high2low order. */
do { do {
--head_gc_id; --head_gc_id;
mdbx_assert(env, mdbx_assert(env,
@ -5479,7 +5512,7 @@ retry:
mdbx_tassert(txn, reservation_gc_id < *env->me_oldest); mdbx_tassert(txn, reservation_gc_id < *env->me_oldest);
if (unlikely(reservation_gc_id < 1 || if (unlikely(reservation_gc_id < 1 ||
reservation_gc_id >= *env->me_oldest)) { reservation_gc_id >= *env->me_oldest)) {
/* LY: not any txn in the past of freedb. */ /* LY: not any txn in the past of GC. */
rc = MDBX_PROBLEM; rc = MDBX_PROBLEM;
goto bailout; goto bailout;
} }
@ -6414,20 +6447,20 @@ static int __cold mdbx_read_header(MDBX_env *env, MDBX_meta *meta,
continue; continue;
} }
/* LY: FreeDB root */ /* LY: GC root */
if (page.mp_meta.mm_dbs[FREE_DBI].md_root == P_INVALID) { if (page.mp_meta.mm_dbs[FREE_DBI].md_root == P_INVALID) {
if (page.mp_meta.mm_dbs[FREE_DBI].md_branch_pages || if (page.mp_meta.mm_dbs[FREE_DBI].md_branch_pages ||
page.mp_meta.mm_dbs[FREE_DBI].md_depth || page.mp_meta.mm_dbs[FREE_DBI].md_depth ||
page.mp_meta.mm_dbs[FREE_DBI].md_entries || page.mp_meta.mm_dbs[FREE_DBI].md_entries ||
page.mp_meta.mm_dbs[FREE_DBI].md_leaf_pages || page.mp_meta.mm_dbs[FREE_DBI].md_leaf_pages ||
page.mp_meta.mm_dbs[FREE_DBI].md_overflow_pages) { page.mp_meta.mm_dbs[FREE_DBI].md_overflow_pages) {
mdbx_notice("meta[%u] has false-empty freedb, skip it", meta_number); mdbx_notice("meta[%u] has false-empty GC, skip it", meta_number);
rc = MDBX_CORRUPTED; rc = MDBX_CORRUPTED;
continue; continue;
} }
} else if (page.mp_meta.mm_dbs[FREE_DBI].md_root >= } else if (page.mp_meta.mm_dbs[FREE_DBI].md_root >=
page.mp_meta.mm_geo.next) { page.mp_meta.mm_geo.next) {
mdbx_notice("meta[%u] has invalid freedb-root %" PRIaPGNO ", skip it", mdbx_notice("meta[%u] has invalid GC-root %" PRIaPGNO ", skip it",
meta_number, page.mp_meta.mm_dbs[FREE_DBI].md_root); meta_number, page.mp_meta.mm_dbs[FREE_DBI].md_root);
rc = MDBX_CORRUPTED; rc = MDBX_CORRUPTED;
continue; continue;
@ -8744,8 +8777,8 @@ __hot static int mdbx_page_search_root(MDBX_cursor *mc, MDBX_val *key,
mdbx_debug("branch page %" PRIaPGNO " has %u keys", mp->mp_pgno, mdbx_debug("branch page %" PRIaPGNO " has %u keys", mp->mp_pgno,
NUMKEYS(mp)); NUMKEYS(mp));
/* Don't assert on branch pages in the FreeDB. We can get here /* Don't assert on branch pages in the GC. We can get here
* while in the process of rebalancing a FreeDB branch page; we must * while in the process of rebalancing a GC branch page; we must
* let that proceed. ITS#8336 */ * let that proceed. ITS#8336 */
mdbx_cassert(mc, !mc->mc_dbi || NUMKEYS(mp) > 1); mdbx_cassert(mc, !mc->mc_dbi || NUMKEYS(mp) > 1);
mdbx_debug("found index 0 to page %" PRIaPGNO, NODEPGNO(NODEPTR(mp, 0))); mdbx_debug("found index 0 to page %" PRIaPGNO, NODEPGNO(NODEPTR(mp, 0)));
@ -11738,7 +11771,7 @@ static int mdbx_page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) {
} }
} }
/* If not operating on FreeDB, allow this page to be reused /* If not operating on GC, allow this page to be reused
* in this txn. Otherwise just add to free list. */ * in this txn. Otherwise just add to free list. */
rc = mdbx_page_retire(csrc, psrc); rc = mdbx_page_retire(csrc, psrc);
if (unlikely(rc)) if (unlikely(rc))

6
src/elements/internals.h
View File

@ -438,7 +438,7 @@ typedef struct MDBX_meta {
* P_META pages contain MDBX_meta, the start point of an MDBX snapshot. * P_META pages contain MDBX_meta, the start point of an MDBX snapshot.
* *
* Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once
* in the snapshot: Either used by a database or listed in a freeDB record. */ * in the snapshot: Either used by a database or listed in a GC record. */
typedef struct MDBX_page { typedef struct MDBX_page {
union { union {
struct MDBX_page *mp_next; /* for in-memory list of freed pages */ struct MDBX_page *mp_next; /* for in-memory list of freed pages */
@ -829,7 +829,7 @@ struct MDBX_txn {
MDBX_reader *reader; MDBX_reader *reader;
} to; } to;
struct { struct {
pgno_t *reclaimed_pglist; /* Reclaimed freeDB pages */ pgno_t *reclaimed_pglist; /* Reclaimed GC pages */
txnid_t last_reclaimed; /* ID of last used record */ txnid_t last_reclaimed; /* ID of last used record */
pgno_t loose_refund_wl /* FIXME: describe */; pgno_t loose_refund_wl /* FIXME: describe */;
/* dirtylist room: Dirty array size - dirty pages visible to this txn. /* dirtylist room: Dirty array size - dirty pages visible to this txn.
@ -904,7 +904,7 @@ struct MDBX_cursor {
#define C_SUB 0x04 /* Cursor is a sub-cursor */ #define C_SUB 0x04 /* Cursor is a sub-cursor */
#define C_DEL 0x08 /* last op was a cursor_del */ #define C_DEL 0x08 /* last op was a cursor_del */
#define C_UNTRACK 0x10 /* Un-track cursor when closing */ #define C_UNTRACK 0x10 /* Un-track cursor when closing */
#define C_RECLAIMING 0x20 /* FreeDB lookup is prohibited */ #define C_RECLAIMING 0x20 /* GC lookup is prohibited */
#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */ #define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */
unsigned mc_flags; /* see mdbx_cursor */ unsigned mc_flags; /* see mdbx_cursor */
MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */