ISAR/libmdbx
3
0
Fork 0
mirror of https://github.com/isar/libmdbx.git synced 2025-01-14 23:44:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

mdbx: новые настройки clang-format (косметика).

Browse Source
This commit is contained in:
Леонид Юрьев (Leonid Yuriev) 2024-12-11 21:22:04 +03:00
parent 3c4d019d00
commit 8867c2ddc2
129 changed files with 6727 additions and 12640 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.clang-format
View File

@ -1,3 +1,3 @@
BasedOnStyle: LLVM
Standard: Cpp11
ReflowComments: true
Standard: c++20
ColumnLimit: 120

39
example/example-mdbx.c
View File

@ -18,8 +18,7 @@
* <http://www.OpenLDAP.org/license.html>.
*/
#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \
!defined(__USE_MINGW_ANSI_STDIO)
#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO)
#define __USE_MINGW_ANSI_STDIO 1
#endif /* MinGW */
@ -59,33 +58,23 @@ int main(int argc, char *argv[]) {
pagesize_min, pagesize_max, pagesize_default);
printf("\tKey size: minimum %zu, maximum ≈¼ pagesize (%zu bytes for default"
" %zuK pagesize, %zu bytes for %zuK pagesize).\n",
(size_t)0, mdbx_limits_keysize_max(-1, MDBX_DB_DEFAULTS),
pagesize_default / 1024,
mdbx_limits_keysize_max(pagesize_max, MDBX_DB_DEFAULTS),
pagesize_max / 1024);
(size_t)0, mdbx_limits_keysize_max(-1, MDBX_DB_DEFAULTS), pagesize_default / 1024,
mdbx_limits_keysize_max(pagesize_max, MDBX_DB_DEFAULTS), pagesize_max / 1024);
printf("\tValue size: minimum %zu, maximum %zu (0x%08zX) bytes for maps,"
" ≈¼ pagesize for multimaps (%zu bytes for default %zuK pagesize,"
" %zu bytes for %zuK pagesize).\n",
(size_t)0, mdbx_limits_valsize_max(pagesize_min, MDBX_DB_DEFAULTS),
mdbx_limits_valsize_max(pagesize_min, MDBX_DB_DEFAULTS),
mdbx_limits_valsize_max(-1, MDBX_DUPSORT), pagesize_default / 1024,
mdbx_limits_valsize_max(pagesize_max, MDBX_DUPSORT),
pagesize_max / 1024);
mdbx_limits_valsize_max(pagesize_min, MDBX_DB_DEFAULTS), mdbx_limits_valsize_max(-1, MDBX_DUPSORT),
pagesize_default / 1024, mdbx_limits_valsize_max(pagesize_max, MDBX_DUPSORT), pagesize_max / 1024);
printf("\tWrite transaction size: up to %zu (0x%zX) pages (%f %s for default "
"%zuK pagesize, %f %s for %zuK pagesize).\n",
mdbx_limits_txnsize_max(pagesize_min) / pagesize_min,
mdbx_limits_txnsize_max(pagesize_min) / pagesize_min,
mdbx_limits_txnsize_max(-1) / scale_factor, scale_unit,
pagesize_default / 1024,
mdbx_limits_txnsize_max(pagesize_max) / scale_factor, scale_unit,
pagesize_max / 1024);
mdbx_limits_txnsize_max(pagesize_min) / pagesize_min, mdbx_limits_txnsize_max(pagesize_min) / pagesize_min,
mdbx_limits_txnsize_max(-1) / scale_factor, scale_unit, pagesize_default / 1024,
mdbx_limits_txnsize_max(pagesize_max) / scale_factor, scale_unit, pagesize_max / 1024);
printf("\tDatabase size: up to %zu pages (%f %s for default %zuK "
"pagesize, %f %s for %zuK pagesize).\n",
mdbx_limits_dbsize_max(pagesize_min) / pagesize_min,
mdbx_limits_dbsize_max(-1) / scale_factor, scale_unit,
pagesize_default / 1024,
mdbx_limits_dbsize_max(pagesize_max) / scale_factor, scale_unit,
pagesize_max / 1024);
mdbx_limits_dbsize_max(pagesize_min) / pagesize_min, mdbx_limits_dbsize_max(-1) / scale_factor, scale_unit,
pagesize_default / 1024, mdbx_limits_dbsize_max(pagesize_max) / scale_factor, scale_unit, pagesize_max / 1024);
printf("\tMaximum sub-databases: %u.\n", MDBX_MAX_DBI);
printf("-----\n");
@ -94,8 +83,7 @@ int main(int argc, char *argv[]) {
fprintf(stderr, "mdbx_env_create: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_env_open(env, "./example-db", MDBX_NOSUBDIR | MDBX_LIFORECLAIM,
0664);
rc = mdbx_env_open(env, "./example-db", MDBX_NOSUBDIR | MDBX_LIFORECLAIM, 0664);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_env_open: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
@ -143,9 +131,8 @@ int main(int argc, char *argv[]) {
int found = 0;
while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n", key.iov_base, (int)key.iov_len,
(char *)key.iov_base, data.iov_base, (int)data.iov_len,
(char *)data.iov_base);
printf("key: %p %.*s, data: %p %.*s\n", key.iov_base, (int)key.iov_len, (char *)key.iov_base, data.iov_base,
(int)data.iov_len, (char *)data.iov_base);
found += 1;
}
if (rc != MDBX_NOTFOUND || found == 0) {

735
mdbx.h
View File

File diff suppressed because it is too large Load Diff

3149
mdbx.h++
View File

File diff suppressed because it is too large Load Diff

177
src/api-cursor.c
View File

@ -14,30 +14,24 @@ MDBX_cursor *mdbx_cursor_create(void *context) {
couple->userctx = context;
couple->outer.top_and_flags = z_poor_mark;
couple->inner.cursor.top_and_flags = z_poor_mark | z_inner;
VALGRIND_MAKE_MEM_DEFINED(&couple->outer.backup,
sizeof(couple->outer.backup));
VALGRIND_MAKE_MEM_DEFINED(&couple->outer.backup, sizeof(couple->outer.backup));
VALGRIND_MAKE_MEM_DEFINED(&couple->outer.tree, sizeof(couple->outer.tree));
VALGRIND_MAKE_MEM_DEFINED(&couple->outer.clc, sizeof(couple->outer.clc));
VALGRIND_MAKE_MEM_DEFINED(&couple->outer.dbi_state,
sizeof(couple->outer.dbi_state));
VALGRIND_MAKE_MEM_DEFINED(&couple->outer.subcur,
sizeof(couple->outer.subcur));
VALGRIND_MAKE_MEM_DEFINED(&couple->outer.dbi_state, sizeof(couple->outer.dbi_state));
VALGRIND_MAKE_MEM_DEFINED(&couple->outer.subcur, sizeof(couple->outer.subcur));
VALGRIND_MAKE_MEM_DEFINED(&couple->outer.txn, sizeof(couple->outer.txn));
return &couple->outer;
}
int mdbx_cursor_renew(const MDBX_txn *txn, MDBX_cursor *mc) {
return likely(mc)
? mdbx_cursor_bind(txn, mc, (kvx_t *)mc->clc - txn->env->kvs)
: LOG_IFERR(MDBX_EINVAL);
return likely(mc) ? mdbx_cursor_bind(txn, mc, (kvx_t *)mc->clc - txn->env->kvs) : LOG_IFERR(MDBX_EINVAL);
}
int mdbx_cursor_reset(MDBX_cursor *mc) {
if (unlikely(!mc))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_ready4dispose &&
mc->signature != cur_signature_live))
if (unlikely(mc->signature != cur_signature_ready4dispose && mc->signature != cur_signature_live))
return LOG_IFERR(MDBX_EBADSIGN);
cursor_couple_t *couple = (cursor_couple_t *)mc;
@ -50,8 +44,7 @@ int mdbx_cursor_bind(const MDBX_txn *txn, MDBX_cursor *mc, MDBX_dbi dbi) {
if (unlikely(!mc))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_ready4dispose &&
mc->signature != cur_signature_live))
if (unlikely(mc->signature != cur_signature_ready4dispose && mc->signature != cur_signature_live))
return LOG_IFERR(MDBX_EBADSIGN);
int rc = check_txn(txn, MDBX_TXN_BLOCKED);
@ -68,16 +61,14 @@ int mdbx_cursor_bind(const MDBX_txn *txn, MDBX_cursor *mc, MDBX_dbi dbi) {
if (unlikely(mc->backup)) /* Cursor from parent transaction */ {
cASSERT(mc, mc->signature == cur_signature_live);
if (unlikely(cursor_dbi(mc) != dbi ||
/* paranoia */ mc->signature != cur_signature_live ||
mc->txn != txn))
/* paranoia */ mc->signature != cur_signature_live || mc->txn != txn))
return LOG_IFERR(MDBX_EINVAL);
cASSERT(mc, mc->tree == &txn->dbs[dbi]);
cASSERT(mc, mc->clc == &txn->env->kvs[dbi].clc);
cASSERT(mc, cursor_dbi(mc) == dbi);
return likely(cursor_dbi(mc) == dbi &&
/* paranoia */ mc->signature == cur_signature_live &&
mc->txn == txn)
/* paranoia */ mc->signature == cur_signature_live && mc->txn == txn)
? MDBX_SUCCESS
: LOG_IFERR(MDBX_EINVAL) /* Disallow change DBI in nested
transactions */
@ -105,9 +96,7 @@ int mdbx_cursor_unbind(MDBX_cursor *mc) {
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_live))
return (mc->signature == cur_signature_ready4dispose)
? MDBX_SUCCESS
: LOG_IFERR(MDBX_EBADSIGN);
return (mc->signature == cur_signature_ready4dispose) ? MDBX_SUCCESS : LOG_IFERR(MDBX_EBADSIGN);
if (unlikely(mc->backup)) /* Cursor from parent transaction */
return LOG_IFERR(MDBX_EINVAL);
@ -116,9 +105,7 @@ int mdbx_cursor_unbind(MDBX_cursor *mc) {
cASSERT(mc, mc->signature == cur_signature_live);
cASSERT(mc, !mc->backup);
if (unlikely(!mc->txn || mc->txn->signature != txn_signature)) {
ERROR("Wrong cursor's transaction %p 0x%x",
__Wpedantic_format_voidptr(mc->txn),
mc->txn ? mc->txn->signature : 0);
ERROR("Wrong cursor's transaction %p 0x%x", __Wpedantic_format_voidptr(mc->txn), mc->txn ? mc->txn->signature : 0);
return LOG_IFERR(MDBX_PROBLEM);
}
if (mc->next != mc) {
@ -160,8 +147,7 @@ int mdbx_cursor_open(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_cursor **ret) {
void mdbx_cursor_close(MDBX_cursor *mc) {
if (likely(mc)) {
ENSURE(nullptr, mc->signature == cur_signature_live ||
mc->signature == cur_signature_ready4dispose);
ENSURE(nullptr, mc->signature == cur_signature_live || mc->signature == cur_signature_ready4dispose);
MDBX_txn *const txn = mc->txn;
if (!mc->backup) {
mc->txn = nullptr;
@ -194,9 +180,7 @@ int mdbx_cursor_copy(const MDBX_cursor *src, MDBX_cursor *dest) {
if (unlikely(!src))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(src->signature != cur_signature_live))
return LOG_IFERR((src->signature == cur_signature_ready4dispose)
? MDBX_EINVAL
: MDBX_EBADSIGN);
return LOG_IFERR((src->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN);
int rc = mdbx_cursor_bind(src->txn, dest, cursor_dbi(src));
if (unlikely(rc != MDBX_SUCCESS))
@ -229,8 +213,7 @@ int mdbx_txn_release_all_cursors(const MDBX_txn *txn, bool unbind) {
TXN_FOREACH_DBI_FROM(txn, i, MAIN_DBI) {
while (txn->cursors[i]) {
MDBX_cursor *mc = txn->cursors[i];
ENSURE(nullptr, mc->signature == cur_signature_live &&
(mc->next != mc) && !mc->backup);
ENSURE(nullptr, mc->signature == cur_signature_live && (mc->next != mc) && !mc->backup);
rc = likely(rc < INT_MAX) ? rc + 1 : rc;
txn->cursors[i] = mc->next;
mc->next = mc;
@ -250,8 +233,7 @@ int mdbx_txn_release_all_cursors(const MDBX_txn *txn, bool unbind) {
return rc;
}
int mdbx_cursor_compare(const MDBX_cursor *l, const MDBX_cursor *r,
bool ignore_multival) {
int mdbx_cursor_compare(const MDBX_cursor *l, const MDBX_cursor *r, bool ignore_multival) {
const int incomparable = INT16_MAX + 1;
if (unlikely(!l))
return r ? -incomparable * 9 : 0;
@ -267,8 +249,7 @@ int mdbx_cursor_compare(const MDBX_cursor *l, const MDBX_cursor *r,
if (l->txn->env != r->txn->env)
return (l->txn->env > r->txn->env) ? incomparable * 7 : -incomparable * 7;
if (l->txn->txnid != r->txn->txnid)
return (l->txn->txnid > r->txn->txnid) ? incomparable * 6
: -incomparable * 6;
return (l->txn->txnid > r->txn->txnid) ? incomparable * 6 : -incomparable * 6;
return (l->clc > r->clc) ? incomparable * 5 : -incomparable * 5;
}
assert(cursor_dbi(l) == cursor_dbi(r));
@ -333,9 +314,7 @@ int mdbx_cursor_count(const MDBX_cursor *mc, size_t *countp) {
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_live))
return LOG_IFERR((mc->signature == cur_signature_ready4dispose)
? MDBX_EINVAL
: MDBX_EBADSIGN);
return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN);
int rc = check_txn(mc->txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
@ -349,9 +328,8 @@ int mdbx_cursor_count(const MDBX_cursor *mc, size_t *countp) {
const page_t *mp = mc->pg[mc->top];
const node_t *node = page_node(mp, mc->ki[mc->top]);
cASSERT(mc, node_flags(node) & N_DUP);
*countp = unlikely(mc->subcur->nested_tree.items > PTRDIFF_MAX)
? PTRDIFF_MAX
: (size_t)mc->subcur->nested_tree.items;
*countp =
unlikely(mc->subcur->nested_tree.items > PTRDIFF_MAX) ? PTRDIFF_MAX : (size_t)mc->subcur->nested_tree.items;
}
}
return MDBX_SUCCESS;
@ -362,9 +340,7 @@ int mdbx_cursor_on_first(const MDBX_cursor *mc) {
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_live))
return LOG_IFERR((mc->signature == cur_signature_ready4dispose)
? MDBX_EINVAL
: MDBX_EBADSIGN);
return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN);
for (intptr_t i = 0; i <= mc->top; ++i) {
if (mc->ki[i])
@ -379,9 +355,7 @@ int mdbx_cursor_on_first_dup(const MDBX_cursor *mc) {
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_live))
return LOG_IFERR((mc->signature == cur_signature_ready4dispose)
? MDBX_EINVAL
: MDBX_EBADSIGN);
return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN);
if (is_filled(mc) && mc->subcur) {
mc = &mc->subcur->cursor;
@ -399,9 +373,7 @@ int mdbx_cursor_on_last(const MDBX_cursor *mc) {
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_live))
return LOG_IFERR((mc->signature == cur_signature_ready4dispose)
? MDBX_EINVAL
: MDBX_EBADSIGN);
return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN);
for (intptr_t i = 0; i <= mc->top; ++i) {
size_t nkeys = page_numkeys(mc->pg[i]);
@ -417,9 +389,7 @@ int mdbx_cursor_on_last_dup(const MDBX_cursor *mc) {
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_live))
return LOG_IFERR((mc->signature == cur_signature_ready4dispose)
? MDBX_EINVAL
: MDBX_EBADSIGN);
return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN);
if (is_filled(mc) && mc->subcur) {
mc = &mc->subcur->cursor;
@ -438,22 +408,17 @@ int mdbx_cursor_eof(const MDBX_cursor *mc) {
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_live))
return LOG_IFERR((mc->signature == cur_signature_ready4dispose)
? MDBX_EINVAL
: MDBX_EBADSIGN);
return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN);
return is_eof(mc) ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE;
}
int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
MDBX_cursor_op op) {
int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) {
if (unlikely(mc == nullptr))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_live))
return LOG_IFERR((mc->signature == cur_signature_ready4dispose)
? MDBX_EINVAL
: MDBX_EBADSIGN);
return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN);
int rc = check_txn(mc->txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
@ -465,8 +430,7 @@ int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
return LOG_IFERR(cursor_ops(mc, key, data, op));
}
__hot static int scan_confinue(MDBX_cursor *mc, MDBX_predicate_func *predicate,
void *context, void *arg, MDBX_val *key,
__hot static int scan_confinue(MDBX_cursor *mc, MDBX_predicate_func *predicate, void *context, void *arg, MDBX_val *key,
MDBX_val *value, MDBX_cursor_op turn_op) {
int rc;
switch (turn_op) {
@ -528,22 +492,19 @@ __hot static int scan_confinue(MDBX_cursor *mc, MDBX_predicate_func *predicate,
}
}
int mdbx_cursor_scan(MDBX_cursor *mc, MDBX_predicate_func *predicate,
void *context, MDBX_cursor_op start_op,
int mdbx_cursor_scan(MDBX_cursor *mc, MDBX_predicate_func *predicate, void *context, MDBX_cursor_op start_op,
MDBX_cursor_op turn_op, void *arg) {
if (unlikely(!predicate))
return LOG_IFERR(MDBX_EINVAL);
const unsigned valid_start_mask =
1 << MDBX_FIRST | 1 << MDBX_FIRST_DUP | 1 << MDBX_LAST |
1 << MDBX_LAST_DUP | 1 << MDBX_GET_CURRENT | 1 << MDBX_GET_MULTIPLE;
const unsigned valid_start_mask = 1 << MDBX_FIRST | 1 << MDBX_FIRST_DUP | 1 << MDBX_LAST | 1 << MDBX_LAST_DUP |
1 << MDBX_GET_CURRENT | 1 << MDBX_GET_MULTIPLE;
if (unlikely(start_op > 30 || ((1 << start_op) & valid_start_mask) == 0))
return LOG_IFERR(MDBX_EINVAL);
const unsigned valid_turn_mask =
1 << MDBX_NEXT | 1 << MDBX_NEXT_DUP | 1 << MDBX_NEXT_NODUP |
1 << MDBX_PREV | 1 << MDBX_PREV_DUP | 1 << MDBX_PREV_NODUP |
1 << MDBX_NEXT_MULTIPLE | 1 << MDBX_PREV_MULTIPLE;
const unsigned valid_turn_mask = 1 << MDBX_NEXT | 1 << MDBX_NEXT_DUP | 1 << MDBX_NEXT_NODUP | 1 << MDBX_PREV |
1 << MDBX_PREV_DUP | 1 << MDBX_PREV_NODUP | 1 << MDBX_NEXT_MULTIPLE |
1 << MDBX_PREV_MULTIPLE;
if (unlikely(turn_op > 30 || ((1 << turn_op) & valid_turn_mask) == 0))
return LOG_IFERR(MDBX_EINVAL);
@ -551,28 +512,22 @@ int mdbx_cursor_scan(MDBX_cursor *mc, MDBX_predicate_func *predicate,
int rc = mdbx_cursor_get(mc, &key, &value, start_op);
if (unlikely(rc != MDBX_SUCCESS))
return LOG_IFERR(rc);
return LOG_IFERR(
scan_confinue(mc, predicate, context, arg, &key, &value, turn_op));
return LOG_IFERR(scan_confinue(mc, predicate, context, arg, &key, &value, turn_op));
}
int mdbx_cursor_scan_from(MDBX_cursor *mc, MDBX_predicate_func *predicate,
void *context, MDBX_cursor_op from_op, MDBX_val *key,
MDBX_val *value, MDBX_cursor_op turn_op, void *arg) {
int mdbx_cursor_scan_from(MDBX_cursor *mc, MDBX_predicate_func *predicate, void *context, MDBX_cursor_op from_op,
MDBX_val *key, MDBX_val *value, MDBX_cursor_op turn_op, void *arg) {
if (unlikely(!predicate || !key))
return LOG_IFERR(MDBX_EINVAL);
const unsigned valid_start_mask =
1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY |
1 << MDBX_GET_MULTIPLE | 1 << MDBX_SET_LOWERBOUND |
1 << MDBX_SET_UPPERBOUND;
if (unlikely(from_op < MDBX_TO_KEY_LESSER_THAN &&
((1 << from_op) & valid_start_mask) == 0))
const unsigned valid_start_mask = 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY |
1 << MDBX_GET_MULTIPLE | 1 << MDBX_SET_LOWERBOUND | 1 << MDBX_SET_UPPERBOUND;
if (unlikely(from_op < MDBX_TO_KEY_LESSER_THAN && ((1 << from_op) & valid_start_mask) == 0))
return LOG_IFERR(MDBX_EINVAL);
const unsigned valid_turn_mask =
1 << MDBX_NEXT | 1 << MDBX_NEXT_DUP | 1 << MDBX_NEXT_NODUP |
1 << MDBX_PREV | 1 << MDBX_PREV_DUP | 1 << MDBX_PREV_NODUP |
1 << MDBX_NEXT_MULTIPLE | 1 << MDBX_PREV_MULTIPLE;
const unsigned valid_turn_mask = 1 << MDBX_NEXT | 1 << MDBX_NEXT_DUP | 1 << MDBX_NEXT_NODUP | 1 << MDBX_PREV |
1 << MDBX_PREV_DUP | 1 << MDBX_PREV_NODUP | 1 << MDBX_NEXT_MULTIPLE |
1 << MDBX_PREV_MULTIPLE;
if (unlikely(turn_op > 30 || ((1 << turn_op) & valid_turn_mask) == 0))
return LOG_IFERR(MDBX_EINVAL);
@ -588,12 +543,10 @@ int mdbx_cursor_scan_from(MDBX_cursor *mc, MDBX_predicate_func *predicate,
if (unlikely(rc != MDBX_SUCCESS))
return LOG_IFERR(rc);
}
return LOG_IFERR(
scan_confinue(mc, predicate, context, arg, key, value, turn_op));
return LOG_IFERR(scan_confinue(mc, predicate, context, arg, key, value, turn_op));
}
int mdbx_cursor_get_batch(MDBX_cursor *mc, size_t *count, MDBX_val *pairs,
size_t limit, MDBX_cursor_op op) {
int mdbx_cursor_get_batch(MDBX_cursor *mc, size_t *count, MDBX_val *pairs, size_t limit, MDBX_cursor_op op) {
if (unlikely(!count))
return LOG_IFERR(MDBX_EINVAL);
@ -602,9 +555,7 @@ int mdbx_cursor_get_batch(MDBX_cursor *mc, size_t *count, MDBX_val *pairs,
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_live))
return LOG_IFERR((mc->signature == cur_signature_ready4dispose)
? MDBX_EINVAL
: MDBX_EBADSIGN);
return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN);
int rc = check_txn(mc->txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
@ -661,11 +612,9 @@ int mdbx_cursor_get_batch(MDBX_cursor *mc, size_t *count, MDBX_val *pairs,
}
mp = mc->pg[mc->top];
DEBUG("next page is %" PRIaPGNO ", key index %u", mp->pgno,
mc->ki[mc->top]);
DEBUG("next page is %" PRIaPGNO ", key index %u", mp->pgno, mc->ki[mc->top]);
if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) {
ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor",
mp->pgno, mp->flags);
ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags);
rc = MDBX_CORRUPTED;
goto bailout;
}
@ -686,8 +635,7 @@ int mdbx_cursor_set_userctx(MDBX_cursor *mc, void *ctx) {
if (unlikely(!mc))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_ready4dispose &&
mc->signature != cur_signature_live))
if (unlikely(mc->signature != cur_signature_ready4dispose && mc->signature != cur_signature_live))
return LOG_IFERR(MDBX_EBADSIGN);
cursor_couple_t *couple = container_of(mc, cursor_couple_t, outer);
@ -699,8 +647,7 @@ void *mdbx_cursor_get_userctx(const MDBX_cursor *mc) {
if (unlikely(!mc))
return nullptr;
if (unlikely(mc->signature != cur_signature_ready4dispose &&
mc->signature != cur_signature_live))
if (unlikely(mc->signature != cur_signature_ready4dispose && mc->signature != cur_signature_live))
return nullptr;
cursor_couple_t *couple = container_of(mc, cursor_couple_t, outer);
@ -726,15 +673,12 @@ MDBX_dbi mdbx_cursor_dbi(const MDBX_cursor *mc) {
/*----------------------------------------------------------------------------*/
int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
MDBX_put_flags_t flags) {
int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, MDBX_put_flags_t flags) {
if (unlikely(mc == nullptr || key == nullptr || data == nullptr))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_live))
return LOG_IFERR((mc->signature == cur_signature_ready4dispose)
? MDBX_EINVAL
: MDBX_EBADSIGN);
return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN);
int rc = check_txn_rw(mc->txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
@ -754,12 +698,9 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
const size_t dcount = data[1].iov_len;
if (unlikely(dcount < 2 || data->iov_len == 0))
return LOG_IFERR(MDBX_BAD_VALSIZE);
if (unlikely(mc->tree->dupfix_size != data->iov_len) &&
mc->tree->dupfix_size)
if (unlikely(mc->tree->dupfix_size != data->iov_len) && mc->tree->dupfix_size)
return LOG_IFERR(MDBX_BAD_VALSIZE);
if (unlikely(dcount >
MAX_MAPSIZE / 2 /
(BRANCH_NODE_MAX(MDBX_MAX_PAGESIZE) - NODESIZE))) {
if (unlikely(dcount > MAX_MAPSIZE / 2 / (BRANCH_NODE_MAX(MDBX_MAX_PAGESIZE) - NODESIZE))) {
/* checking for multiplication overflow */
if (unlikely(dcount > MAX_MAPSIZE / 2 / data->iov_len))
return LOG_IFERR(MDBX_TOO_LARGE);
@ -767,15 +708,13 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
}
if (flags & MDBX_RESERVE) {
if (unlikely(mc->tree->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP |
MDBX_INTEGERDUP | MDBX_DUPFIXED)))
if (unlikely(mc->tree->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_INTEGERDUP | MDBX_DUPFIXED)))
return LOG_IFERR(MDBX_INCOMPATIBLE);
data->iov_base = nullptr;
}
if (unlikely(mc->txn->flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED)))
return LOG_IFERR((mc->txn->flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS
: MDBX_BAD_TXN);
return LOG_IFERR((mc->txn->flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN);
return LOG_IFERR(cursor_put_checklen(mc, key, data, flags));
}
@ -785,9 +724,7 @@ int mdbx_cursor_del(MDBX_cursor *mc, MDBX_put_flags_t flags) {
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_live))
return LOG_IFERR((mc->signature == cur_signature_ready4dispose)
? MDBX_EINVAL
: MDBX_EBADSIGN);
return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN);
int rc = check_txn_rw(mc->txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
@ -804,9 +741,7 @@ __cold int mdbx_cursor_ignord(MDBX_cursor *mc) {
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(mc->signature != cur_signature_live))
return LOG_IFERR((mc->signature == cur_signature_ready4dispose)
? MDBX_EINVAL
: MDBX_EBADSIGN);
return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN);
mc->checking |= z_ignord;
if (mc->subcur)

272
src/api-env.c
View File

@ -7,8 +7,7 @@ __cold static intptr_t reasonable_db_maxsize(void) {
static intptr_t cached_result;
if (cached_result == 0) {
intptr_t pagesize, total_ram_pages;
if (unlikely(mdbx_get_sysraminfo(&pagesize, &total_ram_pages, nullptr) !=
MDBX_SUCCESS))
if (unlikely(mdbx_get_sysraminfo(&pagesize, &total_ram_pages, nullptr) != MDBX_SUCCESS))
/* the 32-bit limit is good enough for fallback */
return cached_result = MAX_MAPSIZE32;
@ -24,8 +23,7 @@ __cold static intptr_t reasonable_db_maxsize(void) {
const size_t floor = floor_powerof2(cached_result, unit);
const size_t ceil = ceil_powerof2(cached_result, unit);
const size_t threshold = (size_t)cached_result >> 4;
const bool down =
cached_result - floor < ceil - cached_result || ceil > MAX_MAPSIZE;
const bool down = cached_result - floor < ceil - cached_result || ceil > MAX_MAPSIZE;
if (threshold < (down ? cached_result - floor : ceil - cached_result))
break;
cached_result = down ? floor : ceil;
@ -39,14 +37,12 @@ __cold static int check_alternative_lck_absent(const pathchar_t *lck_pathname) {
if (unlikely(err != MDBX_RESULT_FALSE)) {
if (err == MDBX_RESULT_TRUE)
err = MDBX_DUPLICATED_CLK;
ERROR("Alternative/Duplicate LCK-file '%" MDBX_PRIsPATH "' error %d",
lck_pathname, err);
ERROR("Alternative/Duplicate LCK-file '%" MDBX_PRIsPATH "' error %d", lck_pathname, err);
}
return err;
}
__cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname,
const mdbx_mode_t mode) {
__cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname, const mdbx_mode_t mode) {
memset(&env->pathname, 0, sizeof(env->pathname));
if (unlikely(!pathname || !*pathname))
return MDBX_EINVAL;
@ -63,8 +59,7 @@ __cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname,
return rc;
/* auto-create directory if requested */
if ((env->flags & MDBX_NOSUBDIR) == 0 &&
!CreateDirectoryW(pathname, nullptr)) {
if ((env->flags & MDBX_NOSUBDIR) == 0 && !CreateDirectoryW(pathname, nullptr)) {
rc = GetLastError();
if (rc != ERROR_ALREADY_EXISTS)
return rc;
@ -87,8 +82,7 @@ __cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname,
/* auto-create directory if requested */
const mdbx_mode_t dir_mode =
(/* inherit read/write permissions for group and others */ mode &
(S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) |
(/* inherit read/write permissions for group and others */ mode & (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) |
/* always add read/write/search for owner */ S_IRWXU |
((mode & S_IRGRP) ? /* +search if readable by group */ S_IXGRP : 0) |
((mode & S_IROTH) ? /* +search if readable by others */ S_IXOTH : 0);
@ -120,15 +114,11 @@ __cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname,
size_t base_len = pathname_len;
static const size_t dxb_name_len = ARRAY_LENGTH(dxb_name) - 1;
if (env->flags & MDBX_NOSUBDIR) {
if (base_len > dxb_name_len &&
osal_pathequal(pathname + base_len - dxb_name_len, dxb_name,
dxb_name_len)) {
if (base_len > dxb_name_len && osal_pathequal(pathname + base_len - dxb_name_len, dxb_name, dxb_name_len)) {
env->flags -= MDBX_NOSUBDIR;
base_len -= dxb_name_len;
} else if (base_len == dxb_name_len - 1 && osal_isdirsep(dxb_name[0]) &&
osal_isdirsep(lck_name[0]) &&
osal_pathequal(pathname + base_len - dxb_name_len + 1,
dxb_name + 1, dxb_name_len - 1)) {
} else if (base_len == dxb_name_len - 1 && osal_isdirsep(dxb_name[0]) && osal_isdirsep(lck_name[0]) &&
osal_pathequal(pathname + base_len - dxb_name_len + 1, dxb_name + 1, dxb_name_len - 1)) {
env->flags -= MDBX_NOSUBDIR;
base_len -= dxb_name_len - 1;
}
@ -136,11 +126,9 @@ __cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname,
const size_t suflen_with_NOSUBDIR = sizeof(lock_suffix) + sizeof(pathchar_t);
const size_t suflen_without_NOSUBDIR = sizeof(lck_name) + sizeof(dxb_name);
const size_t enough4any = (suflen_with_NOSUBDIR > suflen_without_NOSUBDIR)
? suflen_with_NOSUBDIR
: suflen_without_NOSUBDIR;
const size_t bytes_needed =
sizeof(pathchar_t) * (base_len * 2 + pathname_len + 1) + enough4any;
const size_t enough4any =
(suflen_with_NOSUBDIR > suflen_without_NOSUBDIR) ? suflen_with_NOSUBDIR : suflen_without_NOSUBDIR;
const size_t bytes_needed = sizeof(pathchar_t) * (base_len * 2 + pathname_len + 1) + enough4any;
env->pathname.buffer = osal_malloc(bytes_needed);
if (!env->pathname.buffer)
return MDBX_ENOMEM;
@ -153,8 +141,7 @@ __cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname,
if (base_len) {
memcpy(buf, pathname, sizeof(pathchar_t) * pathname_len);
if (env->flags & MDBX_NOSUBDIR) {
const pathchar_t *const lck_ext =
osal_fileext(lck_name, ARRAY_LENGTH(lck_name));
const pathchar_t *const lck_ext = osal_fileext(lck_name, ARRAY_LENGTH(lck_name));
if (lck_ext) {
pathchar_t *pathname_ext = osal_fileext(buf, pathname_len);
memcpy(pathname_ext ? pathname_ext : buf + pathname_len, lck_ext,
@ -181,14 +168,11 @@ __cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname,
memcpy(buf + dxb_name_len - 1, lock_suffix, sizeof(lock_suffix));
rc = check_alternative_lck_absent(buf);
memcpy(env->pathname.dxb, dxb_name + 1,
sizeof(dxb_name) - sizeof(pathchar_t));
memcpy(env->pathname.lck, lck_name + 1,
sizeof(lck_name) - sizeof(pathchar_t));
memcpy(env->pathname.dxb, dxb_name + 1, sizeof(dxb_name) - sizeof(pathchar_t));
memcpy(env->pathname.lck, lck_name + 1, sizeof(lck_name) - sizeof(pathchar_t));
}
memcpy(env->pathname.specified, pathname,
sizeof(pathchar_t) * (pathname_len + 1));
memcpy(env->pathname.specified, pathname, sizeof(pathchar_t) * (pathname_len + 1));
return rc;
}
@ -212,8 +196,7 @@ __cold int mdbx_env_create(MDBX_env **penv) {
#endif /* MDBX_64BIT_ATOMIC */
#endif /* MDBX_HAVE_C11ATOMICS */
if (unlikely(!is_powerof2(globals.sys_pagesize) ||
globals.sys_pagesize < MDBX_MIN_PAGESIZE)) {
if (unlikely(!is_powerof2(globals.sys_pagesize) || globals.sys_pagesize < MDBX_MIN_PAGESIZE)) {
ERROR("unsuitable system pagesize %u", globals.sys_pagesize);
return LOG_IFERR(MDBX_INCOMPATIBLE);
}
@ -222,10 +205,8 @@ __cold int mdbx_env_create(MDBX_env **penv) {
if (unlikely(globals.linux_kernel_version < 0x04000000)) {
/* 2022-09-01: Прошло уже более двух лет после окончания какой-либо
* поддержки самого "долгоиграющего" ядра 3.16.85 ветки 3.x */
ERROR("too old linux kernel %u.%u.%u.%u, the >= 4.0.0 is required",
globals.linux_kernel_version >> 24,
(globals.linux_kernel_version >> 16) & 255,
(globals.linux_kernel_version >> 8) & 255,
ERROR("too old linux kernel %u.%u.%u.%u, the >= 4.0.0 is required", globals.linux_kernel_version >> 24,
(globals.linux_kernel_version >> 16) & 255, (globals.linux_kernel_version >> 8) & 255,
globals.linux_kernel_version & 255);
return LOG_IFERR(MDBX_INCOMPATIBLE);
}
@ -237,14 +218,11 @@ __cold int mdbx_env_create(MDBX_env **penv) {
env->max_readers = DEFAULT_READERS;
env->max_dbi = env->n_dbi = CORE_DBS;
env->lazy_fd = env->dsync_fd = env->fd4meta = env->lck_mmap.fd =
INVALID_HANDLE_VALUE;
env->lazy_fd = env->dsync_fd = env->fd4meta = env->lck_mmap.fd = INVALID_HANDLE_VALUE;
env->stuck_meta = -1;
env_options_init(env);
env_setup_pagesize(env, (globals.sys_pagesize < MDBX_MAX_PAGESIZE)
? globals.sys_pagesize
: MDBX_MAX_PAGESIZE);
env_setup_pagesize(env, (globals.sys_pagesize < MDBX_MAX_PAGESIZE) ? globals.sys_pagesize : MDBX_MAX_PAGESIZE);
int rc = osal_fastmutex_init(&env->dbi_lock);
if (unlikely(rc != MDBX_SUCCESS))
@ -318,8 +296,7 @@ __cold int mdbx_env_turn_for_recovery(MDBX_env *env, unsigned target) {
return LOG_IFERR(meta_override(env, target, new_txnid, target_meta));
}
__cold int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname,
unsigned target_meta, bool writeable) {
__cold int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname, unsigned target_meta, bool writeable) {
#if defined(_WIN32) || defined(_WIN64)
wchar_t *pathnameW = nullptr;
int rc = osal_mb2w(pathname, &pathnameW);
@ -330,8 +307,7 @@ __cold int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname,
return LOG_IFERR(rc);
}
__cold int mdbx_env_open_for_recoveryW(MDBX_env *env, const wchar_t *pathname,
unsigned target_meta, bool writeable) {
__cold int mdbx_env_open_for_recoveryW(MDBX_env *env, const wchar_t *pathname, unsigned target_meta, bool writeable) {
#endif /* Windows */
if (unlikely(target_meta >= NUM_METAS))
@ -349,8 +325,7 @@ __cold int mdbx_env_open_for_recoveryW(MDBX_env *env, const wchar_t *pathname,
#else
mdbx_env_open
#endif /* Windows */
(env, pathname, writeable ? MDBX_EXCLUSIVE : MDBX_EXCLUSIVE | MDBX_RDONLY,
0);
(env, pathname, writeable ? MDBX_EXCLUSIVE : MDBX_EXCLUSIVE | MDBX_RDONLY, 0);
}
__cold int mdbx_env_delete(const char *pathname, MDBX_env_delete_mode_t mode) {
@ -364,8 +339,7 @@ __cold int mdbx_env_delete(const char *pathname, MDBX_env_delete_mode_t mode) {
return LOG_IFERR(rc);
}
__cold int mdbx_env_deleteW(const wchar_t *pathname,
MDBX_env_delete_mode_t mode) {
__cold int mdbx_env_deleteW(const wchar_t *pathname, MDBX_env_delete_mode_t mode) {
#endif /* Windows */
switch (mode) {
@ -383,22 +357,18 @@ __cold int mdbx_env_deleteW(const wchar_t *pathname,
MDBX_env dummy_env_silo, *const dummy_env = &dummy_env_silo;
#endif
memset(dummy_env, 0, sizeof(*dummy_env));
dummy_env->flags =
(mode == MDBX_ENV_ENSURE_UNUSED) ? MDBX_EXCLUSIVE : MDBX_ENV_DEFAULTS;
dummy_env->flags = (mode == MDBX_ENV_ENSURE_UNUSED) ? MDBX_EXCLUSIVE : MDBX_ENV_DEFAULTS;
dummy_env->ps = (unsigned)mdbx_default_pagesize();
STATIC_ASSERT(sizeof(dummy_env->flags) == sizeof(MDBX_env_flags_t));
int rc = MDBX_RESULT_TRUE, err = env_handle_pathname(dummy_env, pathname, 0);
if (likely(err == MDBX_SUCCESS)) {
mdbx_filehandle_t clk_handle = INVALID_HANDLE_VALUE,
dxb_handle = INVALID_HANDLE_VALUE;
mdbx_filehandle_t clk_handle = INVALID_HANDLE_VALUE, dxb_handle = INVALID_HANDLE_VALUE;
if (mode > MDBX_ENV_JUST_DELETE) {
err = osal_openfile(MDBX_OPEN_DELETE, dummy_env, dummy_env->pathname.dxb,
&dxb_handle, 0);
err = osal_openfile(MDBX_OPEN_DELETE, dummy_env, dummy_env->pathname.dxb, &dxb_handle, 0);
err = (err == MDBX_ENOFILE) ? MDBX_SUCCESS : err;
if (err == MDBX_SUCCESS) {
err = osal_openfile(MDBX_OPEN_DELETE, dummy_env,
dummy_env->pathname.lck, &clk_handle, 0);
err = osal_openfile(MDBX_OPEN_DELETE, dummy_env, dummy_env->pathname.lck, &clk_handle, 0);
err = (err == MDBX_ENOFILE) ? MDBX_SUCCESS : err;
}
if (err == MDBX_SUCCESS && clk_handle != INVALID_HANDLE_VALUE)
@ -425,8 +395,7 @@ __cold int mdbx_env_deleteW(const wchar_t *pathname,
if (err == MDBX_SUCCESS && !(dummy_env->flags & MDBX_NOSUBDIR) &&
(/* pathname != "." */ pathname[0] != '.' || pathname[1] != 0) &&
(/* pathname != ".." */ pathname[0] != '.' || pathname[1] != '.' ||
pathname[2] != 0)) {
(/* pathname != ".." */ pathname[0] != '.' || pathname[1] != '.' || pathname[2] != 0)) {
err = osal_removedirectory(pathname);
if (err == MDBX_SUCCESS)
rc = MDBX_SUCCESS;
@ -445,8 +414,7 @@ __cold int mdbx_env_deleteW(const wchar_t *pathname,
return LOG_IFERR((err == MDBX_SUCCESS) ? rc : err);
}
__cold int mdbx_env_open(MDBX_env *env, const char *pathname,
MDBX_env_flags_t flags, mdbx_mode_t mode) {
__cold int mdbx_env_open(MDBX_env *env, const char *pathname, MDBX_env_flags_t flags, mdbx_mode_t mode) {
#if defined(_WIN32) || defined(_WIN64)
wchar_t *pathnameW = nullptr;
int rc = osal_mb2w(pathname, &pathnameW);
@ -460,8 +428,7 @@ __cold int mdbx_env_open(MDBX_env *env, const char *pathname,
return LOG_IFERR(rc);
}
__cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname,
MDBX_env_flags_t flags, mdbx_mode_t mode) {
__cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, MDBX_env_flags_t flags, mdbx_mode_t mode) {
#endif /* Windows */
int rc = check_env(env, false);
@ -471,8 +438,7 @@ __cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname,
if (unlikely(flags & ~ENV_USABLE_FLAGS))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(env->lazy_fd != INVALID_HANDLE_VALUE ||
(env->flags & ENV_ACTIVE) != 0 || env->dxb_mmap.base))
if (unlikely(env->lazy_fd != INVALID_HANDLE_VALUE || (env->flags & ENV_ACTIVE) != 0 || env->dxb_mmap.base))
return LOG_IFERR(MDBX_EPERM);
/* Pickup previously mdbx_env_set_flags(),
@ -482,9 +448,8 @@ __cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname,
if (flags & MDBX_RDONLY) {
/* Silently ignore irrelevant flags when we're only getting read access */
flags &= ~(MDBX_WRITEMAP | DEPRECATED_MAPASYNC | MDBX_SAFE_NOSYNC |
MDBX_NOMETASYNC | DEPRECATED_COALESCE | MDBX_LIFORECLAIM |
MDBX_NOMEMINIT | MDBX_ACCEDE);
flags &= ~(MDBX_WRITEMAP | DEPRECATED_MAPASYNC | MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | DEPRECATED_COALESCE |
MDBX_LIFORECLAIM | MDBX_NOMEMINIT | MDBX_ACCEDE);
mode = 0;
} else {
#if MDBX_MMAP_INCOHERENT_FILE_WRITE
@ -520,16 +485,14 @@ __cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname,
MDBX_txn *txn = nullptr;
const intptr_t bitmap_bytes =
#if MDBX_ENABLE_DBI_SPARSE
ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) /
CHAR_BIT;
ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) / CHAR_BIT;
#else
0;
#endif /* MDBX_ENABLE_DBI_SPARSE */
const size_t base = sizeof(MDBX_txn) + sizeof(cursor_couple_t);
const size_t size =
base + bitmap_bytes +
env->max_dbi * (sizeof(txn->dbs[0]) + sizeof(txn->cursors[0]) +
sizeof(txn->dbi_seqs[0]) + sizeof(txn->dbi_state[0]));
const size_t size = base + bitmap_bytes +
env->max_dbi * (sizeof(txn->dbs[0]) + sizeof(txn->cursors[0]) + sizeof(txn->dbi_seqs[0]) +
sizeof(txn->dbi_state[0]));
txn = osal_calloc(1, size);
if (unlikely(!txn)) {
@ -538,10 +501,8 @@ __cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname,
}
txn->dbs = ptr_disp(txn, base);
txn->cursors = ptr_disp(txn->dbs, env->max_dbi * sizeof(txn->dbs[0]));
txn->dbi_seqs =
ptr_disp(txn->cursors, env->max_dbi * sizeof(txn->cursors[0]));
txn->dbi_state =
ptr_disp(txn, size - env->max_dbi * sizeof(txn->dbi_state[0]));
txn->dbi_seqs = ptr_disp(txn->cursors, env->max_dbi * sizeof(txn->cursors[0]));
txn->dbi_state = ptr_disp(txn, size - env->max_dbi * sizeof(txn->dbi_state[0]));
#if MDBX_ENABLE_DBI_SPARSE
txn->dbi_sparse = ptr_disp(txn->dbi_state, -bitmap_bytes);
#endif /* MDBX_ENABLE_DBI_SPARSE */
@ -566,10 +527,9 @@ __cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname,
const meta_ptr_t head = meta_recent(env, &troika);
const tree_t *db = &head.ptr_c->trees.main;
DEBUG("opened database version %u, pagesize %u",
(uint8_t)unaligned_peek_u64(4, head.ptr_c->magic_and_version), env->ps);
DEBUG("using meta page %" PRIaPGNO ", txn %" PRIaTXN,
data_page(head.ptr_c)->pgno, head.txnid);
DEBUG("opened database version %u, pagesize %u", (uint8_t)unaligned_peek_u64(4, head.ptr_c->magic_and_version),
env->ps);
DEBUG("using meta page %" PRIaPGNO ", txn %" PRIaTXN, data_page(head.ptr_c)->pgno, head.txnid);
DEBUG("depth: %u", db->height);
DEBUG("entries: %" PRIu64, db->items);
DEBUG("branch pages: %" PRIaPGNO, db->branch_pages);
@ -651,8 +611,7 @@ __cold int mdbx_env_close_ex(MDBX_env *env, bool dont_sync) {
env->flags |= ENV_FATAL_ERROR;
#endif /* MDBX_ENV_CHECKPID */
if (env->dxb_mmap.base &&
(env->flags & (MDBX_RDONLY | ENV_FATAL_ERROR)) == 0 && env->basal_txn) {
if (env->dxb_mmap.base && (env->flags & (MDBX_RDONLY | ENV_FATAL_ERROR)) == 0 && env->basal_txn) {
if (env->basal_txn->owner && env->basal_txn->owner != osal_thread_self())
return LOG_IFERR(MDBX_BUSY);
} else
@ -675,8 +634,8 @@ __cold int mdbx_env_close_ex(MDBX_env *env, bool dont_sync) {
rc = errno;
else if (st.st_nlink > 0 /* don't sync deleted files */) {
rc = env_sync(env, true, true);
rc = (rc == MDBX_BUSY || rc == EAGAIN || rc == EACCES || rc == EBUSY ||
rc == EWOULDBLOCK || rc == MDBX_RESULT_TRUE)
rc = (rc == MDBX_BUSY || rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK ||
rc == MDBX_RESULT_TRUE)
? MDBX_SUCCESS
: rc;
}
@ -717,8 +676,7 @@ __cold int mdbx_env_close_ex(MDBX_env *env, bool dont_sync) {
/*----------------------------------------------------------------------------*/
static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn,
MDBX_envinfo *out, const size_t bytes,
static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out, const size_t bytes,
troika_t *const troika) {
const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid);
const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat);
@ -752,8 +710,7 @@ static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn,
#endif
}
*troika =
(txn && !(txn->flags & MDBX_TXN_RDONLY)) ? txn->tw.troika : meta_tap(env);
*troika = (txn && !(txn->flags & MDBX_TXN_RDONLY)) ? txn->tw.troika : meta_tap(env);
const meta_ptr_t head = meta_recent(env, troika);
const meta_t *const meta0 = METAPAGE(env, 0);
const meta_t *const meta1 = METAPAGE(env, 1);
@ -780,9 +737,7 @@ static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn,
out->mi_last_pgno = txn->geo.first_unallocated - 1;
out->mi_geo.current = pgno2bytes(env, txn->geo.end_pgno);
const txnid_t wanna_meta_txnid = (txn->flags & MDBX_TXN_RDONLY)
? txn->txnid
: txn->txnid - xMDBX_TXNID_STEP;
const txnid_t wanna_meta_txnid = (txn->flags & MDBX_TXN_RDONLY) ? txn->txnid : txn->txnid - xMDBX_TXNID_STEP;
txn_meta = (out->mi_meta_txnid[0] == wanna_meta_txnid) ? meta0 : txn_meta;
txn_meta = (out->mi_meta_txnid[1] == wanna_meta_txnid) ? meta1 : txn_meta;
txn_meta = (out->mi_meta_txnid[2] == wanna_meta_txnid) ? meta2 : txn_meta;
@ -795,30 +750,23 @@ static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn,
const lck_t *const lck = env->lck;
out->mi_maxreaders = env->max_readers;
out->mi_numreaders = env->lck_mmap.lck
? atomic_load32(&lck->rdt_length, mo_Relaxed)
: INT32_MAX;
out->mi_numreaders = env->lck_mmap.lck ? atomic_load32(&lck->rdt_length, mo_Relaxed) : INT32_MAX;
out->mi_dxb_pagesize = env->ps;
out->mi_sys_pagesize = globals.sys_pagesize;
if (likely(bytes > size_before_bootid)) {
const uint64_t unsynced_pages =
atomic_load64(&lck->unsynced_pages, mo_Relaxed) +
((uint32_t)out->mi_recent_txnid !=
atomic_load32(&lck->meta_sync_txnid, mo_Relaxed));
((uint32_t)out->mi_recent_txnid != atomic_load32(&lck->meta_sync_txnid, mo_Relaxed));
out->mi_unsync_volume = pgno2bytes(env, (size_t)unsynced_pages);
const uint64_t monotime_now = osal_monotime();
uint64_t ts = atomic_load64(&lck->eoos_timestamp, mo_Relaxed);
out->mi_since_sync_seconds16dot16 =
ts ? osal_monotime_to_16dot16_noUnderflow(monotime_now - ts) : 0;
out->mi_since_sync_seconds16dot16 = ts ? osal_monotime_to_16dot16_noUnderflow(monotime_now - ts) : 0;
ts = atomic_load64(&lck->readers_check_timestamp, mo_Relaxed);
out->mi_since_reader_check_seconds16dot16 =
ts ? osal_monotime_to_16dot16_noUnderflow(monotime_now - ts) : 0;
out->mi_autosync_threshold =
pgno2bytes(env, atomic_load32(&lck->autosync_threshold, mo_Relaxed));
out->mi_since_reader_check_seconds16dot16 = ts ? osal_monotime_to_16dot16_noUnderflow(monotime_now - ts) : 0;
out->mi_autosync_threshold = pgno2bytes(env, atomic_load32(&lck->autosync_threshold, mo_Relaxed));
out->mi_autosync_period_seconds16dot16 =
osal_monotime_to_16dot16_noUnderflow(
atomic_load64(&lck->autosync_period, mo_Relaxed));
osal_monotime_to_16dot16_noUnderflow(atomic_load64(&lck->autosync_period, mo_Relaxed));
out->mi_bootid.current.x = globals.bootid.x;
out->mi_bootid.current.y = globals.bootid.y;
out->mi_mode = env->lck_mmap.lck ? lck->envmode.weak : env->flags;
@ -834,8 +782,7 @@ static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn,
out->mi_pgop_stat.spill = atomic_load64(&lck->pgops.spill, mo_Relaxed);
out->mi_pgop_stat.unspill = atomic_load64(&lck->pgops.unspill, mo_Relaxed);
out->mi_pgop_stat.wops = atomic_load64(&lck->pgops.wops, mo_Relaxed);
out->mi_pgop_stat.prefault =
atomic_load64(&lck->pgops.prefault, mo_Relaxed);
out->mi_pgop_stat.prefault = atomic_load64(&lck->pgops.prefault, mo_Relaxed);
out->mi_pgop_stat.mincore = atomic_load64(&lck->pgops.mincore, mo_Relaxed);
out->mi_pgop_stat.msync = atomic_load64(&lck->pgops.msync, mo_Relaxed);
out->mi_pgop_stat.fsync = atomic_load64(&lck->pgops.fsync, mo_Relaxed);
@ -865,8 +812,7 @@ static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn,
return MDBX_SUCCESS;
}
__cold int env_info(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out,
size_t bytes, troika_t *troika) {
__cold int env_info(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out, size_t bytes, troika_t *troika) {
MDBX_envinfo snap;
int rc = env_info_snap(env, txn, &snap, sizeof(snap), troika);
if (unlikely(rc != MDBX_SUCCESS))
@ -878,24 +824,22 @@ __cold int env_info(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out,
if (unlikely(rc != MDBX_SUCCESS))
return rc;
snap.mi_since_sync_seconds16dot16 = out->mi_since_sync_seconds16dot16;
snap.mi_since_reader_check_seconds16dot16 =
out->mi_since_reader_check_seconds16dot16;
snap.mi_since_reader_check_seconds16dot16 = out->mi_since_reader_check_seconds16dot16;
if (likely(memcmp(&snap, out, bytes) == 0))
return MDBX_SUCCESS;
memcpy(&snap, out, bytes);
}
}
__cold int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn,
MDBX_envinfo *arg, size_t bytes) {
__cold int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *arg, size_t bytes) {
if (unlikely((env == nullptr && txn == nullptr) || arg == nullptr))
return LOG_IFERR(MDBX_EINVAL);
const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid);
const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat);
const size_t size_before_dxbid = offsetof(MDBX_envinfo, mi_dxbid);
if (unlikely(bytes != sizeof(MDBX_envinfo)) && bytes != size_before_bootid &&
bytes != size_before_pgop_stat && bytes != size_before_dxbid)
if (unlikely(bytes != sizeof(MDBX_envinfo)) && bytes != size_before_bootid && bytes != size_before_pgop_stat &&
bytes != size_before_dxbid)
return LOG_IFERR(MDBX_EINVAL);
if (txn) {
@ -917,8 +861,7 @@ __cold int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn,
return LOG_IFERR(env_info(env, txn, arg, bytes, &troika));
}
__cold int mdbx_preopen_snapinfo(const char *pathname, MDBX_envinfo *out,
size_t bytes) {
__cold int mdbx_preopen_snapinfo(const char *pathname, MDBX_envinfo *out, size_t bytes) {
#if defined(_WIN32) || defined(_WIN64)
wchar_t *pathnameW = nullptr;
int rc = osal_mb2w(pathname, &pathnameW);
@ -929,8 +872,7 @@ __cold int mdbx_preopen_snapinfo(const char *pathname, MDBX_envinfo *out,
return LOG_IFERR(rc);
}
__cold int mdbx_preopen_snapinfoW(const wchar_t *pathname, MDBX_envinfo *out,
size_t bytes) {
__cold int mdbx_preopen_snapinfoW(const wchar_t *pathname, MDBX_envinfo *out, size_t bytes) {
#endif /* Windows */
if (unlikely(!out))
return LOG_IFERR(MDBX_EINVAL);
@ -938,8 +880,8 @@ __cold int mdbx_preopen_snapinfoW(const wchar_t *pathname, MDBX_envinfo *out,
const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid);
const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat);
const size_t size_before_dxbid = offsetof(MDBX_envinfo, mi_dxbid);
if (unlikely(bytes != sizeof(MDBX_envinfo)) && bytes != size_before_bootid &&
bytes != size_before_pgop_stat && bytes != size_before_dxbid)
if (unlikely(bytes != sizeof(MDBX_envinfo)) && bytes != size_before_bootid && bytes != size_before_pgop_stat &&
bytes != size_before_dxbid)
return LOG_IFERR(MDBX_EINVAL);
memset(out, 0, bytes);
@ -951,8 +893,7 @@ __cold int mdbx_preopen_snapinfoW(const wchar_t *pathname, MDBX_envinfo *out,
MDBX_env env;
memset(&env, 0, sizeof(env));
env.pid = osal_getpid();
if (unlikely(!is_powerof2(globals.sys_pagesize) ||
globals.sys_pagesize < MDBX_MIN_PAGESIZE)) {
if (unlikely(!is_powerof2(globals.sys_pagesize) || globals.sys_pagesize < MDBX_MIN_PAGESIZE)) {
ERROR("unsuitable system pagesize %u", globals.sys_pagesize);
return LOG_IFERR(MDBX_INCOMPATIBLE);
}
@ -972,8 +913,7 @@ __cold int mdbx_preopen_snapinfoW(const wchar_t *pathname, MDBX_envinfo *out,
int rc = env_handle_pathname(&env, pathname, 0);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
rc = osal_openfile(MDBX_OPEN_DXB_READ, &env, env.pathname.dxb, &env.lazy_fd,
0);
rc = osal_openfile(MDBX_OPEN_DXB_READ, &env, env.pathname.dxb, &env.lazy_fd, 0);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
@ -1006,10 +946,8 @@ bailout:
/*----------------------------------------------------------------------------*/
__cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
intptr_t size_now, intptr_t size_upper,
intptr_t growth_step,
intptr_t shrink_threshold, intptr_t pagesize) {
__cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t size_now, intptr_t size_upper,
intptr_t growth_step, intptr_t shrink_threshold, intptr_t pagesize) {
int rc = check_env(env, false);
if (unlikely(rc != MDBX_SUCCESS))
return LOG_IFERR(rc);
@ -1038,8 +976,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
should_unlock = true;
env->basal_txn->tw.troika = meta_tap(env);
eASSERT(env, !env->txn && !env->basal_txn->nested);
env->basal_txn->txnid =
env->basal_txn->tw.troika.txnid[env->basal_txn->tw.troika.recent];
env->basal_txn->txnid = env->basal_txn->tw.troika.txnid[env->basal_txn->tw.troika.recent];
txn_snapshot_oldest(env->basal_txn);
}
@ -1047,9 +984,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
if (pagesize <= 0 || pagesize >= INT_MAX)
pagesize = env->ps;
const geo_t *const geo =
inside_txn
? &env->txn->geo
: &meta_recent(env, &env->basal_txn->tw.troika).ptr_c->geometry;
inside_txn ? &env->txn->geo : &meta_recent(env, &env->basal_txn->tw.troika).ptr_c->geometry;
if (size_lower < 0)
size_lower = pgno2bytes(env, geo->lower);
if (size_now < 0)
@ -1065,8 +1000,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
rc = MDBX_EINVAL;
goto bailout;
}
const size_t usedbytes =
pgno2bytes(env, mvcc_snapshot_largest(env, geo->first_unallocated));
const size_t usedbytes = pgno2bytes(env, mvcc_snapshot_largest(env, geo->first_unallocated));
if ((size_t)size_upper < usedbytes) {
rc = MDBX_MAP_FULL;
goto bailout;
@ -1101,14 +1035,12 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
else if (top >= (intptr_t)MAX_MAPSIZE /* maximal */)
top = MAX_MAPSIZE;
while (top > pagesize * (int64_t)(MAX_PAGENO + 1) &&
pagesize < MDBX_MAX_PAGESIZE)
while (top > pagesize * (int64_t)(MAX_PAGENO + 1) && pagesize < MDBX_MAX_PAGESIZE)
pagesize <<= 1;
}
}
if (pagesize < (intptr_t)MDBX_MIN_PAGESIZE ||
pagesize > (intptr_t)MDBX_MAX_PAGESIZE || !is_powerof2(pagesize)) {
if (pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || !is_powerof2(pagesize)) {
rc = MDBX_EINVAL;
goto bailout;
}
@ -1140,13 +1072,10 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
size_upper = size_now;
else if (size_now >= reasonable_db_maxsize() / 2)
size_upper = reasonable_db_maxsize();
else if ((size_t)size_now >= MAX_MAPSIZE32 / 2 &&
(size_t)size_now <= MAX_MAPSIZE32 / 4 * 3)
else if ((size_t)size_now >= MAX_MAPSIZE32 / 2 && (size_t)size_now <= MAX_MAPSIZE32 / 4 * 3)
size_upper = MAX_MAPSIZE32;
else {
size_upper = ceil_powerof2(((size_t)size_now < MAX_MAPSIZE / 4)
? size_now + size_now
: size_now + size_now / 2,
size_upper = ceil_powerof2(((size_t)size_now < MAX_MAPSIZE / 4) ? size_now + size_now : size_now + size_now / 2,
MEGABYTE * MDBX_WORDBITS * MDBX_WORDBITS / 32);
if ((size_t)size_upper > MAX_MAPSIZE)
size_upper = MAX_MAPSIZE;
@ -1174,15 +1103,12 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
size_now = size_lower;
}
if (unlikely((size_t)size_upper > MAX_MAPSIZE ||
(uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) {
if (unlikely((size_t)size_upper > MAX_MAPSIZE || (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) {
rc = MDBX_TOO_LARGE;
goto bailout;
}
const size_t unit = (globals.sys_pagesize > (size_t)pagesize)
? globals.sys_pagesize
: (size_t)pagesize;
const size_t unit = (globals.sys_pagesize > (size_t)pagesize) ? globals.sys_pagesize : (size_t)pagesize;
size_lower = ceil_powerof2(size_lower, unit);
size_upper = ceil_powerof2(size_upper, unit);
size_now = ceil_powerof2(size_now, unit);
@ -1190,10 +1116,8 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
/* LY: подбираем значение size_upper:
* - кратное размеру страницы
* - без нарушения MAX_MAPSIZE и MAX_PAGENO */
while (unlikely((size_t)size_upper > MAX_MAPSIZE ||
(uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) {
if ((size_t)size_upper < unit + MIN_MAPSIZE ||
(size_t)size_upper < (size_t)pagesize * (MIN_PAGENO + 1)) {
while (unlikely((size_t)size_upper > MAX_MAPSIZE || (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) {
if ((size_t)size_upper < unit + MIN_MAPSIZE || (size_t)size_upper < (size_t)pagesize * (MIN_PAGENO + 1)) {
/* паранойа на случай переполнения при невероятных значениях */
rc = MDBX_EINVAL;
goto bailout;
@ -1235,10 +1159,8 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
env->geo_in_bytes.lower = size_lower;
env->geo_in_bytes.now = size_now;
env->geo_in_bytes.upper = size_upper;
env->geo_in_bytes.grow =
pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, growth_step))));
env->geo_in_bytes.shrink =
pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, shrink_threshold))));
env->geo_in_bytes.grow = pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, growth_step))));
env->geo_in_bytes.shrink = pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, shrink_threshold))));
env_options_adjust_defaults(env);
ENSURE(env, env->geo_in_bytes.lower >= MIN_MAPSIZE);
@ -1290,15 +1212,13 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
meta_set_txnid(env, &meta, txnid);
}
const geo_t *const current_geo =
&(env->txn ? env->txn : env->basal_txn)->geo;
const geo_t *const current_geo = &(env->txn ? env->txn : env->basal_txn)->geo;
/* update env-geo to avoid influences */
env->geo_in_bytes.now = pgno2bytes(env, current_geo->now);
env->geo_in_bytes.lower = pgno2bytes(env, current_geo->lower);
env->geo_in_bytes.upper = pgno2bytes(env, current_geo->upper);
env->geo_in_bytes.grow = pgno2bytes(env, pv2pages(current_geo->grow_pv));
env->geo_in_bytes.shrink =
pgno2bytes(env, pv2pages(current_geo->shrink_pv));
env->geo_in_bytes.shrink = pgno2bytes(env, pv2pages(current_geo->shrink_pv));
geo_t new_geo;
new_geo.lower = bytes2pgno(env, size_lower);
@ -1326,8 +1246,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
#if defined(_WIN32) || defined(_WIN64)
/* Was DB shrinking disabled before and now it will be enabled? */
if (new_geo.lower < new_geo.upper && new_geo.shrink_pv &&
!(current_geo->lower < current_geo->upper &&
current_geo->shrink_pv)) {
!(current_geo->lower < current_geo->upper && current_geo->shrink_pv)) {
if (!env->lck_mmap.lck) {
rc = MDBX_EPERM;
goto bailout;
@ -1341,9 +1260,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
/* Check if there are any reading threads that do not use the SRWL */
const size_t CurrentTid = GetCurrentThreadId();
const reader_slot_t *const begin = env->lck_mmap.lck->rdt;
const reader_slot_t *const end =
begin +
atomic_load32(&env->lck_mmap.lck->rdt_length, mo_AcquireRelease);
const reader_slot_t *const end = begin + atomic_load32(&env->lck_mmap.lck->rdt_length, mo_AcquireRelease);
for (const reader_slot_t *reader = begin; reader < end; ++reader) {
if (reader->pid.weak == env->pid && reader->tid.weak != CurrentTid) {
/* At least one thread may don't use SRWL */
@ -1358,10 +1275,8 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
}
#endif /* Windows */
if (new_geo.now != current_geo->now ||
new_geo.upper != current_geo->upper) {
rc = dxb_resize(env, current_geo->first_unallocated, new_geo.now,
new_geo.upper, explicit_resize);
if (new_geo.now != current_geo->now || new_geo.upper != current_geo->upper) {
rc = dxb_resize(env, current_geo->first_unallocated, new_geo.now, new_geo.upper, explicit_resize);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
}
@ -1370,13 +1285,10 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
env->txn->flags |= MDBX_TXN_DIRTY;
} else {
meta.geometry = new_geo;
rc =
dxb_sync_locked(env, env->flags, &meta, &env->basal_txn->tw.troika);
rc = dxb_sync_locked(env, env->flags, &meta, &env->basal_txn->tw.troika);
if (likely(rc == MDBX_SUCCESS)) {
env->geo_in_bytes.now =
pgno2bytes(env, new_geo.now = meta.geometry.now);
env->geo_in_bytes.upper =
pgno2bytes(env, new_geo.upper = meta.geometry.upper);
env->geo_in_bytes.now = pgno2bytes(env, new_geo.now = meta.geometry.now);
env->geo_in_bytes.upper = pgno2bytes(env, new_geo.upper = meta.geometry.upper);
}
}
}

36
src/api-extra.c
View File

@ -6,8 +6,7 @@
/*------------------------------------------------------------------------------
* Readers API */
__cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func,
void *ctx) {
__cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func, void *ctx) {
int rc = check_env(env, true);
if (unlikely(rc != MDBX_SUCCESS))
return LOG_IFERR(rc);
@ -19,8 +18,7 @@ __cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func,
int serial = 0;
lck_t *const lck = env->lck_mmap.lck;
if (likely(lck)) {
const size_t snap_nreaders =
atomic_load32(&lck->rdt_length, mo_AcquireRelease);
const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
for (size_t i = 0; i < snap_nreaders; i++) {
const reader_slot_t *r = lck->rdt + i;
retry_reader:;
@ -29,17 +27,12 @@ __cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func,
continue;
txnid_t txnid = safe64_read(&r->txnid);
const uint64_t tid = atomic_load64(&r->tid, mo_Relaxed);
const pgno_t pages_used =
atomic_load32(&r->snapshot_pages_used, mo_Relaxed);
const uint64_t reader_pages_retired =
atomic_load64(&r->snapshot_pages_retired, mo_Relaxed);
if (unlikely(txnid != safe64_read(&r->txnid) ||
pid != atomic_load32(&r->pid, mo_AcquireRelease) ||
const pgno_t pages_used = atomic_load32(&r->snapshot_pages_used, mo_Relaxed);
const uint64_t reader_pages_retired = atomic_load64(&r->snapshot_pages_retired, mo_Relaxed);
if (unlikely(txnid != safe64_read(&r->txnid) || pid != atomic_load32(&r->pid, mo_AcquireRelease) ||
tid != atomic_load64(&r->tid, mo_Relaxed) ||
pages_used !=
atomic_load32(&r->snapshot_pages_used, mo_Relaxed) ||
reader_pages_retired !=
atomic_load64(&r->snapshot_pages_retired, mo_Relaxed)))
pages_used != atomic_load32(&r->snapshot_pages_used, mo_Relaxed) ||
reader_pages_retired != atomic_load64(&r->snapshot_pages_retired, mo_Relaxed)))
goto retry_reader;
eASSERT(env, txnid > 0);
@ -53,22 +46,18 @@ __cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func,
troika_t troika = meta_tap(env);
retry_header:;
const meta_ptr_t head = meta_recent(env, &troika);
const uint64_t head_pages_retired =
unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired);
const uint64_t head_pages_retired = unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired);
if (unlikely(meta_should_retry(env, &troika) ||
head_pages_retired != unaligned_peek_u64_volatile(
4, head.ptr_v->pages_retired)))
head_pages_retired != unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired)))
goto retry_header;
lag = (head.txnid - txnid) / xMDBX_TXNID_STEP;
bytes_used = pgno2bytes(env, pages_used);
bytes_retained = (head_pages_retired > reader_pages_retired)
? pgno2bytes(env, (pgno_t)(head_pages_retired -
reader_pages_retired))
? pgno2bytes(env, (pgno_t)(head_pages_retired - reader_pages_retired))
: 0;
}
rc = func(ctx, ++serial, (unsigned)i, pid, (mdbx_tid_t)((intptr_t)tid),
txnid, lag, bytes_used, bytes_retained);
rc = func(ctx, ++serial, (unsigned)i, pid, (mdbx_tid_t)((intptr_t)tid), txnid, lag, bytes_used, bytes_retained);
if (unlikely(rc != MDBX_SUCCESS))
break;
}
@ -93,8 +82,7 @@ int mdbx_txn_lock(MDBX_env *env, bool dont_wait) {
if (unlikely(env->flags & MDBX_RDONLY))
return LOG_IFERR(MDBX_EACCESS);
if (unlikely(env->basal_txn->owner ||
(env->basal_txn->flags & MDBX_TXN_FINISHED) == 0))
if (unlikely(env->basal_txn->owner || (env->basal_txn->flags & MDBX_TXN_FINISHED) == 0))
return LOG_IFERR(MDBX_BUSY);
return LOG_IFERR(lck_txn_lock(env, dont_wait));

84
src/api-key-transform.c
View File

@ -9,16 +9,14 @@ static inline double key2double(const int64_t key) {
double f;
} casting;
casting.u = (key < 0) ? key + UINT64_C(0x8000000000000000)
: UINT64_C(0xffffFFFFffffFFFF) - key;
casting.u = (key < 0) ? key + UINT64_C(0x8000000000000000) : UINT64_C(0xffffFFFFffffFFFF) - key;
return casting.f;
}
static inline uint64_t double2key(const double *const ptr) {
STATIC_ASSERT(sizeof(double) == sizeof(int64_t));
const int64_t i = *(const int64_t *)ptr;
const uint64_t u = (i < 0) ? UINT64_C(0xffffFFFFffffFFFF) - i
: i + UINT64_C(0x8000000000000000);
const uint64_t u = (i < 0) ? UINT64_C(0xffffFFFFffffFFFF) - i : i + UINT64_C(0x8000000000000000);
if (ASSERT_ENABLED()) {
const double f = key2double(u);
assert(memcmp(&f, ptr, sizeof(double)) == 0);
@ -32,16 +30,14 @@ static inline float key2float(const int32_t key) {
float f;
} casting;
casting.u =
(key < 0) ? key + UINT32_C(0x80000000) : UINT32_C(0xffffFFFF) - key;
casting.u = (key < 0) ? key + UINT32_C(0x80000000) : UINT32_C(0xffffFFFF) - key;
return casting.f;
}
static inline uint32_t float2key(const float *const ptr) {
STATIC_ASSERT(sizeof(float) == sizeof(int32_t));
const int32_t i = *(const int32_t *)ptr;
const uint32_t u =
(i < 0) ? UINT32_C(0xffffFFFF) - i : i + UINT32_C(0x80000000);
const uint32_t u = (i < 0) ? UINT32_C(0xffffFFFF) - i : i + UINT32_C(0x80000000);
if (ASSERT_ENABLED()) {
const float f = key2float(u);
assert(memcmp(&f, ptr, sizeof(float)) == 0);
@ -49,21 +45,13 @@ static inline uint32_t float2key(const float *const ptr) {
return u;
}
uint64_t mdbx_key_from_double(const double ieee754_64bit) {
return double2key(&ieee754_64bit);
}
uint64_t mdbx_key_from_double(const double ieee754_64bit) { return double2key(&ieee754_64bit); }
uint64_t mdbx_key_from_ptrdouble(const double *const ieee754_64bit) {
return double2key(ieee754_64bit);
}
uint64_t mdbx_key_from_ptrdouble(const double *const ieee754_64bit) { return double2key(ieee754_64bit); }
uint32_t mdbx_key_from_float(const float ieee754_32bit) {
return float2key(&ieee754_32bit);
}
uint32_t mdbx_key_from_float(const float ieee754_32bit) { return float2key(&ieee754_32bit); }
uint32_t mdbx_key_from_ptrfloat(const float *const ieee754_32bit) {
return float2key(ieee754_32bit);
}
uint32_t mdbx_key_from_ptrfloat(const float *const ieee754_32bit) { return float2key(ieee754_32bit); }
#define IEEE754_DOUBLE_MANTISSA_SIZE 52
#define IEEE754_DOUBLE_EXPONENTA_BIAS 0x3FF
@ -78,8 +66,7 @@ static inline int clz64(uint64_t value) {
return __builtin_clz(value);
if (sizeof(value) == sizeof(long))
return __builtin_clzl(value);
#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || \
__has_builtin(__builtin_clzll)
#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || __has_builtin(__builtin_clzll)
return __builtin_clzll(value);
#endif /* have(long long) && long long == uint64_t */
#endif /* GNU C */
@ -105,11 +92,10 @@ static inline int clz64(uint64_t value) {
value |= value >> 8;
value |= value >> 16;
value |= value >> 32;
static const uint8_t debruijn_clz64[64] = {
63, 16, 62, 7, 15, 36, 61, 3, 6, 14, 22, 26, 35, 47, 60, 2,
9, 5, 28, 11, 13, 21, 42, 19, 25, 31, 34, 40, 46, 52, 59, 1,
17, 8, 37, 4, 23, 27, 48, 10, 29, 12, 43, 20, 32, 41, 53, 18,
38, 24, 49, 30, 44, 33, 54, 39, 50, 45, 55, 51, 56, 57, 58, 0};
static const uint8_t debruijn_clz64[64] = {63, 16, 62, 7, 15, 36, 61, 3, 6, 14, 22, 26, 35, 47, 60, 2,
9, 5, 28, 11, 13, 21, 42, 19, 25, 31, 34, 40, 46, 52, 59, 1,
17, 8, 37, 4, 23, 27, 48, 10, 29, 12, 43, 20, 32, 41, 53, 18,
38, 24, 49, 30, 44, 33, 54, 39, 50, 45, 55, 51, 56, 57, 58, 0};
return debruijn_clz64[value * UINT64_C(0x03F79D71B4CB0A89) >> 58];
}
@ -134,17 +120,12 @@ uint64_t mdbx_key_from_jsonInteger(const int64_t json_integer) {
mantissa = round_mantissa(u64, --shift);
}
assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD &&
mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX);
const uint64_t exponent = (uint64_t)IEEE754_DOUBLE_EXPONENTA_BIAS +
IEEE754_DOUBLE_MANTISSA_SIZE - shift;
assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD && mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX);
const uint64_t exponent = (uint64_t)IEEE754_DOUBLE_EXPONENTA_BIAS + IEEE754_DOUBLE_MANTISSA_SIZE - shift;
assert(exponent > 0 && exponent <= IEEE754_DOUBLE_EXPONENTA_MAX);
const uint64_t key = bias + (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) +
(mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD);
#if !defined(_MSC_VER) || \
defined( \
_DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \
symbol __except1 referenced in function __ftol3_except */
const uint64_t key = bias + (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) + (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD);
#if !defined(_MSC_VER) || defined(_DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \
symbol __except1 referenced in function __ftol3_except */
assert(key == mdbx_key_from_double((double)json_integer));
#endif /* Workaround for MSVC */
return key;
@ -160,17 +141,13 @@ uint64_t mdbx_key_from_jsonInteger(const int64_t json_integer) {
mantissa = round_mantissa(u64, --shift);
}
assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD &&
mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX);
const uint64_t exponent = (uint64_t)IEEE754_DOUBLE_EXPONENTA_BIAS +
IEEE754_DOUBLE_MANTISSA_SIZE - shift;
assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD && mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX);
const uint64_t exponent = (uint64_t)IEEE754_DOUBLE_EXPONENTA_BIAS + IEEE754_DOUBLE_MANTISSA_SIZE - shift;
assert(exponent > 0 && exponent <= IEEE754_DOUBLE_EXPONENTA_MAX);
const uint64_t key = bias - 1 - (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) -
(mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD);
#if !defined(_MSC_VER) || \
defined( \
_DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \
symbol __except1 referenced in function __ftol3_except */
const uint64_t key =
bias - 1 - (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) - (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD);
#if !defined(_MSC_VER) || defined(_DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \
symbol __except1 referenced in function __ftol3_except */
assert(key == mdbx_key_from_double((double)json_integer));
#endif /* Workaround for MSVC */
return key;
@ -185,21 +162,17 @@ int64_t mdbx_jsonInteger_from_key(const MDBX_val v) {
const uint64_t bias = UINT64_C(0x8000000000000000);
const uint64_t covalent = (key > bias) ? key - bias : bias - key - 1;
const int shift = IEEE754_DOUBLE_EXPONENTA_BIAS + 63 -
(IEEE754_DOUBLE_EXPONENTA_MAX &
(int)(covalent >> IEEE754_DOUBLE_MANTISSA_SIZE));
(IEEE754_DOUBLE_EXPONENTA_MAX & (int)(covalent >> IEEE754_DOUBLE_MANTISSA_SIZE));
if (unlikely(shift < 1))
return (key < bias) ? INT64_MIN : INT64_MAX;
if (unlikely(shift > 63))
return 0;
const uint64_t unscaled = ((covalent & IEEE754_DOUBLE_MANTISSA_MASK)
<< (63 - IEEE754_DOUBLE_MANTISSA_SIZE)) +
bias;
const uint64_t unscaled = ((covalent & IEEE754_DOUBLE_MANTISSA_MASK) << (63 - IEEE754_DOUBLE_MANTISSA_SIZE)) + bias;
const int64_t absolute = unscaled >> shift;
const int64_t value = (key < bias) ? -absolute : absolute;
assert(key == mdbx_key_from_jsonInteger(value) ||
(mdbx_key_from_jsonInteger(value - 1) < key &&
key < mdbx_key_from_jsonInteger(value + 1)));
(mdbx_key_from_jsonInteger(value - 1) < key && key < mdbx_key_from_jsonInteger(value + 1)));
return value;
}
@ -220,6 +193,5 @@ int32_t mdbx_int32_from_key(const MDBX_val v) {
int64_t mdbx_int64_from_key(const MDBX_val v) {
assert(v.iov_len == 8);
return (int64_t)(unaligned_peek_u64(2, v.iov_base) -
UINT64_C(0x8000000000000000));
return (int64_t)(unaligned_peek_u64(2, v.iov_base) - UINT64_C(0x8000000000000000));
}

96
src/api-txn.c
View File

@ -16,9 +16,7 @@ int mdbx_txn_straggler(const MDBX_txn *txn, int *percent)
MDBX_env *env = txn->env;
if (unlikely((txn->flags & MDBX_TXN_RDONLY) == 0)) {
if (percent)
*percent = (int)((txn->geo.first_unallocated * UINT64_C(100) +
txn->geo.end_pgno / 2) /
txn->geo.end_pgno);
*percent = (int)((txn->geo.first_unallocated * UINT64_C(100) + txn->geo.end_pgno / 2) / txn->geo.end_pgno);
return 0;
}
@ -28,9 +26,7 @@ int mdbx_txn_straggler(const MDBX_txn *txn, int *percent)
const meta_ptr_t head = meta_recent(env, &troika);
if (percent) {
const pgno_t maxpg = head.ptr_v->geometry.now;
*percent = (int)((head.ptr_v->geometry.first_unallocated * UINT64_C(100) +
maxpg / 2) /
maxpg);
*percent = (int)((head.ptr_v->geometry.first_unallocated * UINT64_C(100) + maxpg / 2) / maxpg);
}
lag = (head.txnid - txn->txnid) / xMDBX_TXNID_STEP;
} while (unlikely(meta_should_retry(env, &troika)));
@ -38,8 +34,7 @@ int mdbx_txn_straggler(const MDBX_txn *txn, int *percent)
return (lag > INT_MAX) ? INT_MAX : (int)lag;
}
__cold int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi,
uint32_t *mask) {
__cold int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi, uint32_t *mask) {
if (unlikely(!mask))
return LOG_IFERR(MDBX_EINVAL);
@ -58,8 +53,7 @@ __cold int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi,
MDBX_val key, data;
rc = outer_first(&cx.outer, &key, &data);
while (rc == MDBX_SUCCESS) {
const node_t *node =
page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]);
const node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]);
const tree_t *db = node_data(node);
const unsigned flags = node_flags(node);
switch (flags) {
@ -77,8 +71,7 @@ __cold int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi,
*mask |= 1 << UNALIGNED_PEEK_16(db, tree_t, height);
break;
default:
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid node-size", flags);
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid node-size", flags);
return LOG_IFERR(MDBX_CORRUPTED);
}
rc = outer_next(&cx.outer, &key, &data, MDBX_NEXT_NODUP);
@ -101,8 +94,7 @@ int mdbx_canary_get(const MDBX_txn *txn, MDBX_canary *canary) {
return MDBX_SUCCESS;
}
int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
MDBX_val *data) {
int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data) {
DKBUF_DEBUG;
DEBUG("===> get db %u key [%s]", dbi, DKEY_DEBUG(key));
@ -121,8 +113,7 @@ int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
return LOG_IFERR(cursor_seek(&cx.outer, (MDBX_val *)key, data, MDBX_SET).err);
}
int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key,
MDBX_val *data) {
int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data) {
int rc = check_txn(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return LOG_IFERR(rc);
@ -141,8 +132,7 @@ int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key,
return LOG_IFERR(cursor_ops(&cx.outer, key, data, MDBX_SET_LOWERBOUND));
}
int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key,
MDBX_val *data, size_t *values_count) {
int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, size_t *values_count) {
DKBUF_DEBUG;
DEBUG("===> get db %u key [%s]", dbi, DKEY_DEBUG(key));
@ -169,8 +159,7 @@ int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key,
*values_count = 1;
if (inner_pointed(&cx.outer))
*values_count =
(sizeof(*values_count) >= sizeof(cx.inner.nested_tree.items) ||
cx.inner.nested_tree.items <= PTRDIFF_MAX)
(sizeof(*values_count) >= sizeof(cx.inner.nested_tree.items) || cx.inner.nested_tree.items <= PTRDIFF_MAX)
? (size_t)cx.inner.nested_tree.items
: PTRDIFF_MAX;
}
@ -185,8 +174,7 @@ int mdbx_canary_put(MDBX_txn *txn, const MDBX_canary *canary) {
return LOG_IFERR(rc);
if (likely(canary)) {
if (txn->canary.x == canary->x && txn->canary.y == canary->y &&
txn->canary.z == canary->z)
if (txn->canary.x == canary->x && txn->canary.y == canary->y && txn->canary.z == canary->z)
return MDBX_SUCCESS;
txn->canary.x = canary->x;
txn->canary.y = canary->y;
@ -236,17 +224,14 @@ int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr) {
* not to the beginning of a data. */
return LOG_IFERR(MDBX_EINVAL);
}
return ((txn->flags & MDBX_TXN_RDONLY) || !is_modifable(txn, page))
? MDBX_RESULT_FALSE
: MDBX_RESULT_TRUE;
return ((txn->flags & MDBX_TXN_RDONLY) || !is_modifable(txn, page)) ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE;
}
if ((size_t)offset < env->dxb_mmap.limit) {
/* Указатель адресует что-то в пределах mmap, но за границей
* распределенных страниц. Такое может случится если mdbx_is_dirty()
* вызывается после операции, в ходе которой грязная страница была
* возвращена в нераспределенное пространство. */
return (txn->flags & MDBX_TXN_RDONLY) ? LOG_IFERR(MDBX_EINVAL)
: MDBX_RESULT_TRUE;
return (txn->flags & MDBX_TXN_RDONLY) ? LOG_IFERR(MDBX_EINVAL) : MDBX_RESULT_TRUE;
}
}
@ -256,13 +241,10 @@ int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr) {
*
* Для режима MDBX_WRITE_MAP режима страница однозначно "не грязная",
* а для режимов без MDBX_WRITE_MAP однозначно "не чистая". */
return (txn->flags & (MDBX_WRITEMAP | MDBX_TXN_RDONLY))
? LOG_IFERR(MDBX_EINVAL)
: MDBX_RESULT_TRUE;
return (txn->flags & (MDBX_WRITEMAP | MDBX_TXN_RDONLY)) ? LOG_IFERR(MDBX_EINVAL) : MDBX_RESULT_TRUE;
}
int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
const MDBX_val *data) {
int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, const MDBX_val *data) {
int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return LOG_IFERR(rc);
@ -274,8 +256,7 @@ int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
return LOG_IFERR(MDBX_BAD_DBI);
if (unlikely(txn->flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED)))
return LOG_IFERR((txn->flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS
: MDBX_BAD_TXN);
return LOG_IFERR((txn->flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN);
cursor_couple_t cx;
rc = cursor_init(&cx.outer, txn, dbi);
@ -302,8 +283,7 @@ int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
return LOG_IFERR(rc);
}
int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data,
MDBX_put_flags_t flags) {
int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data, MDBX_put_flags_t flags) {
int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return LOG_IFERR(rc);
@ -314,14 +294,12 @@ int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data,
if (unlikely(dbi <= FREE_DBI))
return LOG_IFERR(MDBX_BAD_DBI);
if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS |
MDBX_ALLDUPS | MDBX_RESERVE | MDBX_APPEND |
if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS | MDBX_ALLDUPS | MDBX_RESERVE | MDBX_APPEND |
MDBX_APPENDDUP | MDBX_CURRENT | MDBX_MULTIPLE)))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(txn->flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED)))
return LOG_IFERR((txn->flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS
: MDBX_BAD_TXN);
return LOG_IFERR((txn->flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN);
cursor_couple_t cx;
rc = cursor_init(&cx.outer, txn, dbi);
@ -333,14 +311,11 @@ int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data,
/* LY: support for update (explicit overwrite) */
if (flags & MDBX_CURRENT) {
rc = cursor_seek(&cx.outer, (MDBX_val *)key, nullptr, MDBX_SET).err;
if (likely(rc == MDBX_SUCCESS) && (txn->dbs[dbi].flags & MDBX_DUPSORT) &&
(flags & MDBX_ALLDUPS) == 0) {
if (likely(rc == MDBX_SUCCESS) && (txn->dbs[dbi].flags & MDBX_DUPSORT) && (flags & MDBX_ALLDUPS) == 0) {
/* LY: allows update (explicit overwrite) only for unique keys */
node_t *node =
page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]);
node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]);
if (node_flags(node) & N_DUP) {
tASSERT(txn, inner_pointed(&cx.outer) &&
cx.outer.subcur->nested_tree.items > 1);
tASSERT(txn, inner_pointed(&cx.outer) && cx.outer.subcur->nested_tree.items > 1);
rc = MDBX_EMULTIVAL;
if ((flags & MDBX_NOOVERWRITE) == 0) {
flags -= MDBX_CURRENT;
@ -383,10 +358,8 @@ int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data,
* - получения dirty-статуса страницы по адресу (знать о MUTABLE/WRITEABLE).
*/
int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
MDBX_val *new_data, MDBX_val *old_data,
MDBX_put_flags_t flags, MDBX_preserve_func preserver,
void *preserver_context) {
int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *new_data, MDBX_val *old_data,
MDBX_put_flags_t flags, MDBX_preserve_func preserver, void *preserver_context) {
int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return LOG_IFERR(rc);
@ -397,16 +370,14 @@ int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
if (unlikely(old_data->iov_base == nullptr && old_data->iov_len))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(new_data == nullptr &&
(flags & (MDBX_CURRENT | MDBX_RESERVE)) != MDBX_CURRENT))
if (unlikely(new_data == nullptr && (flags & (MDBX_CURRENT | MDBX_RESERVE)) != MDBX_CURRENT))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(dbi <= FREE_DBI))
return LOG_IFERR(MDBX_BAD_DBI);
if (unlikely(flags &
~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS |
MDBX_RESERVE | MDBX_APPEND | MDBX_APPENDDUP | MDBX_CURRENT)))
if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS | MDBX_RESERVE | MDBX_APPEND |
MDBX_APPENDDUP | MDBX_CURRENT)))
return LOG_IFERR(MDBX_EINVAL);
cursor_couple_t cx;
@ -452,8 +423,7 @@ int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
/* disallow update/delete for multi-values */
node_t *node = page_node(page, cx.outer.ki[cx.outer.top]);
if (node_flags(node) & N_DUP) {
tASSERT(txn, inner_pointed(&cx.outer) &&
cx.outer.subcur->nested_tree.items > 1);
tASSERT(txn, inner_pointed(&cx.outer) && cx.outer.subcur->nested_tree.items > 1);
if (cx.outer.subcur->nested_tree.items > 1) {
rc = MDBX_EMULTIVAL;
goto bailout;
@ -472,8 +442,7 @@ int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
*old_data = *new_data;
goto bailout;
}
rc = preserver ? preserver(preserver_context, old_data,
present_data.iov_base, present_data.iov_len)
rc = preserver ? preserver(preserver_context, old_data, present_data.iov_base, present_data.iov_len)
: MDBX_SUCCESS;
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
@ -494,8 +463,7 @@ bailout:
return LOG_IFERR(rc);
}
static int default_value_preserver(void *context, MDBX_val *target,
const void *src, size_t bytes) {
static int default_value_preserver(void *context, MDBX_val *target, const void *src, size_t bytes) {
(void)context;
if (unlikely(target->iov_len < bytes)) {
target->iov_base = nullptr;
@ -506,9 +474,7 @@ static int default_value_preserver(void *context, MDBX_val *target,
return MDBX_SUCCESS;
}
int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key,
MDBX_val *new_data, MDBX_val *old_data,
int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *new_data, MDBX_val *old_data,
MDBX_put_flags_t flags) {
return mdbx_replace_ex(txn, dbi, key, new_data, old_data, flags,
default_value_preserver, nullptr);
return mdbx_replace_ex(txn, dbi, key, new_data, old_data, flags, default_value_preserver, nullptr);
}

112
src/atomics-ops.h
View File

@ -8,43 +8,36 @@
#ifndef __cplusplus
#ifdef MDBX_HAVE_C11ATOMICS
#define osal_memory_fence(order, write) \
atomic_thread_fence((write) ? mo_c11_store(order) : mo_c11_load(order))
#define osal_memory_fence(order, write) atomic_thread_fence((write) ? mo_c11_store(order) : mo_c11_load(order))
#else /* MDBX_HAVE_C11ATOMICS */
#define osal_memory_fence(order, write) \
do { \
osal_compiler_barrier(); \
if (write && order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed \
: mo_AcquireRelease)) \
osal_memory_barrier(); \
#define osal_memory_fence(order, write) \
do { \
osal_compiler_barrier(); \
if (write && order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed : mo_AcquireRelease)) \
osal_memory_barrier(); \
} while (0)
#endif /* MDBX_HAVE_C11ATOMICS */
#if defined(MDBX_HAVE_C11ATOMICS) && defined(__LCC__)
#define atomic_store32(p, value, order) \
({ \
const uint32_t value_to_store = (value); \
atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value_to_store, \
mo_c11_store(order)); \
value_to_store; \
#define atomic_store32(p, value, order) \
({ \
const uint32_t value_to_store = (value); \
atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value_to_store, mo_c11_store(order)); \
value_to_store; \
})
#define atomic_load32(p, order) \
atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order))
#define atomic_store64(p, value, order) \
({ \
const uint64_t value_to_store = (value); \
atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value_to_store, \
mo_c11_store(order)); \
value_to_store; \
#define atomic_load32(p, order) atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order))
#define atomic_store64(p, value, order) \
({ \
const uint64_t value_to_store = (value); \
atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value_to_store, mo_c11_store(order)); \
value_to_store; \
})
#define atomic_load64(p, order) \
atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order))
#define atomic_load64(p, order) atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order))
#endif /* LCC && MDBX_HAVE_C11ATOMICS */
#ifndef atomic_store32
MDBX_MAYBE_UNUSED static __always_inline uint32_t
atomic_store32(mdbx_atomic_uint32_t *p, const uint32_t value,
enum mdbx_memory_order order) {
MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_store32(mdbx_atomic_uint32_t *p, const uint32_t value,
enum mdbx_memory_order order) {
STATIC_ASSERT(sizeof(mdbx_atomic_uint32_t) == 4);
#ifdef MDBX_HAVE_C11ATOMICS
assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
@ -60,8 +53,8 @@ atomic_store32(mdbx_atomic_uint32_t *p, const uint32_t value,
#endif /* atomic_store32 */
#ifndef atomic_load32
MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32(
const volatile mdbx_atomic_uint32_t *p, enum mdbx_memory_order order) {
MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32(const volatile mdbx_atomic_uint32_t *p,
enum mdbx_memory_order order) {
STATIC_ASSERT(sizeof(mdbx_atomic_uint32_t) == 4);
#ifdef MDBX_HAVE_C11ATOMICS
assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p)));
@ -90,9 +83,8 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32(
#endif /* xMDBX_TXNID_STEP */
#ifndef atomic_store64
MDBX_MAYBE_UNUSED static __always_inline uint64_t
atomic_store64(mdbx_atomic_uint64_t *p, const uint64_t value,
enum mdbx_memory_order order) {
MDBX_MAYBE_UNUSED static __always_inline uint64_t atomic_store64(mdbx_atomic_uint64_t *p, const uint64_t value,
enum mdbx_memory_order order) {
STATIC_ASSERT(sizeof(mdbx_atomic_uint64_t) == 8);
#if MDBX_64BIT_ATOMIC
#if __GNUC_PREREQ(11, 0)
@ -124,8 +116,7 @@ MDBX_MAYBE_UNUSED static
__always_inline
#endif /* MDBX_64BIT_ATOMIC */
uint64_t
atomic_load64(const volatile mdbx_atomic_uint64_t *p,
enum mdbx_memory_order order) {
atomic_load64(const volatile mdbx_atomic_uint64_t *p, enum mdbx_memory_order order) {
STATIC_ASSERT(sizeof(mdbx_atomic_uint64_t) == 8);
#if MDBX_64BIT_ATOMIC
#ifdef MDBX_HAVE_C11ATOMICS
@ -142,15 +133,13 @@ MDBX_MAYBE_UNUSED static
osal_compiler_barrier();
uint64_t value = (uint64_t)atomic_load32(&p->high, order) << 32;
jitter4testing(true);
value |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed
: mo_AcquireRelease);
value |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed : mo_AcquireRelease);
jitter4testing(true);
for (;;) {
osal_compiler_barrier();
uint64_t again = (uint64_t)atomic_load32(&p->high, order) << 32;
jitter4testing(true);
again |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed
: mo_AcquireRelease);
again |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed : mo_AcquireRelease);
jitter4testing(true);
if (likely(value == again))
return value;
@ -171,19 +160,16 @@ MDBX_MAYBE_UNUSED static __always_inline void atomic_yield(void) {
#else
__asm__ __volatile__("hint @pause");
#endif
#elif defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 6) || \
defined(__ARM_ARCH_6K__)
#elif defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 6) || defined(__ARM_ARCH_6K__)
#ifdef __CC_ARM
__yield();
#else
__asm__ __volatile__("yield");
#endif
#elif (defined(__mips64) || defined(__mips64__)) && defined(__mips_isa_rev) && \
__mips_isa_rev >= 2
#elif (defined(__mips64) || defined(__mips64__)) && defined(__mips_isa_rev) && __mips_isa_rev >= 2
__asm__ __volatile__("pause");
#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
defined(__MWERKS__) || defined(__sgi)
#elif defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \
defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi)
__asm__ __volatile__(".word 0x00000140");
#elif defined(__linux__) || defined(__gnu_linux__) || defined(_UNIX03_SOURCE)
sched_yield();
@ -193,8 +179,7 @@ MDBX_MAYBE_UNUSED static __always_inline void atomic_yield(void) {
}
#if MDBX_64BIT_CAS
MDBX_MAYBE_UNUSED static __always_inline bool
atomic_cas64(mdbx_atomic_uint64_t *p, uint64_t c, uint64_t v) {
MDBX_MAYBE_UNUSED static __always_inline bool atomic_cas64(mdbx_atomic_uint64_t *p, uint64_t c, uint64_t v) {
#ifdef MDBX_HAVE_C11ATOMICS
STATIC_ASSERT(sizeof(long long) >= sizeof(uint64_t));
assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p)));
@ -202,8 +187,7 @@ atomic_cas64(mdbx_atomic_uint64_t *p, uint64_t c, uint64_t v) {
#elif defined(__GNUC__) || defined(__clang__)
return __sync_bool_compare_and_swap(&p->weak, c, v);
#elif defined(_MSC_VER)
return c == (uint64_t)_InterlockedCompareExchange64(
(volatile __int64 *)&p->weak, v, c);
return c == (uint64_t)_InterlockedCompareExchange64((volatile __int64 *)&p->weak, v, c);
#elif defined(__APPLE__)
return OSAtomicCompareAndSwap64Barrier(c, v, &p->weak);
#else
@ -212,8 +196,7 @@ atomic_cas64(mdbx_atomic_uint64_t *p, uint64_t c, uint64_t v) {
}
#endif /* MDBX_64BIT_CAS */
MDBX_MAYBE_UNUSED static __always_inline bool
atomic_cas32(mdbx_atomic_uint32_t *p, uint32_t c, uint32_t v) {
MDBX_MAYBE_UNUSED static __always_inline bool atomic_cas32(mdbx_atomic_uint32_t *p, uint32_t c, uint32_t v) {
#ifdef MDBX_HAVE_C11ATOMICS
STATIC_ASSERT(sizeof(int) >= sizeof(uint32_t));
assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
@ -222,8 +205,7 @@ atomic_cas32(mdbx_atomic_uint32_t *p, uint32_t c, uint32_t v) {
return __sync_bool_compare_and_swap(&p->weak, c, v);
#elif defined(_MSC_VER)
STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t));
return c ==
(uint32_t)_InterlockedCompareExchange((volatile long *)&p->weak, v, c);
return c == (uint32_t)_InterlockedCompareExchange((volatile long *)&p->weak, v, c);
#elif defined(__APPLE__)
return OSAtomicCompareAndSwap32Barrier(c, v, &p->weak);
#else
@ -231,8 +213,7 @@ atomic_cas32(mdbx_atomic_uint32_t *p, uint32_t c, uint32_t v) {
#endif
}
MDBX_MAYBE_UNUSED static __always_inline uint32_t
atomic_add32(mdbx_atomic_uint32_t *p, uint32_t v) {
MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_add32(mdbx_atomic_uint32_t *p, uint32_t v) {
#ifdef MDBX_HAVE_C11ATOMICS
STATIC_ASSERT(sizeof(int) >= sizeof(uint32_t));
assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p)));
@ -251,8 +232,7 @@ atomic_add32(mdbx_atomic_uint32_t *p, uint32_t v) {
#define atomic_sub32(p, v) atomic_add32(p, 0 - (v))
MDBX_MAYBE_UNUSED static __always_inline uint64_t
safe64_txnid_next(uint64_t txnid) {
MDBX_MAYBE_UNUSED static __always_inline uint64_t safe64_txnid_next(uint64_t txnid) {
txnid += xMDBX_TXNID_STEP;
#if !MDBX_64BIT_CAS
/* avoid overflow of low-part in safe64_reset() */
@ -262,8 +242,7 @@ safe64_txnid_next(uint64_t txnid) {
}
/* Atomically make target value >= SAFE64_INVALID_THRESHOLD */
MDBX_MAYBE_UNUSED static __always_inline void
safe64_reset(mdbx_atomic_uint64_t *p, bool single_writer) {
MDBX_MAYBE_UNUSED static __always_inline void safe64_reset(mdbx_atomic_uint64_t *p, bool single_writer) {
if (single_writer) {
#if MDBX_64BIT_ATOMIC && MDBX_WORDBITS >= 64
atomic_store64(p, UINT64_MAX, mo_AcquireRelease);
@ -290,8 +269,7 @@ safe64_reset(mdbx_atomic_uint64_t *p, bool single_writer) {
jitter4testing(true);
}
MDBX_MAYBE_UNUSED static __always_inline bool
safe64_reset_compare(mdbx_atomic_uint64_t *p, uint64_t compare) {
MDBX_MAYBE_UNUSED static __always_inline bool safe64_reset_compare(mdbx_atomic_uint64_t *p, uint64_t compare) {
/* LY: This function is used to reset `txnid` from hsr-handler in case
* the asynchronously cancellation of read transaction. Therefore,
* there may be a collision between the cleanup performed here and
@ -307,8 +285,7 @@ safe64_reset_compare(mdbx_atomic_uint64_t *p, uint64_t compare) {
bool rc = false;
if (likely(atomic_load32(&p->low, mo_AcquireRelease) == (uint32_t)compare &&
atomic_cas32(&p->high, (uint32_t)(compare >> 32), UINT32_MAX))) {
if (unlikely(atomic_load32(&p->low, mo_AcquireRelease) !=
(uint32_t)compare))
if (unlikely(atomic_load32(&p->low, mo_AcquireRelease) != (uint32_t)compare))
atomic_cas32(&p->high, UINT32_MAX, (uint32_t)(compare >> 32));
else
rc = true;
@ -318,8 +295,7 @@ safe64_reset_compare(mdbx_atomic_uint64_t *p, uint64_t compare) {
return rc;
}
MDBX_MAYBE_UNUSED static __always_inline void
safe64_write(mdbx_atomic_uint64_t *p, const uint64_t v) {
MDBX_MAYBE_UNUSED static __always_inline void safe64_write(mdbx_atomic_uint64_t *p, const uint64_t v) {
assert(p->weak >= SAFE64_INVALID_THRESHOLD);
#if MDBX_64BIT_ATOMIC && MDBX_64BIT_CAS
atomic_store64(p, v, mo_AcquireRelease);
@ -336,8 +312,7 @@ safe64_write(mdbx_atomic_uint64_t *p, const uint64_t v) {
jitter4testing(true);
}
MDBX_MAYBE_UNUSED static __always_inline uint64_t
safe64_read(const mdbx_atomic_uint64_t *p) {
MDBX_MAYBE_UNUSED static __always_inline uint64_t safe64_read(const mdbx_atomic_uint64_t *p) {
jitter4testing(true);
uint64_t v;
do
@ -366,8 +341,7 @@ MDBX_MAYBE_UNUSED static __always_inline bool
#endif /* unused for now */
/* non-atomic write with safety for reading a half-updated value */
MDBX_MAYBE_UNUSED static __always_inline void
safe64_update(mdbx_atomic_uint64_t *p, const uint64_t v) {
MDBX_MAYBE_UNUSED static __always_inline void safe64_update(mdbx_atomic_uint64_t *p, const uint64_t v) {
#if MDBX_64BIT_ATOMIC
atomic_store64(p, v, mo_Relaxed);
#else

26
src/atomics-types.h
View File

@ -16,21 +16,19 @@
#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include(<cstdatomic>)
#include <cstdatomic>
#define MDBX_HAVE_C11ATOMICS
#elif !defined(__cplusplus) && \
(__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \
!defined(__STDC_NO_ATOMICS__) && \
(__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \
!(defined(__GNUC__) || defined(__clang__)))
#elif !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \
!defined(__STDC_NO_ATOMICS__) && \
(__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || !(defined(__GNUC__) || defined(__clang__)))
#include <stdatomic.h>
#define MDBX_HAVE_C11ATOMICS
#elif defined(__GNUC__) || defined(__clang__)
#elif defined(_MSC_VER)
#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */
#pragma warning(disable : 4133) /* 'function': incompatible types - from \
#pragma warning(disable : 4133) /* 'function': incompatible types - from \
'size_t' to 'LONGLONG' */
#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
'std::size_t', possible loss of data */
#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
'long', possible loss of data */
#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange)
#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64)
@ -85,13 +83,13 @@ typedef union {
#define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a)
#endif /* Crutches for C11 atomic compiler's bugs */
#define mo_c11_store(fence) \
(((fence) == mo_Relaxed) ? memory_order_relaxed \
: ((fence) == mo_AcquireRelease) ? memory_order_release \
#define mo_c11_store(fence) \
(((fence) == mo_Relaxed) ? memory_order_relaxed \
: ((fence) == mo_AcquireRelease) ? memory_order_release \
: memory_order_seq_cst)
#define mo_c11_load(fence) \
(((fence) == mo_Relaxed) ? memory_order_relaxed \
: ((fence) == mo_AcquireRelease) ? memory_order_acquire \
#define mo_c11_load(fence) \
(((fence) == mo_Relaxed) ? memory_order_relaxed \
: ((fence) == mo_AcquireRelease) ? memory_order_acquire \
: memory_order_seq_cst)
#endif /* MDBX_HAVE_C11ATOMICS */

46
src/audit.c
View File

@ -8,29 +8,23 @@ struct audit_ctx {
uint8_t *const done_bitmap;
};
static int audit_dbi(void *ctx, const MDBX_txn *txn, const MDBX_val *name,
MDBX_db_flags_t flags, const struct MDBX_stat *stat,
MDBX_dbi dbi) {
static int audit_dbi(void *ctx, const MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags,
const struct MDBX_stat *stat, MDBX_dbi dbi) {
struct audit_ctx *audit_ctx = ctx;
(void)name;
(void)txn;
(void)flags;
audit_ctx->used += (size_t)stat->ms_branch_pages +
(size_t)stat->ms_leaf_pages +
(size_t)stat->ms_overflow_pages;
audit_ctx->used += (size_t)stat->ms_branch_pages + (size_t)stat->ms_leaf_pages + (size_t)stat->ms_overflow_pages;
if (dbi)
audit_ctx->done_bitmap[dbi / CHAR_BIT] |= 1 << dbi % CHAR_BIT;
return MDBX_SUCCESS;
}
static size_t audit_db_used(const tree_t *db) {
return db ? (size_t)db->branch_pages + (size_t)db->leaf_pages +
(size_t)db->large_pages
: 0;
return db ? (size_t)db->branch_pages + (size_t)db->leaf_pages + (size_t)db->large_pages : 0;
}
__cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored,
bool dont_filter_gc) {
__cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored, bool dont_filter_gc) {
const MDBX_env *const env = txn->env;
size_t pending = 0;
if ((txn->flags & MDBX_TXN_RDONLY) == 0)
@ -48,8 +42,7 @@ __cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored,
while (rc == MDBX_SUCCESS) {
if (!dont_filter_gc) {
if (unlikely(key.iov_len != sizeof(txnid_t))) {
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid GC-key size", (unsigned)key.iov_len);
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-key size", (unsigned)key.iov_len);
return MDBX_CORRUPTED;
}
txnid_t id = unaligned_peek_u64(4, key.iov_base);
@ -68,18 +61,16 @@ __cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored,
const size_t done_bitmap_size = (txn->n_dbi + CHAR_BIT - 1) / CHAR_BIT;
if (txn->parent) {
tASSERT(txn, txn->n_dbi == txn->parent->n_dbi &&
txn->n_dbi == txn->env->txn->n_dbi);
tASSERT(txn, txn->n_dbi == txn->parent->n_dbi && txn->n_dbi == txn->env->txn->n_dbi);
#if MDBX_ENABLE_DBI_SPARSE
tASSERT(txn, txn->dbi_sparse == txn->parent->dbi_sparse &&
txn->dbi_sparse == txn->env->txn->dbi_sparse);
tASSERT(txn, txn->dbi_sparse == txn->parent->dbi_sparse && txn->dbi_sparse == txn->env->txn->dbi_sparse);
#endif /* MDBX_ENABLE_DBI_SPARSE */
}
struct audit_ctx ctx = {0, alloca(done_bitmap_size)};
memset(ctx.done_bitmap, 0, done_bitmap_size);
ctx.used = NUM_METAS + audit_db_used(dbi_dig(txn, FREE_DBI, nullptr)) +
audit_db_used(dbi_dig(txn, MAIN_DBI, nullptr));
ctx.used =
NUM_METAS + audit_db_used(dbi_dig(txn, FREE_DBI, nullptr)) + audit_db_used(dbi_dig(txn, MAIN_DBI, nullptr));
rc = mdbx_enumerate_tables(txn, audit_dbi, &ctx);
tASSERT(txn, rc == MDBX_SUCCESS);
@ -91,11 +82,9 @@ __cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored,
if (db)
ctx.used += audit_db_used(db);
else if (dbi_state(txn, dbi))
WARNING("audit %s@%" PRIaTXN
": unable account dbi %zd / \"%*s\", state 0x%02x",
txn->parent ? "nested-" : "", txn->txnid, dbi,
(int)env->kvs[dbi].name.iov_len,
(const char *)env->kvs[dbi].name.iov_base, dbi_state(txn, dbi));
WARNING("audit %s@%" PRIaTXN ": unable account dbi %zd / \"%*s\", state 0x%02x", txn->parent ? "nested-" : "",
txn->txnid, dbi, (int)env->kvs[dbi].name.iov_len, (const char *)env->kvs[dbi].name.iov_base,
dbi_state(txn, dbi));
}
if (pending + gc + ctx.used == txn->geo.first_unallocated)
@ -104,15 +93,12 @@ __cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored,
if ((txn->flags & MDBX_TXN_RDONLY) == 0)
ERROR("audit @%" PRIaTXN ": %zu(pending) = %zu(loose) + "
"%zu(reclaimed) + %zu(retired-pending) - %zu(retired-stored)",
txn->txnid, pending, txn->tw.loose_count,
MDBX_PNL_GETSIZE(txn->tw.relist),
txn->tw.retired_pages ? MDBX_PNL_GETSIZE(txn->tw.retired_pages) : 0,
retired_stored);
txn->txnid, pending, txn->tw.loose_count, MDBX_PNL_GETSIZE(txn->tw.relist),
txn->tw.retired_pages ? MDBX_PNL_GETSIZE(txn->tw.retired_pages) : 0, retired_stored);
ERROR("audit @%" PRIaTXN ": %zu(pending) + %zu"
"(gc) + %zu(count) = %zu(total) <> %zu"
"(allocated)",
txn->txnid, pending, gc, ctx.used, pending + gc + ctx.used,
(size_t)txn->geo.first_unallocated);
txn->txnid, pending, gc, ctx.used, pending + gc + ctx.used, (size_t)txn->geo.first_unallocated);
return MDBX_PROBLEM;
}

1007
src/chk.c
View File

File diff suppressed because it is too large Load Diff

122
src/cogs.c
View File

@ -74,27 +74,22 @@ __cold bool pv2pages_verify(void) {
/*----------------------------------------------------------------------------*/
MDBX_NOTHROW_PURE_FUNCTION size_t bytes_align2os_bytes(const MDBX_env *env,
size_t bytes) {
return ceil_powerof2(
bytes, (env->ps > globals.sys_pagesize) ? env->ps : globals.sys_pagesize);
MDBX_NOTHROW_PURE_FUNCTION size_t bytes_align2os_bytes(const MDBX_env *env, size_t bytes) {
return ceil_powerof2(bytes, (env->ps > globals.sys_pagesize) ? env->ps : globals.sys_pagesize);
}
MDBX_NOTHROW_PURE_FUNCTION size_t pgno_align2os_bytes(const MDBX_env *env,
size_t pgno) {
MDBX_NOTHROW_PURE_FUNCTION size_t pgno_align2os_bytes(const MDBX_env *env, size_t pgno) {
return ceil_powerof2(pgno2bytes(env, pgno), globals.sys_pagesize);
}
MDBX_NOTHROW_PURE_FUNCTION pgno_t pgno_align2os_pgno(const MDBX_env *env,
size_t pgno) {
MDBX_NOTHROW_PURE_FUNCTION pgno_t pgno_align2os_pgno(const MDBX_env *env, size_t pgno) {
return bytes2pgno(env, pgno_align2os_bytes(env, pgno));
}
/*----------------------------------------------------------------------------*/
MDBX_NOTHROW_PURE_FUNCTION static __always_inline int
cmp_int_inline(const size_t expected_alignment, const MDBX_val *a,
const MDBX_val *b) {
MDBX_NOTHROW_PURE_FUNCTION static __always_inline int cmp_int_inline(const size_t expected_alignment, const MDBX_val *a,
const MDBX_val *b) {
if (likely(a->iov_len == b->iov_len)) {
if (sizeof(size_t) > 7 && likely(a->iov_len == 8))
return CMP2INT(unaligned_peek_u64(expected_alignment, a->iov_base),
@ -106,35 +101,31 @@ cmp_int_inline(const size_t expected_alignment, const MDBX_val *a,
return CMP2INT(unaligned_peek_u64(expected_alignment, a->iov_base),
unaligned_peek_u64(expected_alignment, b->iov_base));
}
ERROR("mismatch and/or invalid size %p.%zu/%p.%zu for INTEGERKEY/INTEGERDUP",
a->iov_base, a->iov_len, b->iov_base, b->iov_len);
ERROR("mismatch and/or invalid size %p.%zu/%p.%zu for INTEGERKEY/INTEGERDUP", a->iov_base, a->iov_len, b->iov_base,
b->iov_len);
return 0;
}
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_unaligned(const MDBX_val *a,
const MDBX_val *b) {
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_unaligned(const MDBX_val *a, const MDBX_val *b) {
return cmp_int_inline(1, a, b);
}
#ifndef cmp_int_align2
/* Compare two items pointing at 2-byte aligned unsigned int's. */
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_align2(const MDBX_val *a,
const MDBX_val *b) {
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_align2(const MDBX_val *a, const MDBX_val *b) {
return cmp_int_inline(2, a, b);
}
#endif /* cmp_int_align2 */
#ifndef cmp_int_align4
/* Compare two items pointing at 4-byte aligned unsigned int's. */
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_align4(const MDBX_val *a,
const MDBX_val *b) {
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_align4(const MDBX_val *a, const MDBX_val *b) {
return cmp_int_inline(4, a, b);
}
#endif /* cmp_int_align4 */
/* Compare two items lexically */
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lexical(const MDBX_val *a,
const MDBX_val *b) {
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lexical(const MDBX_val *a, const MDBX_val *b) {
if (a->iov_len == b->iov_len)
return a->iov_len ? memcmp(a->iov_base, b->iov_base, a->iov_len) : 0;
@ -144,8 +135,7 @@ MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lexical(const MDBX_val *a,
return likely(diff_data) ? diff_data : diff_len;
}
MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned
tail3le(const uint8_t *p, size_t l) {
MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned tail3le(const uint8_t *p, size_t l) {
STATIC_ASSERT(sizeof(unsigned) > 2);
// 1: 0 0 0
// 2: 0 1 1
@ -154,8 +144,7 @@ tail3le(const uint8_t *p, size_t l) {
}
/* Compare two items in reverse byte order */
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_reverse(const MDBX_val *a,
const MDBX_val *b) {
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_reverse(const MDBX_val *a, const MDBX_val *b) {
size_t left = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len;
if (likely(left)) {
const uint8_t *pa = ptr_disp(a->iov_base, a->iov_len);
@ -209,25 +198,19 @@ MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_reverse(const MDBX_val *a,
}
/* Fast non-lexically comparator */
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lenfast(const MDBX_val *a,
const MDBX_val *b) {
MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lenfast(const MDBX_val *a, const MDBX_val *b) {
int diff = CMP2INT(a->iov_len, b->iov_len);
return (likely(diff) || a->iov_len == 0)
? diff
: memcmp(a->iov_base, b->iov_base, a->iov_len);
return (likely(diff) || a->iov_len == 0) ? diff : memcmp(a->iov_base, b->iov_base, a->iov_len);
}
MDBX_NOTHROW_PURE_FUNCTION __hot bool
eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l) {
MDBX_NOTHROW_PURE_FUNCTION __hot bool eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l) {
if (likely(l > 3)) {
if (MDBX_UNALIGNED_OK >= 4 && likely(l < 9))
return ((unaligned_peek_u32(1, a) - unaligned_peek_u32(1, b)) |
(unaligned_peek_u32(1, a + l - 4) -
unaligned_peek_u32(1, b + l - 4))) == 0;
(unaligned_peek_u32(1, a + l - 4) - unaligned_peek_u32(1, b + l - 4))) == 0;
if (MDBX_UNALIGNED_OK >= 8 && sizeof(size_t) > 7 && likely(l < 17))
return ((unaligned_peek_u64(1, a) - unaligned_peek_u64(1, b)) |
(unaligned_peek_u64(1, a + l - 8) -
unaligned_peek_u64(1, b + l - 8))) == 0;
(unaligned_peek_u64(1, a + l - 8) - unaligned_peek_u64(1, b + l - 8))) == 0;
return memcmp(a, b, l) == 0;
}
if (likely(l))
@ -235,31 +218,21 @@ eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l) {
return true;
}
int cmp_equal_or_greater(const MDBX_val *a, const MDBX_val *b) {
return eq_fast(a, b) ? 0 : 1;
}
int cmp_equal_or_greater(const MDBX_val *a, const MDBX_val *b) { return eq_fast(a, b) ? 0 : 1; }
int cmp_equal_or_wrong(const MDBX_val *a, const MDBX_val *b) {
return eq_fast(a, b) ? 0 : -1;
}
int cmp_equal_or_wrong(const MDBX_val *a, const MDBX_val *b) { return eq_fast(a, b) ? 0 : -1; }
/*----------------------------------------------------------------------------*/
__cold void update_mlcnt(const MDBX_env *env,
const pgno_t new_aligned_mlocked_pgno,
const bool lock_not_release) {
__cold void update_mlcnt(const MDBX_env *env, const pgno_t new_aligned_mlocked_pgno, const bool lock_not_release) {
for (;;) {
const pgno_t mlock_pgno_before =
atomic_load32(&env->mlocked_pgno, mo_AcquireRelease);
eASSERT(env,
pgno_align2os_pgno(env, mlock_pgno_before) == mlock_pgno_before);
eASSERT(env, pgno_align2os_pgno(env, new_aligned_mlocked_pgno) ==
new_aligned_mlocked_pgno);
const pgno_t mlock_pgno_before = atomic_load32(&env->mlocked_pgno, mo_AcquireRelease);
eASSERT(env, pgno_align2os_pgno(env, mlock_pgno_before) == mlock_pgno_before);
eASSERT(env, pgno_align2os_pgno(env, new_aligned_mlocked_pgno) == new_aligned_mlocked_pgno);
if (lock_not_release ? (mlock_pgno_before >= new_aligned_mlocked_pgno)
: (mlock_pgno_before <= new_aligned_mlocked_pgno))
break;
if (likely(atomic_cas32(&((MDBX_env *)env)->mlocked_pgno, mlock_pgno_before,
new_aligned_mlocked_pgno)))
if (likely(atomic_cas32(&((MDBX_env *)env)->mlocked_pgno, mlock_pgno_before, new_aligned_mlocked_pgno)))
for (;;) {
mdbx_atomic_uint32_t *const mlcnt = env->lck->mlcnt;
const int32_t snap_locked = atomic_load32(mlcnt + 0, mo_Relaxed);
@ -269,52 +242,39 @@ __cold void update_mlcnt(const MDBX_env *env,
if (unlikely(!atomic_cas32(mlcnt + 0, snap_locked, snap_locked + 1)))
continue;
}
if (new_aligned_mlocked_pgno == 0 &&
(snap_locked - snap_unlocked) > 0) {
if (new_aligned_mlocked_pgno == 0 && (snap_locked - snap_unlocked) > 0) {
eASSERT(env, !lock_not_release);
if (unlikely(
!atomic_cas32(mlcnt + 1, snap_unlocked, snap_unlocked + 1)))
if (unlikely(!atomic_cas32(mlcnt + 1, snap_unlocked, snap_unlocked + 1)))
continue;
}
NOTICE("%s-pages %u..%u, mlocked-process(es) %u -> %u",
lock_not_release ? "lock" : "unlock",
NOTICE("%s-pages %u..%u, mlocked-process(es) %u -> %u", lock_not_release ? "lock" : "unlock",
lock_not_release ? mlock_pgno_before : new_aligned_mlocked_pgno,
lock_not_release ? new_aligned_mlocked_pgno : mlock_pgno_before,
snap_locked - snap_unlocked,
atomic_load32(mlcnt + 0, mo_Relaxed) -
atomic_load32(mlcnt + 1, mo_Relaxed));
lock_not_release ? new_aligned_mlocked_pgno : mlock_pgno_before, snap_locked - snap_unlocked,
atomic_load32(mlcnt + 0, mo_Relaxed) - atomic_load32(mlcnt + 1, mo_Relaxed));
return;
}
}
}
__cold void munlock_after(const MDBX_env *env, const pgno_t aligned_pgno,
const size_t end_bytes) {
__cold void munlock_after(const MDBX_env *env, const pgno_t aligned_pgno, const size_t end_bytes) {
if (atomic_load32(&env->mlocked_pgno, mo_AcquireRelease) > aligned_pgno) {
int err = MDBX_ENOSYS;
const size_t munlock_begin = pgno2bytes(env, aligned_pgno);
const size_t munlock_size = end_bytes - munlock_begin;
eASSERT(env, end_bytes % globals.sys_pagesize == 0 &&
munlock_begin % globals.sys_pagesize == 0 &&
eASSERT(env, end_bytes % globals.sys_pagesize == 0 && munlock_begin % globals.sys_pagesize == 0 &&
munlock_size % globals.sys_pagesize == 0);
#if defined(_WIN32) || defined(_WIN64)
err =
VirtualUnlock(ptr_disp(env->dxb_mmap.base, munlock_begin), munlock_size)
? MDBX_SUCCESS
: (int)GetLastError();
err = VirtualUnlock(ptr_disp(env->dxb_mmap.base, munlock_begin), munlock_size) ? MDBX_SUCCESS : (int)GetLastError();
if (err == ERROR_NOT_LOCKED)
err = MDBX_SUCCESS;
#elif defined(_POSIX_MEMLOCK_RANGE)
err = munlock(ptr_disp(env->dxb_mmap.base, munlock_begin), munlock_size)
? errno
: MDBX_SUCCESS;
err = munlock(ptr_disp(env->dxb_mmap.base, munlock_begin), munlock_size) ? errno : MDBX_SUCCESS;
#endif
if (likely(err == MDBX_SUCCESS))
update_mlcnt(env, aligned_pgno, false);
else {
#if defined(_WIN32) || defined(_WIN64)
WARNING("VirtualUnlock(%zu, %zu) error %d", munlock_begin, munlock_size,
err);
WARNING("VirtualUnlock(%zu, %zu) error %d", munlock_begin, munlock_size, err);
#else
WARNING("munlock(%zu, %zu) error %d", munlock_begin, munlock_size, err);
#endif
@ -332,13 +292,11 @@ uint32_t combine_durability_flags(const uint32_t a, const uint32_t b) {
uint32_t r = a | b;
/* avoid false MDBX_UTTERLY_NOSYNC */
if (F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) &&
!F_ISSET(b, MDBX_UTTERLY_NOSYNC))
if (F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && !F_ISSET(b, MDBX_UTTERLY_NOSYNC))
r = (r - MDBX_UTTERLY_NOSYNC) | MDBX_SAFE_NOSYNC;
/* convert DEPRECATED_MAPASYNC to MDBX_SAFE_NOSYNC */
if ((r & (MDBX_WRITEMAP | DEPRECATED_MAPASYNC)) ==
(MDBX_WRITEMAP | DEPRECATED_MAPASYNC) &&
if ((r & (MDBX_WRITEMAP | DEPRECATED_MAPASYNC)) == (MDBX_WRITEMAP | DEPRECATED_MAPASYNC) &&
!F_ISSET(r, MDBX_UTTERLY_NOSYNC))
r = (r - DEPRECATED_MAPASYNC) | MDBX_SAFE_NOSYNC;
@ -346,8 +304,6 @@ uint32_t combine_durability_flags(const uint32_t a, const uint32_t b) {
if (r & (MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC))
r |= MDBX_NOMETASYNC;
assert(!(F_ISSET(r, MDBX_UTTERLY_NOSYNC) &&
!F_ISSET(a, MDBX_UTTERLY_NOSYNC) &&
!F_ISSET(b, MDBX_UTTERLY_NOSYNC)));
assert(!(F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && !F_ISSET(b, MDBX_UTTERLY_NOSYNC)));
return r;
}

208
src/cogs.h
View File

@ -51,19 +51,15 @@ MDBX_MAYBE_UNUSED MDBX_INTERNAL bool pv2pages_verify(void);
#define PAGESPACE(pagesize) ((pagesize) - PAGEHDRSZ)
#define BRANCH_NODE_MAX(pagesize) \
(EVEN_FLOOR((PAGESPACE(pagesize) - sizeof(indx_t) - NODESIZE) / (3 - 1) - \
sizeof(indx_t)))
#define BRANCH_NODE_MAX(pagesize) \
(EVEN_FLOOR((PAGESPACE(pagesize) - sizeof(indx_t) - NODESIZE) / (3 - 1) - sizeof(indx_t)))
#define LEAF_NODE_MAX(pagesize) \
(EVEN_FLOOR(PAGESPACE(pagesize) / 2) - sizeof(indx_t))
#define LEAF_NODE_MAX(pagesize) (EVEN_FLOOR(PAGESPACE(pagesize) / 2) - sizeof(indx_t))
#define MAX_GC1OVPAGE(pagesize) (PAGESPACE(pagesize) / sizeof(pgno_t) - 1)
MDBX_NOTHROW_CONST_FUNCTION static inline size_t
keysize_max(size_t pagesize, MDBX_db_flags_t flags) {
assert(pagesize >= MDBX_MIN_PAGESIZE && pagesize <= MDBX_MAX_PAGESIZE &&
is_powerof2(pagesize));
MDBX_NOTHROW_CONST_FUNCTION static inline size_t keysize_max(size_t pagesize, MDBX_db_flags_t flags) {
assert(pagesize >= MDBX_MIN_PAGESIZE && pagesize <= MDBX_MAX_PAGESIZE && is_powerof2(pagesize));
STATIC_ASSERT(BRANCH_NODE_MAX(MDBX_MIN_PAGESIZE) - NODESIZE >= 8);
if (flags & MDBX_INTEGERKEY)
return 8 /* sizeof(uint64_t) */;
@ -72,18 +68,14 @@ keysize_max(size_t pagesize, MDBX_db_flags_t flags) {
STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) - NODESIZE -
/* sizeof(uint64) as a key */ 8 >
sizeof(tree_t));
if (flags &
(MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) {
const intptr_t max_dupsort_leaf_key =
LEAF_NODE_MAX(pagesize) - NODESIZE - sizeof(tree_t);
return (max_branch_key < max_dupsort_leaf_key) ? max_branch_key
: max_dupsort_leaf_key;
if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) {
const intptr_t max_dupsort_leaf_key = LEAF_NODE_MAX(pagesize) - NODESIZE - sizeof(tree_t);
return (max_branch_key < max_dupsort_leaf_key) ? max_branch_key : max_dupsort_leaf_key;
}
return max_branch_key;
}
MDBX_NOTHROW_CONST_FUNCTION static inline size_t
env_keysize_max(const MDBX_env *env, MDBX_db_flags_t flags) {
MDBX_NOTHROW_CONST_FUNCTION static inline size_t env_keysize_max(const MDBX_env *env, MDBX_db_flags_t flags) {
size_t size_max;
if (flags & MDBX_INTEGERKEY)
size_max = 8 /* sizeof(uint64_t) */;
@ -92,12 +84,9 @@ env_keysize_max(const MDBX_env *env, MDBX_db_flags_t flags) {
STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) - NODESIZE -
/* sizeof(uint64) as a key */ 8 >
sizeof(tree_t));
if (flags &
(MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) {
const intptr_t max_dupsort_leaf_key =
env->leaf_nodemax - NODESIZE - sizeof(tree_t);
size_max = (max_branch_key < max_dupsort_leaf_key) ? max_branch_key
: max_dupsort_leaf_key;
if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) {
const intptr_t max_dupsort_leaf_key = env->leaf_nodemax - NODESIZE - sizeof(tree_t);
size_max = (max_branch_key < max_dupsort_leaf_key) ? max_branch_key : max_dupsort_leaf_key;
} else
size_max = max_branch_key;
}
@ -105,13 +94,11 @@ env_keysize_max(const MDBX_env *env, MDBX_db_flags_t flags) {
return size_max;
}
MDBX_NOTHROW_CONST_FUNCTION static inline size_t
keysize_min(MDBX_db_flags_t flags) {
MDBX_NOTHROW_CONST_FUNCTION static inline size_t keysize_min(MDBX_db_flags_t flags) {
return (flags & MDBX_INTEGERKEY) ? 4 /* sizeof(uint32_t) */ : 0;
}
MDBX_NOTHROW_CONST_FUNCTION static inline size_t
valsize_min(MDBX_db_flags_t flags) {
MDBX_NOTHROW_CONST_FUNCTION static inline size_t valsize_min(MDBX_db_flags_t flags) {
if (flags & MDBX_INTEGERDUP)
return 4 /* sizeof(uint32_t) */;
else if (flags & MDBX_DUPFIXED)
@ -120,10 +107,8 @@ valsize_min(MDBX_db_flags_t flags) {
return 0;
}
MDBX_NOTHROW_CONST_FUNCTION static inline size_t
valsize_max(size_t pagesize, MDBX_db_flags_t flags) {
assert(pagesize >= MDBX_MIN_PAGESIZE && pagesize <= MDBX_MAX_PAGESIZE &&
is_powerof2(pagesize));
MDBX_NOTHROW_CONST_FUNCTION static inline size_t valsize_max(size_t pagesize, MDBX_db_flags_t flags) {
assert(pagesize >= MDBX_MIN_PAGESIZE && pagesize <= MDBX_MAX_PAGESIZE && is_powerof2(pagesize));
if (flags & MDBX_INTEGERDUP)
return 8 /* sizeof(uint64_t) */;
@ -136,13 +121,11 @@ valsize_max(size_t pagesize, MDBX_db_flags_t flags) {
const size_t hard_pages = hard >> page_ln2;
STATIC_ASSERT(PAGELIST_LIMIT <= MAX_PAGENO);
const size_t pages_limit = PAGELIST_LIMIT / 4;
const size_t limit =
(hard_pages < pages_limit) ? hard : (pages_limit << page_ln2);
const size_t limit = (hard_pages < pages_limit) ? hard : (pages_limit << page_ln2);
return (limit < MAX_MAPSIZE / 2) ? limit : MAX_MAPSIZE / 2;
}
MDBX_NOTHROW_CONST_FUNCTION static inline size_t
env_valsize_max(const MDBX_env *env, MDBX_db_flags_t flags) {
MDBX_NOTHROW_CONST_FUNCTION static inline size_t env_valsize_max(const MDBX_env *env, MDBX_db_flags_t flags) {
size_t size_max;
if (flags & MDBX_INTEGERDUP)
size_max = 8 /* sizeof(uint64_t) */;
@ -153,8 +136,7 @@ env_valsize_max(const MDBX_env *env, MDBX_db_flags_t flags) {
const size_t hard_pages = hard >> env->ps2ln;
STATIC_ASSERT(PAGELIST_LIMIT <= MAX_PAGENO);
const size_t pages_limit = PAGELIST_LIMIT / 4;
const size_t limit =
(hard_pages < pages_limit) ? hard : (pages_limit << env->ps2ln);
const size_t limit = (hard_pages < pages_limit) ? hard : (pages_limit << env->ps2ln);
size_max = (limit < MAX_MAPSIZE / 2) ? limit : MAX_MAPSIZE / 2;
}
eASSERT(env, size_max == valsize_max(env->ps, flags));
@ -163,8 +145,8 @@ env_valsize_max(const MDBX_env *env, MDBX_db_flags_t flags) {
/*----------------------------------------------------------------------------*/
MDBX_NOTHROW_PURE_FUNCTION static inline size_t
leaf_size(const MDBX_env *env, const MDBX_val *key, const MDBX_val *data) {
MDBX_NOTHROW_PURE_FUNCTION static inline size_t leaf_size(const MDBX_env *env, const MDBX_val *key,
const MDBX_val *data) {
size_t node_bytes = node_size(key, data);
if (node_bytes > env->leaf_nodemax)
/* put on large/overflow page */
@ -173,35 +155,30 @@ leaf_size(const MDBX_env *env, const MDBX_val *key, const MDBX_val *data) {
return node_bytes + sizeof(indx_t);
}
MDBX_NOTHROW_PURE_FUNCTION static inline size_t
branch_size(const MDBX_env *env, const MDBX_val *key) {
MDBX_NOTHROW_PURE_FUNCTION static inline size_t branch_size(const MDBX_env *env, const MDBX_val *key) {
/* Size of a node in a branch page with a given key.
* This is just the node header plus the key, there is no data. */
size_t node_bytes = node_size(key, nullptr);
if (unlikely(node_bytes > env->branch_nodemax)) {
/* put on large/overflow page, not implemented */
mdbx_panic("node_size(key) %zu > %u branch_nodemax", node_bytes,
env->branch_nodemax);
mdbx_panic("node_size(key) %zu > %u branch_nodemax", node_bytes, env->branch_nodemax);
node_bytes = node_size(key, nullptr) + sizeof(pgno_t);
}
return node_bytes + sizeof(indx_t);
}
MDBX_NOTHROW_CONST_FUNCTION static inline uint16_t
flags_db2sub(uint16_t db_flags) {
MDBX_NOTHROW_CONST_FUNCTION static inline uint16_t flags_db2sub(uint16_t db_flags) {
uint16_t sub_flags = db_flags & MDBX_DUPFIXED;
/* MDBX_INTEGERDUP => MDBX_INTEGERKEY */
#define SHIFT_INTEGERDUP_TO_INTEGERKEY 2
STATIC_ASSERT((MDBX_INTEGERDUP >> SHIFT_INTEGERDUP_TO_INTEGERKEY) ==
MDBX_INTEGERKEY);
STATIC_ASSERT((MDBX_INTEGERDUP >> SHIFT_INTEGERDUP_TO_INTEGERKEY) == MDBX_INTEGERKEY);
sub_flags |= (db_flags & MDBX_INTEGERDUP) >> SHIFT_INTEGERDUP_TO_INTEGERKEY;
/* MDBX_REVERSEDUP => MDBX_REVERSEKEY */
#define SHIFT_REVERSEDUP_TO_REVERSEKEY 5
STATIC_ASSERT((MDBX_REVERSEDUP >> SHIFT_REVERSEDUP_TO_REVERSEKEY) ==
MDBX_REVERSEKEY);
STATIC_ASSERT((MDBX_REVERSEDUP >> SHIFT_REVERSEDUP_TO_REVERSEKEY) == MDBX_REVERSEKEY);
sub_flags |= (db_flags & MDBX_REVERSEDUP) >> SHIFT_REVERSEDUP_TO_REVERSEKEY;
return sub_flags;
@ -219,41 +196,33 @@ static inline bool check_table_flags(unsigned flags) {
case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP:
case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP:
case MDBX_DB_DEFAULTS:
return (flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)) !=
(MDBX_REVERSEKEY | MDBX_INTEGERKEY);
return (flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)) != (MDBX_REVERSEKEY | MDBX_INTEGERKEY);
}
}
/*----------------------------------------------------------------------------*/
MDBX_NOTHROW_PURE_FUNCTION static inline size_t pgno2bytes(const MDBX_env *env,
size_t pgno) {
MDBX_NOTHROW_PURE_FUNCTION static inline size_t pgno2bytes(const MDBX_env *env, size_t pgno) {
eASSERT(env, (1u << env->ps2ln) == env->ps);
return ((size_t)pgno) << env->ps2ln;
}
MDBX_NOTHROW_PURE_FUNCTION static inline page_t *pgno2page(const MDBX_env *env,
size_t pgno) {
MDBX_NOTHROW_PURE_FUNCTION static inline page_t *pgno2page(const MDBX_env *env, size_t pgno) {
return ptr_disp(env->dxb_mmap.base, pgno2bytes(env, pgno));
}
MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t bytes2pgno(const MDBX_env *env,
size_t bytes) {
MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t bytes2pgno(const MDBX_env *env, size_t bytes) {
eASSERT(env, (env->ps >> env->ps2ln) == 1);
return (pgno_t)(bytes >> env->ps2ln);
}
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t
bytes_align2os_bytes(const MDBX_env *env, size_t bytes);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t bytes_align2os_bytes(const MDBX_env *env, size_t bytes);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t
pgno_align2os_bytes(const MDBX_env *env, size_t pgno);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t pgno_align2os_bytes(const MDBX_env *env, size_t pgno);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL pgno_t
pgno_align2os_pgno(const MDBX_env *env, size_t pgno);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL pgno_t pgno_align2os_pgno(const MDBX_env *env, size_t pgno);
MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t
largechunk_npages(const MDBX_env *env, size_t bytes) {
MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t largechunk_npages(const MDBX_env *env, size_t bytes) {
return bytes2pgno(env, PAGEHDRSZ - 1 + bytes) + 1;
}
@ -264,69 +233,53 @@ MDBX_NOTHROW_PURE_FUNCTION static inline MDBX_val get_key(const node_t *node) {
return key;
}
static inline void get_key_optional(const node_t *node,
MDBX_val *keyptr /* __may_null */) {
static inline void get_key_optional(const node_t *node, MDBX_val *keyptr /* __may_null */) {
if (keyptr)
*keyptr = get_key(node);
}
MDBX_NOTHROW_PURE_FUNCTION static inline void *page_data(const page_t *mp) {
return ptr_disp(mp, PAGEHDRSZ);
}
MDBX_NOTHROW_PURE_FUNCTION static inline void *page_data(const page_t *mp) { return ptr_disp(mp, PAGEHDRSZ); }
MDBX_NOTHROW_PURE_FUNCTION static inline const page_t *
data_page(const void *data) {
MDBX_NOTHROW_PURE_FUNCTION static inline const page_t *data_page(const void *data) {
return container_of(data, page_t, entries);
}
MDBX_NOTHROW_PURE_FUNCTION static inline meta_t *page_meta(page_t *mp) {
return (meta_t *)page_data(mp);
}
MDBX_NOTHROW_PURE_FUNCTION static inline meta_t *page_meta(page_t *mp) { return (meta_t *)page_data(mp); }
MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_numkeys(const page_t *mp) {
return mp->lower >> 1;
}
MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_numkeys(const page_t *mp) { return mp->lower >> 1; }
MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_room(const page_t *mp) {
return mp->upper - mp->lower;
}
MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_room(const page_t *mp) { return mp->upper - mp->lower; }
MDBX_NOTHROW_PURE_FUNCTION static inline size_t
page_space(const MDBX_env *env) {
MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_space(const MDBX_env *env) {
STATIC_ASSERT(PAGEHDRSZ % 2 == 0);
return env->ps - PAGEHDRSZ;
}
MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_used(const MDBX_env *env,
const page_t *mp) {
MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_used(const MDBX_env *env, const page_t *mp) {
return page_space(env) - page_room(mp);
}
/* The percentage of space used in the page, in a percents. */
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline unsigned
page_fill_percentum_x10(const MDBX_env *env, const page_t *mp) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline unsigned page_fill_percentum_x10(const MDBX_env *env,
const page_t *mp) {
const size_t space = page_space(env);
return (unsigned)((page_used(env, mp) * 1000 + space / 2) / space);
}
MDBX_NOTHROW_PURE_FUNCTION static inline node_t *page_node(const page_t *mp,
size_t i) {
MDBX_NOTHROW_PURE_FUNCTION static inline node_t *page_node(const page_t *mp, size_t i) {
assert(page_type_compat(mp) == P_LEAF || page_type(mp) == P_BRANCH);
assert(page_numkeys(mp) > i);
assert(mp->entries[i] % 2 == 0);
return ptr_disp(mp, mp->entries[i] + PAGEHDRSZ);
}
MDBX_NOTHROW_PURE_FUNCTION static inline void *
page_dupfix_ptr(const page_t *mp, size_t i, size_t keysize) {
assert(page_type_compat(mp) == (P_LEAF | P_DUPFIX) && i == (indx_t)i &&
mp->dupfix_ksize == keysize);
MDBX_NOTHROW_PURE_FUNCTION static inline void *page_dupfix_ptr(const page_t *mp, size_t i, size_t keysize) {
assert(page_type_compat(mp) == (P_LEAF | P_DUPFIX) && i == (indx_t)i && mp->dupfix_ksize == keysize);
(void)keysize;
return ptr_disp(mp, PAGEHDRSZ + mp->dupfix_ksize * (indx_t)i);
}
MDBX_NOTHROW_PURE_FUNCTION static inline MDBX_val
page_dupfix_key(const page_t *mp, size_t i, size_t keysize) {
MDBX_NOTHROW_PURE_FUNCTION static inline MDBX_val page_dupfix_key(const page_t *mp, size_t i, size_t keysize) {
MDBX_val r;
r.iov_base = page_dupfix_ptr(mp, i, keysize);
r.iov_len = mp->dupfix_ksize;
@ -335,11 +288,9 @@ page_dupfix_key(const page_t *mp, size_t i, size_t keysize) {
/*----------------------------------------------------------------------------*/
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int
cmp_int_unaligned(const MDBX_val *a, const MDBX_val *b);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_int_unaligned(const MDBX_val *a, const MDBX_val *b);
#if MDBX_UNALIGNED_OK < 2 || \
(MDBX_DEBUG || MDBX_FORCE_ASSERTIONS || !defined(NDEBUG))
#if MDBX_UNALIGNED_OK < 2 || (MDBX_DEBUG || MDBX_FORCE_ASSERTIONS || !defined(NDEBUG))
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int
/* Compare two items pointing at 2-byte aligned unsigned int's. */
cmp_int_align2(const MDBX_val *a, const MDBX_val *b);
@ -347,8 +298,7 @@ cmp_int_align2(const MDBX_val *a, const MDBX_val *b);
#define cmp_int_align2 cmp_int_unaligned
#endif /* !MDBX_UNALIGNED_OK || debug */
#if MDBX_UNALIGNED_OK < 4 || \
(MDBX_DEBUG || MDBX_FORCE_ASSERTIONS || !defined(NDEBUG))
#if MDBX_UNALIGNED_OK < 4 || (MDBX_DEBUG || MDBX_FORCE_ASSERTIONS || !defined(NDEBUG))
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int
/* Compare two items pointing at 4-byte aligned unsigned int's. */
cmp_int_align4(const MDBX_val *a, const MDBX_val *b);
@ -357,50 +307,38 @@ cmp_int_align4(const MDBX_val *a, const MDBX_val *b);
#endif /* !MDBX_UNALIGNED_OK || debug */
/* Compare two items lexically */
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_lexical(const MDBX_val *a,
const MDBX_val *b);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_lexical(const MDBX_val *a, const MDBX_val *b);
/* Compare two items in reverse byte order */
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_reverse(const MDBX_val *a,
const MDBX_val *b);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_reverse(const MDBX_val *a, const MDBX_val *b);
/* Fast non-lexically comparator */
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_lenfast(const MDBX_val *a,
const MDBX_val *b);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_lenfast(const MDBX_val *a, const MDBX_val *b);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL bool
eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL bool eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l);
MDBX_NOTHROW_PURE_FUNCTION static inline bool eq_fast(const MDBX_val *a,
const MDBX_val *b) {
return unlikely(a->iov_len == b->iov_len) &&
eq_fast_slowpath(a->iov_base, b->iov_base, a->iov_len);
MDBX_NOTHROW_PURE_FUNCTION static inline bool eq_fast(const MDBX_val *a, const MDBX_val *b) {
return unlikely(a->iov_len == b->iov_len) && eq_fast_slowpath(a->iov_base, b->iov_base, a->iov_len);
}
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int
cmp_equal_or_greater(const MDBX_val *a, const MDBX_val *b);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_equal_or_greater(const MDBX_val *a, const MDBX_val *b);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int
cmp_equal_or_wrong(const MDBX_val *a, const MDBX_val *b);
MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_equal_or_wrong(const MDBX_val *a, const MDBX_val *b);
static inline MDBX_cmp_func *builtin_keycmp(MDBX_db_flags_t flags) {
return (flags & MDBX_REVERSEKEY) ? cmp_reverse
: (flags & MDBX_INTEGERKEY) ? cmp_int_align2
: cmp_lexical;
return (flags & MDBX_REVERSEKEY) ? cmp_reverse : (flags & MDBX_INTEGERKEY) ? cmp_int_align2 : cmp_lexical;
}
static inline MDBX_cmp_func *builtin_datacmp(MDBX_db_flags_t flags) {
return !(flags & MDBX_DUPSORT)
? cmp_lenfast
: ((flags & MDBX_INTEGERDUP)
? cmp_int_unaligned
: ((flags & MDBX_REVERSEDUP) ? cmp_reverse : cmp_lexical));
: ((flags & MDBX_INTEGERDUP) ? cmp_int_unaligned
: ((flags & MDBX_REVERSEDUP) ? cmp_reverse : cmp_lexical));
}
/*----------------------------------------------------------------------------*/
MDBX_INTERNAL uint32_t combine_durability_flags(const uint32_t a,
const uint32_t b);
MDBX_INTERNAL uint32_t combine_durability_flags(const uint32_t a, const uint32_t b);
MDBX_CONST_FUNCTION static inline lck_t *lckless_stub(const MDBX_env *env) {
uintptr_t stub = (uintptr_t)&env->lckless_placeholder;
@ -477,12 +415,10 @@ static inline int check_txn(const MDBX_txn *txn, int bad_bits) {
}
tASSERT(txn, (txn->flags & MDBX_TXN_FINISHED) ||
(txn->flags & MDBX_NOSTICKYTHREADS) ==
(txn->env->flags & MDBX_NOSTICKYTHREADS));
(txn->flags & MDBX_NOSTICKYTHREADS) == (txn->env->flags & MDBX_NOSTICKYTHREADS));
#if MDBX_TXN_CHECKOWNER
STATIC_ASSERT((long)MDBX_NOSTICKYTHREADS > (long)MDBX_TXN_FINISHED);
if ((txn->flags & (MDBX_NOSTICKYTHREADS | MDBX_TXN_FINISHED)) <
MDBX_TXN_FINISHED &&
if ((txn->flags & (MDBX_NOSTICKYTHREADS | MDBX_TXN_FINISHED)) < MDBX_TXN_FINISHED &&
unlikely(txn->owner != osal_thread_self()))
return txn->owner ? MDBX_THREAD_MISMATCH : MDBX_BAD_TXN;
#endif /* MDBX_TXN_CHECKOWNER */
@ -508,12 +444,10 @@ static inline int check_txn_rw(const MDBX_txn *txn, int bad_bits) {
MDBX_INTERNAL void mincore_clean_cache(const MDBX_env *const env);
MDBX_INTERNAL void update_mlcnt(const MDBX_env *env,
const pgno_t new_aligned_mlocked_pgno,
MDBX_INTERNAL void update_mlcnt(const MDBX_env *env, const pgno_t new_aligned_mlocked_pgno,
const bool lock_not_release);
MDBX_INTERNAL void munlock_after(const MDBX_env *env, const pgno_t aligned_pgno,
const size_t end_bytes);
MDBX_INTERNAL void munlock_after(const MDBX_env *env, const pgno_t aligned_pgno, const size_t end_bytes);
MDBX_INTERNAL void munlock_all(const MDBX_env *env);
@ -527,15 +461,13 @@ MDBX_INTERNAL void munlock_all(const MDBX_env *env);
#define osal_flush_incoherent_cpu_writeback() osal_compiler_barrier()
#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
MDBX_MAYBE_UNUSED static inline void
osal_flush_incoherent_mmap(const void *addr, size_t nbytes,
const intptr_t pagesize) {
MDBX_MAYBE_UNUSED static inline void osal_flush_incoherent_mmap(const void *addr, size_t nbytes,
const intptr_t pagesize) {
#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE
#error "The MDBX_MMAP_INCOHERENT_FILE_WRITE must be defined before"
#elif MDBX_MMAP_INCOHERENT_FILE_WRITE
char *const begin = (char *)(-pagesize & (intptr_t)addr);
char *const end =
(char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1));
char *const end = (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1));
int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0;
eASSERT(nullptr, err == 0);
(void)err;

118
src/coherency.c
View File

@ -4,8 +4,7 @@
#include "internals.h"
/* check against https://libmdbx.dqdkfa.ru/dead-github/issues/269 */
static bool coherency_check(const MDBX_env *env, const txnid_t txnid,
const volatile tree_t *trees,
static bool coherency_check(const MDBX_env *env, const txnid_t txnid, const volatile tree_t *trees,
const volatile meta_t *meta, bool report) {
const txnid_t freedb_mod_txnid = trees[FREE_DBI].mod_txnid;
const txnid_t maindb_mod_txnid = trees[MAIN_DBI].mod_txnid;
@ -13,67 +12,42 @@ static bool coherency_check(const MDBX_env *env, const txnid_t txnid,
const pgno_t freedb_root_pgno = trees[FREE_DBI].root;
const page_t *freedb_root =
(env->dxb_mmap.base && freedb_root_pgno < last_pgno)
? pgno2page(env, freedb_root_pgno)
: nullptr;
(env->dxb_mmap.base && freedb_root_pgno < last_pgno) ? pgno2page(env, freedb_root_pgno) : nullptr;
const pgno_t maindb_root_pgno = trees[MAIN_DBI].root;
const page_t *maindb_root =
(env->dxb_mmap.base && maindb_root_pgno < last_pgno)
? pgno2page(env, maindb_root_pgno)
: nullptr;
const uint64_t magic_and_version =
unaligned_peek_u64_volatile(4, &meta->magic_and_version);
(env->dxb_mmap.base && maindb_root_pgno < last_pgno) ? pgno2page(env, maindb_root_pgno) : nullptr;
const uint64_t magic_and_version = unaligned_peek_u64_volatile(4, &meta->magic_and_version);
bool ok = true;
if (freedb_root_pgno != P_INVALID &&
unlikely(freedb_root_pgno >= last_pgno)) {
if (freedb_root_pgno != P_INVALID && unlikely(freedb_root_pgno >= last_pgno)) {
if (report)
WARNING(
"catch invalid %s-db root %" PRIaPGNO " for meta_txnid %" PRIaTXN
" %s",
"free", freedb_root_pgno, txnid,
(env->stuck_meta < 0)
? "(workaround for incoherent flaw of unified page/buffer cache)"
: "(wagering meta)");
WARNING("catch invalid %s-db root %" PRIaPGNO " for meta_txnid %" PRIaTXN " %s", "free", freedb_root_pgno, txnid,
(env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)"
: "(wagering meta)");
ok = false;
}
if (maindb_root_pgno != P_INVALID &&
unlikely(maindb_root_pgno >= last_pgno)) {
if (maindb_root_pgno != P_INVALID && unlikely(maindb_root_pgno >= last_pgno)) {
if (report)
WARNING(
"catch invalid %s-db root %" PRIaPGNO " for meta_txnid %" PRIaTXN
" %s",
"main", maindb_root_pgno, txnid,
(env->stuck_meta < 0)
? "(workaround for incoherent flaw of unified page/buffer cache)"
: "(wagering meta)");
WARNING("catch invalid %s-db root %" PRIaPGNO " for meta_txnid %" PRIaTXN " %s", "main", maindb_root_pgno, txnid,
(env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)"
: "(wagering meta)");
ok = false;
}
if (unlikely(txnid < freedb_mod_txnid ||
(!freedb_mod_txnid && freedb_root &&
likely(magic_and_version == MDBX_DATA_MAGIC)))) {
(!freedb_mod_txnid && freedb_root && likely(magic_and_version == MDBX_DATA_MAGIC)))) {
if (report)
WARNING(
"catch invalid %s-db.mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN
" %s",
"free", freedb_mod_txnid, txnid,
(env->stuck_meta < 0)
? "(workaround for incoherent flaw of unified page/buffer cache)"
: "(wagering meta)");
"catch invalid %s-db.mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN " %s", "free", freedb_mod_txnid, txnid,
(env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)" : "(wagering meta)");
ok = false;
}
if (unlikely(txnid < maindb_mod_txnid ||
(!maindb_mod_txnid && maindb_root &&
likely(magic_and_version == MDBX_DATA_MAGIC)))) {
(!maindb_mod_txnid && maindb_root && likely(magic_and_version == MDBX_DATA_MAGIC)))) {
if (report)
WARNING(
"catch invalid %s-db.mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN
" %s",
"main", maindb_mod_txnid, txnid,
(env->stuck_meta < 0)
? "(workaround for incoherent flaw of unified page/buffer cache)"
: "(wagering meta)");
"catch invalid %s-db.mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN " %s", "main", maindb_mod_txnid, txnid,
(env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)" : "(wagering meta)");
ok = false;
}
@ -81,15 +55,13 @@ static bool coherency_check(const MDBX_env *env, const txnid_t txnid,
* в пределах текущего отображения. Иначе возможны SIGSEGV до переноса
* вызова coherency_check_head() после dxb_resize() внутри txn_renew(). */
if (likely(freedb_root && freedb_mod_txnid &&
(size_t)ptr_dist(env->dxb_mmap.base, freedb_root) <
env->dxb_mmap.limit)) {
(size_t)ptr_dist(env->dxb_mmap.base, freedb_root) < env->dxb_mmap.limit)) {
VALGRIND_MAKE_MEM_DEFINED(freedb_root, sizeof(freedb_root->txnid));
MDBX_ASAN_UNPOISON_MEMORY_REGION(freedb_root, sizeof(freedb_root->txnid));
const txnid_t root_txnid = freedb_root->txnid;
if (unlikely(root_txnid != freedb_mod_txnid)) {
if (report)
WARNING("catch invalid root_page %" PRIaPGNO " mod_txnid %" PRIaTXN
" for %s-db.mod_txnid %" PRIaTXN " %s",
WARNING("catch invalid root_page %" PRIaPGNO " mod_txnid %" PRIaTXN " for %s-db.mod_txnid %" PRIaTXN " %s",
freedb_root_pgno, root_txnid, "free", freedb_mod_txnid,
(env->stuck_meta < 0) ? "(workaround for incoherent flaw of "
"unified page/buffer cache)"
@ -98,15 +70,13 @@ static bool coherency_check(const MDBX_env *env, const txnid_t txnid,
}
}
if (likely(maindb_root && maindb_mod_txnid &&
(size_t)ptr_dist(env->dxb_mmap.base, maindb_root) <
env->dxb_mmap.limit)) {
(size_t)ptr_dist(env->dxb_mmap.base, maindb_root) < env->dxb_mmap.limit)) {
VALGRIND_MAKE_MEM_DEFINED(maindb_root, sizeof(maindb_root->txnid));
MDBX_ASAN_UNPOISON_MEMORY_REGION(maindb_root, sizeof(maindb_root->txnid));
const txnid_t root_txnid = maindb_root->txnid;
if (unlikely(root_txnid != maindb_mod_txnid)) {
if (report)
WARNING("catch invalid root_page %" PRIaPGNO " mod_txnid %" PRIaTXN
" for %s-db.mod_txnid %" PRIaTXN " %s",
WARNING("catch invalid root_page %" PRIaPGNO " mod_txnid %" PRIaTXN " for %s-db.mod_txnid %" PRIaTXN " %s",
maindb_root_pgno, root_txnid, "main", maindb_mod_txnid,
(env->stuck_meta < 0) ? "(workaround for incoherent flaw of "
"unified page/buffer cache)"
@ -116,24 +86,19 @@ static bool coherency_check(const MDBX_env *env, const txnid_t txnid,
}
if (unlikely(!ok) && report)
env->lck->pgops.incoherence.weak =
(env->lck->pgops.incoherence.weak >= INT32_MAX)
? INT32_MAX
: env->lck->pgops.incoherence.weak + 1;
(env->lck->pgops.incoherence.weak >= INT32_MAX) ? INT32_MAX : env->lck->pgops.incoherence.weak + 1;
return ok;
}
__cold int coherency_timeout(uint64_t *timestamp, intptr_t pgno,
const MDBX_env *env) {
__cold int coherency_timeout(uint64_t *timestamp, intptr_t pgno, const MDBX_env *env) {
if (likely(timestamp && *timestamp == 0))
*timestamp = osal_monotime();
else if (unlikely(!timestamp || osal_monotime() - *timestamp >
osal_16dot16_to_monotime(65536 / 10))) {
else if (unlikely(!timestamp || osal_monotime() - *timestamp > osal_16dot16_to_monotime(65536 / 10))) {
if (pgno >= 0 && pgno != env->stuck_meta)
ERROR("bailout waiting for %" PRIuSIZE " page arrival %s", pgno,
"(workaround for incoherent flaw of unified page/buffer cache)");
else if (env->stuck_meta < 0)
ERROR("bailout waiting for valid snapshot (%s)",
"workaround for incoherent flaw of unified page/buffer cache");
ERROR("bailout waiting for valid snapshot (%s)", "workaround for incoherent flaw of unified page/buffer cache");
return MDBX_PROBLEM;
}
@ -152,28 +117,23 @@ __cold int coherency_timeout(uint64_t *timestamp, intptr_t pgno,
/* check with timeout as the workaround
* for https://libmdbx.dqdkfa.ru/dead-github/issues/269 */
__hot int coherency_fetch_head(MDBX_txn *txn, const meta_ptr_t head,
uint64_t *timestamp) {
__hot int coherency_fetch_head(MDBX_txn *txn, const meta_ptr_t head, uint64_t *timestamp) {
/* Copy the DB info and flags */
txn->txnid = head.txnid;
txn->geo = head.ptr_c->geometry;
memcpy(txn->dbs, &head.ptr_c->trees, sizeof(head.ptr_c->trees));
STATIC_ASSERT(sizeof(head.ptr_c->trees) == CORE_DBS * sizeof(tree_t));
VALGRIND_MAKE_MEM_UNDEFINED(txn->dbs + CORE_DBS,
txn->env->max_dbi - CORE_DBS);
VALGRIND_MAKE_MEM_UNDEFINED(txn->dbs + CORE_DBS, txn->env->max_dbi - CORE_DBS);
txn->canary = head.ptr_c->canary;
if (unlikely(!coherency_check(txn->env, head.txnid, txn->dbs, head.ptr_v,
*timestamp == 0) ||
if (unlikely(!coherency_check(txn->env, head.txnid, txn->dbs, head.ptr_v, *timestamp == 0) ||
txn->txnid != meta_txnid(head.ptr_v)))
return coherency_timeout(timestamp, -1, txn->env);
if (unlikely(txn->dbs[FREE_DBI].flags != MDBX_INTEGERKEY)) {
if ((txn->dbs[FREE_DBI].flags & DB_PERSISTENT_FLAGS) != MDBX_INTEGERKEY ||
unaligned_peek_u64(4, &head.ptr_c->magic_and_version) ==
MDBX_DATA_MAGIC) {
ERROR("unexpected/invalid db-flags 0x%x for %s", txn->dbs[FREE_DBI].flags,
"GC/FreeDB");
unaligned_peek_u64(4, &head.ptr_c->magic_and_version) == MDBX_DATA_MAGIC) {
ERROR("unexpected/invalid db-flags 0x%x for %s", txn->dbs[FREE_DBI].flags, "GC/FreeDB");
return MDBX_INCOMPATIBLE;
}
txn->dbs[FREE_DBI].flags &= DB_PERSISTENT_FLAGS;
@ -183,23 +143,19 @@ __hot int coherency_fetch_head(MDBX_txn *txn, const meta_ptr_t head,
return MDBX_SUCCESS;
}
int coherency_check_written(const MDBX_env *env, const txnid_t txnid,
const volatile meta_t *meta, const intptr_t pgno,
int coherency_check_written(const MDBX_env *env, const txnid_t txnid, const volatile meta_t *meta, const intptr_t pgno,
uint64_t *timestamp) {
const bool report = !(timestamp && *timestamp);
const txnid_t head_txnid = meta_txnid(meta);
if (likely(head_txnid >= MIN_TXNID && head_txnid >= txnid)) {
if (likely(
coherency_check(env, head_txnid, &meta->trees.gc, meta, report))) {
if (likely(coherency_check(env, head_txnid, &meta->trees.gc, meta, report))) {
eASSERT(env, meta->trees.gc.flags == MDBX_INTEGERKEY);
eASSERT(env, check_table_flags(meta->trees.main.flags));
return MDBX_SUCCESS;
}
} else if (report) {
env->lck->pgops.incoherence.weak =
(env->lck->pgops.incoherence.weak >= INT32_MAX)
? INT32_MAX
: env->lck->pgops.incoherence.weak + 1;
(env->lck->pgops.incoherence.weak >= INT32_MAX) ? INT32_MAX : env->lck->pgops.incoherence.weak + 1;
WARNING("catch %s txnid %" PRIaTXN " for meta_%" PRIaPGNO " %s",
(head_txnid < MIN_TXNID) ? "invalid" : "unexpected", head_txnid,
bytes2pgno(env, ptr_dist(meta, env->dxb_mmap.base)),
@ -208,9 +164,7 @@ int coherency_check_written(const MDBX_env *env, const txnid_t txnid,
return coherency_timeout(timestamp, pgno, env);
}
bool coherency_check_meta(const MDBX_env *env, const volatile meta_t *meta,
bool report) {
bool coherency_check_meta(const MDBX_env *env, const volatile meta_t *meta, bool report) {
uint64_t timestamp = 0;
return coherency_check_written(env, 0, meta, -1,
report ? &timestamp : nullptr) == MDBX_SUCCESS;
return coherency_check_written(env, 0, meta, -1, report ? &timestamp : nullptr) == MDBX_SUCCESS;
}

228
src/cold.c
View File

@ -14,8 +14,7 @@ __cold size_t mdbx_default_pagesize(void) {
__cold intptr_t mdbx_limits_dbsize_min(intptr_t pagesize) {
if (pagesize < 1)
pagesize = (intptr_t)mdbx_default_pagesize();
else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE ||
pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
!is_powerof2((size_t)pagesize)))
return -1;
@ -25,8 +24,7 @@ __cold intptr_t mdbx_limits_dbsize_min(intptr_t pagesize) {
__cold intptr_t mdbx_limits_dbsize_max(intptr_t pagesize) {
if (pagesize < 1)
pagesize = (intptr_t)mdbx_default_pagesize();
else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE ||
pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
!is_powerof2((size_t)pagesize)))
return -1;
@ -38,112 +36,90 @@ __cold intptr_t mdbx_limits_dbsize_max(intptr_t pagesize) {
__cold intptr_t mdbx_limits_txnsize_max(intptr_t pagesize) {
if (pagesize < 1)
pagesize = (intptr_t)mdbx_default_pagesize();
else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE ||
pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
!is_powerof2((size_t)pagesize)))
return -1;
STATIC_ASSERT(MAX_MAPSIZE < INTPTR_MAX);
const uint64_t pgl_limit =
pagesize * (uint64_t)(PAGELIST_LIMIT / MDBX_GOLD_RATIO_DBL);
const uint64_t pgl_limit = pagesize * (uint64_t)(PAGELIST_LIMIT / MDBX_GOLD_RATIO_DBL);
const uint64_t map_limit = (uint64_t)(MAX_MAPSIZE / MDBX_GOLD_RATIO_DBL);
return (pgl_limit < map_limit) ? (intptr_t)pgl_limit : (intptr_t)map_limit;
}
__cold intptr_t mdbx_limits_keysize_max(intptr_t pagesize,
MDBX_db_flags_t flags) {
__cold intptr_t mdbx_limits_keysize_max(intptr_t pagesize, MDBX_db_flags_t flags) {
if (pagesize < 1)
pagesize = (intptr_t)mdbx_default_pagesize();
if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE ||
pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
!is_powerof2((size_t)pagesize)))
return -1;
return keysize_max(pagesize, flags);
}
__cold int mdbx_env_get_maxkeysize_ex(const MDBX_env *env,
MDBX_db_flags_t flags) {
__cold int mdbx_env_get_maxkeysize_ex(const MDBX_env *env, MDBX_db_flags_t flags) {
if (unlikely(!env || env->signature.weak != env_signature))
return -1;
return (int)mdbx_limits_keysize_max((intptr_t)env->ps, flags);
}
__cold int mdbx_env_get_maxkeysize(const MDBX_env *env) {
return mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT);
}
__cold int mdbx_env_get_maxkeysize(const MDBX_env *env) { return mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT); }
__cold intptr_t mdbx_limits_keysize_min(MDBX_db_flags_t flags) {
return keysize_min(flags);
}
__cold intptr_t mdbx_limits_keysize_min(MDBX_db_flags_t flags) { return keysize_min(flags); }
__cold intptr_t mdbx_limits_valsize_max(intptr_t pagesize,
MDBX_db_flags_t flags) {
__cold intptr_t mdbx_limits_valsize_max(intptr_t pagesize, MDBX_db_flags_t flags) {
if (pagesize < 1)
pagesize = (intptr_t)mdbx_default_pagesize();
if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE ||
pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
!is_powerof2((size_t)pagesize)))
return -1;
return valsize_max(pagesize, flags);
}
__cold int mdbx_env_get_maxvalsize_ex(const MDBX_env *env,
MDBX_db_flags_t flags) {
__cold int mdbx_env_get_maxvalsize_ex(const MDBX_env *env, MDBX_db_flags_t flags) {
if (unlikely(!env || env->signature.weak != env_signature))
return -1;
return (int)mdbx_limits_valsize_max((intptr_t)env->ps, flags);
}
__cold intptr_t mdbx_limits_valsize_min(MDBX_db_flags_t flags) {
return valsize_min(flags);
}
__cold intptr_t mdbx_limits_valsize_min(MDBX_db_flags_t flags) { return valsize_min(flags); }
__cold intptr_t mdbx_limits_pairsize4page_max(intptr_t pagesize,
MDBX_db_flags_t flags) {
__cold intptr_t mdbx_limits_pairsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags) {
if (pagesize < 1)
pagesize = (intptr_t)mdbx_default_pagesize();
if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE ||
pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
!is_powerof2((size_t)pagesize)))
return -1;
if (flags &
(MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP))
if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP))
return BRANCH_NODE_MAX(pagesize) - NODESIZE;
return LEAF_NODE_MAX(pagesize) - NODESIZE;
}
__cold int mdbx_env_get_pairsize4page_max(const MDBX_env *env,
MDBX_db_flags_t flags) {
__cold int mdbx_env_get_pairsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags) {
if (unlikely(!env || env->signature.weak != env_signature))
return -1;
return (int)mdbx_limits_pairsize4page_max((intptr_t)env->ps, flags);
}
__cold intptr_t mdbx_limits_valsize4page_max(intptr_t pagesize,
MDBX_db_flags_t flags) {
__cold intptr_t mdbx_limits_valsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags) {
if (pagesize < 1)
pagesize = (intptr_t)mdbx_default_pagesize();
if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE ||
pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE ||
!is_powerof2((size_t)pagesize)))
return -1;
if (flags &
(MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP))
if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP))
return valsize_max(pagesize, flags);
return PAGESPACE(pagesize);
}
__cold int mdbx_env_get_valsize4page_max(const MDBX_env *env,
MDBX_db_flags_t flags) {
__cold int mdbx_env_get_valsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags) {
if (unlikely(!env || env->signature.weak != env_signature))
return -1;
@ -152,17 +128,14 @@ __cold int mdbx_env_get_valsize4page_max(const MDBX_env *env,
/*----------------------------------------------------------------------------*/
__cold static void stat_add(const tree_t *db, MDBX_stat *const st,
const size_t bytes) {
__cold static void stat_add(const tree_t *db, MDBX_stat *const st, const size_t bytes) {
st->ms_depth += db->height;
st->ms_branch_pages += db->branch_pages;
st->ms_leaf_pages += db->leaf_pages;
st->ms_overflow_pages += db->large_pages;
st->ms_entries += db->items;
if (likely(bytes >=
offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid)))
st->ms_mod_txnid =
(st->ms_mod_txnid > db->mod_txnid) ? st->ms_mod_txnid : db->mod_txnid;
if (likely(bytes >= offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid)))
st->ms_mod_txnid = (st->ms_mod_txnid > db->mod_txnid) ? st->ms_mod_txnid : db->mod_txnid;
}
__cold static int stat_acc(const MDBX_txn *txn, MDBX_stat *st, size_t bytes) {
@ -179,15 +152,13 @@ __cold static int stat_acc(const MDBX_txn *txn, MDBX_stat *st, size_t bytes) {
const MDBX_env *const env = txn->env;
st->ms_psize = env->ps;
TXN_FOREACH_DBI_FROM(
txn, dbi,
/* assuming GC is internal and not subject for accounting */ MAIN_DBI) {
TXN_FOREACH_DBI_FROM(txn, dbi,
/* assuming GC is internal and not subject for accounting */ MAIN_DBI) {
if ((txn->dbi_state[dbi] & (DBI_VALID | DBI_STALE)) == DBI_VALID)
stat_add(txn->dbs + dbi, st, bytes);
}
if (!(txn->dbs[MAIN_DBI].flags & MDBX_DUPSORT) &&
txn->dbs[MAIN_DBI].items /* TODO: use `md_subs` field */) {
if (!(txn->dbs[MAIN_DBI].flags & MDBX_DUPSORT) && txn->dbs[MAIN_DBI].items /* TODO: use `md_subs` field */) {
/* scan and account not opened named tables */
err = tree_search(&cx.outer, nullptr, Z_FIRST);
@ -198,8 +169,7 @@ __cold static int stat_acc(const MDBX_txn *txn, MDBX_stat *st, size_t bytes) {
if (node_flags(node) != N_TREE)
continue;
if (unlikely(node_ds(node) != sizeof(tree_t))) {
ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid table node size", node_ds(node));
ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid table node size", node_ds(node));
return MDBX_CORRUPTED;
}
@ -228,8 +198,7 @@ __cold static int stat_acc(const MDBX_txn *txn, MDBX_stat *st, size_t bytes) {
return MDBX_SUCCESS;
}
__cold int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn,
MDBX_stat *dest, size_t bytes) {
__cold int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, MDBX_stat *dest, size_t bytes) {
if (unlikely(!dest))
return LOG_IFERR(MDBX_EINVAL);
const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid);
@ -265,18 +234,15 @@ __cold int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn,
/*----------------------------------------------------------------------------*/
static size_t estimate_rss(size_t database_bytes) {
return database_bytes + database_bytes / 64 +
(512 + MDBX_WORDBITS * 16) * MEGABYTE;
return database_bytes + database_bytes / 64 + (512 + MDBX_WORDBITS * 16) * MEGABYTE;
}
__cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn,
MDBX_warmup_flags_t flags,
__cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, MDBX_warmup_flags_t flags,
unsigned timeout_seconds_16dot16) {
if (unlikely(env == nullptr && txn == nullptr))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(flags >
(MDBX_warmup_force | MDBX_warmup_oomsafe | MDBX_warmup_lock |
MDBX_warmup_touchlimit | MDBX_warmup_release)))
if (unlikely(flags > (MDBX_warmup_force | MDBX_warmup_oomsafe | MDBX_warmup_lock | MDBX_warmup_touchlimit |
MDBX_warmup_release)))
return LOG_IFERR(MDBX_EINVAL);
if (txn) {
@ -294,10 +260,9 @@ __cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn,
env = txn->env;
}
const uint64_t timeout_monotime =
(timeout_seconds_16dot16 && (flags & MDBX_warmup_force))
? osal_monotime() + osal_16dot16_to_monotime(timeout_seconds_16dot16)
: 0;
const uint64_t timeout_monotime = (timeout_seconds_16dot16 && (flags & MDBX_warmup_force))
? osal_monotime() + osal_16dot16_to_monotime(timeout_seconds_16dot16)
: 0;
if (flags & MDBX_warmup_release)
munlock_all(env);
@ -317,18 +282,14 @@ __cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn,
const size_t estimated_rss = estimate_rss(used_range);
#if defined(_WIN32) || defined(_WIN64)
SIZE_T current_ws_lower, current_ws_upper;
if (GetProcessWorkingSetSize(GetCurrentProcess(), &current_ws_lower,
&current_ws_upper) &&
if (GetProcessWorkingSetSize(GetCurrentProcess(), &current_ws_lower, &current_ws_upper) &&
current_ws_lower < estimated_rss) {
const SIZE_T ws_lower = estimated_rss;
const SIZE_T ws_upper =
(MDBX_WORDBITS == 32 && ws_lower > MEGABYTE * 2048)
? ws_lower
: ws_lower + MDBX_WORDBITS * MEGABYTE * 32;
(MDBX_WORDBITS == 32 && ws_lower > MEGABYTE * 2048) ? ws_lower : ws_lower + MDBX_WORDBITS * MEGABYTE * 32;
if (!SetProcessWorkingSetSize(GetCurrentProcess(), ws_lower, ws_upper)) {
rc = (int)GetLastError();
WARNING("SetProcessWorkingSetSize(%zu, %zu) error %d", ws_lower,
ws_upper, rc);
WARNING("SetProcessWorkingSetSize(%zu, %zu) error %d", ws_lower, ws_upper, rc);
}
}
#endif /* Windows */
@ -340,23 +301,21 @@ __cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn,
rss.rlim_max = estimated_rss;
if (setrlimit(RLIMIT_RSS, &rss)) {
rc = errno;
WARNING("setrlimit(%s, {%zu, %zu}) error %d", "RLIMIT_RSS",
(size_t)rss.rlim_cur, (size_t)rss.rlim_max, rc);
WARNING("setrlimit(%s, {%zu, %zu}) error %d", "RLIMIT_RSS", (size_t)rss.rlim_cur, (size_t)rss.rlim_max, rc);
}
}
#endif /* RLIMIT_RSS */
#ifdef RLIMIT_MEMLOCK
if (flags & MDBX_warmup_lock) {
struct rlimit memlock;
if (getrlimit(RLIMIT_MEMLOCK, &memlock) == 0 &&
memlock.rlim_cur < estimated_rss) {
if (getrlimit(RLIMIT_MEMLOCK, &memlock) == 0 && memlock.rlim_cur < estimated_rss) {
memlock.rlim_cur = estimated_rss;
if (memlock.rlim_max < estimated_rss)
memlock.rlim_max = estimated_rss;
if (setrlimit(RLIMIT_MEMLOCK, &memlock)) {
rc = errno;
WARNING("setrlimit(%s, {%zu, %zu}) error %d", "RLIMIT_MEMLOCK",
(size_t)memlock.rlim_cur, (size_t)memlock.rlim_max, rc);
WARNING("setrlimit(%s, {%zu, %zu}) error %d", "RLIMIT_MEMLOCK", (size_t)memlock.rlim_cur,
(size_t)memlock.rlim_max, rc);
}
}
}
@ -364,12 +323,10 @@ __cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn,
(void)estimated_rss;
}
#if defined(MLOCK_ONFAULT) && \
((defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 27)) || \
(defined(__ANDROID_API__) && __ANDROID_API__ >= 30)) && \
#if defined(MLOCK_ONFAULT) && \
((defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 27)) || (defined(__ANDROID_API__) && __ANDROID_API__ >= 30)) && \
(defined(__linux__) || defined(__gnu_linux__))
if ((flags & MDBX_warmup_lock) != 0 &&
globals.linux_kernel_version >= 0x04040000 &&
if ((flags & MDBX_warmup_lock) != 0 && globals.linux_kernel_version >= 0x04040000 &&
atomic_load32(&env->mlocked_pgno, mo_AcquireRelease) < mlock_pgno) {
if (mlock2(env->dxb_mmap.base, used_range, MLOCK_ONFAULT)) {
rc = errno;
@ -388,8 +345,7 @@ __cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn,
if (err != MDBX_SUCCESS && rc == MDBX_SUCCESS)
rc = err;
if ((flags & MDBX_warmup_force) != 0 &&
(rc == MDBX_SUCCESS || rc == MDBX_ENOSYS)) {
if ((flags & MDBX_warmup_force) != 0 && (rc == MDBX_SUCCESS || rc == MDBX_ENOSYS)) {
const volatile uint8_t *ptr = env->dxb_mmap.base;
size_t offset = 0, unused = 42;
#if !(defined(_WIN32) || defined(_WIN64))
@ -440,8 +396,7 @@ __cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn,
(void)unused;
}
if ((flags & MDBX_warmup_lock) != 0 &&
(rc == MDBX_SUCCESS || rc == MDBX_ENOSYS) &&
if ((flags & MDBX_warmup_lock) != 0 && (rc == MDBX_SUCCESS || rc == MDBX_ENOSYS) &&
atomic_load32(&env->mlocked_pgno, mo_AcquireRelease) < mlock_pgno) {
#if defined(_WIN32) || defined(_WIN64)
if (VirtualLock(env->dxb_mmap.base, used_range)) {
@ -481,14 +436,12 @@ __cold int mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *arg) {
return MDBX_SUCCESS;
}
__cold int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags,
bool onoff) {
__cold int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags, bool onoff) {
int rc = check_env(env, false);
if (unlikely(rc != MDBX_SUCCESS))
return LOG_IFERR(rc);
if (unlikely(flags & ((env->flags & ENV_ACTIVE) ? ~ENV_CHANGEABLE_FLAGS
: ~ENV_USABLE_FLAGS)))
if (unlikely(flags & ((env->flags & ENV_ACTIVE) ? ~ENV_CHANGEABLE_FLAGS : ~ENV_USABLE_FLAGS)))
return LOG_IFERR(MDBX_EPERM);
if (unlikely(env->flags & MDBX_RDONLY))
@ -536,9 +489,7 @@ __cold int mdbx_env_set_userctx(MDBX_env *env, void *ctx) {
return MDBX_SUCCESS;
}
__cold void *mdbx_env_get_userctx(const MDBX_env *env) {
return env ? env->userctx : nullptr;
}
__cold void *mdbx_env_get_userctx(const MDBX_env *env) { return env ? env->userctx : nullptr; }
__cold int mdbx_env_set_assert(MDBX_env *env, MDBX_assert_func *func) {
int rc = check_env(env, false);
@ -564,8 +515,7 @@ __cold int mdbx_env_set_hsr(MDBX_env *env, MDBX_hsr_func *hsr) {
}
__cold MDBX_hsr_func *mdbx_env_get_hsr(const MDBX_env *env) {
return likely(env && env->signature.weak == env_signature) ? env->hsr_callback
: nullptr;
return likely(env && env->signature.weak == env_signature) ? env->hsr_callback : nullptr;
}
#if defined(_WIN32) || defined(_WIN64)
@ -595,13 +545,10 @@ __cold int mdbx_env_get_path(const MDBX_env *env, const char **arg) {
*arg = nullptr;
DWORD flags = /* WC_ERR_INVALID_CHARS */ 0x80;
size_t mb_len =
WideCharToMultiByte(CP_THREAD_ACP, flags, env->pathname.specified, -1,
nullptr, 0, nullptr, nullptr);
WideCharToMultiByte(CP_THREAD_ACP, flags, env->pathname.specified, -1, nullptr, 0, nullptr, nullptr);
rc = mb_len ? MDBX_SUCCESS : (int)GetLastError();
if (rc == ERROR_INVALID_FLAGS) {
mb_len =
WideCharToMultiByte(CP_THREAD_ACP, flags = 0, env->pathname.specified,
-1, nullptr, 0, nullptr, nullptr);
mb_len = WideCharToMultiByte(CP_THREAD_ACP, flags = 0, env->pathname.specified, -1, nullptr, 0, nullptr, nullptr);
rc = mb_len ? MDBX_SUCCESS : (int)GetLastError();
}
if (unlikely(rc != MDBX_SUCCESS))
@ -610,16 +557,14 @@ __cold int mdbx_env_get_path(const MDBX_env *env, const char **arg) {
char *const mb_pathname = osal_malloc(mb_len);
if (!mb_pathname)
return LOG_IFERR(MDBX_ENOMEM);
if (mb_len != (size_t)WideCharToMultiByte(
CP_THREAD_ACP, flags, env->pathname.specified, -1,
mb_pathname, (int)mb_len, nullptr, nullptr)) {
if (mb_len != (size_t)WideCharToMultiByte(CP_THREAD_ACP, flags, env->pathname.specified, -1, mb_pathname,
(int)mb_len, nullptr, nullptr)) {
rc = (int)GetLastError();
osal_free(mb_pathname);
return LOG_IFERR(rc);
}
if (env->pathname_char ||
InterlockedCompareExchangePointer((PVOID volatile *)&env->pathname_char,
mb_pathname, nullptr))
InterlockedCompareExchangePointer((PVOID volatile *)&env->pathname_char, mb_pathname, nullptr))
osal_free(mb_pathname);
}
*arg = env->pathname_char;
@ -634,41 +579,29 @@ __cold int mdbx_env_get_path(const MDBX_env *env, const char **arg) {
#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API
LIBMDBX_API int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent,
MDBX_txn_flags_t flags, MDBX_txn **ret) {
LIBMDBX_API int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, MDBX_txn **ret) {
return __inline_mdbx_txn_begin(env, parent, flags, ret);
}
LIBMDBX_API int mdbx_txn_commit(MDBX_txn *txn) {
return __inline_mdbx_txn_commit(txn);
}
LIBMDBX_API int mdbx_txn_commit(MDBX_txn *txn) { return __inline_mdbx_txn_commit(txn); }
LIBMDBX_API __cold int mdbx_env_stat(const MDBX_env *env, MDBX_stat *stat,
size_t bytes) {
LIBMDBX_API __cold int mdbx_env_stat(const MDBX_env *env, MDBX_stat *stat, size_t bytes) {
return __inline_mdbx_env_stat(env, stat, bytes);
}
LIBMDBX_API __cold int mdbx_env_info(const MDBX_env *env, MDBX_envinfo *info,
size_t bytes) {
LIBMDBX_API __cold int mdbx_env_info(const MDBX_env *env, MDBX_envinfo *info, size_t bytes) {
return __inline_mdbx_env_info(env, info, bytes);
}
LIBMDBX_API int mdbx_dbi_flags(const MDBX_txn *txn, MDBX_dbi dbi,
unsigned *flags) {
LIBMDBX_API int mdbx_dbi_flags(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags) {
return __inline_mdbx_dbi_flags(txn, dbi, flags);
}
LIBMDBX_API __cold int mdbx_env_sync(MDBX_env *env) {
return __inline_mdbx_env_sync(env);
}
LIBMDBX_API __cold int mdbx_env_sync(MDBX_env *env) { return __inline_mdbx_env_sync(env); }
LIBMDBX_API __cold int mdbx_env_sync_poll(MDBX_env *env) {
return __inline_mdbx_env_sync_poll(env);
}
LIBMDBX_API __cold int mdbx_env_sync_poll(MDBX_env *env) { return __inline_mdbx_env_sync_poll(env); }
LIBMDBX_API __cold int mdbx_env_close(MDBX_env *env) {
return __inline_mdbx_env_close(env);
}
LIBMDBX_API __cold int mdbx_env_close(MDBX_env *env) { return __inline_mdbx_env_close(env); }
LIBMDBX_API __cold int mdbx_env_set_mapsize(MDBX_env *env, size_t size) {
return __inline_mdbx_env_set_mapsize(env, size);
@ -682,13 +615,11 @@ LIBMDBX_API __cold int mdbx_env_get_maxdbs(const MDBX_env *env, MDBX_dbi *dbs) {
return __inline_mdbx_env_get_maxdbs(env, dbs);
}
LIBMDBX_API __cold int mdbx_env_set_maxreaders(MDBX_env *env,
unsigned readers) {
LIBMDBX_API __cold int mdbx_env_set_maxreaders(MDBX_env *env, unsigned readers) {
return __inline_mdbx_env_set_maxreaders(env, readers);
}
LIBMDBX_API __cold int mdbx_env_get_maxreaders(const MDBX_env *env,
unsigned *readers) {
LIBMDBX_API __cold int mdbx_env_get_maxreaders(const MDBX_env *env, unsigned *readers) {
return __inline_mdbx_env_get_maxreaders(env, readers);
}
@ -696,35 +627,24 @@ LIBMDBX_API __cold int mdbx_env_set_syncbytes(MDBX_env *env, size_t threshold) {
return __inline_mdbx_env_set_syncbytes(env, threshold);
}
LIBMDBX_API __cold int mdbx_env_get_syncbytes(const MDBX_env *env,
size_t *threshold) {
LIBMDBX_API __cold int mdbx_env_get_syncbytes(const MDBX_env *env, size_t *threshold) {
return __inline_mdbx_env_get_syncbytes(env, threshold);
}
LIBMDBX_API __cold int mdbx_env_set_syncperiod(MDBX_env *env,
unsigned seconds_16dot16) {
LIBMDBX_API __cold int mdbx_env_set_syncperiod(MDBX_env *env, unsigned seconds_16dot16) {
return __inline_mdbx_env_set_syncperiod(env, seconds_16dot16);
}
LIBMDBX_API __cold int mdbx_env_get_syncperiod(const MDBX_env *env,
unsigned *seconds_16dot16) {
LIBMDBX_API __cold int mdbx_env_get_syncperiod(const MDBX_env *env, unsigned *seconds_16dot16) {
return __inline_mdbx_env_get_syncperiod(env, seconds_16dot16);
}
LIBMDBX_API __cold uint64_t mdbx_key_from_int64(const int64_t i64) {
return __inline_mdbx_key_from_int64(i64);
}
LIBMDBX_API __cold uint64_t mdbx_key_from_int64(const int64_t i64) { return __inline_mdbx_key_from_int64(i64); }
LIBMDBX_API __cold uint32_t mdbx_key_from_int32(const int32_t i32) {
return __inline_mdbx_key_from_int32(i32);
}
LIBMDBX_API __cold uint32_t mdbx_key_from_int32(const int32_t i32) { return __inline_mdbx_key_from_int32(i32); }
LIBMDBX_API __cold intptr_t mdbx_limits_pgsize_min(void) {
return __inline_mdbx_limits_pgsize_min();
}
LIBMDBX_API __cold intptr_t mdbx_limits_pgsize_min(void) { return __inline_mdbx_limits_pgsize_min(); }
LIBMDBX_API __cold intptr_t mdbx_limits_pgsize_max(void) {
return __inline_mdbx_limits_pgsize_max();
}
LIBMDBX_API __cold intptr_t mdbx_limits_pgsize_max(void) { return __inline_mdbx_limits_pgsize_max(); }
#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */

177
src/copy.c
View File

@ -93,8 +93,7 @@ __cold static int compacting_toggle_write_buffers(ctx_t *ctx) {
return ctx->error;
}
static int compacting_put_bytes(ctx_t *ctx, const void *src, size_t bytes,
pgno_t pgno, pgno_t npages) {
static int compacting_put_bytes(ctx_t *ctx, const void *src, size_t bytes, pgno_t pgno, pgno_t npages) {
assert(pgno == 0 || bytes > PAGEHDRSZ);
while (bytes > 0) {
const size_t side = ctx->head & 1;
@ -130,17 +129,14 @@ static int compacting_put_bytes(ctx_t *ctx, const void *src, size_t bytes,
return MDBX_SUCCESS;
}
static int compacting_put_page(ctx_t *ctx, const page_t *mp,
const size_t head_bytes, const size_t tail_bytes,
static int compacting_put_page(ctx_t *ctx, const page_t *mp, const size_t head_bytes, const size_t tail_bytes,
const pgno_t npages) {
if (tail_bytes) {
assert(head_bytes + tail_bytes <= ctx->env->ps);
assert(npages == 1 &&
(page_type(mp) == P_BRANCH || page_type(mp) == P_LEAF));
assert(npages == 1 && (page_type(mp) == P_BRANCH || page_type(mp) == P_LEAF));
} else {
assert(head_bytes <= pgno2bytes(ctx->env, npages));
assert((npages == 1 && page_type(mp) == (P_LEAF | P_DUPFIX)) ||
page_type(mp) == P_LARGE);
assert((npages == 1 && page_type(mp) == (P_LEAF | P_DUPFIX)) || page_type(mp) == P_LARGE);
}
const pgno_t pgno = ctx->first_unallocated;
@ -148,18 +144,13 @@ static int compacting_put_page(ctx_t *ctx, const page_t *mp,
int err = compacting_put_bytes(ctx, mp, head_bytes, pgno, npages);
if (unlikely(err != MDBX_SUCCESS))
return err;
err = compacting_put_bytes(
ctx, nullptr, pgno2bytes(ctx->env, npages) - (head_bytes + tail_bytes), 0,
0);
err = compacting_put_bytes(ctx, nullptr, pgno2bytes(ctx->env, npages) - (head_bytes + tail_bytes), 0, 0);
if (unlikely(err != MDBX_SUCCESS))
return err;
return compacting_put_bytes(ctx, ptr_disp(mp, ctx->env->ps - tail_bytes),
tail_bytes, 0, 0);
return compacting_put_bytes(ctx, ptr_disp(mp, ctx->env->ps - tail_bytes), tail_bytes, 0, 0);
}
__cold static int compacting_walk(ctx_t *ctx, MDBX_cursor *mc,
pgno_t *const parent_pgno,
txnid_t parent_txnid) {
__cold static int compacting_walk(ctx_t *ctx, MDBX_cursor *mc, pgno_t *const parent_pgno, txnid_t parent_txnid) {
mc->top = 0;
mc->ki[0] = 0;
int rc = page_get(mc, *parent_pgno, &mc->pg[0], parent_txnid);
@ -201,22 +192,18 @@ __cold static int compacting_walk(ctx_t *ctx, MDBX_cursor *mc,
node = page_node(mp, i);
}
const pgr_t lp =
page_get_large(mc, node_largedata_pgno(node), mp->txnid);
const pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->txnid);
if (unlikely((rc = lp.err) != MDBX_SUCCESS))
goto bailout;
const size_t datasize = node_ds(node);
const pgno_t npages = largechunk_npages(ctx->env, datasize);
poke_pgno(node_data(node), ctx->first_unallocated);
rc = compacting_put_page(ctx, lp.page, PAGEHDRSZ + datasize, 0,
npages);
rc = compacting_put_page(ctx, lp.page, PAGEHDRSZ + datasize, 0, npages);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
} else if (node_flags(node) & N_TREE) {
if (!MDBX_DISABLE_VALIDATION &&
unlikely(node_ds(node) != sizeof(tree_t))) {
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid dupsort sub-tree node size",
if (!MDBX_DISABLE_VALIDATION && unlikely(node_ds(node) != sizeof(tree_t))) {
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid dupsort sub-tree node size",
(unsigned)node_ds(node));
rc = MDBX_CORRUPTED;
goto bailout;
@ -235,13 +222,11 @@ __cold static int compacting_walk(ctx_t *ctx, MDBX_cursor *mc,
rc = cursor_dupsort_setup(mc, node, mp);
if (likely(rc == MDBX_SUCCESS)) {
nested = &mc->subcur->nested_tree;
rc = compacting_walk(ctx, &mc->subcur->cursor, &nested->root,
mp->txnid);
rc = compacting_walk(ctx, &mc->subcur->cursor, &nested->root, mp->txnid);
}
} else {
cASSERT(mc, (mc->flags & z_inner) == 0 && mc->subcur == 0);
cursor_couple_t *couple =
container_of(mc, cursor_couple_t, outer);
cursor_couple_t *couple = container_of(mc, cursor_couple_t, outer);
nested = &couple->inner.nested_tree;
memcpy(nested, node_data(node), sizeof(tree_t));
rc = compacting_walk_tree(ctx, nested);
@ -280,11 +265,9 @@ __cold static int compacting_walk(ctx_t *ctx, MDBX_cursor *mc,
const pgno_t pgno = ctx->first_unallocated;
if (likely(!is_dupfix_leaf(mp))) {
rc = compacting_put_page(ctx, mp, PAGEHDRSZ + mp->lower,
ctx->env->ps - (PAGEHDRSZ + mp->upper), 1);
rc = compacting_put_page(ctx, mp, PAGEHDRSZ + mp->lower, ctx->env->ps - (PAGEHDRSZ + mp->upper), 1);
} else {
rc = compacting_put_page(
ctx, mp, PAGEHDRSZ + page_numkeys(mp) * mp->dupfix_ksize, 0, 1);
rc = compacting_put_page(ctx, mp, PAGEHDRSZ + page_numkeys(mp) * mp->dupfix_ksize, 0, 1);
}
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
@ -326,19 +309,15 @@ __cold static int compacting_walk_tree(ctx_t *ctx, tree_t *tree) {
__cold static void compacting_fixup_meta(MDBX_env *env, meta_t *meta) {
eASSERT(env, meta->trees.gc.mod_txnid || meta->trees.gc.root == P_INVALID);
eASSERT(env,
meta->trees.main.mod_txnid || meta->trees.main.root == P_INVALID);
eASSERT(env, meta->trees.main.mod_txnid || meta->trees.main.root == P_INVALID);
/* Calculate filesize taking in account shrink/growing thresholds */
if (meta->geometry.first_unallocated != meta->geometry.now) {
meta->geometry.now = meta->geometry.first_unallocated;
const size_t aligner =
pv2pages(meta->geometry.grow_pv ? meta->geometry.grow_pv
: meta->geometry.shrink_pv);
const size_t aligner = pv2pages(meta->geometry.grow_pv ? meta->geometry.grow_pv : meta->geometry.shrink_pv);
if (aligner) {
const pgno_t aligned = pgno_align2os_pgno(
env, meta->geometry.first_unallocated + aligner -
meta->geometry.first_unallocated % aligner);
const pgno_t aligned = pgno_align2os_pgno(env, meta->geometry.first_unallocated + aligner -
meta->geometry.first_unallocated % aligner);
meta->geometry.now = aligned;
}
}
@ -366,13 +345,10 @@ __cold static void meta_make_sizeable(meta_t *meta) {
}
}
__cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn,
mdbx_filehandle_t fd, uint8_t *buffer,
const bool dest_is_pipe,
const MDBX_copy_flags_t flags) {
__cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn, mdbx_filehandle_t fd, uint8_t *buffer,
const bool dest_is_pipe, const MDBX_copy_flags_t flags) {
const size_t meta_bytes = pgno2bytes(env, NUM_METAS);
uint8_t *const data_buffer =
buffer + ceil_powerof2(meta_bytes, globals.sys_pagesize);
uint8_t *const data_buffer = buffer + ceil_powerof2(meta_bytes, globals.sys_pagesize);
meta_t *const meta = meta_init_triplet(env, buffer);
meta_set_txnid(env, meta, txn->txnid);
@ -405,22 +381,17 @@ __cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn,
int rc = cursor_init(&couple.outer, txn, FREE_DBI);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
pgno_t gc_npages = txn->dbs[FREE_DBI].branch_pages +
txn->dbs[FREE_DBI].leaf_pages +
txn->dbs[FREE_DBI].large_pages;
pgno_t gc_npages = txn->dbs[FREE_DBI].branch_pages + txn->dbs[FREE_DBI].leaf_pages + txn->dbs[FREE_DBI].large_pages;
MDBX_val key, data;
rc = outer_first(&couple.outer, &key, &data);
while (rc == MDBX_SUCCESS) {
const pnl_t pnl = data.iov_base;
if (unlikely(data.iov_len % sizeof(pgno_t) ||
data.iov_len < MDBX_PNL_SIZEOF(pnl))) {
ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid GC-record length", data.iov_len);
if (unlikely(data.iov_len % sizeof(pgno_t) || data.iov_len < MDBX_PNL_SIZEOF(pnl))) {
ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-record length", data.iov_len);
return MDBX_CORRUPTED;
}
if (unlikely(!pnl_check(pnl, txn->geo.first_unallocated))) {
ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid GC-record content");
ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-record content");
return MDBX_CORRUPTED;
}
gc_npages += MDBX_PNL_GETSIZE(pnl);
@ -466,21 +437,16 @@ __cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn,
/* toggle to flush non-empty buffers */
compacting_toggle_write_buffers(&ctx);
if (likely(rc == MDBX_SUCCESS) &&
unlikely(meta->geometry.first_unallocated != ctx.first_unallocated)) {
if (likely(rc == MDBX_SUCCESS) && unlikely(meta->geometry.first_unallocated != ctx.first_unallocated)) {
if (ctx.first_unallocated > meta->geometry.first_unallocated) {
ERROR("the source DB %s: post-compactification used pages %" PRIaPGNO
" %c expected %" PRIaPGNO,
"has double-used pages or other corruption",
ctx.first_unallocated, '>', meta->geometry.first_unallocated);
ERROR("the source DB %s: post-compactification used pages %" PRIaPGNO " %c expected %" PRIaPGNO,
"has double-used pages or other corruption", ctx.first_unallocated, '>',
meta->geometry.first_unallocated);
rc = MDBX_CORRUPTED; /* corrupted DB */
}
if (ctx.first_unallocated < meta->geometry.first_unallocated) {
WARNING(
"the source DB %s: post-compactification used pages %" PRIaPGNO
" %c expected %" PRIaPGNO,
"has page leak(s)", ctx.first_unallocated, '<',
meta->geometry.first_unallocated);
WARNING("the source DB %s: post-compactification used pages %" PRIaPGNO " %c expected %" PRIaPGNO,
"has page leak(s)", ctx.first_unallocated, '<', meta->geometry.first_unallocated);
if (dest_is_pipe)
/* the root within already written meta-pages is wrong */
rc = MDBX_CORRUPTED;
@ -493,8 +459,7 @@ __cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn,
eASSERT(env, (ctx.write_len[ctx.head & 1]) == 0);
compacting_toggle_write_buffers(&ctx);
thread_err = osal_thread_join(thread);
eASSERT(env, (ctx.tail == ctx.head && ctx.write_len[ctx.head & 1] == 0) ||
ctx.error);
eASSERT(env, (ctx.tail == ctx.head && ctx.write_len[ctx.head & 1] == 0) || ctx.error);
osal_condpair_destroy(&ctx.condpair);
}
if (unlikely(thread_err != MDBX_SUCCESS))
@ -519,9 +484,8 @@ __cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn,
const size_t used_size = pgno2bytes(env, meta->geometry.first_unallocated);
memset(data_buffer, 0, (size_t)MDBX_ENVCOPY_WRITEBUF);
for (size_t offset = used_size; offset < whole_size;) {
const size_t chunk = ((size_t)MDBX_ENVCOPY_WRITEBUF < whole_size - offset)
? (size_t)MDBX_ENVCOPY_WRITEBUF
: whole_size - offset;
const size_t chunk =
((size_t)MDBX_ENVCOPY_WRITEBUF < whole_size - offset) ? (size_t)MDBX_ENVCOPY_WRITEBUF : whole_size - offset;
int rc = osal_write(fd, data_buffer, chunk);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
@ -533,9 +497,8 @@ __cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn,
//----------------------------------------------------------------------------
__cold static int copy_asis(MDBX_env *env, MDBX_txn *txn, mdbx_filehandle_t fd,
uint8_t *buffer, const bool dest_is_pipe,
const MDBX_copy_flags_t flags) {
__cold static int copy_asis(MDBX_env *env, MDBX_txn *txn, mdbx_filehandle_t fd, uint8_t *buffer,
const bool dest_is_pipe, const MDBX_copy_flags_t flags) {
bool should_unlock = false;
if ((txn->flags & MDBX_TXN_RDONLY) != 0 && (flags & MDBX_CP_RENEW_TXN) != 0) {
/* Try temporarily block writers until we snapshot the meta pages */
@ -566,8 +529,7 @@ retry_snap_meta:
rc = MDBX_MVCC_RETARDED;
for (size_t n = 0; n < NUM_METAS; ++n) {
meta_t *const meta = page_meta(ptr_disp(buffer, pgno2bytes(env, n)));
if (troika.txnid[n] == txn->txnid &&
((/* is_steady */ (troika.fsm >> n) & 1) || rc != MDBX_SUCCESS)) {
if (troika.txnid[n] == txn->txnid && ((/* is_steady */ (troika.fsm >> n) & 1) || rc != MDBX_SUCCESS)) {
rc = MDBX_SUCCESS;
headcopy = meta;
} else if (troika.txnid[n] > txn->txnid)
@ -606,14 +568,12 @@ retry_snap_meta:
if (dest_is_pipe)
rc = osal_write(fd, buffer, meta_bytes);
uint8_t *const data_buffer =
buffer + ceil_powerof2(meta_bytes, globals.sys_pagesize);
uint8_t *const data_buffer = buffer + ceil_powerof2(meta_bytes, globals.sys_pagesize);
#if MDBX_USE_COPYFILERANGE
static bool copyfilerange_unavailable;
bool not_the_same_filesystem = false;
struct statfs statfs_info;
if (fstatfs(fd, &statfs_info) ||
statfs_info.f_type == /* ECRYPTFS_SUPER_MAGIC */ 0xf15f)
if (fstatfs(fd, &statfs_info) || statfs_info.f_type == /* ECRYPTFS_SUPER_MAGIC */ 0xf15f)
/* avoid use copyfilerange_unavailable() to ecryptfs due bugs */
not_the_same_filesystem = true;
#endif /* MDBX_USE_COPYFILERANGE */
@ -629,8 +589,7 @@ retry_snap_meta:
static bool sendfile_unavailable;
if (dest_is_pipe && likely(!sendfile_unavailable)) {
off_t in_offset = offset;
const ssize_t written =
sendfile(fd, env->lazy_fd, &in_offset, used_size - offset);
const ssize_t written = sendfile(fd, env->lazy_fd, &in_offset, used_size - offset);
if (likely(written > 0)) {
offset = in_offset;
if (flags & MDBX_CP_THROTTLE_MVCC)
@ -645,11 +604,9 @@ retry_snap_meta:
#endif /* MDBX_USE_SENDFILE */
#if MDBX_USE_COPYFILERANGE
if (!dest_is_pipe && !not_the_same_filesystem &&
likely(!copyfilerange_unavailable)) {
if (!dest_is_pipe && !not_the_same_filesystem && likely(!copyfilerange_unavailable)) {
off_t in_offset = offset, out_offset = offset;
ssize_t bytes_copied = copy_file_range(
env->lazy_fd, &in_offset, fd, &out_offset, used_size - offset, 0);
ssize_t bytes_copied = copy_file_range(env->lazy_fd, &in_offset, fd, &out_offset, used_size - offset, 0);
if (likely(bytes_copied > 0)) {
offset = in_offset;
if (flags & MDBX_CP_THROTTLE_MVCC)
@ -672,9 +629,8 @@ retry_snap_meta:
#endif /* MDBX_USE_COPYFILERANGE */
/* fallback to portable */
const size_t chunk = ((size_t)MDBX_ENVCOPY_WRITEBUF < used_size - offset)
? (size_t)MDBX_ENVCOPY_WRITEBUF
: used_size - offset;
const size_t chunk =
((size_t)MDBX_ENVCOPY_WRITEBUF < used_size - offset) ? (size_t)MDBX_ENVCOPY_WRITEBUF : used_size - offset;
/* copy to avoid EFAULT in case swapped-out */
memcpy(data_buffer, ptr_disp(env->dxb_mmap.base, offset), chunk);
if (flags & MDBX_CP_THROTTLE_MVCC)
@ -689,12 +645,9 @@ retry_snap_meta:
rc = osal_ftruncate(fd, whole_size);
else {
memset(data_buffer, 0, (size_t)MDBX_ENVCOPY_WRITEBUF);
for (size_t offset = used_size;
rc == MDBX_SUCCESS && offset < whole_size;) {
for (size_t offset = used_size; rc == MDBX_SUCCESS && offset < whole_size;) {
const size_t chunk =
((size_t)MDBX_ENVCOPY_WRITEBUF < whole_size - offset)
? (size_t)MDBX_ENVCOPY_WRITEBUF
: whole_size - offset;
((size_t)MDBX_ENVCOPY_WRITEBUF < whole_size - offset) ? (size_t)MDBX_ENVCOPY_WRITEBUF : whole_size - offset;
rc = osal_write(fd, data_buffer, chunk);
offset += chunk;
}
@ -706,8 +659,7 @@ retry_snap_meta:
//----------------------------------------------------------------------------
__cold static int copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd,
MDBX_copy_flags_t flags) {
__cold static int copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, MDBX_copy_flags_t flags) {
if (unlikely(txn->flags & MDBX_TXN_DIRTY))
return MDBX_BAD_TXN;
@ -734,9 +686,7 @@ __cold static int copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd,
MDBX_env *const env = txn->env;
const size_t buffer_size =
pgno_align2os_bytes(env, NUM_METAS) +
ceil_powerof2(((flags & MDBX_CP_COMPACT)
? 2 * (size_t)MDBX_ENVCOPY_WRITEBUF
: (size_t)MDBX_ENVCOPY_WRITEBUF),
ceil_powerof2(((flags & MDBX_CP_COMPACT) ? 2 * (size_t)MDBX_ENVCOPY_WRITEBUF : (size_t)MDBX_ENVCOPY_WRITEBUF),
globals.sys_pagesize);
uint8_t *buffer = nullptr;
@ -755,8 +705,7 @@ __cold static int copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd,
rc = mdbx_txn_unpark(txn, false);
if (likely(rc == MDBX_SUCCESS)) {
memset(buffer, 0, pgno2bytes(env, NUM_METAS));
rc = ((flags & MDBX_CP_COMPACT) ? copy_with_compacting : copy_asis)(
env, txn, fd, buffer, dest_is_pipe, flags);
rc = ((flags & MDBX_CP_COMPACT) ? copy_with_compacting : copy_asis)(env, txn, fd, buffer, dest_is_pipe, flags);
if (likely(rc == MDBX_SUCCESS))
rc = mdbx_txn_unpark(txn, false);
@ -785,8 +734,7 @@ __cold static int copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd,
return rc;
}
__cold static int copy2pathname(MDBX_txn *txn, const pathchar_t *dest_path,
MDBX_copy_flags_t flags) {
__cold static int copy2pathname(MDBX_txn *txn, const pathchar_t *dest_path, MDBX_copy_flags_t flags) {
if (unlikely(!dest_path || *dest_path == '\0'))
return MDBX_EINVAL;
@ -813,7 +761,7 @@ __cold static int copy2pathname(MDBX_txn *txn, const pathchar_t *dest_path,
lock_op.l_start = 0;
lock_op.l_len = OFF_T_MAX;
if (MDBX_FCNTL(newfd, MDBX_F_SETLK, &lock_op)
#if (defined(__linux__) || defined(__gnu_linux__)) && defined(LOCK_EX) && \
#if (defined(__linux__) || defined(__gnu_linux__)) && defined(LOCK_EX) && \
(!defined(__ANDROID_API__) || __ANDROID_API__ >= 24)
|| flock(newfd, LOCK_EX | LOCK_NB)
#endif /* Linux */
@ -837,8 +785,7 @@ __cold static int copy2pathname(MDBX_txn *txn, const pathchar_t *dest_path,
//----------------------------------------------------------------------------
__cold int mdbx_txn_copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd,
MDBX_copy_flags_t flags) {
__cold int mdbx_txn_copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, MDBX_copy_flags_t flags) {
int rc = check_txn(txn, MDBX_TXN_BLOCKED);
if (likely(rc == MDBX_SUCCESS))
rc = copy2fd(txn, fd, flags);
@ -847,8 +794,7 @@ __cold int mdbx_txn_copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd,
return LOG_IFERR(rc);
}
__cold int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd,
MDBX_copy_flags_t flags) {
__cold int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, MDBX_copy_flags_t flags) {
if (unlikely(flags & (MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN)))
return LOG_IFERR(MDBX_EINVAL);
@ -866,8 +812,7 @@ __cold int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd,
return LOG_IFERR(rc);
}
__cold int mdbx_txn_copy2pathname(MDBX_txn *txn, const char *dest_path,
MDBX_copy_flags_t flags) {
__cold int mdbx_txn_copy2pathname(MDBX_txn *txn, const char *dest_path, MDBX_copy_flags_t flags) {
#if defined(_WIN32) || defined(_WIN64)
wchar_t *dest_pathW = nullptr;
int rc = osal_mb2w(dest_path, &dest_pathW);
@ -878,8 +823,7 @@ __cold int mdbx_txn_copy2pathname(MDBX_txn *txn, const char *dest_path,
return LOG_IFERR(rc);
}
__cold int mdbx_txn_copy2pathnameW(MDBX_txn *txn, const wchar_t *dest_path,
MDBX_copy_flags_t flags) {
__cold int mdbx_txn_copy2pathnameW(MDBX_txn *txn, const wchar_t *dest_path, MDBX_copy_flags_t flags) {
#endif /* Windows */
int rc = check_txn(txn, MDBX_TXN_BLOCKED);
if (likely(rc == MDBX_SUCCESS))
@ -889,8 +833,7 @@ __cold int mdbx_txn_copy2pathnameW(MDBX_txn *txn, const wchar_t *dest_path,
return LOG_IFERR(rc);
}
__cold int mdbx_env_copy(MDBX_env *env, const char *dest_path,
MDBX_copy_flags_t flags) {
__cold int mdbx_env_copy(MDBX_env *env, const char *dest_path, MDBX_copy_flags_t flags) {
#if defined(_WIN32) || defined(_WIN64)
wchar_t *dest_pathW = nullptr;
int rc = osal_mb2w(dest_path, &dest_pathW);
@ -901,8 +844,7 @@ __cold int mdbx_env_copy(MDBX_env *env, const char *dest_path,
return LOG_IFERR(rc);
}
__cold int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest_path,
MDBX_copy_flags_t flags) {
__cold int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest_path, MDBX_copy_flags_t flags) {
#endif /* Windows */
if (unlikely(flags & (MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN)))
return LOG_IFERR(MDBX_EINVAL);
@ -916,8 +858,7 @@ __cold int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest_path,
if (unlikely(rc != MDBX_SUCCESS))
return LOG_IFERR(rc);
rc = copy2pathname(txn, dest_path,
flags | MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN);
rc = copy2pathname(txn, dest_path, flags | MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN);
mdbx_txn_abort(txn);
return LOG_IFERR(rc);
}

393
src/cursor.c
View File

@ -11,14 +11,11 @@ __cold int cursor_check(const MDBX_cursor *mc) {
} else {
cASSERT(mc, (mc->txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
cASSERT(mc, mc->txn->tw.dirtyroom + mc->txn->tw.dirtylist->length ==
(mc->txn->parent ? mc->txn->parent->tw.dirtyroom
: mc->txn->env->options.dp_limit));
(mc->txn->parent ? mc->txn->parent->tw.dirtyroom : mc->txn->env->options.dp_limit));
}
cASSERT(mc, (mc->checking & z_updating) ? mc->top + 1 <= mc->tree->height
: mc->top + 1 == mc->tree->height);
if (unlikely((mc->checking & z_updating) ? mc->top + 1 > mc->tree->height
: mc->top + 1 != mc->tree->height))
cASSERT(mc, (mc->checking & z_updating) ? mc->top + 1 <= mc->tree->height : mc->top + 1 == mc->tree->height);
if (unlikely((mc->checking & z_updating) ? mc->top + 1 > mc->tree->height : mc->top + 1 != mc->tree->height))
return MDBX_CURSOR_FULL;
if (is_pointed(mc) && (mc->checking & z_updating) == 0) {
@ -40,17 +37,14 @@ __cold int cursor_check(const MDBX_cursor *mc) {
page_t *mp = mc->pg[n];
const size_t nkeys = page_numkeys(mp);
const bool expect_branch = (n < mc->tree->height - 1) ? true : false;
const bool expect_nested_leaf =
(n + 1 == mc->tree->height - 1) ? true : false;
const bool expect_nested_leaf = (n + 1 == mc->tree->height - 1) ? true : false;
const bool branch = is_branch(mp) ? true : false;
cASSERT(mc, branch == expect_branch);
if (unlikely(branch != expect_branch))
return MDBX_CURSOR_FULL;
if ((mc->checking & z_updating) == 0) {
cASSERT(mc, nkeys > mc->ki[n] || (!branch && nkeys == mc->ki[n] &&
(mc->flags & z_hollow) != 0));
if (unlikely(nkeys <= mc->ki[n] && !(!branch && nkeys == mc->ki[n] &&
(mc->flags & z_hollow) != 0)))
cASSERT(mc, nkeys > mc->ki[n] || (!branch && nkeys == mc->ki[n] && (mc->flags & z_hollow) != 0));
if (unlikely(nkeys <= mc->ki[n] && !(!branch && nkeys == mc->ki[n] && (mc->flags & z_hollow) != 0)))
return MDBX_CURSOR_FULL;
} else {
cASSERT(mc, nkeys + 1 >= mc->ki[n]);
@ -96,8 +90,7 @@ __cold int cursor_check_updating(MDBX_cursor *mc) {
}
bool cursor_is_tracked(const MDBX_cursor *mc) {
for (MDBX_cursor *scan = mc->txn->cursors[cursor_dbi(mc)]; scan;
scan = scan->next)
for (MDBX_cursor *scan = mc->txn->cursors[cursor_dbi(mc)]; scan; scan = scan->next)
if (mc == ((mc->flags & z_inner) ? &scan->subcur->cursor : scan))
return true;
return false;
@ -121,16 +114,14 @@ static int touch_dbi(MDBX_cursor *mc) {
if (unlikely(rc != MDBX_SUCCESS))
return rc;
mc->txn->dbi_state[MAIN_DBI] |= DBI_DIRTY;
rc = tree_search(&cx.outer, &container_of(mc->clc, kvx_t, clc)->name,
Z_MODIFY);
rc = tree_search(&cx.outer, &container_of(mc->clc, kvx_t, clc)->name, Z_MODIFY);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
}
return MDBX_SUCCESS;
}
__hot int cursor_touch(MDBX_cursor *const mc, const MDBX_val *key,
const MDBX_val *data) {
__hot int cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, const MDBX_val *data) {
cASSERT(mc, (mc->txn->flags & MDBX_TXN_RDONLY) == 0);
cASSERT(mc, is_pointed(mc) || mc->tree->height == 0);
cASSERT(mc, cursor_is_tracked(mc));
@ -193,13 +184,10 @@ __hot int cursor_touch(MDBX_cursor *const mc, const MDBX_val *key,
/*----------------------------------------------------------------------------*/
int cursor_shadow(MDBX_cursor *parent_cursor, MDBX_txn *nested_txn,
const size_t dbi) {
int cursor_shadow(MDBX_cursor *parent_cursor, MDBX_txn *nested_txn, const size_t dbi) {
tASSERT(nested_txn, dbi > FREE_DBI && dbi < nested_txn->n_dbi);
const size_t size = parent_cursor->subcur
? sizeof(MDBX_cursor) + sizeof(subcur_t)
: sizeof(MDBX_cursor);
const size_t size = parent_cursor->subcur ? sizeof(MDBX_cursor) + sizeof(subcur_t) : sizeof(MDBX_cursor);
for (MDBX_cursor *bk; parent_cursor; parent_cursor = bk->next) {
cASSERT(parent_cursor, parent_cursor != parent_cursor->next);
bk = parent_cursor;
@ -235,8 +223,7 @@ int cursor_shadow(MDBX_cursor *parent_cursor, MDBX_txn *nested_txn,
void cursor_eot(MDBX_cursor *mc, const bool merge) {
const unsigned stage = mc->signature;
MDBX_cursor *const bk = mc->backup;
ENSURE(mc->txn->env, stage == cur_signature_live ||
(stage == cur_signature_wait4eot && bk));
ENSURE(mc->txn->env, stage == cur_signature_live || (stage == cur_signature_wait4eot && bk));
if (bk) {
subcur_t *mx = mc->subcur;
cASSERT(mc, mc->txn->parent != nullptr);
@ -274,10 +261,8 @@ void cursor_eot(MDBX_cursor *mc, const bool merge) {
/*----------------------------------------------------------------------------*/
static __always_inline int couple_init(cursor_couple_t *couple,
const MDBX_txn *const txn,
tree_t *const tree, kvx_t *const kvx,
uint8_t *const dbi_state) {
static __always_inline int couple_init(cursor_couple_t *couple, const MDBX_txn *const txn, tree_t *const tree,
kvx_t *const kvx, uint8_t *const dbi_state) {
VALGRIND_MAKE_MEM_UNDEFINED(couple, sizeof(cursor_couple_t));
tASSERT(txn, F_ISSET(*dbi_state, DBI_VALID | DBI_LINDO));
@ -290,12 +275,9 @@ static __always_inline int couple_init(cursor_couple_t *couple,
couple->outer.clc = &kvx->clc;
couple->outer.dbi_state = dbi_state;
couple->outer.top_and_flags = z_fresh_mark;
STATIC_ASSERT((int)z_branch == P_BRANCH && (int)z_leaf == P_LEAF &&
(int)z_largepage == P_LARGE && (int)z_dupfix == P_DUPFIX);
couple->outer.checking =
(AUDIT_ENABLED() || (txn->env->flags & MDBX_VALIDATION))
? z_pagecheck | z_leaf
: z_leaf;
STATIC_ASSERT((int)z_branch == P_BRANCH && (int)z_leaf == P_LEAF && (int)z_largepage == P_LARGE &&
(int)z_dupfix == P_DUPFIX);
couple->outer.checking = (AUDIT_ENABLED() || (txn->env->flags & MDBX_VALIDATION)) ? z_pagecheck | z_leaf : z_leaf;
couple->outer.subcur = nullptr;
if (tree->flags & MDBX_DUPSORT) {
@ -310,8 +292,7 @@ static __always_inline int couple_init(cursor_couple_t *couple,
mx->cursor.dbi_state = dbi_state;
mx->cursor.top_and_flags = z_fresh_mark | z_inner;
STATIC_ASSERT(MDBX_DUPFIXED * 2 == P_DUPFIX);
mx->cursor.checking =
couple->outer.checking + ((tree->flags & MDBX_DUPFIXED) << 1);
mx->cursor.checking = couple->outer.checking + ((tree->flags & MDBX_DUPFIXED) << 1);
}
if (unlikely(*dbi_state & DBI_STALE))
@ -323,8 +304,7 @@ static __always_inline int couple_init(cursor_couple_t *couple,
return MDBX_SUCCESS;
}
__cold int cursor_init4walk(cursor_couple_t *couple, const MDBX_txn *const txn,
tree_t *const tree, kvx_t *const kvx) {
__cold int cursor_init4walk(cursor_couple_t *couple, const MDBX_txn *const txn, tree_t *const tree, kvx_t *const kvx) {
return couple_init(couple, txn, tree, kvx, txn->dbi_state);
}
@ -332,21 +312,19 @@ int cursor_init(MDBX_cursor *mc, const MDBX_txn *txn, size_t dbi) {
STATIC_ASSERT(offsetof(cursor_couple_t, outer) == 0);
int rc = dbi_check(txn, dbi);
if (likely(rc == MDBX_SUCCESS))
rc = couple_init(container_of(mc, cursor_couple_t, outer), txn,
&txn->dbs[dbi], &txn->env->kvs[dbi], &txn->dbi_state[dbi]);
rc = couple_init(container_of(mc, cursor_couple_t, outer), txn, &txn->dbs[dbi], &txn->env->kvs[dbi],
&txn->dbi_state[dbi]);
return rc;
}
__cold static int unexpected_dupsort(MDBX_cursor *mc) {
ERROR("unexpected dupsort-page/node for non-dupsort db/cursor (dbi %zu)",
cursor_dbi(mc));
ERROR("unexpected dupsort-page/node for non-dupsort db/cursor (dbi %zu)", cursor_dbi(mc));
mc->txn->flags |= MDBX_TXN_ERROR;
be_poor(mc);
return MDBX_CORRUPTED;
}
int cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node,
const page_t *mp) {
int cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node, const page_t *mp) {
cASSERT(mc, is_pointed(mc));
subcur_t *mx = mc->subcur;
if (!MDBX_DISABLE_VALIDATION && unlikely(mx == nullptr))
@ -359,16 +337,13 @@ int cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node,
goto bailout;
case N_DUP | N_TREE:
if (!MDBX_DISABLE_VALIDATION && unlikely(node_ds(node) != sizeof(tree_t))) {
ERROR("invalid nested-db record size (%zu, expect %zu)", node_ds(node),
sizeof(tree_t));
ERROR("invalid nested-db record size (%zu, expect %zu)", node_ds(node), sizeof(tree_t));
goto bailout;
}
memcpy(&mx->nested_tree, node_data(node), sizeof(tree_t));
const txnid_t pp_txnid = mp->txnid;
if (!MDBX_DISABLE_VALIDATION &&
unlikely(mx->nested_tree.mod_txnid > pp_txnid)) {
ERROR("nested-db.mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")",
mx->nested_tree.mod_txnid, pp_txnid);
if (!MDBX_DISABLE_VALIDATION && unlikely(mx->nested_tree.mod_txnid > pp_txnid)) {
ERROR("nested-db.mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")", mx->nested_tree.mod_txnid, pp_txnid);
goto bailout;
}
mx->cursor.top_and_flags = z_fresh_mark | z_inner;
@ -390,25 +365,21 @@ int cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node,
mx->cursor.pg[0] = sp;
mx->cursor.ki[0] = 0;
mx->nested_tree.flags = flags_db2sub(mc->tree->flags);
mx->nested_tree.dupfix_size =
(mc->tree->flags & MDBX_DUPFIXED) ? sp->dupfix_ksize : 0;
mx->nested_tree.dupfix_size = (mc->tree->flags & MDBX_DUPFIXED) ? sp->dupfix_ksize : 0;
break;
}
if (unlikely(mx->nested_tree.dupfix_size != mc->tree->dupfix_size)) {
if (!MDBX_DISABLE_VALIDATION && unlikely(mc->tree->dupfix_size != 0)) {
ERROR("cursor mismatched nested-db dupfix_size %u",
mc->tree->dupfix_size);
ERROR("cursor mismatched nested-db dupfix_size %u", mc->tree->dupfix_size);
goto bailout;
}
if (!MDBX_DISABLE_VALIDATION &&
unlikely((mc->tree->flags & MDBX_DUPFIXED) == 0)) {
if (!MDBX_DISABLE_VALIDATION && unlikely((mc->tree->flags & MDBX_DUPFIXED) == 0)) {
ERROR("mismatched nested-db flags %u", mc->tree->flags);
goto bailout;
}
if (!MDBX_DISABLE_VALIDATION &&
unlikely(mx->nested_tree.dupfix_size < mc->clc->v.lmin ||
mx->nested_tree.dupfix_size > mc->clc->v.lmax)) {
unlikely(mx->nested_tree.dupfix_size < mc->clc->v.lmin || mx->nested_tree.dupfix_size > mc->clc->v.lmax)) {
ERROR("mismatched nested-db.dupfix_size (%u) <> min/max value-length "
"(%zu/%zu)",
mx->nested_tree.dupfix_size, mc->clc->v.lmin, mc->clc->v.lmax);
@ -418,8 +389,7 @@ int cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node,
mc->clc->v.lmin = mc->clc->v.lmax = mx->nested_tree.dupfix_size;
}
DEBUG("Sub-db dbi -%zu root page %" PRIaPGNO, cursor_dbi(&mx->cursor),
mx->nested_tree.root);
DEBUG("Sub-db dbi -%zu root page %" PRIaPGNO, cursor_dbi(&mx->cursor), mx->nested_tree.root);
return MDBX_SUCCESS;
bailout:
@ -450,14 +420,11 @@ static __always_inline int sibling(MDBX_cursor *mc, bool right) {
}
cursor_pop(mc);
DEBUG("parent page is page %" PRIaPGNO ", index %u", mc->pg[mc->top]->pgno,
mc->ki[mc->top]);
DEBUG("parent page is page %" PRIaPGNO ", index %u", mc->pg[mc->top]->pgno, mc->ki[mc->top]);
int err;
if (right ? (mc->ki[mc->top] + (size_t)1 >= page_numkeys(mc->pg[mc->top]))
: (mc->ki[mc->top] == 0)) {
DEBUG("no more keys aside, moving to next %s sibling",
right ? "right" : "left");
if (right ? (mc->ki[mc->top] + (size_t)1 >= page_numkeys(mc->pg[mc->top])) : (mc->ki[mc->top] == 0)) {
DEBUG("no more keys aside, moving to next %s sibling", right ? "right" : "left");
err = right ? cursor_sibling_right(mc) : cursor_sibling_left(mc);
if (err != MDBX_SUCCESS) {
if (likely(err == MDBX_NOTFOUND))
@ -467,8 +434,7 @@ static __always_inline int sibling(MDBX_cursor *mc, bool right) {
}
} else {
mc->ki[mc->top] += right ? 1 : -1;
DEBUG("just moving to %s index key %u", right ? "right" : "left",
mc->ki[mc->top]);
DEBUG("just moving to %s index key %u", right ? "right" : "left", mc->ki[mc->top]);
}
cASSERT(mc, is_branch(mc->pg[mc->top]));
@ -515,10 +481,8 @@ __hot int cursor_sibling_right(MDBX_cursor *mc) {
/* Функция-шаблон: Приземляет курсор на данные в текущей позиции.
* В том числе, загружает данные во вложенный курсор при его наличии. */
static __always_inline int cursor_bring(const bool inner, const bool tend2first,
MDBX_cursor *__restrict mc,
MDBX_val *__restrict key,
MDBX_val *__restrict data, bool eof) {
static __always_inline int cursor_bring(const bool inner, const bool tend2first, MDBX_cursor *__restrict mc,
MDBX_val *__restrict key, MDBX_val *__restrict data, bool eof) {
if (inner) {
cASSERT(mc, !data && !mc->subcur && (mc->flags & z_inner) != 0);
} else {
@ -527,8 +491,7 @@ static __always_inline int cursor_bring(const bool inner, const bool tend2first,
const page_t *mp = mc->pg[mc->top];
if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) {
ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor",
mp->pgno, mp->flags);
ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags);
return MDBX_CORRUPTED;
}
@ -554,8 +517,7 @@ static __always_inline int cursor_bring(const bool inner, const bool tend2first,
return err;
MDBX_ANALYSIS_ASSUME(mc->subcur != nullptr);
if (node_flags(node) & N_TREE) {
err = tend2first ? inner_first(&mc->subcur->cursor, data)
: inner_last(&mc->subcur->cursor, data);
err = tend2first ? inner_first(&mc->subcur->cursor, data) : inner_last(&mc->subcur->cursor, data);
if (unlikely(err != MDBX_SUCCESS))
return err;
} else {
@ -592,10 +554,8 @@ static __always_inline int cursor_bring(const bool inner, const bool tend2first,
}
/* Функция-шаблон: Устанавливает курсор в начало или конец. */
static __always_inline int cursor_brim(const bool inner, const bool tend2first,
MDBX_cursor *__restrict mc,
MDBX_val *__restrict key,
MDBX_val *__restrict data) {
static __always_inline int cursor_brim(const bool inner, const bool tend2first, MDBX_cursor *__restrict mc,
MDBX_val *__restrict key, MDBX_val *__restrict data) {
if (mc->top != 0) {
int err = tree_search(mc, nullptr, tend2first ? Z_FIRST : Z_LAST);
if (unlikely(err != MDBX_SUCCESS))
@ -607,13 +567,9 @@ static __always_inline int cursor_brim(const bool inner, const bool tend2first,
return cursor_bring(inner, tend2first, mc, key, data, !tend2first);
}
__hot int inner_first(MDBX_cursor *mc, MDBX_val *data) {
return cursor_brim(true, true, mc, data, nullptr);
}
__hot int inner_first(MDBX_cursor *mc, MDBX_val *data) { return cursor_brim(true, true, mc, data, nullptr); }
__hot int inner_last(MDBX_cursor *mc, MDBX_val *data) {
return cursor_brim(true, false, mc, data, nullptr);
}
__hot int inner_last(MDBX_cursor *mc, MDBX_val *data) { return cursor_brim(true, false, mc, data, nullptr); }
__hot int outer_first(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) {
return cursor_brim(false, true, mc, key, data);
@ -627,23 +583,18 @@ __hot int outer_last(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) {
/* Функция-шаблон: Передвигает курсор на одну позицию.
* При необходимости управляет вложенным курсором. */
static __always_inline int cursor_step(const bool inner, const bool forward,
MDBX_cursor *__restrict mc,
MDBX_val *__restrict key,
MDBX_val *__restrict data,
MDBX_cursor_op op) {
static __always_inline int cursor_step(const bool inner, const bool forward, MDBX_cursor *__restrict mc,
MDBX_val *__restrict key, MDBX_val *__restrict data, MDBX_cursor_op op) {
if (forward) {
if (inner)
cASSERT(mc, op == MDBX_NEXT);
else
cASSERT(mc,
op == MDBX_NEXT || op == MDBX_NEXT_DUP || op == MDBX_NEXT_NODUP);
cASSERT(mc, op == MDBX_NEXT || op == MDBX_NEXT_DUP || op == MDBX_NEXT_NODUP);
} else {
if (inner)
cASSERT(mc, op == MDBX_PREV);
else
cASSERT(mc,
op == MDBX_PREV || op == MDBX_PREV_DUP || op == MDBX_PREV_NODUP);
cASSERT(mc, op == MDBX_PREV || op == MDBX_PREV_DUP || op == MDBX_PREV_NODUP);
}
if (inner) {
cASSERT(mc, !data && !mc->subcur && (mc->flags & z_inner) != 0);
@ -668,15 +619,13 @@ static __always_inline int cursor_step(const bool inner, const bool forward,
cASSERT(mc, nkeys > 0);
intptr_t ki = mc->ki[mc->top];
const uint8_t state =
mc->flags & (z_after_delete | z_hollow | z_eof_hard | z_eof_soft);
const uint8_t state = mc->flags & (z_after_delete | z_hollow | z_eof_hard | z_eof_soft);
if (likely(state == 0)) {
cASSERT(mc, ki < nkeys);
if (!inner && op != (forward ? MDBX_NEXT_NODUP : MDBX_PREV_NODUP)) {
int err = MDBX_NOTFOUND;
if (inner_pointed(mc)) {
err = forward ? inner_next(&mc->subcur->cursor, data)
: inner_prev(&mc->subcur->cursor, data);
err = forward ? inner_next(&mc->subcur->cursor, data) : inner_prev(&mc->subcur->cursor, data);
if (likely(err == MDBX_SUCCESS)) {
get_key_optional(page_node(mp, ki), key);
return MDBX_SUCCESS;
@ -715,9 +664,8 @@ static __always_inline int cursor_step(const bool inner, const bool forward,
}
}
DEBUG("turn-%s: top page was %" PRIaPGNO " in cursor %p, ki %zi of %zi",
forward ? "next" : "prev", mp->pgno, __Wpedantic_format_voidptr(mc), ki,
nkeys);
DEBUG("turn-%s: top page was %" PRIaPGNO " in cursor %p, ki %zi of %zi", forward ? "next" : "prev", mp->pgno,
__Wpedantic_format_voidptr(mc), ki, nkeys);
if (forward) {
if (likely(++ki < nkeys))
mc->ki[mc->top] = (indx_t)ki;
@ -727,8 +675,7 @@ static __always_inline int cursor_step(const bool inner, const bool forward,
if (unlikely(err != MDBX_SUCCESS))
return err;
mp = mc->pg[mc->top];
DEBUG("next page is %" PRIaPGNO ", key index %u", mp->pgno,
mc->ki[mc->top]);
DEBUG("next page is %" PRIaPGNO ", key index %u", mp->pgno, mc->ki[mc->top]);
}
} else {
if (likely(--ki >= 0))
@ -739,47 +686,38 @@ static __always_inline int cursor_step(const bool inner, const bool forward,
if (unlikely(err != MDBX_SUCCESS))
return err;
mp = mc->pg[mc->top];
DEBUG("prev page is %" PRIaPGNO ", key index %u", mp->pgno,
mc->ki[mc->top]);
DEBUG("prev page is %" PRIaPGNO ", key index %u", mp->pgno, mc->ki[mc->top]);
}
}
DEBUG("==> cursor points to page %" PRIaPGNO " with %zu keys, key index %u",
mp->pgno, page_numkeys(mp), mc->ki[mc->top]);
DEBUG("==> cursor points to page %" PRIaPGNO " with %zu keys, key index %u", mp->pgno, page_numkeys(mp),
mc->ki[mc->top]);
bring:
return cursor_bring(inner, forward, mc, key, data, false);
}
__hot int inner_next(MDBX_cursor *mc, MDBX_val *data) {
return cursor_step(true, true, mc, data, nullptr, MDBX_NEXT);
}
__hot int inner_next(MDBX_cursor *mc, MDBX_val *data) { return cursor_step(true, true, mc, data, nullptr, MDBX_NEXT); }
__hot int inner_prev(MDBX_cursor *mc, MDBX_val *data) {
return cursor_step(true, false, mc, data, nullptr, MDBX_PREV);
}
__hot int inner_prev(MDBX_cursor *mc, MDBX_val *data) { return cursor_step(true, false, mc, data, nullptr, MDBX_PREV); }
__hot int outer_next(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
MDBX_cursor_op op) {
__hot int outer_next(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) {
return cursor_step(false, true, mc, key, data, op);
}
__hot int outer_prev(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
MDBX_cursor_op op) {
__hot int outer_prev(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) {
return cursor_step(false, false, mc, key, data, op);
}
/*----------------------------------------------------------------------------*/
__hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
unsigned flags) {
__hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags) {
int err;
DKBUF_DEBUG;
MDBX_env *const env = mc->txn->env;
if (LOG_ENABLED(MDBX_LOG_DEBUG) && (flags & MDBX_RESERVE))
data->iov_base = nullptr;
DEBUG("==> put db %d key [%s], size %" PRIuPTR ", data [%s] size %" PRIuPTR,
cursor_dbi_dbg(mc), DKEY_DEBUG(key), key->iov_len, DVAL_DEBUG(data),
data->iov_len);
DEBUG("==> put db %d key [%s], size %" PRIuPTR ", data [%s] size %" PRIuPTR, cursor_dbi_dbg(mc), DKEY_DEBUG(key),
key->iov_len, DVAL_DEBUG(data), data->iov_len);
if ((flags & MDBX_CURRENT) != 0 && (mc->flags & z_inner) == 0) {
if (unlikely(flags & (MDBX_APPEND | MDBX_NOOVERWRITE)))
@ -805,8 +743,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
/* Если за ключом более одного значения, либо если размер данных
* отличается, то вместо обновления требуется удаление и
* последующая вставка. */
if (mc->subcur->nested_tree.items > 1 ||
current_data.iov_len != data->iov_len) {
if (mc->subcur->nested_tree.items > 1 || current_data.iov_len != data->iov_len) {
drop_current:
err = cursor_del(mc, flags & MDBX_ALLDUPS);
if (unlikely(err != MDBX_SUCCESS))
@ -826,8 +763,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
goto skip_check_samedata;
}
}
if (!(flags & MDBX_RESERVE) &&
unlikely(cmp_lenfast(&current_data, data) == 0))
if (!(flags & MDBX_RESERVE) && unlikely(cmp_lenfast(&current_data, data) == 0))
return MDBX_SUCCESS /* the same data, nothing to update */;
skip_check_samedata:;
}
@ -843,8 +779,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
if ((flags & MDBX_APPEND) && mc->tree->items > 0) {
old_data.iov_base = nullptr;
old_data.iov_len = 0;
rc = (mc->flags & z_inner) ? inner_last(mc, &last_key)
: outer_last(mc, &last_key, &old_data);
rc = (mc->flags & z_inner) ? inner_last(mc, &last_key) : outer_last(mc, &last_key, &old_data);
if (likely(rc == MDBX_SUCCESS)) {
const int cmp = mc->clc->k.cmp(key, &last_key);
if (likely(cmp > 0)) {
@ -875,11 +810,10 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
if (unlikely(mc->flags & z_inner)) {
/* nested subtree of DUPSORT-database with the same key,
* nothing to update */
eASSERT(env,
data->iov_len == 0 && (old_data.iov_len == 0 ||
/* olddata may not be updated in case
DUPFIX-page of dupfix-table */
(mc->tree->flags & MDBX_DUPFIXED)));
eASSERT(env, data->iov_len == 0 && (old_data.iov_len == 0 ||
/* olddata may not be updated in case
DUPFIX-page of dupfix-table */
(mc->tree->flags & MDBX_DUPFIXED)));
return MDBX_SUCCESS;
}
if (unlikely(flags & MDBX_ALLDUPS) && inner_pointed(mc)) {
@ -936,13 +870,11 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
mc->tree->root = npr.page->pgno;
mc->tree->height++;
if (mc->tree->flags & MDBX_INTEGERKEY) {
assert(key->iov_len >= mc->clc->k.lmin &&
key->iov_len <= mc->clc->k.lmax);
assert(key->iov_len >= mc->clc->k.lmin && key->iov_len <= mc->clc->k.lmax);
mc->clc->k.lmin = mc->clc->k.lmax = key->iov_len;
}
if (mc->tree->flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) {
assert(data->iov_len >= mc->clc->v.lmin &&
data->iov_len <= mc->clc->v.lmax);
assert(data->iov_len >= mc->clc->v.lmin && data->iov_len <= mc->clc->v.lmax);
assert(mc->subcur != nullptr);
mc->tree->dupfix_size = /* mc->subcur->nested_tree.dupfix_size = */
(unsigned)(mc->clc->v.lmin = mc->clc->v.lmax = data->iov_len);
@ -966,12 +898,10 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
if (insert_key) {
/* The key does not exist */
DEBUG("inserting key at index %i", mc->ki[mc->top]);
if ((mc->tree->flags & MDBX_DUPSORT) &&
node_size(key, data) > env->leaf_nodemax) {
if ((mc->tree->flags & MDBX_DUPSORT) && node_size(key, data) > env->leaf_nodemax) {
/* Too big for a node, insert in sub-DB. Set up an empty
* "old sub-page" for convert_to_subtree to expand to a full page. */
fp->dupfix_ksize =
(mc->tree->flags & MDBX_DUPFIXED) ? (uint16_t)data->iov_len : 0;
fp->dupfix_ksize = (mc->tree->flags & MDBX_DUPFIXED) ? (uint16_t)data->iov_len : 0;
fp->lower = fp->upper = 0;
old_data.iov_len = PAGEHDRSZ;
goto convert_to_subtree;
@ -1022,9 +952,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
/* Large/Overflow page overwrites need special handling */
if (unlikely(node_flags(node) & N_BIG)) {
const size_t dpages = (node_size(key, data) > env->leaf_nodemax)
? largechunk_npages(env, data->iov_len)
: 0;
const size_t dpages = (node_size(key, data) > env->leaf_nodemax) ? largechunk_npages(env, data->iov_len) : 0;
const pgno_t pgno = node_largedata_pgno(node);
pgr_t lp = page_get_large(mc, pgno, mc->pg[mc->top]->txnid);
@ -1035,11 +963,8 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
/* Is the ov page from this txn (or a parent) and big enough? */
const size_t ovpages = lp.page->pages;
const size_t extra_threshold =
(mc->tree == &mc->txn->dbs[FREE_DBI])
? 1
: /* LY: add configurable threshold to keep reserve space */ 0;
if (!is_frozen(mc->txn, lp.page) && ovpages >= dpages &&
ovpages <= dpages + extra_threshold) {
(mc->tree == &mc->txn->dbs[FREE_DBI]) ? 1 : /* LY: add configurable threshold to keep reserve space */ 0;
if (!is_frozen(mc->txn, lp.page) && ovpages >= dpages && ovpages <= dpages + extra_threshold) {
/* yes, overwrite it. */
if (!is_modifable(mc->txn, lp.page)) {
if (is_spilled(mc->txn, lp.page)) {
@ -1052,10 +977,8 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
if (unlikely(!mc->txn->parent)) {
ERROR("Unexpected not frozen/modifiable/spilled but shadowed %s "
"page %" PRIaPGNO " mod-txnid %" PRIaTXN ","
" without parent transaction, current txn %" PRIaTXN
" front %" PRIaTXN,
"large/overflow", pgno, lp.page->txnid, mc->txn->txnid,
mc->txn->front_txnid);
" without parent transaction, current txn %" PRIaTXN " front %" PRIaTXN,
"large/overflow", pgno, lp.page->txnid, mc->txn->txnid, mc->txn->front_txnid);
return MDBX_PROBLEM;
}
@ -1094,8 +1017,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
} else {
old_data.iov_len = node_ds(node);
old_data.iov_base = node_data(node);
cASSERT(mc, ptr_disp(old_data.iov_base, old_data.iov_len) <=
ptr_disp(mc->pg[mc->top], env->ps));
cASSERT(mc, ptr_disp(old_data.iov_base, old_data.iov_len) <= ptr_disp(mc->pg[mc->top], env->ps));
/* DB has dups? */
if (mc->tree->flags & MDBX_DUPSORT) {
@ -1133,8 +1055,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
}
/* Back up original data item */
memcpy(old_singledup.iov_base = fp + 1, old_data.iov_base,
old_singledup.iov_len = old_data.iov_len);
memcpy(old_singledup.iov_base = fp + 1, old_data.iov_base, old_singledup.iov_len = old_data.iov_len);
/* Make sub-page header for the dup items, with dummy body */
fp->flags = P_LEAF | P_SUBP;
@ -1149,13 +1070,11 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
* не сразу расширять созданную под-страницу.
* Резервирование в целом сомнительно (см ниже), но может сработать
* в плюс (а если в минус то несущественный) при коротких ключах. */
xdata.iov_len += page_subleaf2_reserve(
env, page_room(mc->pg[mc->top]) + old_data.iov_len,
xdata.iov_len, data->iov_len);
xdata.iov_len +=
page_subleaf2_reserve(env, page_room(mc->pg[mc->top]) + old_data.iov_len, xdata.iov_len, data->iov_len);
cASSERT(mc, (xdata.iov_len & 1) == 0);
} else {
xdata.iov_len += 2 * (sizeof(indx_t) + NODESIZE) +
(old_data.iov_len & 1) + (data->iov_len & 1);
xdata.iov_len += 2 * (sizeof(indx_t) + NODESIZE) + (old_data.iov_len & 1) + (data->iov_len & 1);
}
cASSERT(mc, (xdata.iov_len & 1) == 0);
fp->upper = (uint16_t)(xdata.iov_len - PAGEHDRSZ);
@ -1169,9 +1088,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
fp = old_data.iov_base;
switch (flags) {
default:
growth = is_dupfix_leaf(fp)
? fp->dupfix_ksize
: (node_size(data, nullptr) + sizeof(indx_t));
growth = is_dupfix_leaf(fp) ? fp->dupfix_ksize : (node_size(data, nullptr) + sizeof(indx_t));
if (page_room(fp) >= growth) {
/* На текущей под-странице есть место для добавления элемента.
* Оптимальнее продолжить использовать эту страницу, ибо
@ -1241,9 +1158,8 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
* subpage_reserve_prereq = leaf_nodemax (1000‰).
*/
if (is_dupfix_leaf(fp))
growth += page_subleaf2_reserve(
env, page_room(mc->pg[mc->top]) + old_data.iov_len,
xdata.iov_len, data->iov_len);
growth += page_subleaf2_reserve(env, page_room(mc->pg[mc->top]) + old_data.iov_len, xdata.iov_len,
data->iov_len);
else {
/* TODO: Если добавить возможность для пользователя задавать
* min/max размеров ключей/данных, то здесь разумно реализовать
@ -1265,13 +1181,10 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
}
fp_flags = fp->flags;
if (xdata.iov_len > env->subpage_limit ||
node_size_len(node_ks(node), xdata.iov_len) > env->leaf_nodemax ||
if (xdata.iov_len > env->subpage_limit || node_size_len(node_ks(node), xdata.iov_len) > env->leaf_nodemax ||
(env->subpage_room_threshold &&
page_room(mc->pg[mc->top]) +
node_size_len(node_ks(node), old_data.iov_len) <
env->subpage_room_threshold +
node_size_len(node_ks(node), xdata.iov_len))) {
page_room(mc->pg[mc->top]) + node_size_len(node_ks(node), old_data.iov_len) <
env->subpage_room_threshold + node_size_len(node_ks(node), xdata.iov_len))) {
/* Too big for a sub-page, convert to sub-DB */
convert_to_subtree:
fp_flags &= ~P_SUBP;
@ -1310,17 +1223,13 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
cASSERT(mc, fp->upper + growth < UINT16_MAX);
mp->upper = fp->upper + (indx_t)growth;
if (unlikely(fp_flags & P_DUPFIX)) {
memcpy(page_data(mp), page_data(fp),
page_numkeys(fp) * fp->dupfix_ksize);
cASSERT(mc, (((mp->dupfix_ksize & page_numkeys(mp)) ^ mp->upper) &
1) == 0);
memcpy(page_data(mp), page_data(fp), page_numkeys(fp) * fp->dupfix_ksize);
cASSERT(mc, (((mp->dupfix_ksize & page_numkeys(mp)) ^ mp->upper) & 1) == 0);
} else {
cASSERT(mc, (mp->upper & 1) == 0);
memcpy(ptr_disp(mp, mp->upper + PAGEHDRSZ),
ptr_disp(fp, fp->upper + PAGEHDRSZ),
memcpy(ptr_disp(mp, mp->upper + PAGEHDRSZ), ptr_disp(fp, fp->upper + PAGEHDRSZ),
old_data.iov_len - fp->upper - PAGEHDRSZ);
memcpy(mp->entries, fp->entries,
page_numkeys(fp) * sizeof(mp->entries[0]));
memcpy(mp->entries, fp->entries, page_numkeys(fp) * sizeof(mp->entries[0]));
for (size_t i = 0; i < page_numkeys(fp); i++) {
cASSERT(mc, mp->entries[i] + growth <= UINT16_MAX);
mp->entries[i] += (indx_t)growth;
@ -1357,8 +1266,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
cASSERT(mc, key->iov_len < UINT16_MAX);
node_set_ks(node, key->iov_len);
memcpy(node_key(node), key->iov_base, key->iov_len);
cASSERT(mc, ptr_disp(node_key(node), node_ds(node)) <
ptr_disp(mc->pg[mc->top], env->ps));
cASSERT(mc, ptr_disp(node_key(node), node_ds(node)) < ptr_disp(mc->pg[mc->top], env->ps));
goto fix_parent;
}
@ -1377,12 +1285,9 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
insert_node:;
const unsigned naf = flags & NODE_ADD_FLAGS;
size_t nsize = is_dupfix_leaf(mc->pg[mc->top])
? key->iov_len
: leaf_size(env, key, ref_data);
size_t nsize = is_dupfix_leaf(mc->pg[mc->top]) ? key->iov_len : leaf_size(env, key, ref_data);
if (page_room(mc->pg[mc->top]) < nsize) {
rc = page_split(mc, key, ref_data, P_INVALID,
insert_key ? naf : naf | MDBX_SPLIT_REPLACE);
rc = page_split(mc, key, ref_data, P_INVALID, insert_key ? naf : naf | MDBX_SPLIT_REPLACE);
if (rc == MDBX_SUCCESS && AUDIT_ENABLED())
rc = insert_key ? cursor_check(mc) : cursor_check_updating(mc);
} else {
@ -1420,12 +1325,8 @@ insert_node:;
empty.iov_base = nullptr;
node_t *node = page_node(mc->pg[mc->top], mc->ki[mc->top]);
#define SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE 1
STATIC_ASSERT(
(MDBX_NODUPDATA >> SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE) ==
MDBX_NOOVERWRITE);
unsigned inner_flags =
MDBX_CURRENT | ((flags & MDBX_NODUPDATA) >>
SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE);
STATIC_ASSERT((MDBX_NODUPDATA >> SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE) == MDBX_NOOVERWRITE);
unsigned inner_flags = MDBX_CURRENT | ((flags & MDBX_NODUPDATA) >> SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE);
if ((flags & MDBX_CURRENT) == 0) {
inner_flags -= MDBX_CURRENT;
rc = cursor_dupsort_setup(mc, node, mc->pg[mc->top]);
@ -1434,8 +1335,7 @@ insert_node:;
}
subcur_t *const mx = mc->subcur;
if (sub_root) {
cASSERT(mc, mx->nested_tree.height == 1 &&
mx->nested_tree.root == sub_root->pgno);
cASSERT(mc, mx->nested_tree.height == 1 && mx->nested_tree.root == sub_root->pgno);
mx->cursor.flags = z_inner;
mx->cursor.top = 0;
mx->cursor.pg[0] = sub_root;
@ -1470,9 +1370,7 @@ insert_node:;
m2->subcur->cursor.top_and_flags = z_inner;
m2->subcur->cursor.ki[0] = 0;
}
DEBUG("Sub-dbi -%zu root page %" PRIaPGNO,
cursor_dbi(&m2->subcur->cursor),
m2->subcur->nested_tree.root);
DEBUG("Sub-dbi -%zu root page %" PRIaPGNO, cursor_dbi(&m2->subcur->cursor), m2->subcur->nested_tree.root);
} else if (!insert_key && m2->ki[mc->top] < nkeys)
cursor_inner_refresh(m2, mp, m2->ki[mc->top]);
}
@ -1480,10 +1378,8 @@ insert_node:;
cASSERT(mc, mc->subcur->nested_tree.items < PTRDIFF_MAX);
const size_t probe = (size_t)mc->subcur->nested_tree.items;
#define SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND 1
STATIC_ASSERT((MDBX_APPENDDUP >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND) ==
MDBX_APPEND);
inner_flags |=
(flags & MDBX_APPENDDUP) >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND;
STATIC_ASSERT((MDBX_APPENDDUP >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND) == MDBX_APPEND);
inner_flags |= (flags & MDBX_APPENDDUP) >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND;
rc = cursor_put(&mc->subcur->cursor, data, &empty, inner_flags);
if (flags & N_TREE) {
void *db = node_data(node);
@ -1530,16 +1426,13 @@ insert_node:;
return rc;
}
__hot int cursor_put_checklen(MDBX_cursor *mc, const MDBX_val *key,
MDBX_val *data, unsigned flags) {
__hot int cursor_put_checklen(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags) {
cASSERT(mc, (mc->flags & z_inner) == 0);
if (unlikely(key->iov_len > mc->clc->k.lmax ||
key->iov_len < mc->clc->k.lmin)) {
if (unlikely(key->iov_len > mc->clc->k.lmax || key->iov_len < mc->clc->k.lmin)) {
cASSERT(mc, !"Invalid key-size");
return MDBX_BAD_VALSIZE;
}
if (unlikely(data->iov_len > mc->clc->v.lmax ||
data->iov_len < mc->clc->v.lmin)) {
if (unlikely(data->iov_len > mc->clc->v.lmax || data->iov_len < mc->clc->v.lmin)) {
cASSERT(mc, !"Invalid data-size");
return MDBX_BAD_VALSIZE;
}
@ -1611,8 +1504,7 @@ __hot int cursor_del(MDBX_cursor *mc, unsigned flags) {
page_t *mp = mc->pg[mc->top];
cASSERT(mc, is_modifable(mc->txn, mp));
if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) {
ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor",
mp->pgno, mp->flags);
ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags);
return MDBX_CORRUPTED;
}
if (is_dupfix_leaf(mp))
@ -1644,8 +1536,7 @@ __hot int cursor_del(MDBX_cursor *mc, unsigned flags) {
node = node_shrink(mp, mc->ki[mc->top], node);
mc->subcur->cursor.pg[0] = node_data(node);
/* fix other sub-DB cursors pointed at sub-pages on this page */
for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2;
m2 = m2->next) {
for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) {
if (!is_related(mc, m2) || m2->pg[mc->top] != mp)
continue;
const node_t *inner = node;
@ -1664,8 +1555,7 @@ __hot int cursor_del(MDBX_cursor *mc, unsigned flags) {
}
}
mc->tree->items -= 1;
cASSERT(mc, mc->tree->items > 0 && mc->tree->height > 0 &&
mc->tree->root != P_INVALID);
cASSERT(mc, mc->tree->items > 0 && mc->tree->height > 0 && mc->tree->root != P_INVALID);
return rc;
}
/* otherwise fall thru and delete the sub-DB */
@ -1725,9 +1615,7 @@ del_key:
/* DB is totally empty now, just bail out.
* Other cursors adjustments were already done
* by rebalance and aren't needed here. */
cASSERT(mc, mc->tree->items == 0 &&
(mc->tree->root == P_INVALID ||
(is_inner(mc) && !mc->tree->root)) &&
cASSERT(mc, mc->tree->items == 0 && (mc->tree->root == P_INVALID || (is_inner(mc) && !mc->tree->root)) &&
mc->flags < 0);
return MDBX_SUCCESS;
}
@ -1736,9 +1624,7 @@ del_key:
mp = mc->pg[mc->top];
cASSERT(mc, is_leaf(mc->pg[mc->top]));
size_t nkeys = page_numkeys(mp);
cASSERT(mc,
(mc->tree->items > 0 && nkeys > 0) ||
((mc->flags & z_inner) && mc->tree->items == 0 && nkeys == 0));
cASSERT(mc, (mc->tree->items > 0 && nkeys > 0) || ((mc->flags & z_inner) && mc->tree->items == 0 && nkeys == 0));
/* Adjust this and other cursors pointing to mp */
const intptr_t top = /* может быть сброшен в -1 */ mc->top;
@ -1777,8 +1663,7 @@ del_key:
* нужно установить на первый дубликат. */
if (is_pointed(&m3->subcur->cursor)) {
if ((node_flags(node) & N_TREE) == 0) {
cASSERT(m3, m3->subcur->cursor.top == 0 &&
m3->subcur->nested_tree.height == 1);
cASSERT(m3, m3->subcur->cursor.top == 0 && m3->subcur->nested_tree.height == 1);
m3->subcur->cursor.pg[0] = node_data(node);
}
} else {
@ -1808,14 +1693,12 @@ fail:
/*----------------------------------------------------------------------------*/
__hot csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
MDBX_cursor_op op) {
__hot csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) {
DKBUF_DEBUG;
csr_t ret;
ret.exact = false;
if (unlikely(key->iov_len < mc->clc->k.lmin ||
key->iov_len > mc->clc->k.lmax)) {
if (unlikely(key->iov_len < mc->clc->k.lmin || key->iov_len > mc->clc->k.lmax)) {
cASSERT(mc, !"Invalid key-size");
ret.err = MDBX_BAD_VALSIZE;
return ret;
@ -1848,9 +1731,8 @@ __hot csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
const size_t nkeys = page_numkeys(mp);
if (unlikely(nkeys == 0)) {
/* при создании первой листовой страницы */
cASSERT(mc, mc->top == 0 && mc->tree->height == 1 &&
mc->tree->branch_pages == 0 &&
mc->tree->leaf_pages == 1 && mc->ki[0] == 0);
cASSERT(mc, mc->top == 0 && mc->tree->height == 1 && mc->tree->branch_pages == 0 && mc->tree->leaf_pages == 1 &&
mc->ki[0] == 0);
/* Логически верно, но нет смысла, ибо это мимолетная/временная
* ситуация до добавления элемента выше по стеку вызовов:
mc->flags |= z_eof_soft | z_hollow; */
@ -1901,8 +1783,7 @@ __hot csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
* первой/последний и соответственно такое сравнение было выше. */
if (mc->ki[mc->top] > 0 && mc->ki[mc->top] < nkeys - 1) {
if (is_dupfix_leaf(mp)) {
nodekey.iov_base =
page_dupfix_ptr(mp, mc->ki[mc->top], nodekey.iov_len);
nodekey.iov_base = page_dupfix_ptr(mp, mc->ki[mc->top], nodekey.iov_len);
} else {
node = page_node(mp, mc->ki[mc->top]);
nodekey = get_key(node);
@ -1928,8 +1809,7 @@ __hot csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
mc->ki[mc->top] = (indx_t)nkeys;
if (op < MDBX_SET_RANGE) {
target_not_found:
cASSERT(mc, op == MDBX_SET || op == MDBX_SET_KEY ||
op == MDBX_GET_BOTH || op == MDBX_GET_BOTH_RANGE);
cASSERT(mc, op == MDBX_SET || op == MDBX_SET_KEY || op == MDBX_GET_BOTH || op == MDBX_GET_BOTH_RANGE);
/* Операция предполагает поиск конкретного ключа, который не найден.
* Поэтому переводим курсор в неустановленное состояние, но без сброса
* top, что позволяет работать fastpath при последующем поиске по дереву
@ -1991,8 +1871,7 @@ got_node:
cASSERT(mc, is_pointed(mc) && !inner_pointed(mc));
cASSERT(mc, mc->ki[mc->top] < page_numkeys(mc->pg[mc->top]));
if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) {
ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor",
mp->pgno, mp->flags);
ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags);
ret.err = MDBX_CORRUPTED;
return ret;
}
@ -2037,8 +1916,7 @@ got_node:
}
} else if (likely(data)) {
if (op <= MDBX_GET_BOTH_RANGE) {
if (unlikely(data->iov_len < mc->clc->v.lmin ||
data->iov_len > mc->clc->v.lmax)) {
if (unlikely(data->iov_len < mc->clc->v.lmin || data->iov_len > mc->clc->v.lmax)) {
cASSERT(mc, !"Invalid data-size");
ret.err = MDBX_BAD_VALSIZE;
return ret;
@ -2049,13 +1927,11 @@ got_node:
if (aligned_data.iov_len == 8) {
if (unlikely(7 & (uintptr_t)aligned_data.iov_base))
/* copy instead of return error to avoid break compatibility */
aligned_data.iov_base =
bcopy_8(&aligned_databytes, aligned_data.iov_base);
aligned_data.iov_base = bcopy_8(&aligned_databytes, aligned_data.iov_base);
} else if (aligned_data.iov_len == 4) {
if (unlikely(3 & (uintptr_t)aligned_data.iov_base))
/* copy instead of return error to avoid break compatibility */
aligned_data.iov_base =
bcopy_4(&aligned_databytes, aligned_data.iov_base);
aligned_data.iov_base = bcopy_4(&aligned_databytes, aligned_data.iov_base);
} else {
cASSERT(mc, !"data-size is invalid for MDBX_INTEGERDUP");
ret.err = MDBX_BAD_VALSIZE;
@ -2089,18 +1965,15 @@ got_node:
if (op >= MDBX_SET_KEY)
get_key_optional(node, key);
DEBUG("==> cursor placed on key [%s], data [%s]", DKEY_DEBUG(key),
DVAL_DEBUG(data));
DEBUG("==> cursor placed on key [%s], data [%s]", DKEY_DEBUG(key), DVAL_DEBUG(data));
ret.err = MDBX_SUCCESS;
be_filled(mc);
return ret;
}
__hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
const MDBX_cursor_op op) {
__hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, const MDBX_cursor_op op) {
if (op != MDBX_GET_CURRENT)
DEBUG(">> cursor %p(0x%x), ops %u, key %p, value %p",
__Wpedantic_format_voidptr(mc), mc->flags, op,
DEBUG(">> cursor %p(0x%x), ops %u, key %p, value %p", __Wpedantic_format_voidptr(mc), mc->flags, op,
__Wpedantic_format_voidptr(key), __Wpedantic_format_voidptr(data));
int rc;
@ -2163,8 +2036,7 @@ __hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
cASSERT(mc, is_filled(mc));
else if (rc == MDBX_NOTFOUND && mc->tree->items) {
cASSERT(mc, is_pointed(mc));
cASSERT(mc, op == MDBX_SET_RANGE || op == MDBX_GET_BOTH_RANGE ||
is_hollow(mc));
cASSERT(mc, op == MDBX_SET_RANGE || op == MDBX_GET_BOTH_RANGE || is_hollow(mc));
cASSERT(mc, op == MDBX_GET_BOTH_RANGE || inner_hollow(mc));
} else
cASSERT(mc, is_poor(mc) && !is_filled(mc));
@ -2271,8 +2143,7 @@ __hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
if ((node_flags(node) & N_DUP) == 0)
return node_read(mc, node, data, mc->pg[mc->top]);
else if (MDBX_DISABLE_VALIDATION || likely(mc->subcur))
return ((op == MDBX_FIRST_DUP) ? inner_first
: inner_last)(&mc->subcur->cursor, data);
return ((op == MDBX_FIRST_DUP) ? inner_first : inner_last)(&mc->subcur->cursor, data);
else
return unexpected_dupsort(mc);
}
@ -2338,8 +2209,7 @@ __hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
rc = outer_prev(mc, key, data, MDBX_PREV_NODUP);
else if (op == MDBX_TO_KEY_GREATER_THAN)
rc = outer_next(mc, key, data, MDBX_NEXT_NODUP);
} else if (op < MDBX_TO_KEY_EQUAL &&
(rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS))
} else if (op < MDBX_TO_KEY_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS))
rc = outer_prev(mc, key, data, MDBX_PREV_NODUP);
else if (op == MDBX_TO_KEY_EQUAL && rc == MDBX_SUCCESS)
rc = MDBX_NOTFOUND;
@ -2371,8 +2241,7 @@ __hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
rc = inner_prev(mx, data);
else if (op == MDBX_TO_EXACT_KEY_VALUE_GREATER_THAN)
rc = inner_next(mx, data);
} else if (op < MDBX_TO_EXACT_KEY_VALUE_EQUAL &&
(rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS))
} else if (op < MDBX_TO_EXACT_KEY_VALUE_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS))
rc = inner_prev(mx, data);
else if (op == MDBX_TO_EXACT_KEY_VALUE_EQUAL && rc == MDBX_SUCCESS)
rc = MDBX_NOTFOUND;
@ -2425,8 +2294,7 @@ __hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
rc = outer_prev(mc, key, data, MDBX_PREV);
else if (op == MDBX_TO_PAIR_GREATER_THAN)
rc = outer_next(mc, key, data, MDBX_NEXT);
} else if (op < MDBX_TO_PAIR_EQUAL &&
(rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS))
} else if (op < MDBX_TO_PAIR_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS))
rc = outer_prev(mc, key, data, MDBX_PREV);
else if (op == MDBX_TO_PAIR_EQUAL && rc == MDBX_SUCCESS)
rc = MDBX_NOTFOUND;
@ -2458,8 +2326,7 @@ __hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
break;
}
}
} else if (op < MDBX_TO_PAIR_EQUAL &&
(rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS))
} else if (op < MDBX_TO_PAIR_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS))
rc = outer_prev(mc, key, data, MDBX_PREV_NODUP);
else if (op == MDBX_TO_PAIR_EQUAL && rc == MDBX_SUCCESS)
rc = MDBX_NOTFOUND;

144
src/cursor.h
View File

@ -125,13 +125,11 @@ enum cursor_state {
z_fresh_mark = z_poor_mark | z_fresh
};
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
is_inner(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_inner(const MDBX_cursor *mc) {
return (mc->flags & z_inner) != 0;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
is_poor(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_poor(const MDBX_cursor *mc) {
const bool r = mc->top < 0;
cASSERT(mc, r == (mc->top_and_flags < 0));
if (r && mc->subcur)
@ -139,8 +137,7 @@ is_poor(const MDBX_cursor *mc) {
return r;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
is_pointed(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_pointed(const MDBX_cursor *mc) {
const bool r = mc->top >= 0;
cASSERT(mc, r == (mc->top_and_flags >= 0));
if (!r && mc->subcur)
@ -148,49 +145,41 @@ is_pointed(const MDBX_cursor *mc) {
return r;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
is_hollow(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_hollow(const MDBX_cursor *mc) {
const bool r = mc->flags < 0;
if (!r) {
cASSERT(mc, mc->top >= 0);
cASSERT(mc, (mc->flags & z_eof_hard) ||
mc->ki[mc->top] < page_numkeys(mc->pg[mc->top]));
cASSERT(mc, (mc->flags & z_eof_hard) || mc->ki[mc->top] < page_numkeys(mc->pg[mc->top]));
} else if (mc->subcur)
cASSERT(mc, is_poor(&mc->subcur->cursor));
return r;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
is_eof(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_eof(const MDBX_cursor *mc) {
const bool r = z_eof_soft <= (uint8_t)mc->flags;
return r;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
is_filled(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_filled(const MDBX_cursor *mc) {
const bool r = z_eof_hard > (uint8_t)mc->flags;
return r;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
inner_filled(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool inner_filled(const MDBX_cursor *mc) {
return mc->subcur && is_filled(&mc->subcur->cursor);
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
inner_pointed(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool inner_pointed(const MDBX_cursor *mc) {
return mc->subcur && is_pointed(&mc->subcur->cursor);
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
inner_hollow(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool inner_hollow(const MDBX_cursor *mc) {
return !mc->subcur || is_hollow(&mc->subcur->cursor);
}
MDBX_MAYBE_UNUSED static inline void inner_gone(MDBX_cursor *mc) {
if (mc->subcur) {
TRACE("reset inner cursor %p",
__Wpedantic_format_voidptr(&mc->subcur->cursor));
TRACE("reset inner cursor %p", __Wpedantic_format_voidptr(&mc->subcur->cursor));
mc->subcur->nested_tree.root = 0;
mc->subcur->cursor.top_and_flags = z_inner | z_poor_mark;
}
@ -218,8 +207,7 @@ MDBX_MAYBE_UNUSED static inline void be_filled(MDBX_cursor *mc) {
cASSERT(mc, inner == is_inner(mc));
}
MDBX_MAYBE_UNUSED static inline bool is_related(const MDBX_cursor *base,
const MDBX_cursor *scan) {
MDBX_MAYBE_UNUSED static inline bool is_related(const MDBX_cursor *base, const MDBX_cursor *scan) {
cASSERT(base, base->top >= 0);
return base->top <= scan->top && base != scan;
}
@ -238,36 +226,30 @@ enum cursor_checking {
MDBX_INTERNAL int __must_check_result cursor_check(const MDBX_cursor *mc);
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline size_t
cursor_dbi(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline size_t cursor_dbi(const MDBX_cursor *mc) {
cASSERT(mc, mc->txn && mc->txn->signature == txn_signature);
size_t dbi = mc->dbi_state - mc->txn->dbi_state;
cASSERT(mc, dbi < mc->txn->env->n_dbi);
return dbi;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
cursor_dbi_changed(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_dbi_changed(const MDBX_cursor *mc) {
return dbi_changed(mc->txn, cursor_dbi(mc));
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t *
cursor_dbi_state(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t *cursor_dbi_state(const MDBX_cursor *mc) {
return mc->dbi_state;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
cursor_is_gc(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_is_gc(const MDBX_cursor *mc) {
return mc->dbi_state == mc->txn->dbi_state + FREE_DBI;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
cursor_is_main(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_is_main(const MDBX_cursor *mc) {
return mc->dbi_state == mc->txn->dbi_state + MAIN_DBI;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
cursor_is_core(const MDBX_cursor *mc) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_is_core(const MDBX_cursor *mc) {
return mc->dbi_state < mc->txn->dbi_state + CORE_DBS;
}
@ -277,10 +259,8 @@ MDBX_MAYBE_UNUSED static inline int cursor_dbi_dbg(const MDBX_cursor *mc) {
return (mc->flags & z_inner) ? -dbi : dbi;
}
MDBX_MAYBE_UNUSED static inline int __must_check_result
cursor_push(MDBX_cursor *mc, page_t *mp, indx_t ki) {
TRACE("pushing page %" PRIaPGNO " on db %d cursor %p", mp->pgno,
cursor_dbi_dbg(mc), __Wpedantic_format_voidptr(mc));
MDBX_MAYBE_UNUSED static inline int __must_check_result cursor_push(MDBX_cursor *mc, page_t *mp, indx_t ki) {
TRACE("pushing page %" PRIaPGNO " on db %d cursor %p", mp->pgno, cursor_dbi_dbg(mc), __Wpedantic_format_voidptr(mc));
if (unlikely(mc->top >= CURSOR_STACK_SIZE - 1)) {
be_poor(mc);
mc->txn->flags |= MDBX_TXN_ERROR;
@ -293,43 +273,32 @@ cursor_push(MDBX_cursor *mc, page_t *mp, indx_t ki) {
}
MDBX_MAYBE_UNUSED static inline void cursor_pop(MDBX_cursor *mc) {
TRACE("popped page %" PRIaPGNO " off db %d cursor %p", mc->pg[mc->top]->pgno,
cursor_dbi_dbg(mc), __Wpedantic_format_voidptr(mc));
TRACE("popped page %" PRIaPGNO " off db %d cursor %p", mc->pg[mc->top]->pgno, cursor_dbi_dbg(mc),
__Wpedantic_format_voidptr(mc));
cASSERT(mc, mc->top >= 0);
mc->top -= 1;
}
MDBX_NOTHROW_PURE_FUNCTION static inline bool
check_leaf_type(const MDBX_cursor *mc, const page_t *mp) {
return (((page_type(mp) ^ mc->checking) &
(z_branch | z_leaf | z_largepage | z_dupfix)) == 0);
MDBX_NOTHROW_PURE_FUNCTION static inline bool check_leaf_type(const MDBX_cursor *mc, const page_t *mp) {
return (((page_type(mp) ^ mc->checking) & (z_branch | z_leaf | z_largepage | z_dupfix)) == 0);
}
MDBX_INTERNAL void cursor_eot(MDBX_cursor *mc, const bool merge);
MDBX_INTERNAL int cursor_shadow(MDBX_cursor *parent_cursor,
MDBX_txn *nested_txn, const size_t dbi);
MDBX_INTERNAL int cursor_shadow(MDBX_cursor *parent_cursor, MDBX_txn *nested_txn, const size_t dbi);
MDBX_INTERNAL MDBX_cursor *cursor_cpstk(const MDBX_cursor *csrc,
MDBX_cursor *cdst);
MDBX_INTERNAL MDBX_cursor *cursor_cpstk(const MDBX_cursor *csrc, MDBX_cursor *cdst);
MDBX_INTERNAL int __must_check_result cursor_ops(MDBX_cursor *mc, MDBX_val *key,
MDBX_val *data,
MDBX_INTERNAL int __must_check_result cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
const MDBX_cursor_op op);
MDBX_INTERNAL int __must_check_result cursor_put_checklen(MDBX_cursor *mc,
const MDBX_val *key,
MDBX_val *data,
MDBX_INTERNAL int __must_check_result cursor_put_checklen(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
unsigned flags);
MDBX_INTERNAL int __must_check_result cursor_put(MDBX_cursor *mc,
const MDBX_val *key,
MDBX_val *data,
unsigned flags);
MDBX_INTERNAL int __must_check_result cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags);
MDBX_INTERNAL int __must_check_result cursor_check_updating(MDBX_cursor *mc);
MDBX_INTERNAL int __must_check_result cursor_del(MDBX_cursor *mc,
unsigned flags);
MDBX_INTERNAL int __must_check_result cursor_del(MDBX_cursor *mc, unsigned flags);
MDBX_INTERNAL int __must_check_result cursor_sibling_left(MDBX_cursor *mc);
MDBX_INTERNAL int __must_check_result cursor_sibling_right(MDBX_cursor *mc);
@ -339,56 +308,37 @@ typedef struct cursor_set_result {
bool exact;
} csr_t;
MDBX_INTERNAL csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
MDBX_cursor_op op);
MDBX_INTERNAL csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op);
MDBX_INTERNAL int __must_check_result inner_first(MDBX_cursor *__restrict mc,
MDBX_INTERNAL int __must_check_result inner_first(MDBX_cursor *__restrict mc, MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result inner_last(MDBX_cursor *__restrict mc, MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result outer_first(MDBX_cursor *__restrict mc, MDBX_val *__restrict key,
MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result inner_last(MDBX_cursor *__restrict mc,
MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result outer_first(MDBX_cursor *__restrict mc,
MDBX_val *__restrict key,
MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result outer_last(MDBX_cursor *__restrict mc,
MDBX_val *__restrict key,
MDBX_INTERNAL int __must_check_result outer_last(MDBX_cursor *__restrict mc, MDBX_val *__restrict key,
MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result inner_next(MDBX_cursor *__restrict mc,
MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result inner_prev(MDBX_cursor *__restrict mc,
MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result outer_next(MDBX_cursor *__restrict mc,
MDBX_val *__restrict key,
MDBX_val *__restrict data,
MDBX_cursor_op op);
MDBX_INTERNAL int __must_check_result outer_prev(MDBX_cursor *__restrict mc,
MDBX_val *__restrict key,
MDBX_val *__restrict data,
MDBX_cursor_op op);
MDBX_INTERNAL int __must_check_result inner_next(MDBX_cursor *__restrict mc, MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result inner_prev(MDBX_cursor *__restrict mc, MDBX_val *__restrict data);
MDBX_INTERNAL int __must_check_result outer_next(MDBX_cursor *__restrict mc, MDBX_val *__restrict key,
MDBX_val *__restrict data, MDBX_cursor_op op);
MDBX_INTERNAL int __must_check_result outer_prev(MDBX_cursor *__restrict mc, MDBX_val *__restrict key,
MDBX_val *__restrict data, MDBX_cursor_op op);
MDBX_INTERNAL int cursor_init4walk(cursor_couple_t *couple,
const MDBX_txn *const txn,
tree_t *const tree, kvx_t *const kvx);
MDBX_INTERNAL int cursor_init4walk(cursor_couple_t *couple, const MDBX_txn *const txn, tree_t *const tree,
kvx_t *const kvx);
MDBX_INTERNAL int __must_check_result cursor_init(MDBX_cursor *mc,
const MDBX_txn *txn,
size_t dbi);
MDBX_INTERNAL int __must_check_result cursor_init(MDBX_cursor *mc, const MDBX_txn *txn, size_t dbi);
MDBX_INTERNAL int __must_check_result cursor_dupsort_setup(MDBX_cursor *mc,
const node_t *node,
const page_t *mp);
MDBX_INTERNAL int __must_check_result cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node, const page_t *mp);
MDBX_INTERNAL int __must_check_result cursor_touch(MDBX_cursor *const mc,
const MDBX_val *key,
const MDBX_val *data);
MDBX_INTERNAL int __must_check_result cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, const MDBX_val *data);
/*----------------------------------------------------------------------------*/
/* Update sub-page pointer, if any, in mc->subcur.
* Needed when the node which contains the sub-page may have moved.
* Called with mp = mc->pg[mc->top], ki = mc->ki[mc->top]. */
MDBX_MAYBE_UNUSED static inline void
cursor_inner_refresh(const MDBX_cursor *mc, const page_t *mp, unsigned ki) {
MDBX_MAYBE_UNUSED static inline void cursor_inner_refresh(const MDBX_cursor *mc, const page_t *mp, unsigned ki) {
cASSERT(mc, is_leaf(mp));
const node_t *node = page_node(mp, ki);
if ((node_flags(node) & (N_DUP | N_TREE)) == N_DUP)

238
src/dbi.c
View File

@ -8,16 +8,14 @@ size_t dbi_bitmap_ctz_fallback(const MDBX_txn *txn, intptr_t bmi) {
tASSERT(txn, bmi > 0);
bmi &= -bmi;
if (sizeof(txn->dbi_sparse[0]) > 4) {
static const uint8_t debruijn_ctz64[64] = {
0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28,
62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11,
63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10,
51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12};
static const uint8_t debruijn_ctz64[64] = {0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28,
62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11,
63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10,
51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12};
return debruijn_ctz64[(UINT64_C(0x022FDD63CC95386D) * (uint64_t)bmi) >> 58];
} else {
static const uint8_t debruijn_ctz32[32] = {
0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9};
static const uint8_t debruijn_ctz32[32] = {0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9};
return debruijn_ctz32[(UINT32_C(0x077CB531) * (uint32_t)bmi) >> 27];
}
}
@ -45,8 +43,7 @@ __noinline int dbi_import(MDBX_txn *txn, const size_t dbi) {
const size_t bitmap_indx = dbi / bitmap_chunk;
const size_t bitmap_mask = (size_t)1 << dbi % bitmap_chunk;
if (dbi >= txn->n_dbi) {
for (size_t i = (txn->n_dbi + bitmap_chunk - 1) / bitmap_chunk;
bitmap_indx >= i; ++i)
for (size_t i = (txn->n_dbi + bitmap_chunk - 1) / bitmap_chunk; bitmap_indx >= i; ++i)
txn->dbi_sparse[i] = 0;
eASSERT(env, (txn->dbi_sparse[bitmap_indx] & bitmap_mask) == 0);
MDBX_txn *scan = txn;
@ -92,8 +89,7 @@ __noinline int dbi_import(MDBX_txn *txn, const size_t dbi) {
int rc = dbi_check(parent, dbi);
/* копируем состояние table очищая new-флаги. */
eASSERT(env, txn->dbi_seqs == parent->dbi_seqs);
txn->dbi_state[dbi] =
parent->dbi_state[dbi] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY);
txn->dbi_state[dbi] = parent->dbi_state[dbi] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY);
if (likely(rc == MDBX_SUCCESS)) {
txn->dbs[dbi] = parent->dbs[dbi];
if (parent->cursors[dbi]) {
@ -111,8 +107,7 @@ __noinline int dbi_import(MDBX_txn *txn, const size_t dbi) {
txn->dbi_state[dbi] = DBI_LINDO;
} else {
eASSERT(env, txn->dbi_seqs[dbi] != env->dbi_seqs[dbi].weak);
if (unlikely((txn->dbi_state[dbi] & (DBI_VALID | DBI_OLDEN)) ||
txn->cursors[dbi])) {
if (unlikely((txn->dbi_state[dbi] & (DBI_VALID | DBI_OLDEN)) || txn->cursors[dbi])) {
/* хендл уже использовался в транзакции, но был закрыт или переоткрыт,
* либо при явном пере-открытии хендла есть висячие курсоры */
eASSERT(env, (txn->dbi_state[dbi] & DBI_STALE) == 0);
@ -137,8 +132,7 @@ __noinline int dbi_import(MDBX_txn *txn, const size_t dbi) {
return MDBX_BAD_DBI;
}
static int defer_and_release(MDBX_env *const env,
defer_free_item_t *const chain) {
static int defer_and_release(MDBX_env *const env, defer_free_item_t *const chain) {
size_t length = 0;
defer_free_item_t *obsolete_chain = nullptr;
#if MDBX_ENABLE_DBI_LOCKFREE
@ -232,8 +226,7 @@ int dbi_update(MDBX_txn *txn, int keep) {
while ((env->dbs_flags[i - 1] & DB_VALID) == 0) {
--i;
eASSERT(env, i >= CORE_DBS);
eASSERT(env, !env->dbs_flags[i] && !env->kvs[i].name.iov_len &&
!env->kvs[i].name.iov_base);
eASSERT(env, !env->dbs_flags[i] && !env->kvs[i].name.iov_len && !env->kvs[i].name.iov_base);
}
env->n_dbi = (unsigned)i;
defer_and_release(env, defer_chain);
@ -241,21 +234,17 @@ int dbi_update(MDBX_txn *txn, int keep) {
return MDBX_SUCCESS;
}
int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags,
MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) {
int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) {
const MDBX_env *const env = txn->env;
eASSERT(env, dbi < txn->n_dbi && dbi < env->n_dbi);
eASSERT(env, dbi_state(txn, dbi) & DBI_LINDO);
eASSERT(env, env->dbs_flags[dbi] != DB_POISON);
if ((env->dbs_flags[dbi] & DB_VALID) == 0) {
eASSERT(env, !env->kvs[dbi].clc.k.cmp && !env->kvs[dbi].clc.v.cmp &&
!env->kvs[dbi].name.iov_len &&
!env->kvs[dbi].name.iov_base &&
!env->kvs[dbi].clc.k.lmax && !env->kvs[dbi].clc.k.lmin &&
eASSERT(env, !env->kvs[dbi].clc.k.cmp && !env->kvs[dbi].clc.v.cmp && !env->kvs[dbi].name.iov_len &&
!env->kvs[dbi].name.iov_base && !env->kvs[dbi].clc.k.lmax && !env->kvs[dbi].clc.k.lmin &&
!env->kvs[dbi].clc.v.lmax && !env->kvs[dbi].clc.v.lmin);
} else {
eASSERT(env, !(txn->dbi_state[dbi] & DBI_VALID) ||
(txn->dbs[dbi].flags | DB_VALID) == env->dbs_flags[dbi]);
eASSERT(env, !(txn->dbi_state[dbi] & DBI_VALID) || (txn->dbs[dbi].flags | DB_VALID) == env->dbs_flags[dbi]);
eASSERT(env, env->kvs[dbi].name.iov_base || dbi < CORE_DBS);
}
@ -271,8 +260,7 @@ int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags,
* 4) user_flags отличаются, но table пустая и задан флаг MDBX_CREATE
* = предполагаем что пользователь пересоздает table;
*/
if ((user_flags & ~MDBX_CREATE) !=
(unsigned)(env->dbs_flags[dbi] & DB_PERSISTENT_FLAGS)) {
if ((user_flags & ~MDBX_CREATE) != (unsigned)(env->dbs_flags[dbi] & DB_PERSISTENT_FLAGS)) {
/* flags are differs, check other conditions */
if ((!user_flags && (!keycmp || keycmp == env->kvs[dbi].clc.k.cmp) &&
(!datacmp || datacmp == env->kvs[dbi].clc.v.cmp)) ||
@ -287,11 +275,8 @@ int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags,
if (unlikely(err == MDBX_SUCCESS))
return err;
}
eASSERT(env, ((env->dbs_flags[dbi] ^ txn->dbs[dbi].flags) &
DB_PERSISTENT_FLAGS) == 0);
eASSERT(env,
(txn->dbi_state[dbi] & (DBI_LINDO | DBI_VALID | DBI_STALE)) ==
(DBI_LINDO | DBI_VALID));
eASSERT(env, ((env->dbs_flags[dbi] ^ txn->dbs[dbi].flags) & DB_PERSISTENT_FLAGS) == 0);
eASSERT(env, (txn->dbi_state[dbi] & (DBI_LINDO | DBI_VALID | DBI_STALE)) == (DBI_LINDO | DBI_VALID));
if (unlikely(txn->dbs[dbi].leaf_pages))
return /* FIXME: return extended info */ MDBX_INCOMPATIBLE;
@ -299,13 +284,11 @@ int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags,
if (unlikely(txn->cursors[dbi]))
return MDBX_DANGLING_DBI;
env->dbs_flags[dbi] = DB_POISON;
atomic_store32(&env->dbi_seqs[dbi], dbi_seq_next(env, dbi),
mo_AcquireRelease);
atomic_store32(&env->dbi_seqs[dbi], dbi_seq_next(env, dbi), mo_AcquireRelease);
const uint32_t seq = dbi_seq_next(env, dbi);
const uint16_t db_flags = user_flags & DB_PERSISTENT_FLAGS;
eASSERT(env, txn->dbs[dbi].height == 0 && txn->dbs[dbi].items == 0 &&
txn->dbs[dbi].root == P_INVALID);
eASSERT(env, txn->dbs[dbi].height == 0 && txn->dbs[dbi].items == 0 && txn->dbs[dbi].root == P_INVALID);
env->kvs[dbi].clc.k.cmp = keycmp ? keycmp : builtin_keycmp(user_flags);
env->kvs[dbi].clc.v.cmp = datacmp ? datacmp : builtin_datacmp(user_flags);
txn->dbs[dbi].flags = db_flags;
@ -325,8 +308,7 @@ int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags,
}
if (!keycmp)
keycmp = (env->dbs_flags[dbi] & DB_VALID) ? env->kvs[dbi].clc.k.cmp
: builtin_keycmp(user_flags);
keycmp = (env->dbs_flags[dbi] & DB_VALID) ? env->kvs[dbi].clc.k.cmp : builtin_keycmp(user_flags);
if (env->kvs[dbi].clc.k.cmp != keycmp) {
if (env->dbs_flags[dbi] & DB_VALID)
return MDBX_EINVAL;
@ -334,8 +316,7 @@ int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags,
}
if (!datacmp)
datacmp = (env->dbs_flags[dbi] & DB_VALID) ? env->kvs[dbi].clc.v.cmp
: builtin_datacmp(user_flags);
datacmp = (env->dbs_flags[dbi] & DB_VALID) ? env->kvs[dbi].clc.v.cmp : builtin_datacmp(user_flags);
if (env->kvs[dbi].clc.v.cmp != datacmp) {
if (env->dbs_flags[dbi] & DB_VALID)
return MDBX_EINVAL;
@ -346,19 +327,15 @@ int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags,
}
static inline size_t dbi_namelen(const MDBX_val name) {
return (name.iov_len > sizeof(defer_free_item_t)) ? name.iov_len
: sizeof(defer_free_item_t);
return (name.iov_len > sizeof(defer_free_item_t)) ? name.iov_len : sizeof(defer_free_item_t);
}
static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi,
MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp,
MDBX_val name) {
static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp,
MDBX_cmp_func *datacmp, MDBX_val name) {
MDBX_env *const env = txn->env;
/* Cannot mix named table(s) with DUPSORT flags */
tASSERT(txn,
(txn->dbi_state[MAIN_DBI] & (DBI_LINDO | DBI_VALID | DBI_STALE)) ==
(DBI_LINDO | DBI_VALID));
tASSERT(txn, (txn->dbi_state[MAIN_DBI] & (DBI_LINDO | DBI_VALID | DBI_STALE)) == (DBI_LINDO | DBI_VALID));
if (unlikely(txn->dbs[MAIN_DBI].flags & MDBX_DUPSORT)) {
if (unlikely((user_flags & MDBX_CREATE) == 0))
return MDBX_NOTFOUND;
@ -367,18 +344,15 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi,
return MDBX_INCOMPATIBLE;
/* Пересоздаём MainDB когда там пусто. */
tASSERT(txn, txn->dbs[MAIN_DBI].height == 0 &&
txn->dbs[MAIN_DBI].items == 0 &&
txn->dbs[MAIN_DBI].root == P_INVALID);
tASSERT(txn,
txn->dbs[MAIN_DBI].height == 0 && txn->dbs[MAIN_DBI].items == 0 && txn->dbs[MAIN_DBI].root == P_INVALID);
if (unlikely(txn->cursors[MAIN_DBI]))
return MDBX_DANGLING_DBI;
env->dbs_flags[MAIN_DBI] = DB_POISON;
atomic_store32(&env->dbi_seqs[MAIN_DBI], dbi_seq_next(env, MAIN_DBI),
mo_AcquireRelease);
atomic_store32(&env->dbi_seqs[MAIN_DBI], dbi_seq_next(env, MAIN_DBI), mo_AcquireRelease);
const uint32_t seq = dbi_seq_next(env, MAIN_DBI);
const uint16_t main_flags =
txn->dbs[MAIN_DBI].flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY);
const uint16_t main_flags = txn->dbs[MAIN_DBI].flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY);
env->kvs[MAIN_DBI].clc.k.cmp = builtin_keycmp(main_flags);
env->kvs[MAIN_DBI].clc.v.cmp = builtin_datacmp(main_flags);
txn->dbs[MAIN_DBI].flags = main_flags;
@ -391,8 +365,7 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi,
return err;
}
env->dbs_flags[MAIN_DBI] = main_flags | DB_VALID;
txn->dbi_seqs[MAIN_DBI] =
atomic_store32(&env->dbi_seqs[MAIN_DBI], seq, mo_AcquireRelease);
txn->dbi_seqs[MAIN_DBI] = atomic_store32(&env->dbi_seqs[MAIN_DBI], seq, mo_AcquireRelease);
txn->dbi_state[MAIN_DBI] |= DBI_DIRTY;
txn->flags |= MDBX_TXN_DIRTY;
}
@ -410,8 +383,7 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi,
if (!env->kvs[MAIN_DBI].clc.k.cmp(&name, &env->kvs[scan].name)) {
slot = scan;
int err = dbi_check(txn, slot);
if (err == MDBX_BAD_DBI &&
txn->dbi_state[slot] == (DBI_OLDEN | DBI_LINDO)) {
if (err == MDBX_BAD_DBI && txn->dbi_state[slot] == (DBI_OLDEN | DBI_LINDO)) {
/* хендл использовался, стал невалидным,
* но теперь явно пере-открывается в этой транзакци */
eASSERT(env, !txn->cursors[slot]);
@ -433,12 +405,10 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi,
return MDBX_DBS_FULL;
if (env->n_dbi == slot)
eASSERT(env, !env->dbs_flags[slot] && !env->kvs[slot].name.iov_len &&
!env->kvs[slot].name.iov_base);
eASSERT(env, !env->dbs_flags[slot] && !env->kvs[slot].name.iov_len && !env->kvs[slot].name.iov_base);
env->dbs_flags[slot] = DB_POISON;
atomic_store32(&env->dbi_seqs[slot], dbi_seq_next(env, slot),
mo_AcquireRelease);
atomic_store32(&env->dbi_seqs[slot], dbi_seq_next(env, slot), mo_AcquireRelease);
memset(&env->kvs[slot], 0, sizeof(env->kvs[slot]));
if (env->n_dbi == slot)
env->n_dbi = (unsigned)slot + 1;
@ -461,13 +431,11 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi,
return rc;
} else {
/* make sure this is actually a table */
node_t *node =
page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]);
node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]);
if (unlikely((node_flags(node) & (N_DUP | N_TREE)) != N_TREE))
return MDBX_INCOMPATIBLE;
if (!MDBX_DISABLE_VALIDATION && unlikely(body.iov_len != sizeof(tree_t))) {
ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid table node size", body.iov_len);
ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid table node size", body.iov_len);
return MDBX_CORRUPTED;
}
memcpy(&txn->dbs[slot], body.iov_base, sizeof(tree_t));
@ -490,8 +458,7 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi,
txn->dbs[slot].flags = user_flags & DB_PERSISTENT_FLAGS;
cx.outer.next = txn->cursors[MAIN_DBI];
txn->cursors[MAIN_DBI] = &cx.outer;
rc =
cursor_put_checklen(&cx.outer, &name, &body, N_TREE | MDBX_NOOVERWRITE);
rc = cursor_put_checklen(&cx.outer, &name, &body, N_TREE | MDBX_NOOVERWRITE);
txn->cursors[MAIN_DBI] = cx.outer.next;
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
@ -503,9 +470,8 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi,
/* Got info, register DBI in this txn */
const uint32_t seq = dbi_seq_next(env, slot);
eASSERT(env,
env->dbs_flags[slot] == DB_POISON && !txn->cursors[slot] &&
(txn->dbi_state[slot] & (DBI_LINDO | DBI_VALID)) == DBI_LINDO);
eASSERT(env, env->dbs_flags[slot] == DB_POISON && !txn->cursors[slot] &&
(txn->dbi_state[slot] & (DBI_LINDO | DBI_VALID)) == DBI_LINDO);
txn->dbi_state[slot] = dbi_state;
memcpy(&txn->dbs[slot], body.iov_base, sizeof(txn->dbs[slot]));
env->dbs_flags[slot] = txn->dbs[slot].flags;
@ -515,8 +481,7 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi,
env->kvs[slot].name = name;
env->dbs_flags[slot] = txn->dbs[slot].flags | DB_VALID;
txn->dbi_seqs[slot] =
atomic_store32(&env->dbi_seqs[slot], seq, mo_AcquireRelease);
txn->dbi_seqs[slot] = atomic_store32(&env->dbi_seqs[slot], seq, mo_AcquireRelease);
done:
*dbi = (MDBX_dbi)slot;
@ -525,8 +490,7 @@ done:
return MDBX_SUCCESS;
bailout:
eASSERT(env, !txn->cursors[slot] && !env->kvs[slot].name.iov_len &&
!env->kvs[slot].name.iov_base);
eASSERT(env, !txn->cursors[slot] && !env->kvs[slot].name.iov_len && !env->kvs[slot].name.iov_base);
txn->dbi_state[slot] &= DBI_LINDO | DBI_OLDEN;
env->dbs_flags[slot] = 0;
osal_free(clone);
@ -535,14 +499,13 @@ bailout:
return rc;
}
int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags,
MDBX_dbi *dbi, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) {
int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp,
MDBX_cmp_func *datacmp) {
if (unlikely(!dbi))
return MDBX_EINVAL;
*dbi = 0;
if (user_flags != MDBX_ACCEDE &&
unlikely(!check_table_flags(user_flags & ~MDBX_CREATE)))
if (user_flags != MDBX_ACCEDE && unlikely(!check_table_flags(user_flags & ~MDBX_CREATE)))
return MDBX_EINVAL;
int rc = check_txn(txn, MDBX_TXN_BLOCKED);
@ -567,8 +530,7 @@ int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags,
}
if (unlikely(name == MDBX_CHK_META || name->iov_base == MDBX_CHK_META))
return MDBX_EINVAL;
if (unlikely(name->iov_len >
txn->env->leaf_nodemax - NODESIZE - sizeof(tree_t)))
if (unlikely(name->iov_len > txn->env->leaf_nodemax - NODESIZE - sizeof(tree_t)))
return MDBX_EINVAL;
#if MDBX_ENABLE_DBI_LOCKFREE
@ -582,31 +544,24 @@ int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags,
continue;
}
const uint32_t snap_seq =
atomic_load32(&env->dbi_seqs[i], mo_AcquireRelease);
const uint32_t snap_seq = atomic_load32(&env->dbi_seqs[i], mo_AcquireRelease);
const uint16_t snap_flags = env->dbs_flags[i];
const MDBX_val snap_name = env->kvs[i].name;
if (user_flags != MDBX_ACCEDE &&
(((user_flags ^ snap_flags) & DB_PERSISTENT_FLAGS) ||
(keycmp && keycmp != env->kvs[i].clc.k.cmp) ||
(((user_flags ^ snap_flags) & DB_PERSISTENT_FLAGS) || (keycmp && keycmp != env->kvs[i].clc.k.cmp) ||
(datacmp && datacmp != env->kvs[i].clc.v.cmp)))
continue;
const uint32_t main_seq =
atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease);
const uint32_t main_seq = atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease);
MDBX_cmp_func *const snap_cmp = env->kvs[MAIN_DBI].clc.k.cmp;
if (unlikely(!(snap_flags & DB_VALID) || !snap_name.iov_base ||
!snap_name.iov_len || !snap_cmp))
if (unlikely(!(snap_flags & DB_VALID) || !snap_name.iov_base || !snap_name.iov_len || !snap_cmp))
continue;
const bool name_match = snap_cmp(&snap_name, name) == 0;
osal_flush_incoherent_cpu_writeback();
if (unlikely(
snap_seq != atomic_load32(&env->dbi_seqs[i], mo_AcquireRelease) ||
main_seq !=
atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease) ||
snap_flags != env->dbs_flags[i] ||
snap_name.iov_base != env->kvs[i].name.iov_base ||
snap_name.iov_len != env->kvs[i].name.iov_len))
if (unlikely(snap_seq != atomic_load32(&env->dbi_seqs[i], mo_AcquireRelease) ||
main_seq != atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease) ||
snap_flags != env->dbs_flags[i] || snap_name.iov_base != env->kvs[i].name.iov_base ||
snap_name.iov_len != env->kvs[i].name.iov_len))
goto retry;
if (name_match) {
rc = dbi_check(txn, i);
@ -634,18 +589,15 @@ int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags,
rc = osal_fastmutex_acquire(&txn->env->dbi_lock);
if (likely(rc == MDBX_SUCCESS)) {
rc = dbi_open_locked(txn, user_flags, dbi, keycmp, datacmp, *name);
ENSURE(txn->env,
osal_fastmutex_release(&txn->env->dbi_lock) == MDBX_SUCCESS);
ENSURE(txn->env, osal_fastmutex_release(&txn->env->dbi_lock) == MDBX_SUCCESS);
}
return rc;
}
static int dbi_open_cstr(MDBX_txn *txn, const char *name_cstr,
MDBX_db_flags_t flags, MDBX_dbi *dbi,
static int dbi_open_cstr(MDBX_txn *txn, const char *name_cstr, MDBX_db_flags_t flags, MDBX_dbi *dbi,
MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) {
MDBX_val thunk, *name;
if (name_cstr == MDBX_CHK_MAIN || name_cstr == MDBX_CHK_GC ||
name_cstr == MDBX_CHK_META)
if (name_cstr == MDBX_CHK_MAIN || name_cstr == MDBX_CHK_GC || name_cstr == MDBX_CHK_META)
name = (void *)name_cstr;
else {
thunk.iov_len = strlen(name_cstr);
@ -660,8 +612,7 @@ struct dbi_rename_result {
int err;
};
__cold static struct dbi_rename_result
dbi_rename_locked(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val new_name) {
__cold static struct dbi_rename_result dbi_rename_locked(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val new_name) {
struct dbi_rename_result pair;
pair.defer = nullptr;
pair.err = dbi_check(txn, dbi);
@ -670,8 +621,7 @@ dbi_rename_locked(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val new_name) {
MDBX_env *const env = txn->env;
MDBX_val old_name = env->kvs[dbi].name;
if (env->kvs[MAIN_DBI].clc.k.cmp(&new_name, &old_name) == 0 &&
MDBX_DEBUG == 0)
if (env->kvs[MAIN_DBI].clc.k.cmp(&new_name, &old_name) == 0 && MDBX_DEBUG == 0)
return pair;
cursor_couple_t cx;
@ -695,8 +645,7 @@ dbi_rename_locked(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val new_name) {
txn->cursors[MAIN_DBI] = &cx.outer;
MDBX_val data = {&txn->dbs[dbi], sizeof(tree_t)};
pair.err = cursor_put_checklen(&cx.outer, &new_name, &data,
N_TREE | MDBX_NOOVERWRITE);
pair.err = cursor_put_checklen(&cx.outer, &new_name, &data, N_TREE | MDBX_NOOVERWRITE);
if (likely(pair.err == MDBX_SUCCESS)) {
pair.err = cursor_seek(&cx.outer, &old_name, nullptr, MDBX_SET).err;
if (likely(pair.err == MDBX_SUCCESS))
@ -732,8 +681,7 @@ static defer_free_item_t *dbi_close_locked(MDBX_env *env, MDBX_dbi dbi) {
do {
--i;
eASSERT(env, i >= CORE_DBS);
eASSERT(env, !env->dbs_flags[i] && !env->kvs[i].name.iov_len &&
!env->kvs[i].name.iov_base);
eASSERT(env, !env->dbs_flags[i] && !env->kvs[i].name.iov_len && !env->kvs[i].name.iov_base);
} while (i > CORE_DBS && !env->kvs[i - 1].name.iov_base);
env->n_dbi = (unsigned)i;
}
@ -745,25 +693,21 @@ static defer_free_item_t *dbi_close_locked(MDBX_env *env, MDBX_dbi dbi) {
/*----------------------------------------------------------------------------*/
/* API */
int mdbx_dbi_open(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags,
MDBX_dbi *dbi) {
int mdbx_dbi_open(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, MDBX_dbi *dbi) {
return LOG_IFERR(dbi_open_cstr(txn, name, flags, dbi, nullptr, nullptr));
}
int mdbx_dbi_open2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags,
MDBX_dbi *dbi) {
int mdbx_dbi_open2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, MDBX_dbi *dbi) {
return LOG_IFERR(dbi_open(txn, name, flags, dbi, nullptr, nullptr));
}
int mdbx_dbi_open_ex(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags,
MDBX_dbi *dbi, MDBX_cmp_func *keycmp,
int mdbx_dbi_open_ex(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp,
MDBX_cmp_func *datacmp) {
return LOG_IFERR(dbi_open_cstr(txn, name, flags, dbi, keycmp, datacmp));
}
int mdbx_dbi_open_ex2(MDBX_txn *txn, const MDBX_val *name,
MDBX_db_flags_t flags, MDBX_dbi *dbi,
MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) {
int mdbx_dbi_open_ex2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp,
MDBX_cmp_func *datacmp) {
return LOG_IFERR(dbi_open(txn, name, flags, dbi, keycmp, datacmp));
}
@ -780,8 +724,7 @@ __cold int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, bool del) {
if (txn->dbs[dbi].height) {
cx.outer.next = txn->cursors[dbi];
txn->cursors[dbi] = &cx.outer;
rc = tree_drop(&cx.outer,
dbi == MAIN_DBI || (cx.outer.tree->flags & MDBX_DUPSORT));
rc = tree_drop(&cx.outer, dbi == MAIN_DBI || (cx.outer.tree->flags & MDBX_DUPSORT));
txn->cursors[dbi] = cx.outer.next;
if (unlikely(rc != MDBX_SUCCESS))
return LOG_IFERR(rc);
@ -832,8 +775,7 @@ __cold int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, bool del) {
__cold int mdbx_dbi_rename(MDBX_txn *txn, MDBX_dbi dbi, const char *name_cstr) {
MDBX_val thunk, *name;
if (name_cstr == MDBX_CHK_MAIN || name_cstr == MDBX_CHK_GC ||
name_cstr == MDBX_CHK_META)
if (name_cstr == MDBX_CHK_MAIN || name_cstr == MDBX_CHK_GC || name_cstr == MDBX_CHK_META)
name = (void *)name_cstr;
else {
thunk.iov_len = strlen(name_cstr);
@ -860,8 +802,7 @@ int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) {
rc = osal_fastmutex_acquire(&env->dbi_lock);
if (likely(rc == MDBX_SUCCESS && dbi < env->n_dbi)) {
retry:
if (env->basal_txn && (env->dbs_flags[dbi] & DB_VALID) &&
(env->basal_txn->flags & MDBX_TXN_FINISHED) == 0) {
if (env->basal_txn && (env->dbs_flags[dbi] & DB_VALID) && (env->basal_txn->flags & MDBX_TXN_FINISHED) == 0) {
/* LY: Опасный код, так как env->txn может быть изменено в другом потоке.
* К сожалению тут нет надежного решения и может быть падение при неверном
* использовании API (вызове mdbx_dbi_close конкурентно с завершением
@ -884,8 +825,7 @@ int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) {
* транзакции, и поэтому этот путь потенциально более опасен. */
const MDBX_txn *const hazard = env->txn;
osal_compiler_barrier();
if ((dbi_state(env->basal_txn, dbi) &
(DBI_LINDO | DBI_DIRTY | DBI_CREAT)) > DBI_LINDO) {
if ((dbi_state(env->basal_txn, dbi) & (DBI_LINDO | DBI_DIRTY | DBI_CREAT)) > DBI_LINDO) {
bailout_dirty_dbi:
osal_fastmutex_release(&env->dbi_lock);
return LOG_IFERR(MDBX_DANGLING_DBI);
@ -893,11 +833,9 @@ int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) {
osal_memory_barrier();
if (unlikely(hazard != env->txn))
goto retry;
if (hazard != env->basal_txn && hazard &&
(hazard->flags & MDBX_TXN_FINISHED) == 0 &&
if (hazard != env->basal_txn && hazard && (hazard->flags & MDBX_TXN_FINISHED) == 0 &&
hazard->signature == txn_signature &&
(dbi_state(hazard, dbi) & (DBI_LINDO | DBI_DIRTY | DBI_CREAT)) >
DBI_LINDO)
(dbi_state(hazard, dbi) & (DBI_LINDO | DBI_DIRTY | DBI_CREAT)) > DBI_LINDO)
goto bailout_dirty_dbi;
osal_compiler_barrier();
if (unlikely(hazard != env->txn))
@ -908,8 +846,7 @@ int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) {
return LOG_IFERR(rc);
}
int mdbx_dbi_flags_ex(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags,
unsigned *state) {
int mdbx_dbi_flags_ex(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags, unsigned *state) {
if (unlikely(!flags || !state))
return LOG_IFERR(MDBX_EINVAL);
@ -928,21 +865,17 @@ int mdbx_dbi_flags_ex(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags,
}
*flags = txn->dbs[dbi].flags & DB_PERSISTENT_FLAGS;
*state =
txn->dbi_state[dbi] & (DBI_FRESH | DBI_CREAT | DBI_DIRTY | DBI_STALE);
*state = txn->dbi_state[dbi] & (DBI_FRESH | DBI_CREAT | DBI_DIRTY | DBI_STALE);
return MDBX_SUCCESS;
}
__cold int mdbx_dbi_rename2(MDBX_txn *txn, MDBX_dbi dbi,
const MDBX_val *new_name) {
__cold int mdbx_dbi_rename2(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *new_name) {
int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return LOG_IFERR(rc);
if (unlikely(new_name == MDBX_CHK_MAIN ||
new_name->iov_base == MDBX_CHK_MAIN || new_name == MDBX_CHK_GC ||
new_name->iov_base == MDBX_CHK_GC || new_name == MDBX_CHK_META ||
new_name->iov_base == MDBX_CHK_META))
if (unlikely(new_name == MDBX_CHK_MAIN || new_name->iov_base == MDBX_CHK_MAIN || new_name == MDBX_CHK_GC ||
new_name->iov_base == MDBX_CHK_GC || new_name == MDBX_CHK_META || new_name->iov_base == MDBX_CHK_META))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(dbi < CORE_DBS))
@ -968,13 +901,11 @@ static void stat_get(const tree_t *db, MDBX_stat *st, size_t bytes) {
st->ms_leaf_pages = db->leaf_pages;
st->ms_overflow_pages = db->large_pages;
st->ms_entries = db->items;
if (likely(bytes >=
offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid)))
if (likely(bytes >= offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid)))
st->ms_mod_txnid = db->mod_txnid;
}
__cold int mdbx_dbi_stat(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *dest,
size_t bytes) {
__cold int mdbx_dbi_stat(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *dest, size_t bytes) {
if (unlikely(!dest))
return LOG_IFERR(MDBX_EINVAL);
@ -1012,8 +943,7 @@ bailout:
return LOG_IFERR(rc);
}
__cold const tree_t *dbi_dig(const MDBX_txn *txn, const size_t dbi,
tree_t *fallback) {
__cold const tree_t *dbi_dig(const MDBX_txn *txn, const size_t dbi, tree_t *fallback) {
const MDBX_txn *dig = txn;
do {
tASSERT(txn, txn->n_dbi == dig->n_dbi);
@ -1036,8 +966,7 @@ __cold const tree_t *dbi_dig(const MDBX_txn *txn, const size_t dbi,
return fallback;
}
__cold int mdbx_enumerate_tables(const MDBX_txn *txn,
MDBX_table_enum_func *func, void *ctx) {
__cold int mdbx_enumerate_tables(const MDBX_txn *txn, MDBX_table_enum_func *func, void *ctx) {
if (unlikely(!func))
return LOG_IFERR(MDBX_EINVAL);
@ -1054,13 +983,12 @@ __cold int mdbx_enumerate_tables(const MDBX_txn *txn,
txn->cursors[MAIN_DBI] = &cx.outer;
for (rc = outer_first(&cx.outer, nullptr, nullptr); rc == MDBX_SUCCESS;
rc = outer_next(&cx.outer, nullptr, nullptr, MDBX_NEXT_NODUP)) {
node_t *node =
page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]);
node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]);
if (node_flags(node) != N_TREE)
continue;
if (unlikely(node_ds(node) != sizeof(tree_t))) {
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid dupsort sub-tree node size", (unsigned)node_ds(node));
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid dupsort sub-tree node size",
(unsigned)node_ds(node));
rc = MDBX_CORRUPTED;
break;
}

67
src/dbi.h
View File

@ -7,8 +7,8 @@
#if MDBX_ENABLE_DBI_SPARSE
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL size_t
dbi_bitmap_ctz_fallback(const MDBX_txn *txn, intptr_t bmi);
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL size_t dbi_bitmap_ctz_fallback(const MDBX_txn *txn,
intptr_t bmi);
static inline size_t dbi_bitmap_ctz(const MDBX_txn *txn, intptr_t bmi) {
tASSERT(txn, bmi > 0);
@ -18,8 +18,7 @@ static inline size_t dbi_bitmap_ctz(const MDBX_txn *txn, intptr_t bmi) {
return __builtin_ctz((int)bmi);
if (sizeof(txn->dbi_sparse[0]) == sizeof(long))
return __builtin_ctzl((long)bmi);
#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || \
__has_builtin(__builtin_ctzll)
#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || __has_builtin(__builtin_ctzll)
return __builtin_ctzll(bmi);
#endif /* have(long long) && long long == uint64_t */
#endif /* GNU C */
@ -46,27 +45,27 @@ static inline size_t dbi_bitmap_ctz(const MDBX_txn *txn, intptr_t bmi) {
/* LY: Макрос целенаправленно сделан с одним циклом, чтобы сохранить возможность
* использования оператора break */
#define TXN_FOREACH_DBI_FROM(TXN, I, FROM) \
for (size_t bitmap_chunk = CHAR_BIT * sizeof(TXN->dbi_sparse[0]), \
bitmap_item = TXN->dbi_sparse[0] >> FROM, I = FROM; \
I < TXN->n_dbi; ++I) \
if (bitmap_item == 0) { \
I = (I - 1) | (bitmap_chunk - 1); \
bitmap_item = TXN->dbi_sparse[(1 + I) / bitmap_chunk]; \
if (!bitmap_item) \
I += bitmap_chunk; \
continue; \
} else if ((bitmap_item & 1) == 0) { \
size_t bitmap_skip = dbi_bitmap_ctz(txn, bitmap_item); \
bitmap_item >>= bitmap_skip; \
I += bitmap_skip - 1; \
continue; \
#define TXN_FOREACH_DBI_FROM(TXN, I, FROM) \
for (size_t bitmap_chunk = CHAR_BIT * sizeof(TXN->dbi_sparse[0]), bitmap_item = TXN->dbi_sparse[0] >> FROM, \
I = FROM; \
I < TXN->n_dbi; ++I) \
if (bitmap_item == 0) { \
I = (I - 1) | (bitmap_chunk - 1); \
bitmap_item = TXN->dbi_sparse[(1 + I) / bitmap_chunk]; \
if (!bitmap_item) \
I += bitmap_chunk; \
continue; \
} else if ((bitmap_item & 1) == 0) { \
size_t bitmap_skip = dbi_bitmap_ctz(txn, bitmap_item); \
bitmap_item >>= bitmap_skip; \
I += bitmap_skip - 1; \
continue; \
} else if (bitmap_item >>= 1, TXN->dbi_state[I])
#else
#define TXN_FOREACH_DBI_FROM(TXN, I, SKIP) \
for (size_t I = SKIP; I < TXN->n_dbi; ++I) \
#define TXN_FOREACH_DBI_FROM(TXN, I, SKIP) \
for (size_t I = SKIP; I < TXN->n_dbi; ++I) \
if (TXN->dbi_state[I])
#endif /* MDBX_ENABLE_DBI_SPARSE */
@ -80,24 +79,19 @@ struct dbi_snap_result {
uint32_t sequence;
unsigned flags;
};
MDBX_INTERNAL struct dbi_snap_result dbi_snap(const MDBX_env *env,
const size_t dbi);
MDBX_INTERNAL struct dbi_snap_result dbi_snap(const MDBX_env *env, const size_t dbi);
MDBX_INTERNAL int dbi_update(MDBX_txn *txn, int keep);
static inline uint8_t dbi_state(const MDBX_txn *txn, const size_t dbi) {
STATIC_ASSERT(
(int)DBI_DIRTY == MDBX_DBI_DIRTY && (int)DBI_STALE == MDBX_DBI_STALE &&
(int)DBI_FRESH == MDBX_DBI_FRESH && (int)DBI_CREAT == MDBX_DBI_CREAT);
STATIC_ASSERT((int)DBI_DIRTY == MDBX_DBI_DIRTY && (int)DBI_STALE == MDBX_DBI_STALE &&
(int)DBI_FRESH == MDBX_DBI_FRESH && (int)DBI_CREAT == MDBX_DBI_CREAT);
#if MDBX_ENABLE_DBI_SPARSE
const size_t bitmap_chunk = CHAR_BIT * sizeof(txn->dbi_sparse[0]);
const size_t bitmap_indx = dbi / bitmap_chunk;
const size_t bitmap_mask = (size_t)1 << dbi % bitmap_chunk;
return likely(dbi < txn->n_dbi &&
(txn->dbi_sparse[bitmap_indx] & bitmap_mask) != 0)
? txn->dbi_state[dbi]
: 0;
return likely(dbi < txn->n_dbi && (txn->dbi_sparse[bitmap_indx] & bitmap_mask) != 0) ? txn->dbi_state[dbi] : 0;
#else
return likely(dbi < txn->n_dbi) ? txn->dbi_state[dbi] : 0;
#endif /* MDBX_ENABLE_DBI_SPARSE */
@ -106,8 +100,7 @@ static inline uint8_t dbi_state(const MDBX_txn *txn, const size_t dbi) {
static inline bool dbi_changed(const MDBX_txn *txn, const size_t dbi) {
const MDBX_env *const env = txn->env;
eASSERT(env, dbi_state(txn, dbi) & DBI_LINDO);
const uint32_t snap_seq =
atomic_load32(&env->dbi_seqs[dbi], mo_AcquireRelease);
const uint32_t snap_seq = atomic_load32(&env->dbi_seqs[dbi], mo_AcquireRelease);
return snap_seq != txn->dbi_seqs[dbi];
}
@ -125,12 +118,10 @@ static inline uint32_t dbi_seq_next(const MDBX_env *const env, size_t dbi) {
return v ? v : 1;
}
MDBX_INTERNAL int dbi_open(MDBX_txn *txn, const MDBX_val *const name,
unsigned user_flags, MDBX_dbi *dbi,
MDBX_INTERNAL int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags, MDBX_dbi *dbi,
MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp);
MDBX_INTERNAL int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags,
MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp);
MDBX_INTERNAL int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, MDBX_cmp_func *keycmp,
MDBX_cmp_func *datacmp);
MDBX_INTERNAL const tree_t *dbi_dig(const MDBX_txn *txn, const size_t dbi,
tree_t *fallback);
MDBX_INTERNAL const tree_t *dbi_dig(const MDBX_txn *txn, const size_t dbi, tree_t *fallback);

18
src/debug_begin.h
View File

@ -9,28 +9,22 @@
#pragma push_macro("eASSERT")
#undef TRACE
#define TRACE(fmt, ...) \
debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__)
#define TRACE(fmt, ...) debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__)
#undef DEBUG
#define DEBUG(fmt, ...) \
debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__)
#define DEBUG(fmt, ...) debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__)
#undef VERBOSE
#define VERBOSE(fmt, ...) \
debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__)
#define VERBOSE(fmt, ...) debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__)
#undef NOTICE
#define NOTICE(fmt, ...) \
debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__)
#define NOTICE(fmt, ...) debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__)
#undef WARNING
#define WARNING(fmt, ...) \
debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__)
#define WARNING(fmt, ...) debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__)
#undef ERROR
#define ERROR(fmt, ...) \
debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__)
#define ERROR(fmt, ...) debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__)
#undef eASSERT
#define eASSERT(env, expr) ENSURE(env, expr)

104
src/dpl.c
View File

@ -9,12 +9,10 @@ static inline size_t dpl_size2bytes(ptrdiff_t size) {
size += size;
#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */
STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(dpl_t) +
(PAGELIST_LIMIT * (MDBX_DPL_PREALLOC_FOR_RADIXSORT + 1)) *
sizeof(dp_t) +
(PAGELIST_LIMIT * (MDBX_DPL_PREALLOC_FOR_RADIXSORT + 1)) * sizeof(dp_t) +
MDBX_PNL_GRANULATE * sizeof(void *) * 2 <
SIZE_MAX / 4 * 3);
size_t bytes = ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(dpl_t) +
size * sizeof(dp_t),
size_t bytes = ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(dpl_t) + size * sizeof(dp_t),
MDBX_PNL_GRANULATE * sizeof(void *) * 2) -
MDBX_ASSUME_MALLOC_OVERHEAD;
return bytes;
@ -22,8 +20,7 @@ static inline size_t dpl_size2bytes(ptrdiff_t size) {
static inline size_t dpl_bytes2size(const ptrdiff_t bytes) {
size_t size = (bytes - sizeof(dpl_t)) / sizeof(dp_t);
assert(size > CURSOR_STACK_SIZE &&
size <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE);
assert(size > CURSOR_STACK_SIZE && size <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE);
#if MDBX_DPL_PREALLOC_FOR_RADIXSORT
size >>= 1;
#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */
@ -41,8 +38,7 @@ dpl_t *dpl_reserve(MDBX_txn *txn, size_t size) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
size_t bytes =
dpl_size2bytes((size < PAGELIST_LIMIT) ? size : PAGELIST_LIMIT);
size_t bytes = dpl_size2bytes((size < PAGELIST_LIMIT) ? size : PAGELIST_LIMIT);
dpl_t *const dl = osal_realloc(txn->tw.dirtylist, bytes);
if (likely(dl)) {
#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size)
@ -59,16 +55,13 @@ int dpl_alloc(MDBX_txn *txn) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
const size_t wanna = (txn->env->options.dp_initial < txn->geo.upper)
? txn->env->options.dp_initial
: txn->geo.upper;
const size_t wanna = (txn->env->options.dp_initial < txn->geo.upper) ? txn->env->options.dp_initial : txn->geo.upper;
#if MDBX_FORCE_ASSERTIONS || MDBX_DEBUG
if (txn->tw.dirtylist)
/* обнуляем чтобы не сработал ассерт внутри dpl_reserve() */
txn->tw.dirtylist->sorted = txn->tw.dirtylist->length = 0;
#endif /* asertions enabled */
if (unlikely(!txn->tw.dirtylist || txn->tw.dirtylist->detent < wanna ||
txn->tw.dirtylist->detent > wanna + wanna) &&
if (unlikely(!txn->tw.dirtylist || txn->tw.dirtylist->detent < wanna || txn->tw.dirtylist->detent > wanna + wanna) &&
unlikely(!dpl_reserve(txn, wanna)))
return MDBX_ENOMEM;
@ -77,8 +70,7 @@ int dpl_alloc(MDBX_txn *txn) {
}
#define MDBX_DPL_EXTRACT_KEY(ptr) ((ptr)->pgno)
RADIXSORT_IMPL(dp, dp_t, MDBX_DPL_EXTRACT_KEY, MDBX_DPL_PREALLOC_FOR_RADIXSORT,
1)
RADIXSORT_IMPL(dp, dp_t, MDBX_DPL_EXTRACT_KEY, MDBX_DPL_PREALLOC_FOR_RADIXSORT, 1)
#define DP_SORT_CMP(first, last) ((first).pgno < (last).pgno)
SORT_IMPL(dp_sort, false, dp_t, DP_SORT_CMP)
@ -90,16 +82,13 @@ __hot __noinline dpl_t *dpl_sort_slowpath(const MDBX_txn *txn) {
dpl_t *dl = txn->tw.dirtylist;
assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
const size_t unsorted = dl->length - dl->sorted;
if (likely(unsorted < MDBX_RADIXSORT_THRESHOLD) ||
unlikely(!dp_radixsort(dl->items + 1, dl->length))) {
if (likely(unsorted < MDBX_RADIXSORT_THRESHOLD) || unlikely(!dp_radixsort(dl->items + 1, dl->length))) {
if (dl->sorted > unsorted / 4 + 4 &&
(MDBX_DPL_PREALLOC_FOR_RADIXSORT ||
dl->length + unsorted < dl->detent + dpl_gap_mergesort)) {
(MDBX_DPL_PREALLOC_FOR_RADIXSORT || dl->length + unsorted < dl->detent + dpl_gap_mergesort)) {
dp_t *const sorted_begin = dl->items + 1;
dp_t *const sorted_end = sorted_begin + dl->sorted;
dp_t *const end = dl->items + (MDBX_DPL_PREALLOC_FOR_RADIXSORT
? dl->length + dl->length + 1
: dl->detent + dpl_reserve_gap);
dp_t *const end =
dl->items + (MDBX_DPL_PREALLOC_FOR_RADIXSORT ? dl->length + dl->length + 1 : dl->detent + dpl_reserve_gap);
dp_t *const tmp = end - unsorted;
assert(dl->items + dl->length + 1 < tmp);
/* copy unsorted to the end of allocated space and sort it */
@ -120,19 +109,16 @@ __hot __noinline dpl_t *dpl_sort_slowpath(const MDBX_txn *txn) {
#endif
} while (likely(--w > l));
assert(r == tmp - 1);
assert(dl->items[0].pgno == 0 &&
dl->items[dl->length + 1].pgno == P_INVALID);
assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
if (ASSERT_ENABLED())
for (size_t i = 0; i <= dl->length; ++i)
assert(dl->items[i].pgno < dl->items[i + 1].pgno);
} else {
dp_sort(dl->items + 1, dl->items + dl->length + 1);
assert(dl->items[0].pgno == 0 &&
dl->items[dl->length + 1].pgno == P_INVALID);
assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
}
} else {
assert(dl->items[0].pgno == 0 &&
dl->items[dl->length + 1].pgno == P_INVALID);
assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
}
dl->sorted = dl->length;
return dl;
@ -143,8 +129,7 @@ __hot __noinline dpl_t *dpl_sort_slowpath(const MDBX_txn *txn) {
#define DP_SEARCH_CMP(dp, id) ((dp).pgno < (id))
SEARCH_IMPL(dp_bsearch, dp_t, pgno_t, DP_SEARCH_CMP)
__hot __noinline MDBX_INTERNAL size_t dpl_search(const MDBX_txn *txn,
pgno_t pgno) {
__hot __noinline MDBX_INTERNAL size_t dpl_search(const MDBX_txn *txn, pgno_t pgno) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
@ -166,10 +151,10 @@ __hot __noinline MDBX_INTERNAL size_t dpl_search(const MDBX_txn *txn,
/* whole sorted cases */
break;
#define LINEAR_SEARCH_CASE(N) \
case N: \
if (dl->items[dl->length - N + 1].pgno == pgno) \
return dl->length - N + 1; \
#define LINEAR_SEARCH_CASE(N) \
case N: \
if (dl->items[dl->length - N + 1].pgno == pgno) \
return dl->length - N + 1; \
__fallthrough
/* use linear scan until the threshold */
@ -193,8 +178,7 @@ const page_t *debug_dpl_find(const MDBX_txn *txn, const pgno_t pgno) {
const dpl_t *dl = txn->tw.dirtylist;
if (dl) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
assert(dl->items[0].pgno == 0 &&
dl->items[dl->length + 1].pgno == P_INVALID);
assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
for (size_t i = dl->length; i > dl->sorted; --i)
if (dl->items[i].pgno == pgno)
return dl->items[i].ptr;
@ -220,13 +204,11 @@ void dpl_remove_ex(const MDBX_txn *txn, size_t i, size_t npages) {
dl->pages_including_loose -= npages;
dl->sorted -= dl->sorted >= i;
dl->length -= 1;
memmove(dl->items + i, dl->items + i + 1,
(dl->length - i + 2) * sizeof(dl->items[0]));
memmove(dl->items + i, dl->items + i + 1, (dl->length - i + 2) * sizeof(dl->items[0]));
assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
}
int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page,
size_t npages) {
int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, size_t npages) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
const dp_t dp = {page, pgno, (pgno_t)npages};
@ -237,8 +219,7 @@ int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page,
dpl_t *dl = txn->tw.dirtylist;
tASSERT(txn, dl->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE);
tASSERT(txn, dl->items[0].pgno == 0 &&
dl->items[dl->length + 1].pgno == P_INVALID);
tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
if (AUDIT_ENABLED()) {
for (size_t i = dl->length; i > 0; --i) {
assert(dl->items[i].pgno != dp.pgno);
@ -254,9 +235,7 @@ int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page,
ERROR("DPL is full (PAGELIST_LIMIT %zu)", PAGELIST_LIMIT);
return MDBX_TXN_FULL;
}
const size_t size = (dl->detent < MDBX_PNL_INITIAL * 42)
? dl->detent + dl->detent
: dl->detent + dl->detent / 2;
const size_t size = (dl->detent < MDBX_PNL_INITIAL * 42) ? dl->detent + dl->detent : dl->detent + dl->detent / 2;
dl = dpl_reserve(txn, size);
if (unlikely(!dl))
return MDBX_ENOMEM;
@ -288,10 +267,7 @@ int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page,
const ptrdiff_t pivot = (ptrdiff_t)dl->length - dpl_insertion_threshold;
#if MDBX_HAVE_CMOV
const pgno_t pivot_pgno =
dl->items[(dl->length < dpl_insertion_threshold)
? 0
: dl->length - dpl_insertion_threshold]
.pgno;
dl->items[(dl->length < dpl_insertion_threshold) ? 0 : dl->length - dpl_insertion_threshold].pgno;
#endif /* MDBX_HAVE_CMOV */
/* copy the stub beyond the end */
@ -310,9 +286,7 @@ int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page,
while (i >= dl->items + dl->sorted) {
#if !defined(__GNUC__) /* пытаемся избежать вызова memmove() */
i[1] = *i;
#elif MDBX_WORDBITS == 64 && \
(defined(__SIZEOF_INT128__) || \
(defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128))
#elif MDBX_WORDBITS == 64 && (defined(__SIZEOF_INT128__) || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128))
STATIC_ASSERT(sizeof(dp) == sizeof(__uint128_t));
((__uint128_t *)i)[1] = *(volatile __uint128_t *)i;
#else
@ -347,9 +321,8 @@ __cold bool dpl_check(MDBX_txn *txn) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
tASSERT(txn, txn->tw.dirtyroom + dl->length ==
(txn->parent ? txn->parent->tw.dirtyroom
: txn->env->options.dp_limit));
tASSERT(txn,
txn->tw.dirtyroom + dl->length == (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
if (!AUDIT_ENABLED())
return true;
@ -389,16 +362,12 @@ __cold bool dpl_check(MDBX_txn *txn) {
return false;
}
const size_t rpa =
pnl_search(txn->tw.relist, dp->pgno, txn->geo.first_unallocated);
tASSERT(txn, rpa > MDBX_PNL_GETSIZE(txn->tw.relist) ||
txn->tw.relist[rpa] != dp->pgno);
if (rpa <= MDBX_PNL_GETSIZE(txn->tw.relist) &&
unlikely(txn->tw.relist[rpa] == dp->pgno))
const size_t rpa = pnl_search(txn->tw.relist, dp->pgno, txn->geo.first_unallocated);
tASSERT(txn, rpa > MDBX_PNL_GETSIZE(txn->tw.relist) || txn->tw.relist[rpa] != dp->pgno);
if (rpa <= MDBX_PNL_GETSIZE(txn->tw.relist) && unlikely(txn->tw.relist[rpa] == dp->pgno))
return false;
if (num > 1) {
const size_t rpb = pnl_search(txn->tw.relist, dp->pgno + num - 1,
txn->geo.first_unallocated);
const size_t rpb = pnl_search(txn->tw.relist, dp->pgno + num - 1, txn->geo.first_unallocated);
tASSERT(txn, rpa == rpb);
if (unlikely(rpa != rpb))
return false;
@ -432,8 +401,7 @@ __noinline void dpl_lru_reduce(MDBX_txn *txn) {
txn->tw.dirtylru >>= 1;
dpl_t *dl = txn->tw.dirtylist;
for (size_t i = 1; i <= dl->length; ++i) {
size_t *const ptr =
ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t));
size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t));
*ptr >>= 1;
}
txn = txn->parent;
@ -444,8 +412,7 @@ void dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
if (MDBX_PNL_GETSIZE(pl) && txn->tw.dirtylist->length) {
tASSERT(txn, pnl_check_allocated(pl, (size_t)txn->geo.first_unallocated
<< spilled));
tASSERT(txn, pnl_check_allocated(pl, (size_t)txn->geo.first_unallocated << spilled));
dpl_t *dl = dpl_sort(txn);
/* Scanning in ascend order */
@ -501,8 +468,7 @@ void dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled) {
dl->sorted = dpl_setlen(dl, w - 1);
txn->tw.dirtyroom += r - w;
tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
(txn->parent ? txn->parent->tw.dirtyroom
: txn->env->options.dp_limit));
(txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
return;
}
}

33
src/dpl.h
View File

@ -49,38 +49,32 @@ static inline dpl_t *dpl_sort(const MDBX_txn *txn) {
dpl_t *dl = txn->tw.dirtylist;
tASSERT(txn, dl->length <= PAGELIST_LIMIT);
tASSERT(txn, dl->sorted <= dl->length);
tASSERT(txn, dl->items[0].pgno == 0 &&
dl->items[dl->length + 1].pgno == P_INVALID);
tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
return likely(dl->sorted == dl->length) ? dl : dpl_sort_slowpath(txn);
}
MDBX_INTERNAL __noinline size_t dpl_search(const MDBX_txn *txn, pgno_t pgno);
MDBX_MAYBE_UNUSED MDBX_INTERNAL const page_t *
debug_dpl_find(const MDBX_txn *txn, const pgno_t pgno);
MDBX_MAYBE_UNUSED MDBX_INTERNAL const page_t *debug_dpl_find(const MDBX_txn *txn, const pgno_t pgno);
MDBX_NOTHROW_PURE_FUNCTION static inline unsigned dpl_npages(const dpl_t *dl,
size_t i) {
MDBX_NOTHROW_PURE_FUNCTION static inline unsigned dpl_npages(const dpl_t *dl, size_t i) {
assert(0 <= (intptr_t)i && i <= dl->length);
unsigned n = dl->items[i].npages;
assert(n == (is_largepage(dl->items[i].ptr) ? dl->items[i].ptr->pages : 1));
return n;
}
MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t dpl_endpgno(const dpl_t *dl,
size_t i) {
MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t dpl_endpgno(const dpl_t *dl, size_t i) {
return dpl_npages(dl, i) + dl->items[i].pgno;
}
static inline bool dpl_intersect(const MDBX_txn *txn, pgno_t pgno,
size_t npages) {
static inline bool dpl_intersect(const MDBX_txn *txn, pgno_t pgno, size_t npages) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
dpl_t *dl = txn->tw.dirtylist;
tASSERT(txn, dl->sorted == dl->length);
tASSERT(txn, dl->items[0].pgno == 0 &&
dl->items[dl->length + 1].pgno == P_INVALID);
tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID);
size_t const n = dpl_search(txn, pgno);
tASSERT(txn, n >= 1 && n <= dl->length + 1);
tASSERT(txn, pgno <= dl->items[n].pgno);
@ -92,8 +86,7 @@ static inline bool dpl_intersect(const MDBX_txn *txn, pgno_t pgno,
bool check = false;
for (size_t i = 1; i <= dl->length; ++i) {
const page_t *const dp = dl->items[i].ptr;
if (!(dp->pgno /* begin */ >= /* end */ pgno + npages ||
dpl_endpgno(dl, i) /* end */ <= /* begin */ pgno))
if (!(dp->pgno /* begin */ >= /* end */ pgno + npages || dpl_endpgno(dl, i) /* end */ <= /* begin */ pgno))
check |= true;
}
tASSERT(txn, check == rc);
@ -101,8 +94,7 @@ static inline bool dpl_intersect(const MDBX_txn *txn, pgno_t pgno,
return rc;
}
MDBX_NOTHROW_PURE_FUNCTION static inline size_t dpl_exist(const MDBX_txn *txn,
pgno_t pgno) {
MDBX_NOTHROW_PURE_FUNCTION static inline size_t dpl_exist(const MDBX_txn *txn, pgno_t pgno) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
dpl_t *dl = txn->tw.dirtylist;
size_t i = dpl_search(txn, pgno);
@ -116,13 +108,11 @@ static inline void dpl_remove(const MDBX_txn *txn, size_t i) {
dpl_remove_ex(txn, i, dpl_npages(txn->tw.dirtylist, i));
}
MDBX_INTERNAL int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno,
page_t *page, size_t npages);
MDBX_INTERNAL int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, size_t npages);
MDBX_MAYBE_UNUSED MDBX_INTERNAL bool dpl_check(MDBX_txn *txn);
MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t dpl_age(const MDBX_txn *txn,
size_t i) {
MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t dpl_age(const MDBX_txn *txn, size_t i) {
tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0);
const dpl_t *dl = txn->tw.dirtylist;
assert((intptr_t)i > 0 && i <= dl->length);
@ -134,8 +124,7 @@ MDBX_INTERNAL void dpl_lru_reduce(MDBX_txn *txn);
static inline uint32_t dpl_lru_turn(MDBX_txn *txn) {
txn->tw.dirtylru += 1;
if (unlikely(txn->tw.dirtylru > UINT32_MAX / 3) &&
(txn->flags & MDBX_WRITEMAP) == 0)
if (unlikely(txn->tw.dirtylru > UINT32_MAX / 3) && (txn->flags & MDBX_WRITEMAP) == 0)
dpl_lru_reduce(txn);
return txn->tw.dirtylru;
}

712
src/dxb.c
View File

File diff suppressed because it is too large Load Diff

54
src/env-opts.c
View File

@ -6,17 +6,11 @@
__cold static unsigned default_rp_augment_limit(const MDBX_env *env) {
const size_t timeframe = /* 16 секунд */ 16 << 16;
const size_t remain_1sec =
(env->options.gc_time_limit < timeframe)
? timeframe - (size_t)env->options.gc_time_limit
: 0;
const size_t minimum = (env->maxgc_large1page * 2 > MDBX_PNL_INITIAL)
? env->maxgc_large1page * 2
: MDBX_PNL_INITIAL;
(env->options.gc_time_limit < timeframe) ? timeframe - (size_t)env->options.gc_time_limit : 0;
const size_t minimum = (env->maxgc_large1page * 2 > MDBX_PNL_INITIAL) ? env->maxgc_large1page * 2 : MDBX_PNL_INITIAL;
const size_t one_third = env->geo_in_bytes.now / 3 >> env->ps2ln;
const size_t augment_limit =
(one_third > minimum)
? minimum + (one_third - minimum) / timeframe * remain_1sec
: minimum;
(one_third > minimum) ? minimum + (one_third - minimum) / timeframe * remain_1sec : minimum;
eASSERT(env, augment_limit < PAGELIST_LIMIT);
return pnl_bytes2size(pnl_size2bytes(augment_limit));
}
@ -86,29 +80,23 @@ void env_options_adjust_defaults(MDBX_env *env) {
const size_t basis = env->geo_in_bytes.now;
/* TODO: use options? */
const unsigned factor = 9;
size_t threshold = (basis < ((size_t)65536 << factor))
? 65536 /* minimal threshold */
: (basis > (MEGABYTE * 4 << factor))
? MEGABYTE * 4 /* maximal threshold */
: basis >> factor;
size_t threshold = (basis < ((size_t)65536 << factor)) ? 65536 /* minimal threshold */
: (basis > (MEGABYTE * 4 << factor)) ? MEGABYTE * 4 /* maximal threshold */
: basis >> factor;
threshold =
(threshold < env->geo_in_bytes.shrink || !env->geo_in_bytes.shrink)
? threshold
: env->geo_in_bytes.shrink;
(threshold < env->geo_in_bytes.shrink || !env->geo_in_bytes.shrink) ? threshold : env->geo_in_bytes.shrink;
env->madv_threshold = bytes2pgno(env, bytes_align2os_bytes(env, threshold));
}
//------------------------------------------------------------------------------
__cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option,
uint64_t value) {
__cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, uint64_t value) {
int err = check_env(env, false);
if (unlikely(err != MDBX_SUCCESS))
return LOG_IFERR(err);
const bool lock_needed =
((env->flags & ENV_ACTIVE) && env->basal_txn && !env_txn0_owned(env));
const bool lock_needed = ((env->flags & ENV_ACTIVE) && env->basal_txn && !env_txn0_owned(env));
bool should_unlock = false;
switch (option) {
case MDBX_opt_sync_bytes:
@ -121,10 +109,8 @@ __cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option,
if (unlikely(value > SIZE_MAX - 65536))
return LOG_IFERR(MDBX_EINVAL);
value = bytes2pgno(env, (size_t)value + env->ps - 1);
if ((uint32_t)value !=
atomic_load32(&env->lck->autosync_threshold, mo_AcquireRelease) &&
atomic_store32(&env->lck->autosync_threshold, (uint32_t)value,
mo_Relaxed)
if ((uint32_t)value != atomic_load32(&env->lck->autosync_threshold, mo_AcquireRelease) &&
atomic_store32(&env->lck->autosync_threshold, (uint32_t)value, mo_Relaxed)
/* Дергаем sync(force=off) только если задано новое не-нулевое значение
* и мы вне транзакции */
&& lock_needed) {
@ -248,8 +234,7 @@ __cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option,
err = MDBX_EPERM /* unable change during transaction */;
else {
const pgno_t value32 = (pgno_t)value;
if (option == MDBX_opt_txn_dp_initial &&
env->options.dp_initial != value32) {
if (option == MDBX_opt_txn_dp_initial && env->options.dp_initial != value32) {
env->options.dp_initial = value32;
if (env->options.dp_limit < value32) {
env->options.dp_limit = value32;
@ -308,8 +293,7 @@ __cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option,
#if defined(_WIN32) || defined(_WIN64)
/* позволяем "установить" значение по-умолчанию и совпадающее
* с поведением соответствующим текущей установке MDBX_NOMETASYNC */
if (value == /* default */ UINT64_MAX &&
value != ((env->flags & MDBX_NOMETASYNC) ? 0 : UINT_MAX))
if (value == /* default */ UINT64_MAX && value != ((env->flags & MDBX_NOMETASYNC) ? 0 : UINT_MAX))
err = MDBX_EINVAL;
#else
if (value == /* default */ UINT64_MAX)
@ -335,8 +319,7 @@ __cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option,
case MDBX_opt_prefer_waf_insteadof_balance:
if (value == /* default */ UINT64_MAX)
env->options.prefer_waf_insteadof_balance =
default_prefer_waf_insteadof_balance(env);
env->options.prefer_waf_insteadof_balance = default_prefer_waf_insteadof_balance(env);
else if (value > 1)
err = MDBX_EINVAL;
else
@ -400,8 +383,7 @@ __cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option,
return LOG_IFERR(err);
}
__cold int mdbx_env_get_option(const MDBX_env *env, const MDBX_option_t option,
uint64_t *pvalue) {
__cold int mdbx_env_get_option(const MDBX_env *env, const MDBX_option_t option, uint64_t *pvalue) {
int err = check_env(env, false);
if (unlikely(err != MDBX_SUCCESS))
return LOG_IFERR(err);
@ -412,15 +394,13 @@ __cold int mdbx_env_get_option(const MDBX_env *env, const MDBX_option_t option,
case MDBX_opt_sync_bytes:
if (unlikely(!(env->flags & ENV_ACTIVE)))
return LOG_IFERR(MDBX_EPERM);
*pvalue = pgno2bytes(
env, atomic_load32(&env->lck->autosync_threshold, mo_Relaxed));
*pvalue = pgno2bytes(env, atomic_load32(&env->lck->autosync_threshold, mo_Relaxed));
break;
case MDBX_opt_sync_period:
if (unlikely(!(env->flags & ENV_ACTIVE)))
return LOG_IFERR(MDBX_EPERM);
*pvalue = osal_monotime_to_16dot16(
atomic_load64(&env->lck->autosync_period, mo_Relaxed));
*pvalue = osal_monotime_to_16dot16(atomic_load64(&env->lck->autosync_period, mo_Relaxed));
break;
case MDBX_opt_max_db:

161
src/env.c
View File

@ -4,17 +4,14 @@
#include "internals.h"
bool env_txn0_owned(const MDBX_env *env) {
return (env->flags & MDBX_NOSTICKYTHREADS)
? (env->basal_txn->owner != 0)
: (env->basal_txn->owner == osal_thread_self());
return (env->flags & MDBX_NOSTICKYTHREADS) ? (env->basal_txn->owner != 0)
: (env->basal_txn->owner == osal_thread_self());
}
int env_page_auxbuffer(MDBX_env *env) {
const int err =
env->page_auxbuf
? MDBX_SUCCESS
: osal_memalign_alloc(globals.sys_pagesize,
env->ps * (size_t)NUM_METAS, &env->page_auxbuf);
const int err = env->page_auxbuf
? MDBX_SUCCESS
: osal_memalign_alloc(globals.sys_pagesize, env->ps * (size_t)NUM_METAS, &env->page_auxbuf);
if (likely(err == MDBX_SUCCESS)) {
memset(env->page_auxbuf, -1, env->ps * (size_t)2);
memset(ptr_disp(env->page_auxbuf, env->ps * (size_t)2), 0, env->ps);
@ -34,26 +31,19 @@ __cold unsigned env_setup_pagesize(MDBX_env *env, const size_t pagesize) {
STATIC_ASSERT(MAX_GC1OVPAGE(MDBX_MIN_PAGESIZE) > 4);
STATIC_ASSERT(MAX_GC1OVPAGE(MDBX_MAX_PAGESIZE) < PAGELIST_LIMIT);
const intptr_t maxgc_ov1page = (pagesize - PAGEHDRSZ) / sizeof(pgno_t) - 1;
ENSURE(env,
maxgc_ov1page > 42 && maxgc_ov1page < (intptr_t)PAGELIST_LIMIT / 4);
ENSURE(env, maxgc_ov1page > 42 && maxgc_ov1page < (intptr_t)PAGELIST_LIMIT / 4);
env->maxgc_large1page = (unsigned)maxgc_ov1page;
env->maxgc_per_branch =
(unsigned)((pagesize - PAGEHDRSZ) /
(sizeof(indx_t) + sizeof(node_t) + sizeof(txnid_t)));
env->maxgc_per_branch = (unsigned)((pagesize - PAGEHDRSZ) / (sizeof(indx_t) + sizeof(node_t) + sizeof(txnid_t)));
STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) >
sizeof(tree_t) + NODESIZE + 42);
STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) > sizeof(tree_t) + NODESIZE + 42);
STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MAX_PAGESIZE) < UINT16_MAX);
STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) >=
BRANCH_NODE_MAX(MDBX_MIN_PAGESIZE));
STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) >= BRANCH_NODE_MAX(MDBX_MIN_PAGESIZE));
STATIC_ASSERT(BRANCH_NODE_MAX(MDBX_MAX_PAGESIZE) > NODESIZE + 42);
STATIC_ASSERT(BRANCH_NODE_MAX(MDBX_MAX_PAGESIZE) < UINT16_MAX);
const intptr_t branch_nodemax = BRANCH_NODE_MAX(pagesize);
const intptr_t leaf_nodemax = LEAF_NODE_MAX(pagesize);
ENSURE(env, branch_nodemax > (intptr_t)(NODESIZE + 42) &&
branch_nodemax % 2 == 0 &&
leaf_nodemax > (intptr_t)(sizeof(tree_t) + NODESIZE + 42) &&
leaf_nodemax >= branch_nodemax &&
ENSURE(env, branch_nodemax > (intptr_t)(NODESIZE + 42) && branch_nodemax % 2 == 0 &&
leaf_nodemax > (intptr_t)(sizeof(tree_t) + NODESIZE + 42) && leaf_nodemax >= branch_nodemax &&
leaf_nodemax < (int)UINT16_MAX && leaf_nodemax % 2 == 0);
env->leaf_nodemax = (uint16_t)leaf_nodemax;
env->branch_nodemax = (uint16_t)branch_nodemax;
@ -71,18 +61,14 @@ __cold unsigned env_setup_pagesize(MDBX_env *env, const size_t pagesize) {
if (unlikely(err != MDBX_SUCCESS))
ERROR("mdbx_get_sysraminfo(), rc %d", err);
else {
size_t reasonable_dpl_limit =
(size_t)(total_ram_pages + avail_ram_pages) / 42;
size_t reasonable_dpl_limit = (size_t)(total_ram_pages + avail_ram_pages) / 42;
if (pagesize > globals.sys_pagesize)
reasonable_dpl_limit /= pagesize / globals.sys_pagesize;
else if (pagesize < globals.sys_pagesize)
reasonable_dpl_limit *= globals.sys_pagesize / pagesize;
reasonable_dpl_limit = (reasonable_dpl_limit < PAGELIST_LIMIT)
? reasonable_dpl_limit
: PAGELIST_LIMIT;
reasonable_dpl_limit = (reasonable_dpl_limit > CURSOR_STACK_SIZE * 4)
? reasonable_dpl_limit
: CURSOR_STACK_SIZE * 4;
reasonable_dpl_limit = (reasonable_dpl_limit < PAGELIST_LIMIT) ? reasonable_dpl_limit : PAGELIST_LIMIT;
reasonable_dpl_limit =
(reasonable_dpl_limit > CURSOR_STACK_SIZE * 4) ? reasonable_dpl_limit : CURSOR_STACK_SIZE * 4;
env->options.dp_limit = (unsigned)reasonable_dpl_limit;
}
}
@ -108,46 +94,36 @@ retry:;
goto bailout;
}
const troika_t troika =
(txn0_owned | should_unlock) ? env->basal_txn->tw.troika : meta_tap(env);
const troika_t troika = (txn0_owned | should_unlock) ? env->basal_txn->tw.troika : meta_tap(env);
const meta_ptr_t head = meta_recent(env, &troika);
const uint64_t unsynced_pages =
atomic_load64(&env->lck->unsynced_pages, mo_Relaxed);
const uint64_t unsynced_pages = atomic_load64(&env->lck->unsynced_pages, mo_Relaxed);
if (unsynced_pages == 0) {
const uint32_t synched_meta_txnid_u32 =
atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed);
const uint32_t synched_meta_txnid_u32 = atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed);
if (synched_meta_txnid_u32 == (uint32_t)head.txnid && head.is_steady)
goto bailout;
}
if (should_unlock && (env->flags & MDBX_WRITEMAP) &&
unlikely(head.ptr_c->geometry.first_unallocated >
bytes2pgno(env, env->dxb_mmap.current))) {
unlikely(head.ptr_c->geometry.first_unallocated > bytes2pgno(env, env->dxb_mmap.current))) {
if (unlikely(env->stuck_meta >= 0) &&
troika.recent != (uint8_t)env->stuck_meta) {
if (unlikely(env->stuck_meta >= 0) && troika.recent != (uint8_t)env->stuck_meta) {
NOTICE("skip %s since wagering meta-page (%u) is mispatch the recent "
"meta-page (%u)",
"sync datafile", env->stuck_meta, troika.recent);
rc = MDBX_RESULT_TRUE;
} else {
rc = dxb_resize(env, head.ptr_c->geometry.first_unallocated,
head.ptr_c->geometry.now, head.ptr_c->geometry.upper,
rc = dxb_resize(env, head.ptr_c->geometry.first_unallocated, head.ptr_c->geometry.now, head.ptr_c->geometry.upper,
implicit_grow);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
}
}
const size_t autosync_threshold =
atomic_load32(&env->lck->autosync_threshold, mo_Relaxed);
const uint64_t autosync_period =
atomic_load64(&env->lck->autosync_period, mo_Relaxed);
const size_t autosync_threshold = atomic_load32(&env->lck->autosync_threshold, mo_Relaxed);
const uint64_t autosync_period = atomic_load64(&env->lck->autosync_period, mo_Relaxed);
uint64_t eoos_timestamp;
if (force || (autosync_threshold && unsynced_pages >= autosync_threshold) ||
(autosync_period &&
(eoos_timestamp =
atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) &&
(autosync_period && (eoos_timestamp = atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) &&
osal_monotime() - eoos_timestamp >= autosync_period))
flags &= MDBX_WRITEMAP /* clear flags for full steady sync */;
@ -159,8 +135,7 @@ retry:;
int err;
/* pre-sync to avoid latency for writer */
if (unsynced_pages > /* FIXME: define threshold */ 42 &&
(flags & MDBX_SAFE_NOSYNC) == 0) {
if (unsynced_pages > /* FIXME: define threshold */ 42 && (flags & MDBX_SAFE_NOSYNC) == 0) {
eASSERT(env, ((flags ^ env->flags) & MDBX_WRITEMAP) == 0);
if (flags & MDBX_WRITEMAP) {
/* Acquire guard to avoid collision with remap */
@ -171,8 +146,7 @@ retry:;
if (unlikely(err != MDBX_SUCCESS))
return err;
#endif
const size_t usedbytes =
pgno_align2os_bytes(env, head.ptr_c->geometry.first_unallocated);
const size_t usedbytes = pgno_align2os_bytes(env, head.ptr_c->geometry.first_unallocated);
err = osal_msync(&env->dxb_mmap, 0, usedbytes, MDBX_SYNC_DATA);
#if defined(_WIN32) || defined(_WIN64)
imports.srwl_ReleaseShared(&env->remap_guard);
@ -215,8 +189,7 @@ retry:;
eASSERT(env, txn0_owned || should_unlock);
eASSERT(env, !txn0_owned || (flags & txn_shrink_allowed) == 0);
if (!head.is_steady && unlikely(env->stuck_meta >= 0) &&
troika.recent != (uint8_t)env->stuck_meta) {
if (!head.is_steady && unlikely(env->stuck_meta >= 0) && troika.recent != (uint8_t)env->stuck_meta) {
NOTICE("skip %s since wagering meta-page (%u) is mispatch the recent "
"meta-page (%u)",
"sync datafile", env->stuck_meta, troika.recent);
@ -224,9 +197,8 @@ retry:;
goto bailout;
}
if (!head.is_steady || ((flags & MDBX_SAFE_NOSYNC) == 0 && unsynced_pages)) {
DEBUG("meta-head %" PRIaPGNO ", %s, sync_pending %" PRIu64,
data_page(head.ptr_c)->pgno, durable_caption(head.ptr_c),
unsynced_pages);
DEBUG("meta-head %" PRIaPGNO ", %s, sync_pending %" PRIu64, data_page(head.ptr_c)->pgno,
durable_caption(head.ptr_c), unsynced_pages);
meta_t meta = *head.ptr_c;
rc = dxb_sync_locked(env, flags, &meta, &env->basal_txn->tw.troika);
if (unlikely(rc != MDBX_SUCCESS))
@ -235,8 +207,7 @@ retry:;
/* LY: sync meta-pages if MDBX_NOMETASYNC enabled
* and someone was not synced above. */
if (atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed) !=
(uint32_t)head.txnid)
if (atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed) != (uint32_t)head.txnid)
rc = meta_sync(env, head);
bailout:
@ -334,9 +305,8 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) {
*/
env->pid = osal_getpid();
int rc = osal_openfile((env->flags & MDBX_RDONLY) ? MDBX_OPEN_DXB_READ
: MDBX_OPEN_DXB_LAZY,
env, env->pathname.dxb, &env->lazy_fd, mode);
int rc = osal_openfile((env->flags & MDBX_RDONLY) ? MDBX_OPEN_DXB_READ : MDBX_OPEN_DXB_LAZY, env, env->pathname.dxb,
&env->lazy_fd, mode);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
@ -355,8 +325,7 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) {
#if defined(_WIN32) || defined(_WIN64)
eASSERT(env, env->ioring.overlapped_fd == 0);
bool ior_direct = false;
if (!(env->flags &
(MDBX_RDONLY | MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_EXCLUSIVE))) {
if (!(env->flags & (MDBX_RDONLY | MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_EXCLUSIVE))) {
if (MDBX_AVOID_MSYNC && (env->flags & MDBX_WRITEMAP)) {
/* Запрошен режим MDBX_SYNC_DURABLE | MDBX_WRITEMAP при активной опции
* MDBX_AVOID_MSYNC.
@ -383,8 +352,7 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) {
int err = dxb_read_header(env, &header, MDBX_SUCCESS, true);
if ((err == MDBX_SUCCESS && header.pagesize >= globals.sys_pagesize) ||
(err == MDBX_ENODATA && mode && env->ps >= globals.sys_pagesize &&
osal_filesize(env->lazy_fd, &dxb_filesize) == MDBX_SUCCESS &&
dxb_filesize == 0))
osal_filesize(env->lazy_fd, &dxb_filesize) == MDBX_SUCCESS && dxb_filesize == 0))
/* Может быть коллизия, если два процесса пытаются одновременно создать
* БД с разным размером страницы, который у одного меньше системной
* страницы, а у другого НЕ меньше. Эта допустимая, но очень странная
@ -392,9 +360,8 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) {
ior_direct = true;
}
rc = osal_openfile(ior_direct ? MDBX_OPEN_DXB_OVERLAPPED_DIRECT
: MDBX_OPEN_DXB_OVERLAPPED,
env, env->pathname.dxb, &env->ioring.overlapped_fd, 0);
rc = osal_openfile(ior_direct ? MDBX_OPEN_DXB_OVERLAPPED_DIRECT : MDBX_OPEN_DXB_OVERLAPPED, env, env->pathname.dxb,
&env->ioring.overlapped_fd, 0);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
env->dxb_lock_event = CreateEventW(nullptr, true, false, nullptr);
@ -410,8 +377,7 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) {
return errno;
mode = st.st_mode;
}
mode = (/* inherit read permissions for group and others */ mode &
(S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) |
mode = (/* inherit read permissions for group and others */ mode & (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) |
/* always add read/write for owner */ S_IRUSR | S_IWUSR |
((mode & S_IRGRP) ? /* +write if readable by group */ S_IWGRP : 0) |
((mode & S_IROTH) ? /* +write if readable by others */ S_IWOTH : 0);
@ -428,8 +394,7 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) {
| MDBX_EXCLUSIVE
#endif /* !Windows */
))) {
rc = osal_openfile(MDBX_OPEN_DXB_DSYNC, env, env->pathname.dxb,
&env->dsync_fd, 0);
rc = osal_openfile(MDBX_OPEN_DXB_DSYNC, env, env->pathname.dxb, &env->dsync_fd, 0);
if (unlikely(MDBX_IS_ERROR(rc)))
return rc;
if (env->dsync_fd != INVALID_HANDLE_VALUE) {
@ -439,19 +404,14 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) {
}
}
const MDBX_env_flags_t lazy_flags =
MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC | MDBX_NOMETASYNC;
const MDBX_env_flags_t mode_flags = lazy_flags | MDBX_LIFORECLAIM |
MDBX_NORDAHEAD | MDBX_RDONLY |
MDBX_WRITEMAP;
const MDBX_env_flags_t lazy_flags = MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC | MDBX_NOMETASYNC;
const MDBX_env_flags_t mode_flags = lazy_flags | MDBX_LIFORECLAIM | MDBX_NORDAHEAD | MDBX_RDONLY | MDBX_WRITEMAP;
lck_t *const lck = env->lck_mmap.lck;
if (lck && lck_rc != MDBX_RESULT_TRUE && (env->flags & MDBX_RDONLY) == 0) {
MDBX_env_flags_t snap_flags;
while ((snap_flags = atomic_load32(&lck->envmode, mo_AcquireRelease)) ==
MDBX_RDONLY) {
if (atomic_cas32(&lck->envmode, MDBX_RDONLY,
(snap_flags = (env->flags & mode_flags)))) {
while ((snap_flags = atomic_load32(&lck->envmode, mo_AcquireRelease)) == MDBX_RDONLY) {
if (atomic_cas32(&lck->envmode, MDBX_RDONLY, (snap_flags = (env->flags & mode_flags)))) {
/* The case:
* - let's assume that for some reason the DB file is smaller
* than it should be according to the geometry,
@ -471,12 +431,9 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) {
if (env->flags & MDBX_ACCEDE) {
/* Pickup current mode-flags (MDBX_LIFORECLAIM, MDBX_NORDAHEAD, etc). */
const MDBX_env_flags_t diff =
(snap_flags ^ env->flags) &
((snap_flags & lazy_flags) ? mode_flags
: mode_flags & ~MDBX_WRITEMAP);
(snap_flags ^ env->flags) & ((snap_flags & lazy_flags) ? mode_flags : mode_flags & ~MDBX_WRITEMAP);
env->flags ^= diff;
NOTICE("accede mode-flags: 0x%X, 0x%X -> 0x%X", diff, env->flags ^ diff,
env->flags);
NOTICE("accede mode-flags: 0x%X, 0x%X -> 0x%X", diff, env->flags ^ diff, env->flags);
}
/* Ранее упущенный не очевидный момент: При работе БД в режимах
@ -498,12 +455,10 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) {
* В результате, требуется либо запретить совместную работу процессам с
* разным MDBX_WRITEMAP в режиме отложенной записи, либо отслеживать такое
* смешивание и блокировать steady-пометки - что контрпродуктивно. */
const MDBX_env_flags_t rigorous_flags =
(snap_flags & lazy_flags)
? MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC | MDBX_WRITEMAP
: MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC;
const MDBX_env_flags_t rigorous_diff =
(snap_flags ^ env->flags) & rigorous_flags;
const MDBX_env_flags_t rigorous_flags = (snap_flags & lazy_flags)
? MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC | MDBX_WRITEMAP
: MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC;
const MDBX_env_flags_t rigorous_diff = (snap_flags ^ env->flags) & rigorous_flags;
if (rigorous_diff) {
ERROR("current mode/flags 0x%X incompatible with requested 0x%X, "
"rigorous diff 0x%X",
@ -529,8 +484,7 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) {
}
if (unlikely(/* recovery mode */ env->stuck_meta >= 0) &&
(lck_rc != /* exclusive */ MDBX_RESULT_TRUE ||
(env->flags & MDBX_EXCLUSIVE) == 0)) {
(lck_rc != /* exclusive */ MDBX_RESULT_TRUE || (env->flags & MDBX_EXCLUSIVE) == 0)) {
ERROR("%s", "recovery requires exclusive mode");
return MDBX_BUSY;
}
@ -545,8 +499,7 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) {
if (lck) {
if (lck_rc == MDBX_RESULT_TRUE) {
rc = lck_downgrade(env);
DEBUG("lck-downgrade-%s: rc %i",
(env->flags & MDBX_EXCLUSIVE) ? "partial" : "full", rc);
DEBUG("lck-downgrade-%s: rc %i", (env->flags & MDBX_EXCLUSIVE) ? "partial" : "full", rc);
if (rc != MDBX_SUCCESS)
return rc;
} else {
@ -556,14 +509,13 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) {
}
}
rc = (env->flags & MDBX_RDONLY)
? MDBX_SUCCESS
: osal_ioring_create(&env->ioring
rc = (env->flags & MDBX_RDONLY) ? MDBX_SUCCESS
: osal_ioring_create(&env->ioring
#if defined(_WIN32) || defined(_WIN64)
,
ior_direct, env->ioring.overlapped_fd
,
ior_direct, env->ioring.overlapped_fd
#endif /* Windows */
);
);
return rc;
}
@ -606,8 +558,7 @@ __cold int env_close(MDBX_env *env, bool resurrect_after_fork) {
}
#if defined(_WIN32) || defined(_WIN64)
eASSERT(env, !env->ioring.overlapped_fd ||
env->ioring.overlapped_fd == INVALID_HANDLE_VALUE);
eASSERT(env, !env->ioring.overlapped_fd || env->ioring.overlapped_fd == INVALID_HANDLE_VALUE);
if (env->dxb_lock_event != INVALID_HANDLE_VALUE) {
CloseHandle(env->dxb_lock_event);
env->dxb_lock_event = INVALID_HANDLE_VALUE;

15
src/essentials.h
View File

@ -110,27 +110,22 @@ extern struct libmdbx_imports imports;
extern LIBMDBX_API const char *const mdbx_sourcery_anchor;
#endif
#define MDBX_IS_ERROR(rc) \
((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE)
#define MDBX_IS_ERROR(rc) ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE)
/*----------------------------------------------------------------------------*/
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t
int64pgno(int64_t i64) {
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t int64pgno(int64_t i64) {
if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1))
return (pgno_t)i64;
return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO;
}
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t
pgno_add(size_t base, size_t augend) {
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_add(size_t base, size_t augend) {
assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO);
return int64pgno((int64_t)base + (int64_t)augend);
}
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t
pgno_sub(size_t base, size_t subtrahend) {
assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 &&
subtrahend < MAX_PAGENO);
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_sub(size_t base, size_t subtrahend) {
assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && subtrahend < MAX_PAGENO);
return int64pgno((int64_t)base - (int64_t)subtrahend);
}

351
src/gc-get.c
View File

@ -52,8 +52,7 @@ static bool mincore_fetch(MDBX_env *const env, const size_t unit_begin) {
env->lck->pgops.mincore.weak += 1;
#endif /* MDBX_ENABLE_PGOP_STAT */
uint8_t *const vector = alloca(pages);
if (unlikely(mincore(ptr_disp(env->dxb_mmap.base, offset), length,
(void *)vector))) {
if (unlikely(mincore(ptr_disp(env->dxb_mmap.base, offset), length, (void *)vector))) {
NOTICE("mincore(+%zu, %zu), err %d", offset, length, errno);
return false;
}
@ -79,14 +78,10 @@ static bool mincore_fetch(MDBX_env *const env, const size_t unit_begin) {
}
#endif /* MDBX_USE_MINCORE */
MDBX_MAYBE_UNUSED static inline bool mincore_probe(MDBX_env *const env,
const pgno_t pgno) {
MDBX_MAYBE_UNUSED static inline bool mincore_probe(MDBX_env *const env, const pgno_t pgno) {
#if MDBX_USE_MINCORE
const size_t offset_aligned =
floor_powerof2(pgno2bytes(env, pgno), globals.sys_pagesize);
const unsigned unit_log2 = (env->ps2ln > globals.sys_pagesize_ln2)
? env->ps2ln
: globals.sys_pagesize_ln2;
const size_t offset_aligned = floor_powerof2(pgno2bytes(env, pgno), globals.sys_pagesize);
const unsigned unit_log2 = (env->ps2ln > globals.sys_pagesize_ln2) ? env->ps2ln : globals.sys_pagesize_ln2;
const size_t unit_begin = offset_aligned >> unit_log2;
eASSERT(env, (unit_begin << unit_log2) == offset_aligned);
const ptrdiff_t dist = unit_begin - env->lck->mincore_cache.begin[0];
@ -102,8 +97,7 @@ MDBX_MAYBE_UNUSED static inline bool mincore_probe(MDBX_env *const env,
/*----------------------------------------------------------------------------*/
MDBX_MAYBE_UNUSED __hot static pgno_t *
scan4seq_fallback(pgno_t *range, const size_t len, const size_t seq) {
MDBX_MAYBE_UNUSED __hot static pgno_t *scan4seq_fallback(pgno_t *range, const size_t len, const size_t seq) {
assert(seq > 0 && len > seq);
#if MDBX_PNL_ASCENDING
assert(range[-1] == len);
@ -167,8 +161,7 @@ scan4seq_fallback(pgno_t *range, const size_t len, const size_t seq) {
return nullptr;
}
MDBX_MAYBE_UNUSED static const pgno_t *scan4range_checker(const pnl_t pnl,
const size_t seq) {
MDBX_MAYBE_UNUSED static const pgno_t *scan4range_checker(const pnl_t pnl, const size_t seq) {
size_t begin = MDBX_PNL_ASCENDING ? 1 : MDBX_PNL_GETSIZE(pnl);
#if MDBX_PNL_ASCENDING
while (seq <= MDBX_PNL_GETSIZE(pnl) - begin) {
@ -186,8 +179,7 @@ MDBX_MAYBE_UNUSED static const pgno_t *scan4range_checker(const pnl_t pnl,
return nullptr;
}
#if defined(_MSC_VER) && !defined(__builtin_clz) && \
!__has_builtin(__builtin_clz)
#if defined(_MSC_VER) && !defined(__builtin_clz) && !__has_builtin(__builtin_clz)
MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clz(uint32_t value) {
unsigned long index;
_BitScanReverse(&index, value);
@ -195,8 +187,7 @@ MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clz(uint32_t value) {
}
#endif /* _MSC_VER */
#if defined(_MSC_VER) && !defined(__builtin_clzl) && \
!__has_builtin(__builtin_clzl)
#if defined(_MSC_VER) && !defined(__builtin_clzl) && !__has_builtin(__builtin_clzl)
MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clzl(size_t value) {
unsigned long index;
#ifdef _WIN64
@ -213,8 +204,7 @@ MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clzl(size_t value) {
#if !MDBX_PNL_ASCENDING
#if !defined(MDBX_ATTRIBUTE_TARGET) && \
(__has_attribute(__target__) || __GNUC_PREREQ(5, 0))
#if !defined(MDBX_ATTRIBUTE_TARGET) && (__has_attribute(__target__) || __GNUC_PREREQ(5, 0))
#define MDBX_ATTRIBUTE_TARGET(target) __attribute__((__target__(target)))
#endif /* MDBX_ATTRIBUTE_TARGET */
@ -223,9 +213,8 @@ MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clzl(size_t value) {
* gcc/i686-buildroot-linux-gnu/12.2.0/include/xmmintrin.h:814:1:
* error: inlining failed in call to 'always_inline' '_mm_movemask_ps':
* target specific option mismatch */
#if !defined(__FAST_MATH__) || !__FAST_MATH__ || !defined(__GNUC__) || \
defined(__e2k__) || defined(__clang__) || defined(__amd64__) || \
defined(__SSE2__)
#if !defined(__FAST_MATH__) || !__FAST_MATH__ || !defined(__GNUC__) || defined(__e2k__) || defined(__clang__) || \
defined(__amd64__) || defined(__SSE2__)
#define MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND 0
#else
#define MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND 1
@ -237,41 +226,36 @@ MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clzl(size_t value) {
#elif (defined(_M_IX86_FP) && _M_IX86_FP >= 2) || defined(__amd64__)
#define __SSE2__
#define MDBX_ATTRIBUTE_TARGET_SSE2 /* nope */
#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && \
!MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND
#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND
#define MDBX_ATTRIBUTE_TARGET_SSE2 MDBX_ATTRIBUTE_TARGET("sse,sse2")
#endif /* __SSE2__ */
#if defined(__AVX2__)
#define MDBX_ATTRIBUTE_TARGET_AVX2 /* nope */
#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && \
!MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND
#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND
#define MDBX_ATTRIBUTE_TARGET_AVX2 MDBX_ATTRIBUTE_TARGET("sse,sse2,avx,avx2")
#endif /* __AVX2__ */
#if defined(MDBX_ATTRIBUTE_TARGET_AVX2)
#if defined(__AVX512BW__)
#define MDBX_ATTRIBUTE_TARGET_AVX512BW /* nope */
#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && \
!MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND && \
#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND && \
(__GNUC_PREREQ(6, 0) || __CLANG_PREREQ(5, 0))
#define MDBX_ATTRIBUTE_TARGET_AVX512BW \
MDBX_ATTRIBUTE_TARGET("sse,sse2,avx,avx2,avx512bw")
#define MDBX_ATTRIBUTE_TARGET_AVX512BW MDBX_ATTRIBUTE_TARGET("sse,sse2,avx,avx2,avx512bw")
#endif /* __AVX512BW__ */
#endif /* MDBX_ATTRIBUTE_TARGET_AVX2 for MDBX_ATTRIBUTE_TARGET_AVX512BW */
#ifdef MDBX_ATTRIBUTE_TARGET_SSE2
MDBX_ATTRIBUTE_TARGET_SSE2 static __always_inline unsigned
diffcmp2mask_sse2(const pgno_t *const ptr, const ptrdiff_t offset,
const __m128i pattern) {
diffcmp2mask_sse2(const pgno_t *const ptr, const ptrdiff_t offset, const __m128i pattern) {
const __m128i f = _mm_loadu_si128((const __m128i *)ptr);
const __m128i l = _mm_loadu_si128((const __m128i *)(ptr + offset));
const __m128i cmp = _mm_cmpeq_epi32(_mm_sub_epi32(f, l), pattern);
return _mm_movemask_ps(*(const __m128 *)&cmp);
}
MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_SSE2 static pgno_t *
scan4seq_sse2(pgno_t *range, const size_t len, const size_t seq) {
MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_SSE2 static pgno_t *scan4seq_sse2(pgno_t *range, const size_t len,
const size_t seq) {
assert(seq > 0 && len > seq);
#if MDBX_PNL_ASCENDING
#error "FIXME: Not implemented"
@ -303,8 +287,7 @@ scan4seq_sse2(pgno_t *range, const size_t len, const size_t seq) {
* Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */
#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__)
const unsigned on_page_safe_mask = 0xff0 /* enough for '-15' bytes offset */;
if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) &&
!RUNNING_ON_VALGRIND) {
if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) {
const unsigned extra = (unsigned)(detent + 4 - range);
assert(extra > 0 && extra < 4);
mask = 0xF << extra;
@ -324,8 +307,7 @@ scan4seq_sse2(pgno_t *range, const size_t len, const size_t seq) {
#ifdef MDBX_ATTRIBUTE_TARGET_AVX2
MDBX_ATTRIBUTE_TARGET_AVX2 static __always_inline unsigned
diffcmp2mask_avx2(const pgno_t *const ptr, const ptrdiff_t offset,
const __m256i pattern) {
diffcmp2mask_avx2(const pgno_t *const ptr, const ptrdiff_t offset, const __m256i pattern) {
const __m256i f = _mm256_loadu_si256((const __m256i *)ptr);
const __m256i l = _mm256_loadu_si256((const __m256i *)(ptr + offset));
const __m256i cmp = _mm256_cmpeq_epi32(_mm256_sub_epi32(f, l), pattern);
@ -333,16 +315,15 @@ diffcmp2mask_avx2(const pgno_t *const ptr, const ptrdiff_t offset,
}
MDBX_ATTRIBUTE_TARGET_AVX2 static __always_inline unsigned
diffcmp2mask_sse2avx(const pgno_t *const ptr, const ptrdiff_t offset,
const __m128i pattern) {
diffcmp2mask_sse2avx(const pgno_t *const ptr, const ptrdiff_t offset, const __m128i pattern) {
const __m128i f = _mm_loadu_si128((const __m128i *)ptr);
const __m128i l = _mm_loadu_si128((const __m128i *)(ptr + offset));
const __m128i cmp = _mm_cmpeq_epi32(_mm_sub_epi32(f, l), pattern);
return _mm_movemask_ps(*(const __m128 *)&cmp);
}
MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_AVX2 static pgno_t *
scan4seq_avx2(pgno_t *range, const size_t len, const size_t seq) {
MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_AVX2 static pgno_t *scan4seq_avx2(pgno_t *range, const size_t len,
const size_t seq) {
assert(seq > 0 && len > seq);
#if MDBX_PNL_ASCENDING
#error "FIXME: Not implemented"
@ -374,8 +355,7 @@ scan4seq_avx2(pgno_t *range, const size_t len, const size_t seq) {
* Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */
#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__)
const unsigned on_page_safe_mask = 0xfe0 /* enough for '-31' bytes offset */;
if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) &&
!RUNNING_ON_VALGRIND) {
if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) {
const unsigned extra = (unsigned)(detent + 8 - range);
assert(extra > 0 && extra < 8);
mask = 0xFF << extra;
@ -402,15 +382,14 @@ scan4seq_avx2(pgno_t *range, const size_t len, const size_t seq) {
#ifdef MDBX_ATTRIBUTE_TARGET_AVX512BW
MDBX_ATTRIBUTE_TARGET_AVX512BW static __always_inline unsigned
diffcmp2mask_avx512bw(const pgno_t *const ptr, const ptrdiff_t offset,
const __m512i pattern) {
diffcmp2mask_avx512bw(const pgno_t *const ptr, const ptrdiff_t offset, const __m512i pattern) {
const __m512i f = _mm512_loadu_si512((const __m512i *)ptr);
const __m512i l = _mm512_loadu_si512((const __m512i *)(ptr + offset));
return _mm512_cmpeq_epi32_mask(_mm512_sub_epi32(f, l), pattern);
}
MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_AVX512BW static pgno_t *
scan4seq_avx512bw(pgno_t *range, const size_t len, const size_t seq) {
MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_AVX512BW static pgno_t *scan4seq_avx512bw(pgno_t *range, const size_t len,
const size_t seq) {
assert(seq > 0 && len > seq);
#if MDBX_PNL_ASCENDING
#error "FIXME: Not implemented"
@ -442,8 +421,7 @@ scan4seq_avx512bw(pgno_t *range, const size_t len, const size_t seq) {
* Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */
#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__)
const unsigned on_page_safe_mask = 0xfc0 /* enough for '-63' bytes offset */;
if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) &&
!RUNNING_ON_VALGRIND) {
if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) {
const unsigned extra = (unsigned)(detent + 16 - range);
assert(extra > 0 && extra < 16);
mask = 0xFFFF << extra;
@ -474,10 +452,8 @@ scan4seq_avx512bw(pgno_t *range, const size_t len, const size_t seq) {
}
#endif /* MDBX_ATTRIBUTE_TARGET_AVX512BW */
#if (defined(__ARM_NEON) || defined(__ARM_NEON__)) && \
(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
static __always_inline size_t diffcmp2mask_neon(const pgno_t *const ptr,
const ptrdiff_t offset,
#if (defined(__ARM_NEON) || defined(__ARM_NEON__)) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
static __always_inline size_t diffcmp2mask_neon(const pgno_t *const ptr, const ptrdiff_t offset,
const uint32x4_t pattern) {
const uint32x4_t f = vld1q_u32(ptr);
const uint32x4_t l = vld1q_u32(ptr + offset);
@ -485,12 +461,10 @@ static __always_inline size_t diffcmp2mask_neon(const pgno_t *const ptr,
if (sizeof(size_t) > 7)
return vget_lane_u64(vreinterpret_u64_u16(cmp), 0);
else
return vget_lane_u32(vreinterpret_u32_u8(vmovn_u16(vcombine_u16(cmp, cmp))),
0);
return vget_lane_u32(vreinterpret_u32_u8(vmovn_u16(vcombine_u16(cmp, cmp))), 0);
}
__hot static pgno_t *scan4seq_neon(pgno_t *range, const size_t len,
const size_t seq) {
__hot static pgno_t *scan4seq_neon(pgno_t *range, const size_t len, const size_t seq) {
assert(seq > 0 && len > seq);
#if MDBX_PNL_ASCENDING
#error "FIXME: Not implemented"
@ -522,8 +496,7 @@ __hot static pgno_t *scan4seq_neon(pgno_t *range, const size_t len,
* Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */
#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__)
const unsigned on_page_safe_mask = 0xff0 /* enough for '-15' bytes offset */;
if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) &&
!RUNNING_ON_VALGRIND) {
if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) {
const unsigned extra = (unsigned)(detent + 4 - range);
assert(extra > 0 && extra < 4);
mask = (~(size_t)0) << (extra * sizeof(size_t) * 2);
@ -548,8 +521,7 @@ __hot static pgno_t *scan4seq_neon(pgno_t *range, const size_t len,
#define scan4seq_default scan4seq_avx2
#elif defined(__SSE2__) && defined(MDBX_ATTRIBUTE_TARGET_SSE2)
#define scan4seq_default scan4seq_sse2
#elif (defined(__ARM_NEON) || defined(__ARM_NEON__)) && \
(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#elif (defined(__ARM_NEON) || defined(__ARM_NEON__)) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#define scan4seq_default scan4seq_neon
/* Choosing of another variants should be added here. */
#endif /* scan4seq_default */
@ -570,17 +542,12 @@ __hot static pgno_t *scan4seq_neon(pgno_t *range, const size_t len,
#else
/* Selecting the most appropriate implementation at runtime,
* depending on the available CPU features. */
static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len,
const size_t seq);
static pgno_t *(*scan4seq_impl)(pgno_t *range, const size_t len,
const size_t seq) = scan4seq_resolver;
static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len, const size_t seq);
static pgno_t *(*scan4seq_impl)(pgno_t *range, const size_t len, const size_t seq) = scan4seq_resolver;
static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len,
const size_t seq) {
pgno_t *(*choice)(pgno_t *range, const size_t len, const size_t seq) =
nullptr;
#if __has_builtin(__builtin_cpu_init) || defined(__BUILTIN_CPU_INIT__) || \
__GNUC_PREREQ(4, 8)
static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len, const size_t seq) {
pgno_t *(*choice)(pgno_t *range, const size_t len, const size_t seq) = nullptr;
#if __has_builtin(__builtin_cpu_init) || defined(__BUILTIN_CPU_INIT__) || __GNUC_PREREQ(4, 8)
__builtin_cpu_init();
#endif /* __builtin_cpu_init() */
#ifdef MDBX_ATTRIBUTE_TARGET_SSE2
@ -607,12 +574,10 @@ static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len,
#define ALLOC_SHOULD_SCAN 8 /* внутреннее состояние */
#define ALLOC_LIFO 16 /* внутреннее состояние */
static inline bool is_gc_usable(MDBX_txn *txn, const MDBX_cursor *mc,
const uint8_t flags) {
static inline bool is_gc_usable(MDBX_txn *txn, const MDBX_cursor *mc, const uint8_t flags) {
/* If txn is updating the GC, then the retired-list cannot play catch-up with
* itself by growing while trying to save it. */
if (mc->tree == &txn->dbs[FREE_DBI] && !(flags & ALLOC_RESERVE) &&
!(mc->flags & z_gcu_preparation))
if (mc->tree == &txn->dbs[FREE_DBI] && !(flags & ALLOC_RESERVE) && !(mc->flags & z_gcu_preparation))
return false;
/* avoid search inside empty tree and while tree is updating,
@ -690,8 +655,7 @@ __hot static pgno_t relist_get_single(MDBX_txn *txn) {
#ifndef MDBX_ENABLE_SAVING_SEQUENCES
#define MDBX_ENABLE_SAVING_SEQUENCES 0
#endif
if (MDBX_ENABLE_SAVING_SEQUENCES && unlikely(target[dir] == *target + 1) &&
len > 2) {
if (MDBX_ENABLE_SAVING_SEQUENCES && unlikely(target[dir] == *target + 1) && len > 2) {
/* Пытаемся пропускать последовательности при наличии одиночных элементов.
* TODO: необходимо кэшировать пропускаемые последовательности
* чтобы не сканировать список сначала при каждом выделении. */
@ -719,8 +683,7 @@ __hot static pgno_t relist_get_single(MDBX_txn *txn) {
#if MDBX_PNL_ASCENDING
/* вырезаем элемент с перемещением хвоста */
MDBX_PNL_SETSIZE(txn->tw.relist, len - 1);
for (const pgno_t *const end = txn->tw.relist + len - 1; target <= end;
++target)
for (const pgno_t *const end = txn->tw.relist + len - 1; target <= end; ++target)
*target = target[1];
#else
/* перемещать хвост не нужно, просто усекам список */
@ -729,8 +692,7 @@ __hot static pgno_t relist_get_single(MDBX_txn *txn) {
return pgno;
}
__hot static pgno_t relist_get_sequence(MDBX_txn *txn, const size_t num,
uint8_t flags) {
__hot static pgno_t relist_get_sequence(MDBX_txn *txn, const size_t num, uint8_t flags) {
const size_t len = MDBX_PNL_GETSIZE(txn->tw.relist);
pgno_t *edge = MDBX_PNL_EDGE(txn->tw.relist);
assert(len >= num && num > 1);
@ -754,8 +716,7 @@ __hot static pgno_t relist_get_sequence(MDBX_txn *txn, const size_t num,
/* вырезаем найденную последовательность с перемещением хвоста */
MDBX_PNL_SETSIZE(txn->tw.relist, len - num);
#if MDBX_PNL_ASCENDING
for (const pgno_t *const end = txn->tw.relist + len - num; target <= end;
++target)
for (const pgno_t *const end = txn->tw.relist + len - num; target <= end; ++target)
*target = target[num];
#else
for (const pgno_t *const end = txn->tw.relist + len; ++target <= end;)
@ -766,16 +727,13 @@ __hot static pgno_t relist_get_sequence(MDBX_txn *txn, const size_t num,
return 0;
}
static inline pgr_t page_alloc_finalize(MDBX_env *const env,
MDBX_txn *const txn,
const MDBX_cursor *const mc,
static inline pgr_t page_alloc_finalize(MDBX_env *const env, MDBX_txn *const txn, const MDBX_cursor *const mc,
const pgno_t pgno, const size_t num) {
#if MDBX_ENABLE_PROFGC
size_t majflt_before;
const uint64_t cputime_before = osal_cputime(&majflt_before);
gc_prof_stat_t *const prof = (cursor_dbi(mc) == FREE_DBI)
? &env->lck->pgops.gc_prof.self
: &env->lck->pgops.gc_prof.work;
gc_prof_stat_t *const prof =
(cursor_dbi(mc) == FREE_DBI) ? &env->lck->pgops.gc_prof.self : &env->lck->pgops.gc_prof.work;
#else
(void)mc;
#endif /* MDBX_ENABLE_PROFGC */
@ -811,8 +769,7 @@ static inline pgr_t page_alloc_finalize(MDBX_env *const env,
* грязной I/O очереди. Из-за этого штраф за лишнюю запись может быть
* сравним с избегаемым ненужным чтением. */
if (env->prefault_write_activated) {
void *const pattern =
ptr_disp(env->page_auxbuf, need_clean ? env->ps : env->ps * 2);
void *const pattern = ptr_disp(env->page_auxbuf, need_clean ? env->ps : env->ps * 2);
size_t file_offset = pgno2bytes(env, pgno);
if (likely(num == 1)) {
if (!mincore_probe(env, pgno)) {
@ -831,8 +788,7 @@ static inline pgr_t page_alloc_finalize(MDBX_env *const env,
iov[n].iov_len = env->ps;
iov[n].iov_base = pattern;
if (unlikely(++n == MDBX_AUXILARY_IOV_MAX)) {
osal_pwritev(env->lazy_fd, iov, MDBX_AUXILARY_IOV_MAX,
file_offset);
osal_pwritev(env->lazy_fd, iov, MDBX_AUXILARY_IOV_MAX, file_offset);
#if MDBX_ENABLE_PGOP_STAT
env->lck->pgops.prefault.weak += 1;
#endif /* MDBX_ENABLE_PGOP_STAT */
@ -873,8 +829,7 @@ static inline pgr_t page_alloc_finalize(MDBX_env *const env,
ret.err = page_dirty(txn, ret.page, (pgno_t)num);
bailout:
tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated -
MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
#if MDBX_ENABLE_PROFGC
size_t majflt_after;
prof->xtime_cpu += osal_cputime(&majflt_after) - cputime_before;
@ -883,32 +838,25 @@ bailout:
return ret;
}
pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num,
uint8_t flags) {
pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags) {
pgr_t ret;
MDBX_txn *const txn = mc->txn;
MDBX_env *const env = txn->env;
#if MDBX_ENABLE_PROFGC
gc_prof_stat_t *const prof = (cursor_dbi(mc) == FREE_DBI)
? &env->lck->pgops.gc_prof.self
: &env->lck->pgops.gc_prof.work;
gc_prof_stat_t *const prof =
(cursor_dbi(mc) == FREE_DBI) ? &env->lck->pgops.gc_prof.self : &env->lck->pgops.gc_prof.work;
prof->spe_counter += 1;
#endif /* MDBX_ENABLE_PROFGC */
eASSERT(env, num > 0 || (flags & ALLOC_RESERVE));
eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated -
MDBX_ENABLE_REFUND));
eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
size_t newnext;
const uint64_t monotime_begin =
(MDBX_ENABLE_PROFGC || (num > 1 && env->options.gc_time_limit))
? osal_monotime()
: 0;
const uint64_t monotime_begin = (MDBX_ENABLE_PROFGC || (num > 1 && env->options.gc_time_limit)) ? osal_monotime() : 0;
struct monotime_cache now_cache;
now_cache.expire_countdown =
1 /* старт с 1 позволяет избавиться как от лишних системных вызовов когда
лимит времени задан нулевой или уже исчерпан, так и от подсчета
времени при не-достижении rp_augment_limit */
now_cache.expire_countdown = 1 /* старт с 1 позволяет избавиться как от лишних системных вызовов когда
лимит времени задан нулевой или уже исчерпан, так и от подсчета
времени при не-достижении rp_augment_limit */
;
now_cache.value = monotime_begin;
pgno_t pgno = 0;
@ -917,9 +865,8 @@ pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num,
prof->xpages += 1;
#endif /* MDBX_ENABLE_PROFGC */
if (MDBX_PNL_GETSIZE(txn->tw.relist) >= num) {
eASSERT(env,
MDBX_PNL_LAST(txn->tw.relist) < txn->geo.first_unallocated &&
MDBX_PNL_FIRST(txn->tw.relist) < txn->geo.first_unallocated);
eASSERT(env, MDBX_PNL_LAST(txn->tw.relist) < txn->geo.first_unallocated &&
MDBX_PNL_FIRST(txn->tw.relist) < txn->geo.first_unallocated);
pgno = relist_get_sequence(txn, num, flags);
if (likely(pgno))
goto done;
@ -936,16 +883,14 @@ pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num,
goto no_gc;
}
eASSERT(env,
(flags & (ALLOC_COALESCE | ALLOC_LIFO | ALLOC_SHOULD_SCAN)) == 0);
eASSERT(env, (flags & (ALLOC_COALESCE | ALLOC_LIFO | ALLOC_SHOULD_SCAN)) == 0);
flags += (env->flags & MDBX_LIFORECLAIM) ? ALLOC_LIFO : 0;
if (/* Не коагулируем записи при подготовке резерва для обновления GC.
* Иначе попытка увеличить резерв может приводить к необходимости ещё
* большего резерва из-за увеличения списка переработанных страниц. */
(flags & ALLOC_RESERVE) == 0) {
if (txn->dbs[FREE_DBI].branch_pages &&
MDBX_PNL_GETSIZE(txn->tw.relist) < env->maxgc_large1page / 2)
if (txn->dbs[FREE_DBI].branch_pages && MDBX_PNL_GETSIZE(txn->tw.relist) < env->maxgc_large1page / 2)
flags += ALLOC_COALESCE;
}
@ -976,9 +921,7 @@ retry_gc_refresh_oldest:;
txnid_t oldest = txn_snapshot_oldest(txn);
retry_gc_have_oldest:
if (unlikely(oldest >= txn->txnid)) {
ERROR("unexpected/invalid oldest-readed txnid %" PRIaTXN
" for current-txnid %" PRIaTXN,
oldest, txn->txnid);
ERROR("unexpected/invalid oldest-readed txnid %" PRIaTXN " for current-txnid %" PRIaTXN, oldest, txn->txnid);
ret.err = MDBX_PROBLEM;
goto fail;
}
@ -1026,8 +969,7 @@ next_gc:;
goto depleted_gc;
}
if (unlikely(key.iov_len != sizeof(txnid_t))) {
ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid GC key-length");
ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC key-length");
ret.err = MDBX_CORRUPTED;
goto fail;
}
@ -1046,26 +988,21 @@ next_gc:;
/* Reading next GC record */
MDBX_val data;
page_t *const mp = gc->pg[gc->top];
if (unlikely((ret.err = node_read(gc, page_node(mp, gc->ki[gc->top]), &data,
mp)) != MDBX_SUCCESS))
if (unlikely((ret.err = node_read(gc, page_node(mp, gc->ki[gc->top]), &data, mp)) != MDBX_SUCCESS))
goto fail;
pgno_t *gc_pnl = (pgno_t *)data.iov_base;
if (unlikely(data.iov_len % sizeof(pgno_t) ||
data.iov_len < MDBX_PNL_SIZEOF(gc_pnl) ||
if (unlikely(data.iov_len % sizeof(pgno_t) || data.iov_len < MDBX_PNL_SIZEOF(gc_pnl) ||
!pnl_check(gc_pnl, txn->geo.first_unallocated))) {
ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid GC value-length");
ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC value-length");
ret.err = MDBX_CORRUPTED;
goto fail;
}
const size_t gc_len = MDBX_PNL_GETSIZE(gc_pnl);
TRACE("gc-read: id #%" PRIaTXN " len %zu, re-list will %zu ", id, gc_len,
gc_len + MDBX_PNL_GETSIZE(txn->tw.relist));
TRACE("gc-read: id #%" PRIaTXN " len %zu, re-list will %zu ", id, gc_len, gc_len + MDBX_PNL_GETSIZE(txn->tw.relist));
if (unlikely(gc_len + MDBX_PNL_GETSIZE(txn->tw.relist) >=
env->maxgc_large1page)) {
if (unlikely(gc_len + MDBX_PNL_GETSIZE(txn->tw.relist) >= env->maxgc_large1page)) {
/* Don't try to coalesce too much. */
if (flags & ALLOC_SHOULD_SCAN) {
eASSERT(env, flags & ALLOC_COALESCE);
@ -1076,10 +1013,8 @@ next_gc:;
#endif /* MDBX_ENABLE_PROFGC */
TRACE("clear %s %s", "ALLOC_COALESCE", "since got threshold");
if (MDBX_PNL_GETSIZE(txn->tw.relist) >= num) {
eASSERT(env,
MDBX_PNL_LAST(txn->tw.relist) < txn->geo.first_unallocated &&
MDBX_PNL_FIRST(txn->tw.relist) <
txn->geo.first_unallocated);
eASSERT(env, MDBX_PNL_LAST(txn->tw.relist) < txn->geo.first_unallocated &&
MDBX_PNL_FIRST(txn->tw.relist) < txn->geo.first_unallocated);
if (likely(num == 1)) {
pgno = relist_get_single(txn);
goto done;
@ -1090,25 +1025,19 @@ next_gc:;
}
flags -= ALLOC_COALESCE | ALLOC_SHOULD_SCAN;
}
if (unlikely(/* list is too long already */ MDBX_PNL_GETSIZE(
txn->tw.relist) >= env->options.rp_augment_limit) &&
if (unlikely(/* list is too long already */ MDBX_PNL_GETSIZE(txn->tw.relist) >= env->options.rp_augment_limit) &&
((/* not a slot-request from gc-update */ num &&
/* have enough unallocated space */ txn->geo.upper >=
txn->geo.first_unallocated + num &&
monotime_since_cached(monotime_begin, &now_cache) +
txn->tw.gc.time_acc >=
env->options.gc_time_limit) ||
/* have enough unallocated space */ txn->geo.upper >= txn->geo.first_unallocated + num &&
monotime_since_cached(monotime_begin, &now_cache) + txn->tw.gc.time_acc >= env->options.gc_time_limit) ||
gc_len + MDBX_PNL_GETSIZE(txn->tw.relist) >= PAGELIST_LIMIT)) {
/* Stop reclaiming to avoid large/overflow the page list. This is a rare
* case while search for a continuously multi-page region in a
* large database, see https://libmdbx.dqdkfa.ru/dead-github/issues/123 */
NOTICE("stop reclaiming %s: %zu (current) + %zu "
"(chunk) -> %zu, rp_augment_limit %u",
likely(gc_len + MDBX_PNL_GETSIZE(txn->tw.relist) < PAGELIST_LIMIT)
? "since rp_augment_limit was reached"
: "to avoid PNL overflow",
MDBX_PNL_GETSIZE(txn->tw.relist), gc_len,
gc_len + MDBX_PNL_GETSIZE(txn->tw.relist),
likely(gc_len + MDBX_PNL_GETSIZE(txn->tw.relist) < PAGELIST_LIMIT) ? "since rp_augment_limit was reached"
: "to avoid PNL overflow",
MDBX_PNL_GETSIZE(txn->tw.relist), gc_len, gc_len + MDBX_PNL_GETSIZE(txn->tw.relist),
env->options.rp_augment_limit);
goto depleted_gc;
}
@ -1128,9 +1057,7 @@ next_gc:;
goto fail;
if (LOG_ENABLED(MDBX_LOG_EXTRA)) {
DEBUG_EXTRA("readed GC-pnl txn %" PRIaTXN " root %" PRIaPGNO
" len %zu, PNL",
id, txn->dbs[FREE_DBI].root, gc_len);
DEBUG_EXTRA("readed GC-pnl txn %" PRIaTXN " root %" PRIaPGNO " len %zu, PNL", id, txn->dbs[FREE_DBI].root, gc_len);
for (size_t i = gc_len; i; i--)
DEBUG_EXTRA_PRINT(" %" PRIaPGNO, gc_pnl[i]);
DEBUG_EXTRA_PRINT(", first_unallocated %u\n", txn->geo.first_unallocated);
@ -1141,33 +1068,27 @@ next_gc:;
flags |= ALLOC_SHOULD_SCAN;
if (AUDIT_ENABLED()) {
if (unlikely(!pnl_check(txn->tw.relist, txn->geo.first_unallocated))) {
ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid txn retired-list");
ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid txn retired-list");
ret.err = MDBX_CORRUPTED;
goto fail;
}
} else {
eASSERT(env,
pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated));
eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated));
}
eASSERT(env, dpl_check(txn));
eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.relist) == 0 ||
MDBX_PNL_MOST(txn->tw.relist) < txn->geo.first_unallocated);
eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.relist) == 0 || MDBX_PNL_MOST(txn->tw.relist) < txn->geo.first_unallocated);
if (MDBX_ENABLE_REFUND && MDBX_PNL_GETSIZE(txn->tw.relist) &&
unlikely(MDBX_PNL_MOST(txn->tw.relist) ==
txn->geo.first_unallocated - 1)) {
unlikely(MDBX_PNL_MOST(txn->tw.relist) == txn->geo.first_unallocated - 1)) {
/* Refund suitable pages into "unallocated" space */
txn_refund(txn);
}
eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated -
MDBX_ENABLE_REFUND));
eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
/* Done for a kick-reclaim mode, actually no page needed */
if (unlikely(num == 0)) {
eASSERT(env, ret.err == MDBX_SUCCESS);
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "early-exit for slot", id,
MDBX_PNL_GETSIZE(txn->tw.relist));
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "early-exit for slot", id, MDBX_PNL_GETSIZE(txn->tw.relist));
goto early_exit;
}
@ -1175,8 +1096,7 @@ next_gc:;
eASSERT(env, op == MDBX_PREV || op == MDBX_NEXT);
if (flags & ALLOC_COALESCE) {
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "coalesce-continue", id,
MDBX_PNL_GETSIZE(txn->tw.relist));
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "coalesce-continue", id, MDBX_PNL_GETSIZE(txn->tw.relist));
goto next_gc;
}
@ -1184,9 +1104,8 @@ scan:
eASSERT(env, flags & ALLOC_SHOULD_SCAN);
eASSERT(env, num > 0);
if (MDBX_PNL_GETSIZE(txn->tw.relist) >= num) {
eASSERT(env,
MDBX_PNL_LAST(txn->tw.relist) < txn->geo.first_unallocated &&
MDBX_PNL_FIRST(txn->tw.relist) < txn->geo.first_unallocated);
eASSERT(env, MDBX_PNL_LAST(txn->tw.relist) < txn->geo.first_unallocated &&
MDBX_PNL_FIRST(txn->tw.relist) < txn->geo.first_unallocated);
if (likely(num == 1)) {
eASSERT(env, !(flags & ALLOC_RESERVE));
pgno = relist_get_single(txn);
@ -1198,14 +1117,12 @@ scan:
}
flags -= ALLOC_SHOULD_SCAN;
if (ret.err == MDBX_SUCCESS) {
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "continue-search", id,
MDBX_PNL_GETSIZE(txn->tw.relist));
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "continue-search", id, MDBX_PNL_GETSIZE(txn->tw.relist));
goto next_gc;
}
depleted_gc:
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "gc-depleted", id,
MDBX_PNL_GETSIZE(txn->tw.relist));
TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "gc-depleted", id, MDBX_PNL_GETSIZE(txn->tw.relist));
ret.err = MDBX_NOTFOUND;
if (flags & ALLOC_SHOULD_SCAN)
goto scan;
@ -1226,16 +1143,11 @@ depleted_gc:
/* Does reclaiming stopped at the last steady point? */
const meta_ptr_t recent = meta_recent(env, &txn->tw.troika);
const meta_ptr_t prefer_steady = meta_prefer_steady(env, &txn->tw.troika);
if (recent.ptr_c != prefer_steady.ptr_c && prefer_steady.is_steady &&
detent == prefer_steady.txnid + 1) {
DEBUG("gc-kick-steady: recent %" PRIaTXN "-%s, steady %" PRIaTXN
"-%s, detent %" PRIaTXN,
recent.txnid, durable_caption(recent.ptr_c), prefer_steady.txnid,
durable_caption(prefer_steady.ptr_c), detent);
const pgno_t autosync_threshold =
atomic_load32(&env->lck->autosync_threshold, mo_Relaxed);
const uint64_t autosync_period =
atomic_load64(&env->lck->autosync_period, mo_Relaxed);
if (recent.ptr_c != prefer_steady.ptr_c && prefer_steady.is_steady && detent == prefer_steady.txnid + 1) {
DEBUG("gc-kick-steady: recent %" PRIaTXN "-%s, steady %" PRIaTXN "-%s, detent %" PRIaTXN, recent.txnid,
durable_caption(recent.ptr_c), prefer_steady.txnid, durable_caption(prefer_steady.ptr_c), detent);
const pgno_t autosync_threshold = atomic_load32(&env->lck->autosync_threshold, mo_Relaxed);
const uint64_t autosync_period = atomic_load64(&env->lck->autosync_period, mo_Relaxed);
uint64_t eoos_timestamp;
/* wipe the last steady-point if one of:
* - UTTERLY_NOSYNC mode AND auto-sync threshold is NOT specified
@ -1246,8 +1158,7 @@ depleted_gc:
* - database is full (with the current file size)
* AND auto-sync threshold it NOT specified */
if (F_ISSET(env->flags, MDBX_UTTERLY_NOSYNC) &&
((autosync_threshold | autosync_period) == 0 ||
newnext >= prefer_steady.ptr_c->geometry.now)) {
((autosync_threshold | autosync_period) == 0 || newnext >= prefer_steady.ptr_c->geometry.now)) {
/* wipe steady checkpoint in MDBX_UTTERLY_NOSYNC mode
* without any auto-sync threshold(s). */
#if MDBX_ENABLE_PROFGC
@ -1257,39 +1168,30 @@ depleted_gc:
DEBUG("gc-wipe-steady, rc %d", ret.err);
if (unlikely(ret.err != MDBX_SUCCESS))
goto fail;
eASSERT(env, prefer_steady.ptr_c !=
meta_prefer_steady(env, &txn->tw.troika).ptr_c);
eASSERT(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->tw.troika).ptr_c);
goto retry_gc_refresh_oldest;
}
if ((autosync_threshold &&
atomic_load64(&env->lck->unsynced_pages, mo_Relaxed) >=
autosync_threshold) ||
(autosync_period &&
(eoos_timestamp =
atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) &&
if ((autosync_threshold && atomic_load64(&env->lck->unsynced_pages, mo_Relaxed) >= autosync_threshold) ||
(autosync_period && (eoos_timestamp = atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) &&
osal_monotime() - eoos_timestamp >= autosync_period) ||
newnext >= txn->geo.upper ||
((num == 0 || newnext >= txn->geo.end_pgno) &&
(autosync_threshold | autosync_period) == 0)) {
((num == 0 || newnext >= txn->geo.end_pgno) && (autosync_threshold | autosync_period) == 0)) {
/* make steady checkpoint. */
#if MDBX_ENABLE_PROFGC
env->lck->pgops.gc_prof.flushes += 1;
#endif /* MDBX_ENABLE_PROFGC */
meta_t meta = *recent.ptr_c;
ret.err = dxb_sync_locked(env, env->flags & MDBX_WRITEMAP, &meta,
&txn->tw.troika);
ret.err = dxb_sync_locked(env, env->flags & MDBX_WRITEMAP, &meta, &txn->tw.troika);
DEBUG("gc-make-steady, rc %d", ret.err);
eASSERT(env, ret.err != MDBX_RESULT_TRUE);
if (unlikely(ret.err != MDBX_SUCCESS))
goto fail;
eASSERT(env, prefer_steady.ptr_c !=
meta_prefer_steady(env, &txn->tw.troika).ptr_c);
eASSERT(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->tw.troika).ptr_c);
goto retry_gc_refresh_oldest;
}
}
if (unlikely(true ==
atomic_load32(&env->lck->rdt_refresh_flag, mo_AcquireRelease))) {
if (unlikely(true == atomic_load32(&env->lck->rdt_refresh_flag, mo_AcquireRelease))) {
oldest = txn_snapshot_oldest(txn);
if (oldest >= detent)
goto retry_gc_have_oldest;
@ -1315,8 +1217,7 @@ no_gc:
#ifndef MDBX_ENABLE_BACKLOG_DEPLETED
#define MDBX_ENABLE_BACKLOG_DEPLETED 0
#endif /* MDBX_ENABLE_BACKLOG_DEPLETED*/
if (MDBX_ENABLE_BACKLOG_DEPLETED &&
unlikely(!(txn->flags & txn_gc_drained))) {
if (MDBX_ENABLE_BACKLOG_DEPLETED && unlikely(!(txn->flags & txn_gc_drained))) {
ret.err = MDBX_BACKLOG_DEPLETED;
goto fail;
}
@ -1338,20 +1239,16 @@ no_gc:
eASSERT(env, newnext > txn->geo.end_pgno);
const size_t grow_step = pv2pages(txn->geo.grow_pv);
size_t aligned = pgno_align2os_pgno(
env, (pgno_t)(newnext + grow_step - newnext % grow_step));
size_t aligned = pgno_align2os_pgno(env, (pgno_t)(newnext + grow_step - newnext % grow_step));
if (aligned > txn->geo.upper)
aligned = txn->geo.upper;
eASSERT(env, aligned >= newnext);
VERBOSE("try growth datafile to %zu pages (+%zu)", aligned,
aligned - txn->geo.end_pgno);
ret.err = dxb_resize(env, txn->geo.first_unallocated, (pgno_t)aligned,
txn->geo.upper, implicit_grow);
VERBOSE("try growth datafile to %zu pages (+%zu)", aligned, aligned - txn->geo.end_pgno);
ret.err = dxb_resize(env, txn->geo.first_unallocated, (pgno_t)aligned, txn->geo.upper, implicit_grow);
if (ret.err != MDBX_SUCCESS) {
ERROR("unable growth datafile to %zu pages (+%zu), errcode %d", aligned,
aligned - txn->geo.end_pgno, ret.err);
ERROR("unable growth datafile to %zu pages (+%zu), errcode %d", aligned, aligned - txn->geo.end_pgno, ret.err);
goto fail;
}
env->txn->geo.end_pgno = (pgno_t)aligned;
@ -1363,26 +1260,20 @@ done:
ret.err = MDBX_SUCCESS;
if (likely((flags & ALLOC_RESERVE) == 0)) {
if (pgno) {
eASSERT(env,
pgno + num <= txn->geo.first_unallocated && pgno >= NUM_METAS);
eASSERT(env,
pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated -
MDBX_ENABLE_REFUND));
eASSERT(env, pgno + num <= txn->geo.first_unallocated && pgno >= NUM_METAS);
eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
} else {
pgno = txn->geo.first_unallocated;
txn->geo.first_unallocated += (pgno_t)num;
eASSERT(env, txn->geo.first_unallocated <= txn->geo.end_pgno);
eASSERT(env,
pgno >= NUM_METAS && pgno + num <= txn->geo.first_unallocated);
eASSERT(env, pgno >= NUM_METAS && pgno + num <= txn->geo.first_unallocated);
}
ret = page_alloc_finalize(env, txn, mc, pgno, num);
if (unlikely(ret.err != MDBX_SUCCESS)) {
fail:
eASSERT(env, ret.err != MDBX_SUCCESS);
eASSERT(env,
pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated -
MDBX_ENABLE_REFUND));
eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
int level;
const char *what;
if (flags & ALLOC_RESERVE) {
@ -1398,12 +1289,9 @@ done:
"unable alloc %zu %s, alloc-flags 0x%x, err %d, txn-flags "
"0x%x, re-list-len %zu, loose-count %zu, gc: height %u, "
"branch %zu, leaf %zu, large %zu, entries %zu\n",
num, what, flags, ret.err, txn->flags,
MDBX_PNL_GETSIZE(txn->tw.relist), txn->tw.loose_count,
txn->dbs[FREE_DBI].height,
(size_t)txn->dbs[FREE_DBI].branch_pages,
(size_t)txn->dbs[FREE_DBI].leaf_pages,
(size_t)txn->dbs[FREE_DBI].large_pages,
num, what, flags, ret.err, txn->flags, MDBX_PNL_GETSIZE(txn->tw.relist), txn->tw.loose_count,
txn->dbs[FREE_DBI].height, (size_t)txn->dbs[FREE_DBI].branch_pages,
(size_t)txn->dbs[FREE_DBI].leaf_pages, (size_t)txn->dbs[FREE_DBI].large_pages,
(size_t)txn->dbs[FREE_DBI].items);
ret.page = nullptr;
}
@ -1411,8 +1299,7 @@ done:
txn->tw.gc.time_acc += monotime_since_cached(monotime_begin, &now_cache);
} else {
early_exit:
DEBUG("return nullptr for %zu pages for ALLOC_%s, rc %d", num,
num ? "RESERVE" : "SLOT", ret.err);
DEBUG("return nullptr for %zu pages for ALLOC_%s, rc %d", num, num ? "RESERVE" : "SLOT", ret.err);
ret.page = nullptr;
}
@ -1425,8 +1312,7 @@ done:
__hot pgr_t gc_alloc_single(const MDBX_cursor *const mc) {
MDBX_txn *const txn = mc->txn;
tASSERT(txn, mc->txn->flags & MDBX_TXN_DIRTY);
tASSERT(txn,
F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY));
tASSERT(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY));
/* If there are any loose pages, just use them */
while (likely(txn->tw.loose_pages)) {
@ -1443,8 +1329,7 @@ __hot pgr_t gc_alloc_single(const MDBX_cursor *const mc) {
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
txn->tw.loose_pages = page_next(lp);
txn->tw.loose_count--;
DEBUG_EXTRA("db %d use loose page %" PRIaPGNO, cursor_dbi_dbg(mc),
lp->pgno);
DEBUG_EXTRA("db %d use loose page %" PRIaPGNO, cursor_dbi_dbg(mc), lp->pgno);
tASSERT(txn, lp->pgno < txn->geo.first_unallocated);
tASSERT(txn, lp->pgno >= NUM_METAS);
VALGRIND_MAKE_MEM_UNDEFINED(page_data(lp), page_space(txn->env));

357
src/gc-put.c
View File

@ -11,9 +11,7 @@ MDBX_MAYBE_UNUSED static inline const char *dbg_prefix(const gcu_t *ctx) {
return is_lifo(ctx->cursor.txn) ? " lifo" : " fifo";
}
static inline size_t backlog_size(MDBX_txn *txn) {
return MDBX_PNL_GETSIZE(txn->tw.relist) + txn->tw.loose_count;
}
static inline size_t backlog_size(MDBX_txn *txn) { return MDBX_PNL_GETSIZE(txn->tw.relist) + txn->tw.loose_count; }
static int clean_stored_retired(MDBX_txn *txn, gcu_t *ctx) {
int err = MDBX_SUCCESS;
@ -53,8 +51,7 @@ static int clean_stored_retired(MDBX_txn *txn, gcu_t *ctx) {
}
static int touch_gc(gcu_t *ctx) {
tASSERT(ctx->cursor.txn, is_pointed(&ctx->cursor) ||
ctx->cursor.txn->dbs[FREE_DBI].leaf_pages == 0);
tASSERT(ctx->cursor.txn, is_pointed(&ctx->cursor) || ctx->cursor.txn->dbs[FREE_DBI].leaf_pages == 0);
MDBX_val key, val;
key.iov_base = val.iov_base = nullptr;
key.iov_len = sizeof(txnid_t);
@ -70,24 +67,19 @@ static int touch_gc(gcu_t *ctx) {
* during a deleting, when GC tree is unbalanced. */
static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) {
const size_t for_cow = txn->dbs[FREE_DBI].height;
const size_t for_rebalance =
for_cow + 1 +
(txn->dbs[FREE_DBI].height + 1ul >= txn->dbs[FREE_DBI].branch_pages);
const size_t for_rebalance = for_cow + 1 + (txn->dbs[FREE_DBI].height + 1ul >= txn->dbs[FREE_DBI].branch_pages);
size_t for_split = ctx->retired_stored == 0;
tASSERT(txn, is_pointed(&ctx->cursor) || txn->dbs[FREE_DBI].leaf_pages == 0);
const intptr_t retired_left =
MDBX_PNL_SIZEOF(txn->tw.retired_pages) - ctx->retired_stored;
const intptr_t retired_left = MDBX_PNL_SIZEOF(txn->tw.retired_pages) - ctx->retired_stored;
size_t for_relist = 0;
if (MDBX_ENABLE_BIGFOOT && retired_left > 0) {
for_relist = (retired_left + txn->env->maxgc_large1page - 1) /
txn->env->maxgc_large1page;
for_relist = (retired_left + txn->env->maxgc_large1page - 1) / txn->env->maxgc_large1page;
const size_t per_branch_page = txn->env->maxgc_per_branch;
for (size_t entries = for_relist; entries > 1; for_split += entries)
entries = (entries + per_branch_page - 1) / per_branch_page;
} else if (!MDBX_ENABLE_BIGFOOT && retired_left != 0) {
for_relist =
largechunk_npages(txn->env, MDBX_PNL_SIZEOF(txn->tw.retired_pages));
for_relist = largechunk_npages(txn->env, MDBX_PNL_SIZEOF(txn->tw.retired_pages));
}
const size_t for_tree_before_touch = for_cow + for_rebalance + for_split;
@ -96,23 +88,20 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) {
const size_t for_all_after_touch = for_relist + for_tree_after_touch;
if (likely(for_relist < 2 && backlog_size(txn) > for_all_before_touch) &&
(ctx->cursor.top < 0 ||
is_modifable(txn, ctx->cursor.pg[ctx->cursor.top])))
(ctx->cursor.top < 0 || is_modifable(txn, ctx->cursor.pg[ctx->cursor.top])))
return MDBX_SUCCESS;
TRACE(">> retired-stored %zu, left %zi, backlog %zu, need %zu (4list %zu, "
"4split %zu, "
"4cow %zu, 4tree %zu)",
ctx->retired_stored, retired_left, backlog_size(txn),
for_all_before_touch, for_relist, for_split, for_cow,
ctx->retired_stored, retired_left, backlog_size(txn), for_all_before_touch, for_relist, for_split, for_cow,
for_tree_before_touch);
int err = touch_gc(ctx);
TRACE("== after-touch, backlog %zu, err %d", backlog_size(txn), err);
if (!MDBX_ENABLE_BIGFOOT && unlikely(for_relist > 1) &&
MDBX_PNL_GETSIZE(txn->tw.retired_pages) != ctx->retired_stored &&
err == MDBX_SUCCESS) {
MDBX_PNL_GETSIZE(txn->tw.retired_pages) != ctx->retired_stored && err == MDBX_SUCCESS) {
if (unlikely(ctx->retired_stored)) {
err = clean_stored_retired(txn, ctx);
if (unlikely(err != MDBX_SUCCESS))
@ -122,8 +111,7 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) {
}
err = gc_alloc_ex(&ctx->cursor, for_relist, ALLOC_RESERVE).err;
TRACE("== after-4linear, backlog %zu, err %d", backlog_size(txn), err);
cASSERT(&ctx->cursor,
backlog_size(txn) >= for_relist || err != MDBX_SUCCESS);
cASSERT(&ctx->cursor, backlog_size(txn) >= for_relist || err != MDBX_SUCCESS);
}
while (backlog_size(txn) < for_all_after_touch && err == MDBX_SUCCESS)
@ -131,10 +119,8 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) {
TRACE("<< backlog %zu, err %d, gc: height %u, branch %zu, leaf %zu, large "
"%zu, entries %zu",
backlog_size(txn), err, txn->dbs[FREE_DBI].height,
(size_t)txn->dbs[FREE_DBI].branch_pages,
(size_t)txn->dbs[FREE_DBI].leaf_pages,
(size_t)txn->dbs[FREE_DBI].large_pages,
backlog_size(txn), err, txn->dbs[FREE_DBI].height, (size_t)txn->dbs[FREE_DBI].branch_pages,
(size_t)txn->dbs[FREE_DBI].leaf_pages, (size_t)txn->dbs[FREE_DBI].large_pages,
(size_t)txn->dbs[FREE_DBI].items);
tASSERT(txn, err != MDBX_NOTFOUND || (txn->flags & txn_gc_drained) != 0);
return (err != MDBX_NOTFOUND) ? err : MDBX_SUCCESS;
@ -164,12 +150,10 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
* though usually none are left at this point.
* The pages themselves remain in dirtylist. */
if (unlikely(!txn->tw.gc.reclaimed && txn->tw.gc.last_reclaimed < 1)) {
TRACE("%s: try allocate gc-slot for %zu loose-pages", dbg_prefix(ctx),
txn->tw.loose_count);
TRACE("%s: try allocate gc-slot for %zu loose-pages", dbg_prefix(ctx), txn->tw.loose_count);
int err = gc_alloc_ex(&ctx->cursor, 0, ALLOC_RESERVE).err;
if (err == MDBX_SUCCESS) {
TRACE("%s: retry since gc-slot for %zu loose-pages available",
dbg_prefix(ctx), txn->tw.loose_count);
TRACE("%s: retry since gc-slot for %zu loose-pages available", dbg_prefix(ctx), txn->tw.loose_count);
return MDBX_RESULT_TRUE;
}
@ -183,15 +167,13 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
}
TRACE("%s: append %zu loose-pages to retired-pages", dbg_prefix(ctx),
txn->tw.loose_count);
TRACE("%s: append %zu loose-pages to retired-pages", dbg_prefix(ctx), txn->tw.loose_count);
} else {
/* Room for loose pages + temp PNL with same */
int err = pnl_need(&txn->tw.relist, 2 * txn->tw.loose_count + 2);
if (unlikely(err != MDBX_SUCCESS))
return err;
pnl_t loose = txn->tw.relist + MDBX_PNL_ALLOCLEN(txn->tw.relist) -
txn->tw.loose_count - 1;
pnl_t loose = txn->tw.relist + MDBX_PNL_ALLOCLEN(txn->tw.relist) - txn->tw.loose_count - 1;
size_t count = 0;
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
tASSERT(txn, lp->flags == P_LOOSE);
@ -203,8 +185,7 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
MDBX_PNL_SETSIZE(loose, count);
pnl_sort(loose, txn->geo.first_unallocated);
pnl_merge(txn->tw.relist, loose);
TRACE("%s: append %zu loose-pages to reclaimed-pages", dbg_prefix(ctx),
txn->tw.loose_count);
TRACE("%s: append %zu loose-pages to reclaimed-pages", dbg_prefix(ctx), txn->tw.loose_count);
}
/* filter-out list of dirty-pages from loose-pages */
@ -227,8 +208,7 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
page_shadow_release(txn->env, dp, 1);
}
}
TRACE("%s: filtered-out loose-pages from %zu -> %zu dirty-pages",
dbg_prefix(ctx), dl->length, w);
TRACE("%s: filtered-out loose-pages from %zu -> %zu dirty-pages", dbg_prefix(ctx), dl->length, w);
tASSERT(txn, txn->tw.loose_count == dl->length - w);
dl->sorted -= sorted_out;
tASSERT(txn, dl->sorted <= w);
@ -236,8 +216,7 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
dl->pages_including_loose -= txn->tw.loose_count;
txn->tw.dirtyroom += txn->tw.loose_count;
tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
(txn->parent ? txn->parent->tw.dirtyroom
: txn->env->options.dp_limit));
(txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
} else {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC);
}
@ -276,8 +255,8 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
if (unlikely(err != MDBX_SUCCESS))
return err;
if (retired_pages_before != MDBX_PNL_GETSIZE(txn->tw.retired_pages)) {
TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx),
retired_pages_before, MDBX_PNL_GETSIZE(txn->tw.retired_pages));
TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx), retired_pages_before,
MDBX_PNL_GETSIZE(txn->tw.retired_pages));
break;
}
@ -290,19 +269,16 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
if (unlikely(err != MDBX_SUCCESS))
return err;
if (ctx->retired_stored >= MDBX_PNL_GETSIZE(txn->tw.retired_pages)) {
TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx),
retired_pages_before, MDBX_PNL_GETSIZE(txn->tw.retired_pages));
TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx), retired_pages_before,
MDBX_PNL_GETSIZE(txn->tw.retired_pages));
break;
}
}
key.iov_len = sizeof(txnid_t);
key.iov_base = &ctx->bigfoot;
const size_t left =
MDBX_PNL_GETSIZE(txn->tw.retired_pages) - ctx->retired_stored;
const size_t left = MDBX_PNL_GETSIZE(txn->tw.retired_pages) - ctx->retired_stored;
const size_t chunk =
(left > txn->env->maxgc_large1page && ctx->bigfoot < MAX_TXNID)
? txn->env->maxgc_large1page
: left;
(left > txn->env->maxgc_large1page && ctx->bigfoot < MAX_TXNID) ? txn->env->maxgc_large1page : left;
data.iov_len = (chunk + 1) * sizeof(pgno_t);
err = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE);
if (unlikely(err != MDBX_SUCCESS))
@ -318,9 +294,7 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
#endif /* MDBX_DEBUG && (ENABLE_MEMCHECK || __SANITIZE_ADDRESS__) */
if (retired_pages_before == MDBX_PNL_GETSIZE(txn->tw.retired_pages)) {
const size_t at = (is_lifo(txn) == MDBX_PNL_ASCENDING)
? left - chunk
: ctx->retired_stored;
const size_t at = (is_lifo(txn) == MDBX_PNL_ASCENDING) ? left - chunk : ctx->retired_stored;
pgno_t *const begin = txn->tw.retired_pages + at;
/* MDBX_PNL_ASCENDING == false && LIFO == false:
* - the larger pgno is at the beginning of retired list
@ -332,15 +306,11 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
*begin = (pgno_t)chunk;
memcpy(data.iov_base, begin, data.iov_len);
*begin = save;
TRACE("%s: put-retired/bigfoot @ %" PRIaTXN
" (slice #%u) #%zu [%zu..%zu] of %zu",
dbg_prefix(ctx), ctx->bigfoot,
(unsigned)(ctx->bigfoot - txn->txnid), chunk, at, at + chunk,
retired_pages_before);
TRACE("%s: put-retired/bigfoot @ %" PRIaTXN " (slice #%u) #%zu [%zu..%zu] of %zu", dbg_prefix(ctx),
ctx->bigfoot, (unsigned)(ctx->bigfoot - txn->txnid), chunk, at, at + chunk, retired_pages_before);
}
ctx->retired_stored += chunk;
} while (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages) &&
(++ctx->bigfoot, true));
} while (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages) && (++ctx->bigfoot, true));
} while (retired_pages_before != MDBX_PNL_GETSIZE(txn->tw.retired_pages));
#else
/* Write to last page of GC */
@ -369,13 +339,11 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) {
tASSERT(txn, data.iov_len == MDBX_PNL_SIZEOF(txn->tw.retired_pages));
memcpy(data.iov_base, txn->tw.retired_pages, data.iov_len);
TRACE("%s: put-retired #%zu @ %" PRIaTXN, dbg_prefix(ctx),
ctx->retired_stored, txn->txnid);
TRACE("%s: put-retired #%zu @ %" PRIaTXN, dbg_prefix(ctx), ctx->retired_stored, txn->txnid);
#endif /* MDBX_ENABLE_BIGFOOT */
if (LOG_ENABLED(MDBX_LOG_EXTRA)) {
size_t i = ctx->retired_stored;
DEBUG_EXTRA("txn %" PRIaTXN " root %" PRIaPGNO " num %zu, retired-PNL",
txn->txnid, txn->dbs[FREE_DBI].root, i);
DEBUG_EXTRA("txn %" PRIaTXN " root %" PRIaPGNO " num %zu, retired-PNL", txn->txnid, txn->dbs[FREE_DBI].root, i);
for (; i; i--)
DEBUG_EXTRA_PRINT(" %" PRIaPGNO, txn->tw.retired_pages[i]);
DEBUG_EXTRA_PRINT("%s\n", ".");
@ -388,8 +356,7 @@ typedef struct gcu_rid_result {
txnid_t rid;
} rid_t;
static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx,
const size_t left) {
static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left) {
rid_t r;
if (is_lifo(txn)) {
if (txn->tw.gc.reclaimed == nullptr) {
@ -400,8 +367,7 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx,
}
}
if (MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) < txl_max &&
left > (MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot) *
txn->env->maxgc_large1page &&
left > (MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot) * txn->env->maxgc_large1page &&
!ctx->dense) {
/* Hужен свободный для для сохранения списка страниц. */
bool need_cleanup = false;
@ -411,15 +377,11 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx,
r.err = gc_alloc_ex(&ctx->cursor, 0, ALLOC_RESERVE).err;
snap_oldest = txn->env->lck->cached_oldest.weak;
if (likely(r.err == MDBX_SUCCESS)) {
TRACE("%s: took @%" PRIaTXN " from GC", dbg_prefix(ctx),
MDBX_PNL_LAST(txn->tw.gc.reclaimed));
TRACE("%s: took @%" PRIaTXN " from GC", dbg_prefix(ctx), MDBX_PNL_LAST(txn->tw.gc.reclaimed));
need_cleanup = true;
}
} while (r.err == MDBX_SUCCESS &&
MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) < txl_max &&
left >
(MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot) *
txn->env->maxgc_large1page);
} while (r.err == MDBX_SUCCESS && MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) < txl_max &&
left > (MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot) * txn->env->maxgc_large1page);
if (likely(r.err == MDBX_SUCCESS)) {
TRACE("%s: got enough from GC.", dbg_prefix(ctx));
@ -443,20 +405,16 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx,
/* no reclaimable GC entries,
* therefore no entries with ID < mdbx_find_oldest(txn) */
txn->tw.gc.last_reclaimed = ctx->rid = snap_oldest;
TRACE("%s: none recycled yet, set rid to @%" PRIaTXN, dbg_prefix(ctx),
ctx->rid);
TRACE("%s: none recycled yet, set rid to @%" PRIaTXN, dbg_prefix(ctx), ctx->rid);
}
/* В GC нет годных к переработке записей,
* будем использовать свободные id в обратном порядке. */
while (MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) < txl_max &&
left >
(MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot) *
txn->env->maxgc_large1page) {
left > (MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot) * txn->env->maxgc_large1page) {
if (unlikely(ctx->rid <= MIN_TXNID)) {
ctx->dense = true;
if (unlikely(MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) <=
ctx->reused_slot)) {
if (unlikely(MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) <= ctx->reused_slot)) {
NOTICE("** restart: reserve depleted (reused_gc_slot %zu >= "
"gc.reclaimed %zu)",
ctx->reused_slot, MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed));
@ -470,21 +428,16 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx,
MDBX_val key = {&ctx->rid, sizeof(ctx->rid)}, data;
r.err = cursor_seek(&ctx->cursor, &key, &data, MDBX_SET_KEY).err;
if (unlikely(r.err == MDBX_SUCCESS)) {
DEBUG("%s: GC's id %" PRIaTXN " is present, going to first",
dbg_prefix(ctx), ctx->rid);
DEBUG("%s: GC's id %" PRIaTXN " is present, going to first", dbg_prefix(ctx), ctx->rid);
r.err = outer_first(&ctx->cursor, &key, nullptr);
if (unlikely(r.err != MDBX_SUCCESS ||
key.iov_len != sizeof(txnid_t))) {
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid GC-key size", (unsigned)key.iov_len);
if (unlikely(r.err != MDBX_SUCCESS || key.iov_len != sizeof(txnid_t))) {
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-key size", (unsigned)key.iov_len);
r.err = MDBX_CORRUPTED;
goto return_error;
}
const txnid_t gc_first = unaligned_peek_u64(4, key.iov_base);
if (unlikely(gc_first <= INITIAL_TXNID)) {
NOTICE("%s: no free GC's id(s) less than %" PRIaTXN
" (going dense-mode)",
dbg_prefix(ctx), ctx->rid);
NOTICE("%s: no free GC's id(s) less than %" PRIaTXN " (going dense-mode)", dbg_prefix(ctx), ctx->rid);
ctx->dense = true;
goto return_restart;
}
@ -501,18 +454,15 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx,
* with less fragmentation. */
need_cleanup = true;
else
ctx->cleaned_slot +=
1 /* mark cleanup is not needed for added slot. */;
ctx->cleaned_slot += 1 /* mark cleanup is not needed for added slot. */;
TRACE("%s: append @%" PRIaTXN
" to lifo-reclaimed, cleaned-gc-slot = %zu",
dbg_prefix(ctx), ctx->rid, ctx->cleaned_slot);
TRACE("%s: append @%" PRIaTXN " to lifo-reclaimed, cleaned-gc-slot = %zu", dbg_prefix(ctx), ctx->rid,
ctx->cleaned_slot);
}
if (need_cleanup) {
if (ctx->cleaned_slot) {
TRACE("%s: restart to clear and re-create GC entries",
dbg_prefix(ctx));
TRACE("%s: restart to clear and re-create GC entries", dbg_prefix(ctx));
goto return_restart;
}
goto return_continue;
@ -522,8 +472,7 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx,
const size_t i = MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot;
tASSERT(txn, i > 0 && i <= MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed));
r.rid = txn->tw.gc.reclaimed[i];
TRACE("%s: take @%" PRIaTXN " from lifo-reclaimed[%zu]", dbg_prefix(ctx),
r.rid, i);
TRACE("%s: take @%" PRIaTXN " from lifo-reclaimed[%zu]", dbg_prefix(ctx), r.rid, i);
} else {
tASSERT(txn, txn->tw.gc.reclaimed == nullptr);
if (unlikely(ctx->rid == 0)) {
@ -532,8 +481,7 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx,
r.err = outer_first(&ctx->cursor, &key, nullptr);
if (likely(r.err == MDBX_SUCCESS)) {
if (unlikely(key.iov_len != sizeof(txnid_t))) {
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid GC-key size", (unsigned)key.iov_len);
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-key size", (unsigned)key.iov_len);
r.err = MDBX_CORRUPTED;
goto return_error;
}
@ -600,18 +548,15 @@ retry:
ctx->loop += !(ctx->prev_first_unallocated > txn->geo.first_unallocated);
TRACE(">> restart, loop %u", ctx->loop);
tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated -
MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, dpl_check(txn));
if (unlikely(/* paranoia */ ctx->loop > ((MDBX_DEBUG > 0) ? 12 : 42))) {
ERROR("txn #%" PRIaTXN " too more loops %u, bailout", txn->txnid,
ctx->loop);
ERROR("txn #%" PRIaTXN " too more loops %u, bailout", txn->txnid, ctx->loop);
rc = MDBX_PROBLEM;
goto bailout;
}
if (unlikely(ctx->dense ||
ctx->prev_first_unallocated > txn->geo.first_unallocated)) {
if (unlikely(ctx->dense || ctx->prev_first_unallocated > txn->geo.first_unallocated)) {
rc = clean_stored_retired(txn, ctx);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
@ -630,13 +575,10 @@ retry:
/* Come back here after each Put() in case retired-list changed */
TRACE("%s", " >> continue");
tASSERT(txn,
pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated -
MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
MDBX_val key, data;
if (is_lifo(txn)) {
if (ctx->cleaned_slot <
(txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) : 0)) {
if (ctx->cleaned_slot < (txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) : 0)) {
ctx->reserved = 0;
ctx->cleaned_slot = 0;
ctx->reused_slot = 0;
@ -644,8 +586,7 @@ retry:
/* LY: cleanup reclaimed records. */
do {
ctx->cleaned_id = txn->tw.gc.reclaimed[++ctx->cleaned_slot];
tASSERT(txn, ctx->cleaned_slot > 0 &&
ctx->cleaned_id <= env->lck->cached_oldest.weak);
tASSERT(txn, ctx->cleaned_slot > 0 && ctx->cleaned_id <= env->lck->cached_oldest.weak);
key.iov_base = &ctx->cleaned_id;
key.iov_len = sizeof(ctx->cleaned_id);
rc = cursor_seek(&ctx->cursor, &key, nullptr, MDBX_SET).err;
@ -657,8 +598,7 @@ retry:
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
tASSERT(txn, ctx->cleaned_id <= env->lck->cached_oldest.weak);
TRACE("%s: cleanup-reclaimed-id [%zu]%" PRIaTXN, dbg_prefix(ctx),
ctx->cleaned_slot, ctx->cleaned_id);
TRACE("%s: cleanup-reclaimed-id [%zu]%" PRIaTXN, dbg_prefix(ctx), ctx->cleaned_slot, ctx->cleaned_id);
tASSERT(txn, *txn->cursors == &ctx->cursor);
rc = cursor_del(&ctx->cursor, 0);
if (unlikely(rc != MDBX_SUCCESS))
@ -668,8 +608,7 @@ retry:
}
} else {
/* Удаляем оставшиеся вынутые из GC записи. */
while (txn->tw.gc.last_reclaimed &&
ctx->cleaned_id <= txn->tw.gc.last_reclaimed) {
while (txn->tw.gc.last_reclaimed && ctx->cleaned_id <= txn->tw.gc.last_reclaimed) {
rc = outer_first(&ctx->cursor, &key, nullptr);
if (rc == MDBX_NOTFOUND) {
ctx->cleaned_id = txn->tw.gc.last_reclaimed + 1;
@ -680,10 +619,8 @@ retry:
}
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
if (!MDBX_DISABLE_VALIDATION &&
unlikely(key.iov_len != sizeof(txnid_t))) {
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid GC-key size", (unsigned)key.iov_len);
if (!MDBX_DISABLE_VALIDATION && unlikely(key.iov_len != sizeof(txnid_t))) {
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-key size", (unsigned)key.iov_len);
rc = MDBX_CORRUPTED;
goto bailout;
}
@ -700,8 +637,7 @@ retry:
goto bailout;
tASSERT(txn, ctx->cleaned_id <= txn->tw.gc.last_reclaimed);
tASSERT(txn, ctx->cleaned_id <= env->lck->cached_oldest.weak);
TRACE("%s: cleanup-reclaimed-id %" PRIaTXN, dbg_prefix(ctx),
ctx->cleaned_id);
TRACE("%s: cleanup-reclaimed-id %" PRIaTXN, dbg_prefix(ctx), ctx->cleaned_id);
tASSERT(txn, *txn->cursors == &ctx->cursor);
rc = cursor_del(&ctx->cursor, 0);
if (unlikely(rc != MDBX_SUCCESS))
@ -709,9 +645,7 @@ retry:
}
}
tASSERT(txn,
pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated -
MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, dpl_check(txn));
if (AUDIT_ENABLED()) {
rc = audit_ex(txn, ctx->retired_stored, false);
@ -721,9 +655,7 @@ retry:
/* return suitable into unallocated space */
if (txn_refund(txn)) {
tASSERT(txn,
pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated -
MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
if (AUDIT_ENABLED()) {
rc = audit_ex(txn, ctx->retired_stored, false);
if (unlikely(rc != MDBX_SUCCESS))
@ -743,10 +675,9 @@ retry:
}
if (unlikely(ctx->reserved > MDBX_PNL_GETSIZE(txn->tw.relist)) &&
(ctx->loop < 5 || ctx->reserved - MDBX_PNL_GETSIZE(txn->tw.relist) >
env->maxgc_large1page / 2)) {
TRACE("%s: reclaimed-list changed %zu -> %zu, retry", dbg_prefix(ctx),
ctx->amount, MDBX_PNL_GETSIZE(txn->tw.relist));
(ctx->loop < 5 || ctx->reserved - MDBX_PNL_GETSIZE(txn->tw.relist) > env->maxgc_large1page / 2)) {
TRACE("%s: reclaimed-list changed %zu -> %zu, retry", dbg_prefix(ctx), ctx->amount,
MDBX_PNL_GETSIZE(txn->tw.relist));
ctx->reserve_adj += ctx->reserved - MDBX_PNL_GETSIZE(txn->tw.relist);
goto retry;
}
@ -760,9 +691,7 @@ retry:
continue;
}
tASSERT(txn,
pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated -
MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, txn->tw.loose_count == 0);
TRACE("%s", " >> reserving");
@ -776,8 +705,7 @@ retry:
"lifo-reclaimed-slots %zu, "
"reused-gc-slots %zu",
dbg_prefix(ctx), ctx->amount, ctx->reserved, ctx->reserve_adj, left,
txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) : 0,
ctx->reused_slot);
txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) : 0, ctx->reused_slot);
if (0 >= (intptr_t)left)
break;
@ -795,59 +723,46 @@ retry:
size_t chunk = left;
if (unlikely(left > env->maxgc_large1page)) {
const size_t avail_gc_slots =
txn->tw.gc.reclaimed
? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot + 1
: (ctx->rid < INT16_MAX) ? (size_t)ctx->rid
: INT16_MAX;
const size_t avail_gc_slots = txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot + 1
: (ctx->rid < INT16_MAX) ? (size_t)ctx->rid
: INT16_MAX;
if (likely(avail_gc_slots > 1)) {
#if MDBX_ENABLE_BIGFOOT
chunk = env->maxgc_large1page;
if (avail_gc_slots < INT16_MAX &&
unlikely(left > env->maxgc_large1page * avail_gc_slots))
if (avail_gc_slots < INT16_MAX && unlikely(left > env->maxgc_large1page * avail_gc_slots))
/* TODO: Можно смотреть последовательности какой длины есть в relist
* и пробовать нарезать куски соответствующего размера.
* Смысл в том, чтобы не дробить последовательности страниц,
* а использовать целиком. */
chunk = env->maxgc_large1page +
left / (env->maxgc_large1page * avail_gc_slots) *
env->maxgc_large1page;
chunk = env->maxgc_large1page + left / (env->maxgc_large1page * avail_gc_slots) * env->maxgc_large1page;
#else
if (chunk < env->maxgc_large1page * 2)
chunk /= 2;
else {
const size_t prefer_max_scatter = 257;
const size_t threshold =
env->maxgc_large1page * ((avail_gc_slots < prefer_max_scatter)
? avail_gc_slots
: prefer_max_scatter);
env->maxgc_large1page * ((avail_gc_slots < prefer_max_scatter) ? avail_gc_slots : prefer_max_scatter);
if (left < threshold)
chunk = env->maxgc_large1page;
else {
const size_t tail = left - threshold + env->maxgc_large1page + 1;
size_t span = 1;
size_t avail = ((pgno2bytes(env, span) - PAGEHDRSZ) /
sizeof(pgno_t)) /* - 1 + span */;
size_t avail = ((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) /* - 1 + span */;
if (tail > avail) {
for (size_t i = ctx->amount - span; i > 0; --i) {
if (MDBX_PNL_ASCENDING ? (txn->tw.relist[i] + span)
: (txn->tw.relist[i] - span) ==
txn->tw.relist[i + span]) {
: (txn->tw.relist[i] - span) == txn->tw.relist[i + span]) {
span += 1;
avail =
((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) -
1 + span;
avail = ((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) - 1 + span;
if (avail >= tail)
break;
}
}
}
chunk = (avail >= tail) ? tail - span
: (avail_gc_slots > 3 &&
ctx->reused_slot < prefer_max_scatter - 3)
? avail - span
: tail;
chunk = (avail >= tail) ? tail - span
: (avail_gc_slots > 3 && ctx->reused_slot < prefer_max_scatter - 3) ? avail - span
: tail;
}
}
#endif /* MDBX_ENABLE_BIGFOOT */
@ -859,57 +774,43 @@ retry:
"%" PRIaTXN,
dbg_prefix(ctx), ctx->rid, ctx->reused_slot, reservation_gc_id);
TRACE("%s: chunk %zu, gc-per-ovpage %u", dbg_prefix(ctx), chunk,
env->maxgc_large1page);
TRACE("%s: chunk %zu, gc-per-ovpage %u", dbg_prefix(ctx), chunk, env->maxgc_large1page);
tASSERT(txn, reservation_gc_id <= env->lck->cached_oldest.weak);
if (unlikely(reservation_gc_id < MIN_TXNID ||
reservation_gc_id >
atomic_load64(&env->lck->cached_oldest, mo_Relaxed))) {
ERROR("** internal error (reservation_gc_id %" PRIaTXN ")",
reservation_gc_id);
reservation_gc_id > atomic_load64(&env->lck->cached_oldest, mo_Relaxed))) {
ERROR("** internal error (reservation_gc_id %" PRIaTXN ")", reservation_gc_id);
rc = MDBX_PROBLEM;
goto bailout;
}
tASSERT(txn,
reservation_gc_id >= MIN_TXNID && reservation_gc_id <= MAX_TXNID);
tASSERT(txn, reservation_gc_id >= MIN_TXNID && reservation_gc_id <= MAX_TXNID);
key.iov_len = sizeof(reservation_gc_id);
key.iov_base = (void *)&reservation_gc_id;
data.iov_len = (chunk + 1) * sizeof(pgno_t);
TRACE("%s: reserve %zu [%zu...%zu) @%" PRIaTXN, dbg_prefix(ctx), chunk,
ctx->reserved + 1, ctx->reserved + chunk + 1, reservation_gc_id);
TRACE("%s: reserve %zu [%zu...%zu) @%" PRIaTXN, dbg_prefix(ctx), chunk, ctx->reserved + 1,
ctx->reserved + chunk + 1, reservation_gc_id);
prepare_backlog(txn, ctx);
rc = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE | MDBX_NOOVERWRITE);
tASSERT(txn,
pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated -
MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
zeroize_reserved(env, data);
ctx->reserved += chunk;
TRACE("%s: reserved %zu (+%zu), continue", dbg_prefix(ctx), ctx->reserved,
chunk);
TRACE("%s: reserved %zu (+%zu), continue", dbg_prefix(ctx), ctx->reserved, chunk);
continue;
}
tASSERT(
txn,
ctx->cleaned_slot ==
(txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) : 0));
tASSERT(txn, ctx->cleaned_slot == (txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) : 0));
TRACE("%s", " >> filling");
/* Fill in the reserved records */
size_t excess_slots = 0;
ctx->fill_idx =
txn->tw.gc.reclaimed
? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot
: ctx->reused_slot;
ctx->fill_idx = txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot : ctx->reused_slot;
rc = MDBX_SUCCESS;
tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated -
MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, dpl_check(txn));
if (ctx->amount) {
MDBX_val key, data;
@ -930,20 +831,17 @@ retry:
while (true) {
txnid_t fill_gc_id;
TRACE("%s: left %zu of %zu", dbg_prefix(ctx), left,
MDBX_PNL_GETSIZE(txn->tw.relist));
TRACE("%s: left %zu of %zu", dbg_prefix(ctx), left, MDBX_PNL_GETSIZE(txn->tw.relist));
if (txn->tw.gc.reclaimed == nullptr) {
tASSERT(txn, is_lifo(txn) == 0);
fill_gc_id =
key.iov_base ? unaligned_peek_u64(4, key.iov_base) : MIN_TXNID;
fill_gc_id = key.iov_base ? unaligned_peek_u64(4, key.iov_base) : MIN_TXNID;
if (ctx->fill_idx == 0 || fill_gc_id > txn->tw.gc.last_reclaimed) {
if (!left)
break;
NOTICE("** restart: reserve depleted (fill_idx %zu, fill_id %" PRIaTXN
" > last_reclaimed %" PRIaTXN ", left %zu",
NOTICE("** restart: reserve depleted (fill_idx %zu, fill_id %" PRIaTXN " > last_reclaimed %" PRIaTXN
", left %zu",
ctx->fill_idx, fill_gc_id, txn->tw.gc.last_reclaimed, left);
ctx->reserve_adj =
(ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0;
ctx->reserve_adj = (ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0;
goto retry;
}
ctx->fill_idx -= 1;
@ -955,26 +853,20 @@ retry:
NOTICE("** restart: reserve depleted (fill_idx %zu >= "
"gc.reclaimed %zu, left %zu",
ctx->fill_idx, MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed), left);
ctx->reserve_adj =
(ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0;
ctx->reserve_adj = (ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0;
goto retry;
}
ctx->fill_idx += 1;
fill_gc_id = txn->tw.gc.reclaimed[ctx->fill_idx];
TRACE("%s: seek-reservation @%" PRIaTXN " at gc.reclaimed[%zu]",
dbg_prefix(ctx), fill_gc_id, ctx->fill_idx);
TRACE("%s: seek-reservation @%" PRIaTXN " at gc.reclaimed[%zu]", dbg_prefix(ctx), fill_gc_id, ctx->fill_idx);
key.iov_base = &fill_gc_id;
key.iov_len = sizeof(fill_gc_id);
rc = cursor_seek(&ctx->cursor, &key, &data, MDBX_SET_KEY).err;
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
}
tASSERT(txn,
ctx->cleaned_slot == (txn->tw.gc.reclaimed
? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed)
: 0));
tASSERT(txn,
fill_gc_id > 0 && fill_gc_id <= env->lck->cached_oldest.weak);
tASSERT(txn, ctx->cleaned_slot == (txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) : 0));
tASSERT(txn, fill_gc_id > 0 && fill_gc_id <= env->lck->cached_oldest.weak);
key.iov_base = &fill_gc_id;
key.iov_len = sizeof(fill_gc_id);
@ -983,14 +875,12 @@ retry:
if (unlikely(chunk > left)) {
const size_t delta = chunk - left;
excess += delta;
TRACE("%s: chunk %zu > left %zu, @%" PRIaTXN, dbg_prefix(ctx), chunk,
left, fill_gc_id);
TRACE("%s: chunk %zu > left %zu, @%" PRIaTXN, dbg_prefix(ctx), chunk, left, fill_gc_id);
if (!left) {
excess_slots += 1;
goto next;
}
if ((ctx->loop < 5 && delta > (ctx->loop / 2)) ||
delta > env->maxgc_large1page)
if ((ctx->loop < 5 && delta > (ctx->loop / 2)) || delta > env->maxgc_large1page)
data.iov_len = (left + 1) * sizeof(pgno_t);
chunk = left;
}
@ -999,29 +889,23 @@ retry:
goto bailout;
zeroize_reserved(env, data);
if (unlikely(txn->tw.loose_count ||
ctx->amount != MDBX_PNL_GETSIZE(txn->tw.relist))) {
NOTICE("** restart: reclaimed-list changed (%zu -> %zu, loose +%zu)",
ctx->amount, MDBX_PNL_GETSIZE(txn->tw.relist),
txn->tw.loose_count);
if (unlikely(txn->tw.loose_count || ctx->amount != MDBX_PNL_GETSIZE(txn->tw.relist))) {
NOTICE("** restart: reclaimed-list changed (%zu -> %zu, loose +%zu)", ctx->amount,
MDBX_PNL_GETSIZE(txn->tw.relist), txn->tw.loose_count);
if (ctx->loop < 5 || (ctx->loop > 10 && (ctx->loop & 1)))
goto retry_clean_adj;
goto retry;
}
if (unlikely(txn->tw.gc.reclaimed
? ctx->cleaned_slot <
MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed)
: ctx->cleaned_id < txn->tw.gc.last_reclaimed)) {
if (unlikely(txn->tw.gc.reclaimed ? ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed)
: ctx->cleaned_id < txn->tw.gc.last_reclaimed)) {
NOTICE("%s", "** restart: reclaimed-slots changed");
goto retry;
}
if (unlikely(ctx->retired_stored !=
MDBX_PNL_GETSIZE(txn->tw.retired_pages))) {
tASSERT(txn,
ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages));
NOTICE("** restart: retired-list growth (%zu -> %zu)",
ctx->retired_stored, MDBX_PNL_GETSIZE(txn->tw.retired_pages));
if (unlikely(ctx->retired_stored != MDBX_PNL_GETSIZE(txn->tw.retired_pages))) {
tASSERT(txn, ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages));
NOTICE("** restart: retired-list growth (%zu -> %zu)", ctx->retired_stored,
MDBX_PNL_GETSIZE(txn->tw.retired_pages));
goto retry;
}
@ -1030,9 +914,8 @@ retry:
pgno_t *src = MDBX_PNL_BEGIN(txn->tw.relist) + left - chunk;
memcpy(dst, src, chunk * sizeof(pgno_t));
pgno_t *from = src, *to = src + chunk;
TRACE("%s: fill %zu [ %zu:%" PRIaPGNO "...%zu:%" PRIaPGNO "] @%" PRIaTXN,
dbg_prefix(ctx), chunk, from - txn->tw.relist, from[0],
to - txn->tw.relist, to[-1], fill_gc_id);
TRACE("%s: fill %zu [ %zu:%" PRIaPGNO "...%zu:%" PRIaPGNO "] @%" PRIaTXN, dbg_prefix(ctx), chunk,
from - txn->tw.relist, from[0], to - txn->tw.relist, to[-1], fill_gc_id);
left -= chunk;
if (AUDIT_ENABLED()) {
@ -1063,16 +946,14 @@ retry:
while (n >= env->maxgc_large1page)
adj -= n /= env->maxgc_large1page;
ctx->reserve_adj += adj;
TRACE("%s: extra %zu reserved space, adj +%zu (%zu)", dbg_prefix(ctx),
excess, adj, ctx->reserve_adj);
TRACE("%s: extra %zu reserved space, adj +%zu (%zu)", dbg_prefix(ctx), excess, adj, ctx->reserve_adj);
}
}
tASSERT(txn, rc == MDBX_SUCCESS);
if (unlikely(txn->tw.loose_count != 0 ||
ctx->amount != MDBX_PNL_GETSIZE(txn->tw.relist))) {
NOTICE("** restart: got %zu loose pages (reclaimed-list %zu -> %zu)",
txn->tw.loose_count, ctx->amount, MDBX_PNL_GETSIZE(txn->tw.relist));
if (unlikely(txn->tw.loose_count != 0 || ctx->amount != MDBX_PNL_GETSIZE(txn->tw.relist))) {
NOTICE("** restart: got %zu loose pages (reclaimed-list %zu -> %zu)", txn->tw.loose_count, ctx->amount,
MDBX_PNL_GETSIZE(txn->tw.relist));
goto retry;
}
@ -1080,14 +961,12 @@ retry:
const bool will_retry = ctx->loop < 5 || excess_slots > 1;
NOTICE("** %s: reserve excess (excess-slots %zu, filled-slot %zu, adj %zu, "
"loop %u)",
will_retry ? "restart" : "ignore", excess_slots, ctx->fill_idx,
ctx->reserve_adj, ctx->loop);
will_retry ? "restart" : "ignore", excess_slots, ctx->fill_idx, ctx->reserve_adj, ctx->loop);
if (will_retry)
goto retry;
}
tASSERT(txn, txn->tw.gc.reclaimed == nullptr ||
ctx->cleaned_slot == MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed));
tASSERT(txn, txn->tw.gc.reclaimed == nullptr || ctx->cleaned_slot == MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed));
bailout:
txn->cursors[FREE_DBI] = ctx->cursor.next;

3
src/gc.h
View File

@ -34,8 +34,7 @@ static inline int gc_update_init(MDBX_txn *txn, gcu_t *ctx) {
#define ALLOC_DEFAULT 0
#define ALLOC_RESERVE 1
#define ALLOC_UNIMPORTANT 2
MDBX_INTERNAL pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num,
uint8_t flags);
MDBX_INTERNAL pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags);
MDBX_INTERNAL pgr_t gc_alloc_single(const MDBX_cursor *const mc);
MDBX_INTERNAL int gc_update(MDBX_txn *txn, gcu_t *ctx);

17
src/global.c
View File

@ -122,8 +122,7 @@ extern void __gmon_start__(void) __attribute__((__weak__));
#endif /* ENABLE_GPROF */
MDBX_EXCLUDE_FOR_GPROF
__cold static __attribute__((__constructor__)) void
mdbx_global_constructor(void) {
__cold static __attribute__((__constructor__)) void mdbx_global_constructor(void) {
#ifdef ENABLE_GPROF
if (!&__gmon_start__)
monstartup((uintptr_t)&_init, (uintptr_t)&_fini);
@ -154,9 +153,8 @@ mdbx_global_constructor(void) {
* So, the REQUIREMENTS for this code:
* 1. MUST detect WSL1 without false-negatives.
* 2. DESIRABLE detect WSL2 but without the risk of violating the first. */
globals.running_on_WSL1 = probe_for_WSL(buffer.version) == 1 ||
probe_for_WSL(buffer.sysname) == 1 ||
probe_for_WSL(buffer.release) == 1;
globals.running_on_WSL1 =
probe_for_WSL(buffer.version) == 1 || probe_for_WSL(buffer.sysname) == 1 || probe_for_WSL(buffer.release) == 1;
}
#endif /* Linux */
@ -164,8 +162,7 @@ mdbx_global_constructor(void) {
}
MDBX_EXCLUDE_FOR_GPROF
__cold static __attribute__((__destructor__)) void
mdbx_global_destructor(void) {
__cold static __attribute__((__destructor__)) void mdbx_global_destructor(void) {
mdbx_fini();
#ifdef ENABLE_GPROF
if (!&__gmon_start__)
@ -180,13 +177,11 @@ mdbx_global_destructor(void) {
struct libmdbx_globals globals;
__cold static void mdbx_init(void) {
globals.runtime_flags = ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT +
((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT;
globals.runtime_flags = ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT;
globals.loglevel = MDBX_LOG_FATAL;
ENSURE(nullptr, osal_fastmutex_init(&globals.debug_lock) == 0);
osal_ctor();
assert(globals.sys_pagesize > 0 &&
(globals.sys_pagesize & (globals.sys_pagesize - 1)) == 0);
assert(globals.sys_pagesize > 0 && (globals.sys_pagesize & (globals.sys_pagesize - 1)) == 0);
rthc_ctor();
#if MDBX_DEBUG
ENSURE(nullptr, troika_verify_fsm());

54
src/internals.h
View File

@ -156,9 +156,8 @@ enum txn_flags {
txn_shrink_allowed = UINT32_C(0x40000000),
txn_parked = MDBX_TXN_PARKED,
txn_gc_drained = 0x40 /* GC was depleted up to oldest reader */,
txn_state_flags = MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY |
MDBX_TXN_SPILLS | MDBX_TXN_HAS_CHILD | MDBX_TXN_INVALID |
txn_gc_drained
txn_state_flags = MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | MDBX_TXN_HAS_CHILD |
MDBX_TXN_INVALID | txn_gc_drained
};
/* A database transaction.
@ -336,12 +335,9 @@ enum env_flags {
/* Only a subset of the mdbx_env flags can be changed
* at runtime. Changing other flags requires closing the
* environment and re-opening it with the new flags. */
ENV_CHANGEABLE_FLAGS = MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC |
DEPRECATED_MAPASYNC | MDBX_NOMEMINIT |
DEPRECATED_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE |
MDBX_VALIDATION,
ENV_CHANGELESS_FLAGS = MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP |
MDBX_NOSTICKYTHREADS | MDBX_NORDAHEAD |
ENV_CHANGEABLE_FLAGS = MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | DEPRECATED_MAPASYNC | MDBX_NOMEMINIT |
DEPRECATED_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE | MDBX_VALIDATION,
ENV_CHANGELESS_FLAGS = MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOSTICKYTHREADS | MDBX_NORDAHEAD |
MDBX_LIFORECLAIM | MDBX_EXCLUSIVE,
ENV_USABLE_FLAGS = ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS
};
@ -368,8 +364,8 @@ struct MDBX_env {
uint16_t subpage_reserve_prereq;
uint16_t subpage_reserve_limit;
atomic_pgno_t mlocked_pgno;
uint8_t ps2ln; /* log2 of DB page size */
int8_t stuck_meta; /* recovery-only: target meta page or less that zero */
uint8_t ps2ln; /* log2 of DB page size */
int8_t stuck_meta; /* recovery-only: target meta page or less that zero */
uint16_t merge_threshold, merge_threshold_gc; /* pages emptier than this are
candidates for merging */
unsigned max_readers; /* size of the reader table */
@ -385,7 +381,7 @@ struct MDBX_env {
kvx_t *kvs; /* array of auxiliary key-value properties */
uint8_t *__restrict dbs_flags; /* array of flags from tree_t.flags */
mdbx_atomic_uint32_t *dbi_seqs; /* array of dbi sequence numbers */
unsigned maxgc_large1page; /* Number of pgno_t fit in a single large page */
unsigned maxgc_large1page; /* Number of pgno_t fit in a single large page */
unsigned maxgc_per_branch;
uint32_t registered_reader_pid; /* have liveness lock in reader table */
void *userctx; /* User-settable context */
@ -492,9 +488,7 @@ struct MDBX_env {
#endif
/* ------------------------------------------------- stub for lck-less mode */
mdbx_atomic_uint64_t
lckless_placeholder[(sizeof(lck_t) + MDBX_CACHELINE_SIZE - 1) /
sizeof(mdbx_atomic_uint64_t)];
mdbx_atomic_uint64_t lckless_placeholder[(sizeof(lck_t) + MDBX_CACHELINE_SIZE - 1) / sizeof(mdbx_atomic_uint64_t)];
};
/*----------------------------------------------------------------------------*/
@ -509,8 +503,8 @@ struct MDBX_env {
#define DEFAULT_READERS 61
enum db_flags {
DB_PERSISTENT_FLAGS = MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY |
MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP,
DB_PERSISTENT_FLAGS =
MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP,
/* mdbx_dbi_open() flags */
DB_USABLE_FLAGS = DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE,
@ -524,27 +518,19 @@ enum db_flags {
MDBX_MAYBE_UNUSED static void static_checks(void) {
STATIC_ASSERT(MDBX_WORDBITS == sizeof(void *) * CHAR_BIT);
STATIC_ASSERT(UINT64_C(0x80000000) == (uint32_t)ENV_FATAL_ERROR);
STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI,
"Oops, MDBX_MAX_DBI or CORE_DBS?");
STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, "Oops, MDBX_MAX_DBI or CORE_DBS?");
STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) ==
((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) &
(ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)),
"Oops, some flags overlapped or wrong");
STATIC_ASSERT_MSG((DB_INTERNAL_FLAGS & DB_USABLE_FLAGS) == 0,
"Oops, some flags overlapped or wrong");
STATIC_ASSERT_MSG((DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS) == 0,
((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)),
"Oops, some flags overlapped or wrong");
STATIC_ASSERT_MSG((DB_INTERNAL_FLAGS & DB_USABLE_FLAGS) == 0, "Oops, some flags overlapped or wrong");
STATIC_ASSERT_MSG((DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS) == 0, "Oops, some flags overlapped or wrong");
STATIC_ASSERT(DB_PERSISTENT_FLAGS <= UINT8_MAX);
STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0,
"Oops, some flags overlapped or wrong");
STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, "Oops, some flags overlapped or wrong");
STATIC_ASSERT_MSG(
(txn_state_flags & (txn_rw_begin_flags | txn_ro_begin_flags)) == 0,
"Oops, some txn flags overlapped or wrong");
STATIC_ASSERT_MSG(
((txn_rw_begin_flags | txn_ro_begin_flags | txn_state_flags) &
txn_shrink_allowed) == 0,
"Oops, some txn flags overlapped or wrong");
STATIC_ASSERT_MSG((txn_state_flags & (txn_rw_begin_flags | txn_ro_begin_flags)) == 0,
"Oops, some txn flags overlapped or wrong");
STATIC_ASSERT_MSG(((txn_rw_begin_flags | txn_ro_begin_flags | txn_state_flags) & txn_shrink_allowed) == 0,
"Oops, some txn flags overlapped or wrong");
STATIC_ASSERT(sizeof(reader_slot_t) == 32);
#if MDBX_LOCKING > 0

32
src/layout-dxb.h
View File

@ -17,10 +17,8 @@
/* FROZEN: The version number for a database's datafile format. */
#define MDBX_DATA_VERSION 3
#define MDBX_DATA_MAGIC \
((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION)
#define MDBX_DATA_MAGIC_LEGACY_COMPAT \
((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2)
#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION)
#define MDBX_DATA_MAGIC_LEGACY_COMPAT ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2)
#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255)
/* handle for the DB used to track free pages. */
@ -261,40 +259,30 @@ typedef enum node_flags {
#pragma pack(pop)
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t
page_type(const page_t *mp) {
return mp->flags;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type(const page_t *mp) { return mp->flags; }
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t
page_type_compat(const page_t *mp) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type_compat(const page_t *mp) {
/* Drop legacy P_DIRTY flag for sub-pages for compatilibity,
* for assertions only. */
return unlikely(mp->flags & P_SUBP) ? mp->flags & ~(P_SUBP | P_LEGACY_DIRTY)
: mp->flags;
return unlikely(mp->flags & P_SUBP) ? mp->flags & ~(P_SUBP | P_LEGACY_DIRTY) : mp->flags;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
is_leaf(const page_t *mp) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_leaf(const page_t *mp) {
return (mp->flags & P_LEAF) != 0;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
is_dupfix_leaf(const page_t *mp) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_dupfix_leaf(const page_t *mp) {
return (mp->flags & P_DUPFIX) != 0;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
is_branch(const page_t *mp) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_branch(const page_t *mp) {
return (mp->flags & P_BRANCH) != 0;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
is_largepage(const page_t *mp) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_largepage(const page_t *mp) {
return (mp->flags & P_LARGE) != 0;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
is_subpage(const page_t *mp) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_subpage(const page_t *mp) {
return (mp->flags & P_SUBP) != 0;
}

30
src/layout-lck.h
View File

@ -19,8 +19,7 @@ typedef void osal_ipclock_t;
#define MDBX_LCK_SIGN UINT32_C(0xF18D)
typedef mdbx_pid_t osal_ipclock_t;
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
MDBX_LOCKING == MDBX_LOCKING_POSIX2008
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008
#define MDBX_LCK_SIGN UINT32_C(0x8017)
typedef pthread_mutex_t osal_ipclock_t;
@ -64,19 +63,15 @@ typedef struct pgops {
mdbx_atomic_uint64_t merge; /* Page merges */
mdbx_atomic_uint64_t spill; /* Quantity of spilled dirty pages */
mdbx_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */
mdbx_atomic_uint64_t
wops; /* Number of explicit write operations (not a pages) to a disk */
mdbx_atomic_uint64_t
msync; /* Number of explicit msync/flush-to-disk operations */
mdbx_atomic_uint64_t
fsync; /* Number of explicit fsync/flush-to-disk operations */
mdbx_atomic_uint64_t wops; /* Number of explicit write operations (not a pages) to a disk */
mdbx_atomic_uint64_t msync; /* Number of explicit msync/flush-to-disk operations */
mdbx_atomic_uint64_t fsync; /* Number of explicit fsync/flush-to-disk operations */
mdbx_atomic_uint64_t prefault; /* Number of prefault write operations */
mdbx_atomic_uint64_t mincore; /* Number of mincore() calls */
mdbx_atomic_uint32_t
incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269
caught */
mdbx_atomic_uint32_t incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269
caught */
mdbx_atomic_uint32_t reserved;
/* Статистика для профилирования GC.
@ -202,8 +197,7 @@ typedef struct shared_lck {
* i.e. for sync-polling in the MDBX_NOMETASYNC mode. */
#define MDBX_NOMETASYNC_LAZY_UNK (UINT32_MAX / 3)
#define MDBX_NOMETASYNC_LAZY_FD (MDBX_NOMETASYNC_LAZY_UNK + UINT32_MAX / 8)
#define MDBX_NOMETASYNC_LAZY_WRITEMAP \
(MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8)
#define MDBX_NOMETASYNC_LAZY_WRITEMAP (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8)
mdbx_atomic_uint32_t meta_sync_txnid;
/* Period for timed auto-sync feature, i.e. at the every steady checkpoint
@ -277,12 +271,10 @@ typedef struct shared_lck {
reader_slot_t rdt[] /* dynamic size */;
/* Lockfile format signature: version, features and field layout */
#define MDBX_LOCK_FORMAT \
(MDBX_LCK_SIGN * 27733 + (unsigned)sizeof(reader_slot_t) * 13 + \
(unsigned)offsetof(reader_slot_t, snapshot_pages_used) * 251 + \
(unsigned)offsetof(lck_t, cached_oldest) * 83 + \
(unsigned)offsetof(lck_t, rdt_length) * 37 + \
(unsigned)offsetof(lck_t, rdt) * 29)
#define MDBX_LOCK_FORMAT \
(MDBX_LCK_SIGN * 27733 + (unsigned)sizeof(reader_slot_t) * 13 + \
(unsigned)offsetof(reader_slot_t, snapshot_pages_used) * 251 + (unsigned)offsetof(lck_t, cached_oldest) * 83 + \
(unsigned)offsetof(lck_t, rdt_length) * 37 + (unsigned)offsetof(lck_t, rdt) * 29)
#endif /* FLEXIBLE_ARRAY_MEMBERS */
} lck_t;

161
src/lck-posix.c
View File

@ -71,11 +71,10 @@ __cold static void choice_fcntl(void) {
assert(!op_setlk && !op_setlkw && !op_getlk);
if ((globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0
#if defined(__linux__) || defined(__gnu_linux__)
&& globals.linux_kernel_version >
0x030f0000 /* OFD locks are available since 3.15, but engages here
only for 3.16 and later kernels (i.e. LTS) because
of reliability reasons */
#endif /* linux */
&& globals.linux_kernel_version > 0x030f0000 /* OFD locks are available since 3.15, but engages here
only for 3.16 and later kernels (i.e. LTS) because
of reliability reasons */
#endif /* linux */
) {
op_setlk = MDBX_F_OFD_SETLK;
op_setlkw = MDBX_F_OFD_SETLKW;
@ -92,32 +91,25 @@ __cold static void choice_fcntl(void) {
#define op_getlk MDBX_F_GETLK
#endif /* MDBX_USE_OFDLOCKS */
static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck,
const off_t offset, off_t len) {
STATIC_ASSERT(sizeof(off_t) >= sizeof(void *) &&
sizeof(off_t) >= sizeof(size_t));
static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck, const off_t offset, off_t len) {
STATIC_ASSERT(sizeof(off_t) >= sizeof(void *) && sizeof(off_t) >= sizeof(size_t));
#ifdef __ANDROID_API__
STATIC_ASSERT_MSG((sizeof(off_t) * 8 == MDBX_WORDBITS),
"The bitness of system `off_t` type is mismatch. Please "
"fix build and/or NDK configuration.");
STATIC_ASSERT_MSG((sizeof(off_t) * 8 == MDBX_WORDBITS), "The bitness of system `off_t` type is mismatch. Please "
"fix build and/or NDK configuration.");
#endif /* Android */
assert(offset >= 0 && len > 0);
assert((uint64_t)offset < (uint64_t)INT64_MAX &&
(uint64_t)len < (uint64_t)INT64_MAX &&
assert((uint64_t)offset < (uint64_t)INT64_MAX && (uint64_t)len < (uint64_t)INT64_MAX &&
(uint64_t)(offset + len) > (uint64_t)offset);
assert((uint64_t)offset < (uint64_t)OFF_T_MAX &&
(uint64_t)len <= (uint64_t)OFF_T_MAX &&
assert((uint64_t)offset < (uint64_t)OFF_T_MAX && (uint64_t)len <= (uint64_t)OFF_T_MAX &&
(uint64_t)(offset + len) <= (uint64_t)OFF_T_MAX);
assert((uint64_t)((off_t)((uint64_t)offset + (uint64_t)len)) ==
((uint64_t)offset + (uint64_t)len));
assert((uint64_t)((off_t)((uint64_t)offset + (uint64_t)len)) == ((uint64_t)offset + (uint64_t)len));
jitter4testing(true);
for (;;) {
MDBX_STRUCT_FLOCK lock_op;
STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(lock_op.l_start) &&
sizeof(off_t) <= sizeof(lock_op.l_len) &&
STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(lock_op.l_start) && sizeof(off_t) <= sizeof(lock_op.l_len) &&
OFF_T_MAX == (off_t)OFF_T_MAX,
"Support for large/64-bit-sized files is misconfigured "
"for the target system and/or toolchain. "
@ -134,15 +126,13 @@ static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck,
/* Checks reader by pid. Returns:
* MDBX_RESULT_TRUE - if pid is live (reader holds a lock).
* MDBX_RESULT_FALSE - if pid is dead (a lock could be placed). */
return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE
: MDBX_RESULT_TRUE;
return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE;
}
return MDBX_SUCCESS;
}
rc = errno;
#if MDBX_USE_OFDLOCKS
if (rc == EINVAL && (cmd == MDBX_F_OFD_SETLK || cmd == MDBX_F_OFD_SETLKW ||
cmd == MDBX_F_OFD_GETLK)) {
if (rc == EINVAL && (cmd == MDBX_F_OFD_SETLK || cmd == MDBX_F_OFD_SETLKW || cmd == MDBX_F_OFD_GETLK)) {
/* fallback to non-OFD locks */
if (cmd == MDBX_F_OFD_SETLK)
cmd = MDBX_F_SETLK;
@ -197,8 +187,7 @@ MDBX_INTERNAL int lck_rpid_check(MDBX_env *env, uint32_t pid) {
MDBX_INTERNAL int lck_ipclock_stubinit(osal_ipclock_t *ipc) {
#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988
return sem_init(ipc, false, 1) ? errno : 0;
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
MDBX_LOCKING == MDBX_LOCKING_POSIX2008
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008
return pthread_mutex_init(ipc, nullptr);
#else
#error "FIXME"
@ -208,8 +197,7 @@ MDBX_INTERNAL int lck_ipclock_stubinit(osal_ipclock_t *ipc) {
MDBX_INTERNAL int lck_ipclock_destroy(osal_ipclock_t *ipc) {
#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988
return sem_destroy(ipc) ? errno : 0;
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
MDBX_LOCKING == MDBX_LOCKING_POSIX2008
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008
return pthread_mutex_destroy(ipc);
#else
#error "FIXME"
@ -233,14 +221,12 @@ static int check_fstat(MDBX_env *env) {
#else
rc = EPERM;
#endif
ERROR("%s %s, err %d", "DXB",
(st.st_nlink < 1) ? "file was removed" : "not a regular file", rc);
ERROR("%s %s, err %d", "DXB", (st.st_nlink < 1) ? "file was removed" : "not a regular file", rc);
return rc;
}
if (st.st_size < (off_t)(MDBX_MIN_PAGESIZE * NUM_METAS)) {
VERBOSE("dxb-file is too short (%u), exclusive-lock needed",
(unsigned)st.st_size);
VERBOSE("dxb-file is too short (%u), exclusive-lock needed", (unsigned)st.st_size);
rc = MDBX_RESULT_TRUE;
}
@ -258,16 +244,14 @@ static int check_fstat(MDBX_env *env) {
#else
rc = EPERM;
#endif
ERROR("%s %s, err %d", "LCK",
(st.st_nlink < 1) ? "file was removed" : "not a regular file", rc);
ERROR("%s %s, err %d", "LCK", (st.st_nlink < 1) ? "file was removed" : "not a regular file", rc);
return rc;
}
/* Checking file size for detect the situation when we got the shared lock
* immediately after lck_destroy(). */
if (st.st_size < (off_t)(sizeof(lck_t) + sizeof(reader_slot_t))) {
VERBOSE("lck-file is too short (%u), exclusive-lock needed",
(unsigned)st.st_size);
VERBOSE("lck-file is too short (%u), exclusive-lock needed", (unsigned)st.st_size);
rc = MDBX_RESULT_TRUE;
}
@ -298,8 +282,7 @@ __cold MDBX_INTERNAL int lck_seize(MDBX_env *env) {
if (env->lck_mmap.fd == INVALID_HANDLE_VALUE) {
/* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */
rc = lck_op(env->lazy_fd, op_setlk,
(env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX);
rc = lck_op(env->lazy_fd, op_setlk, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX);
if (rc != MDBX_SUCCESS) {
ERROR("%s, err %u", "without-lck", rc);
eASSERT(env, MDBX_IS_ERROR(rc));
@ -329,8 +312,7 @@ retry:
return rc;
continue_dxb_exclusive:
rc = lck_op(env->lazy_fd, op_setlk,
(env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX);
rc = lck_op(env->lazy_fd, op_setlk, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX);
if (rc == MDBX_SUCCESS)
return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */;
@ -339,16 +321,14 @@ retry:
return err;
/* the cause may be a collision with POSIX's file-lock recovery. */
if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK ||
rc == EDEADLK)) {
if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || rc == EDEADLK)) {
ERROR("%s, err %u", "dxb-exclusive", rc);
eASSERT(env, MDBX_IS_ERROR(rc));
return rc;
}
/* Fallback to lck-shared */
} else if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY ||
rc == EWOULDBLOCK || rc == EDEADLK)) {
} else if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || rc == EDEADLK)) {
ERROR("%s, err %u", "try-exclusive", rc);
eASSERT(env, MDBX_IS_ERROR(rc));
return rc;
@ -384,16 +364,14 @@ retry:
if (rc == MDBX_SUCCESS)
goto continue_dxb_exclusive;
if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK ||
rc == EDEADLK)) {
if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || rc == EDEADLK)) {
ERROR("%s, err %u", "try-exclusive", rc);
eASSERT(env, MDBX_IS_ERROR(rc));
return rc;
}
/* Lock against another process operating in without-lck or exclusive mode. */
rc = lck_op(env->lazy_fd, op_setlk,
(env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, env->pid, 1);
rc = lck_op(env->lazy_fd, op_setlk, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, env->pid, 1);
if (rc != MDBX_SUCCESS) {
ERROR("%s, err %u", "lock-against-without-lck", rc);
eASSERT(env, MDBX_IS_ERROR(rc));
@ -413,8 +391,7 @@ MDBX_INTERNAL int lck_downgrade(MDBX_env *env) {
if ((env->flags & MDBX_EXCLUSIVE) == 0) {
rc = lck_op(env->lazy_fd, op_setlk, F_UNLCK, 0, env->pid);
if (rc == MDBX_SUCCESS)
rc = lck_op(env->lazy_fd, op_setlk, F_UNLCK, env->pid + 1,
OFF_T_MAX - env->pid - 1);
rc = lck_op(env->lazy_fd, op_setlk, F_UNLCK, env->pid + 1, OFF_T_MAX - env->pid - 1);
}
if (rc == MDBX_SUCCESS)
rc = lck_op(env->lck_mmap.fd, op_setlk, F_RDLCK, 0, 1);
@ -433,13 +410,10 @@ MDBX_INTERNAL int lck_upgrade(MDBX_env *env, bool dont_wait) {
const int cmd = dont_wait ? op_setlk : op_setlkw;
int rc = lck_op(env->lck_mmap.fd, cmd, F_WRLCK, 0, 1);
if (rc == MDBX_SUCCESS && (env->flags & MDBX_EXCLUSIVE) == 0) {
rc = (env->pid > 1) ? lck_op(env->lazy_fd, cmd, F_WRLCK, 0, env->pid - 1)
: MDBX_SUCCESS;
rc = (env->pid > 1) ? lck_op(env->lazy_fd, cmd, F_WRLCK, 0, env->pid - 1) : MDBX_SUCCESS;
if (rc == MDBX_SUCCESS) {
rc = lck_op(env->lazy_fd, cmd, F_WRLCK, env->pid + 1,
OFF_T_MAX - env->pid - 1);
if (rc != MDBX_SUCCESS && env->pid > 1 &&
lck_op(env->lazy_fd, op_setlk, F_UNLCK, 0, env->pid - 1))
rc = lck_op(env->lazy_fd, cmd, F_WRLCK, env->pid + 1, OFF_T_MAX - env->pid - 1);
if (rc != MDBX_SUCCESS && env->pid > 1 && lck_op(env->lazy_fd, op_setlk, F_UNLCK, 0, env->pid - 1))
rc = MDBX_PANIC;
}
if (rc != MDBX_SUCCESS && lck_op(env->lck_mmap.fd, op_setlk, F_RDLCK, 0, 1))
@ -452,9 +426,7 @@ MDBX_INTERNAL int lck_upgrade(MDBX_env *env, bool dont_wait) {
return rc;
}
__cold MDBX_INTERNAL int lck_destroy(MDBX_env *env,
MDBX_env *inprocess_neighbor,
const uint32_t current_pid) {
__cold MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, const uint32_t current_pid) {
eASSERT(env, osal_getpid() == current_pid);
int rc = MDBX_SUCCESS;
struct stat lck_info;
@ -464,9 +436,7 @@ __cold MDBX_INTERNAL int lck_destroy(MDBX_env *env,
lck_op(env->lck_mmap.fd, op_setlk, F_WRLCK, 0, OFF_T_MAX) == 0 &&
/* if LCK was not removed */
fstat(env->lck_mmap.fd, &lck_info) == 0 && lck_info.st_nlink > 0 &&
lck_op(env->lazy_fd, op_setlk,
(env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0,
OFF_T_MAX) == 0) {
lck_op(env->lazy_fd, op_setlk, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX) == 0) {
VERBOSE("%p got exclusive, drown ipc-locks", (void *)env);
eASSERT(env, current_pid == env->pid);
@ -492,8 +462,7 @@ __cold MDBX_INTERNAL int lck_destroy(MDBX_env *env,
if (current_pid != env->pid) {
eASSERT(env, !inprocess_neighbor);
NOTICE("drown env %p after-fork pid %d -> %d",
__Wpedantic_format_voidptr(env), env->pid, current_pid);
NOTICE("drown env %p after-fork pid %d -> %d", __Wpedantic_format_voidptr(env), env->pid, current_pid);
inprocess_neighbor = nullptr;
}
@ -516,11 +485,8 @@ __cold MDBX_INTERNAL int lck_destroy(MDBX_env *env,
env->lazy_fd = INVALID_HANDLE_VALUE;
if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) {
/* restore file-lock */
rc = lck_op(inprocess_neighbor->lazy_fd, F_SETLKW,
(inprocess_neighbor->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK,
(inprocess_neighbor->flags & MDBX_EXCLUSIVE)
? 0
: inprocess_neighbor->pid,
rc = lck_op(inprocess_neighbor->lazy_fd, F_SETLKW, (inprocess_neighbor->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK,
(inprocess_neighbor->flags & MDBX_EXCLUSIVE) ? 0 : inprocess_neighbor->pid,
(inprocess_neighbor->flags & MDBX_EXCLUSIVE) ? OFF_T_MAX : 1);
}
}
@ -545,8 +511,7 @@ __cold MDBX_INTERNAL int lck_destroy(MDBX_env *env,
/*---------------------------------------------------------------------------*/
__cold MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor,
int global_uniqueness_flag) {
__cold MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, int global_uniqueness_flag) {
#if MDBX_LOCKING == MDBX_LOCKING_SYSV
int semid = -1;
/* don't initialize semaphores twice */
@ -556,9 +521,7 @@ __cold MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor,
if (fstat(env->lazy_fd, &st))
return errno;
sysv_retry_create:
semid = semget(env->me_sysv_ipc.key, 2,
IPC_CREAT | IPC_EXCL |
(st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)));
semid = semget(env->me_sysv_ipc.key, 2, IPC_CREAT | IPC_EXCL | (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)));
if (unlikely(semid == -1)) {
int err = errno;
if (err != EEXIST)
@ -614,8 +577,7 @@ __cold MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor,
}
return MDBX_SUCCESS;
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
MDBX_LOCKING == MDBX_LOCKING_POSIX2008
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008
if (inprocess_neighbor)
return MDBX_SUCCESS /* don't need any initialization for mutexes
if LCK already opened/used inside current process */
@ -653,8 +615,7 @@ __cold MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor,
#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008
#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutexattr_setrobust)
rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST);
#elif defined(PTHREAD_MUTEX_ROBUST_NP) || \
defined(pthread_mutexattr_setrobust_np)
#elif defined(PTHREAD_MUTEX_ROBUST_NP) || defined(pthread_mutexattr_setrobust_np)
rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP);
#elif _POSIX_THREAD_PROCESS_SHARED < 200809L
rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP);
@ -665,8 +626,7 @@ __cold MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor,
goto bailout;
#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */
#if defined(_POSIX_THREAD_PRIO_INHERIT) && _POSIX_THREAD_PRIO_INHERIT >= 0 && \
!defined(MDBX_SAFE4QEMU)
#if defined(_POSIX_THREAD_PRIO_INHERIT) && _POSIX_THREAD_PRIO_INHERIT >= 0 && !defined(MDBX_SAFE4QEMU)
rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT);
if (rc == ENOTSUP)
rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE);
@ -691,8 +651,7 @@ bailout:
#endif /* MDBX_LOCKING > 0 */
}
__cold static int osal_ipclock_failed(MDBX_env *env, osal_ipclock_t *ipc,
const int err) {
__cold static int osal_ipclock_failed(MDBX_env *env, osal_ipclock_t *ipc, const int err) {
int rc = err;
#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008 || MDBX_LOCKING == MDBX_LOCKING_SYSV
@ -712,8 +671,7 @@ __cold static int osal_ipclock_failed(MDBX_env *env, osal_ipclock_t *ipc,
rc = MDBX_PANIC;
}
}
WARNING("%clock owner died, %s", (rlocked ? 'r' : 'w'),
(rc ? "this process' env is hosed" : "recovering"));
WARNING("%clock owner died, %s", (rlocked ? 'r' : 'w'), (rc ? "this process' env is hosed" : "recovering"));
int check_rc = mvcc_cleanup_dead(env, rlocked, nullptr);
check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc;
@ -781,10 +739,8 @@ MDBX_INTERNAL int osal_check_tid4bionic(void) {
}
#endif /* __ANDROID_API__ || ANDROID) || BIONIC */
static int osal_ipclock_lock(MDBX_env *env, osal_ipclock_t *ipc,
const bool dont_wait) {
#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
MDBX_LOCKING == MDBX_LOCKING_POSIX2008
static int osal_ipclock_lock(MDBX_env *env, osal_ipclock_t *ipc, const bool dont_wait) {
#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008
int rc = osal_check_tid4bionic();
if (likely(rc == 0))
rc = dont_wait ? pthread_mutex_trylock(ipc) : pthread_mutex_lock(ipc);
@ -800,9 +756,8 @@ static int osal_ipclock_lock(MDBX_env *env, osal_ipclock_t *ipc,
} else if (sem_wait(ipc))
rc = errno;
#elif MDBX_LOCKING == MDBX_LOCKING_SYSV
struct sembuf op = {.sem_num = (ipc != &env->lck->wrt_lock),
.sem_op = -1,
.sem_flg = dont_wait ? IPC_NOWAIT | SEM_UNDO : SEM_UNDO};
struct sembuf op = {
.sem_num = (ipc != &env->lck->wrt_lock), .sem_op = -1, .sem_flg = dont_wait ? IPC_NOWAIT | SEM_UNDO : SEM_UNDO};
int rc;
if (semop(env->me_sysv_ipc.semid, &op, 1)) {
rc = errno;
@ -823,8 +778,7 @@ static int osal_ipclock_lock(MDBX_env *env, osal_ipclock_t *ipc,
int osal_ipclock_unlock(MDBX_env *env, osal_ipclock_t *ipc) {
int err = MDBX_ENOSYS;
#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \
MDBX_LOCKING == MDBX_LOCKING_POSIX2008
#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008
err = pthread_mutex_unlock(ipc);
#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988
err = sem_post(ipc) ? errno : MDBX_SUCCESS;
@ -833,9 +787,7 @@ int osal_ipclock_unlock(MDBX_env *env, osal_ipclock_t *ipc) {
err = EPERM;
else {
*ipc = 0;
struct sembuf op = {.sem_num = (ipc != &env->lck->wrt_lock),
.sem_op = 1,
.sem_flg = SEM_UNDO};
struct sembuf op = {.sem_num = (ipc != &env->lck->wrt_lock), .sem_op = 1, .sem_flg = SEM_UNDO};
err = semop(env->me_sysv_ipc.semid, &op, 1) ? errno : MDBX_SUCCESS;
}
#else
@ -845,13 +797,9 @@ int osal_ipclock_unlock(MDBX_env *env, osal_ipclock_t *ipc) {
if (unlikely(rc != MDBX_SUCCESS)) {
const uint32_t current_pid = osal_getpid();
if (current_pid == env->pid || LOG_ENABLED(MDBX_LOG_NOTICE))
debug_log((current_pid == env->pid)
? MDBX_LOG_FATAL
: (rc = MDBX_SUCCESS, MDBX_LOG_NOTICE),
"ipc-unlock()", __LINE__, "failed: env %p, lck-%s %p, err %d\n",
__Wpedantic_format_voidptr(env),
(env->lck == env->lck_mmap.lck) ? "mmap" : "stub",
__Wpedantic_format_voidptr(env->lck), err);
debug_log((current_pid == env->pid) ? MDBX_LOG_FATAL : (rc = MDBX_SUCCESS, MDBX_LOG_NOTICE), "ipc-unlock()",
__LINE__, "failed: env %p, lck-%s %p, err %d\n", __Wpedantic_format_voidptr(env),
(env->lck == env->lck_mmap.lck) ? "mmap" : "stub", __Wpedantic_format_voidptr(env->lck), err);
}
return rc;
}
@ -879,10 +827,9 @@ int lck_txn_lock(MDBX_env *env, bool dont_wait) {
const int err = osal_ipclock_lock(env, &env->lck->wrt_lock, dont_wait);
int rc = err;
if (likely(!MDBX_IS_ERROR(err))) {
eASSERT(env, !env->basal_txn->owner ||
err == /* если другой поток в этом-же процессе завершился
не освободив блокировку */
MDBX_RESULT_TRUE);
eASSERT(env, !env->basal_txn->owner || err == /* если другой поток в этом-же процессе завершился
не освободив блокировку */
MDBX_RESULT_TRUE);
env->basal_txn->owner = osal_thread_self();
rc = MDBX_SUCCESS;
}

161
src/lck-windows.c
View File

@ -16,10 +16,8 @@
#define LCK_WAITFOR 0
#define LCK_DONTWAIT LOCKFILE_FAIL_IMMEDIATELY
static int flock_with_event(HANDLE fd, HANDLE event, unsigned flags,
size_t offset, size_t bytes) {
TRACE("lock>>: fd %p, event %p, flags 0x%x offset %zu, bytes %zu >>", fd,
event, flags, offset, bytes);
static int flock_with_event(HANDLE fd, HANDLE event, unsigned flags, size_t offset, size_t bytes) {
TRACE("lock>>: fd %p, event %p, flags 0x%x offset %zu, bytes %zu >>", fd, event, flags, offset, bytes);
OVERLAPPED ov;
ov.Internal = 0;
ov.InternalHigh = 0;
@ -27,8 +25,7 @@ static int flock_with_event(HANDLE fd, HANDLE event, unsigned flags,
ov.Offset = (DWORD)offset;
ov.OffsetHigh = HIGH_DWORD(offset);
if (LockFileEx(fd, flags, 0, (DWORD)bytes, HIGH_DWORD(bytes), &ov)) {
TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << %s", fd,
event, flags, offset, bytes, "done");
TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << %s", fd, event, flags, offset, bytes, "done");
return MDBX_SUCCESS;
}
@ -36,37 +33,32 @@ static int flock_with_event(HANDLE fd, HANDLE event, unsigned flags,
if (rc == ERROR_IO_PENDING) {
if (event) {
if (GetOverlappedResult(fd, &ov, &rc, true)) {
TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << %s",
fd, event, flags, offset, bytes, "overlapped-done");
TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << %s", fd, event, flags, offset, bytes,
"overlapped-done");
return MDBX_SUCCESS;
}
rc = GetLastError();
} else
CancelIo(fd);
}
TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << err %d",
fd, event, flags, offset, bytes, (int)rc);
TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << err %d", fd, event, flags, offset, bytes,
(int)rc);
return (int)rc;
}
static inline int flock(HANDLE fd, unsigned flags, size_t offset,
size_t bytes) {
static inline int flock(HANDLE fd, unsigned flags, size_t offset, size_t bytes) {
return flock_with_event(fd, 0, flags, offset, bytes);
}
static inline int flock_data(const MDBX_env *env, unsigned flags, size_t offset,
size_t bytes) {
const HANDLE fd4data =
env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
static inline int flock_data(const MDBX_env *env, unsigned flags, size_t offset, size_t bytes) {
const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
return flock_with_event(fd4data, env->dxb_lock_event, flags, offset, bytes);
}
static int funlock(mdbx_filehandle_t fd, size_t offset, size_t bytes) {
TRACE("unlock: fd %p, offset %zu, bytes %zu", fd, offset, bytes);
return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes,
HIGH_DWORD(bytes))
? MDBX_SUCCESS
: (int)GetLastError();
return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes, HIGH_DWORD(bytes)) ? MDBX_SUCCESS
: (int)GetLastError();
}
/*----------------------------------------------------------------------------*/
@ -88,9 +80,7 @@ int lck_txn_lock(MDBX_env *env, bool dontwait) {
} else {
__try {
EnterCriticalSection(&env->windowsbug_lock);
}
__except ((GetExceptionCode() ==
0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */)
} __except ((GetExceptionCode() == 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */)
? EXCEPTION_EXECUTE_HANDLER
: EXCEPTION_CONTINUE_SEARCH) {
return MDBX_EDEADLK;
@ -101,20 +91,15 @@ int lck_txn_lock(MDBX_env *env, bool dontwait) {
if (env->flags & MDBX_EXCLUSIVE)
goto done;
const HANDLE fd4data =
env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
int rc = flock_with_event(fd4data, env->dxb_lock_event,
dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT)
: (LCK_EXCLUSIVE | LCK_WAITFOR),
DXB_BODY);
dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT) : (LCK_EXCLUSIVE | LCK_WAITFOR), DXB_BODY);
if (rc == ERROR_LOCK_VIOLATION && dontwait) {
SleepEx(0, true);
rc = flock_with_event(fd4data, env->dxb_lock_event,
LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_BODY);
rc = flock_with_event(fd4data, env->dxb_lock_event, LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_BODY);
if (rc == ERROR_LOCK_VIOLATION) {
SleepEx(0, true);
rc = flock_with_event(fd4data, env->dxb_lock_event,
LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_BODY);
rc = flock_with_event(fd4data, env->dxb_lock_event, LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_BODY);
}
}
if (rc == MDBX_SUCCESS) {
@ -133,8 +118,7 @@ int lck_txn_lock(MDBX_env *env, bool dontwait) {
void lck_txn_unlock(MDBX_env *env) {
eASSERT(env, env->basal_txn->owner == osal_thread_self());
if ((env->flags & MDBX_EXCLUSIVE) == 0) {
const HANDLE fd4data =
env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
int err = funlock(fd4data, DXB_BODY);
if (err != MDBX_SUCCESS)
mdbx_panic("%s failed: err %u", __func__, err);
@ -173,8 +157,7 @@ MDBX_INTERNAL int lck_rdt_lock(MDBX_env *env) {
}
MDBX_INTERNAL void lck_rdt_unlock(MDBX_env *env) {
if (env->lck_mmap.fd != INVALID_HANDLE_VALUE &&
(env->flags & MDBX_EXCLUSIVE) == 0) {
if (env->lck_mmap.fd != INVALID_HANDLE_VALUE && (env->flags & MDBX_EXCLUSIVE) == 0) {
/* transition from S-E (locked) to S-? (used), e.g. unlock upper-part */
int err = funlock(env->lck_mmap.fd, LCK_UPPER);
if (err != MDBX_SUCCESS)
@ -184,22 +167,15 @@ MDBX_INTERNAL void lck_rdt_unlock(MDBX_env *env) {
}
MDBX_INTERNAL int osal_lockfile(mdbx_filehandle_t fd, bool wait) {
return flock(
fd, wait ? LCK_EXCLUSIVE | LCK_WAITFOR : LCK_EXCLUSIVE | LCK_DONTWAIT, 0,
DXB_MAXLEN);
return flock(fd, wait ? LCK_EXCLUSIVE | LCK_WAITFOR : LCK_EXCLUSIVE | LCK_DONTWAIT, 0, DXB_MAXLEN);
}
static int suspend_and_append(mdbx_handle_array_t **array,
const DWORD ThreadId) {
static int suspend_and_append(mdbx_handle_array_t **array, const DWORD ThreadId) {
const unsigned limit = (*array)->limit;
if ((*array)->count == limit) {
mdbx_handle_array_t *const ptr =
osal_realloc((limit > ARRAY_LENGTH((*array)->handles))
? *array
: /* don't free initial array on the stack */ nullptr,
sizeof(mdbx_handle_array_t) +
sizeof(HANDLE) * (limit * (size_t)2 -
ARRAY_LENGTH((*array)->handles)));
mdbx_handle_array_t *const ptr = osal_realloc(
(limit > ARRAY_LENGTH((*array)->handles)) ? *array : /* don't free initial array on the stack */ nullptr,
sizeof(mdbx_handle_array_t) + sizeof(HANDLE) * (limit * (size_t)2 - ARRAY_LENGTH((*array)->handles)));
if (!ptr)
return MDBX_ENOMEM;
if (limit == ARRAY_LENGTH((*array)->handles))
@ -208,16 +184,15 @@ static int suspend_and_append(mdbx_handle_array_t **array,
(*array)->limit = limit * 2;
}
HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION,
FALSE, ThreadId);
HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION, FALSE, ThreadId);
if (hThread == nullptr)
return (int)GetLastError();
if (SuspendThread(hThread) == (DWORD)-1) {
int err = (int)GetLastError();
DWORD ExitCode;
if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED ||
!GetExitCodeThread(hThread, &ExitCode) || ExitCode != STILL_ACTIVE)
if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED || !GetExitCodeThread(hThread, &ExitCode) ||
ExitCode != STILL_ACTIVE)
err = MDBX_SUCCESS;
CloseHandle(hThread);
return err;
@ -227,21 +202,17 @@ static int suspend_and_append(mdbx_handle_array_t **array,
return MDBX_SUCCESS;
}
MDBX_INTERNAL int
osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) {
MDBX_INTERNAL int osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) {
eASSERT(env, (env->flags & MDBX_NOSTICKYTHREADS) == 0);
const uintptr_t CurrentTid = GetCurrentThreadId();
int rc;
if (env->lck_mmap.lck) {
/* Scan LCK for threads of the current process */
const reader_slot_t *const begin = env->lck_mmap.lck->rdt;
const reader_slot_t *const end =
begin +
atomic_load32(&env->lck_mmap.lck->rdt_length, mo_AcquireRelease);
const reader_slot_t *const end = begin + atomic_load32(&env->lck_mmap.lck->rdt_length, mo_AcquireRelease);
const uintptr_t WriteTxnOwner = env->basal_txn ? env->basal_txn->owner : 0;
for (const reader_slot_t *reader = begin; reader < end; ++reader) {
if (reader->pid.weak != env->pid || !reader->tid.weak ||
reader->tid.weak >= MDBX_TID_TXN_OUSTED) {
if (reader->pid.weak != env->pid || !reader->tid.weak || reader->tid.weak >= MDBX_TID_TXN_OUSTED) {
skip_lck:
continue;
}
@ -280,8 +251,7 @@ osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) {
}
do {
if (entry.th32OwnerProcessID != env->pid ||
entry.th32ThreadID == CurrentTid)
if (entry.th32OwnerProcessID != env->pid || entry.th32ThreadID == CurrentTid)
continue;
rc = suspend_and_append(array, entry.th32ThreadID);
@ -306,8 +276,8 @@ MDBX_INTERNAL int osal_resume_threads_after_remap(mdbx_handle_array_t *array) {
if (ResumeThread(hThread) == (DWORD)-1) {
const int err = (int)GetLastError();
DWORD ExitCode;
if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED &&
GetExitCodeThread(hThread, &ExitCode) && ExitCode == STILL_ACTIVE)
if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED && GetExitCodeThread(hThread, &ExitCode) &&
ExitCode == STILL_ACTIVE)
rc = err;
}
CloseHandle(hThread);
@ -358,35 +328,30 @@ static void lck_unlock(MDBX_env *env) {
do
err = funlock(env->lck_mmap.fd, LCK_LOWER);
while (err == MDBX_SUCCESS);
assert(err == ERROR_NOT_LOCKED ||
(globals.running_under_Wine && err == ERROR_LOCK_VIOLATION));
assert(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION));
SetLastError(ERROR_SUCCESS);
do
err = funlock(env->lck_mmap.fd, LCK_UPPER);
while (err == MDBX_SUCCESS);
assert(err == ERROR_NOT_LOCKED ||
(globals.running_under_Wine && err == ERROR_LOCK_VIOLATION));
assert(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION));
SetLastError(ERROR_SUCCESS);
}
const HANDLE fd4data =
env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
if (fd4data != INVALID_HANDLE_VALUE) {
/* explicitly unlock to avoid latency for other processes (windows kernel
* releases such locks via deferred queues) */
do
err = funlock(fd4data, DXB_BODY);
while (err == MDBX_SUCCESS);
assert(err == ERROR_NOT_LOCKED ||
(globals.running_under_Wine && err == ERROR_LOCK_VIOLATION));
assert(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION));
SetLastError(ERROR_SUCCESS);
do
err = funlock(fd4data, DXB_WHOLE);
while (err == MDBX_SUCCESS);
assert(err == ERROR_NOT_LOCKED ||
(globals.running_under_Wine && err == ERROR_LOCK_VIOLATION));
assert(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION));
SetLastError(ERROR_SUCCESS);
}
}
@ -418,8 +383,7 @@ static int internal_seize_lck(HANDLE lfd) {
/* 6) something went wrong, give up */
rc = funlock(lfd, LCK_UPPER);
if (rc != MDBX_SUCCESS)
mdbx_panic("%s(%s) failed: err %u", __func__, "?-E(middle) >> ?-?(free)",
rc);
mdbx_panic("%s(%s) failed: err %u", __func__, "?-E(middle) >> ?-?(free)", rc);
return rc;
}
@ -435,16 +399,14 @@ static int internal_seize_lck(HANDLE lfd) {
* transition to S-? (used) or ?-? (free) */
int err = funlock(lfd, LCK_UPPER);
if (err != MDBX_SUCCESS)
mdbx_panic("%s(%s) failed: err %u", __func__,
"X-E(locked/middle) >> X-?(used/free)", err);
mdbx_panic("%s(%s) failed: err %u", __func__, "X-E(locked/middle) >> X-?(used/free)", err);
/* 9) now on S-? (used, DONE) or ?-? (free, FAILURE) */
return rc;
}
MDBX_INTERNAL int lck_seize(MDBX_env *env) {
const HANDLE fd4data =
env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
assert(fd4data != INVALID_HANDLE_VALUE);
if (env->flags & MDBX_EXCLUSIVE)
return MDBX_RESULT_TRUE /* nope since files were must be opened
@ -479,16 +441,14 @@ MDBX_INTERNAL int lck_seize(MDBX_env *env) {
jitter4testing(false);
err = funlock(fd4data, DXB_WHOLE);
if (err != MDBX_SUCCESS)
mdbx_panic("%s(%s) failed: err %u", __func__,
"unlock-against-without-lck", err);
mdbx_panic("%s(%s) failed: err %u", __func__, "unlock-against-without-lck", err);
}
return rc;
}
MDBX_INTERNAL int lck_downgrade(MDBX_env *env) {
const HANDLE fd4data =
env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd;
/* Transite from exclusive-write state (E-E) to used (S-?) */
assert(fd4data != INVALID_HANDLE_VALUE);
assert(env->lck_mmap.fd != INVALID_HANDLE_VALUE);
@ -499,8 +459,7 @@ MDBX_INTERNAL int lck_downgrade(MDBX_env *env) {
/* 1) now at E-E (exclusive-write), transition to ?_E (middle) */
int rc = funlock(env->lck_mmap.fd, LCK_LOWER);
if (rc != MDBX_SUCCESS)
mdbx_panic("%s(%s) failed: err %u", __func__,
"E-E(exclusive-write) >> ?-E(middle)", rc);
mdbx_panic("%s(%s) failed: err %u", __func__, "E-E(exclusive-write) >> ?-E(middle)", rc);
/* 2) now at ?-E (middle), transition to S-E (locked) */
rc = flock(env->lck_mmap.fd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER);
@ -513,8 +472,7 @@ MDBX_INTERNAL int lck_downgrade(MDBX_env *env) {
/* 4) got S-E (locked), continue transition to S-? (used) */
rc = funlock(env->lck_mmap.fd, LCK_UPPER);
if (rc != MDBX_SUCCESS)
mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> S-?(used)",
rc);
mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> S-?(used)", rc);
return MDBX_SUCCESS /* 5) now at S-? (used), done */;
}
@ -529,9 +487,7 @@ MDBX_INTERNAL int lck_upgrade(MDBX_env *env, bool dont_wait) {
/* 1) now on S-? (used), try S-E (locked) */
jitter4testing(false);
int rc = flock(env->lck_mmap.fd,
dont_wait ? LCK_EXCLUSIVE | LCK_DONTWAIT : LCK_EXCLUSIVE,
LCK_UPPER);
int rc = flock(env->lck_mmap.fd, dont_wait ? LCK_EXCLUSIVE | LCK_DONTWAIT : LCK_EXCLUSIVE, LCK_UPPER);
if (rc != MDBX_SUCCESS) {
/* 2) something went wrong, give up */;
VERBOSE("%s, err %u", "S-?(used) >> S-E(locked)", rc);
@ -541,14 +497,11 @@ MDBX_INTERNAL int lck_upgrade(MDBX_env *env, bool dont_wait) {
/* 3) now on S-E (locked), transition to ?-E (middle) */
rc = funlock(env->lck_mmap.fd, LCK_LOWER);
if (rc != MDBX_SUCCESS)
mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> ?-E(middle)",
rc);
mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> ?-E(middle)", rc);
/* 4) now on ?-E (middle), try E-E (exclusive-write) */
jitter4testing(false);
rc = flock(env->lck_mmap.fd,
dont_wait ? LCK_EXCLUSIVE | LCK_DONTWAIT : LCK_EXCLUSIVE,
LCK_LOWER);
rc = flock(env->lck_mmap.fd, dont_wait ? LCK_EXCLUSIVE | LCK_DONTWAIT : LCK_EXCLUSIVE, LCK_LOWER);
if (rc != MDBX_SUCCESS) {
/* 5) something went wrong, give up */;
VERBOSE("%s, err %u", "?-E(middle) >> E-E(exclusive-write)", rc);
@ -558,8 +511,7 @@ MDBX_INTERNAL int lck_upgrade(MDBX_env *env, bool dont_wait) {
return MDBX_SUCCESS /* 6) now at E-E (exclusive-write), done */;
}
MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor,
int global_uniqueness_flag) {
MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, int global_uniqueness_flag) {
(void)env;
(void)inprocess_neighbor;
(void)global_uniqueness_flag;
@ -568,12 +520,9 @@ MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor,
TOKEN_PRIVILEGES privileges;
privileges.PrivilegeCount = 1;
privileges.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES,
&token) ||
!LookupPrivilegeValue(nullptr, SE_LOCK_MEMORY_NAME,
&privileges.Privileges[0].Luid) ||
!AdjustTokenPrivileges(token, FALSE, &privileges, sizeof(privileges),
nullptr, nullptr) ||
if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &token) ||
!LookupPrivilegeValue(nullptr, SE_LOCK_MEMORY_NAME, &privileges.Privileges[0].Luid) ||
!AdjustTokenPrivileges(token, FALSE, &privileges, sizeof(privileges), nullptr, nullptr) ||
GetLastError() != ERROR_SUCCESS)
imports.SetFileIoOverlappedRange = nullptr;
@ -583,8 +532,7 @@ MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor,
return MDBX_SUCCESS;
}
MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor,
const uint32_t current_pid) {
MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, const uint32_t current_pid) {
(void)current_pid;
/* LY: should unmap before releasing the locks to avoid race condition and
* STATUS_USER_MAPPED_FILE/ERROR_USER_MAPPED_FILE */
@ -593,8 +541,7 @@ MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor,
if (env->lck_mmap.lck) {
const bool synced = env->lck_mmap.lck->unsynced_pages.weak == 0;
osal_munmap(&env->lck_mmap);
if (synced && !inprocess_neighbor &&
env->lck_mmap.fd != INVALID_HANDLE_VALUE &&
if (synced && !inprocess_neighbor && env->lck_mmap.fd != INVALID_HANDLE_VALUE &&
lck_upgrade(env, true) == MDBX_SUCCESS)
/* this will fail if LCK is used/mmapped by other process(es) */
osal_ftruncate(env->lck_mmap.fd, 0);

60
src/lck.c
View File

@ -15,14 +15,12 @@ __cold static int lck_setup_locked(MDBX_env *env) {
if (env->lck_mmap.fd == INVALID_HANDLE_VALUE) {
env->lck = lckless_stub(env);
env->max_readers = UINT_MAX;
DEBUG("lck-setup:%s%s%s", " lck-less",
(env->flags & MDBX_RDONLY) ? " readonly" : "",
DEBUG("lck-setup:%s%s%s", " lck-less", (env->flags & MDBX_RDONLY) ? " readonly" : "",
(lck_seize_rc == MDBX_RESULT_TRUE) ? " exclusive" : " cooperative");
return lck_seize_rc;
}
DEBUG("lck-setup:%s%s%s", " with-lck",
(env->flags & MDBX_RDONLY) ? " readonly" : "",
DEBUG("lck-setup:%s%s%s", " with-lck", (env->flags & MDBX_RDONLY) ? " readonly" : "",
(lck_seize_rc == MDBX_RESULT_TRUE) ? " exclusive" : " cooperative");
MDBX_env *inprocess_neighbor = nullptr;
@ -30,8 +28,7 @@ __cold static int lck_setup_locked(MDBX_env *env) {
if (unlikely(MDBX_IS_ERROR(err)))
return err;
if (inprocess_neighbor) {
if ((globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 ||
(inprocess_neighbor->flags & MDBX_EXCLUSIVE) != 0)
if ((globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 || (inprocess_neighbor->flags & MDBX_EXCLUSIVE) != 0)
return MDBX_BUSY;
if (lck_seize_rc == MDBX_RESULT_TRUE) {
err = lck_downgrade(env);
@ -47,52 +44,41 @@ __cold static int lck_setup_locked(MDBX_env *env) {
return err;
if (lck_seize_rc == MDBX_RESULT_TRUE) {
size =
ceil_powerof2(env->max_readers * sizeof(reader_slot_t) + sizeof(lck_t),
globals.sys_pagesize);
size = ceil_powerof2(env->max_readers * sizeof(reader_slot_t) + sizeof(lck_t), globals.sys_pagesize);
jitter4testing(false);
} else {
if (env->flags & MDBX_EXCLUSIVE)
return MDBX_BUSY;
if (size > INT_MAX || (size & (globals.sys_pagesize - 1)) != 0 ||
size < globals.sys_pagesize) {
if (size > INT_MAX || (size & (globals.sys_pagesize - 1)) != 0 || size < globals.sys_pagesize) {
ERROR("lck-file has invalid size %" PRIu64 " bytes", size);
return MDBX_PROBLEM;
}
}
const size_t maxreaders =
((size_t)size - sizeof(lck_t)) / sizeof(reader_slot_t);
const size_t maxreaders = ((size_t)size - sizeof(lck_t)) / sizeof(reader_slot_t);
if (maxreaders < 4) {
ERROR("lck-size too small (up to %" PRIuPTR " readers)", maxreaders);
return MDBX_PROBLEM;
}
env->max_readers = (maxreaders <= MDBX_READERS_LIMIT)
? (unsigned)maxreaders
: (unsigned)MDBX_READERS_LIMIT;
env->max_readers = (maxreaders <= MDBX_READERS_LIMIT) ? (unsigned)maxreaders : (unsigned)MDBX_READERS_LIMIT;
err = osal_mmap((env->flags & MDBX_EXCLUSIVE) | MDBX_WRITEMAP, &env->lck_mmap,
(size_t)size, (size_t)size,
lck_seize_rc ? MMAP_OPTION_TRUNCATE | MMAP_OPTION_SEMAPHORE
: MMAP_OPTION_SEMAPHORE);
err = osal_mmap((env->flags & MDBX_EXCLUSIVE) | MDBX_WRITEMAP, &env->lck_mmap, (size_t)size, (size_t)size,
lck_seize_rc ? MMAP_OPTION_TRUNCATE | MMAP_OPTION_SEMAPHORE : MMAP_OPTION_SEMAPHORE);
if (unlikely(err != MDBX_SUCCESS))
return err;
#ifdef MADV_DODUMP
err = madvise(env->lck_mmap.lck, size, MADV_DODUMP) ? ignore_enosys(errno)
: MDBX_SUCCESS;
err = madvise(env->lck_mmap.lck, size, MADV_DODUMP) ? ignore_enosys(errno) : MDBX_SUCCESS;
if (unlikely(MDBX_IS_ERROR(err)))
return err;
#endif /* MADV_DODUMP */
#ifdef MADV_WILLNEED
err = madvise(env->lck_mmap.lck, size, MADV_WILLNEED) ? ignore_enosys(errno)
: MDBX_SUCCESS;
err = madvise(env->lck_mmap.lck, size, MADV_WILLNEED) ? ignore_enosys(errno) : MDBX_SUCCESS;
if (unlikely(MDBX_IS_ERROR(err)))
return err;
#elif defined(POSIX_MADV_WILLNEED)
err = ignore_enosys(
posix_madvise(env->lck_mmap.lck, size, POSIX_MADV_WILLNEED));
err = ignore_enosys(posix_madvise(env->lck_mmap.lck, size, POSIX_MADV_WILLNEED));
if (unlikely(MDBX_IS_ERROR(err)))
return err;
#endif /* MADV_WILLNEED */
@ -108,8 +94,7 @@ __cold static int lck_setup_locked(MDBX_env *env) {
#if MDBX_ENABLE_PGOP_STAT
lck->pgops.wops.weak = 1;
#endif /* MDBX_ENABLE_PGOP_STAT */
err = osal_msync(&env->lck_mmap, 0, (size_t)size,
MDBX_SYNC_DATA | MDBX_SYNC_SIZE);
err = osal_msync(&env->lck_mmap, 0, (size_t)size, MDBX_SYNC_DATA | MDBX_SYNC_SIZE);
if (unlikely(err != MDBX_SUCCESS)) {
ERROR("initial-%s for lck-file failed, err %d", "msync/fsync", err);
eASSERT(env, MDBX_IS_ERROR(err));
@ -118,17 +103,14 @@ __cold static int lck_setup_locked(MDBX_env *env) {
} else {
if (lck->magic_and_version != MDBX_LOCK_MAGIC) {
const bool invalid = (lck->magic_and_version >> 8) != MDBX_MAGIC;
ERROR("lock region has %s",
invalid
? "invalid magic"
: "incompatible version (only applications with nearly or the "
"same versions of libmdbx can share the same database)");
ERROR("lock region has %s", invalid ? "invalid magic"
: "incompatible version (only applications with nearly or the "
"same versions of libmdbx can share the same database)");
return invalid ? MDBX_INVALID : MDBX_VERSION_MISMATCH;
}
if (lck->os_and_format != MDBX_LOCK_FORMAT) {
ERROR("lock region has os/format signature 0x%" PRIx32
", expected 0x%" PRIx32,
lck->os_and_format, MDBX_LOCK_FORMAT);
ERROR("lock region has os/format signature 0x%" PRIx32 ", expected 0x%" PRIx32, lck->os_and_format,
MDBX_LOCK_FORMAT);
return MDBX_VERSION_MISMATCH;
}
}
@ -148,8 +130,7 @@ __cold int lck_setup(MDBX_env *env, mdbx_mode_t mode) {
eASSERT(env, env->lazy_fd != INVALID_HANDLE_VALUE);
eASSERT(env, env->lck_mmap.fd == INVALID_HANDLE_VALUE);
int err = osal_openfile(MDBX_OPEN_LCK, env, env->pathname.lck,
&env->lck_mmap.fd, mode);
int err = osal_openfile(MDBX_OPEN_LCK, env, env->pathname.lck, &env->lck_mmap.fd, mode);
if (err != MDBX_SUCCESS) {
switch (err) {
default:
@ -186,6 +167,5 @@ __cold int lck_setup(MDBX_env *env, mdbx_mode_t mode) {
}
void mincore_clean_cache(const MDBX_env *const env) {
memset(env->lck->mincore_cache.begin, -1,
sizeof(env->lck->mincore_cache.begin));
memset(env->lck->mincore_cache.begin, -1, sizeof(env->lck->mincore_cache.begin));
}

6
src/lck.h
View File

@ -23,8 +23,7 @@ MDBX_INTERNAL int lck_ipclock_destroy(osal_ipclock_t *ipc);
/// MUST NOT initialize shared synchronization objects in memory-mapped
/// LCK-file that are already in use.
/// \return Error code or zero on success.
MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor,
int global_uniqueness_flag);
MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, int global_uniqueness_flag);
/// \brief Disconnects from shared interprocess objects and destructs
/// synchronization objects linked with MDBX_env instance
@ -43,8 +42,7 @@ MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor,
/// of other instances of MDBX_env within the current process, e.g.
/// restore POSIX-fcntl locks after the closing of file descriptors.
/// \return Error code (MDBX_PANIC) or zero on success.
MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor,
const uint32_t current_pid);
MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, const uint32_t current_pid);
/// \brief Connects to shared interprocess locking objects and tries to acquire
/// the maximum lock level (shared if exclusive is not available)

67
src/logging_and_debug.c
View File

@ -3,15 +3,13 @@
#include "internals.h"
__cold void debug_log_va(int level, const char *function, int line,
const char *fmt, va_list args) {
__cold void debug_log_va(int level, const char *function, int line, const char *fmt, va_list args) {
ENSURE(nullptr, osal_fastmutex_acquire(&globals.debug_lock) == 0);
if (globals.logger.ptr) {
if (globals.logger_buffer == nullptr)
globals.logger.fmt(level, function, line, fmt, args);
else {
const int len = vsnprintf(globals.logger_buffer,
globals.logger_buffer_size, fmt, args);
const int len = vsnprintf(globals.logger_buffer, globals.logger_buffer_size, fmt, args);
if (len > 0)
globals.logger.nofmt(level, function, line, globals.logger_buffer, len);
}
@ -51,8 +49,7 @@ __cold void debug_log_va(int level, const char *function, int line,
ENSURE(nullptr, osal_fastmutex_release(&globals.debug_lock) == 0);
}
__cold void debug_log(int level, const char *function, int line,
const char *fmt, ...) {
__cold void debug_log(int level, const char *function, int line, const char *fmt, ...) {
va_list args;
va_start(args, fmt);
debug_log_va(level, function, line, fmt, args);
@ -62,18 +59,15 @@ __cold void debug_log(int level, const char *function, int line,
__cold int log_error(const int err, const char *func, unsigned line) {
assert(err != MDBX_SUCCESS);
if (unlikely(globals.loglevel >= MDBX_LOG_DEBUG) &&
(globals.loglevel >= MDBX_LOG_TRACE ||
!(err == MDBX_RESULT_TRUE || err == MDBX_NOTFOUND))) {
(globals.loglevel >= MDBX_LOG_TRACE || !(err == MDBX_RESULT_TRUE || err == MDBX_NOTFOUND))) {
char buf[256];
debug_log(MDBX_LOG_ERROR, func, line, "error %d (%s)\n", err,
mdbx_strerror_r(err, buf, sizeof(buf)));
debug_log(MDBX_LOG_ERROR, func, line, "error %d (%s)\n", err, mdbx_strerror_r(err, buf, sizeof(buf)));
}
return err;
}
/* Dump a val in ascii or hexadecimal. */
__cold const char *mdbx_dump_val(const MDBX_val *val, char *const buf,
const size_t bufsize) {
__cold const char *mdbx_dump_val(const MDBX_val *val, char *const buf, const size_t bufsize) {
if (!val)
return "<null>";
if (!val->iov_len)
@ -97,9 +91,7 @@ __cold const char *mdbx_dump_val(const MDBX_val *val, char *const buf,
}
if (is_ascii) {
int len =
snprintf(buf, bufsize, "%.*s",
(val->iov_len > INT_MAX) ? INT_MAX : (int)val->iov_len, data);
int len = snprintf(buf, bufsize, "%.*s", (val->iov_len > INT_MAX) ? INT_MAX : (int)val->iov_len, data);
assert(len > 0 && (size_t)len < bufsize);
(void)len;
} else {
@ -107,8 +99,7 @@ __cold const char *mdbx_dump_val(const MDBX_val *val, char *const buf,
char *ptr = buf;
*ptr++ = '<';
for (size_t i = 0; i < val->iov_len && ptr < detent; i++) {
const char hex[16] = {'0', '1', '2', '3', '4', '5', '6', '7',
'8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
const char hex[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
*ptr++ = hex[data[i] >> 4];
*ptr++ = hex[data[i] & 15];
}
@ -145,11 +136,8 @@ __cold const char *pagetype_caption(const uint8_t type, char buf4unknown[16]) {
}
__cold static const char *leafnode_type(node_t *n) {
static const char *const tp[2][2] = {{"", ": DB"},
{": sub-page", ": sub-DB"}};
return (node_flags(n) & N_BIG)
? ": large page"
: tp[!!(node_flags(n) & N_DUP)][!!(node_flags(n) & N_TREE)];
static const char *const tp[2][2] = {{"", ": DB"}, {": sub-page", ": sub-DB"}};
return (node_flags(n) & N_BIG) ? ": large page" : tp[!!(node_flags(n) & N_DUP)][!!(node_flags(n) & N_TREE)];
}
/* Display all the keys in the page. */
@ -181,8 +169,7 @@ __cold void page_list(page_t *mp) {
VERBOSE("Overflow page %" PRIaPGNO " pages %u\n", pgno, mp->pages);
return;
case P_META:
VERBOSE("Meta-page %" PRIaPGNO " txnid %" PRIu64 "\n", pgno,
unaligned_peek_u64(4, page_meta(mp)->txnid_a));
VERBOSE("Meta-page %" PRIaPGNO " txnid %" PRIu64 "\n", pgno, unaligned_peek_u64(4, page_meta(mp)->txnid_a));
return;
default:
VERBOSE("Bad page %" PRIaPGNO " flags 0x%X\n", pgno, mp->flags);
@ -193,8 +180,7 @@ __cold void page_list(page_t *mp) {
VERBOSE("%s %" PRIaPGNO " numkeys %zu\n", type, pgno, nkeys);
for (i = 0; i < nkeys; i++) {
if (is_dupfix_leaf(
mp)) { /* DUPFIX pages have no entries[] or node headers */
if (is_dupfix_leaf(mp)) { /* DUPFIX pages have no entries[] or node headers */
key = page_dupfix_key(mp, i, nsize = mp->dupfix_ksize);
total += nsize;
VERBOSE("key %zu: nsize %zu, %s\n", i, nsize, DKEY(&key));
@ -205,8 +191,7 @@ __cold void page_list(page_t *mp) {
key.iov_base = node->payload;
nsize = NODESIZE + key.iov_len;
if (is_branch(mp)) {
VERBOSE("key %zu: page %" PRIaPGNO ", %s\n", i, node_pgno(node),
DKEY(&key));
VERBOSE("key %zu: page %" PRIaPGNO ", %s\n", i, node_pgno(node), DKEY(&key));
total += nsize;
} else {
if (node_flags(node) & N_BIG)
@ -215,18 +200,15 @@ __cold void page_list(page_t *mp) {
nsize += node_ds(node);
total += nsize;
nsize += sizeof(indx_t);
VERBOSE("key %zu: nsize %zu, %s%s\n", i, nsize, DKEY(&key),
leafnode_type(node));
VERBOSE("key %zu: nsize %zu, %s%s\n", i, nsize, DKEY(&key), leafnode_type(node));
}
total = EVEN_CEIL(total);
}
VERBOSE("Total: header %u + contents %zu + unused %zu\n",
is_dupfix_leaf(mp) ? PAGEHDRSZ : PAGEHDRSZ + mp->lower, total,
page_room(mp));
VERBOSE("Total: header %u + contents %zu + unused %zu\n", is_dupfix_leaf(mp) ? PAGEHDRSZ : PAGEHDRSZ + mp->lower,
total, page_room(mp));
}
__cold static int setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags,
union logger_union logger, char *buffer,
__cold static int setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags, union logger_union logger, char *buffer,
size_t buffer_size) {
ENSURE(nullptr, osal_fastmutex_acquire(&globals.debug_lock) == 0);
@ -239,8 +221,7 @@ __cold static int setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags,
#if MDBX_DEBUG
MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER |
#endif
MDBX_DBG_DUMP | MDBX_DBG_LEGACY_MULTIOPEN | MDBX_DBG_LEGACY_OVERLAP |
MDBX_DBG_DONT_UPGRADE;
MDBX_DBG_DUMP | MDBX_DBG_LEGACY_MULTIOPEN | MDBX_DBG_LEGACY_OVERLAP | MDBX_DBG_DONT_UPGRADE;
globals.runtime_flags = (uint8_t)flags;
}
@ -255,18 +236,14 @@ __cold static int setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags,
return rc;
}
__cold int mdbx_setup_debug_nofmt(MDBX_log_level_t level,
MDBX_debug_flags_t flags,
MDBX_debug_func_nofmt *logger, char *buffer,
size_t buffer_size) {
__cold int mdbx_setup_debug_nofmt(MDBX_log_level_t level, MDBX_debug_flags_t flags, MDBX_debug_func_nofmt *logger,
char *buffer, size_t buffer_size) {
union logger_union thunk;
thunk.nofmt =
(logger && buffer && buffer_size) ? logger : MDBX_LOGGER_NOFMT_DONTCHANGE;
thunk.nofmt = (logger && buffer && buffer_size) ? logger : MDBX_LOGGER_NOFMT_DONTCHANGE;
return setup_debug(level, flags, thunk, buffer, buffer_size);
}
__cold int mdbx_setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags,
MDBX_debug_func *logger) {
__cold int mdbx_setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags, MDBX_debug_func *logger) {
union logger_union thunk;
thunk.fmt = logger;
return setup_debug(level, flags, thunk, nullptr, 0);

124
src/logging_and_debug.h
View File

@ -6,23 +6,17 @@
#include "essentials.h"
#ifndef __Wpedantic_format_voidptr
MDBX_MAYBE_UNUSED static inline const void *
__Wpedantic_format_voidptr(const void *ptr) {
return ptr;
}
MDBX_MAYBE_UNUSED static inline const void *__Wpedantic_format_voidptr(const void *ptr) { return ptr; }
#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG)
#endif /* __Wpedantic_format_voidptr */
MDBX_INTERNAL void MDBX_PRINTF_ARGS(4, 5)
debug_log(int level, const char *function, int line, const char *fmt, ...)
MDBX_PRINTF_ARGS(4, 5);
MDBX_INTERNAL void debug_log_va(int level, const char *function, int line,
const char *fmt, va_list args);
MDBX_INTERNAL void MDBX_PRINTF_ARGS(4, 5) debug_log(int level, const char *function, int line, const char *fmt, ...)
MDBX_PRINTF_ARGS(4, 5);
MDBX_INTERNAL void debug_log_va(int level, const char *function, int line, const char *fmt, va_list args);
#if MDBX_DEBUG
#define LOG_ENABLED(LVL) unlikely(LVL <= globals.loglevel)
#define AUDIT_ENABLED() \
unlikely((globals.runtime_flags & (unsigned)MDBX_DBG_AUDIT))
#define AUDIT_ENABLED() unlikely((globals.runtime_flags & (unsigned)MDBX_DBG_AUDIT))
#else /* MDBX_DEBUG */
#define LOG_ENABLED(LVL) (LVL < MDBX_LOG_VERBOSE && LVL <= globals.loglevel)
#define AUDIT_ENABLED() (0)
@ -31,91 +25,88 @@ MDBX_INTERNAL void debug_log_va(int level, const char *function, int line,
#if MDBX_FORCE_ASSERTIONS
#define ASSERT_ENABLED() (1)
#elif MDBX_DEBUG
#define ASSERT_ENABLED() \
likely((globals.runtime_flags & (unsigned)MDBX_DBG_ASSERT))
#define ASSERT_ENABLED() likely((globals.runtime_flags & (unsigned)MDBX_DBG_ASSERT))
#else
#define ASSERT_ENABLED() (0)
#endif /* ASSERT_ENABLED() */
#define DEBUG_EXTRA(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_EXTRA)) \
debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \
#define DEBUG_EXTRA(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_EXTRA)) \
debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \
} while (0)
#define DEBUG_EXTRA_PRINT(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_EXTRA)) \
debug_log(MDBX_LOG_EXTRA, nullptr, 0, fmt, __VA_ARGS__); \
#define DEBUG_EXTRA_PRINT(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_EXTRA)) \
debug_log(MDBX_LOG_EXTRA, nullptr, 0, fmt, __VA_ARGS__); \
} while (0)
#define TRACE(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_TRACE)) \
debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \
#define TRACE(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_TRACE)) \
debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \
} while (0)
#define DEBUG(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_DEBUG)) \
debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \
#define DEBUG(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_DEBUG)) \
debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \
} while (0)
#define VERBOSE(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \
debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \
#define VERBOSE(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \
debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \
} while (0)
#define NOTICE(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_NOTICE)) \
debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \
#define NOTICE(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_NOTICE)) \
debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \
} while (0)
#define WARNING(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_WARN)) \
debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \
#define WARNING(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_WARN)) \
debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \
} while (0)
#undef ERROR /* wingdi.h \
#undef ERROR /* wingdi.h \
Yeah, morons from M$ put such definition to the public header. */
#define ERROR(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_ERROR)) \
debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \
#define ERROR(fmt, ...) \
do { \
if (LOG_ENABLED(MDBX_LOG_ERROR)) \
debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \
} while (0)
#define FATAL(fmt, ...) \
debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__);
#define FATAL(fmt, ...) debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__);
#if MDBX_DEBUG
#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line)
#else /* MDBX_DEBUG */
MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func,
unsigned line);
#define ASSERT_FAIL(env, msg, func, line) \
do { \
(void)(env); \
assert_fail(msg, func, line); \
MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, unsigned line);
#define ASSERT_FAIL(env, msg, func, line) \
do { \
(void)(env); \
assert_fail(msg, func, line); \
} while (0)
#endif /* MDBX_DEBUG */
#define ENSURE_MSG(env, expr, msg) \
do { \
if (unlikely(!(expr))) \
ASSERT_FAIL(env, msg, __func__, __LINE__); \
#define ENSURE_MSG(env, expr, msg) \
do { \
if (unlikely(!(expr))) \
ASSERT_FAIL(env, msg, __func__, __LINE__); \
} while (0)
#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr)
/* assert(3) variant in environment context */
#define eASSERT(env, expr) \
do { \
if (ASSERT_ENABLED()) \
ENSURE(env, expr); \
#define eASSERT(env, expr) \
do { \
if (ASSERT_ENABLED()) \
ENSURE(env, expr); \
} while (0)
/* assert(3) variant in cursor context */
@ -140,14 +131,12 @@ MDBX_MAYBE_UNUSED static inline void jitter4testing(bool tiny) {
MDBX_MAYBE_UNUSED MDBX_INTERNAL void page_list(page_t *mp);
MDBX_INTERNAL const char *pagetype_caption(const uint8_t type,
char buf4unknown[16]);
MDBX_INTERNAL const char *pagetype_caption(const uint8_t type, char buf4unknown[16]);
/* Key size which fits in a DKBUF (debug key buffer). */
#define DKBUF_MAX 127
#define DKBUF char dbg_kbuf[DKBUF_MAX * 4 + 2]
#define DKEY(x) mdbx_dump_val(x, dbg_kbuf, DKBUF_MAX * 2 + 1)
#define DVAL(x) \
mdbx_dump_val(x, dbg_kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1)
#define DVAL(x) mdbx_dump_val(x, dbg_kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1)
#if MDBX_DEBUG
#define DKBUF_DEBUG DKBUF
@ -161,8 +150,7 @@ MDBX_INTERNAL const char *pagetype_caption(const uint8_t type,
MDBX_INTERNAL int log_error(const int err, const char *func, unsigned line);
MDBX_MAYBE_UNUSED static inline int
log_if_error(const int err, const char *func, unsigned line) {
MDBX_MAYBE_UNUSED static inline int log_if_error(const int err, const char *func, unsigned line) {
if (likely(err == MDBX_SUCCESS))
return err;
int rc = log_error(err, func, line);

374
src/mdbx.c++
View File

@ -187,21 +187,19 @@ __cold bug::~bug() noexcept {}
throw bug(what_and_where);
}
#define RAISE_BUG(line, condition, function, file) \
do { \
static MDBX_CXX11_CONSTEXPR_VAR trouble_location bug(line, condition, \
function, file); \
raise_bug(bug); \
#define RAISE_BUG(line, condition, function, file) \
do { \
static MDBX_CXX11_CONSTEXPR_VAR trouble_location bug(line, condition, function, file); \
raise_bug(bug); \
} while (0)
#define ENSURE(condition) \
do \
if (MDBX_UNLIKELY(!(condition))) \
MDBX_CXX20_UNLIKELY RAISE_BUG(__LINE__, #condition, __func__, __FILE__); \
#define ENSURE(condition) \
do \
if (MDBX_UNLIKELY(!(condition))) \
MDBX_CXX20_UNLIKELY RAISE_BUG(__LINE__, #condition, __func__, __FILE__); \
while (0)
#define NOT_IMPLEMENTED() \
RAISE_BUG(__LINE__, "not_implemented", __func__, __FILE__);
#define NOT_IMPLEMENTED() RAISE_BUG(__LINE__, "not_implemented", __func__, __FILE__);
#endif /* Unused*/
@ -226,14 +224,12 @@ struct line_wrapper {
}
};
template <typename TYPE, unsigned INPLACE_BYTES = unsigned(sizeof(void *) * 64)>
struct temp_buffer {
template <typename TYPE, unsigned INPLACE_BYTES = unsigned(sizeof(void *) * 64)> struct temp_buffer {
TYPE inplace[(INPLACE_BYTES + sizeof(TYPE) - 1) / sizeof(TYPE)];
const size_t size;
TYPE *const area;
temp_buffer(size_t bytes)
: size((bytes + sizeof(TYPE) - 1) / sizeof(TYPE)),
area((bytes > sizeof(inplace)) ? new TYPE[size] : inplace) {
: size((bytes + sizeof(TYPE) - 1) / sizeof(TYPE)), area((bytes > sizeof(inplace)) ? new TYPE[size] : inplace) {
memset(area, 0, sizeof(TYPE) * size);
}
~temp_buffer() {
@ -265,8 +261,7 @@ struct temp_buffer {
namespace mdbx {
[[noreturn]] __cold void throw_max_length_exceeded() {
throw std::length_error(
"mdbx:: Exceeded the maximal length of data/slice/buffer.");
throw std::length_error("mdbx:: Exceeded the maximal length of data/slice/buffer.");
}
[[noreturn]] __cold void throw_too_small_target_buffer() {
@ -279,38 +274,31 @@ namespace mdbx {
}
[[noreturn]] __cold void throw_allocators_mismatch() {
throw std::logic_error(
"mdbx:: An allocators mismatch, so an object could not be transferred "
"into an incompatible memory allocation scheme.");
throw std::logic_error("mdbx:: An allocators mismatch, so an object could not be transferred "
"into an incompatible memory allocation scheme.");
}
[[noreturn]] __cold void throw_incomparable_cursors() {
throw std::logic_error(
"mdbx:: incomparable and/or invalid cursors to compare positions.");
throw std::logic_error("mdbx:: incomparable and/or invalid cursors to compare positions.");
}
[[noreturn]] __cold void throw_bad_value_size() {
throw bad_value_size(MDBX_BAD_VALSIZE);
}
[[noreturn]] __cold void throw_bad_value_size() { throw bad_value_size(MDBX_BAD_VALSIZE); }
__cold exception::exception(const ::mdbx::error &error) noexcept
: base(error.what()), error_(error) {}
__cold exception::exception(const ::mdbx::error &error) noexcept : base(error.what()), error_(error) {}
__cold exception::~exception() noexcept {}
static std::atomic_int fatal_countdown;
__cold fatal::fatal(const ::mdbx::error &error) noexcept : base(error) {
++fatal_countdown;
}
__cold fatal::fatal(const ::mdbx::error &error) noexcept : base(error) { ++fatal_countdown; }
__cold fatal::~fatal() noexcept {
if (--fatal_countdown == 0)
std::terminate();
}
#define DEFINE_EXCEPTION(NAME) \
__cold NAME::NAME(const ::mdbx::error &rc) : exception(rc) {} \
#define DEFINE_EXCEPTION(NAME) \
__cold NAME::NAME(const ::mdbx::error &rc) : exception(rc) {} \
__cold NAME::~NAME() noexcept {}
DEFINE_EXCEPTION(bad_map_id)
@ -352,8 +340,8 @@ __cold const char *error::what() const noexcept {
return mdbx_liberr2str(code());
switch (code()) {
#define ERROR_CASE(CODE) \
case CODE: \
#define ERROR_CASE(CODE) \
case CODE: \
return MDBX_STRINGIFY(CODE)
ERROR_CASE(MDBX_ENODATA);
ERROR_CASE(MDBX_EINVAL);
@ -379,8 +367,7 @@ __cold std::string error::message() const {
return std::string(msg ? msg : "unknown");
}
[[noreturn]] __cold void error::panic(const char *context,
const char *func) const noexcept {
[[noreturn]] __cold void error::panic(const char *context, const char *func) const noexcept {
assert(code() != MDBX_SUCCESS);
::mdbx_panic("mdbx::%s.%s(): \"%s\" (%d)", context, func, what(), code());
std::terminate();
@ -397,8 +384,8 @@ __cold void error::throw_exception() const {
throw std::logic_error("MDBX_SUCCESS (MDBX_RESULT_FALSE)");
case MDBX_RESULT_TRUE:
throw std::logic_error("MDBX_RESULT_TRUE");
#define CASE_EXCEPTION(NAME, CODE) \
case CODE: \
#define CASE_EXCEPTION(NAME, CODE) \
case CODE: \
throw NAME(code())
CASE_EXCEPTION(bad_map_id, MDBX_BAD_DBI);
CASE_EXCEPTION(bad_transaction, MDBX_BAD_TXN);
@ -702,27 +689,23 @@ char *to_hex::write_bytes(char *__restrict const dest, size_t dest_size) const {
return out;
}
char *from_hex::write_bytes(char *__restrict const dest,
size_t dest_size) const {
char *from_hex::write_bytes(char *__restrict const dest, size_t dest_size) const {
if (MDBX_UNLIKELY(source.length() % 2 && !ignore_spaces))
MDBX_CXX20_UNLIKELY throw std::domain_error(
"mdbx::from_hex:: odd length of hexadecimal string");
MDBX_CXX20_UNLIKELY throw std::domain_error("mdbx::from_hex:: odd length of hexadecimal string");
if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
auto ptr = dest;
auto src = source.byte_ptr();
for (auto left = source.length(); left > 0;) {
if (MDBX_UNLIKELY(*src <= ' ') &&
MDBX_LIKELY(ignore_spaces && isspace(*src))) {
if (MDBX_UNLIKELY(*src <= ' ') && MDBX_LIKELY(ignore_spaces && isspace(*src))) {
++src;
--left;
continue;
}
if (MDBX_UNLIKELY(left < 1 || !isxdigit(src[0]) || !isxdigit(src[1])))
MDBX_CXX20_UNLIKELY throw std::domain_error(
"mdbx::from_hex:: invalid hexadecimal string");
MDBX_CXX20_UNLIKELY throw std::domain_error("mdbx::from_hex:: invalid hexadecimal string");
int8_t hi = src[0];
hi = (hi | 0x20) - 'a';
@ -747,8 +730,7 @@ bool from_hex::is_erroneous() const noexcept {
bool got = false;
auto src = source.byte_ptr();
for (auto left = source.length(); left > 0;) {
if (MDBX_UNLIKELY(*src <= ' ') &&
MDBX_LIKELY(ignore_spaces && isspace(*src))) {
if (MDBX_UNLIKELY(*src <= ' ') && MDBX_LIKELY(ignore_spaces && isspace(*src))) {
++src;
--left;
continue;
@ -780,25 +762,21 @@ using b58_uint = uint_fast32_t;
#endif
struct b58_buffer : public temp_buffer<b58_uint> {
b58_buffer(size_t bytes, size_t estimation_ratio_numerator,
size_t estimation_ratio_denominator, size_t extra = 0)
: temp_buffer((/* пересчитываем по указанной пропорции */
bytes = (bytes * estimation_ratio_numerator +
estimation_ratio_denominator - 1) /
estimation_ratio_denominator,
/* учитываем резервный старший байт в каждом слове */
((bytes + sizeof(b58_uint) - 2) / (sizeof(b58_uint) - 1) *
sizeof(b58_uint) +
extra) *
sizeof(b58_uint))) {}
b58_buffer(size_t bytes, size_t estimation_ratio_numerator, size_t estimation_ratio_denominator, size_t extra = 0)
: temp_buffer(
(/* пересчитываем по указанной пропорции */
bytes =
(bytes * estimation_ratio_numerator + estimation_ratio_denominator - 1) / estimation_ratio_denominator,
/* учитываем резервный старший байт в каждом слове */
((bytes + sizeof(b58_uint) - 2) / (sizeof(b58_uint) - 1) * sizeof(b58_uint) + extra) * sizeof(b58_uint))) {
}
};
static byte b58_8to11(b58_uint &v) noexcept {
static const char b58_alphabet[58] = {
'1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F',
'G', 'H', 'J', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W',
'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'm',
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'};
static const char b58_alphabet[58] = {'1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F',
'G', 'H', 'J', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W',
'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'm',
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'};
const auto i = size_t(v % 58);
v /= 58;
@ -807,9 +785,8 @@ static byte b58_8to11(b58_uint &v) noexcept {
static slice b58_encode(b58_buffer &buf, const byte *begin, const byte *end) {
auto high = buf.end();
const auto modulo =
b58_uint((sizeof(b58_uint) > 4) ? UINT64_C(0x1A636A90B07A00) /* 58^9 */
: UINT32_C(0xACAD10) /* 58^4 */);
const auto modulo = b58_uint((sizeof(b58_uint) > 4) ? UINT64_C(0x1A636A90B07A00) /* 58^9 */
: UINT32_C(0xACAD10) /* 58^4 */);
static_assert(sizeof(modulo) == 4 || sizeof(modulo) == 8, "WTF?");
while (begin < end) {
b58_uint carry = *begin++;
@ -855,8 +832,7 @@ static slice b58_encode(b58_buffer &buf, const byte *begin, const byte *end) {
return slice(output, ptr);
}
char *to_base58::write_bytes(char *__restrict const dest,
size_t dest_size) const {
char *to_base58::write_bytes(char *__restrict const dest, size_t dest_size) const {
if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
@ -927,8 +903,7 @@ const signed char b58_map[256] = {
IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL // f0
};
static slice b58_decode(b58_buffer &buf, const byte *begin, const byte *end,
bool ignore_spaces) {
static slice b58_decode(b58_buffer &buf, const byte *begin, const byte *end, bool ignore_spaces) {
auto high = buf.end();
while (begin < end) {
const auto c = b58_map[*begin++];
@ -969,8 +944,7 @@ static slice b58_decode(b58_buffer &buf, const byte *begin, const byte *end,
return slice(output, ptr);
}
char *from_base58::write_bytes(char *__restrict const dest,
size_t dest_size) const {
char *from_base58::write_bytes(char *__restrict const dest, size_t dest_size) const {
if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
@ -996,8 +970,7 @@ bool from_base58::is_erroneous() const noexcept {
auto begin = source.byte_ptr();
auto const end = source.end_byte_ptr();
while (begin < end) {
if (MDBX_UNLIKELY(b58_map[*begin] < 0 &&
!(ignore_spaces && isspace(*begin))))
if (MDBX_UNLIKELY(b58_map[*begin] < 0 && !(ignore_spaces && isspace(*begin))))
return true;
++begin;
}
@ -1006,22 +979,18 @@ bool from_base58::is_erroneous() const noexcept {
//------------------------------------------------------------------------------
static inline void b64_3to4(const byte x, const byte y, const byte z,
char *__restrict dest) noexcept {
static const byte alphabet[64] = {
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'};
static inline void b64_3to4(const byte x, const byte y, const byte z, char *__restrict dest) noexcept {
static const byte alphabet[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'};
dest[0] = alphabet[(x & 0xfc) >> 2];
dest[1] = alphabet[((x & 0x03) << 4) + ((y & 0xf0) >> 4)];
dest[2] = alphabet[((y & 0x0f) << 2) + ((z & 0xc0) >> 6)];
dest[3] = alphabet[z & 0x3f];
}
char *to_base64::write_bytes(char *__restrict const dest,
size_t dest_size) const {
char *to_base64::write_bytes(char *__restrict const dest, size_t dest_size) const {
if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
@ -1115,8 +1084,7 @@ static const signed char b64_map[256] = {
IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL // f0
};
static inline signed char b64_4to3(signed char a, signed char b, signed char c,
signed char d,
static inline signed char b64_4to3(signed char a, signed char b, signed char c, signed char d,
char *__restrict dest) noexcept {
dest[0] = byte((a << 2) + ((b & 0x30) >> 4));
dest[1] = byte(((b & 0xf) << 4) + ((c & 0x3c) >> 2));
@ -1124,19 +1092,16 @@ static inline signed char b64_4to3(signed char a, signed char b, signed char c,
return a | b | c | d;
}
char *from_base64::write_bytes(char *__restrict const dest,
size_t dest_size) const {
char *from_base64::write_bytes(char *__restrict const dest, size_t dest_size) const {
if (MDBX_UNLIKELY(source.length() % 4 && !ignore_spaces))
MDBX_CXX20_UNLIKELY throw std::domain_error(
"mdbx::from_base64:: odd length of base64 string");
MDBX_CXX20_UNLIKELY throw std::domain_error("mdbx::from_base64:: odd length of base64 string");
if (MDBX_UNLIKELY(envisage_result_length() > dest_size))
MDBX_CXX20_UNLIKELY throw_too_small_target_buffer();
auto ptr = dest;
auto src = source.byte_ptr();
for (auto left = source.length(); left > 0;) {
if (MDBX_UNLIKELY(*src <= ' ') &&
MDBX_LIKELY(ignore_spaces && isspace(*src))) {
if (MDBX_UNLIKELY(*src <= ' ') && MDBX_LIKELY(ignore_spaces && isspace(*src))) {
++src;
--left;
continue;
@ -1147,8 +1112,7 @@ char *from_base64::write_bytes(char *__restrict const dest,
bailout:
throw std::domain_error("mdbx::from_base64:: invalid base64 string");
}
const signed char a = b64_map[src[0]], b = b64_map[src[1]],
c = b64_map[src[2]], d = b64_map[src[3]];
const signed char a = b64_map[src[0]], b = b64_map[src[1]], c = b64_map[src[2]], d = b64_map[src[3]];
if (MDBX_UNLIKELY(b64_4to3(a, b, c, d, ptr) < 0)) {
if (left == 4 && (a | b) >= 0 && d == EQ) {
if (c >= 0) {
@ -1177,8 +1141,7 @@ bool from_base64::is_erroneous() const noexcept {
bool got = false;
auto src = source.byte_ptr();
for (auto left = source.length(); left > 0;) {
if (MDBX_UNLIKELY(*src <= ' ') &&
MDBX_LIKELY(ignore_spaces && isspace(*src))) {
if (MDBX_UNLIKELY(*src <= ' ') && MDBX_LIKELY(ignore_spaces && isspace(*src))) {
++src;
--left;
continue;
@ -1186,8 +1149,7 @@ bool from_base64::is_erroneous() const noexcept {
if (MDBX_UNLIKELY(left < 3))
MDBX_CXX20_UNLIKELY return false;
const signed char a = b64_map[src[0]], b = b64_map[src[1]],
c = b64_map[src[2]], d = b64_map[src[3]];
const signed char a = b64_map[src[0]], b = b64_map[src[1]], c = b64_map[src[2]], d = b64_map[src[3]];
if (MDBX_UNLIKELY((a | b | c | d) < 0))
MDBX_CXX20_UNLIKELY {
if (left == 4 && (a | b) >= 0 && d == EQ && (c >= 0 || c == d))
@ -1205,8 +1167,7 @@ bool from_base64::is_erroneous() const noexcept {
template class LIBMDBX_API_TYPE buffer<legacy_allocator>;
#if defined(__cpp_lib_memory_resource) && \
__cpp_lib_memory_resource >= 201603L && _GLIBCXX_USE_CXX11_ABI
#if defined(__cpp_lib_memory_resource) && __cpp_lib_memory_resource >= 201603L && _GLIBCXX_USE_CXX11_ABI
template class LIBMDBX_API_TYPE buffer<polymorphic_allocator>;
#endif /* __cpp_lib_memory_resource >= 201603L */
@ -1225,8 +1186,7 @@ static inline MDBX_env_flags_t mode2flags(env::mode mode) {
}
}
__cold MDBX_env_flags_t
env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const {
__cold MDBX_env_flags_t env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const {
MDBX_env_flags_t flags = mode2flags(mode);
if (accede)
flags |= MDBX_ACCEDE;
@ -1252,8 +1212,7 @@ env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const {
flags |= MDBX_LIFORECLAIM;
switch (durability) {
default:
MDBX_CXX20_UNLIKELY throw std::invalid_argument(
"db::durability is invalid");
MDBX_CXX20_UNLIKELY throw std::invalid_argument("db::durability is invalid");
case env::durability::robust_synchronous:
break;
case env::durability::half_synchronous_weak_last:
@ -1271,16 +1230,13 @@ env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const {
return flags;
}
env::mode
env::operate_parameters::mode_from_flags(MDBX_env_flags_t flags) noexcept {
env::mode env::operate_parameters::mode_from_flags(MDBX_env_flags_t flags) noexcept {
if (flags & MDBX_RDONLY)
return env::mode::readonly;
return (flags & MDBX_WRITEMAP) ? env::mode::write_mapped_io
: env::mode::write_file_io;
return (flags & MDBX_WRITEMAP) ? env::mode::write_mapped_io : env::mode::write_file_io;
}
env::durability env::operate_parameters::durability_from_flags(
MDBX_env_flags_t flags) noexcept {
env::durability env::operate_parameters::durability_from_flags(MDBX_env_flags_t flags) noexcept {
if ((flags & MDBX_UTTERLY_NOSYNC) == MDBX_UTTERLY_NOSYNC)
return env::durability::whole_fragile;
if (flags & MDBX_SAFE_NOSYNC)
@ -1291,71 +1247,51 @@ env::durability env::operate_parameters::durability_from_flags(
}
env::reclaiming_options::reclaiming_options(MDBX_env_flags_t flags) noexcept
: lifo((flags & MDBX_LIFORECLAIM) ? true : false),
coalesce((flags & MDBX_COALESCE) ? true : false) {}
: lifo((flags & MDBX_LIFORECLAIM) ? true : false), coalesce((flags & MDBX_COALESCE) ? true : false) {}
env::operate_options::operate_options(MDBX_env_flags_t flags) noexcept
: no_sticky_threads(((flags & (MDBX_NOSTICKYTHREADS | MDBX_EXCLUSIVE)) ==
MDBX_NOSTICKYTHREADS)
? true
: false),
nested_write_transactions((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) ? false
: true),
exclusive((flags & MDBX_EXCLUSIVE) ? true : false),
disable_readahead((flags & MDBX_NORDAHEAD) ? true : false),
: no_sticky_threads(((flags & (MDBX_NOSTICKYTHREADS | MDBX_EXCLUSIVE)) == MDBX_NOSTICKYTHREADS) ? true : false),
nested_write_transactions((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) ? false : true),
exclusive((flags & MDBX_EXCLUSIVE) ? true : false), disable_readahead((flags & MDBX_NORDAHEAD) ? true : false),
disable_clear_memory((flags & MDBX_NOMEMINIT) ? true : false) {}
bool env::is_pristine() const {
return get_stat().ms_mod_txnid == 0 &&
get_info().mi_recent_txnid == INITIAL_TXNID;
}
bool env::is_pristine() const { return get_stat().ms_mod_txnid == 0 && get_info().mi_recent_txnid == INITIAL_TXNID; }
bool env::is_empty() const { return get_stat().ms_leaf_pages == 0; }
__cold env &env::copy(filehandle fd, bool compactify, bool force_dynamic_size) {
error::success_or_throw(
::mdbx_env_copy2fd(handle_, fd,
(compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
(force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE
: MDBX_CP_DEFAULTS)));
error::success_or_throw(::mdbx_env_copy2fd(handle_, fd,
(compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
(force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE : MDBX_CP_DEFAULTS)));
return *this;
}
__cold env &env::copy(const char *destination, bool compactify,
bool force_dynamic_size) {
error::success_or_throw(
::mdbx_env_copy(handle_, destination,
(compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
(force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE
: MDBX_CP_DEFAULTS)));
__cold env &env::copy(const char *destination, bool compactify, bool force_dynamic_size) {
error::success_or_throw(::mdbx_env_copy(handle_, destination,
(compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
(force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE : MDBX_CP_DEFAULTS)));
return *this;
}
__cold env &env::copy(const ::std::string &destination, bool compactify,
bool force_dynamic_size) {
__cold env &env::copy(const ::std::string &destination, bool compactify, bool force_dynamic_size) {
return copy(destination.c_str(), compactify, force_dynamic_size);
}
#if defined(_WIN32) || defined(_WIN64)
__cold env &env::copy(const wchar_t *destination, bool compactify,
bool force_dynamic_size) {
error::success_or_throw(
::mdbx_env_copyW(handle_, destination,
(compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
(force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE
: MDBX_CP_DEFAULTS)));
__cold env &env::copy(const wchar_t *destination, bool compactify, bool force_dynamic_size) {
error::success_or_throw(::mdbx_env_copyW(handle_, destination,
(compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
(force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE : MDBX_CP_DEFAULTS)));
return *this;
}
env &env::copy(const ::std::wstring &destination, bool compactify,
bool force_dynamic_size) {
env &env::copy(const ::std::wstring &destination, bool compactify, bool force_dynamic_size) {
return copy(destination.c_str(), compactify, force_dynamic_size);
}
#endif /* Windows */
#ifdef MDBX_STD_FILESYSTEM_PATH
__cold env &env::copy(const MDBX_STD_FILESYSTEM_PATH &destination,
bool compactify, bool force_dynamic_size) {
__cold env &env::copy(const MDBX_STD_FILESYSTEM_PATH &destination, bool compactify, bool force_dynamic_size) {
return copy(destination.native(), compactify, force_dynamic_size);
}
#endif /* MDBX_STD_FILESYSTEM_PATH */
@ -1375,8 +1311,7 @@ __cold path env::get_path() const {
}
__cold bool env::remove(const char *pathname, const remove_mode mode) {
return !error::boolean_or_throw(
::mdbx_env_delete(pathname, MDBX_env_delete_mode_t(mode)));
return !error::boolean_or_throw(::mdbx_env_delete(pathname, MDBX_env_delete_mode_t(mode)));
}
__cold bool env::remove(const ::std::string &pathname, const remove_mode mode) {
@ -1385,19 +1320,16 @@ __cold bool env::remove(const ::std::string &pathname, const remove_mode mode) {
#if defined(_WIN32) || defined(_WIN64)
__cold bool env::remove(const wchar_t *pathname, const remove_mode mode) {
return !error::boolean_or_throw(
::mdbx_env_deleteW(pathname, MDBX_env_delete_mode_t(mode)));
return !error::boolean_or_throw(::mdbx_env_deleteW(pathname, MDBX_env_delete_mode_t(mode)));
}
__cold bool env::remove(const ::std::wstring &pathname,
const remove_mode mode) {
__cold bool env::remove(const ::std::wstring &pathname, const remove_mode mode) {
return remove(pathname.c_str(), mode);
}
#endif /* Windows */
#ifdef MDBX_STD_FILESYSTEM_PATH
__cold bool env::remove(const MDBX_STD_FILESYSTEM_PATH &pathname,
const remove_mode mode) {
__cold bool env::remove(const MDBX_STD_FILESYSTEM_PATH &pathname, const remove_mode mode) {
return remove(pathname.native(), mode);
}
#endif /* MDBX_STD_FILESYSTEM_PATH */
@ -1413,13 +1345,11 @@ static inline MDBX_env *create_env() {
__cold env_managed::~env_managed() noexcept {
if (MDBX_UNLIKELY(handle_))
MDBX_CXX20_UNLIKELY error::success_or_panic(
::mdbx_env_close(handle_), "mdbx::~env()", "mdbx_env_close");
MDBX_CXX20_UNLIKELY error::success_or_panic(::mdbx_env_close(handle_), "mdbx::~env()", "mdbx_env_close");
}
__cold void env_managed::close(bool dont_sync) {
const error rc =
static_cast<MDBX_error_t>(::mdbx_env_close_ex(handle_, dont_sync));
const error rc = static_cast<MDBX_error_t>(::mdbx_env_close_ex(handle_, dont_sync));
switch (rc.code()) {
case MDBX_EBADSIGN:
MDBX_CXX20_UNLIKELY handle_ = nullptr;
@ -1438,87 +1368,69 @@ __cold void env_managed::setup(unsigned max_maps, unsigned max_readers) {
error::success_or_throw(::mdbx_env_set_maxdbs(handle_, max_maps));
}
__cold env_managed::env_managed(const char *pathname,
const operate_parameters &op, bool accede)
__cold env_managed::env_managed(const char *pathname, const operate_parameters &op, bool accede)
: env_managed(create_env()) {
setup(op.max_maps, op.max_readers);
error::success_or_throw(
::mdbx_env_open(handle_, pathname, op.make_flags(accede), 0));
error::success_or_throw(::mdbx_env_open(handle_, pathname, op.make_flags(accede), 0));
if (op.options.nested_write_transactions &&
!get_options().nested_write_transactions)
if (op.options.nested_write_transactions && !get_options().nested_write_transactions)
MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
}
__cold env_managed::env_managed(const char *pathname,
const env_managed::create_parameters &cp,
__cold env_managed::env_managed(const char *pathname, const env_managed::create_parameters &cp,
const env::operate_parameters &op, bool accede)
: env_managed(create_env()) {
setup(op.max_maps, op.max_readers);
set_geometry(cp.geometry);
error::success_or_throw(::mdbx_env_open(
handle_, pathname, op.make_flags(accede, cp.use_subdirectory),
cp.file_mode_bits));
error::success_or_throw(
::mdbx_env_open(handle_, pathname, op.make_flags(accede, cp.use_subdirectory), cp.file_mode_bits));
if (op.options.nested_write_transactions &&
!get_options().nested_write_transactions)
if (op.options.nested_write_transactions && !get_options().nested_write_transactions)
MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
}
__cold env_managed::env_managed(const ::std::string &pathname,
const operate_parameters &op, bool accede)
__cold env_managed::env_managed(const ::std::string &pathname, const operate_parameters &op, bool accede)
: env_managed(pathname.c_str(), op, accede) {}
__cold env_managed::env_managed(const ::std::string &pathname,
const env_managed::create_parameters &cp,
__cold env_managed::env_managed(const ::std::string &pathname, const env_managed::create_parameters &cp,
const env::operate_parameters &op, bool accede)
: env_managed(pathname.c_str(), cp, op, accede) {}
#if defined(_WIN32) || defined(_WIN64)
__cold env_managed::env_managed(const wchar_t *pathname,
const operate_parameters &op, bool accede)
__cold env_managed::env_managed(const wchar_t *pathname, const operate_parameters &op, bool accede)
: env_managed(create_env()) {
setup(op.max_maps, op.max_readers);
error::success_or_throw(
::mdbx_env_openW(handle_, pathname, op.make_flags(accede), 0));
error::success_or_throw(::mdbx_env_openW(handle_, pathname, op.make_flags(accede), 0));
if (op.options.nested_write_transactions &&
!get_options().nested_write_transactions)
if (op.options.nested_write_transactions && !get_options().nested_write_transactions)
MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
}
__cold env_managed::env_managed(const wchar_t *pathname,
const env_managed::create_parameters &cp,
__cold env_managed::env_managed(const wchar_t *pathname, const env_managed::create_parameters &cp,
const env::operate_parameters &op, bool accede)
: env_managed(create_env()) {
setup(op.max_maps, op.max_readers);
set_geometry(cp.geometry);
error::success_or_throw(::mdbx_env_openW(
handle_, pathname, op.make_flags(accede, cp.use_subdirectory),
cp.file_mode_bits));
error::success_or_throw(
::mdbx_env_openW(handle_, pathname, op.make_flags(accede, cp.use_subdirectory), cp.file_mode_bits));
if (op.options.nested_write_transactions &&
!get_options().nested_write_transactions)
if (op.options.nested_write_transactions && !get_options().nested_write_transactions)
MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
}
__cold env_managed::env_managed(const ::std::wstring &pathname,
const operate_parameters &op, bool accede)
__cold env_managed::env_managed(const ::std::wstring &pathname, const operate_parameters &op, bool accede)
: env_managed(pathname.c_str(), op, accede) {}
__cold env_managed::env_managed(const ::std::wstring &pathname,
const env_managed::create_parameters &cp,
__cold env_managed::env_managed(const ::std::wstring &pathname, const env_managed::create_parameters &cp,
const env::operate_parameters &op, bool accede)
: env_managed(pathname.c_str(), cp, op, accede) {}
#endif /* Windows */
#ifdef MDBX_STD_FILESYSTEM_PATH
__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname,
const operate_parameters &op, bool accede)
__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, const operate_parameters &op, bool accede)
: env_managed(pathname.native(), op, accede) {}
__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname,
const env_managed::create_parameters &cp,
__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, const env_managed::create_parameters &cp,
const env::operate_parameters &op, bool accede)
: env_managed(pathname.native(), cp, op, accede) {}
#endif /* MDBX_STD_FILESYSTEM_PATH */
@ -1528,16 +1440,14 @@ __cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname,
txn_managed txn::start_nested() {
MDBX_txn *nested;
error::throw_on_nullptr(handle_, MDBX_BAD_TXN);
error::success_or_throw(::mdbx_txn_begin(mdbx_txn_env(handle_), handle_,
MDBX_TXN_READWRITE, &nested));
error::success_or_throw(::mdbx_txn_begin(mdbx_txn_env(handle_), handle_, MDBX_TXN_READWRITE, &nested));
assert(nested != nullptr);
return txn_managed(nested);
}
txn_managed::~txn_managed() noexcept {
if (MDBX_UNLIKELY(handle_))
MDBX_CXX20_UNLIKELY error::success_or_panic(::mdbx_txn_abort(handle_),
"mdbx::~txn", "mdbx_txn_abort");
MDBX_CXX20_UNLIKELY error::success_or_panic(::mdbx_txn_abort(handle_), "mdbx::~txn", "mdbx_txn_abort");
}
void txn_managed::abort() {
@ -1557,8 +1467,7 @@ void txn_managed::commit() {
}
void txn_managed::commit(commit_latency *latency) {
const error err =
static_cast<MDBX_error_t>(::mdbx_txn_commit_ex(handle_, latency));
const error err = static_cast<MDBX_error_t>(::mdbx_txn_commit_ex(handle_, latency));
if (MDBX_LIKELY(err.code() != MDBX_THREAD_MISMATCH))
MDBX_CXX20_LIKELY handle_ = nullptr;
if (MDBX_UNLIKELY(err.code() != MDBX_SUCCESS))
@ -1568,8 +1477,7 @@ void txn_managed::commit(commit_latency *latency) {
void txn_managed::commit_embark_read() {
auto env = this->env();
commit();
error::success_or_throw(
::mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &handle_));
error::success_or_throw(::mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &handle_));
}
//------------------------------------------------------------------------------
@ -1608,8 +1516,7 @@ __cold bool txn::clear_map(const char *name, bool throw_if_absent) {
}
}
__cold bool txn::rename_map(const char *old_name, const char *new_name,
bool throw_if_absent) {
__cold bool txn::rename_map(const char *old_name, const char *new_name, bool throw_if_absent) {
map_handle map;
const int err = ::mdbx_dbi_open(handle_, old_name, MDBX_DB_ACCEDE, &map.dbi);
switch (err) {
@ -1660,9 +1567,7 @@ __cold bool txn::clear_map(const ::mdbx::slice &name, bool throw_if_absent) {
}
}
__cold bool txn::rename_map(const ::mdbx::slice &old_name,
const ::mdbx::slice &new_name,
bool throw_if_absent) {
__cold bool txn::rename_map(const ::mdbx::slice &old_name, const ::mdbx::slice &new_name, bool throw_if_absent) {
map_handle map;
const int err = ::mdbx_dbi_open2(handle_, old_name, MDBX_DB_ACCEDE, &map.dbi);
switch (err) {
@ -1679,11 +1584,8 @@ __cold bool txn::rename_map(const ::mdbx::slice &old_name,
}
}
__cold bool txn::rename_map(const ::std::string &old_name,
const ::std::string &new_name,
bool throw_if_absent) {
return rename_map(::mdbx::slice(old_name), ::mdbx::slice(new_name),
throw_if_absent);
__cold bool txn::rename_map(const ::std::string &old_name, const ::std::string &new_name, bool throw_if_absent) {
return rename_map(::mdbx::slice(old_name), ::mdbx::slice(new_name), throw_if_absent);
}
//------------------------------------------------------------------------------
@ -1723,12 +1625,10 @@ __cold ::std::ostream &operator<<(::std::ostream &out, const pair &it) {
}
__cold ::std::ostream &operator<<(::std::ostream &out, const pair_result &it) {
return out << "{" << (it.done ? "done: " : "non-done: ") << it.key << " => "
<< it.value << "}";
return out << "{" << (it.done ? "done: " : "non-done: ") << it.key << " => " << it.value << "}";
}
__cold ::std::ostream &operator<<(::std::ostream &out,
const ::mdbx::env::geometry::size &it) {
__cold ::std::ostream &operator<<(::std::ostream &out, const ::mdbx::env::geometry::size &it) {
switch (it.bytes) {
case ::mdbx::env::geometry::default_value:
return out << "default";
@ -1738,8 +1638,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out,
return out << "maximal";
}
const auto bytes = (it.bytes < 0) ? out << "-",
size_t(-it.bytes) : size_t(it.bytes);
const auto bytes = (it.bytes < 0) ? out << "-", size_t(-it.bytes) : size_t(it.bytes);
struct {
size_t one;
const char *suffix;
@ -1769,8 +1668,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out,
return out;
}
__cold ::std::ostream &operator<<(::std::ostream &out,
const env::geometry &it) {
__cold ::std::ostream &operator<<(::std::ostream &out, const env::geometry &it) {
return //
out << "\tlower " << env::geometry::size(it.size_lower) //
<< ",\n\tnow " << env::geometry::size(it.size_now) //
@ -1780,8 +1678,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out,
<< ",\n\tpagesize " << env::geometry::size(it.pagesize) << "\n";
}
__cold ::std::ostream &operator<<(::std::ostream &out,
const env::operate_parameters &it) {
__cold ::std::ostream &operator<<(::std::ostream &out, const env::operate_parameters &it) {
return out << "{\n" //
<< "\tmax_maps " << it.max_maps //
<< ",\n\tmax_readers " << it.max_readers //
@ -1805,8 +1702,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, const env::mode &it) {
}
}
__cold ::std::ostream &operator<<(::std::ostream &out,
const env::durability &it) {
__cold ::std::ostream &operator<<(::std::ostream &out, const env::durability &it) {
switch (it) {
case env::durability::robust_synchronous:
return out << "robust_synchronous";
@ -1821,16 +1717,14 @@ __cold ::std::ostream &operator<<(::std::ostream &out,
}
}
__cold ::std::ostream &operator<<(::std::ostream &out,
const env::reclaiming_options &it) {
__cold ::std::ostream &operator<<(::std::ostream &out, const env::reclaiming_options &it) {
return out << "{" //
<< "lifo: " << (it.lifo ? "yes" : "no") //
<< ", coalesce: " << (it.coalesce ? "yes" : "no") //
<< "}";
}
__cold ::std::ostream &operator<<(::std::ostream &out,
const env::operate_options &it) {
__cold ::std::ostream &operator<<(::std::ostream &out, const env::operate_options &it) {
static const char comma[] = ", ";
const char *delimiter = "";
out << "{";
@ -1859,8 +1753,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out,
return out << "}";
}
__cold ::std::ostream &operator<<(::std::ostream &out,
const env_managed::create_parameters &it) {
__cold ::std::ostream &operator<<(::std::ostream &out, const env_managed::create_parameters &it) {
return out << "{\n" //
<< "\tfile_mode " << std::oct << it.file_mode_bits << std::dec //
<< ",\n\tsubdirectory " << (it.use_subdirectory ? "yes" : "no") //
@ -1868,8 +1761,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out,
<< it.geometry << "}";
}
__cold ::std::ostream &operator<<(::std::ostream &out,
const MDBX_log_level_t &it) {
__cold ::std::ostream &operator<<(::std::ostream &out, const MDBX_log_level_t &it) {
switch (it) {
case MDBX_LOG_FATAL:
return out << "LOG_FATAL";
@ -1894,8 +1786,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out,
}
}
__cold ::std::ostream &operator<<(::std::ostream &out,
const MDBX_debug_flags_t &it) {
__cold ::std::ostream &operator<<(::std::ostream &out, const MDBX_debug_flags_t &it) {
if (it == MDBX_DBG_DONTCHANGE)
return out << "DBG_DONTCHANGE";
@ -1931,8 +1822,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out,
return out << "}";
}
__cold ::std::ostream &operator<<(::std::ostream &out,
const ::mdbx::error &err) {
__cold ::std::ostream &operator<<(::std::ostream &out, const ::mdbx::error &err) {
return out << err.what() << " (" << long(err.code()) << ")";
}

322
src/meta.c
View File

@ -9,15 +9,11 @@ typedef struct meta_snap {
} meta_snap_t;
static inline txnid_t fetch_txnid(const volatile mdbx_atomic_uint32_t *ptr) {
#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && \
MDBX_UNALIGNED_OK >= 8
return atomic_load64((const volatile mdbx_atomic_uint64_t *)ptr,
mo_AcquireRelease);
#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && MDBX_UNALIGNED_OK >= 8
return atomic_load64((const volatile mdbx_atomic_uint64_t *)ptr, mo_AcquireRelease);
#else
const uint32_t l = atomic_load32(
&ptr[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], mo_AcquireRelease);
const uint32_t h = atomic_load32(
&ptr[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], mo_AcquireRelease);
const uint32_t l = atomic_load32(&ptr[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], mo_AcquireRelease);
const uint32_t h = atomic_load32(&ptr[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], mo_AcquireRelease);
return (uint64_t)h << 32 | l;
#endif
}
@ -33,9 +29,7 @@ static inline meta_snap_t meta_snap(const volatile meta_t *meta) {
return r;
}
txnid_t meta_txnid(const volatile meta_t *meta) {
return meta_snap(meta).txnid;
}
txnid_t meta_txnid(const volatile meta_t *meta) { return meta_snap(meta).txnid; }
meta_ptr_t meta_ptr(const MDBX_env *env, unsigned n) {
eASSERT(env, n < NUM_METAS);
@ -46,16 +40,13 @@ meta_ptr_t meta_ptr(const MDBX_env *env, unsigned n) {
return r;
}
static uint8_t meta_cmp2pack(uint8_t c01, uint8_t c02, uint8_t c12, bool s0,
bool s1, bool s2) {
static uint8_t meta_cmp2pack(uint8_t c01, uint8_t c02, uint8_t c12, bool s0, bool s1, bool s2) {
assert(c01 < 3 && c02 < 3 && c12 < 3);
/* assert(s0 < 2 && s1 < 2 && s2 < 2); */
const uint8_t recent = meta_cmp2recent(c01, s0, s1)
? (meta_cmp2recent(c02, s0, s2) ? 0 : 2)
: (meta_cmp2recent(c12, s1, s2) ? 1 : 2);
const uint8_t prefer_steady = meta_cmp2steady(c01, s0, s1)
? (meta_cmp2steady(c02, s0, s2) ? 0 : 2)
: (meta_cmp2steady(c12, s1, s2) ? 1 : 2);
const uint8_t recent =
meta_cmp2recent(c01, s0, s1) ? (meta_cmp2recent(c02, s0, s2) ? 0 : 2) : (meta_cmp2recent(c12, s1, s2) ? 1 : 2);
const uint8_t prefer_steady =
meta_cmp2steady(c01, s0, s1) ? (meta_cmp2steady(c02, s0, s2) ? 0 : 2) : (meta_cmp2steady(c12, s1, s2) ? 1 : 2);
uint8_t tail;
if (recent == 0)
@ -65,10 +56,8 @@ static uint8_t meta_cmp2pack(uint8_t c01, uint8_t c02, uint8_t c12, bool s0,
else
tail = meta_cmp2steady(c01, s0, s1) ? 1 : 0;
const bool valid =
c01 != 1 || s0 != s1 || c02 != 1 || s0 != s2 || c12 != 1 || s1 != s2;
const bool strict = (c01 != 1 || s0 != s1) && (c02 != 1 || s0 != s2) &&
(c12 != 1 || s1 != s2);
const bool valid = c01 != 1 || s0 != s1 || c02 != 1 || s0 != s2 || c12 != 1 || s1 != s2;
const bool strict = (c01 != 1 || s0 != s1) && (c02 != 1 || s0 != s2) && (c12 != 1 || s1 != s2);
return tail | recent << 2 | prefer_steady << 4 | strict << 6 | valid << 7;
}
@ -82,21 +71,16 @@ static inline void meta_troika_unpack(troika_t *troika, const uint8_t packed) {
}
static const uint8_t troika_fsm_map[2 * 2 * 2 * 3 * 3 * 3] = {
232, 201, 216, 216, 232, 233, 232, 232, 168, 201, 216, 152, 168, 233, 232,
168, 233, 201, 216, 201, 233, 233, 232, 233, 168, 201, 152, 216, 232, 169,
232, 168, 168, 193, 152, 152, 168, 169, 232, 168, 169, 193, 152, 194, 233,
169, 232, 169, 232, 201, 216, 216, 232, 201, 232, 232, 168, 193, 216, 152,
168, 193, 232, 168, 193, 193, 210, 194, 225, 193, 225, 193, 168, 137, 212,
214, 232, 233, 168, 168, 168, 137, 212, 150, 168, 233, 168, 168, 169, 137,
216, 201, 233, 233, 168, 169, 168, 137, 148, 214, 232, 169, 168, 168, 40,
129, 148, 150, 168, 169, 168, 40, 169, 129, 152, 194, 233, 169, 168, 169,
168, 137, 214, 214, 232, 201, 168, 168, 168, 129, 214, 150, 168, 193, 168,
168, 129, 129, 210, 194, 225, 193, 161, 129, 212, 198, 212, 214, 228, 228,
212, 212, 148, 201, 212, 150, 164, 233, 212, 148, 233, 201, 216, 201, 233,
233, 216, 233, 148, 198, 148, 214, 228, 164, 212, 148, 148, 194, 148, 150,
164, 169, 212, 148, 169, 194, 152, 194, 233, 169, 216, 169, 214, 198, 214,
214, 228, 198, 212, 214, 150, 194, 214, 150, 164, 193, 212, 150, 194, 194,
210, 194, 225, 193, 210, 194};
232, 201, 216, 216, 232, 233, 232, 232, 168, 201, 216, 152, 168, 233, 232, 168, 233, 201, 216, 201, 233, 233,
232, 233, 168, 201, 152, 216, 232, 169, 232, 168, 168, 193, 152, 152, 168, 169, 232, 168, 169, 193, 152, 194,
233, 169, 232, 169, 232, 201, 216, 216, 232, 201, 232, 232, 168, 193, 216, 152, 168, 193, 232, 168, 193, 193,
210, 194, 225, 193, 225, 193, 168, 137, 212, 214, 232, 233, 168, 168, 168, 137, 212, 150, 168, 233, 168, 168,
169, 137, 216, 201, 233, 233, 168, 169, 168, 137, 148, 214, 232, 169, 168, 168, 40, 129, 148, 150, 168, 169,
168, 40, 169, 129, 152, 194, 233, 169, 168, 169, 168, 137, 214, 214, 232, 201, 168, 168, 168, 129, 214, 150,
168, 193, 168, 168, 129, 129, 210, 194, 225, 193, 161, 129, 212, 198, 212, 214, 228, 228, 212, 212, 148, 201,
212, 150, 164, 233, 212, 148, 233, 201, 216, 201, 233, 233, 216, 233, 148, 198, 148, 214, 228, 164, 212, 148,
148, 194, 148, 150, 164, 169, 212, 148, 169, 194, 152, 194, 233, 169, 216, 169, 214, 198, 214, 214, 228, 198,
212, 214, 150, 194, 214, 150, 164, 193, 212, 150, 194, 194, 210, 194, 225, 193, 210, 194};
__cold bool troika_verify_fsm(void) {
bool ok = true;
@ -117,12 +101,10 @@ __cold bool troika_verify_fsm(void) {
const bool strict = TROIKA_STRICT_VALID(&troika);
const bool valid = TROIKA_VALID(&troika);
const uint8_t recent_chk = meta_cmp2recent(c01, s0, s1)
? (meta_cmp2recent(c02, s0, s2) ? 0 : 2)
: (meta_cmp2recent(c12, s1, s2) ? 1 : 2);
const uint8_t recent_chk =
meta_cmp2recent(c01, s0, s1) ? (meta_cmp2recent(c02, s0, s2) ? 0 : 2) : (meta_cmp2recent(c12, s1, s2) ? 1 : 2);
const uint8_t prefer_steady_chk =
meta_cmp2steady(c01, s0, s1) ? (meta_cmp2steady(c02, s0, s2) ? 0 : 2)
: (meta_cmp2steady(c12, s1, s2) ? 1 : 2);
meta_cmp2steady(c01, s0, s1) ? (meta_cmp2steady(c02, s0, s2) ? 0 : 2) : (meta_cmp2steady(c12, s1, s2) ? 1 : 2);
uint8_t tail_chk;
if (recent_chk == 0)
@ -132,20 +114,16 @@ __cold bool troika_verify_fsm(void) {
else
tail_chk = meta_cmp2steady(c01, s0, s1) ? 1 : 0;
const bool valid_chk =
c01 != 1 || s0 != s1 || c02 != 1 || s0 != s2 || c12 != 1 || s1 != s2;
const bool strict_chk = (c01 != 1 || s0 != s1) && (c02 != 1 || s0 != s2) &&
(c12 != 1 || s1 != s2);
const bool valid_chk = c01 != 1 || s0 != s1 || c02 != 1 || s0 != s2 || c12 != 1 || s1 != s2;
const bool strict_chk = (c01 != 1 || s0 != s1) && (c02 != 1 || s0 != s2) && (c12 != 1 || s1 != s2);
assert(troika.recent == recent_chk);
assert(troika.prefer_steady == prefer_steady_chk);
assert(tail == tail_chk);
assert(valid == valid_chk);
assert(strict == strict_chk);
assert(troika_fsm_map[troika.fsm] == packed);
if (troika.recent != recent_chk ||
troika.prefer_steady != prefer_steady_chk || tail != tail_chk ||
valid != valid_chk || strict != strict_chk ||
troika_fsm_map[troika.fsm] != packed) {
if (troika.recent != recent_chk || troika.prefer_steady != prefer_steady_chk || tail != tail_chk ||
valid != valid_chk || strict != strict_chk || troika_fsm_map[troika.fsm] != packed) {
ok = false;
}
}
@ -181,27 +159,24 @@ txnid_t recent_committed_txnid(const MDBX_env *env) {
static inline bool meta_eq(const troika_t *troika, size_t a, size_t b) {
assert(a < NUM_METAS && b < NUM_METAS);
return troika->txnid[a] == troika->txnid[b] &&
(((troika->fsm >> a) ^ (troika->fsm >> b)) & 1) == 0 &&
return troika->txnid[a] == troika->txnid[b] && (((troika->fsm >> a) ^ (troika->fsm >> b)) & 1) == 0 &&
troika->txnid[a];
}
unsigned meta_eq_mask(const troika_t *troika) {
return meta_eq(troika, 0, 1) | meta_eq(troika, 1, 2) << 1 |
meta_eq(troika, 2, 0) << 2;
return meta_eq(troika, 0, 1) | meta_eq(troika, 1, 2) << 1 | meta_eq(troika, 2, 0) << 2;
}
__hot bool meta_should_retry(const MDBX_env *env, troika_t *troika) {
const troika_t prev = *troika;
*troika = meta_tap(env);
return prev.fsm != troika->fsm || prev.txnid[0] != troika->txnid[0] ||
prev.txnid[1] != troika->txnid[1] || prev.txnid[2] != troika->txnid[2];
return prev.fsm != troika->fsm || prev.txnid[0] != troika->txnid[0] || prev.txnid[1] != troika->txnid[1] ||
prev.txnid[2] != troika->txnid[2];
}
const char *durable_caption(const meta_t *const meta) {
if (meta_is_steady(meta))
return (meta_sign_get(meta) == meta_sign_calculate(meta)) ? "Steady"
: "Tainted";
return (meta_sign_get(meta) == meta_sign_calculate(meta)) ? "Steady" : "Tainted";
return "Weak";
}
@ -214,20 +189,16 @@ __cold void meta_troika_dump(const MDBX_env *env, const troika_t *troika) {
"base=%d-%" PRIaTXN ".%c, "
"tail=%d-%" PRIaTXN ".%c, "
"valid %c, strict %c",
troika->txnid[0], (troika->fsm & 1) ? 's' : 'w', troika->txnid[1],
(troika->fsm & 2) ? 's' : 'w', troika->txnid[2],
(troika->fsm & 4) ? 's' : 'w', troika->fsm, troika->recent,
recent.txnid, recent.is_steady ? 's' : 'w', troika->prefer_steady,
prefer_steady.txnid, prefer_steady.is_steady ? 's' : 'w',
troika->tail_and_flags % NUM_METAS, tail.txnid,
tail.is_steady ? 's' : 'w', TROIKA_VALID(troika) ? 'Y' : 'N',
troika->txnid[0], (troika->fsm & 1) ? 's' : 'w', troika->txnid[1], (troika->fsm & 2) ? 's' : 'w',
troika->txnid[2], (troika->fsm & 4) ? 's' : 'w', troika->fsm, troika->recent, recent.txnid,
recent.is_steady ? 's' : 'w', troika->prefer_steady, prefer_steady.txnid, prefer_steady.is_steady ? 's' : 'w',
troika->tail_and_flags % NUM_METAS, tail.txnid, tail.is_steady ? 's' : 'w', TROIKA_VALID(troika) ? 'Y' : 'N',
TROIKA_STRICT_VALID(troika) ? 'Y' : 'N');
}
/*----------------------------------------------------------------------------*/
static int meta_unsteady(MDBX_env *env, const txnid_t inclusive_upto,
const pgno_t pgno) {
static int meta_unsteady(MDBX_env *env, const txnid_t inclusive_upto, const pgno_t pgno) {
meta_t *const meta = METAPAGE(env, pgno);
const txnid_t txnid = constmeta_txnid(meta);
if (!meta_is_steady(meta) || txnid > inclusive_upto)
@ -236,8 +207,7 @@ static int meta_unsteady(MDBX_env *env, const txnid_t inclusive_upto,
WARNING("wipe txn #%" PRIaTXN ", meta %" PRIaPGNO, txnid, pgno);
const uint64_t wipe = DATASIGN_NONE;
const void *ptr = &wipe;
size_t bytes = sizeof(meta->sign),
offset = ptr_dist(&meta->sign, env->dxb_mmap.base);
size_t bytes = sizeof(meta->sign), offset = ptr_dist(&meta->sign, env->dxb_mmap.base);
if (env->flags & MDBX_WRITEMAP) {
unaligned_poke_u64(4, meta->sign, wipe);
osal_flush_incoherent_cpu_writeback();
@ -265,8 +235,7 @@ __cold int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto) {
if (err == MDBX_RESULT_TRUE) {
err = MDBX_SUCCESS;
if (!MDBX_AVOID_MSYNC && (env->flags & MDBX_WRITEMAP)) {
err = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS),
MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
err = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
#if MDBX_ENABLE_PGOP_STAT
env->lck->pgops.msync.weak += 1;
#endif /* MDBX_ENABLE_PGOP_STAT */
@ -278,8 +247,7 @@ __cold int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto) {
}
}
osal_flush_incoherent_mmap(env->dxb_mmap.base, pgno2bytes(env, NUM_METAS),
globals.sys_pagesize);
osal_flush_incoherent_mmap(env->dxb_mmap.base, pgno2bytes(env, NUM_METAS), globals.sys_pagesize);
/* force oldest refresh */
atomic_store32(&env->lck->rdt_refresh_flag, true, mo_Relaxed);
@ -291,8 +259,7 @@ __cold int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto) {
}
int meta_sync(const MDBX_env *env, const meta_ptr_t head) {
eASSERT(env, atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed) !=
(uint32_t)head.txnid);
eASSERT(env, atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed) != (uint32_t)head.txnid);
/* Функция может вызываться (в том числе) при (env->flags &
* MDBX_NOMETASYNC) == 0 и env->fd4meta == env->dsync_fd, например если
* предыдущая транзакция была выполненна с флагом MDBX_NOMETASYNC. */
@ -300,8 +267,7 @@ int meta_sync(const MDBX_env *env, const meta_ptr_t head) {
int rc = MDBX_RESULT_TRUE;
if (env->flags & MDBX_WRITEMAP) {
if (!MDBX_AVOID_MSYNC) {
rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS),
MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
#if MDBX_ENABLE_PGOP_STAT
env->lck->pgops.msync.weak += 1;
#endif /* MDBX_ENABLE_PGOP_STAT */
@ -310,8 +276,7 @@ int meta_sync(const MDBX_env *env, const meta_ptr_t head) {
env->lck->pgops.wops.weak += 1;
#endif /* MDBX_ENABLE_PGOP_STAT */
const page_t *page = data_page(head.ptr_c);
rc = osal_pwrite(env->fd4meta, page, env->ps,
ptr_dist(page, env->dxb_mmap.base));
rc = osal_pwrite(env->fd4meta, page, env->ps, ptr_dist(page, env->dxb_mmap.base));
if (likely(rc == MDBX_SUCCESS) && env->fd4meta == env->lazy_fd) {
rc = osal_fsync(env->lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
@ -332,8 +297,7 @@ int meta_sync(const MDBX_env *env, const meta_ptr_t head) {
return rc;
}
__cold static page_t *meta_model(const MDBX_env *env, page_t *model, size_t num,
const bin128_t *guid) {
__cold static page_t *meta_model(const MDBX_env *env, page_t *model, size_t num, const bin128_t *guid) {
ENSURE(env, is_powerof2(env->ps));
ENSURE(env, env->ps >= MDBX_MIN_PAGESIZE);
ENSURE(env, env->ps <= MDBX_MAX_PAGESIZE);
@ -350,10 +314,8 @@ __cold static page_t *meta_model(const MDBX_env *env, page_t *model, size_t num,
model_meta->geometry.lower = bytes2pgno(env, env->geo_in_bytes.lower);
model_meta->geometry.upper = bytes2pgno(env, env->geo_in_bytes.upper);
model_meta->geometry.grow_pv =
pages2pv(bytes2pgno(env, env->geo_in_bytes.grow));
model_meta->geometry.shrink_pv =
pages2pv(bytes2pgno(env, env->geo_in_bytes.shrink));
model_meta->geometry.grow_pv = pages2pv(bytes2pgno(env, env->geo_in_bytes.grow));
model_meta->geometry.shrink_pv = pages2pv(bytes2pgno(env, env->geo_in_bytes.shrink));
model_meta->geometry.now = bytes2pgno(env, env->geo_in_bytes.now);
model_meta->geometry.first_unallocated = NUM_METAS;
@ -362,12 +324,9 @@ __cold static page_t *meta_model(const MDBX_env *env, page_t *model, size_t num,
ENSURE(env, model_meta->geometry.now >= model_meta->geometry.lower);
ENSURE(env, model_meta->geometry.now <= model_meta->geometry.upper);
ENSURE(env, model_meta->geometry.first_unallocated >= MIN_PAGENO);
ENSURE(env,
model_meta->geometry.first_unallocated <= model_meta->geometry.now);
ENSURE(env, model_meta->geometry.grow_pv ==
pages2pv(pv2pages(model_meta->geometry.grow_pv)));
ENSURE(env, model_meta->geometry.shrink_pv ==
pages2pv(pv2pages(model_meta->geometry.shrink_pv)));
ENSURE(env, model_meta->geometry.first_unallocated <= model_meta->geometry.now);
ENSURE(env, model_meta->geometry.grow_pv == pages2pv(pv2pages(model_meta->geometry.grow_pv)));
ENSURE(env, model_meta->geometry.shrink_pv == pages2pv(pv2pages(model_meta->geometry.shrink_pv)));
model_meta->pagesize = env->ps;
model_meta->trees.gc.flags = MDBX_INTEGERKEY;
@ -389,12 +348,9 @@ __cold meta_t *meta_init_triplet(const MDBX_env *env, void *buffer) {
return page_meta(page2);
}
__cold int __must_check_result meta_override(MDBX_env *env, size_t target,
txnid_t txnid,
const meta_t *shape) {
__cold int __must_check_result meta_override(MDBX_env *env, size_t target, txnid_t txnid, const meta_t *shape) {
page_t *const page = env->page_auxbuf;
meta_model(env, page, target,
&((target == 0 && shape) ? shape : METAPAGE(env, 0))->dxbid);
meta_model(env, page, target, &((target == 0 && shape) ? shape : METAPAGE(env, 0))->dxbid);
meta_t *const model = page_meta(page);
meta_set_txnid(env, model, txnid);
if (txnid)
@ -407,21 +363,18 @@ __cold int __must_check_result meta_override(MDBX_env *env, size_t target,
return MDBX_PROBLEM;
}
if (globals.runtime_flags & MDBX_DBG_DONT_UPGRADE)
memcpy(&model->magic_and_version, &shape->magic_and_version,
sizeof(model->magic_and_version));
memcpy(&model->magic_and_version, &shape->magic_and_version, sizeof(model->magic_and_version));
model->reserve16 = shape->reserve16;
model->validator_id = shape->validator_id;
model->extra_pagehdr = shape->extra_pagehdr;
memcpy(&model->geometry, &shape->geometry, sizeof(model->geometry));
memcpy(&model->trees, &shape->trees, sizeof(model->trees));
memcpy(&model->canary, &shape->canary, sizeof(model->canary));
memcpy(&model->pages_retired, &shape->pages_retired,
sizeof(model->pages_retired));
memcpy(&model->pages_retired, &shape->pages_retired, sizeof(model->pages_retired));
if (txnid) {
if ((!model->trees.gc.mod_txnid && model->trees.gc.root != P_INVALID) ||
(!model->trees.main.mod_txnid && model->trees.main.root != P_INVALID))
memcpy(&model->magic_and_version, &shape->magic_and_version,
sizeof(model->magic_and_version));
memcpy(&model->magic_and_version, &shape->magic_and_version, sizeof(model->magic_and_version));
if (unlikely(!coherency_check_meta(env, model, false))) {
ERROR("bailout overriding meta-%zu since model failed "
"FreeDB/MainDB %s-check for txnid #%" PRIaTXN,
@ -452,8 +405,7 @@ __cold int __must_check_result meta_override(MDBX_env *env, size_t target,
#if MDBX_ENABLE_PGOP_STAT
env->lck->pgops.msync.weak += 1;
#endif /* MDBX_ENABLE_PGOP_STAT */
rc = osal_msync(&env->dxb_mmap, 0,
pgno_align2os_bytes(env, model->geometry.first_unallocated),
rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, model->geometry.first_unallocated),
MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
@ -465,8 +417,7 @@ __cold int __must_check_result meta_override(MDBX_env *env, size_t target,
#if MDBX_ENABLE_PGOP_STAT
env->lck->pgops.msync.weak += 1;
#endif /* MDBX_ENABLE_PGOP_STAT */
rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, target + 1),
MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, target + 1), MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
} else {
#if MDBX_ENABLE_PGOP_STAT
env->lck->pgops.wops.weak += 1;
@ -478,28 +429,20 @@ __cold int __must_check_result meta_override(MDBX_env *env, size_t target,
#endif /* MDBX_ENABLE_PGOP_STAT */
rc = osal_fsync(env->lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ);
}
osal_flush_incoherent_mmap(env->dxb_mmap.base, pgno2bytes(env, NUM_METAS),
globals.sys_pagesize);
osal_flush_incoherent_mmap(env->dxb_mmap.base, pgno2bytes(env, NUM_METAS), globals.sys_pagesize);
}
eASSERT(env, (!env->txn && (env->flags & ENV_ACTIVE) == 0) ||
(env->stuck_meta == (int)target &&
(env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) ==
MDBX_EXCLUSIVE));
(env->stuck_meta == (int)target && (env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == MDBX_EXCLUSIVE));
return rc;
}
__cold int meta_validate(MDBX_env *env, meta_t *const meta,
const page_t *const page, const unsigned meta_number,
__cold int meta_validate(MDBX_env *env, meta_t *const meta, const page_t *const page, const unsigned meta_number,
unsigned *guess_pagesize) {
const uint64_t magic_and_version =
unaligned_peek_u64(4, &meta->magic_and_version);
if (unlikely(magic_and_version != MDBX_DATA_MAGIC &&
magic_and_version != MDBX_DATA_MAGIC_LEGACY_COMPAT &&
const uint64_t magic_and_version = unaligned_peek_u64(4, &meta->magic_and_version);
if (unlikely(magic_and_version != MDBX_DATA_MAGIC && magic_and_version != MDBX_DATA_MAGIC_LEGACY_COMPAT &&
magic_and_version != MDBX_DATA_MAGIC_LEGACY_DEVEL)) {
ERROR("meta[%u] has invalid magic/version %" PRIx64, meta_number,
magic_and_version);
return ((magic_and_version >> 8) != MDBX_MAGIC) ? MDBX_INVALID
: MDBX_VERSION_MISMATCH;
ERROR("meta[%u] has invalid magic/version %" PRIx64, meta_number, magic_and_version);
return ((magic_and_version >> 8) != MDBX_MAGIC) ? MDBX_INVALID : MDBX_VERSION_MISMATCH;
}
if (unlikely(page->pgno != meta_number)) {
@ -512,11 +455,9 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta,
return MDBX_INVALID;
}
if (unlikely(!is_powerof2(meta->pagesize) ||
meta->pagesize < MDBX_MIN_PAGESIZE ||
if (unlikely(!is_powerof2(meta->pagesize) || meta->pagesize < MDBX_MIN_PAGESIZE ||
meta->pagesize > MDBX_MAX_PAGESIZE)) {
WARNING("meta[%u] has invalid pagesize (%u), skip it", meta_number,
meta->pagesize);
WARNING("meta[%u] has invalid pagesize (%u), skip it", meta_number, meta->pagesize);
return is_powerof2(meta->pagesize) ? MDBX_VERSION_MISMATCH : MDBX_INVALID;
}
@ -535,81 +476,63 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta,
const uint64_t sign = meta_sign_get(meta);
const uint64_t sign_stready = meta_sign_calculate(meta);
if (SIGN_IS_STEADY(sign) && unlikely(sign != sign_stready)) {
WARNING("meta[%u] has invalid steady-checksum (0x%" PRIx64 " != 0x%" PRIx64
"), skip it",
meta_number, sign, sign_stready);
WARNING("meta[%u] has invalid steady-checksum (0x%" PRIx64 " != 0x%" PRIx64 "), skip it", meta_number, sign,
sign_stready);
return MDBX_RESULT_TRUE;
}
if (unlikely(meta->trees.gc.flags != MDBX_INTEGERKEY) &&
((meta->trees.gc.flags & DB_PERSISTENT_FLAGS) != MDBX_INTEGERKEY ||
magic_and_version == MDBX_DATA_MAGIC)) {
WARNING("meta[%u] has invalid %s flags 0x%x, skip it", meta_number,
"GC/FreeDB", meta->trees.gc.flags);
((meta->trees.gc.flags & DB_PERSISTENT_FLAGS) != MDBX_INTEGERKEY || magic_and_version == MDBX_DATA_MAGIC)) {
WARNING("meta[%u] has invalid %s flags 0x%x, skip it", meta_number, "GC/FreeDB", meta->trees.gc.flags);
return MDBX_INCOMPATIBLE;
}
if (unlikely(!check_table_flags(meta->trees.main.flags))) {
WARNING("meta[%u] has invalid %s flags 0x%x, skip it", meta_number,
"MainDB", meta->trees.main.flags);
WARNING("meta[%u] has invalid %s flags 0x%x, skip it", meta_number, "MainDB", meta->trees.main.flags);
return MDBX_INCOMPATIBLE;
}
DEBUG("checking meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO
", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO
" +%u -%u, txn_id %" PRIaTXN ", %s",
page->pgno, meta->trees.main.root, meta->trees.gc.root,
meta->geometry.lower, meta->geometry.first_unallocated,
meta->geometry.now, meta->geometry.upper,
pv2pages(meta->geometry.grow_pv), pv2pages(meta->geometry.shrink_pv),
DEBUG("checking meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO
"/%" PRIaPGNO " +%u -%u, txn_id %" PRIaTXN ", %s",
page->pgno, meta->trees.main.root, meta->trees.gc.root, meta->geometry.lower, meta->geometry.first_unallocated,
meta->geometry.now, meta->geometry.upper, pv2pages(meta->geometry.grow_pv), pv2pages(meta->geometry.shrink_pv),
txnid, durable_caption(meta));
if (unlikely(txnid < MIN_TXNID || txnid > MAX_TXNID)) {
WARNING("meta[%u] has invalid txnid %" PRIaTXN ", skip it", meta_number,
txnid);
WARNING("meta[%u] has invalid txnid %" PRIaTXN ", skip it", meta_number, txnid);
return MDBX_RESULT_TRUE;
}
if (unlikely(meta->geometry.lower < MIN_PAGENO ||
meta->geometry.lower > MAX_PAGENO + 1)) {
WARNING("meta[%u] has invalid min-pages (%" PRIaPGNO "), skip it",
meta_number, meta->geometry.lower);
if (unlikely(meta->geometry.lower < MIN_PAGENO || meta->geometry.lower > MAX_PAGENO + 1)) {
WARNING("meta[%u] has invalid min-pages (%" PRIaPGNO "), skip it", meta_number, meta->geometry.lower);
return MDBX_INVALID;
}
if (unlikely(meta->geometry.upper < MIN_PAGENO ||
meta->geometry.upper > MAX_PAGENO + 1 ||
if (unlikely(meta->geometry.upper < MIN_PAGENO || meta->geometry.upper > MAX_PAGENO + 1 ||
meta->geometry.upper < meta->geometry.lower)) {
WARNING("meta[%u] has invalid max-pages (%" PRIaPGNO "), skip it",
meta_number, meta->geometry.upper);
WARNING("meta[%u] has invalid max-pages (%" PRIaPGNO "), skip it", meta_number, meta->geometry.upper);
return MDBX_INVALID;
}
if (unlikely(meta->geometry.first_unallocated < MIN_PAGENO ||
meta->geometry.first_unallocated - 1 > MAX_PAGENO)) {
WARNING("meta[%u] has invalid next-pageno (%" PRIaPGNO "), skip it",
meta_number, meta->geometry.first_unallocated);
if (unlikely(meta->geometry.first_unallocated < MIN_PAGENO || meta->geometry.first_unallocated - 1 > MAX_PAGENO)) {
WARNING("meta[%u] has invalid next-pageno (%" PRIaPGNO "), skip it", meta_number, meta->geometry.first_unallocated);
return MDBX_CORRUPTED;
}
const uint64_t used_bytes =
meta->geometry.first_unallocated * (uint64_t)meta->pagesize;
const uint64_t used_bytes = meta->geometry.first_unallocated * (uint64_t)meta->pagesize;
if (unlikely(used_bytes > env->dxb_mmap.filesize)) {
/* Here could be a race with DB-shrinking performed by other process */
int err = osal_filesize(env->lazy_fd, &env->dxb_mmap.filesize);
if (unlikely(err != MDBX_SUCCESS))
return err;
if (unlikely(used_bytes > env->dxb_mmap.filesize)) {
WARNING("meta[%u] used-bytes (%" PRIu64 ") beyond filesize (%" PRIu64
"), skip it",
meta_number, used_bytes, env->dxb_mmap.filesize);
WARNING("meta[%u] used-bytes (%" PRIu64 ") beyond filesize (%" PRIu64 "), skip it", meta_number, used_bytes,
env->dxb_mmap.filesize);
return MDBX_CORRUPTED;
}
}
if (unlikely(meta->geometry.first_unallocated - 1 > MAX_PAGENO ||
used_bytes > MAX_MAPSIZE)) {
WARNING("meta[%u] has too large used-space (%" PRIu64 "), skip it",
meta_number, used_bytes);
if (unlikely(meta->geometry.first_unallocated - 1 > MAX_PAGENO || used_bytes > MAX_MAPSIZE)) {
WARNING("meta[%u] has too large used-space (%" PRIu64 "), skip it", meta_number, used_bytes);
return MDBX_TOO_LARGE;
}
@ -617,13 +540,10 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta,
uint64_t mapsize_min = geo_lower * (uint64_t)meta->pagesize;
STATIC_ASSERT(MAX_MAPSIZE < PTRDIFF_MAX - MDBX_MAX_PAGESIZE);
STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE);
STATIC_ASSERT((uint64_t)(MAX_PAGENO + 1) * MDBX_MIN_PAGESIZE % (4ul << 20) ==
0);
STATIC_ASSERT((uint64_t)(MAX_PAGENO + 1) * MDBX_MIN_PAGESIZE % (4ul << 20) == 0);
if (unlikely(mapsize_min < MIN_MAPSIZE || mapsize_min > MAX_MAPSIZE)) {
if (MAX_MAPSIZE != MAX_MAPSIZE64 && mapsize_min > MAX_MAPSIZE &&
mapsize_min <= MAX_MAPSIZE64) {
eASSERT(env, meta->geometry.first_unallocated - 1 <= MAX_PAGENO &&
used_bytes <= MAX_MAPSIZE);
if (MAX_MAPSIZE != MAX_MAPSIZE64 && mapsize_min > MAX_MAPSIZE && mapsize_min <= MAX_MAPSIZE64) {
eASSERT(env, meta->geometry.first_unallocated - 1 <= MAX_PAGENO && used_bytes <= MAX_MAPSIZE);
WARNING("meta[%u] has too large min-mapsize (%" PRIu64 "), "
"but size of used space still acceptable (%" PRIu64 ")",
meta_number, mapsize_min, used_bytes);
@ -632,14 +552,12 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta,
geo_lower = MAX_PAGENO + 1;
mapsize_min = geo_lower * (uint64_t)meta->pagesize;
}
WARNING("meta[%u] consider get-%s pageno is %" PRIaPGNO
" instead of wrong %" PRIaPGNO
WARNING("meta[%u] consider get-%s pageno is %" PRIaPGNO " instead of wrong %" PRIaPGNO
", will be corrected on next commit(s)",
meta_number, "lower", geo_lower, meta->geometry.lower);
meta->geometry.lower = geo_lower;
} else {
WARNING("meta[%u] has invalid min-mapsize (%" PRIu64 "), skip it",
meta_number, mapsize_min);
WARNING("meta[%u] has invalid min-mapsize (%" PRIu64 "), skip it", meta_number, mapsize_min);
return MDBX_VERSION_MISMATCH;
}
}
@ -648,17 +566,13 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta,
uint64_t mapsize_max = geo_upper * (uint64_t)meta->pagesize;
STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE);
if (unlikely(mapsize_max > MAX_MAPSIZE ||
(MAX_PAGENO + 1) <
ceil_powerof2((size_t)mapsize_max, globals.sys_pagesize) /
(size_t)meta->pagesize)) {
(MAX_PAGENO + 1) < ceil_powerof2((size_t)mapsize_max, globals.sys_pagesize) / (size_t)meta->pagesize)) {
if (mapsize_max > MAX_MAPSIZE64) {
WARNING("meta[%u] has invalid max-mapsize (%" PRIu64 "), skip it",
meta_number, mapsize_max);
WARNING("meta[%u] has invalid max-mapsize (%" PRIu64 "), skip it", meta_number, mapsize_max);
return MDBX_VERSION_MISMATCH;
}
/* allow to open large DB from a 32-bit environment */
eASSERT(env, meta->geometry.first_unallocated - 1 <= MAX_PAGENO &&
used_bytes <= MAX_MAPSIZE);
eASSERT(env, meta->geometry.first_unallocated - 1 <= MAX_PAGENO && used_bytes <= MAX_MAPSIZE);
WARNING("meta[%u] has too large max-mapsize (%" PRIu64 "), "
"but size of used space still acceptable (%" PRIu64 ")",
meta_number, mapsize_max, used_bytes);
@ -667,8 +581,7 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta,
geo_upper = MAX_PAGENO + 1;
mapsize_max = geo_upper * (uint64_t)meta->pagesize;
}
WARNING("meta[%u] consider get-%s pageno is %" PRIaPGNO
" instead of wrong %" PRIaPGNO
WARNING("meta[%u] consider get-%s pageno is %" PRIaPGNO " instead of wrong %" PRIaPGNO
", will be corrected on next commit(s)",
meta_number, "upper", geo_upper, meta->geometry.upper);
meta->geometry.upper = geo_upper;
@ -688,14 +601,12 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta,
geo_now = geo_upper;
if (unlikely(meta->geometry.first_unallocated > geo_now)) {
WARNING("meta[%u] next-pageno (%" PRIaPGNO
") is beyond end-pgno (%" PRIaPGNO "), skip it",
meta_number, meta->geometry.first_unallocated, geo_now);
WARNING("meta[%u] next-pageno (%" PRIaPGNO ") is beyond end-pgno (%" PRIaPGNO "), skip it", meta_number,
meta->geometry.first_unallocated, geo_now);
return MDBX_CORRUPTED;
}
if (meta->geometry.now != geo_now) {
WARNING("meta[%u] consider geo-%s pageno is %" PRIaPGNO
" instead of wrong %" PRIaPGNO
WARNING("meta[%u] consider geo-%s pageno is %" PRIaPGNO " instead of wrong %" PRIaPGNO
", will be corrected on next commit(s)",
meta_number, "now", geo_now, meta->geometry.now);
meta->geometry.now = geo_now;
@ -703,43 +614,36 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta,
/* GC */
if (meta->trees.gc.root == P_INVALID) {
if (unlikely(meta->trees.gc.branch_pages || meta->trees.gc.height ||
meta->trees.gc.items || meta->trees.gc.leaf_pages ||
meta->trees.gc.large_pages)) {
if (unlikely(meta->trees.gc.branch_pages || meta->trees.gc.height || meta->trees.gc.items ||
meta->trees.gc.leaf_pages || meta->trees.gc.large_pages)) {
WARNING("meta[%u] has false-empty %s, skip it", meta_number, "GC");
return MDBX_CORRUPTED;
}
} else if (unlikely(meta->trees.gc.root >=
meta->geometry.first_unallocated)) {
WARNING("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it", meta_number,
"GC", meta->trees.gc.root);
} else if (unlikely(meta->trees.gc.root >= meta->geometry.first_unallocated)) {
WARNING("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it", meta_number, "GC", meta->trees.gc.root);
return MDBX_CORRUPTED;
}
/* MainDB */
if (meta->trees.main.root == P_INVALID) {
if (unlikely(meta->trees.main.branch_pages || meta->trees.main.height ||
meta->trees.main.items || meta->trees.main.leaf_pages ||
meta->trees.main.large_pages)) {
if (unlikely(meta->trees.main.branch_pages || meta->trees.main.height || meta->trees.main.items ||
meta->trees.main.leaf_pages || meta->trees.main.large_pages)) {
WARNING("meta[%u] has false-empty %s", meta_number, "MainDB");
return MDBX_CORRUPTED;
}
} else if (unlikely(meta->trees.main.root >=
meta->geometry.first_unallocated)) {
WARNING("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it", meta_number,
"MainDB", meta->trees.main.root);
} else if (unlikely(meta->trees.main.root >= meta->geometry.first_unallocated)) {
WARNING("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it", meta_number, "MainDB", meta->trees.main.root);
return MDBX_CORRUPTED;
}
if (unlikely(meta->trees.gc.mod_txnid > txnid)) {
WARNING("meta[%u] has wrong mod_txnid %" PRIaTXN " for %s, skip it",
meta_number, meta->trees.gc.mod_txnid, "GC");
WARNING("meta[%u] has wrong mod_txnid %" PRIaTXN " for %s, skip it", meta_number, meta->trees.gc.mod_txnid, "GC");
return MDBX_CORRUPTED;
}
if (unlikely(meta->trees.main.mod_txnid > txnid)) {
WARNING("meta[%u] has wrong mod_txnid %" PRIaTXN " for %s, skip it",
meta_number, meta->trees.main.mod_txnid, "MainDB");
WARNING("meta[%u] has wrong mod_txnid %" PRIaTXN " for %s, skip it", meta_number, meta->trees.main.mod_txnid,
"MainDB");
return MDBX_CORRUPTED;
}
@ -748,7 +652,5 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta,
__cold int meta_validate_copy(MDBX_env *env, const meta_t *meta, meta_t *dest) {
*dest = *meta;
return meta_validate(env, dest, data_page(meta),
bytes2pgno(env, ptr_dist(meta, env->dxb_mmap.base)),
nullptr);
return meta_validate(env, dest, data_page(meta), bytes2pgno(env, ptr_dist(meta, env->dxb_mmap.base)), nullptr);
}

106
src/meta.h
View File

@ -16,17 +16,11 @@ static inline uint64_t meta_sign_calculate(const meta_t *meta) {
return (sign > DATASIGN_WEAK) ? sign : ~sign;
}
static inline uint64_t meta_sign_get(const volatile meta_t *meta) {
return unaligned_peek_u64_volatile(4, meta->sign);
}
static inline uint64_t meta_sign_get(const volatile meta_t *meta) { return unaligned_peek_u64_volatile(4, meta->sign); }
static inline void meta_sign_as_steady(meta_t *meta) {
unaligned_poke_u64(4, meta->sign, meta_sign_calculate(meta));
}
static inline void meta_sign_as_steady(meta_t *meta) { unaligned_poke_u64(4, meta->sign, meta_sign_calculate(meta)); }
static inline bool meta_is_steady(const volatile meta_t *meta) {
return SIGN_IS_STEADY(meta_sign_get(meta));
}
static inline bool meta_is_steady(const volatile meta_t *meta) { return SIGN_IS_STEADY(meta_sign_get(meta)); }
MDBX_INTERNAL troika_t meta_tap(const MDBX_env *env);
MDBX_INTERNAL unsigned meta_eq_mask(const troika_t *troika);
@ -48,14 +42,12 @@ MDBX_INTERNAL txnid_t recent_committed_txnid(const MDBX_env *env);
MDBX_INTERNAL int meta_sync(const MDBX_env *env, const meta_ptr_t head);
MDBX_INTERNAL const char *durable_caption(const meta_t *const meta);
MDBX_INTERNAL void meta_troika_dump(const MDBX_env *env,
const troika_t *troika);
MDBX_INTERNAL void meta_troika_dump(const MDBX_env *env, const troika_t *troika);
#define METAPAGE(env, n) page_meta(pgno2page(env, n))
#define METAPAGE_END(env) METAPAGE(env, NUM_METAS)
static inline meta_ptr_t meta_recent(const MDBX_env *env,
const troika_t *troika) {
static inline meta_ptr_t meta_recent(const MDBX_env *env, const troika_t *troika) {
meta_ptr_t r;
r.txnid = troika->txnid[troika->recent];
r.ptr_v = METAPAGE(env, troika->recent);
@ -63,8 +55,7 @@ static inline meta_ptr_t meta_recent(const MDBX_env *env,
return r;
}
static inline meta_ptr_t meta_prefer_steady(const MDBX_env *env,
const troika_t *troika) {
static inline meta_ptr_t meta_prefer_steady(const MDBX_env *env, const troika_t *troika) {
meta_ptr_t r;
r.txnid = troika->txnid[troika->prefer_steady];
r.ptr_v = METAPAGE(env, troika->prefer_steady);
@ -72,8 +63,7 @@ static inline meta_ptr_t meta_prefer_steady(const MDBX_env *env,
return r;
}
static inline meta_ptr_t meta_tail(const MDBX_env *env,
const troika_t *troika) {
static inline meta_ptr_t meta_tail(const MDBX_env *env, const troika_t *troika) {
const uint8_t tail = troika->tail_and_flags & 3;
MDBX_ANALYSIS_ASSUME(tail < NUM_METAS);
meta_ptr_t r;
@ -89,72 +79,53 @@ static inline bool meta_is_used(const troika_t *troika, unsigned n) {
static inline bool meta_bootid_match(const meta_t *meta) {
return memcmp(&meta->bootid, &globals.bootid, 16) == 0 &&
(globals.bootid.x | globals.bootid.y) != 0;
return memcmp(&meta->bootid, &globals.bootid, 16) == 0 && (globals.bootid.x | globals.bootid.y) != 0;
}
static inline bool meta_weak_acceptable(const MDBX_env *env, const meta_t *meta,
const int lck_exclusive) {
static inline bool meta_weak_acceptable(const MDBX_env *env, const meta_t *meta, const int lck_exclusive) {
return lck_exclusive
? /* exclusive lock */ meta_bootid_match(meta)
: /* db already opened */ env->lck_mmap.lck &&
(env->lck_mmap.lck->envmode.weak & MDBX_RDONLY) == 0;
: /* db already opened */ env->lck_mmap.lck && (env->lck_mmap.lck->envmode.weak & MDBX_RDONLY) == 0;
}
MDBX_NOTHROW_PURE_FUNCTION static inline txnid_t
constmeta_txnid(const meta_t *meta) {
MDBX_NOTHROW_PURE_FUNCTION static inline txnid_t constmeta_txnid(const meta_t *meta) {
const txnid_t a = unaligned_peek_u64(4, &meta->txnid_a);
const txnid_t b = unaligned_peek_u64(4, &meta->txnid_b);
return likely(a == b) ? a : 0;
}
static inline void meta_update_begin(const MDBX_env *env, meta_t *meta,
txnid_t txnid) {
static inline void meta_update_begin(const MDBX_env *env, meta_t *meta, txnid_t txnid) {
eASSERT(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env));
eASSERT(env, unaligned_peek_u64(4, meta->txnid_a) < txnid &&
unaligned_peek_u64(4, meta->txnid_b) < txnid);
eASSERT(env, unaligned_peek_u64(4, meta->txnid_a) < txnid && unaligned_peek_u64(4, meta->txnid_b) < txnid);
(void)env;
#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && \
MDBX_UNALIGNED_OK >= 8
#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && MDBX_UNALIGNED_OK >= 8
atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_b, 0, mo_AcquireRelease);
atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_a, txnid,
mo_AcquireRelease);
atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_a, txnid, mo_AcquireRelease);
#else
atomic_store32(&meta->txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], 0,
mo_AcquireRelease);
atomic_store32(&meta->txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], 0,
mo_AcquireRelease);
atomic_store32(&meta->txnid_a[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__],
(uint32_t)txnid, mo_AcquireRelease);
atomic_store32(&meta->txnid_a[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__],
(uint32_t)(txnid >> 32), mo_AcquireRelease);
atomic_store32(&meta->txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], 0, mo_AcquireRelease);
atomic_store32(&meta->txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], 0, mo_AcquireRelease);
atomic_store32(&meta->txnid_a[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], (uint32_t)txnid, mo_AcquireRelease);
atomic_store32(&meta->txnid_a[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], (uint32_t)(txnid >> 32), mo_AcquireRelease);
#endif
}
static inline void meta_update_end(const MDBX_env *env, meta_t *meta,
txnid_t txnid) {
static inline void meta_update_end(const MDBX_env *env, meta_t *meta, txnid_t txnid) {
eASSERT(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env));
eASSERT(env, unaligned_peek_u64(4, meta->txnid_a) == txnid);
eASSERT(env, unaligned_peek_u64(4, meta->txnid_b) < txnid);
(void)env;
jitter4testing(true);
memcpy(&meta->bootid, &globals.bootid, 16);
#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && \
MDBX_UNALIGNED_OK >= 8
atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_b, txnid,
mo_AcquireRelease);
#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && MDBX_UNALIGNED_OK >= 8
atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_b, txnid, mo_AcquireRelease);
#else
atomic_store32(&meta->txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__],
(uint32_t)txnid, mo_AcquireRelease);
atomic_store32(&meta->txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__],
(uint32_t)(txnid >> 32), mo_AcquireRelease);
atomic_store32(&meta->txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], (uint32_t)txnid, mo_AcquireRelease);
atomic_store32(&meta->txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], (uint32_t)(txnid >> 32), mo_AcquireRelease);
#endif
}
static inline void meta_set_txnid(const MDBX_env *env, meta_t *meta,
const txnid_t txnid) {
eASSERT(env, !env->dxb_mmap.base || meta < METAPAGE(env, 0) ||
meta >= METAPAGE_END(env));
static inline void meta_set_txnid(const MDBX_env *env, meta_t *meta, const txnid_t txnid) {
eASSERT(env, !env->dxb_mmap.base || meta < METAPAGE(env, 0) || meta >= METAPAGE_END(env));
(void)env;
/* update inconsistently since this function used ONLY for filling meta-image
* for writing, but not the actual meta-page */
@ -167,42 +138,31 @@ static inline uint8_t meta_cmp2int(txnid_t a, txnid_t b, uint8_t s) {
return unlikely(a == b) ? 1 * s : (a > b) ? 2 * s : 0 * s;
}
static inline uint8_t meta_cmp2recent(uint8_t ab_cmp2int, bool a_steady,
bool b_steady) {
static inline uint8_t meta_cmp2recent(uint8_t ab_cmp2int, bool a_steady, bool b_steady) {
assert(ab_cmp2int < 3 /* && a_steady< 2 && b_steady < 2 */);
return ab_cmp2int > 1 || (ab_cmp2int == 1 && a_steady > b_steady);
}
static inline uint8_t meta_cmp2steady(uint8_t ab_cmp2int, bool a_steady,
bool b_steady) {
static inline uint8_t meta_cmp2steady(uint8_t ab_cmp2int, bool a_steady, bool b_steady) {
assert(ab_cmp2int < 3 /* && a_steady< 2 && b_steady < 2 */);
return a_steady > b_steady || (a_steady == b_steady && ab_cmp2int > 1);
}
static inline bool meta_choice_recent(txnid_t a_txnid, bool a_steady,
txnid_t b_txnid, bool b_steady) {
static inline bool meta_choice_recent(txnid_t a_txnid, bool a_steady, txnid_t b_txnid, bool b_steady) {
return meta_cmp2recent(meta_cmp2int(a_txnid, b_txnid, 1), a_steady, b_steady);
}
static inline bool meta_choice_steady(txnid_t a_txnid, bool a_steady,
txnid_t b_txnid, bool b_steady) {
static inline bool meta_choice_steady(txnid_t a_txnid, bool a_steady, txnid_t b_txnid, bool b_steady) {
return meta_cmp2steady(meta_cmp2int(a_txnid, b_txnid, 1), a_steady, b_steady);
}
MDBX_INTERNAL meta_t *meta_init_triplet(const MDBX_env *env, void *buffer);
MDBX_INTERNAL int meta_validate(MDBX_env *env, meta_t *const meta,
const page_t *const page,
const unsigned meta_number,
MDBX_INTERNAL int meta_validate(MDBX_env *env, meta_t *const meta, const page_t *const page, const unsigned meta_number,
unsigned *guess_pagesize);
MDBX_INTERNAL int __must_check_result meta_validate_copy(MDBX_env *env,
const meta_t *meta,
meta_t *dest);
MDBX_INTERNAL int __must_check_result meta_validate_copy(MDBX_env *env, const meta_t *meta, meta_t *dest);
MDBX_INTERNAL int __must_check_result meta_override(MDBX_env *env,
size_t target,
txnid_t txnid,
const meta_t *shape);
MDBX_INTERNAL int __must_check_result meta_override(MDBX_env *env, size_t target, txnid_t txnid, const meta_t *shape);
MDBX_INTERNAL int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto);

44
src/misc.c
View File

@ -14,11 +14,9 @@ __cold int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy) {
const int log2page = log2n_powerof2(pagesize);
const intptr_t volume_pages = (volume + pagesize - 1) >> log2page;
const intptr_t redundancy_pages =
(redundancy < 0) ? -(intptr_t)((-redundancy + pagesize - 1) >> log2page)
: (intptr_t)(redundancy + pagesize - 1) >> log2page;
if (volume_pages >= total_ram_pages ||
volume_pages + redundancy_pages >= total_ram_pages)
const intptr_t redundancy_pages = (redundancy < 0) ? -(intptr_t)((-redundancy + pagesize - 1) >> log2page)
: (intptr_t)(redundancy + pagesize - 1) >> log2page;
if (volume_pages >= total_ram_pages || volume_pages + redundancy_pages >= total_ram_pages)
return MDBX_RESULT_FALSE;
intptr_t avail_ram_pages;
@ -26,13 +24,10 @@ __cold int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy) {
if (unlikely(err != MDBX_SUCCESS))
return LOG_IFERR(err);
return (volume_pages + redundancy_pages >= avail_ram_pages)
? MDBX_RESULT_FALSE
: MDBX_RESULT_TRUE;
return (volume_pages + redundancy_pages >= avail_ram_pages) ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE;
}
int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result,
uint64_t increment) {
int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, uint64_t increment) {
int rc = check_txn(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS)) {
bailout:
@ -111,30 +106,23 @@ int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result,
return MDBX_SUCCESS;
}
int mdbx_cmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a,
const MDBX_val *b) {
int mdbx_cmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, const MDBX_val *b) {
eASSERT(nullptr, txn->signature == txn_signature);
tASSERT(txn, (dbi_state(txn, dbi) & DBI_VALID) && !dbi_changed(txn, dbi));
tASSERT(txn,
dbi < txn->env->n_dbi && (txn->env->dbs_flags[dbi] & DB_VALID) != 0);
tASSERT(txn, dbi < txn->env->n_dbi && (txn->env->dbs_flags[dbi] & DB_VALID) != 0);
return txn->env->kvs[dbi].clc.k.cmp(a, b);
}
int mdbx_dcmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a,
const MDBX_val *b) {
int mdbx_dcmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, const MDBX_val *b) {
eASSERT(nullptr, txn->signature == txn_signature);
tASSERT(txn, (dbi_state(txn, dbi) & DBI_VALID) && !dbi_changed(txn, dbi));
tASSERT(txn, dbi < txn->env->n_dbi && (txn->env->dbs_flags[dbi] & DB_VALID));
return txn->env->kvs[dbi].clc.v.cmp(a, b);
}
__cold MDBX_cmp_func *mdbx_get_keycmp(MDBX_db_flags_t flags) {
return builtin_keycmp(flags);
}
__cold MDBX_cmp_func *mdbx_get_keycmp(MDBX_db_flags_t flags) { return builtin_keycmp(flags); }
__cold MDBX_cmp_func *mdbx_get_datacmp(MDBX_db_flags_t flags) {
return builtin_datacmp(flags);
}
__cold MDBX_cmp_func *mdbx_get_datacmp(MDBX_db_flags_t flags) { return builtin_datacmp(flags); }
/*----------------------------------------------------------------------------*/
@ -227,10 +215,8 @@ __cold const char *mdbx_strerror_r(int errnum, char *buf, size_t buflen) {
const char *msg = mdbx_liberr2str(errnum);
if (!msg && buflen > 0 && buflen < INT_MAX) {
#if defined(_WIN32) || defined(_WIN64)
DWORD size = FormatMessageA(
FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr,
errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen,
nullptr);
DWORD size = FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, errnum,
MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, nullptr);
while (size && buf[size - 1] <= ' ')
--size;
buf[size] = 0;
@ -284,10 +270,8 @@ __cold const char *mdbx_strerror(int errnum) {
const char *mdbx_strerror_r_ANSI2OEM(int errnum, char *buf, size_t buflen) {
const char *msg = mdbx_liberr2str(errnum);
if (!msg && buflen > 0 && buflen < INT_MAX) {
DWORD size = FormatMessageA(
FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr,
errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen,
nullptr);
DWORD size = FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, errnum,
MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, nullptr);
while (size && buf[size - 1] <= ' ')
--size;
buf[size] = 0;

126
src/mvcc-readers.c
View File

@ -45,8 +45,7 @@ bsr_t mvcc_bind_slot(MDBX_env *env) {
result.err = mvcc_cleanup_dead(env, true, nullptr);
if (result.err != MDBX_RESULT_TRUE) {
lck_rdt_unlock(env);
result.err =
(result.err == MDBX_SUCCESS) ? MDBX_READERS_FULL : result.err;
result.err = (result.err == MDBX_SUCCESS) ? MDBX_READERS_FULL : result.err;
return result;
}
}
@ -61,8 +60,7 @@ bsr_t mvcc_bind_slot(MDBX_env *env) {
safe64_reset(&result.rslot->txnid, true);
if (slot == nreaders)
env->lck->rdt_length.weak = (uint32_t)++nreaders;
result.rslot->tid.weak =
(env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self();
result.rslot->tid.weak = (env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self();
atomic_store32(&result.rslot->pid, env->pid, mo_AcquireRelease);
lck_rdt_unlock(env);
@ -84,17 +82,14 @@ __hot txnid_t mvcc_shapshot_oldest(MDBX_env *const env, const txnid_t steady) {
return env->lck->cached_oldest.weak = steady;
}
const txnid_t prev_oldest =
atomic_load64(&lck->cached_oldest, mo_AcquireRelease);
const txnid_t prev_oldest = atomic_load64(&lck->cached_oldest, mo_AcquireRelease);
eASSERT(env, steady >= prev_oldest);
txnid_t new_oldest = prev_oldest;
while (nothing_changed !=
atomic_load32(&lck->rdt_refresh_flag, mo_AcquireRelease)) {
while (nothing_changed != atomic_load32(&lck->rdt_refresh_flag, mo_AcquireRelease)) {
lck->rdt_refresh_flag.weak = nothing_changed;
jitter4testing(false);
const size_t snap_nreaders =
atomic_load32(&lck->rdt_length, mo_AcquireRelease);
const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
new_oldest = steady;
for (size_t i = 0; i < snap_nreaders; ++i) {
@ -105,11 +100,9 @@ __hot txnid_t mvcc_shapshot_oldest(MDBX_env *const env, const txnid_t steady) {
const txnid_t rtxn = safe64_read(&lck->rdt[i].txnid);
if (unlikely(rtxn < prev_oldest)) {
if (unlikely(nothing_changed == atomic_load32(&lck->rdt_refresh_flag,
mo_AcquireRelease)) &&
if (unlikely(nothing_changed == atomic_load32(&lck->rdt_refresh_flag, mo_AcquireRelease)) &&
safe64_reset_compare(&lck->rdt[i].txnid, rtxn)) {
NOTICE("kick stuck reader[%zu of %zu].pid_%u %" PRIaTXN
" < prev-oldest %" PRIaTXN ", steady-txn %" PRIaTXN,
NOTICE("kick stuck reader[%zu of %zu].pid_%u %" PRIaTXN " < prev-oldest %" PRIaTXN ", steady-txn %" PRIaTXN,
i, snap_nreaders, pid, rtxn, prev_oldest, steady);
}
continue;
@ -135,17 +128,13 @@ pgno_t mvcc_snapshot_largest(const MDBX_env *env, pgno_t last_used_page) {
lck_t *const lck = env->lck_mmap.lck;
if (likely(lck != nullptr /* check for exclusive without-lck mode */)) {
retry:;
const size_t snap_nreaders =
atomic_load32(&lck->rdt_length, mo_AcquireRelease);
const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
for (size_t i = 0; i < snap_nreaders; ++i) {
if (atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease)) {
/* jitter4testing(true); */
const pgno_t snap_pages =
atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_Relaxed);
const pgno_t snap_pages = atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_Relaxed);
const txnid_t snap_txnid = safe64_read(&lck->rdt[i].txnid);
if (unlikely(snap_pages !=
atomic_load32(&lck->rdt[i].snapshot_pages_used,
mo_AcquireRelease) ||
if (unlikely(snap_pages != atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_AcquireRelease) ||
snap_txnid != safe64_read(&lck->rdt[i].txnid)))
goto retry;
if (last_used_page < snap_pages && snap_txnid <= env->basal_txn->txnid)
@ -161,18 +150,14 @@ pgno_t mvcc_snapshot_largest(const MDBX_env *env, pgno_t last_used_page) {
pgno_t mvcc_largest_this(MDBX_env *env, pgno_t largest) {
lck_t *const lck = env->lck_mmap.lck;
if (likely(lck != nullptr /* exclusive mode */)) {
const size_t snap_nreaders =
atomic_load32(&lck->rdt_length, mo_AcquireRelease);
const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
for (size_t i = 0; i < snap_nreaders; ++i) {
retry:
if (atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease) == env->pid) {
/* jitter4testing(true); */
const pgno_t snap_pages =
atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_Relaxed);
const pgno_t snap_pages = atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_Relaxed);
const txnid_t snap_txnid = safe64_read(&lck->rdt[i].txnid);
if (unlikely(snap_pages !=
atomic_load32(&lck->rdt[i].snapshot_pages_used,
mo_AcquireRelease) ||
if (unlikely(snap_pages != atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_AcquireRelease) ||
snap_txnid != safe64_read(&lck->rdt[i].txnid)))
goto retry;
if (largest < snap_pages &&
@ -219,8 +204,7 @@ static bool pid_insert(uint32_t *list, uint32_t pid) {
return true;
}
__cold MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rdt_locked,
int *dead) {
__cold MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rdt_locked, int *dead) {
int rc = check_env(env, true);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
@ -234,13 +218,11 @@ __cold MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rdt_locked,
return MDBX_SUCCESS;
}
const size_t snap_nreaders =
atomic_load32(&lck->rdt_length, mo_AcquireRelease);
const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
uint32_t pidsbuf_onstask[142];
uint32_t *const pids =
(snap_nreaders < ARRAY_LENGTH(pidsbuf_onstask))
? pidsbuf_onstask
: osal_malloc((snap_nreaders + 1) * sizeof(uint32_t));
uint32_t *const pids = (snap_nreaders < ARRAY_LENGTH(pidsbuf_onstask))
? pidsbuf_onstask
: osal_malloc((snap_nreaders + 1) * sizeof(uint32_t));
if (unlikely(!pids))
return MDBX_ENOMEM;
@ -296,8 +278,7 @@ __cold MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rdt_locked,
/* clean it */
for (size_t ii = i; ii < snap_nreaders; ii++) {
if (lck->rdt[ii].pid.weak == pid) {
DEBUG("clear stale reader pid %" PRIuPTR " txn %" PRIaTXN, (size_t)pid,
lck->rdt[ii].txnid.weak);
DEBUG("clear stale reader pid %" PRIuPTR " txn %" PRIaTXN, (size_t)pid, lck->rdt[ii].txnid.weak);
atomic_store32(&lck->rdt[ii].pid, 0, mo_Relaxed);
atomic_store32(&lck->rdt_refresh_flag, true, mo_AcquireRelease);
count++;
@ -321,11 +302,9 @@ __cold MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rdt_locked,
int txn_park(MDBX_txn *txn, bool autounpark) {
reader_slot_t *const rslot = txn->to.reader;
tASSERT(txn, (txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY |
MDBX_TXN_PARKED)) == MDBX_TXN_RDONLY);
tASSERT(txn, (txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) == MDBX_TXN_RDONLY);
tASSERT(txn, txn->to.reader->tid.weak < MDBX_TID_TXN_OUSTED);
if (unlikely((txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY |
MDBX_TXN_PARKED)) != MDBX_TXN_RDONLY))
if (unlikely((txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) != MDBX_TXN_RDONLY))
return MDBX_BAD_TXN;
const uint32_t pid = atomic_load32(&rslot->pid, mo_Relaxed);
@ -344,14 +323,12 @@ int txn_park(MDBX_txn *txn, bool autounpark) {
atomic_store64(&rslot->tid, MDBX_TID_TXN_PARKED, mo_AcquireRelease);
atomic_store32(&txn->env->lck->rdt_refresh_flag, true, mo_Relaxed);
txn->flags +=
autounpark ? MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK : MDBX_TXN_PARKED;
txn->flags += autounpark ? MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK : MDBX_TXN_PARKED;
return MDBX_SUCCESS;
}
int txn_unpark(MDBX_txn *txn) {
if (unlikely((txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD |
MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) !=
if (unlikely((txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD | MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) !=
(MDBX_TXN_RDONLY | MDBX_TXN_PARKED)))
return MDBX_BAD_TXN;
@ -363,14 +340,11 @@ int txn_unpark(MDBX_txn *txn) {
ERROR("unexpected pid %u%s%u", pid, " != expected ", txn->env->pid);
return MDBX_PROBLEM;
}
if (unlikely(tid == MDBX_TID_TXN_OUSTED ||
txnid >= SAFE64_INVALID_THRESHOLD))
if (unlikely(tid == MDBX_TID_TXN_OUSTED || txnid >= SAFE64_INVALID_THRESHOLD))
break;
if (unlikely(tid != MDBX_TID_TXN_PARKED || txnid != txn->txnid)) {
ERROR("unexpected thread-id 0x%" PRIx64 "%s0x%" PRIx64
" and/or txn-id %" PRIaTXN "%s%" PRIaTXN,
tid, " != must ", MDBX_TID_TXN_OUSTED, txnid, " != must ",
txn->txnid);
ERROR("unexpected thread-id 0x%" PRIx64 "%s0x%" PRIx64 " and/or txn-id %" PRIaTXN "%s%" PRIaTXN, tid, " != must ",
MDBX_TID_TXN_OUSTED, txnid, " != must ", txn->txnid);
break;
}
if (unlikely((txn->flags & MDBX_TXN_ERROR)))
@ -380,12 +354,9 @@ int txn_unpark(MDBX_txn *txn) {
if (unlikely(!atomic_cas64(&rslot->tid, MDBX_TID_TXN_PARKED, txn->owner)))
continue;
#else
atomic_store32(&rslot->tid.high, (uint32_t)((uint64_t)txn->owner >> 32),
mo_Relaxed);
if (unlikely(!atomic_cas32(&rslot->tid.low, (uint32_t)MDBX_TID_TXN_PARKED,
(uint32_t)txn->owner))) {
atomic_store32(&rslot->tid.high, (uint32_t)(MDBX_TID_TXN_PARKED >> 32),
mo_AcquireRelease);
atomic_store32(&rslot->tid.high, (uint32_t)((uint64_t)txn->owner >> 32), mo_Relaxed);
if (unlikely(!atomic_cas32(&rslot->tid.low, (uint32_t)MDBX_TID_TXN_PARKED, (uint32_t)txn->owner))) {
atomic_store32(&rslot->tid.high, (uint32_t)(MDBX_TID_TXN_PARKED >> 32), mo_AcquireRelease);
continue;
}
#endif
@ -413,8 +384,7 @@ __cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) {
bool notify_eof_of_loop = false;
int retry = 0;
do {
const txnid_t steady =
env->txn->tw.troika.txnid[env->txn->tw.troika.prefer_steady];
const txnid_t steady = env->txn->tw.troika.txnid[env->txn->tw.troika.prefer_steady];
env->lck->rdt_refresh_flag.weak = /* force refresh */ true;
oldest = mvcc_shapshot_oldest(env, steady);
eASSERT(env, oldest < env->basal_txn->txnid);
@ -435,8 +405,7 @@ __cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) {
reader_slot_t *const rslot = &lck->rdt[i];
txnid_t rtxn = safe64_read(&rslot->txnid);
retry:
if (rtxn == straggler &&
(pid = atomic_load32(&rslot->pid, mo_AcquireRelease)) != 0) {
if (rtxn == straggler && (pid = atomic_load32(&rslot->pid, mo_AcquireRelease)) != 0) {
const uint64_t tid = safe64_read(&rslot->tid);
if (tid == MDBX_TID_TXN_PARKED) {
/* Читающая транзакция была помечена владельцем как "припаркованная",
@ -454,25 +423,21 @@ __cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) {
*/
bool ousted =
#if MDBX_64BIT_CAS
atomic_cas64(&rslot->tid, MDBX_TID_TXN_PARKED,
MDBX_TID_TXN_OUSTED);
atomic_cas64(&rslot->tid, MDBX_TID_TXN_PARKED, MDBX_TID_TXN_OUSTED);
#else
atomic_cas32(&rslot->tid.low, (uint32_t)MDBX_TID_TXN_PARKED,
(uint32_t)MDBX_TID_TXN_OUSTED);
atomic_cas32(&rslot->tid.low, (uint32_t)MDBX_TID_TXN_PARKED, (uint32_t)MDBX_TID_TXN_OUSTED);
#endif
if (likely(ousted)) {
ousted = safe64_reset_compare(&rslot->txnid, rtxn);
NOTICE("ousted-%s parked read-txn %" PRIaTXN
", pid %u, tid 0x%" PRIx64,
ousted ? "complete" : "half", rtxn, pid, tid);
NOTICE("ousted-%s parked read-txn %" PRIaTXN ", pid %u, tid 0x%" PRIx64, ousted ? "complete" : "half", rtxn,
pid, tid);
eASSERT(env, ousted || safe64_read(&rslot->txnid) > straggler);
continue;
}
rtxn = safe64_read(&rslot->txnid);
goto retry;
}
hold_retired =
atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_Relaxed);
hold_retired = atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_Relaxed);
stucked = rslot;
}
}
@ -487,15 +452,10 @@ __cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) {
const meta_ptr_t head = meta_recent(env, &env->txn->tw.troika);
const txnid_t gap = (head.txnid - straggler) / xMDBX_TXNID_STEP;
const uint64_t head_retired =
unaligned_peek_u64(4, head.ptr_c->pages_retired);
const size_t space =
(head_retired > hold_retired)
? pgno2bytes(env, (pgno_t)(head_retired - hold_retired))
: 0;
int rc =
callback(env, env->txn, pid, (mdbx_tid_t)((intptr_t)tid), straggler,
(gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, space, retry);
const uint64_t head_retired = unaligned_peek_u64(4, head.ptr_c->pages_retired);
const size_t space = (head_retired > hold_retired) ? pgno2bytes(env, (pgno_t)(head_retired - hold_retired)) : 0;
int rc = callback(env, env->txn, pid, (mdbx_tid_t)((intptr_t)tid), straggler,
(gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, space, retry);
if (rc < 0)
/* hsr returned error and/or agree MDBX_MAP_FULL error */
break;
@ -523,10 +483,8 @@ __cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) {
/* notify end of hsr-loop */
const txnid_t turn = oldest - straggler;
if (turn)
NOTICE("hsr-kick: done turn %" PRIaTXN " -> %" PRIaTXN " +%" PRIaTXN,
straggler, oldest, turn);
callback(env, env->txn, 0, 0, straggler,
(turn < UINT_MAX) ? (unsigned)turn : UINT_MAX, 0, -retry);
NOTICE("hsr-kick: done turn %" PRIaTXN " -> %" PRIaTXN " +%" PRIaTXN, straggler, oldest, turn);
callback(env, env->txn, 0, 0, straggler, (turn < UINT_MAX) ? (unsigned)turn : UINT_MAX, 0, -retry);
}
return oldest;
}

89
src/node.c
View File

@ -5,15 +5,13 @@
#include "internals.h"
__hot int __must_check_result node_add_dupfix(MDBX_cursor *mc, size_t indx,
const MDBX_val *key) {
__hot int __must_check_result node_add_dupfix(MDBX_cursor *mc, size_t indx, const MDBX_val *key) {
page_t *mp = mc->pg[mc->top];
MDBX_ANALYSIS_ASSUME(key != nullptr);
DKBUF_DEBUG;
DEBUG("add to leaf2-%spage %" PRIaPGNO " index %zi, "
" key size %" PRIuPTR " [%s]",
is_subpage(mp) ? "sub-" : "", mp->pgno, indx, key ? key->iov_len : 0,
DKEY_DEBUG(key));
is_subpage(mp) ? "sub-" : "", mp->pgno, indx, key ? key->iov_len : 0, DKEY_DEBUG(key));
cASSERT(mc, key);
cASSERT(mc, page_type_compat(mp) == (P_LEAF | P_DUPFIX));
@ -45,14 +43,11 @@ __hot int __must_check_result node_add_dupfix(MDBX_cursor *mc, size_t indx,
return MDBX_SUCCESS;
}
int __must_check_result node_add_branch(MDBX_cursor *mc, size_t indx,
const MDBX_val *key, pgno_t pgno) {
int __must_check_result node_add_branch(MDBX_cursor *mc, size_t indx, const MDBX_val *key, pgno_t pgno) {
page_t *mp = mc->pg[mc->top];
DKBUF_DEBUG;
DEBUG("add to branch-%spage %" PRIaPGNO " index %zi, node-pgno %" PRIaPGNO
" key size %" PRIuPTR " [%s]",
is_subpage(mp) ? "sub-" : "", mp->pgno, indx, pgno,
key ? key->iov_len : 0, DKEY_DEBUG(key));
DEBUG("add to branch-%spage %" PRIaPGNO " index %zi, node-pgno %" PRIaPGNO " key size %" PRIuPTR " [%s]",
is_subpage(mp) ? "sub-" : "", mp->pgno, indx, pgno, key ? key->iov_len : 0, DKEY_DEBUG(key));
cASSERT(mc, page_type(mp) == P_BRANCH);
STATIC_ASSERT(NODESIZE % 2 == 0);
@ -87,17 +82,15 @@ int __must_check_result node_add_branch(MDBX_cursor *mc, size_t indx,
return MDBX_SUCCESS;
}
__hot int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx,
const MDBX_val *key, MDBX_val *data,
__hot int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx, const MDBX_val *key, MDBX_val *data,
unsigned flags) {
MDBX_ANALYSIS_ASSUME(key != nullptr);
MDBX_ANALYSIS_ASSUME(data != nullptr);
page_t *mp = mc->pg[mc->top];
DKBUF_DEBUG;
DEBUG("add to leaf-%spage %" PRIaPGNO " index %zi, data size %" PRIuPTR
" key size %" PRIuPTR " [%s]",
is_subpage(mp) ? "sub-" : "", mp->pgno, indx, data ? data->iov_len : 0,
key ? key->iov_len : 0, DKEY_DEBUG(key));
DEBUG("add to leaf-%spage %" PRIaPGNO " index %zi, data size %" PRIuPTR " key size %" PRIuPTR " [%s]",
is_subpage(mp) ? "sub-" : "", mp->pgno, indx, data ? data->iov_len : 0, key ? key->iov_len : 0,
DKEY_DEBUG(key));
cASSERT(mc, key != nullptr && data != nullptr);
cASSERT(mc, page_type_compat(mp) == P_LEAF);
page_t *largepage = nullptr;
@ -106,19 +99,16 @@ __hot int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx,
if (unlikely(flags & N_BIG)) {
/* Data already on large/overflow page. */
STATIC_ASSERT(sizeof(pgno_t) % 2 == 0);
node_bytes =
node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t);
node_bytes = node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t);
cASSERT(mc, page_room(mp) >= node_bytes);
} else if (unlikely(node_size(key, data) > mc->txn->env->leaf_nodemax)) {
/* Put data on large/overflow page. */
if (unlikely(mc->tree->flags & MDBX_DUPSORT)) {
ERROR("Unexpected target %s flags 0x%x for large data-item", "dupsort-db",
mc->tree->flags);
ERROR("Unexpected target %s flags 0x%x for large data-item", "dupsort-db", mc->tree->flags);
return MDBX_PROBLEM;
}
if (unlikely(flags & (N_DUP | N_TREE))) {
ERROR("Unexpected target %s flags 0x%x for large data-item", "node",
flags);
ERROR("Unexpected target %s flags 0x%x for large data-item", "node", flags);
return MDBX_PROBLEM;
}
cASSERT(mc, page_room(mp) >= leaf_size(mc->txn->env, key, data));
@ -127,12 +117,10 @@ __hot int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx,
if (unlikely(npr.err != MDBX_SUCCESS))
return npr.err;
largepage = npr.page;
DEBUG("allocated %u large/overflow page(s) %" PRIaPGNO "for %" PRIuPTR
" data bytes",
largepage->pages, largepage->pgno, data->iov_len);
DEBUG("allocated %u large/overflow page(s) %" PRIaPGNO "for %" PRIuPTR " data bytes", largepage->pages,
largepage->pgno, data->iov_len);
flags |= N_BIG;
node_bytes =
node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t);
node_bytes = node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t);
cASSERT(mc, node_bytes == leaf_size(mc->txn->env, key, data));
} else {
cASSERT(mc, page_room(mp) >= leaf_size(mc->txn->env, key, data));
@ -186,8 +174,7 @@ __hot void node_del(MDBX_cursor *mc, size_t ksize) {
const size_t hole = mc->ki[mc->top];
const size_t nkeys = page_numkeys(mp);
DEBUG("delete node %zu on %s page %" PRIaPGNO, hole,
is_leaf(mp) ? "leaf" : "branch", mp->pgno);
DEBUG("delete node %zu on %s page %" PRIaPGNO, hole, is_leaf(mp) ? "leaf" : "branch", mp->pgno);
cASSERT(mc, hole < nkeys);
if (is_dupfix_leaf(mp)) {
@ -215,9 +202,7 @@ __hot void node_del(MDBX_cursor *mc, size_t ksize) {
size_t r, w;
for (r = w = 0; r < nkeys; r++)
if (r != hole)
mp->entries[w++] = (mp->entries[r] < hole_offset)
? mp->entries[r] + (indx_t)hole_size
: mp->entries[r];
mp->entries[w++] = (mp->entries[r] < hole_offset) ? mp->entries[r] + (indx_t)hole_size : mp->entries[r];
void *const base = ptr_disp(mp, mp->upper + PAGEHDRSZ);
memmove(ptr_disp(base, hole_size), base, hole_offset - mp->upper);
@ -236,14 +221,12 @@ __hot void node_del(MDBX_cursor *mc, size_t ksize) {
}
}
__noinline int node_read_bigdata(MDBX_cursor *mc, const node_t *node,
MDBX_val *data, const page_t *mp) {
__noinline int node_read_bigdata(MDBX_cursor *mc, const node_t *node, MDBX_val *data, const page_t *mp) {
cASSERT(mc, node_flags(node) == N_BIG && data->iov_len == node_ds(node));
pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->txnid);
if (unlikely((lp.err != MDBX_SUCCESS))) {
DEBUG("read large/overflow page %" PRIaPGNO " failed",
node_largedata_pgno(node));
DEBUG("read large/overflow page %" PRIaPGNO " failed", node_largedata_pgno(node));
return lp.err;
}
@ -254,9 +237,7 @@ __noinline int node_read_bigdata(MDBX_cursor *mc, const node_t *node,
const size_t dsize = data->iov_len;
const unsigned npages = largechunk_npages(env, dsize);
if (unlikely(lp.page->pages < npages))
return bad_page(lp.page,
"too less n-pages %u for bigdata-node (%zu bytes)",
lp.page->pages, dsize);
return bad_page(lp.page, "too less n-pages %u for bigdata-node (%zu bytes)", lp.page->pages, dsize);
}
return MDBX_SUCCESS;
}
@ -265,8 +246,7 @@ node_t *node_shrink(page_t *mp, size_t indx, node_t *node) {
assert(node == page_node(mp, indx));
page_t *sp = (page_t *)node_data(node);
assert(is_subpage(sp) && page_numkeys(sp) > 0);
const size_t delta =
EVEN_FLOOR(page_room(sp) /* avoid the node uneven-sized */);
const size_t delta = EVEN_FLOOR(page_room(sp) /* avoid the node uneven-sized */);
if (unlikely(delta) == 0)
return node;
@ -303,15 +283,13 @@ node_t *node_shrink(page_t *mp, size_t indx, node_t *node) {
return ptr_disp(node, delta);
}
__hot struct node_search_result node_search(MDBX_cursor *mc,
const MDBX_val *key) {
__hot struct node_search_result node_search(MDBX_cursor *mc, const MDBX_val *key) {
page_t *mp = mc->pg[mc->top];
const intptr_t nkeys = page_numkeys(mp);
DKBUF_DEBUG;
DEBUG("searching %zu keys in %s %spage %" PRIaPGNO, nkeys,
is_leaf(mp) ? "leaf" : "branch", is_subpage(mp) ? "sub-" : "",
mp->pgno);
DEBUG("searching %zu keys in %s %spage %" PRIaPGNO, nkeys, is_leaf(mp) ? "leaf" : "branch",
is_subpage(mp) ? "sub-" : "", mp->pgno);
struct node_search_result ret;
ret.exact = false;
@ -333,8 +311,7 @@ __hot struct node_search_result node_search(MDBX_cursor *mc,
do {
i = (low + high) >> 1;
nodekey.iov_base = page_dupfix_ptr(mp, i, nodekey.iov_len);
cASSERT(mc, ptr_disp(mp, mc->txn->env->ps) >=
ptr_disp(nodekey.iov_base, nodekey.iov_len));
cASSERT(mc, ptr_disp(mp, mc->txn->env->ps) >= ptr_disp(nodekey.iov_base, nodekey.iov_len));
int cr = cmp(key, &nodekey);
DEBUG("found leaf index %zu [%s], rc = %i", i, DKEY_DEBUG(&nodekey), cr);
if (cr > 0)
@ -349,10 +326,8 @@ __hot struct node_search_result node_search(MDBX_cursor *mc,
/* store the key index */
mc->ki[mc->top] = (indx_t)i;
ret.node =
(i < nkeys)
? /* fake for DUPFIX */ (node_t *)(intptr_t)-1
: /* There is no entry larger or equal to the key. */ nullptr;
ret.node = (i < nkeys) ? /* fake for DUPFIX */ (node_t *)(intptr_t)-1
: /* There is no entry larger or equal to the key. */ nullptr;
return ret;
}
@ -367,14 +342,12 @@ __hot struct node_search_result node_search(MDBX_cursor *mc,
node = page_node(mp, i);
nodekey.iov_len = node_ks(node);
nodekey.iov_base = node_key(node);
cASSERT(mc, ptr_disp(mp, mc->txn->env->ps) >=
ptr_disp(nodekey.iov_base, nodekey.iov_len));
cASSERT(mc, ptr_disp(mp, mc->txn->env->ps) >= ptr_disp(nodekey.iov_base, nodekey.iov_len));
int cr = cmp(key, &nodekey);
if (is_leaf(mp))
DEBUG("found leaf index %zu [%s], rc = %i", i, DKEY_DEBUG(&nodekey), cr);
else
DEBUG("found branch index %zu [%s -> %" PRIaPGNO "], rc = %i", i,
DKEY_DEBUG(&nodekey), node_pgno(node), cr);
DEBUG("found branch index %zu [%s -> %" PRIaPGNO "], rc = %i", i, DKEY_DEBUG(&nodekey), node_pgno(node), cr);
if (cr > 0)
low = ++i;
else if (cr < 0)
@ -387,8 +360,6 @@ __hot struct node_search_result node_search(MDBX_cursor *mc,
/* store the key index */
mc->ki[mc->top] = (indx_t)i;
ret.node = (i < nkeys)
? page_node(mp, i)
: /* There is no entry larger or equal to the key. */ nullptr;
ret.node = (i < nkeys) ? page_node(mp, i) : /* There is no entry larger or equal to the key. */ nullptr;
return ret;
}

53
src/node.h
View File

@ -9,8 +9,7 @@
#define NODE_ADD_FLAGS (N_DUP | N_TREE | MDBX_RESERVE | MDBX_APPEND)
/* Get the page number pointed to by a branch node */
MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t
node_pgno(const node_t *const __restrict node) {
MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t node_pgno(const node_t *const __restrict node) {
pgno_t pgno = UNALIGNED_PEEK_32(node, node_t, child_pgno);
return pgno;
}
@ -23,8 +22,7 @@ static inline void node_set_pgno(node_t *const __restrict node, pgno_t pgno) {
}
/* Get the size of the data in a leaf node */
MDBX_NOTHROW_PURE_FUNCTION static inline size_t
node_ds(const node_t *const __restrict node) {
MDBX_NOTHROW_PURE_FUNCTION static inline size_t node_ds(const node_t *const __restrict node) {
return UNALIGNED_PEEK_32(node, node_t, dsize);
}
@ -35,8 +33,7 @@ static inline void node_set_ds(node_t *const __restrict node, size_t size) {
}
/* The size of a key in a node */
MDBX_NOTHROW_PURE_FUNCTION static inline size_t
node_ks(const node_t *const __restrict node) {
MDBX_NOTHROW_PURE_FUNCTION static inline size_t node_ks(const node_t *const __restrict node) {
return UNALIGNED_PEEK_16(node, node_t, ksize);
}
@ -46,54 +43,42 @@ static inline void node_set_ks(node_t *const __restrict node, size_t size) {
UNALIGNED_POKE_16(node, node_t, ksize, (uint16_t)size);
}
MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t
node_flags(const node_t *const __restrict node) {
MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t node_flags(const node_t *const __restrict node) {
return UNALIGNED_PEEK_8(node, node_t, flags);
}
static inline void node_set_flags(node_t *const __restrict node,
uint8_t flags) {
static inline void node_set_flags(node_t *const __restrict node, uint8_t flags) {
UNALIGNED_POKE_8(node, node_t, flags, flags);
}
/* Address of the key for the node */
MDBX_NOTHROW_PURE_FUNCTION static inline void *
node_key(const node_t *const __restrict node) {
MDBX_NOTHROW_PURE_FUNCTION static inline void *node_key(const node_t *const __restrict node) {
return ptr_disp(node, NODESIZE);
}
/* Address of the data for a node */
MDBX_NOTHROW_PURE_FUNCTION static inline void *
node_data(const node_t *const __restrict node) {
MDBX_NOTHROW_PURE_FUNCTION static inline void *node_data(const node_t *const __restrict node) {
return ptr_disp(node_key(node), node_ks(node));
}
/* Size of a node in a leaf page with a given key and data.
* This is node header plus key plus data size. */
MDBX_NOTHROW_CONST_FUNCTION static inline size_t
node_size_len(const size_t key_len, const size_t value_len) {
MDBX_NOTHROW_CONST_FUNCTION static inline size_t node_size_len(const size_t key_len, const size_t value_len) {
return NODESIZE + EVEN_CEIL(key_len + value_len);
}
MDBX_NOTHROW_PURE_FUNCTION static inline size_t
node_size(const MDBX_val *key, const MDBX_val *value) {
MDBX_NOTHROW_PURE_FUNCTION static inline size_t node_size(const MDBX_val *key, const MDBX_val *value) {
return node_size_len(key ? key->iov_len : 0, value ? value->iov_len : 0);
}
MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t
node_largedata_pgno(const node_t *const __restrict node) {
MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t node_largedata_pgno(const node_t *const __restrict node) {
assert(node_flags(node) & N_BIG);
return peek_pgno(node_data(node));
}
MDBX_INTERNAL int __must_check_result node_read_bigdata(MDBX_cursor *mc,
const node_t *node,
MDBX_val *data,
MDBX_INTERNAL int __must_check_result node_read_bigdata(MDBX_cursor *mc, const node_t *node, MDBX_val *data,
const page_t *mp);
static inline int __must_check_result node_read(MDBX_cursor *mc,
const node_t *node,
MDBX_val *data,
const page_t *mp) {
static inline int __must_check_result node_read(MDBX_cursor *mc, const node_t *node, MDBX_val *data, const page_t *mp) {
data->iov_len = node_ds(node);
data->iov_base = node_data(node);
if (likely(node_flags(node) != N_BIG))
@ -105,20 +90,12 @@ static inline int __must_check_result node_read(MDBX_cursor *mc,
MDBX_INTERNAL nsr_t node_search(MDBX_cursor *mc, const MDBX_val *key);
MDBX_INTERNAL int __must_check_result node_add_branch(MDBX_cursor *mc,
size_t indx,
const MDBX_val *key,
pgno_t pgno);
MDBX_INTERNAL int __must_check_result node_add_branch(MDBX_cursor *mc, size_t indx, const MDBX_val *key, pgno_t pgno);
MDBX_INTERNAL int __must_check_result node_add_leaf(MDBX_cursor *mc,
size_t indx,
const MDBX_val *key,
MDBX_val *data,
MDBX_INTERNAL int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx, const MDBX_val *key, MDBX_val *data,
unsigned flags);
MDBX_INTERNAL int __must_check_result node_add_dupfix(MDBX_cursor *mc,
size_t indx,
const MDBX_val *key);
MDBX_INTERNAL int __must_check_result node_add_dupfix(MDBX_cursor *mc, size_t indx, const MDBX_val *key);
MDBX_INTERNAL void node_del(MDBX_cursor *mc, size_t ksize);

72
src/options.h
View File

@ -66,8 +66,7 @@
/** Does a system have battery-backed Real-Time Clock or just a fake. */
#ifndef MDBX_TRUST_RTC
#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \
defined(__OpenBSD__)
#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || defined(__OpenBSD__)
#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */
#else
#define MDBX_TRUST_RTC 1
@ -131,15 +130,13 @@
#ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT
#define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1
#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \
MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1)
#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1)
#error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */
#ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT
#define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1
#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \
MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1)
#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1)
#error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1
#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */
@ -204,8 +201,7 @@
/** Size of buffer used during copying a environment/database file. */
#ifndef MDBX_ENVCOPY_WRITEBUF
#define MDBX_ENVCOPY_WRITEBUF 1048576u
#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \
MDBX_ENVCOPY_WRITEBUF % 65536u
#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || MDBX_ENVCOPY_WRITEBUF % 65536u
#error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536
#endif /* MDBX_ENVCOPY_WRITEBUF */
@ -224,8 +220,7 @@
#else
#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u)
#endif
#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \
MDBX_ASSUME_MALLOC_OVERHEAD % 4
#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || MDBX_ASSUME_MALLOC_OVERHEAD % 4
#error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4
#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */
@ -252,15 +247,13 @@
#define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0
#elif defined(__e2k__)
#define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0
#elif __has_builtin(__builtin_cpu_supports) || \
defined(__BUILTIN_CPU_SUPPORTS__) || \
#elif __has_builtin(__builtin_cpu_supports) || defined(__BUILTIN_CPU_SUPPORTS__) || \
(defined(__ia32__) && __GNUC_PREREQ(4, 8) && __GLIBC_PREREQ(2, 23))
#define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 1
#else
#define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0
#endif
#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || \
MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1)
#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1)
#error MDBX_HAVE_BUILTIN_CPU_SUPPORTS must be defined as 0 or 1
#endif /* MDBX_HAVE_BUILTIN_CPU_SUPPORTS */
@ -286,19 +279,15 @@
#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES
#else
#ifndef MDBX_LOCKING
#if defined(_POSIX_THREAD_PROCESS_SHARED) && \
_POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__)
#if defined(_POSIX_THREAD_PROCESS_SHARED) && _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__)
/* Some platforms define the EOWNERDEAD error code even though they
* don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */
#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \
((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \
_POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \
(defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \
_POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \
defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \
(!defined(__GLIBC__) || \
__GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */)
#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \
((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \
(defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \
defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \
(!defined(__GLIBC__) || __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */)
#define MDBX_LOCKING MDBX_LOCKING_POSIX2008
#else
#define MDBX_LOCKING MDBX_LOCKING_POSIX2001
@ -316,12 +305,9 @@
/** Advanced: Using POSIX OFD-locks (autodetection by default). */
#ifndef MDBX_USE_OFDLOCKS
#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && \
defined(F_OFD_GETLK)) || \
(defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \
defined(F_OFD_GETLK64))) && \
!defined(MDBX_SAFE4QEMU) && \
!defined(__sun) /* OFD-lock are broken on Solaris */
#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK)) || \
(defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64))) && \
!defined(MDBX_SAFE4QEMU) && !defined(__sun) /* OFD-lock are broken on Solaris */
#define MDBX_USE_OFDLOCKS 1
#else
#define MDBX_USE_OFDLOCKS 0
@ -335,8 +321,7 @@
/** Advanced: Using sendfile() syscall (autodetection by default). */
#ifndef MDBX_USE_SENDFILE
#if ((defined(__linux__) || defined(__gnu_linux__)) && \
!defined(__ANDROID_API__)) || \
#if ((defined(__linux__) || defined(__gnu_linux__)) && !defined(__ANDROID_API__)) || \
(defined(__ANDROID_API__) && __ANDROID_API__ >= 21)
#define MDBX_USE_SENDFILE 1
#else
@ -360,14 +345,12 @@
//------------------------------------------------------------------------------
#ifndef MDBX_CPU_WRITEBACK_INCOHERENT
#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
defined(__hppa__) || defined(DOXYGEN)
#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || defined(__hppa__) || defined(DOXYGEN)
#define MDBX_CPU_WRITEBACK_INCOHERENT 0
#else
#define MDBX_CPU_WRITEBACK_INCOHERENT 1
#endif
#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || \
MDBX_CPU_WRITEBACK_INCOHERENT == 1)
#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || MDBX_CPU_WRITEBACK_INCOHERENT == 1)
#error MDBX_CPU_WRITEBACK_INCOHERENT must be defined as 0 or 1
#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */
@ -377,31 +360,27 @@
#else
#define MDBX_MMAP_INCOHERENT_FILE_WRITE 0
#endif
#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || \
MDBX_MMAP_INCOHERENT_FILE_WRITE == 1)
#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || MDBX_MMAP_INCOHERENT_FILE_WRITE == 1)
#error MDBX_MMAP_INCOHERENT_FILE_WRITE must be defined as 0 or 1
#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */
#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE
#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
defined(__MWERKS__) || defined(__sgi)
#if defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \
defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi)
/* MIPS has cache coherency issues. */
#define MDBX_MMAP_INCOHERENT_CPU_CACHE 1
#else
/* LY: assume no relevant mmap/dcache issues. */
#define MDBX_MMAP_INCOHERENT_CPU_CACHE 0
#endif
#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || \
MDBX_MMAP_INCOHERENT_CPU_CACHE == 1)
#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || MDBX_MMAP_INCOHERENT_CPU_CACHE == 1)
#error MDBX_MMAP_INCOHERENT_CPU_CACHE must be defined as 0 or 1
#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
/** Assume system needs explicit syscall to sync/flush/write modified mapped
* memory. */
#ifndef MDBX_MMAP_NEEDS_JOLT
#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE || \
!(defined(__linux__) || defined(__gnu_linux__))
#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE || !(defined(__linux__) || defined(__gnu_linux__))
#define MDBX_MMAP_NEEDS_JOLT 1
#else
#define MDBX_MMAP_NEEDS_JOLT 0
@ -456,8 +435,7 @@
#endif /* MDBX_64BIT_CAS */
#ifndef MDBX_UNALIGNED_OK
#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || \
defined(ENABLE_UBSAN)
#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || defined(ENABLE_UBSAN)
#define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */
#elif defined(__ARM_FEATURE_UNALIGNED)
#define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */

842
src/osal.c
View File

File diff suppressed because it is too large Load Diff

148
src/osal.h
View File

@ -12,9 +12,8 @@
#if __has_include(<sys/cachectl.h>)
#include <sys/cachectl.h>
#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
defined(__MWERKS__) || defined(__sgi)
#elif defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \
defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi)
/* MIPS should have explicit cache control */
#include <sys/cachectl.h>
#endif
@ -28,11 +27,9 @@ MDBX_MAYBE_UNUSED static inline void osal_compiler_barrier(void) {
__memory_barrier();
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
__compiler_barrier();
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64))
_Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
defined(__ppc64__) || defined(__powerpc64__)
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__)
__fence();
#else
#error "Could not guess the kind of compiler, please report to us."
@ -60,11 +57,9 @@ MDBX_MAYBE_UNUSED static inline void osal_memory_barrier(void) {
#endif
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
__machine_rw_barrier();
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64))
_Asm_mf();
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
defined(__ppc64__) || defined(__powerpc64__)
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__)
__lwsync();
#else
#error "Could not guess the kind of compiler, please report to us."
@ -101,9 +96,7 @@ typedef CRITICAL_SECTION osal_fastmutex_t;
#if MDBX_WITHOUT_MSVC_CRT
#ifndef osal_malloc
static inline void *osal_malloc(size_t bytes) {
return HeapAlloc(GetProcessHeap(), 0, bytes);
}
static inline void *osal_malloc(size_t bytes) { return HeapAlloc(GetProcessHeap(), 0, bytes); }
#endif /* osal_malloc */
#ifndef osal_calloc
@ -114,8 +107,7 @@ static inline void *osal_calloc(size_t nelem, size_t size) {
#ifndef osal_realloc
static inline void *osal_realloc(void *ptr, size_t bytes) {
return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes)
: HeapAlloc(GetProcessHeap(), 0, bytes);
return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) : HeapAlloc(GetProcessHeap(), 0, bytes);
}
#endif /* osal_realloc */
@ -208,7 +200,7 @@ typedef struct osal_mmap {
#elif defined(__APPLE__) || defined(__MACH__) || defined(_DARWIN_C_SOURCE)
#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \
#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \
MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0
/* FIXME: add checks for IOS versions, etc */
#define MDBX_HAVE_PWRITEV 1
@ -279,39 +271,29 @@ typedef struct osal_ioring {
MDBX_INTERNAL int osal_ioring_create(osal_ioring_t *
#if defined(_WIN32) || defined(_WIN64)
,
bool enable_direct,
mdbx_filehandle_t overlapped_fd
bool enable_direct, mdbx_filehandle_t overlapped_fd
#endif /* Windows */
);
MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *, size_t items);
MDBX_INTERNAL void osal_ioring_destroy(osal_ioring_t *);
MDBX_INTERNAL void osal_ioring_reset(osal_ioring_t *);
MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ctx, const size_t offset,
void *data, const size_t bytes);
MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, void *data, const size_t bytes);
typedef struct osal_ioring_write_result {
int err;
unsigned wops;
} osal_ioring_write_result_t;
MDBX_INTERNAL osal_ioring_write_result_t
osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd);
MDBX_INTERNAL osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd);
MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx,
void (*callback)(iov_ctx_t *ctx,
size_t offset, void *data,
size_t bytes));
void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes));
MDBX_MAYBE_UNUSED static inline unsigned
osal_ioring_left(const osal_ioring_t *ior) {
return ior->slots_left;
}
MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_left(const osal_ioring_t *ior) { return ior->slots_left; }
MDBX_MAYBE_UNUSED static inline unsigned
osal_ioring_used(const osal_ioring_t *ior) {
MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_used(const osal_ioring_t *ior) {
return ior->allocated - ior->slots_left;
}
MDBX_MAYBE_UNUSED static inline int
osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) {
MDBX_MAYBE_UNUSED static inline int osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) {
items = (items > 32) ? items : 32;
#if defined(_WIN32) || defined(_WIN64)
if (ior->direct) {
@ -330,13 +312,11 @@ osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) {
/*----------------------------------------------------------------------------*/
/* libc compatibility stuff */
#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \
(defined(_GNU_SOURCE) || defined(_BSD_SOURCE))
#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && (defined(_GNU_SOURCE) || defined(_BSD_SOURCE))
#define osal_asprintf asprintf
#define osal_vasprintf vasprintf
#else
MDBX_MAYBE_UNUSED MDBX_INTERNAL
MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...);
MDBX_MAYBE_UNUSED MDBX_INTERNAL MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...);
MDBX_INTERNAL int osal_vasprintf(char **strp, const char *fmt, va_list ap);
#endif
@ -358,14 +338,12 @@ MDBX_MAYBE_UNUSED MDBX_INTERNAL void osal_jitter(bool tiny);
#else
#define MAX_WRITE UINT32_C(0x3f000000)
#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && \
!defined(__ANDROID_API__)
#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && !defined(__ANDROID_API__)
#define MDBX_F_SETLK F_SETLK64
#define MDBX_F_SETLKW F_SETLKW64
#define MDBX_F_GETLK F_GETLK64
#if (__GLIBC_PREREQ(2, 28) && \
(defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \
defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \
#if (__GLIBC_PREREQ(2, 28) && (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \
defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \
defined(fcntl64)
#define MDBX_FCNTL fcntl64
#else
@ -383,8 +361,7 @@ MDBX_MAYBE_UNUSED MDBX_INTERNAL void osal_jitter(bool tiny);
#define MDBX_STRUCT_FLOCK struct flock
#endif /* MDBX_F_SETLK, MDBX_F_SETLKW, MDBX_F_GETLK */
#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \
defined(F_OFD_GETLK64) && !defined(__ANDROID_API__)
#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64) && !defined(__ANDROID_API__)
#define MDBX_F_OFD_SETLK F_OFD_SETLK64
#define MDBX_F_OFD_SETLKW F_OFD_SETLKW64
#define MDBX_F_OFD_GETLK F_OFD_GETLK64
@ -393,8 +370,7 @@ MDBX_MAYBE_UNUSED MDBX_INTERNAL void osal_jitter(bool tiny);
#define MDBX_F_OFD_SETLKW F_OFD_SETLKW
#define MDBX_F_OFD_GETLK F_OFD_GETLK
#ifndef OFF_T_MAX
#define OFF_T_MAX \
(((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff)
#define OFF_T_MAX (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff)
#endif /* OFF_T_MAX */
#endif /* MDBX_F_OFD_SETLK64, MDBX_F_OFD_SETLKW64, MDBX_F_OFD_GETLK64 */
@ -414,8 +390,7 @@ MDBX_MAYBE_UNUSED static inline int osal_get_errno(void) {
}
#ifndef osal_memalign_alloc
MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes,
void **result);
MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, void **result);
#endif
#ifndef osal_memalign_free
MDBX_INTERNAL void osal_memalign_free(void *ptr);
@ -433,19 +408,13 @@ MDBX_INTERNAL int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex);
MDBX_INTERNAL int osal_fastmutex_release(osal_fastmutex_t *fastmutex);
MDBX_INTERNAL int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex);
MDBX_INTERNAL int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov,
size_t sgvcnt, uint64_t offset);
MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count,
uint64_t offset);
MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf,
size_t count, uint64_t offset);
MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf,
size_t count);
MDBX_INTERNAL int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset);
MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, uint64_t offset);
MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t count, uint64_t offset);
MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, size_t count);
MDBX_INTERNAL int
osal_thread_create(osal_thread_t *thread,
THREAD_RESULT(THREAD_CALL *start_routine)(void *),
void *arg);
MDBX_INTERNAL int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *),
void *arg);
MDBX_INTERNAL int osal_thread_join(osal_thread_t thread);
enum osal_syncmode_bits {
@ -456,8 +425,7 @@ enum osal_syncmode_bits {
MDBX_SYNC_IODQ = 8
};
MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd,
const enum osal_syncmode_bits mode_bits);
MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, const enum osal_syncmode_bits mode_bits);
MDBX_INTERNAL int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length);
MDBX_INTERNAL int osal_fseek(mdbx_filehandle_t fd, uint64_t pos);
MDBX_INTERNAL int osal_filesize(mdbx_filehandle_t fd, uint64_t *length);
@ -483,14 +451,11 @@ MDBX_MAYBE_UNUSED static inline bool osal_isdirsep(pathchar_t c) {
c == '/';
}
MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r,
size_t len);
MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len);
MDBX_INTERNAL pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len);
MDBX_INTERNAL int osal_fileexists(const pathchar_t *pathname);
MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose,
const MDBX_env *env, const pathchar_t *pathname,
mdbx_filehandle_t *fd,
mdbx_mode_t unix_mode_bits);
MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env,
const pathchar_t *pathname, mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits);
MDBX_INTERNAL int osal_closefile(mdbx_filehandle_t fd);
MDBX_INTERNAL int osal_removefile(const pathchar_t *pathname);
MDBX_INTERNAL int osal_removedirectory(const pathchar_t *pathname);
@ -499,26 +464,21 @@ MDBX_INTERNAL int osal_lockfile(mdbx_filehandle_t fd, bool wait);
#define MMAP_OPTION_TRUNCATE 1
#define MMAP_OPTION_SEMAPHORE 2
MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size,
const size_t limit, const unsigned options);
MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, const unsigned options);
MDBX_INTERNAL int osal_munmap(osal_mmap_t *map);
#define MDBX_MRESIZE_MAY_MOVE 0x00000100
#define MDBX_MRESIZE_MAY_UNMAP 0x00000200
MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size,
size_t limit);
MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit);
#if defined(_WIN32) || defined(_WIN64)
typedef struct {
unsigned limit, count;
HANDLE handles[31];
} mdbx_handle_array_t;
MDBX_INTERNAL int
osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array);
MDBX_INTERNAL int osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array);
MDBX_INTERNAL int osal_resume_threads_after_remap(mdbx_handle_array_t *array);
#endif /* Windows */
MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset,
size_t length, enum osal_syncmode_bits mode_bits);
MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle,
const pathchar_t *pathname, int err);
MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, size_t length, enum osal_syncmode_bits mode_bits);
MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err);
MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle);
MDBX_MAYBE_UNUSED static inline uint32_t osal_getpid(void) {
@ -549,8 +509,7 @@ MDBX_INTERNAL int osal_check_tid4bionic(void);
static inline int osal_check_tid4bionic(void) { return 0; }
#endif /* __ANDROID_API__ || ANDROID) || BIONIC */
MDBX_MAYBE_UNUSED static inline int
osal_pthread_mutex_lock(pthread_mutex_t *mutex) {
MDBX_MAYBE_UNUSED static inline int osal_pthread_mutex_lock(pthread_mutex_t *mutex) {
int err = osal_check_tid4bionic();
return unlikely(err) ? err : pthread_mutex_lock(mutex);
}
@ -561,8 +520,7 @@ MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults);
MDBX_INTERNAL uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16);
MDBX_INTERNAL uint32_t osal_monotime_to_16dot16(uint64_t monotime);
MDBX_MAYBE_UNUSED static inline uint32_t
osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) {
MDBX_MAYBE_UNUSED static inline uint32_t osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) {
uint32_t seconds_16dot16 = osal_monotime_to_16dot16(monotime);
return seconds_16dot16 ? seconds_16dot16 : /* fix underflow */ (monotime > 0);
}
@ -589,10 +547,8 @@ MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *);
/*----------------------------------------------------------------------------*/
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t
osal_bswap64(uint64_t v) {
#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \
__has_builtin(__builtin_bswap64)
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t osal_bswap64(uint64_t v) {
#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap64)
return __builtin_bswap64(v);
#elif defined(_MSC_VER) && !defined(__clang__)
return _byteswap_uint64(v);
@ -601,19 +557,14 @@ osal_bswap64(uint64_t v) {
#elif defined(bswap_64)
return bswap_64(v);
#else
return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) |
((v << 24) & UINT64_C(0x0000ff0000000000)) |
((v << 8) & UINT64_C(0x000000ff00000000)) |
((v >> 8) & UINT64_C(0x00000000ff000000)) |
((v >> 24) & UINT64_C(0x0000000000ff0000)) |
((v >> 40) & UINT64_C(0x000000000000ff00));
return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | ((v << 24) & UINT64_C(0x0000ff0000000000)) |
((v << 8) & UINT64_C(0x000000ff00000000)) | ((v >> 8) & UINT64_C(0x00000000ff000000)) |
((v >> 24) & UINT64_C(0x0000000000ff0000)) | ((v >> 40) & UINT64_C(0x000000000000ff00));
#endif
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t
osal_bswap32(uint32_t v) {
#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \
__has_builtin(__builtin_bswap32)
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t osal_bswap32(uint32_t v) {
#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap32)
return __builtin_bswap32(v);
#elif defined(_MSC_VER) && !defined(__clang__)
return _byteswap_ulong(v);
@ -622,7 +573,6 @@ osal_bswap32(uint32_t v) {
#elif defined(bswap_32)
return bswap_32(v);
#else
return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) |
((v >> 8) & UINT32_C(0x0000ff00));
return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | ((v >> 8) & UINT32_C(0x0000ff00));
#endif
}

280
src/page-get.c
View File

@ -3,17 +3,14 @@
#include "internals.h"
__cold int MDBX_PRINTF_ARGS(2, 3)
bad_page(const page_t *mp, const char *fmt, ...) {
__cold int MDBX_PRINTF_ARGS(2, 3) bad_page(const page_t *mp, const char *fmt, ...) {
if (LOG_ENABLED(MDBX_LOG_ERROR)) {
static const page_t *prev;
if (prev != mp) {
char buf4unknown[16];
prev = mp;
debug_log(MDBX_LOG_ERROR, "badpage", 0,
"corrupted %s-page #%u, mod-txnid %" PRIaTXN "\n",
pagetype_caption(page_type(mp), buf4unknown), mp->pgno,
mp->txnid);
debug_log(MDBX_LOG_ERROR, "badpage", 0, "corrupted %s-page #%u, mod-txnid %" PRIaTXN "\n",
pagetype_caption(page_type(mp), buf4unknown), mp->pgno, mp->txnid);
}
va_list args;
@ -24,17 +21,14 @@ __cold int MDBX_PRINTF_ARGS(2, 3)
return MDBX_CORRUPTED;
}
__cold void MDBX_PRINTF_ARGS(2, 3)
poor_page(const page_t *mp, const char *fmt, ...) {
__cold void MDBX_PRINTF_ARGS(2, 3) poor_page(const page_t *mp, const char *fmt, ...) {
if (LOG_ENABLED(MDBX_LOG_NOTICE)) {
static const page_t *prev;
if (prev != mp) {
char buf4unknown[16];
prev = mp;
debug_log(MDBX_LOG_NOTICE, "poorpage", 0,
"suboptimal %s-page #%u, mod-txnid %" PRIaTXN "\n",
pagetype_caption(page_type(mp), buf4unknown), mp->pgno,
mp->txnid);
debug_log(MDBX_LOG_NOTICE, "poorpage", 0, "suboptimal %s-page #%u, mod-txnid %" PRIaTXN "\n",
pagetype_caption(page_type(mp), buf4unknown), mp->pgno, mp->txnid);
}
va_list args;
@ -63,21 +57,17 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) {
const ptrdiff_t offset = ptr_dist(mp, env->dxb_mmap.base);
unsigned flags_mask = P_ILL_BITS;
unsigned flags_expected = 0;
if (offset < 0 ||
offset > (ptrdiff_t)(pgno2bytes(env, mc->txn->geo.first_unallocated) -
((mp->flags & P_SUBP) ? PAGEHDRSZ + 1 : env->ps))) {
if (offset < 0 || offset > (ptrdiff_t)(pgno2bytes(env, mc->txn->geo.first_unallocated) -
((mp->flags & P_SUBP) ? PAGEHDRSZ + 1 : env->ps))) {
/* should be dirty page without MDBX_WRITEMAP, or a subpage of. */
flags_mask -= P_SUBP;
if ((env->flags & MDBX_WRITEMAP) != 0 ||
(!is_shadowed(mc->txn, mp) && !(mp->flags & P_SUBP)))
rc = bad_page(mp, "invalid page-address %p, offset %zi\n",
__Wpedantic_format_voidptr(mp), offset);
if ((env->flags & MDBX_WRITEMAP) != 0 || (!is_shadowed(mc->txn, mp) && !(mp->flags & P_SUBP)))
rc = bad_page(mp, "invalid page-address %p, offset %zi\n", __Wpedantic_format_voidptr(mp), offset);
} else if (offset & (env->ps - 1))
flags_expected = P_SUBP;
if (unlikely((mp->flags & flags_mask) != flags_expected))
rc = bad_page(mp, "unknown/extra page-flags (have 0x%x, expect 0x%x)\n",
mp->flags & flags_mask, flags_expected);
rc = bad_page(mp, "unknown/extra page-flags (have 0x%x, expect 0x%x)\n", mp->flags & flags_mask, flags_expected);
cASSERT(mc, (mc->checking & z_dupfix) == 0 || (mc->flags & z_inner) != 0);
const uint8_t type = page_type(mp);
@ -86,82 +76,62 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) {
return bad_page(mp, "invalid type (%u)\n", type);
case P_LARGE:
if (unlikely(mc->flags & z_inner))
rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "large",
"nested dupsort tree", mc->tree->flags);
rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "large", "nested dupsort tree", mc->tree->flags);
const pgno_t npages = mp->pages;
if (unlikely(npages < 1 || npages >= MAX_PAGENO / 2))
rc = bad_page(mp, "invalid n-pages (%u) for large-page\n", npages);
if (unlikely(mp->pgno + npages > mc->txn->geo.first_unallocated))
rc = bad_page(
mp, "end of large-page beyond (%u) allocated space (%u next-pgno)\n",
mp->pgno + npages, mc->txn->geo.first_unallocated);
rc = bad_page(mp, "end of large-page beyond (%u) allocated space (%u next-pgno)\n", mp->pgno + npages,
mc->txn->geo.first_unallocated);
return rc; //-------------------------- end of large/overflow page handling
case P_LEAF | P_SUBP:
if (unlikely(mc->tree->height != 1))
rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n",
"leaf-sub", "nested dupsort db", mc->tree->flags);
rc =
bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "leaf-sub", "nested dupsort db", mc->tree->flags);
/* fall through */
__fallthrough;
case P_LEAF:
if (unlikely((mc->checking & z_dupfix) != 0))
rc = bad_page(mp,
"unexpected leaf-page for dupfix subtree (db-lags 0x%x)\n",
mc->tree->flags);
rc = bad_page(mp, "unexpected leaf-page for dupfix subtree (db-lags 0x%x)\n", mc->tree->flags);
break;
case P_LEAF | P_DUPFIX | P_SUBP:
if (unlikely(mc->tree->height != 1))
rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n",
"leaf2-sub", "nested dupsort db", mc->tree->flags);
rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "leaf2-sub", "nested dupsort db",
mc->tree->flags);
/* fall through */
__fallthrough;
case P_LEAF | P_DUPFIX:
if (unlikely((mc->checking & z_dupfix) == 0))
rc = bad_page(
mp,
"unexpected leaf2-page for non-dupfix (sub)tree (db-flags 0x%x)\n",
mc->tree->flags);
rc = bad_page(mp, "unexpected leaf2-page for non-dupfix (sub)tree (db-flags 0x%x)\n", mc->tree->flags);
break;
case P_BRANCH:
break;
}
if (unlikely(mp->upper < mp->lower || (mp->lower & 1) ||
PAGEHDRSZ + mp->upper > env->ps))
rc = bad_page(mp, "invalid page lower(%u)/upper(%u) with limit %zu\n",
mp->lower, mp->upper, page_space(env));
if (unlikely(mp->upper < mp->lower || (mp->lower & 1) || PAGEHDRSZ + mp->upper > env->ps))
rc = bad_page(mp, "invalid page lower(%u)/upper(%u) with limit %zu\n", mp->lower, mp->upper, page_space(env));
const char *const end_of_page = ptr_disp(mp, env->ps);
const size_t nkeys = page_numkeys(mp);
STATIC_ASSERT(P_BRANCH == 1);
if (unlikely(nkeys <= (uint8_t)(mp->flags & P_BRANCH))) {
if ((!(mc->flags & z_inner) || mc->tree->items) &&
(!(mc->checking & z_updating) ||
!(is_modifable(mc->txn, mp) || (mp->flags & P_SUBP))))
rc =
bad_page(mp, "%s-page nkeys (%zu) < %u\n",
is_branch(mp) ? "branch" : "leaf", nkeys, 1 + is_branch(mp));
(!(mc->checking & z_updating) || !(is_modifable(mc->txn, mp) || (mp->flags & P_SUBP))))
rc = bad_page(mp, "%s-page nkeys (%zu) < %u\n", is_branch(mp) ? "branch" : "leaf", nkeys, 1 + is_branch(mp));
}
const size_t ksize_max = keysize_max(env->ps, 0);
const size_t leaf2_ksize = mp->dupfix_ksize;
if (is_dupfix_leaf(mp)) {
if (unlikely((mc->flags & z_inner) == 0 ||
(mc->tree->flags & MDBX_DUPFIXED) == 0))
rc = bad_page(mp, "unexpected leaf2-page (db-flags 0x%x)\n",
mc->tree->flags);
if (unlikely((mc->flags & z_inner) == 0 || (mc->tree->flags & MDBX_DUPFIXED) == 0))
rc = bad_page(mp, "unexpected leaf2-page (db-flags 0x%x)\n", mc->tree->flags);
else if (unlikely(leaf2_ksize != mc->tree->dupfix_size))
rc = bad_page(mp, "invalid leaf2_ksize %zu\n", leaf2_ksize);
else if (unlikely(((leaf2_ksize & nkeys) ^ mp->upper) & 1))
rc = bad_page(
mp, "invalid page upper (%u) for nkeys %zu with leaf2-length %zu\n",
mp->upper, nkeys, leaf2_ksize);
rc = bad_page(mp, "invalid page upper (%u) for nkeys %zu with leaf2-length %zu\n", mp->upper, nkeys, leaf2_ksize);
} else {
if (unlikely((mp->upper & 1) ||
PAGEHDRSZ + mp->upper + nkeys * sizeof(node_t) + nkeys - 1 >
env->ps))
rc =
bad_page(mp, "invalid page upper (%u) for nkeys %zu with limit %zu\n",
mp->upper, nkeys, page_space(env));
if (unlikely((mp->upper & 1) || PAGEHDRSZ + mp->upper + nkeys * sizeof(node_t) + nkeys - 1 > env->ps))
rc = bad_page(mp, "invalid page upper (%u) for nkeys %zu with limit %zu\n", mp->upper, nkeys, page_space(env));
}
MDBX_val here, prev = {0, 0};
@ -170,17 +140,14 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) {
if (is_dupfix_leaf(mp)) {
const char *const key = page_dupfix_ptr(mp, i, mc->tree->dupfix_size);
if (unlikely(end_of_page < key + leaf2_ksize)) {
rc = bad_page(mp, "leaf2-item beyond (%zu) page-end\n",
key + leaf2_ksize - end_of_page);
rc = bad_page(mp, "leaf2-item beyond (%zu) page-end\n", key + leaf2_ksize - end_of_page);
continue;
}
if (unlikely(leaf2_ksize != mc->clc->k.lmin)) {
if (unlikely(leaf2_ksize < mc->clc->k.lmin ||
leaf2_ksize > mc->clc->k.lmax))
rc = bad_page(mp,
"leaf2-item size (%zu) <> min/max length (%zu/%zu)\n",
leaf2_ksize, mc->clc->k.lmin, mc->clc->k.lmax);
if (unlikely(leaf2_ksize < mc->clc->k.lmin || leaf2_ksize > mc->clc->k.lmax))
rc = bad_page(mp, "leaf2-item size (%zu) <> min/max length (%zu/%zu)\n", leaf2_ksize, mc->clc->k.lmin,
mc->clc->k.lmax);
else
mc->clc->k.lmin = mc->clc->k.lmax = leaf2_ksize;
}
@ -188,16 +155,14 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) {
here.iov_base = (void *)key;
here.iov_len = leaf2_ksize;
if (prev.iov_base && unlikely(mc->clc->k.cmp(&prev, &here) >= 0))
rc = bad_page(mp, "leaf2-item #%zu wrong order (%s >= %s)\n", i,
DKEY(&prev), DVAL(&here));
rc = bad_page(mp, "leaf2-item #%zu wrong order (%s >= %s)\n", i, DKEY(&prev), DVAL(&here));
prev = here;
}
} else {
const node_t *const node = page_node(mp, i);
const char *const node_end = ptr_disp(node, NODESIZE);
if (unlikely(node_end > end_of_page)) {
rc = bad_page(mp, "node[%zu] (%zu) beyond page-end\n", i,
node_end - end_of_page);
rc = bad_page(mp, "node[%zu] (%zu) beyond page-end\n", i, node_end - end_of_page);
continue;
}
const size_t ksize = node_ks(node);
@ -205,44 +170,36 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) {
rc = bad_page(mp, "node[%zu] too long key (%zu)\n", i, ksize);
const char *const key = node_key(node);
if (unlikely(end_of_page < key + ksize)) {
rc = bad_page(mp, "node[%zu] key (%zu) beyond page-end\n", i,
key + ksize - end_of_page);
rc = bad_page(mp, "node[%zu] key (%zu) beyond page-end\n", i, key + ksize - end_of_page);
continue;
}
if ((is_leaf(mp) || i > 0)) {
if (unlikely(ksize < mc->clc->k.lmin || ksize > mc->clc->k.lmax))
rc = bad_page(
mp, "node[%zu] key size (%zu) <> min/max key-length (%zu/%zu)\n",
i, ksize, mc->clc->k.lmin, mc->clc->k.lmax);
rc = bad_page(mp, "node[%zu] key size (%zu) <> min/max key-length (%zu/%zu)\n", i, ksize, mc->clc->k.lmin,
mc->clc->k.lmax);
if ((mc->checking & z_ignord) == 0) {
here.iov_base = (void *)key;
here.iov_len = ksize;
if (prev.iov_base && unlikely(mc->clc->k.cmp(&prev, &here) >= 0))
rc = bad_page(mp, "node[%zu] key wrong order (%s >= %s)\n", i,
DKEY(&prev), DVAL(&here));
rc = bad_page(mp, "node[%zu] key wrong order (%s >= %s)\n", i, DKEY(&prev), DVAL(&here));
prev = here;
}
}
if (is_branch(mp)) {
if ((mc->checking & z_updating) == 0 && i == 0 && unlikely(ksize != 0))
rc = bad_page(mp, "branch-node[%zu] wrong 0-node key-length (%zu)\n",
i, ksize);
rc = bad_page(mp, "branch-node[%zu] wrong 0-node key-length (%zu)\n", i, ksize);
const pgno_t ref = node_pgno(node);
if (unlikely(ref < MIN_PAGENO) ||
(unlikely(ref >= mc->txn->geo.first_unallocated) &&
(unlikely(ref >= mc->txn->geo.now) ||
!(mc->checking & z_retiring))))
if (unlikely(ref < MIN_PAGENO) || (unlikely(ref >= mc->txn->geo.first_unallocated) &&
(unlikely(ref >= mc->txn->geo.now) || !(mc->checking & z_retiring))))
rc = bad_page(mp, "branch-node[%zu] wrong pgno (%u)\n", i, ref);
if (unlikely(node_flags(node)))
rc = bad_page(mp, "branch-node[%zu] wrong flags (%u)\n", i,
node_flags(node));
rc = bad_page(mp, "branch-node[%zu] wrong flags (%u)\n", i, node_flags(node));
continue;
}
switch (node_flags(node)) {
default:
rc =
bad_page(mp, "invalid node[%zu] flags (%u)\n", i, node_flags(node));
rc = bad_page(mp, "invalid node[%zu] flags (%u)\n", i, node_flags(node));
break;
case N_BIG /* data on large-page */:
case 0 /* usual */:
@ -256,46 +213,36 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) {
const char *const data = node_data(node);
if (node_flags(node) & N_BIG) {
if (unlikely(end_of_page < data + sizeof(pgno_t))) {
rc = bad_page(
mp, "node-%s(%zu of %zu, %zu bytes) beyond (%zu) page-end\n",
"bigdata-pgno", i, nkeys, dsize, data + dsize - end_of_page);
rc = bad_page(mp, "node-%s(%zu of %zu, %zu bytes) beyond (%zu) page-end\n", "bigdata-pgno", i, nkeys, dsize,
data + dsize - end_of_page);
continue;
}
if (unlikely(dsize <= v_clc.lmin || dsize > v_clc.lmax))
rc = bad_page(
mp,
"big-node data size (%zu) <> min/max value-length (%zu/%zu)\n",
dsize, v_clc.lmin, v_clc.lmax);
if (unlikely(node_size_len(node_ks(node), dsize) <=
mc->txn->env->leaf_nodemax) &&
rc = bad_page(mp, "big-node data size (%zu) <> min/max value-length (%zu/%zu)\n", dsize, v_clc.lmin,
v_clc.lmax);
if (unlikely(node_size_len(node_ks(node), dsize) <= mc->txn->env->leaf_nodemax) &&
mc->tree != &mc->txn->dbs[FREE_DBI])
poor_page(mp, "too small data (%zu bytes) for bigdata-node", dsize);
if ((mc->checking & z_retiring) == 0) {
const pgr_t lp =
page_get_large(mc, node_largedata_pgno(node), mp->txnid);
const pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->txnid);
if (unlikely(lp.err != MDBX_SUCCESS))
return lp.err;
cASSERT(mc, page_type(lp.page) == P_LARGE);
const unsigned npages = largechunk_npages(env, dsize);
if (unlikely(lp.page->pages != npages)) {
if (lp.page->pages < npages)
rc = bad_page(lp.page,
"too less n-pages %u for bigdata-node (%zu bytes)",
lp.page->pages, dsize);
rc = bad_page(lp.page, "too less n-pages %u for bigdata-node (%zu bytes)", lp.page->pages, dsize);
else if (mc->tree != &mc->txn->dbs[FREE_DBI])
poor_page(lp.page,
"extra n-pages %u for bigdata-node (%zu bytes)",
lp.page->pages, dsize);
poor_page(lp.page, "extra n-pages %u for bigdata-node (%zu bytes)", lp.page->pages, dsize);
}
}
continue;
}
if (unlikely(end_of_page < data + dsize)) {
rc = bad_page(mp,
"node-%s(%zu of %zu, %zu bytes) beyond (%zu) page-end\n",
"data", i, nkeys, dsize, data + dsize - end_of_page);
rc = bad_page(mp, "node-%s(%zu of %zu, %zu bytes) beyond (%zu) page-end\n", "data", i, nkeys, dsize,
data + dsize - end_of_page);
continue;
}
@ -305,9 +252,7 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) {
continue;
case 0 /* usual */:
if (unlikely(dsize < v_clc.lmin || dsize > v_clc.lmax)) {
rc = bad_page(
mp, "node-data size (%zu) <> min/max value-length (%zu/%zu)\n",
dsize, v_clc.lmin, v_clc.lmax);
rc = bad_page(mp, "node-data size (%zu) <> min/max value-length (%zu/%zu)\n", dsize, v_clc.lmin, v_clc.lmax);
continue;
}
break;
@ -319,15 +264,13 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) {
break;
case N_TREE | N_DUP /* dupsorted sub-tree */:
if (unlikely(dsize != sizeof(tree_t))) {
rc = bad_page(mp, "invalid nested-db record size (%zu, expect %zu)\n",
dsize, sizeof(tree_t));
rc = bad_page(mp, "invalid nested-db record size (%zu, expect %zu)\n", dsize, sizeof(tree_t));
continue;
}
break;
case N_DUP /* short sub-page */:
if (unlikely(dsize <= PAGEHDRSZ)) {
rc = bad_page(mp, "invalid nested/sub-page record size (%zu)\n",
dsize);
rc = bad_page(mp, "invalid nested/sub-page record size (%zu)\n", dsize);
continue;
} else {
const page_t *const sp = (page_t *)data;
@ -337,28 +280,23 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) {
case P_LEAF | P_DUPFIX | P_SUBP:
break;
default:
rc = bad_page(mp, "invalid nested/sub-page flags (0x%02x)\n",
sp->flags);
rc = bad_page(mp, "invalid nested/sub-page flags (0x%02x)\n", sp->flags);
continue;
}
const char *const end_of_subpage = data + dsize;
const intptr_t nsubkeys = page_numkeys(sp);
if (unlikely(nsubkeys == 0) && !(mc->checking & z_updating) &&
mc->tree->items)
rc = bad_page(mp, "no keys on a %s-page\n",
is_dupfix_leaf(sp) ? "leaf2-sub" : "leaf-sub");
if (unlikely(nsubkeys == 0) && !(mc->checking & z_updating) && mc->tree->items)
rc = bad_page(mp, "no keys on a %s-page\n", is_dupfix_leaf(sp) ? "leaf2-sub" : "leaf-sub");
MDBX_val sub_here, sub_prev = {0, 0};
for (int ii = 0; ii < nsubkeys; ii++) {
if (is_dupfix_leaf(sp)) {
/* DUPFIX pages have no entries[] or node headers */
const size_t sub_ksize = sp->dupfix_ksize;
const char *const sub_key =
page_dupfix_ptr(sp, ii, mc->tree->dupfix_size);
const char *const sub_key = page_dupfix_ptr(sp, ii, mc->tree->dupfix_size);
if (unlikely(end_of_subpage < sub_key + sub_ksize)) {
rc = bad_page(mp, "nested-leaf2-key beyond (%zu) nested-page\n",
sub_key + sub_ksize - end_of_subpage);
rc = bad_page(mp, "nested-leaf2-key beyond (%zu) nested-page\n", sub_key + sub_ksize - end_of_subpage);
continue;
}
@ -374,24 +312,20 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) {
if ((mc->checking & z_ignord) == 0) {
sub_here.iov_base = (void *)sub_key;
sub_here.iov_len = sub_ksize;
if (sub_prev.iov_base &&
unlikely(v_clc.cmp(&sub_prev, &sub_here) >= 0))
rc = bad_page(mp,
"nested-leaf2-key #%u wrong order (%s >= %s)\n",
ii, DKEY(&sub_prev), DVAL(&sub_here));
if (sub_prev.iov_base && unlikely(v_clc.cmp(&sub_prev, &sub_here) >= 0))
rc = bad_page(mp, "nested-leaf2-key #%u wrong order (%s >= %s)\n", ii, DKEY(&sub_prev),
DVAL(&sub_here));
sub_prev = sub_here;
}
} else {
const node_t *const sub_node = page_node(sp, ii);
const char *const sub_node_end = ptr_disp(sub_node, NODESIZE);
if (unlikely(sub_node_end > end_of_subpage)) {
rc = bad_page(mp, "nested-node beyond (%zu) nested-page\n",
end_of_subpage - sub_node_end);
rc = bad_page(mp, "nested-node beyond (%zu) nested-page\n", end_of_subpage - sub_node_end);
continue;
}
if (unlikely(node_flags(sub_node) != 0))
rc = bad_page(mp, "nested-node invalid flags (%u)\n",
node_flags(sub_node));
rc = bad_page(mp, "nested-node invalid flags (%u)\n", node_flags(sub_node));
const size_t sub_ksize = node_ks(sub_node);
const char *const sub_key = node_key(sub_node);
@ -406,19 +340,15 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) {
if ((mc->checking & z_ignord) == 0) {
sub_here.iov_base = (void *)sub_key;
sub_here.iov_len = sub_ksize;
if (sub_prev.iov_base &&
unlikely(v_clc.cmp(&sub_prev, &sub_here) >= 0))
rc = bad_page(mp,
"nested-node-key #%u wrong order (%s >= %s)\n",
ii, DKEY(&sub_prev), DVAL(&sub_here));
if (sub_prev.iov_base && unlikely(v_clc.cmp(&sub_prev, &sub_here) >= 0))
rc = bad_page(mp, "nested-node-key #%u wrong order (%s >= %s)\n", ii, DKEY(&sub_prev),
DVAL(&sub_here));
sub_prev = sub_here;
}
if (unlikely(sub_dsize != 0))
rc = bad_page(mp, "nested-node non-empty data size (%zu)\n",
sub_dsize);
rc = bad_page(mp, "nested-node non-empty data size (%zu)\n", sub_dsize);
if (unlikely(end_of_subpage < sub_key + sub_ksize))
rc = bad_page(mp, "nested-node-key beyond (%zu) nested-page\n",
sub_key + sub_ksize - end_of_subpage);
rc = bad_page(mp, "nested-node-key beyond (%zu) nested-page\n", sub_key + sub_ksize - end_of_subpage);
}
}
}
@ -429,9 +359,7 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) {
return rc;
}
static __always_inline int check_page_header(const uint16_t ILL,
const page_t *page,
MDBX_txn *const txn,
static __always_inline int check_page_header(const uint16_t ILL, const page_t *page, MDBX_txn *const txn,
const txnid_t front) {
if (unlikely(page->flags & ILL)) {
if (ILL == P_ILL_BITS || (page->flags & P_ILL_BITS))
@ -439,59 +367,44 @@ static __always_inline int check_page_header(const uint16_t ILL,
else if (ILL & P_LARGE) {
assert((ILL & (P_BRANCH | P_LEAF | P_DUPFIX)) == 0);
assert(page->flags & (P_BRANCH | P_LEAF | P_DUPFIX));
return bad_page(page, "unexpected %s instead of %s (%u)\n",
"large/overflow", "branch/leaf/leaf2", page->flags);
return bad_page(page, "unexpected %s instead of %s (%u)\n", "large/overflow", "branch/leaf/leaf2", page->flags);
} else if (ILL & (P_BRANCH | P_LEAF | P_DUPFIX)) {
assert((ILL & P_BRANCH) && (ILL & P_LEAF) && (ILL & P_DUPFIX));
assert(page->flags & (P_BRANCH | P_LEAF | P_DUPFIX));
return bad_page(page, "unexpected %s instead of %s (%u)\n",
"branch/leaf/leaf2", "large/overflow", page->flags);
return bad_page(page, "unexpected %s instead of %s (%u)\n", "branch/leaf/leaf2", "large/overflow", page->flags);
} else {
assert(false);
}
}
if (unlikely(page->txnid > front) &&
unlikely(page->txnid > txn->front_txnid || front < txn->txnid))
return bad_page(
page,
"invalid page' txnid (%" PRIaTXN ") for %s' txnid (%" PRIaTXN ")\n",
page->txnid,
(front == txn->front_txnid && front != txn->txnid) ? "front-txn"
: "parent-page",
front);
if (unlikely(page->txnid > front) && unlikely(page->txnid > txn->front_txnid || front < txn->txnid))
return bad_page(page, "invalid page' txnid (%" PRIaTXN ") for %s' txnid (%" PRIaTXN ")\n", page->txnid,
(front == txn->front_txnid && front != txn->txnid) ? "front-txn" : "parent-page", front);
if (((ILL & P_LARGE) || !is_largepage(page)) &&
(ILL & (P_BRANCH | P_LEAF | P_DUPFIX)) == 0) {
if (((ILL & P_LARGE) || !is_largepage(page)) && (ILL & (P_BRANCH | P_LEAF | P_DUPFIX)) == 0) {
/* Контроль четности page->upper тут либо приводит к ложным ошибкам,
* либо слишком дорог по количеству операций. Заковырка в том, что upper
* может быть нечетным на DUPFIX-страницах, при нечетном количестве
* элементов нечетной длины. Поэтому четность page->upper здесь не
* проверяется, но соответствующие полные проверки есть в page_check(). */
if (unlikely(page->upper < page->lower || (page->lower & 1) ||
PAGEHDRSZ + page->upper > txn->env->ps))
return bad_page(page,
"invalid page' lower(%u)/upper(%u) with limit %zu\n",
page->lower, page->upper, page_space(txn->env));
if (unlikely(page->upper < page->lower || (page->lower & 1) || PAGEHDRSZ + page->upper > txn->env->ps))
return bad_page(page, "invalid page' lower(%u)/upper(%u) with limit %zu\n", page->lower, page->upper,
page_space(txn->env));
} else if ((ILL & P_LARGE) == 0) {
const pgno_t npages = page->pages;
if (unlikely(npages < 1) || unlikely(npages >= MAX_PAGENO / 2))
return bad_page(page, "invalid n-pages (%u) for large-page\n", npages);
if (unlikely(page->pgno + npages > txn->geo.first_unallocated))
return bad_page(
page,
"end of large-page beyond (%u) allocated space (%u next-pgno)\n",
page->pgno + npages, txn->geo.first_unallocated);
return bad_page(page, "end of large-page beyond (%u) allocated space (%u next-pgno)\n", page->pgno + npages,
txn->geo.first_unallocated);
} else {
assert(false);
}
return MDBX_SUCCESS;
}
__cold static __noinline pgr_t check_page_complete(const uint16_t ILL,
page_t *page,
const MDBX_cursor *const mc,
__cold static __noinline pgr_t check_page_complete(const uint16_t ILL, page_t *page, const MDBX_cursor *const mc,
const txnid_t front) {
pgr_t r = {page, check_page_header(ILL, page, mc->txn, front)};
if (likely(r.err == MDBX_SUCCESS))
@ -501,9 +414,7 @@ __cold static __noinline pgr_t check_page_complete(const uint16_t ILL,
return r;
}
static __always_inline pgr_t page_get_inline(const uint16_t ILL,
const MDBX_cursor *const mc,
const pgno_t pgno,
static __always_inline pgr_t page_get_inline(const uint16_t ILL, const MDBX_cursor *const mc, const pgno_t pgno,
const txnid_t front) {
MDBX_txn *const txn = mc->txn;
tASSERT(txn, front <= txn->front_txnid);
@ -527,8 +438,7 @@ static __always_inline pgr_t page_get_inline(const uint16_t ILL,
* because the dirty list got full. Bring this page
* back in from the map (but don't unspill it here,
* leave that unless page_touch happens again). */
if (unlikely(spiller->flags & MDBX_TXN_SPILLS) &&
spill_search(spiller, pgno))
if (unlikely(spiller->flags & MDBX_TXN_SPILLS) && spill_search(spiller, pgno))
break;
const size_t i = dpl_search(spiller, pgno);
@ -543,9 +453,7 @@ static __always_inline pgr_t page_get_inline(const uint16_t ILL,
}
if (unlikely(r.page->pgno != pgno)) {
r.err = bad_page(
r.page, "pgno mismatch (%" PRIaPGNO ") != expected (%" PRIaPGNO ")\n",
r.page->pgno, pgno);
r.err = bad_page(r.page, "pgno mismatch (%" PRIaPGNO ") != expected (%" PRIaPGNO ")\n", r.page->pgno, pgno);
goto bailout;
}
@ -562,18 +470,14 @@ static __always_inline pgr_t page_get_inline(const uint16_t ILL,
return r;
}
pgr_t page_get_any(const MDBX_cursor *const mc, const pgno_t pgno,
const txnid_t front) {
pgr_t page_get_any(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front) {
return page_get_inline(P_ILL_BITS, mc, pgno, front);
}
__hot pgr_t page_get_three(const MDBX_cursor *const mc, const pgno_t pgno,
const txnid_t front) {
__hot pgr_t page_get_three(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front) {
return page_get_inline(P_ILL_BITS | P_LARGE, mc, pgno, front);
}
pgr_t page_get_large(const MDBX_cursor *const mc, const pgno_t pgno,
const txnid_t front) {
return page_get_inline(P_ILL_BITS | P_BRANCH | P_LEAF | P_DUPFIX, mc, pgno,
front);
pgr_t page_get_large(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front) {
return page_get_inline(P_ILL_BITS | P_BRANCH | P_LEAF | P_DUPFIX, mc, pgno, front);
}

37
src/page-iov.c
View File

@ -3,17 +3,14 @@
#include "internals.h"
int iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, size_t items, size_t npages,
mdbx_filehandle_t fd, bool check_coherence) {
int iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, size_t items, size_t npages, mdbx_filehandle_t fd,
bool check_coherence) {
ctx->env = txn->env;
ctx->ior = &txn->env->ioring;
ctx->fd = fd;
ctx->coherency_timestamp =
(check_coherence || txn->env->lck->pgops.incoherence.weak)
? 0
: UINT64_MAX /* не выполнять сверку */;
ctx->err = osal_ioring_prepare(ctx->ior, items,
pgno_align2os_bytes(txn->env, npages));
(check_coherence || txn->env->lck->pgops.incoherence.weak) ? 0 : UINT64_MAX /* не выполнять сверку */;
ctx->err = osal_ioring_prepare(ctx->ior, items, pgno_align2os_bytes(txn->env, npages));
if (likely(ctx->err == MDBX_SUCCESS)) {
#if MDBX_NEED_WRITTEN_RANGE
ctx->flush_begin = MAX_PAGENO;
@ -24,8 +21,7 @@ int iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, size_t items, size_t npages,
return ctx->err;
}
static void iov_callback4dirtypages(iov_ctx_t *ctx, size_t offset, void *data,
size_t bytes) {
static void iov_callback4dirtypages(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes) {
MDBX_env *const env = ctx->env;
eASSERT(env, (env->flags & MDBX_WRITEMAP) == 0);
@ -89,19 +85,15 @@ static void iov_callback4dirtypages(iov_ctx_t *ctx, size_t offset, void *data,
#ifndef MDBX_FORCE_CHECK_MMAP_COHERENCY
#define MDBX_FORCE_CHECK_MMAP_COHERENCY 0
#endif /* MDBX_FORCE_CHECK_MMAP_COHERENCY */
if ((MDBX_FORCE_CHECK_MMAP_COHERENCY ||
ctx->coherency_timestamp != UINT64_MAX) &&
if ((MDBX_FORCE_CHECK_MMAP_COHERENCY || ctx->coherency_timestamp != UINT64_MAX) &&
unlikely(memcmp(wp, rp, bytes))) {
ctx->coherency_timestamp = 0;
env->lck->pgops.incoherence.weak =
(env->lck->pgops.incoherence.weak >= INT32_MAX)
? INT32_MAX
: env->lck->pgops.incoherence.weak + 1;
(env->lck->pgops.incoherence.weak >= INT32_MAX) ? INT32_MAX : env->lck->pgops.incoherence.weak + 1;
WARNING("catch delayed/non-arrived page %" PRIaPGNO " %s", wp->pgno,
"(workaround for incoherent flaw of unified page/buffer cache)");
do
if (coherency_timeout(&ctx->coherency_timestamp, wp->pgno, env) !=
MDBX_RESULT_TRUE) {
if (coherency_timeout(&ctx->coherency_timestamp, wp->pgno, env) != MDBX_RESULT_TRUE) {
ctx->err = MDBX_PROBLEM;
break;
}
@ -160,8 +152,7 @@ int iov_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, size_t npages) {
#if MDBX_AVOID_MSYNC
doit:;
#endif /* MDBX_AVOID_MSYNC */
int err = osal_ioring_add(ctx->ior, pgno2bytes(env, dp->pgno), dp,
pgno2bytes(env, npages));
int err = osal_ioring_add(ctx->ior, pgno2bytes(env, dp->pgno), dp, pgno2bytes(env, npages));
if (unlikely(err != MDBX_SUCCESS)) {
ctx->err = err;
if (unlikely(err != MDBX_RESULT_TRUE)) {
@ -171,8 +162,7 @@ int iov_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, size_t npages) {
err = iov_write(ctx);
tASSERT(txn, iov_empty(ctx));
if (likely(err == MDBX_SUCCESS)) {
err = osal_ioring_add(ctx->ior, pgno2bytes(env, dp->pgno), dp,
pgno2bytes(env, npages));
err = osal_ioring_add(ctx->ior, pgno2bytes(env, dp->pgno), dp, pgno2bytes(env, npages));
if (unlikely(err != MDBX_SUCCESS)) {
iov_complete(ctx);
return ctx->err = err;
@ -188,11 +178,8 @@ int iov_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, size_t npages) {
}
#if MDBX_NEED_WRITTEN_RANGE
ctx->flush_begin =
(ctx->flush_begin < dp->pgno) ? ctx->flush_begin : dp->pgno;
ctx->flush_end = (ctx->flush_end > dp->pgno + (pgno_t)npages)
? ctx->flush_end
: dp->pgno + (pgno_t)npages;
ctx->flush_begin = (ctx->flush_begin < dp->pgno) ? ctx->flush_begin : dp->pgno;
ctx->flush_end = (ctx->flush_end > dp->pgno + (pgno_t)npages) ? ctx->flush_end : dp->pgno + (pgno_t)npages;
#endif /* MDBX_NEED_WRITTEN_RANGE */
return MDBX_SUCCESS;
}

12
src/page-iov.h
View File

@ -24,15 +24,11 @@ struct iov_ctx {
uint64_t coherency_timestamp;
};
MDBX_INTERNAL __must_check_result int
iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, size_t items, size_t npages,
mdbx_filehandle_t fd, bool check_coherence);
MDBX_INTERNAL __must_check_result int iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, size_t items, size_t npages,
mdbx_filehandle_t fd, bool check_coherence);
static inline bool iov_empty(const iov_ctx_t *ctx) {
return osal_ioring_used(ctx->ior) == 0;
}
static inline bool iov_empty(const iov_ctx_t *ctx) { return osal_ioring_used(ctx->ior) == 0; }
MDBX_INTERNAL __must_check_result int iov_page(MDBX_txn *txn, iov_ctx_t *ctx,
page_t *dp, size_t npages);
MDBX_INTERNAL __must_check_result int iov_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, size_t npages);
MDBX_INTERNAL __must_check_result int iov_write(iov_ctx_t *ctx);

141
src/page-ops.c
View File

@ -42,13 +42,11 @@ pgr_t page_new(MDBX_cursor *mc, const unsigned flags) {
}
pgr_t page_new_large(MDBX_cursor *mc, const size_t npages) {
pgr_t ret = likely(npages == 1) ? gc_alloc_single(mc)
: gc_alloc_ex(mc, npages, ALLOC_DEFAULT);
pgr_t ret = likely(npages == 1) ? gc_alloc_single(mc) : gc_alloc_ex(mc, npages, ALLOC_DEFAULT);
if (unlikely(ret.err != MDBX_SUCCESS))
return ret;
DEBUG("dbi %zu allocated new large-page %" PRIaPGNO ", num %zu",
cursor_dbi(mc), ret.page->pgno, npages);
DEBUG("dbi %zu allocated new large-page %" PRIaPGNO ", num %zu", cursor_dbi(mc), ret.page->pgno, npages);
ret.page->flags = P_LARGE;
cASSERT(mc, *cursor_dbi_state(mc) & DBI_DIRTY);
cASSERT(mc, mc->txn->flags & MDBX_TXN_DIRTY);
@ -62,8 +60,7 @@ pgr_t page_new_large(MDBX_cursor *mc, const size_t npages) {
return ret;
}
__hot void page_copy(page_t *const dst, const page_t *const src,
const size_t size) {
__hot void page_copy(page_t *const dst, const page_t *const src, const size_t size) {
STATIC_ASSERT(UINT16_MAX > MDBX_MAX_PAGESIZE - PAGEHDRSZ);
STATIC_ASSERT(MDBX_MIN_PAGESIZE > PAGEHDRSZ + NODESIZE * 4);
void *copy_dst = dst;
@ -94,17 +91,14 @@ __hot void page_copy(page_t *const dst, const page_t *const src,
bailout:
if (src->flags & P_DUPFIX)
bad_page(src, "%s addr %p, n-keys %zu, ksize %u",
"invalid/corrupted source page", __Wpedantic_format_voidptr(src),
bad_page(src, "%s addr %p, n-keys %zu, ksize %u", "invalid/corrupted source page", __Wpedantic_format_voidptr(src),
page_numkeys(src), src->dupfix_ksize);
else
bad_page(src, "%s addr %p, upper %u", "invalid/corrupted source page",
__Wpedantic_format_voidptr(src), src->upper);
bad_page(src, "%s addr %p, upper %u", "invalid/corrupted source page", __Wpedantic_format_voidptr(src), src->upper);
memset(dst, -1, size);
}
__cold pgr_t __must_check_result page_unspill(MDBX_txn *const txn,
const page_t *const mp) {
__cold pgr_t __must_check_result page_unspill(MDBX_txn *const txn, const page_t *const mp) {
VERBOSE("unspill page %" PRIaPGNO, mp->pgno);
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0);
tASSERT(txn, is_spilled(txn, mp));
@ -139,13 +133,11 @@ __cold pgr_t __must_check_result page_unspill(MDBX_txn *const txn,
ret.page->flags |= (scan == txn) ? 0 : P_SPILLED;
ret.err = MDBX_SUCCESS;
return ret;
} while (likely((scan = scan->parent) != nullptr &&
(scan->flags & MDBX_TXN_SPILLS) != 0));
ERROR("Page %" PRIaPGNO " mod-txnid %" PRIaTXN
" not found in the spill-list(s), current txn %" PRIaTXN
} while (likely((scan = scan->parent) != nullptr && (scan->flags & MDBX_TXN_SPILLS) != 0));
ERROR("Page %" PRIaPGNO " mod-txnid %" PRIaTXN " not found in the spill-list(s), current txn %" PRIaTXN
" front %" PRIaTXN ", root txn %" PRIaTXN " front %" PRIaTXN,
mp->pgno, mp->txnid, txn->txnid, txn->front_txnid,
txn->env->basal_txn->txnid, txn->env->basal_txn->front_txnid);
mp->pgno, mp->txnid, txn->txnid, txn->front_txnid, txn->env->basal_txn->txnid,
txn->env->basal_txn->front_txnid);
ret.err = MDBX_PROBLEM;
ret.page = nullptr;
return ret;
@ -157,8 +149,7 @@ __hot int page_touch_modifable(MDBX_txn *txn, const page_t *const mp) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
const size_t n = dpl_search(txn, mp->pgno);
if (MDBX_AVOID_MSYNC &&
unlikely(txn->tw.dirtylist->items[n].pgno != mp->pgno)) {
if (MDBX_AVOID_MSYNC && unlikely(txn->tw.dirtylist->items[n].pgno != mp->pgno)) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP));
tASSERT(txn, n > 0 && n <= txn->tw.dirtylist->length + 1);
VERBOSE("unspill page %" PRIaPGNO, mp->pgno);
@ -169,18 +160,15 @@ __hot int page_touch_modifable(MDBX_txn *txn, const page_t *const mp) {
}
tASSERT(txn, n > 0 && n <= txn->tw.dirtylist->length);
tASSERT(txn, txn->tw.dirtylist->items[n].pgno == mp->pgno &&
txn->tw.dirtylist->items[n].ptr == mp);
tASSERT(txn, txn->tw.dirtylist->items[n].pgno == mp->pgno && txn->tw.dirtylist->items[n].ptr == mp);
if (!MDBX_AVOID_MSYNC || (txn->flags & MDBX_WRITEMAP) == 0) {
size_t *const ptr =
ptr_disp(txn->tw.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t));
size_t *const ptr = ptr_disp(txn->tw.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t));
*ptr = txn->tw.dirtylru;
}
return MDBX_SUCCESS;
}
__hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc,
const page_t *const mp) {
__hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *const mp) {
tASSERT(txn, !is_modifable(txn, mp) && !is_largepage(mp));
if (is_subpage(mp)) {
((page_t *)mp)->txnid = txn->front_txnid;
@ -201,8 +189,7 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc,
goto fail;
const pgno_t pgno = np->pgno;
DEBUG("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, cursor_dbi_dbg(mc),
mp->pgno, pgno);
DEBUG("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno, pgno);
tASSERT(txn, mp->pgno != pgno);
pnl_append_prereserved(txn->tw.retired_pages, mp->pgno);
/* Update the parent page, if any, to point to the new page */
@ -233,17 +220,14 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc,
if (unlikely(!txn->parent)) {
ERROR("Unexpected not frozen/modifiable/spilled but shadowed %s "
"page %" PRIaPGNO " mod-txnid %" PRIaTXN ","
" without parent transaction, current txn %" PRIaTXN
" front %" PRIaTXN,
is_branch(mp) ? "branch" : "leaf", mp->pgno, mp->txnid,
mc->txn->txnid, mc->txn->front_txnid);
" without parent transaction, current txn %" PRIaTXN " front %" PRIaTXN,
is_branch(mp) ? "branch" : "leaf", mp->pgno, mp->txnid, mc->txn->txnid, mc->txn->front_txnid);
rc = MDBX_PROBLEM;
goto fail;
}
DEBUG("clone db %d page %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno);
tASSERT(txn,
txn->tw.dirtylist->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE);
tASSERT(txn, txn->tw.dirtylist->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE);
/* No - copy it */
np = page_shadow_alloc(txn, 1);
if (unlikely(!np)) {
@ -299,8 +283,7 @@ page_t *page_shadow_alloc(MDBX_txn *txn, size_t num) {
if (likely(num == 1 && np)) {
eASSERT(env, env->shadow_reserve_len > 0);
MDBX_ASAN_UNPOISON_MEMORY_REGION(np, size);
VALGRIND_MEMPOOL_ALLOC(env, ptr_disp(np, -(ptrdiff_t)sizeof(size_t)),
size + sizeof(size_t));
VALGRIND_MEMPOOL_ALLOC(env, ptr_disp(np, -(ptrdiff_t)sizeof(size_t)), size + sizeof(size_t));
VALGRIND_MAKE_MEM_DEFINED(&page_next(np), sizeof(page_t *));
env->shadow_reserve = page_next(np);
env->shadow_reserve_len -= 1;
@ -338,8 +321,7 @@ void page_shadow_release(MDBX_env *env, page_t *dp, size_t npages) {
MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, pgno2bytes(env, npages));
if (unlikely(env->flags & MDBX_PAGEPERTURB))
memset(dp, -1, pgno2bytes(env, npages));
if (likely(npages == 1 &&
env->shadow_reserve_len < env->options.dp_reserve_limit)) {
if (likely(npages == 1 && env->shadow_reserve_len < env->options.dp_reserve_limit)) {
MDBX_ASAN_POISON_MEMORY_REGION(dp, env->ps);
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(dp), sizeof(page_t *));
page_next(dp) = env->shadow_reserve;
@ -354,8 +336,7 @@ void page_shadow_release(MDBX_env *env, page_t *dp, size_t npages) {
}
}
__cold static void page_kill(MDBX_txn *txn, page_t *mp, pgno_t pgno,
size_t npages) {
__cold static void page_kill(MDBX_txn *txn, page_t *mp, pgno_t pgno, size_t npages) {
MDBX_env *const env = txn->env;
DEBUG("kill %zu page(s) %" PRIaPGNO, npages, pgno);
eASSERT(env, pgno >= NUM_METAS && npages);
@ -391,8 +372,7 @@ static inline bool suitable4loose(const MDBX_txn *txn, pgno_t pgno) {
return txn->tw.loose_count < txn->env->options.dp_loose_limit &&
(!MDBX_ENABLE_REFUND ||
/* skip pages near to the end in favor of compactification */
txn->geo.first_unallocated >
pgno + txn->env->options.dp_loose_limit ||
txn->geo.first_unallocated > pgno + txn->env->options.dp_loose_limit ||
txn->geo.first_unallocated <= txn->env->options.dp_loose_limit);
}
@ -404,8 +384,7 @@ static inline bool suitable4loose(const MDBX_txn *txn, pgno_t pgno) {
*
* If the page wasn't dirtied in this txn, just add it
* to this txn's free list. */
int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno,
page_t *mp /* maybe null */,
int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null */,
unsigned pageflags /* maybe unknown/zero */) {
int rc;
MDBX_txn *const txn = mc->txn;
@ -423,13 +402,7 @@ int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno,
* So for flexibility and avoid extra internal dependencies we just
* fallback to reading if dirty list was not allocated yet. */
size_t di = 0, si = 0, npages = 1;
enum page_status {
unknown,
frozen,
spilled,
shadowed,
modifable
} status = unknown;
enum page_status { unknown, frozen, spilled, shadowed, modifable } status = unknown;
if (unlikely(!mp)) {
if (ASSERT_ENABLED() && pageflags) {
@ -437,8 +410,7 @@ int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno,
check = page_get_any(mc, pgno, txn->front_txnid);
if (unlikely(check.err != MDBX_SUCCESS))
return check.err;
tASSERT(txn, ((unsigned)check.page->flags & ~P_SPILLED) ==
(pageflags & ~P_FROZEN));
tASSERT(txn, ((unsigned)check.page->flags & ~P_SPILLED) == (pageflags & ~P_FROZEN));
tASSERT(txn, !(pageflags & P_FROZEN) || is_frozen(txn, check.page));
}
if (pageflags & P_FROZEN) {
@ -540,8 +512,7 @@ status_done:
/* Возврат страниц в нераспределенный "хвост" БД.
* Содержимое страниц не уничтожается, а для вложенных транзакций граница
* нераспределенного "хвоста" БД сдвигается только при их коммите. */
if (MDBX_ENABLE_REFUND &&
unlikely(pgno + npages == txn->geo.first_unallocated)) {
if (MDBX_ENABLE_REFUND && unlikely(pgno + npages == txn->geo.first_unallocated)) {
const char *kind = nullptr;
if (status == modifable) {
/* Страница испачкана в этой транзакции, но до этого могла быть
@ -589,8 +560,7 @@ status_done:
if (status == modifable) {
/* Dirty page from this transaction */
/* If suitable we can reuse it through loose list */
if (likely(npages == 1 && suitable4loose(txn, pgno)) &&
(di || !txn->tw.dirtylist)) {
if (likely(npages == 1 && suitable4loose(txn, pgno)) && (di || !txn->tw.dirtylist)) {
DEBUG("loosen dirty page %" PRIaPGNO, pgno);
if (MDBX_DEBUG != 0 || unlikely(txn->env->flags & MDBX_PAGEPERTURB))
memset(page_data(mp), -1, txn->env->ps - PAGEHDRSZ);
@ -600,9 +570,7 @@ status_done:
txn->tw.loose_pages = mp;
txn->tw.loose_count++;
#if MDBX_ENABLE_REFUND
txn->tw.loose_refund_wl = (pgno + 2 > txn->tw.loose_refund_wl)
? pgno + 2
: txn->tw.loose_refund_wl;
txn->tw.loose_refund_wl = (pgno + 2 > txn->tw.loose_refund_wl) ? pgno + 2 : txn->tw.loose_refund_wl;
#endif /* MDBX_ENABLE_REFUND */
VALGRIND_MAKE_MEM_NOACCESS(page_data(mp), txn->env->ps - PAGEHDRSZ);
MDBX_ASAN_POISON_MEMORY_REGION(page_data(mp), txn->env->ps - PAGEHDRSZ);
@ -617,9 +585,7 @@ status_done:
* в том числе, позже выгружена и затем снова загружена и изменена.
* В обоих случаях её нельзя затирать на диске и помечать недоступной
* в asan и/или valgrind */
for (MDBX_txn *parent = txn->parent;
parent && (parent->flags & MDBX_TXN_SPILLS);
parent = parent->parent) {
for (MDBX_txn *parent = txn->parent; parent && (parent->flags & MDBX_TXN_SPILLS); parent = parent->parent) {
if (spill_intersect(parent, pgno, npages))
goto skip_invalidate;
if (dpl_intersect(parent, pgno, npages))
@ -631,11 +597,8 @@ status_done:
#endif
page_kill(txn, mp, pgno, npages);
if ((txn->flags & MDBX_WRITEMAP) == 0) {
VALGRIND_MAKE_MEM_NOACCESS(page_data(pgno2page(txn->env, pgno)),
pgno2bytes(txn->env, npages) - PAGEHDRSZ);
MDBX_ASAN_POISON_MEMORY_REGION(page_data(pgno2page(txn->env, pgno)),
pgno2bytes(txn->env, npages) -
PAGEHDRSZ);
VALGRIND_MAKE_MEM_NOACCESS(page_data(pgno2page(txn->env, pgno)), pgno2bytes(txn->env, npages) - PAGEHDRSZ);
MDBX_ASAN_POISON_MEMORY_REGION(page_data(pgno2page(txn->env, pgno)), pgno2bytes(txn->env, npages) - PAGEHDRSZ);
}
}
skip_invalidate:
@ -646,9 +609,7 @@ status_done:
reclaim:
DEBUG("reclaim %zu %s page %" PRIaPGNO, npages, "dirty", pgno);
rc = pnl_insert_span(&txn->tw.relist, pgno, npages);
tASSERT(txn,
pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated -
MDBX_ENABLE_REFUND));
tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND));
tASSERT(txn, dpl_check(txn));
return rc;
}
@ -675,8 +636,7 @@ status_done:
if (ASSERT_ENABLED()) {
const page_t *parent_dp = nullptr;
/* Check parent(s)'s dirty lists. */
for (MDBX_txn *parent = txn->parent; parent && !parent_dp;
parent = parent->parent) {
for (MDBX_txn *parent = txn->parent; parent && !parent_dp; parent = parent->parent) {
tASSERT(txn, !spill_search(parent, pgno));
parent_dp = debug_dpl_find(parent, pgno);
}
@ -697,8 +657,7 @@ status_done:
goto retire;
}
__hot int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp,
size_t npages) {
__hot int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, size_t npages) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
mp->txnid = txn->front_txnid;
if (!txn->tw.dirtylist) {
@ -756,37 +715,27 @@ void recalculate_subpage_thresholds(MDBX_env *env) {
size_t whole = env->leaf_nodemax - NODESIZE;
env->subpage_limit = (whole * env->options.subpage.limit + 32767) >> 16;
whole = env->subpage_limit;
env->subpage_reserve_limit =
(whole * env->options.subpage.reserve_limit + 32767) >> 16;
env->subpage_reserve_limit = (whole * env->options.subpage.reserve_limit + 32767) >> 16;
eASSERT(env, env->leaf_nodemax >= env->subpage_limit + NODESIZE);
eASSERT(env, env->subpage_limit >= env->subpage_reserve_limit);
whole = env->leaf_nodemax;
env->subpage_room_threshold =
(whole * env->options.subpage.room_threshold + 32767) >> 16;
env->subpage_reserve_prereq =
(whole * env->options.subpage.reserve_prereq + 32767) >> 16;
if (env->subpage_room_threshold + env->subpage_reserve_limit >
(intptr_t)page_space(env))
env->subpage_room_threshold = (whole * env->options.subpage.room_threshold + 32767) >> 16;
env->subpage_reserve_prereq = (whole * env->options.subpage.reserve_prereq + 32767) >> 16;
if (env->subpage_room_threshold + env->subpage_reserve_limit > (intptr_t)page_space(env))
env->subpage_reserve_prereq = page_space(env);
else if (env->subpage_reserve_prereq <
env->subpage_room_threshold + env->subpage_reserve_limit)
env->subpage_reserve_prereq =
env->subpage_room_threshold + env->subpage_reserve_limit;
eASSERT(env, env->subpage_reserve_prereq >
env->subpage_room_threshold + env->subpage_reserve_limit);
else if (env->subpage_reserve_prereq < env->subpage_room_threshold + env->subpage_reserve_limit)
env->subpage_reserve_prereq = env->subpage_room_threshold + env->subpage_reserve_limit;
eASSERT(env, env->subpage_reserve_prereq > env->subpage_room_threshold + env->subpage_reserve_limit);
}
size_t page_subleaf2_reserve(const MDBX_env *env, size_t host_page_room,
size_t subpage_len, size_t item_len) {
size_t page_subleaf2_reserve(const MDBX_env *env, size_t host_page_room, size_t subpage_len, size_t item_len) {
eASSERT(env, (subpage_len & 1) == 0);
eASSERT(env, env->leaf_nodemax >= env->subpage_limit + NODESIZE);
size_t reserve = 0;
for (size_t n = 0;
n < 5 && reserve + item_len <= env->subpage_reserve_limit &&
EVEN_CEIL(subpage_len + item_len) <= env->subpage_limit &&
host_page_room >=
env->subpage_reserve_prereq + EVEN_CEIL(subpage_len + item_len);
for (size_t n = 0; n < 5 && reserve + item_len <= env->subpage_reserve_limit &&
EVEN_CEIL(subpage_len + item_len) <= env->subpage_limit &&
host_page_room >= env->subpage_reserve_prereq + EVEN_CEIL(subpage_len + item_len);
++n) {
subpage_len += item_len;
reserve += item_len;

103
src/page-ops.h
View File

@ -5,9 +5,7 @@
#include "essentials.h"
MDBX_INTERNAL int __must_check_result tree_search_finalize(MDBX_cursor *mc,
const MDBX_val *key,
int flags);
MDBX_INTERNAL int __must_check_result tree_search_finalize(MDBX_cursor *mc, const MDBX_val *key, int flags);
MDBX_INTERNAL int tree_search_lowest(MDBX_cursor *mc);
enum page_search_flags {
@ -16,64 +14,47 @@ enum page_search_flags {
Z_FIRST = 4,
Z_LAST = 8,
};
MDBX_INTERNAL int __must_check_result tree_search(MDBX_cursor *mc,
const MDBX_val *key,
int flags);
MDBX_INTERNAL int __must_check_result tree_search(MDBX_cursor *mc, const MDBX_val *key, int flags);
#define MDBX_SPLIT_REPLACE MDBX_APPENDDUP /* newkey is not new */
MDBX_INTERNAL int __must_check_result page_split(MDBX_cursor *mc,
const MDBX_val *const newkey,
MDBX_val *const newdata,
pgno_t newpgno,
const unsigned naf);
MDBX_INTERNAL int __must_check_result page_split(MDBX_cursor *mc, const MDBX_val *const newkey, MDBX_val *const newdata,
pgno_t newpgno, const unsigned naf);
/*----------------------------------------------------------------------------*/
MDBX_INTERNAL int MDBX_PRINTF_ARGS(2, 3)
bad_page(const page_t *mp, const char *fmt, ...);
MDBX_INTERNAL int MDBX_PRINTF_ARGS(2, 3) bad_page(const page_t *mp, const char *fmt, ...);
MDBX_INTERNAL void MDBX_PRINTF_ARGS(2, 3)
poor_page(const page_t *mp, const char *fmt, ...);
MDBX_INTERNAL void MDBX_PRINTF_ARGS(2, 3) poor_page(const page_t *mp, const char *fmt, ...);
MDBX_NOTHROW_PURE_FUNCTION static inline bool is_frozen(const MDBX_txn *txn,
const page_t *mp) {
MDBX_NOTHROW_PURE_FUNCTION static inline bool is_frozen(const MDBX_txn *txn, const page_t *mp) {
return mp->txnid < txn->txnid;
}
MDBX_NOTHROW_PURE_FUNCTION static inline bool is_spilled(const MDBX_txn *txn,
const page_t *mp) {
MDBX_NOTHROW_PURE_FUNCTION static inline bool is_spilled(const MDBX_txn *txn, const page_t *mp) {
return mp->txnid == txn->txnid;
}
MDBX_NOTHROW_PURE_FUNCTION static inline bool is_shadowed(const MDBX_txn *txn,
const page_t *mp) {
MDBX_NOTHROW_PURE_FUNCTION static inline bool is_shadowed(const MDBX_txn *txn, const page_t *mp) {
return mp->txnid > txn->txnid;
}
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool
is_correct(const MDBX_txn *txn, const page_t *mp) {
MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_correct(const MDBX_txn *txn, const page_t *mp) {
return mp->txnid <= txn->front_txnid;
}
MDBX_NOTHROW_PURE_FUNCTION static inline bool is_modifable(const MDBX_txn *txn,
const page_t *mp) {
MDBX_NOTHROW_PURE_FUNCTION static inline bool is_modifable(const MDBX_txn *txn, const page_t *mp) {
return mp->txnid == txn->front_txnid;
}
MDBX_INTERNAL int __must_check_result page_check(const MDBX_cursor *const mc,
const page_t *const mp);
MDBX_INTERNAL int __must_check_result page_check(const MDBX_cursor *const mc, const page_t *const mp);
MDBX_INTERNAL pgr_t page_get_any(const MDBX_cursor *const mc, const pgno_t pgno,
const txnid_t front);
MDBX_INTERNAL pgr_t page_get_any(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front);
MDBX_INTERNAL pgr_t page_get_three(const MDBX_cursor *const mc,
const pgno_t pgno, const txnid_t front);
MDBX_INTERNAL pgr_t page_get_three(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front);
MDBX_INTERNAL pgr_t page_get_large(const MDBX_cursor *const mc,
const pgno_t pgno, const txnid_t front);
MDBX_INTERNAL pgr_t page_get_large(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front);
static inline int __must_check_result page_get(const MDBX_cursor *mc,
const pgno_t pgno, page_t **mp,
static inline int __must_check_result page_get(const MDBX_cursor *mc, const pgno_t pgno, page_t **mp,
const txnid_t front) {
pgr_t ret = page_get_three(mc, pgno, front);
*mp = ret.page;
@ -82,21 +63,18 @@ static inline int __must_check_result page_get(const MDBX_cursor *mc,
/*----------------------------------------------------------------------------*/
MDBX_INTERNAL int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp,
size_t npages);
MDBX_INTERNAL int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, size_t npages);
MDBX_INTERNAL pgr_t page_new(MDBX_cursor *mc, const unsigned flags);
MDBX_INTERNAL pgr_t page_new_large(MDBX_cursor *mc, const size_t npages);
MDBX_INTERNAL int page_touch_modifable(MDBX_txn *txn, const page_t *const mp);
MDBX_INTERNAL int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc,
const page_t *const mp);
MDBX_INTERNAL int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *const mp);
static inline int page_touch(MDBX_cursor *mc) {
page_t *const mp = mc->pg[mc->top];
MDBX_txn *txn = mc->txn;
tASSERT(txn, mc->txn->flags & MDBX_TXN_DIRTY);
tASSERT(txn,
F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY));
tASSERT(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY));
tASSERT(txn, !is_largepage(mp));
if (ASSERT_ENABLED()) {
if (mc->flags & z_inner) {
@ -119,40 +97,31 @@ static inline int page_touch(MDBX_cursor *mc) {
return page_touch_unmodifable(txn, mc, mp);
}
MDBX_INTERNAL void page_copy(page_t *const dst, const page_t *const src,
const size_t size);
MDBX_INTERNAL pgr_t __must_check_result page_unspill(MDBX_txn *const txn,
const page_t *const mp);
MDBX_INTERNAL void page_copy(page_t *const dst, const page_t *const src, const size_t size);
MDBX_INTERNAL pgr_t __must_check_result page_unspill(MDBX_txn *const txn, const page_t *const mp);
MDBX_INTERNAL page_t *page_shadow_alloc(MDBX_txn *txn, size_t num);
MDBX_INTERNAL void page_shadow_release(MDBX_env *env, page_t *dp,
size_t npages);
MDBX_INTERNAL void page_shadow_release(MDBX_env *env, page_t *dp, size_t npages);
MDBX_INTERNAL int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno,
page_t *mp /* maybe null */,
MDBX_INTERNAL int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null */,
unsigned pageflags /* maybe unknown/zero */);
static inline int page_retire(MDBX_cursor *mc, page_t *mp) {
return page_retire_ex(mc, mp->pgno, mp, mp->flags);
}
static inline int page_retire(MDBX_cursor *mc, page_t *mp) { return page_retire_ex(mc, mp->pgno, mp, mp->flags); }
static inline void page_wash(MDBX_txn *txn, size_t di, page_t *const mp,
const size_t npages) {
static inline void page_wash(MDBX_txn *txn, size_t di, page_t *const mp, const size_t npages) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
mp->txnid = INVALID_TXNID;
mp->flags = P_BAD;
if (txn->tw.dirtylist) {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC);
tASSERT(txn,
MDBX_AVOID_MSYNC || (di && txn->tw.dirtylist->items[di].ptr == mp));
tASSERT(txn, MDBX_AVOID_MSYNC || (di && txn->tw.dirtylist->items[di].ptr == mp));
if (!MDBX_AVOID_MSYNC || di) {
dpl_remove_ex(txn, di, npages);
txn->tw.dirtyroom++;
tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
(txn->parent ? txn->parent->tw.dirtyroom
: txn->env->options.dp_limit));
(txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP)) {
page_shadow_release(txn->env, mp, npages);
return;
@ -160,20 +129,14 @@ static inline void page_wash(MDBX_txn *txn, size_t di, page_t *const mp,
}
} else {
tASSERT(txn, (txn->flags & MDBX_WRITEMAP) && !MDBX_AVOID_MSYNC && !di);
txn->tw.writemap_dirty_npages -= (txn->tw.writemap_dirty_npages > npages)
? npages
: txn->tw.writemap_dirty_npages;
txn->tw.writemap_dirty_npages -= (txn->tw.writemap_dirty_npages > npages) ? npages : txn->tw.writemap_dirty_npages;
}
VALGRIND_MAKE_MEM_UNDEFINED(mp, PAGEHDRSZ);
VALGRIND_MAKE_MEM_NOACCESS(page_data(mp),
pgno2bytes(txn->env, npages) - PAGEHDRSZ);
MDBX_ASAN_POISON_MEMORY_REGION(page_data(mp),
pgno2bytes(txn->env, npages) - PAGEHDRSZ);
VALGRIND_MAKE_MEM_NOACCESS(page_data(mp), pgno2bytes(txn->env, npages) - PAGEHDRSZ);
MDBX_ASAN_POISON_MEMORY_REGION(page_data(mp), pgno2bytes(txn->env, npages) - PAGEHDRSZ);
}
MDBX_INTERNAL size_t page_subleaf2_reserve(const MDBX_env *env,
size_t host_page_room,
size_t subpage_len, size_t item_len);
MDBX_INTERNAL size_t page_subleaf2_reserve(const MDBX_env *env, size_t host_page_room, size_t subpage_len,
size_t item_len);
#define page_next(mp) \
(*(page_t **)ptr_disp((mp)->entries, sizeof(void *) - sizeof(uint32_t)))
#define page_next(mp) (*(page_t **)ptr_disp((mp)->entries, sizeof(void *) - sizeof(uint32_t)))

19
src/page-search.c
View File

@ -54,14 +54,11 @@ __hot int tree_search(MDBX_cursor *mc, const MDBX_val *key, int flags) {
cASSERT(mc, root >= NUM_METAS && root < mc->txn->geo.first_unallocated);
if (mc->top < 0 || mc->pg[0]->pgno != root) {
txnid_t pp_txnid = mc->tree->mod_txnid;
pp_txnid = /* tree->mod_txnid maybe zero in a legacy DB */ pp_txnid
? pp_txnid
: mc->txn->txnid;
pp_txnid = /* tree->mod_txnid maybe zero in a legacy DB */ pp_txnid ? pp_txnid : mc->txn->txnid;
if ((mc->txn->flags & MDBX_TXN_RDONLY) == 0) {
MDBX_txn *scan = mc->txn;
do
if ((scan->flags & MDBX_TXN_DIRTY) &&
(dbi == MAIN_DBI || (scan->dbi_state[dbi] & DBI_DIRTY))) {
if ((scan->flags & MDBX_TXN_DIRTY) && (dbi == MAIN_DBI || (scan->dbi_state[dbi] & DBI_DIRTY))) {
/* После коммита вложенных тразакций может быть mod_txnid > front */
pp_txnid = scan->front_txnid;
break;
@ -75,8 +72,7 @@ __hot int tree_search(MDBX_cursor *mc, const MDBX_val *key, int flags) {
mc->top = 0;
mc->ki[0] = (flags & Z_LAST) ? page_numkeys(mc->pg[0]) - 1 : 0;
DEBUG("db %d root page %" PRIaPGNO " has flags 0x%X", cursor_dbi_dbg(mc),
root, mc->pg[0]->flags);
DEBUG("db %d root page %" PRIaPGNO " has flags 0x%X", cursor_dbi_dbg(mc), root, mc->pg[0]->flags);
if (flags & Z_MODIFY) {
err = page_touch(mc);
@ -90,8 +86,7 @@ __hot int tree_search(MDBX_cursor *mc, const MDBX_val *key, int flags) {
return tree_search_finalize(mc, key, flags);
}
__hot __noinline int tree_search_finalize(MDBX_cursor *mc, const MDBX_val *key,
int flags) {
__hot __noinline int tree_search_finalize(MDBX_cursor *mc, const MDBX_val *key, int flags) {
cASSERT(mc, !is_poor(mc));
DKBUF_DEBUG;
int err;
@ -128,16 +123,14 @@ __hot __noinline int tree_search_finalize(MDBX_cursor *mc, const MDBX_val *key,
}
if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) {
ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor",
mp->pgno, mp->flags);
ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags);
err = MDBX_CORRUPTED;
bailout:
be_poor(mc);
return err;
}
DEBUG("found leaf page %" PRIaPGNO " for key [%s]", mp->pgno,
DKEY_DEBUG(key));
DEBUG("found leaf page %" PRIaPGNO " for key [%s]", mp->pgno, DKEY_DEBUG(key));
/* Логически верно, но (в текущем понимании) нет необходимости.
Однако, стоит ещё по-проверять/по-тестировать.
Возможно есть сценарий, в котором очистка флагов всё-таки требуется.

60
src/pnl.c
View File

@ -25,14 +25,11 @@ MDBX_INTERNAL void pnl_free(pnl_t pnl) {
MDBX_INTERNAL void pnl_shrink(pnl_t __restrict *__restrict ppnl) {
assert(pnl_bytes2size(pnl_size2bytes(MDBX_PNL_INITIAL)) >= MDBX_PNL_INITIAL &&
pnl_bytes2size(pnl_size2bytes(MDBX_PNL_INITIAL)) <
MDBX_PNL_INITIAL * 3 / 2);
assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT &&
MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl));
pnl_bytes2size(pnl_size2bytes(MDBX_PNL_INITIAL)) < MDBX_PNL_INITIAL * 3 / 2);
assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl));
MDBX_PNL_SETSIZE(*ppnl, 0);
if (unlikely(MDBX_PNL_ALLOCLEN(*ppnl) >
MDBX_PNL_INITIAL * (MDBX_PNL_PREALLOC_FOR_RADIXSORT ? 8 : 4) -
MDBX_CACHELINE_SIZE / sizeof(pgno_t))) {
MDBX_PNL_INITIAL * (MDBX_PNL_PREALLOC_FOR_RADIXSORT ? 8 : 4) - MDBX_CACHELINE_SIZE / sizeof(pgno_t))) {
size_t bytes = pnl_size2bytes(MDBX_PNL_INITIAL * 2);
pnl_t pnl = osal_realloc(*ppnl - 1, bytes);
if (likely(pnl)) {
@ -45,11 +42,9 @@ MDBX_INTERNAL void pnl_shrink(pnl_t __restrict *__restrict ppnl) {
}
}
MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl,
const size_t wanna) {
MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, const size_t wanna) {
const size_t allocated = MDBX_PNL_ALLOCLEN(*ppnl);
assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT &&
MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl));
assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl));
if (likely(allocated >= wanna))
return MDBX_SUCCESS;
@ -58,9 +53,7 @@ MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl,
return MDBX_TXN_FULL;
}
const size_t size = (wanna + wanna - allocated < PAGELIST_LIMIT)
? wanna + wanna - allocated
: PAGELIST_LIMIT;
const size_t size = (wanna + wanna - allocated < PAGELIST_LIMIT) ? wanna + wanna - allocated : PAGELIST_LIMIT;
size_t bytes = pnl_size2bytes(size);
pnl_t pnl = osal_realloc(*ppnl - 1, bytes);
if (likely(pnl)) {
@ -75,8 +68,8 @@ MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl,
return MDBX_ENOMEM;
}
static __always_inline int __must_check_result pnl_append_stepped(
unsigned step, __restrict pnl_t *ppnl, pgno_t pgno, size_t n) {
static __always_inline int __must_check_result pnl_append_stepped(unsigned step, __restrict pnl_t *ppnl, pgno_t pgno,
size_t n) {
assert(n > 0);
int rc = pnl_need(ppnl, n);
if (unlikely(rc != MDBX_SUCCESS))
@ -106,18 +99,15 @@ static __always_inline int __must_check_result pnl_append_stepped(
return MDBX_SUCCESS;
}
__hot MDBX_INTERNAL int __must_check_result
spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) {
__hot MDBX_INTERNAL int __must_check_result spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) {
return pnl_append_stepped(2, ppnl, pgno << 1, n);
}
__hot MDBX_INTERNAL int __must_check_result
pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) {
__hot MDBX_INTERNAL int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) {
return pnl_append_stepped(1, ppnl, pgno, n);
}
__hot MDBX_INTERNAL int __must_check_result
pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) {
__hot MDBX_INTERNAL int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) {
assert(n > 0);
int rc = pnl_need(ppnl, n);
if (unlikely(rc != MDBX_SUCCESS))
@ -135,8 +125,7 @@ pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) {
return MDBX_SUCCESS;
}
__hot __noinline MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl,
const size_t limit) {
__hot __noinline MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, const size_t limit) {
assert(limit >= MIN_PAGENO - MDBX_ENABLE_REFUND);
if (likely(MDBX_PNL_GETSIZE(pnl))) {
if (unlikely(MDBX_PNL_GETSIZE(pnl) > PAGELIST_LIMIT))
@ -146,8 +135,7 @@ __hot __noinline MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl,
if (unlikely(MDBX_PNL_MOST(pnl) >= limit))
return false;
if ((!MDBX_DISABLE_VALIDATION || AUDIT_ENABLED()) &&
likely(MDBX_PNL_GETSIZE(pnl) > 1)) {
if ((!MDBX_DISABLE_VALIDATION || AUDIT_ENABLED()) && likely(MDBX_PNL_GETSIZE(pnl) > 1)) {
const pgno_t *scan = MDBX_PNL_BEGIN(pnl);
const pgno_t *const end = MDBX_PNL_END(pnl);
pgno_t prev = *scan++;
@ -161,10 +149,9 @@ __hot __noinline MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl,
return true;
}
static __always_inline void
pnl_merge_inner(pgno_t *__restrict dst, const pgno_t *__restrict src_a,
const pgno_t *__restrict src_b,
const pgno_t *__restrict const src_b_detent) {
static __always_inline void pnl_merge_inner(pgno_t *__restrict dst, const pgno_t *__restrict src_a,
const pgno_t *__restrict src_b,
const pgno_t *__restrict const src_b_detent) {
do {
#if MDBX_HAVE_CMOV
const bool flag = MDBX_PNL_ORDERED(*src_b, *src_a);
@ -203,14 +190,11 @@ __hot MDBX_INTERNAL size_t pnl_merge(pnl_t dst, const pnl_t src) {
total += src_len;
if (!MDBX_DEBUG && total < (MDBX_HAVE_CMOV ? 21 : 12))
goto avoid_call_libc_for_short_cases;
if (dst_len == 0 ||
MDBX_PNL_ORDERED(MDBX_PNL_LAST(dst), MDBX_PNL_FIRST(src)))
if (dst_len == 0 || MDBX_PNL_ORDERED(MDBX_PNL_LAST(dst), MDBX_PNL_FIRST(src)))
memcpy(MDBX_PNL_END(dst), MDBX_PNL_BEGIN(src), src_len * sizeof(pgno_t));
else if (MDBX_PNL_ORDERED(MDBX_PNL_LAST(src), MDBX_PNL_FIRST(dst))) {
memmove(MDBX_PNL_BEGIN(dst) + src_len, MDBX_PNL_BEGIN(dst),
dst_len * sizeof(pgno_t));
memcpy(MDBX_PNL_BEGIN(dst), MDBX_PNL_BEGIN(src),
src_len * sizeof(pgno_t));
memmove(MDBX_PNL_BEGIN(dst) + src_len, MDBX_PNL_BEGIN(dst), dst_len * sizeof(pgno_t));
memcpy(MDBX_PNL_BEGIN(dst), MDBX_PNL_BEGIN(src), src_len * sizeof(pgno_t));
} else {
avoid_call_libc_for_short_cases:
dst[0] = /* the detent */ (MDBX_PNL_ASCENDING ? 0 : P_INVALID);
@ -227,8 +211,7 @@ __hot MDBX_INTERNAL size_t pnl_merge(pnl_t dst, const pnl_t src) {
#else
#define MDBX_PNL_EXTRACT_KEY(ptr) (P_INVALID - *(ptr))
#endif
RADIXSORT_IMPL(pgno, pgno_t, MDBX_PNL_EXTRACT_KEY,
MDBX_PNL_PREALLOC_FOR_RADIXSORT, 0)
RADIXSORT_IMPL(pgno, pgno_t, MDBX_PNL_EXTRACT_KEY, MDBX_PNL_PREALLOC_FOR_RADIXSORT, 0)
SORT_IMPL(pgno_sort, false, pgno_t, MDBX_PNL_ORDERED)
@ -240,8 +223,7 @@ __hot __noinline MDBX_INTERNAL void pnl_sort_nochk(pnl_t pnl) {
SEARCH_IMPL(pgno_bsearch, pgno_t, pgno_t, MDBX_PNL_ORDERED)
__hot __noinline MDBX_INTERNAL size_t pnl_search_nochk(const pnl_t pnl,
pgno_t pgno) {
__hot __noinline MDBX_INTERNAL size_t pnl_search_nochk(const pnl_t pnl, pgno_t pgno) {
const pgno_t *begin = MDBX_PNL_BEGIN(pnl);
const pgno_t *it = pgno_bsearch(begin, MDBX_PNL_GETSIZE(pnl), pgno);
const pgno_t *end = begin + MDBX_PNL_GETSIZE(pnl);

53
src/pnl.h
View File

@ -26,16 +26,15 @@ typedef const pgno_t *const_pnl_t;
#define MDBX_PNL_GRANULATE_LOG2 10
#define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2)
#define MDBX_PNL_INITIAL \
(MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t))
#define MDBX_PNL_INITIAL (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t))
#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1])
#define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0]))
#define MDBX_PNL_SETSIZE(pl, size) \
do { \
const size_t __size = size; \
assert(__size < INT_MAX); \
(pl)[0] = (pgno_t)__size; \
#define MDBX_PNL_SETSIZE(pl, size) \
do { \
const size_t __size = size; \
assert(__size < INT_MAX); \
(pl)[0] = (pgno_t)__size; \
} while (0)
#define MDBX_PNL_FIRST(pl) ((pl)[1])
#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)])
@ -62,13 +61,10 @@ MDBX_MAYBE_UNUSED static inline size_t pnl_size2bytes(size_t size) {
size += size;
#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */
STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD +
(PAGELIST_LIMIT * (MDBX_PNL_PREALLOC_FOR_RADIXSORT + 1) +
MDBX_PNL_GRANULATE + 3) *
sizeof(pgno_t) <
(PAGELIST_LIMIT * (MDBX_PNL_PREALLOC_FOR_RADIXSORT + 1) + MDBX_PNL_GRANULATE + 3) * sizeof(pgno_t) <
SIZE_MAX / 4 * 3);
size_t bytes =
ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(pgno_t) * (size + 3),
MDBX_PNL_GRANULATE * sizeof(pgno_t)) -
ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(pgno_t) * (size + 3), MDBX_PNL_GRANULATE * sizeof(pgno_t)) -
MDBX_ASSUME_MALLOC_OVERHEAD;
return bytes;
}
@ -87,21 +83,16 @@ MDBX_INTERNAL pnl_t pnl_alloc(size_t size);
MDBX_INTERNAL void pnl_free(pnl_t pnl);
MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl,
const size_t wanna);
MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, const size_t wanna);
MDBX_MAYBE_UNUSED static inline int __must_check_result
pnl_need(pnl_t __restrict *__restrict ppnl, size_t num) {
assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT &&
MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl));
MDBX_MAYBE_UNUSED static inline int __must_check_result pnl_need(pnl_t __restrict *__restrict ppnl, size_t num) {
assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl));
assert(num <= PAGELIST_LIMIT);
const size_t wanna = MDBX_PNL_GETSIZE(*ppnl) + num;
return likely(MDBX_PNL_ALLOCLEN(*ppnl) >= wanna) ? MDBX_SUCCESS
: pnl_reserve(ppnl, wanna);
return likely(MDBX_PNL_ALLOCLEN(*ppnl) >= wanna) ? MDBX_SUCCESS : pnl_reserve(ppnl, wanna);
}
MDBX_MAYBE_UNUSED static inline void
pnl_append_prereserved(__restrict pnl_t pnl, pgno_t pgno) {
MDBX_MAYBE_UNUSED static inline void pnl_append_prereserved(__restrict pnl_t pnl, pgno_t pgno) {
assert(MDBX_PNL_GETSIZE(pnl) < MDBX_PNL_ALLOCLEN(pnl));
if (AUDIT_ENABLED()) {
for (size_t i = MDBX_PNL_GETSIZE(pnl); i > 0; --i)
@ -113,14 +104,11 @@ pnl_append_prereserved(__restrict pnl_t pnl, pgno_t pgno) {
MDBX_INTERNAL void pnl_shrink(pnl_t __restrict *__restrict ppnl);
MDBX_INTERNAL int __must_check_result spill_append_span(__restrict pnl_t *ppnl,
pgno_t pgno, size_t n);
MDBX_INTERNAL int __must_check_result spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n);
MDBX_INTERNAL int __must_check_result pnl_append_span(__restrict pnl_t *ppnl,
pgno_t pgno, size_t n);
MDBX_INTERNAL int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n);
MDBX_INTERNAL int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl,
pgno_t pgno, size_t n);
MDBX_INTERNAL int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n);
MDBX_INTERNAL size_t pnl_search_nochk(const pnl_t pnl, pgno_t pgno);
@ -128,10 +116,8 @@ MDBX_INTERNAL void pnl_sort_nochk(pnl_t pnl);
MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, const size_t limit);
MDBX_MAYBE_UNUSED static inline bool pnl_check_allocated(const const_pnl_t pnl,
const size_t limit) {
return pnl == nullptr || (MDBX_PNL_ALLOCLEN(pnl) >= MDBX_PNL_GETSIZE(pnl) &&
pnl_check(pnl, limit));
MDBX_MAYBE_UNUSED static inline bool pnl_check_allocated(const const_pnl_t pnl, const size_t limit) {
return pnl == nullptr || (MDBX_PNL_ALLOCLEN(pnl) >= MDBX_PNL_GETSIZE(pnl) && pnl_check(pnl, limit));
}
MDBX_MAYBE_UNUSED static inline void pnl_sort(pnl_t pnl, size_t limit4check) {
@ -140,8 +126,7 @@ MDBX_MAYBE_UNUSED static inline void pnl_sort(pnl_t pnl, size_t limit4check) {
(void)limit4check;
}
MDBX_MAYBE_UNUSED static inline size_t pnl_search(const pnl_t pnl, pgno_t pgno,
size_t limit) {
MDBX_MAYBE_UNUSED static inline size_t pnl_search(const pnl_t pnl, pgno_t pgno, size_t limit) {
assert(pnl_check_allocated(pnl, limit));
if (MDBX_HAVE_CMOV) {
/* cmov-ускоренный бинарный поиск может читать (но не использовать) один

244
src/preface.h
View File

@ -4,8 +4,7 @@
#pragma once
/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || \
(defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS)
#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS)
#undef NDEBUG
#ifndef MDBX_DEBUG
/* Чтобы избежать включения отладки только из-за включения assert-проверок */
@ -29,8 +28,7 @@
#endif /* MDBX_DISABLE_GNU_SOURCE */
/* Should be defined before any includes */
#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \
!defined(ANDROID)
#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && !defined(ANDROID)
#define _FILE_OFFSET_BITS 64
#endif /* _FILE_OFFSET_BITS */
@ -38,8 +36,7 @@
#define _DARWIN_C_SOURCE
#endif /* _DARWIN_C_SOURCE */
#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \
!defined(__USE_MINGW_ANSI_STDIO)
#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO)
#define __USE_MINGW_ANSI_STDIO 1
#endif /* MinGW */
@ -56,8 +53,7 @@
#define UNICODE
#endif /* UNICODE */
#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \
!defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT
#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT
#define _NO_CRT_STDIO_INLINE
#endif /* _NO_CRT_STDIO_INLINE */
@ -72,8 +68,7 @@
#endif /* NOMINMAX */
/* Workaround for modern libstdc++ with CLANG < 4.x */
#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && \
defined(__clang__) && __clang_major__ < 4
#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && defined(__clang__) && __clang_major__ < 4
#define __GLIBCXX_BITSIZE_INT_N_0 128
#define __GLIBCXX_TYPE_INT_N_0 __int128
#endif /* Workaround for modern libstdc++ with CLANG < 4.x */
@ -107,8 +102,7 @@
* and how to and where you can obtain the latest "Visual Studio 2015" build
* with all fixes.
*/
#error \
"At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required."
#error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required."
#endif
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
@ -117,9 +111,8 @@
#pragma warning(disable : 5045) /* will insert Spectre mitigation... */
#endif
#if _MSC_VER > 1914
#pragma warning( \
disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \
producing 'defined' has undefined behavior */
#pragma warning(disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \
producing 'defined' has undefined behavior */
#endif
#if _MSC_VER < 1920
/* avoid "error C2219: syntax error: type qualifier must be after '*'" */
@ -127,33 +120,32 @@
#endif
#if _MSC_VER > 1930
#pragma warning(disable : 6235) /* <expression> is always a constant */
#pragma warning(disable : 6237) /* <expression> is never evaluated and might \
#pragma warning(disable : 6237) /* <expression> is never evaluated and might \
have side effects */
#endif
#pragma warning(disable : 4710) /* 'xyz': function not inlined */
#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \
#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \
inline expansion */
#pragma warning(disable : 4201) /* nonstandard extension used: nameless \
#pragma warning(disable : 4201) /* nonstandard extension used: nameless \
struct/union */
#pragma warning(disable : 4702) /* unreachable code */
#pragma warning(disable : 4706) /* assignment within conditional expression */
#pragma warning(disable : 4127) /* conditional expression is constant */
#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \
#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \
alignment specifier */
#pragma warning(disable : 4310) /* cast truncates constant value */
#pragma warning(disable : 4820) /* bytes padding added after data member for \
#pragma warning(disable : 4820) /* bytes padding added after data member for \
alignment */
#pragma warning(disable : 4548) /* expression before comma has no effect; \
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \
#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \
unaligned */
#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \
#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \
array in struct/union */
#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \
#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \
aggregate initializer */
#pragma warning( \
disable : 4505) /* unreferenced local function has been removed */
#endif /* _MSC_VER (warnings) */
#pragma warning(disable : 4505) /* unreferenced local function has been removed */
#endif /* _MSC_VER (warnings) */
#if defined(__GNUC__) && __GNUC__ < 9
#pragma GCC diagnostic ignored "-Wattributes"
@ -166,12 +158,12 @@
#ifdef _MSC_VER
#pragma warning(push, 1)
#pragma warning(disable : 4548) /* expression before comma has no effect; \
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
* semantics are not enabled. Specify /EHsc */
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
* mode specified; termination on exception is \
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
* mode specified; termination on exception is \
* not guaranteed. Specify /EHsc */
#endif /* _MSC_VER (warnings) */
@ -232,8 +224,7 @@
#ifndef __GNUC_PREREQ
#if defined(__GNUC__) && defined(__GNUC_MINOR__)
#define __GNUC_PREREQ(maj, min) \
((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
#define __GNUC_PREREQ(maj, min) ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
#else
#define __GNUC_PREREQ(maj, min) (0)
#endif
@ -241,8 +232,7 @@
#ifndef __CLANG_PREREQ
#ifdef __clang__
#define __CLANG_PREREQ(maj, min) \
((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
#define __CLANG_PREREQ(maj, min) ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
#else
#define __CLANG_PREREQ(maj, min) (0)
#endif
@ -250,8 +240,7 @@
#ifndef __GLIBC_PREREQ
#if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
#define __GLIBC_PREREQ(maj, min) \
((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
#define __GLIBC_PREREQ(maj, min) ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
#else
#define __GLIBC_PREREQ(maj, min) (0)
#endif
@ -261,8 +250,7 @@
/* pre-requirements */
#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
#error \
"Sanity checking failed: Two's complement, reasonably sized integer types"
#error "Sanity checking failed: Two's complement, reasonably sized integer types"
#endif
#ifndef SSIZE_MAX
@ -294,8 +282,7 @@
#endif
#ifdef __SANITIZE_THREAD__
#warning \
"libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues."
#warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues."
#endif /* __SANITIZE_THREAD__ */
/*----------------------------------------------------------------------------*/
@ -327,8 +314,7 @@
#endif
#endif /* __extern_C */
#if !defined(nullptr) && !defined(__cplusplus) || \
(__cplusplus < 201103L && !defined(_MSC_VER))
#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
#define nullptr NULL
#endif
@ -340,9 +326,8 @@
#endif
#endif /* Apple OSX & iOS */
#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \
defined(__APPLE__) || defined(__MACH__)
#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \
defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__)
#include <sys/cdefs.h>
#include <sys/mount.h>
#include <sys/sysctl.h>
@ -359,8 +344,7 @@
#endif
#else
#include <malloc.h>
#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \
defined(_WIN32) || defined(_WIN64))
#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(_WIN32) || defined(_WIN64))
#include <mntent.h>
#endif /* !Solaris */
#endif /* !xBSD */
@ -469,43 +453,38 @@ __extern_C key_t ftok(const char *, int);
/*----------------------------------------------------------------------------*/
/* Byteorder */
#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
defined(i486) || defined(__i486) || defined(__i486__) || defined(i586) || \
defined(__i586) || defined(__i586__) || defined(i686) || \
defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || defined(i486) || defined(__i486) || \
defined(__i486__) || defined(i586) || defined(__i586) || defined(__i586__) || defined(i686) || defined(__i686) || \
defined(__i686__) || defined(_M_IX86) || defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \
defined(_M_X64) || defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
#ifndef __ia32__
/* LY: define neutral __ia32__ for x86 and x86-64 */
#define __ia32__ 1
#endif /* __ia32__ */
#if !defined(__amd64__) && \
(defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || \
defined(_M_X64) || defined(_M_AMD64))
#if !defined(__amd64__) && \
(defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64) || defined(_M_AMD64))
/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
#define __amd64__ 1
#endif /* __amd64__ */
#endif /* all x86 */
#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
!defined(__ORDER_BIG_ENDIAN__)
#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || !defined(__ORDER_BIG_ENDIAN__)
#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || \
defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || \
__has_include(<endian.h>)
#include <endian.h>
#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || defined(HAVE_MACHINE_ENDIAN_H) || \
__has_include(<machine/endian.h>)
#include <machine/endian.h>
#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
#include <sys/isa_defs.h>
#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
(__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
#include <sys/endian.h>
#include <sys/types.h>
#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
defined(__NetBSD__) || defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || \
defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
#include <sys/param.h>
#endif /* OS */
@ -521,27 +500,19 @@ __extern_C key_t ftok(const char *, int);
#define __ORDER_LITTLE_ENDIAN__ 1234
#define __ORDER_BIG_ENDIAN__ 4321
#if defined(__LITTLE_ENDIAN__) || \
(defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
defined(__WINDOWS__)
#if defined(__LITTLE_ENDIAN__) || (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || defined(__ARMEL__) || \
defined(__THUMBEL__) || defined(__AARCH64EL__) || defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || defined(__elbrus_4c__) || defined(__elbrus_8c__) || \
defined(__bfin__) || defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || defined(__IA64__) || \
defined(__ia64) || defined(_M_IA64) || defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || defined(__WINDOWS__)
#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
#elif defined(__BIG_ENDIAN__) || \
(defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
#elif defined(__BIG_ENDIAN__) || (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || defined(__ARMEB__) || \
defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
defined(__m68k__) || defined(M68000) || defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \
defined(__sparc__) || defined(__sparc) || defined(__370__) || defined(__THW_370__) || defined(__s390__) || \
defined(__s390x__) || defined(__SYSC_ZARCH__)
#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
#else
@ -561,17 +532,14 @@ __extern_C key_t ftok(const char *, int);
#define MDBX_HAVE_CMOV 1
#elif defined(__thumb__) || defined(__thumb) || defined(__TARGET_ARCH_THUMB)
#define MDBX_HAVE_CMOV 0
#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || \
defined(__aarch64) || defined(__arm__) || defined(__arm) || \
defined(__CC_ARM)
#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || defined(__aarch64) || defined(__arm__) || \
defined(__arm) || defined(__CC_ARM)
#define MDBX_HAVE_CMOV 1
#elif (defined(__riscv__) || defined(__riscv64)) && \
(defined(__riscv_b) || defined(__riscv_bitmanip))
#elif (defined(__riscv__) || defined(__riscv64)) && (defined(__riscv_b) || defined(__riscv_bitmanip))
#define MDBX_HAVE_CMOV 1
#elif defined(i686) || defined(__i686) || defined(__i686__) || \
(defined(_M_IX86) && _M_IX86 > 600) || defined(__x86_64) || \
defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \
defined(_M_X64) || defined(_M_AMD64)
#elif defined(i686) || defined(__i686) || defined(__i686__) || (defined(_M_IX86) && _M_IX86 > 600) || \
defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
defined(_M_AMD64)
#define MDBX_HAVE_CMOV 1
#else
#define MDBX_HAVE_CMOV 0
@ -597,8 +565,7 @@ __extern_C key_t ftok(const char *, int);
#endif
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
#include <mbarrier.h>
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64))
#include <machine/sys/inline.h>
#elif defined(__IBMC__) && defined(__powerpc)
#include <atomic.h>
@ -620,29 +587,26 @@ __extern_C key_t ftok(const char *, int);
#endif /* Compiler */
#if !defined(__noop) && !defined(_MSC_VER)
#define __noop \
do { \
#define __noop \
do { \
} while (0)
#endif /* __noop */
#if defined(__fallthrough) && \
(defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
#if defined(__fallthrough) && (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
#undef __fallthrough
#endif /* __fallthrough workaround for MinGW */
#ifndef __fallthrough
#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \
(!defined(__clang__) || __clang__ > 4)) || \
#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && (!defined(__clang__) || __clang__ > 4)) || \
__cplusplus >= 201703L
#define __fallthrough [[fallthrough]]
#elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L
#define __fallthrough [[fallthrough]]
#elif __GNUC_PREREQ(7, 0) && \
(!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \
(__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126))
#elif __GNUC_PREREQ(7, 0) && (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \
(__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126))
#define __fallthrough __attribute__((__fallthrough__))
#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && \
__has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough")
#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && __has_feature(cxx_attributes) && \
__has_warning("-Wimplicit-fallthrough")
#define __fallthrough [[clang::fallthrough]]
#else
#define __fallthrough
@ -655,8 +619,8 @@ __extern_C key_t ftok(const char *, int);
#elif defined(_MSC_VER)
#define __unreachable() __assume(0)
#else
#define __unreachable() \
do { \
#define __unreachable() \
do { \
} while (1)
#endif
#endif /* __unreachable */
@ -665,9 +629,9 @@ __extern_C key_t ftok(const char *, int);
#if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch)
#define __prefetch(ptr) __builtin_prefetch(ptr)
#else
#define __prefetch(ptr) \
do { \
(void)(ptr); \
#define __prefetch(ptr) \
do { \
(void)(ptr); \
} while (0)
#endif
#endif /* __prefetch */
@ -677,8 +641,7 @@ __extern_C key_t ftok(const char *, int);
#endif /* offsetof */
#ifndef container_of
#define container_of(ptr, type, member) \
((type *)((char *)(ptr) - offsetof(type, member)))
#define container_of(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member)))
#endif /* container_of */
/*----------------------------------------------------------------------------*/
@ -750,8 +713,7 @@ __extern_C key_t ftok(const char *, int);
#ifndef __hot
#if defined(__OPTIMIZE__)
#if defined(__clang__) && !__has_attribute(__hot__) && \
__has_attribute(__section__) && \
#if defined(__clang__) && !__has_attribute(__hot__) && __has_attribute(__section__) && \
(defined(__linux__) || defined(__gnu_linux__))
/* just put frequently used functions in separate section */
#define __hot __attribute__((__section__("text.hot"))) __optimize("O3")
@ -767,8 +729,7 @@ __extern_C key_t ftok(const char *, int);
#ifndef __cold
#if defined(__OPTIMIZE__)
#if defined(__clang__) && !__has_attribute(__cold__) && \
__has_attribute(__section__) && \
#if defined(__clang__) && !__has_attribute(__cold__) && __has_attribute(__section__) && \
(defined(__linux__) || defined(__gnu_linux__))
/* just put infrequently used functions in separate section */
#define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os")
@ -791,8 +752,7 @@ __extern_C key_t ftok(const char *, int);
#endif /* __flatten */
#ifndef likely
#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \
!defined(__COVERITY__)
#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
#define likely(cond) __builtin_expect(!!(cond), 1)
#else
#define likely(x) (!!(x))
@ -800,8 +760,7 @@ __extern_C key_t ftok(const char *, int);
#endif /* likely */
#ifndef unlikely
#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \
!defined(__COVERITY__)
#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__)
#define unlikely(cond) __builtin_expect(!!(cond), 0)
#else
#define unlikely(x) (!!(x))
@ -821,8 +780,7 @@ __extern_C key_t ftok(const char *, int);
#define MDBX_WEAK_IMPORT_ATTRIBUTE WEAK_IMPORT_ATTRIBUTE
#elif __has_attribute(__weak__) && __has_attribute(__weak_import__)
#define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__, __weak_import__))
#elif __has_attribute(__weak__) || \
(defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__))
#elif __has_attribute(__weak__) || (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__))
#define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__))
#else
#define MDBX_WEAK_IMPORT_ATTRIBUTE
@ -835,9 +793,7 @@ __extern_C key_t ftok(const char *, int);
#ifndef MDBX_EXCLUDE_FOR_GPROF
#ifdef ENABLE_GPROF
#define MDBX_EXCLUDE_FOR_GPROF \
__attribute__((__no_instrument_function__, \
__no_profile_instrument_function__))
#define MDBX_EXCLUDE_FOR_GPROF __attribute__((__no_instrument_function__, __no_profile_instrument_function__))
#else
#define MDBX_EXCLUDE_FOR_GPROF
#endif /* ENABLE_GPROF */
@ -846,10 +802,9 @@ __extern_C key_t ftok(const char *, int);
/*----------------------------------------------------------------------------*/
#ifndef expect_with_probability
#if defined(__builtin_expect_with_probability) || \
__has_builtin(__builtin_expect_with_probability) || __GNUC_PREREQ(9, 0)
#define expect_with_probability(expr, value, prob) \
__builtin_expect_with_probability(expr, value, prob)
#if defined(__builtin_expect_with_probability) || __has_builtin(__builtin_expect_with_probability) || \
__GNUC_PREREQ(9, 0)
#define expect_with_probability(expr, value, prob) __builtin_expect_with_probability(expr, value, prob)
#else
#define expect_with_probability(expr, value, prob) (expr)
#endif
@ -866,11 +821,9 @@ __extern_C key_t ftok(const char *, int);
#if MDBX_GOOFY_MSVC_STATIC_ANALYZER || (defined(_MSC_VER) && _MSC_VER > 1919)
#define MDBX_ANALYSIS_ASSUME(expr) __analysis_assume(expr)
#ifdef _PREFAST_
#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \
__pragma(prefast(suppress : warn_id))
#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(prefast(suppress : warn_id))
#else
#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \
__pragma(warning(suppress : warn_id))
#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(warning(suppress : warn_id))
#endif
#else
#define MDBX_ANALYSIS_ASSUME(expr) assert(expr)
@ -878,8 +831,7 @@ __extern_C key_t ftok(const char *, int);
#endif /* MDBX_GOOFY_MSVC_STATIC_ANALYZER */
#ifndef FLEXIBLE_ARRAY_MEMBERS
#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \
(!defined(__cplusplus) && defined(_MSC_VER))
#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || (!defined(__cplusplus) && defined(_MSC_VER))
#define FLEXIBLE_ARRAY_MEMBERS 1
#else
#define FLEXIBLE_ARRAY_MEMBERS 0
@ -938,8 +890,7 @@ template <typename T, size_t N> char (&__ArraySizeHelper(T (&array)[N]))[N];
#define CONCAT(a, b) a##b
#define XCONCAT(a, b) CONCAT(a, b)
#define MDBX_TETRAD(a, b, c, d) \
((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
#define MDBX_TETRAD(a, b, c, d) ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3])
@ -953,14 +904,13 @@ template <typename T, size_t N> char (&__ArraySizeHelper(T (&array)[N]))[N];
#elif defined(_MSC_VER)
#include <crtdbg.h>
#define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \
__has_feature(c_static_assert)
#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __has_feature(c_static_assert)
#define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg)
#else
#define STATIC_ASSERT_MSG(expr, msg) \
switch (0) { \
case 0: \
case (expr):; \
#define STATIC_ASSERT_MSG(expr, msg) \
switch (0) { \
case 0: \
case (expr):; \
}
#endif
#endif /* STATIC_ASSERT */

69
src/proto.h
View File

@ -8,39 +8,26 @@
/* Internal prototypes */
/* audit.c */
MDBX_INTERNAL int audit_ex(MDBX_txn *txn, size_t retired_stored,
bool dont_filter_gc);
MDBX_INTERNAL int audit_ex(MDBX_txn *txn, size_t retired_stored, bool dont_filter_gc);
/* mvcc-readers.c */
MDBX_INTERNAL bsr_t mvcc_bind_slot(MDBX_env *env);
MDBX_MAYBE_UNUSED MDBX_INTERNAL pgno_t mvcc_largest_this(MDBX_env *env,
pgno_t largest);
MDBX_INTERNAL txnid_t mvcc_shapshot_oldest(MDBX_env *const env,
const txnid_t steady);
MDBX_INTERNAL pgno_t mvcc_snapshot_largest(const MDBX_env *env,
pgno_t last_used_page);
MDBX_INTERNAL txnid_t mvcc_kick_laggards(MDBX_env *env,
const txnid_t straggler);
MDBX_MAYBE_UNUSED MDBX_INTERNAL pgno_t mvcc_largest_this(MDBX_env *env, pgno_t largest);
MDBX_INTERNAL txnid_t mvcc_shapshot_oldest(MDBX_env *const env, const txnid_t steady);
MDBX_INTERNAL pgno_t mvcc_snapshot_largest(const MDBX_env *env, pgno_t last_used_page);
MDBX_INTERNAL txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler);
MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rlocked, int *dead);
MDBX_INTERNAL txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t laggard);
/* dxb.c */
MDBX_INTERNAL int dxb_setup(MDBX_env *env, const int lck_rc,
const mdbx_mode_t mode_bits);
MDBX_INTERNAL int __must_check_result
dxb_read_header(MDBX_env *env, meta_t *meta, const int lck_exclusive,
const mdbx_mode_t mode_bits);
MDBX_INTERNAL int dxb_setup(MDBX_env *env, const int lck_rc, const mdbx_mode_t mode_bits);
MDBX_INTERNAL int __must_check_result dxb_read_header(MDBX_env *env, meta_t *meta, const int lck_exclusive,
const mdbx_mode_t mode_bits);
enum resize_mode { implicit_grow, impilict_shrink, explicit_resize };
MDBX_INTERNAL int __must_check_result dxb_resize(MDBX_env *const env,
const pgno_t used_pgno,
const pgno_t size_pgno,
pgno_t limit_pgno,
const enum resize_mode mode);
MDBX_INTERNAL int dxb_set_readahead(const MDBX_env *env, const pgno_t edge,
const bool enable, const bool force_whole);
MDBX_INTERNAL int __must_check_result dxb_sync_locked(MDBX_env *env,
unsigned flags,
meta_t *const pending,
MDBX_INTERNAL int __must_check_result dxb_resize(MDBX_env *const env, const pgno_t used_pgno, const pgno_t size_pgno,
pgno_t limit_pgno, const enum resize_mode mode);
MDBX_INTERNAL int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, const bool enable, const bool force_whole);
MDBX_INTERNAL int __must_check_result dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending,
troika_t *const troika);
#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__)
MDBX_INTERNAL void dxb_sanitize_tail(MDBX_env *env, MDBX_txn *txn);
@ -60,9 +47,8 @@ MDBX_INTERNAL int txn_park(MDBX_txn *txn, bool autounpark);
MDBX_INTERNAL int txn_unpark(MDBX_txn *txn);
MDBX_INTERNAL int txn_check_badbits_parked(const MDBX_txn *txn, int bad_bits);
#define TXN_END_NAMES \
{"committed", "empty-commit", "abort", "reset", \
"fail-begin", "fail-beginchild", "ousted", nullptr}
#define TXN_END_NAMES \
{"committed", "empty-commit", "abort", "reset", "fail-begin", "fail-beginchild", "ousted", nullptr}
enum {
/* txn_end operation number, for logging */
TXN_END_COMMITTED,
@ -84,8 +70,7 @@ MDBX_INTERNAL int txn_write(MDBX_txn *txn, iov_ctx_t *ctx);
/* env.c */
MDBX_INTERNAL int env_open(MDBX_env *env, mdbx_mode_t mode);
MDBX_INTERNAL int env_info(const MDBX_env *env, const MDBX_txn *txn,
MDBX_envinfo *out, size_t bytes, troika_t *troika);
MDBX_INTERNAL int env_info(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out, size_t bytes, troika_t *troika);
MDBX_INTERNAL int env_sync(MDBX_env *env, bool force, bool nonblock);
MDBX_INTERNAL int env_close(MDBX_env *env, bool resurrect_after_fork);
MDBX_INTERNAL bool env_txn0_owned(const MDBX_env *env);
@ -97,27 +82,17 @@ MDBX_INTERNAL unsigned env_setup_pagesize(MDBX_env *env, const size_t pagesize);
/* tree.c */
MDBX_INTERNAL int tree_drop(MDBX_cursor *mc, const bool may_have_tables);
MDBX_INTERNAL int __must_check_result tree_rebalance(MDBX_cursor *mc);
MDBX_INTERNAL int __must_check_result tree_propagate_key(MDBX_cursor *mc,
const MDBX_val *key);
MDBX_INTERNAL int __must_check_result tree_propagate_key(MDBX_cursor *mc, const MDBX_val *key);
MDBX_INTERNAL void recalculate_merge_thresholds(MDBX_env *env);
MDBX_INTERNAL void recalculate_subpage_thresholds(MDBX_env *env);
/* table.c */
MDBX_INTERNAL int __must_check_result tbl_fetch(MDBX_txn *txn, size_t dbi);
MDBX_INTERNAL int __must_check_result tbl_setup(const MDBX_env *env,
kvx_t *const kvx,
const tree_t *const db);
MDBX_INTERNAL int __must_check_result tbl_setup(const MDBX_env *env, kvx_t *const kvx, const tree_t *const db);
/* coherency.c */
MDBX_INTERNAL bool coherency_check_meta(const MDBX_env *env,
const volatile meta_t *meta,
bool report);
MDBX_INTERNAL int coherency_fetch_head(MDBX_txn *txn, const meta_ptr_t head,
uint64_t *timestamp);
MDBX_INTERNAL int coherency_check_written(const MDBX_env *env,
const txnid_t txnid,
const volatile meta_t *meta,
const intptr_t pgno,
uint64_t *timestamp);
MDBX_INTERNAL int coherency_timeout(uint64_t *timestamp, intptr_t pgno,
const MDBX_env *env);
MDBX_INTERNAL bool coherency_check_meta(const MDBX_env *env, const volatile meta_t *meta, bool report);
MDBX_INTERNAL int coherency_fetch_head(MDBX_txn *txn, const meta_ptr_t head, uint64_t *timestamp);
MDBX_INTERNAL int coherency_check_written(const MDBX_env *env, const txnid_t txnid, const volatile meta_t *meta,
const intptr_t pgno, uint64_t *timestamp);
MDBX_INTERNAL int coherency_timeout(uint64_t *timestamp, intptr_t pgno, const MDBX_env *env);

95
src/range-estimate.c
View File

@ -10,20 +10,17 @@ typedef struct diff_result {
} diff_t;
/* calculates: r = x - y */
__hot static int cursor_diff(const MDBX_cursor *const __restrict x,
const MDBX_cursor *const __restrict y,
__hot static int cursor_diff(const MDBX_cursor *const __restrict x, const MDBX_cursor *const __restrict y,
diff_t *const __restrict r) {
r->diff = 0;
r->level = 0;
r->root_nkeys = 0;
if (unlikely(x->signature != cur_signature_live))
return (x->signature == cur_signature_ready4dispose) ? MDBX_EINVAL
: MDBX_EBADSIGN;
return (x->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN;
if (unlikely(y->signature != cur_signature_live))
return (y->signature == cur_signature_ready4dispose) ? MDBX_EINVAL
: MDBX_EBADSIGN;
return (y->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN;
int rc = check_txn(x->txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
@ -86,8 +83,7 @@ __hot static int cursor_diff(const MDBX_cursor *const __restrict x,
return MDBX_SUCCESS;
}
__hot static ptrdiff_t estimate(const tree_t *tree,
diff_t *const __restrict dr) {
__hot static ptrdiff_t estimate(const tree_t *tree, diff_t *const __restrict dr) {
/* root: branch-page => scale = leaf-factor * branch-factor^(N-1)
* level-1: branch-page(s) => scale = leaf-factor * branch-factor^2
* level-2: branch-page(s) => scale = leaf-factor * branch-factor
@ -98,8 +94,7 @@ __hot static ptrdiff_t estimate(const tree_t *tree,
if (btree_power < 0)
return dr->diff;
ptrdiff_t estimated =
(ptrdiff_t)tree->items * dr->diff / (ptrdiff_t)tree->leaf_pages;
ptrdiff_t estimated = (ptrdiff_t)tree->items * dr->diff / (ptrdiff_t)tree->leaf_pages;
if (btree_power == 0)
return estimated;
@ -112,9 +107,7 @@ __hot static ptrdiff_t estimate(const tree_t *tree,
total(branch_entries) = leaf_pages + branch_pages - 1 (root page) */
const size_t log2_fixedpoint = sizeof(size_t) - 1;
const size_t half = UINT64_C(1) << (log2_fixedpoint - 1);
const size_t factor =
((tree->leaf_pages + tree->branch_pages - 1) << log2_fixedpoint) /
tree->branch_pages;
const size_t factor = ((tree->leaf_pages + tree->branch_pages - 1) << log2_fixedpoint) / tree->branch_pages;
while (1) {
switch ((size_t)btree_power) {
default: {
@ -149,11 +142,8 @@ __hot static ptrdiff_t estimate(const tree_t *tree,
}
}
__hot int mdbx_estimate_distance(const MDBX_cursor *first,
const MDBX_cursor *last,
ptrdiff_t *distance_items) {
if (unlikely(first == nullptr || last == nullptr ||
distance_items == nullptr))
__hot int mdbx_estimate_distance(const MDBX_cursor *first, const MDBX_cursor *last, ptrdiff_t *distance_items) {
if (unlikely(first == nullptr || last == nullptr || distance_items == nullptr))
return LOG_IFERR(MDBX_EINVAL);
*distance_items = 0;
@ -177,17 +167,14 @@ __hot int mdbx_estimate_distance(const MDBX_cursor *first,
return MDBX_SUCCESS;
}
__hot int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key,
MDBX_val *data, MDBX_cursor_op move_op,
__hot int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, MDBX_cursor_op move_op,
ptrdiff_t *distance_items) {
if (unlikely(cursor == nullptr || distance_items == nullptr ||
move_op == MDBX_GET_CURRENT || move_op == MDBX_GET_MULTIPLE))
if (unlikely(cursor == nullptr || distance_items == nullptr || move_op == MDBX_GET_CURRENT ||
move_op == MDBX_GET_MULTIPLE))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(cursor->signature != cur_signature_live))
return LOG_IFERR((cursor->signature == cur_signature_ready4dispose)
? MDBX_EINVAL
: MDBX_EBADSIGN);
return LOG_IFERR((cursor->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN);
int rc = check_txn(cursor->txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
@ -209,8 +196,7 @@ __hot int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key,
MDBX_val stub_data;
if (data == nullptr) {
const unsigned mask =
1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY;
const unsigned mask = 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY;
if (unlikely(mask & (1 << move_op)))
return LOG_IFERR(MDBX_EINVAL);
stub_data.iov_base = nullptr;
@ -220,9 +206,8 @@ __hot int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key,
MDBX_val stub_key;
if (key == nullptr) {
const unsigned mask = 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE |
1 << MDBX_SET_KEY | 1 << MDBX_SET |
1 << MDBX_SET_RANGE;
const unsigned mask =
1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY | 1 << MDBX_SET | 1 << MDBX_SET_RANGE;
if (unlikely(mask & (1 << move_op)))
return LOG_IFERR(MDBX_EINVAL);
stub_key.iov_base = nullptr;
@ -232,8 +217,7 @@ __hot int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key,
next.outer.signature = cur_signature_live;
rc = cursor_ops(&next.outer, key, data, move_op);
if (unlikely(rc != MDBX_SUCCESS &&
(rc != MDBX_NOTFOUND || !is_pointed(&next.outer))))
if (unlikely(rc != MDBX_SUCCESS && (rc != MDBX_NOTFOUND || !is_pointed(&next.outer))))
return LOG_IFERR(rc);
if (move_op == MDBX_LAST) {
@ -243,11 +227,8 @@ __hot int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key,
return mdbx_estimate_distance(cursor, &next.outer, distance_items);
}
__hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi,
const MDBX_val *begin_key,
const MDBX_val *begin_data,
const MDBX_val *end_key, const MDBX_val *end_data,
ptrdiff_t *size_items) {
__hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *begin_key, const MDBX_val *begin_data,
const MDBX_val *end_key, const MDBX_val *end_data, ptrdiff_t *size_items) {
int rc = check_txn(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return LOG_IFERR(rc);
@ -255,8 +236,7 @@ __hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi,
if (unlikely(!size_items))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(begin_data &&
(begin_key == nullptr || begin_key == MDBX_EPSILON)))
if (unlikely(begin_data && (begin_key == nullptr || begin_key == MDBX_EPSILON)))
return LOG_IFERR(MDBX_EINVAL);
if (unlikely(end_data && (end_key == nullptr || end_key == MDBX_EPSILON)))
@ -285,20 +265,14 @@ __hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi,
rc = outer_first(&begin.outer, nullptr, nullptr);
if (unlikely(end_key == MDBX_EPSILON)) {
/* LY: FIRST..+epsilon case */
return LOG_IFERR(
(rc == MDBX_SUCCESS)
? mdbx_cursor_count(&begin.outer, (size_t *)size_items)
: rc);
return LOG_IFERR((rc == MDBX_SUCCESS) ? mdbx_cursor_count(&begin.outer, (size_t *)size_items) : rc);
}
} else {
if (unlikely(begin_key == MDBX_EPSILON)) {
if (end_key == nullptr) {
/* LY: -epsilon..LAST case */
rc = outer_last(&begin.outer, nullptr, nullptr);
return LOG_IFERR(
(rc == MDBX_SUCCESS)
? mdbx_cursor_count(&begin.outer, (size_t *)size_items)
: rc);
return LOG_IFERR((rc == MDBX_SUCCESS) ? mdbx_cursor_count(&begin.outer, (size_t *)size_items) : rc);
}
/* LY: -epsilon..value case */
assert(end_key != MDBX_EPSILON);
@ -309,22 +283,19 @@ __hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi,
end_key = begin_key;
}
if (end_key && !begin_data && !end_data &&
(begin_key == end_key ||
begin.outer.clc->k.cmp(begin_key, end_key) == 0)) {
(begin_key == end_key || begin.outer.clc->k.cmp(begin_key, end_key) == 0)) {
/* LY: single key case */
rc = cursor_seek(&begin.outer, (MDBX_val *)begin_key, nullptr, MDBX_SET)
.err;
rc = cursor_seek(&begin.outer, (MDBX_val *)begin_key, nullptr, MDBX_SET).err;
if (unlikely(rc != MDBX_SUCCESS)) {
*size_items = 0;
return LOG_IFERR((rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc);
}
*size_items = 1;
if (inner_pointed(&begin.outer))
*size_items =
(sizeof(*size_items) >= sizeof(begin.inner.nested_tree.items) ||
begin.inner.nested_tree.items <= PTRDIFF_MAX)
? (size_t)begin.inner.nested_tree.items
: PTRDIFF_MAX;
*size_items = (sizeof(*size_items) >= sizeof(begin.inner.nested_tree.items) ||
begin.inner.nested_tree.items <= PTRDIFF_MAX)
? (size_t)begin.inner.nested_tree.items
: PTRDIFF_MAX;
return MDBX_SUCCESS;
} else {
@ -332,9 +303,7 @@ __hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi,
MDBX_val proxy_data = {nullptr, 0};
if (begin_data)
proxy_data = *begin_data;
rc = LOG_IFERR(cursor_seek(&begin.outer, &proxy_key, &proxy_data,
MDBX_SET_LOWERBOUND)
.err);
rc = LOG_IFERR(cursor_seek(&begin.outer, &proxy_key, &proxy_data, MDBX_SET_LOWERBOUND).err);
}
}
@ -356,8 +325,7 @@ __hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi,
MDBX_val proxy_data = {nullptr, 0};
if (end_data)
proxy_data = *end_data;
rc = cursor_seek(&end.outer, &proxy_key, &proxy_data, MDBX_SET_LOWERBOUND)
.err;
rc = cursor_seek(&end.outer, &proxy_key, &proxy_data, MDBX_SET_LOWERBOUND).err;
}
if (unlikely(rc != MDBX_SUCCESS)) {
if (rc != MDBX_NOTFOUND || !is_pointed(&end.outer))
@ -367,10 +335,9 @@ __hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi,
rc = mdbx_estimate_distance(&begin.outer, &end.outer, size_items);
if (unlikely(rc != MDBX_SUCCESS))
return LOG_IFERR(rc);
assert(*size_items >= -(ptrdiff_t)begin.outer.tree->items &&
*size_items <= (ptrdiff_t)begin.outer.tree->items);
assert(*size_items >= -(ptrdiff_t)begin.outer.tree->items && *size_items <= (ptrdiff_t)begin.outer.tree->items);
#if 0 /* LY: Was decided to returns as-is (i.e. negative) the estimation \
#if 0 /* LY: Was decided to returns as-is (i.e. negative) the estimation \
* results for an inverted ranges. */
/* Commit 8ddfd1f34ad7cf7a3c4aa75d2e248ca7e639ed63

49
src/refund.c
View File

@ -8,8 +8,7 @@ static void refund_reclaimed(MDBX_txn *txn) {
/* Scanning in descend order */
pgno_t first_unallocated = txn->geo.first_unallocated;
const pnl_t pnl = txn->tw.relist;
tASSERT(txn,
MDBX_PNL_GETSIZE(pnl) && MDBX_PNL_MOST(pnl) == first_unallocated - 1);
tASSERT(txn, MDBX_PNL_GETSIZE(pnl) && MDBX_PNL_MOST(pnl) == first_unallocated - 1);
#if MDBX_PNL_ASCENDING
size_t i = MDBX_PNL_GETSIZE(pnl);
tASSERT(txn, pnl[i] == first_unallocated - 1);
@ -26,12 +25,10 @@ static void refund_reclaimed(MDBX_txn *txn) {
for (size_t move = 0; move < len; ++move)
pnl[1 + move] = pnl[i + move];
#endif
VERBOSE("refunded %" PRIaPGNO " pages: %" PRIaPGNO " -> %" PRIaPGNO,
txn->geo.first_unallocated - first_unallocated,
VERBOSE("refunded %" PRIaPGNO " pages: %" PRIaPGNO " -> %" PRIaPGNO, txn->geo.first_unallocated - first_unallocated,
txn->geo.first_unallocated, first_unallocated);
txn->geo.first_unallocated = first_unallocated;
tASSERT(txn,
pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - 1));
tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - 1));
}
static void refund_loose(MDBX_txn *txn) {
@ -58,18 +55,14 @@ static void refund_loose(MDBX_txn *txn) {
}
/* Collect loose-pages which may be refunded. */
tASSERT(txn,
txn->geo.first_unallocated >= MIN_PAGENO + txn->tw.loose_count);
tASSERT(txn, txn->geo.first_unallocated >= MIN_PAGENO + txn->tw.loose_count);
pgno_t most = MIN_PAGENO;
size_t w = 0;
for (const page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
tASSERT(txn, lp->flags == P_LOOSE);
tASSERT(txn, txn->geo.first_unallocated > lp->pgno);
if (likely(txn->geo.first_unallocated - txn->tw.loose_count <=
lp->pgno)) {
tASSERT(txn,
w < ((suitable == onstack) ? pnl_bytes2size(sizeof(onstack))
: MDBX_PNL_ALLOCLEN(suitable)));
if (likely(txn->geo.first_unallocated - txn->tw.loose_count <= lp->pgno)) {
tASSERT(txn, w < ((suitable == onstack) ? pnl_bytes2size(sizeof(onstack)) : MDBX_PNL_ALLOCLEN(suitable)));
suitable[++w] = lp->pgno;
most = (lp->pgno > most) ? lp->pgno : most;
}
@ -84,10 +77,8 @@ static void refund_loose(MDBX_txn *txn) {
/* Scanning in descend order */
const intptr_t step = MDBX_PNL_ASCENDING ? -1 : 1;
const intptr_t begin =
MDBX_PNL_ASCENDING ? MDBX_PNL_GETSIZE(suitable) : 1;
const intptr_t end =
MDBX_PNL_ASCENDING ? 0 : MDBX_PNL_GETSIZE(suitable) + 1;
const intptr_t begin = MDBX_PNL_ASCENDING ? MDBX_PNL_GETSIZE(suitable) : 1;
const intptr_t end = MDBX_PNL_ASCENDING ? 0 : MDBX_PNL_GETSIZE(suitable) + 1;
tASSERT(txn, suitable[begin] >= suitable[end - step]);
tASSERT(txn, most == suitable[begin]);
@ -97,8 +88,7 @@ static void refund_loose(MDBX_txn *txn) {
most -= 1;
}
const size_t refunded = txn->geo.first_unallocated - most;
DEBUG("refund-suitable %zu pages %" PRIaPGNO " -> %" PRIaPGNO, refunded,
most, txn->geo.first_unallocated);
DEBUG("refund-suitable %zu pages %" PRIaPGNO " -> %" PRIaPGNO, refunded, most, txn->geo.first_unallocated);
txn->geo.first_unallocated = most;
txn->tw.loose_count -= refunded;
if (dl) {
@ -126,22 +116,19 @@ static void refund_loose(MDBX_txn *txn) {
}
dpl_setlen(dl, w);
tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
(txn->parent ? txn->parent->tw.dirtyroom
: txn->env->options.dp_limit));
(txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
}
goto unlink_loose;
}
} else {
/* Dirtylist is mostly sorted, just refund loose pages at the end. */
dpl_sort(txn);
tASSERT(txn,
dl->length < 2 || dl->items[1].pgno < dl->items[dl->length].pgno);
tASSERT(txn, dl->length < 2 || dl->items[1].pgno < dl->items[dl->length].pgno);
tASSERT(txn, dl->sorted == dl->length);
/* Scan dirtylist tail-forward and cutoff suitable pages. */
size_t n;
for (n = dl->length; dl->items[n].pgno == txn->geo.first_unallocated - 1 &&
dl->items[n].ptr->flags == P_LOOSE;
for (n = dl->length; dl->items[n].pgno == txn->geo.first_unallocated - 1 && dl->items[n].ptr->flags == P_LOOSE;
--n) {
tASSERT(txn, n > 0);
page_t *dp = dl->items[n].ptr;
@ -158,8 +145,7 @@ static void refund_loose(MDBX_txn *txn) {
txn->tw.dirtyroom += refunded;
dl->pages_including_loose -= refunded;
tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length ==
(txn->parent ? txn->parent->tw.dirtyroom
: txn->env->options.dp_limit));
(txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit));
/* Filter-out loose chain & dispose refunded pages. */
unlink_loose:
@ -188,18 +174,15 @@ static void refund_loose(MDBX_txn *txn) {
bool txn_refund(MDBX_txn *txn) {
const pgno_t before = txn->geo.first_unallocated;
if (txn->tw.loose_pages &&
txn->tw.loose_refund_wl > txn->geo.first_unallocated)
if (txn->tw.loose_pages && txn->tw.loose_refund_wl > txn->geo.first_unallocated)
refund_loose(txn);
while (true) {
if (MDBX_PNL_GETSIZE(txn->tw.relist) == 0 ||
MDBX_PNL_MOST(txn->tw.relist) != txn->geo.first_unallocated - 1)
if (MDBX_PNL_GETSIZE(txn->tw.relist) == 0 || MDBX_PNL_MOST(txn->tw.relist) != txn->geo.first_unallocated - 1)
break;
refund_reclaimed(txn);
if (!txn->tw.loose_pages ||
txn->tw.loose_refund_wl <= txn->geo.first_unallocated)
if (!txn->tw.loose_pages || txn->tw.loose_refund_wl <= txn->geo.first_unallocated)
break;
const pgno_t memo = txn->geo.first_unallocated;

551
src/sort.h
View File

@ -14,21 +14,21 @@
* Thanks to John M. Gamble for the http://pages.ripco.net/~jgamble/nw.html */
#if MDBX_HAVE_CMOV
#define SORT_CMP_SWAP(TYPE, CMP, a, b) \
do { \
const TYPE swap_tmp = (a); \
const bool swap_cmp = expect_with_probability(CMP(swap_tmp, b), 0, .5); \
(a) = swap_cmp ? swap_tmp : b; \
(b) = swap_cmp ? b : swap_tmp; \
#define SORT_CMP_SWAP(TYPE, CMP, a, b) \
do { \
const TYPE swap_tmp = (a); \
const bool swap_cmp = expect_with_probability(CMP(swap_tmp, b), 0, .5); \
(a) = swap_cmp ? swap_tmp : b; \
(b) = swap_cmp ? b : swap_tmp; \
} while (0)
#else
#define SORT_CMP_SWAP(TYPE, CMP, a, b) \
do \
if (expect_with_probability(!CMP(a, b), 0, .5)) { \
const TYPE swap_tmp = (a); \
(a) = (b); \
(b) = swap_tmp; \
} \
#define SORT_CMP_SWAP(TYPE, CMP, a, b) \
do \
if (expect_with_probability(!CMP(a, b), 0, .5)) { \
const TYPE swap_tmp = (a); \
(a) = (b); \
(b) = swap_tmp; \
} \
while (0)
#endif
@ -42,11 +42,11 @@
// [[1,2]]
// [[0,2]]
// [[0,1]]
#define SORT_NETWORK_3(TYPE, CMP, begin) \
do { \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
#define SORT_NETWORK_3(TYPE, CMP, begin) \
do { \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
} while (0)
// 5 comparators, 3 parallel operations
@ -61,13 +61,13 @@
// [[0,1],[2,3]]
// [[0,2],[1,3]]
// [[1,2]]
#define SORT_NETWORK_4(TYPE, CMP, begin) \
do { \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \
#define SORT_NETWORK_4(TYPE, CMP, begin) \
do { \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \
} while (0)
// 9 comparators, 5 parallel operations
@ -86,17 +86,17 @@
// [[2,4],[0,1]]
// [[2,3],[1,4]]
// [[1,2],[3,4]]
#define SORT_NETWORK_5(TYPE, CMP, begin) \
do { \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \
#define SORT_NETWORK_5(TYPE, CMP, begin) \
do { \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \
} while (0)
// 12 comparators, 6 parallel operations
@ -118,20 +118,20 @@
// [[0,3],[1,4]]
// [[2,4],[1,3]]
// [[2,3]]
#define SORT_NETWORK_6(TYPE, CMP, begin) \
do { \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \
#define SORT_NETWORK_6(TYPE, CMP, begin) \
do { \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \
} while (0)
// 16 comparators, 6 parallel operations
@ -155,24 +155,24 @@
// [[2,3],[4,5]]
// [[1,4],[3,6]]
// [[1,2],[3,4],[5,6]]
#define SORT_NETWORK_7(TYPE, CMP, begin) \
do { \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[6]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[6]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[6]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[6]); \
#define SORT_NETWORK_7(TYPE, CMP, begin) \
do { \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[6]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[6]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[6]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[6]); \
} while (0)
// 19 comparators, 6 parallel operations
@ -198,237 +198,236 @@
// [[2,3],[4,5]]
// [[1,4],[3,6]]
// [[1,2],[3,4],[5,6]]
#define SORT_NETWORK_8(TYPE, CMP, begin) \
do { \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[6]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[7]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[6]); \
SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[7]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
SORT_CMP_SWAP(TYPE, CMP, begin[6], begin[7]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[6]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[6]); \
#define SORT_NETWORK_8(TYPE, CMP, begin) \
do { \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[6]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[7]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[6]); \
SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[7]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
SORT_CMP_SWAP(TYPE, CMP, begin[6], begin[7]); \
SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \
SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[6]); \
SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \
SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \
SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[6]); \
} while (0)
#define SORT_INNER(TYPE, CMP, begin, end, len) \
switch (len) { \
default: \
assert(false); \
__unreachable(); \
case 0: \
case 1: \
break; \
case 2: \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
break; \
case 3: \
SORT_NETWORK_3(TYPE, CMP, begin); \
break; \
case 4: \
SORT_NETWORK_4(TYPE, CMP, begin); \
break; \
case 5: \
SORT_NETWORK_5(TYPE, CMP, begin); \
break; \
case 6: \
SORT_NETWORK_6(TYPE, CMP, begin); \
break; \
case 7: \
SORT_NETWORK_7(TYPE, CMP, begin); \
break; \
case 8: \
SORT_NETWORK_8(TYPE, CMP, begin); \
break; \
#define SORT_INNER(TYPE, CMP, begin, end, len) \
switch (len) { \
default: \
assert(false); \
__unreachable(); \
case 0: \
case 1: \
break; \
case 2: \
SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \
break; \
case 3: \
SORT_NETWORK_3(TYPE, CMP, begin); \
break; \
case 4: \
SORT_NETWORK_4(TYPE, CMP, begin); \
break; \
case 5: \
SORT_NETWORK_5(TYPE, CMP, begin); \
break; \
case 6: \
SORT_NETWORK_6(TYPE, CMP, begin); \
break; \
case 7: \
SORT_NETWORK_7(TYPE, CMP, begin); \
break; \
case 8: \
SORT_NETWORK_8(TYPE, CMP, begin); \
break; \
}
#define SORT_SWAP(TYPE, a, b) \
do { \
const TYPE swap_tmp = (a); \
(a) = (b); \
(b) = swap_tmp; \
#define SORT_SWAP(TYPE, a, b) \
do { \
const TYPE swap_tmp = (a); \
(a) = (b); \
(b) = swap_tmp; \
} while (0)
#define SORT_PUSH(low, high) \
do { \
top->lo = (low); \
top->hi = (high); \
++top; \
#define SORT_PUSH(low, high) \
do { \
top->lo = (low); \
top->hi = (high); \
++top; \
} while (0)
#define SORT_POP(low, high) \
do { \
--top; \
low = top->lo; \
high = top->hi; \
#define SORT_POP(low, high) \
do { \
--top; \
low = top->lo; \
high = top->hi; \
} while (0)
#define SORT_IMPL(NAME, EXPECT_LOW_CARDINALITY_OR_PRESORTED, TYPE, CMP) \
\
static inline bool NAME##_is_sorted(const TYPE *first, const TYPE *last) { \
while (++first <= last) \
if (expect_with_probability(CMP(first[0], first[-1]), 1, .1)) \
return false; \
return true; \
} \
\
typedef struct { \
TYPE *lo, *hi; \
} NAME##_stack; \
\
__hot static void NAME(TYPE *const __restrict begin, \
TYPE *const __restrict end) { \
NAME##_stack stack[sizeof(size_t) * CHAR_BIT], *__restrict top = stack; \
\
TYPE *__restrict hi = end - 1; \
TYPE *__restrict lo = begin; \
while (true) { \
const ptrdiff_t len = hi - lo; \
if (len < 8) { \
SORT_INNER(TYPE, CMP, lo, hi + 1, len + 1); \
if (unlikely(top == stack)) \
break; \
SORT_POP(lo, hi); \
continue; \
} \
\
TYPE *__restrict mid = lo + (len >> 1); \
SORT_CMP_SWAP(TYPE, CMP, *lo, *mid); \
SORT_CMP_SWAP(TYPE, CMP, *mid, *hi); \
SORT_CMP_SWAP(TYPE, CMP, *lo, *mid); \
\
TYPE *right = hi - 1; \
TYPE *left = lo + 1; \
while (1) { \
while (expect_with_probability(CMP(*left, *mid), 0, .5)) \
++left; \
while (expect_with_probability(CMP(*mid, *right), 0, .5)) \
--right; \
if (unlikely(left > right)) { \
if (EXPECT_LOW_CARDINALITY_OR_PRESORTED) { \
if (NAME##_is_sorted(lo, right)) \
lo = right + 1; \
if (NAME##_is_sorted(left, hi)) \
hi = left; \
} \
break; \
} \
SORT_SWAP(TYPE, *left, *right); \
mid = (mid == left) ? right : (mid == right) ? left : mid; \
++left; \
--right; \
} \
\
if (right - lo > hi - left) { \
SORT_PUSH(lo, right); \
lo = left; \
} else { \
SORT_PUSH(left, hi); \
hi = right; \
} \
} \
\
if (AUDIT_ENABLED()) { \
for (TYPE *scan = begin + 1; scan < end; ++scan) \
assert(CMP(scan[-1], scan[0])); \
} \
#define SORT_IMPL(NAME, EXPECT_LOW_CARDINALITY_OR_PRESORTED, TYPE, CMP) \
\
static inline bool NAME##_is_sorted(const TYPE *first, const TYPE *last) { \
while (++first <= last) \
if (expect_with_probability(CMP(first[0], first[-1]), 1, .1)) \
return false; \
return true; \
} \
\
typedef struct { \
TYPE *lo, *hi; \
} NAME##_stack; \
\
__hot static void NAME(TYPE *const __restrict begin, TYPE *const __restrict end) { \
NAME##_stack stack[sizeof(size_t) * CHAR_BIT], *__restrict top = stack; \
\
TYPE *__restrict hi = end - 1; \
TYPE *__restrict lo = begin; \
while (true) { \
const ptrdiff_t len = hi - lo; \
if (len < 8) { \
SORT_INNER(TYPE, CMP, lo, hi + 1, len + 1); \
if (unlikely(top == stack)) \
break; \
SORT_POP(lo, hi); \
continue; \
} \
\
TYPE *__restrict mid = lo + (len >> 1); \
SORT_CMP_SWAP(TYPE, CMP, *lo, *mid); \
SORT_CMP_SWAP(TYPE, CMP, *mid, *hi); \
SORT_CMP_SWAP(TYPE, CMP, *lo, *mid); \
\
TYPE *right = hi - 1; \
TYPE *left = lo + 1; \
while (1) { \
while (expect_with_probability(CMP(*left, *mid), 0, .5)) \
++left; \
while (expect_with_probability(CMP(*mid, *right), 0, .5)) \
--right; \
if (unlikely(left > right)) { \
if (EXPECT_LOW_CARDINALITY_OR_PRESORTED) { \
if (NAME##_is_sorted(lo, right)) \
lo = right + 1; \
if (NAME##_is_sorted(left, hi)) \
hi = left; \
} \
break; \
} \
SORT_SWAP(TYPE, *left, *right); \
mid = (mid == left) ? right : (mid == right) ? left : mid; \
++left; \
--right; \
} \
\
if (right - lo > hi - left) { \
SORT_PUSH(lo, right); \
lo = left; \
} else { \
SORT_PUSH(left, hi); \
hi = right; \
} \
} \
\
if (AUDIT_ENABLED()) { \
for (TYPE *scan = begin + 1; scan < end; ++scan) \
assert(CMP(scan[-1], scan[0])); \
} \
}
/*------------------------------------------------------------------------------
* LY: radix sort for large chunks */
#define RADIXSORT_IMPL(NAME, TYPE, EXTRACT_KEY, BUFFER_PREALLOCATED, END_GAP) \
\
__hot static bool NAME##_radixsort(TYPE *const begin, const size_t length) { \
TYPE *tmp; \
if (BUFFER_PREALLOCATED) { \
tmp = begin + length + END_GAP; \
/* memset(tmp, 0xDeadBeef, sizeof(TYPE) * length); */ \
} else { \
tmp = osal_malloc(sizeof(TYPE) * length); \
if (unlikely(!tmp)) \
return false; \
} \
\
size_t key_shift = 0, key_diff_mask; \
do { \
struct { \
pgno_t a[256], b[256]; \
} counters; \
memset(&counters, 0, sizeof(counters)); \
\
key_diff_mask = 0; \
size_t prev_key = EXTRACT_KEY(begin) >> key_shift; \
TYPE *r = begin, *end = begin + length; \
do { \
const size_t key = EXTRACT_KEY(r) >> key_shift; \
counters.a[key & 255]++; \
counters.b[(key >> 8) & 255]++; \
key_diff_mask |= prev_key ^ key; \
prev_key = key; \
} while (++r != end); \
\
pgno_t ta = 0, tb = 0; \
for (size_t i = 0; i < 256; ++i) { \
const pgno_t ia = counters.a[i]; \
counters.a[i] = ta; \
ta += ia; \
const pgno_t ib = counters.b[i]; \
counters.b[i] = tb; \
tb += ib; \
} \
\
r = begin; \
do { \
const size_t key = EXTRACT_KEY(r) >> key_shift; \
tmp[counters.a[key & 255]++] = *r; \
} while (++r != end); \
\
if (unlikely(key_diff_mask < 256)) { \
memcpy(begin, tmp, ptr_dist(end, begin)); \
break; \
} \
end = (r = tmp) + length; \
do { \
const size_t key = EXTRACT_KEY(r) >> key_shift; \
begin[counters.b[(key >> 8) & 255]++] = *r; \
} while (++r != end); \
\
key_shift += 16; \
} while (key_diff_mask >> 16); \
\
if (!(BUFFER_PREALLOCATED)) \
osal_free(tmp); \
return true; \
#define RADIXSORT_IMPL(NAME, TYPE, EXTRACT_KEY, BUFFER_PREALLOCATED, END_GAP) \
\
__hot static bool NAME##_radixsort(TYPE *const begin, const size_t length) { \
TYPE *tmp; \
if (BUFFER_PREALLOCATED) { \
tmp = begin + length + END_GAP; \
/* memset(tmp, 0xDeadBeef, sizeof(TYPE) * length); */ \
} else { \
tmp = osal_malloc(sizeof(TYPE) * length); \
if (unlikely(!tmp)) \
return false; \
} \
\
size_t key_shift = 0, key_diff_mask; \
do { \
struct { \
pgno_t a[256], b[256]; \
} counters; \
memset(&counters, 0, sizeof(counters)); \
\
key_diff_mask = 0; \
size_t prev_key = EXTRACT_KEY(begin) >> key_shift; \
TYPE *r = begin, *end = begin + length; \
do { \
const size_t key = EXTRACT_KEY(r) >> key_shift; \
counters.a[key & 255]++; \
counters.b[(key >> 8) & 255]++; \
key_diff_mask |= prev_key ^ key; \
prev_key = key; \
} while (++r != end); \
\
pgno_t ta = 0, tb = 0; \
for (size_t i = 0; i < 256; ++i) { \
const pgno_t ia = counters.a[i]; \
counters.a[i] = ta; \
ta += ia; \
const pgno_t ib = counters.b[i]; \
counters.b[i] = tb; \
tb += ib; \
} \
\
r = begin; \
do { \
const size_t key = EXTRACT_KEY(r) >> key_shift; \
tmp[counters.a[key & 255]++] = *r; \
} while (++r != end); \
\
if (unlikely(key_diff_mask < 256)) { \
memcpy(begin, tmp, ptr_dist(end, begin)); \
break; \
} \
end = (r = tmp) + length; \
do { \
const size_t key = EXTRACT_KEY(r) >> key_shift; \
begin[counters.b[(key >> 8) & 255]++] = *r; \
} while (++r != end); \
\
key_shift += 16; \
} while (key_diff_mask >> 16); \
\
if (!(BUFFER_PREALLOCATED)) \
osal_free(tmp); \
return true; \
}
/*------------------------------------------------------------------------------
* LY: Binary search */
#if defined(__clang__) && __clang_major__ > 4 && defined(__ia32__)
#define WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(size, flag) \
do \
__asm __volatile("" \
: "+r"(size) \
: "r" /* the `b` constraint is more suitable here, but \
cause CLANG to allocate and push/pop an one more \
register, so using the `r` which avoids this. */ \
(flag)); \
#define WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(size, flag) \
do \
__asm __volatile("" \
: "+r"(size) \
: "r" /* the `b` constraint is more suitable here, but \
cause CLANG to allocate and push/pop an one more \
register, so using the `r` which avoids this. */ \
(flag)); \
while (0)
#else
#define WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(size, flag) \
do { \
/* nope for non-clang or non-x86 */; \
#define WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(size, flag) \
do { \
/* nope for non-clang or non-x86 */; \
} while (0)
#endif /* Workaround for CLANG */

163
src/spill.c
View File

@ -4,33 +4,25 @@
#include "internals.h"
void spill_remove(MDBX_txn *txn, size_t idx, size_t npages) {
tASSERT(txn, idx > 0 && idx <= MDBX_PNL_GETSIZE(txn->tw.spilled.list) &&
txn->tw.spilled.least_removed > 0);
txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed)
? idx
: txn->tw.spilled.least_removed;
tASSERT(txn, idx > 0 && idx <= MDBX_PNL_GETSIZE(txn->tw.spilled.list) && txn->tw.spilled.least_removed > 0);
txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed) ? idx : txn->tw.spilled.least_removed;
txn->tw.spilled.list[idx] |= 1;
MDBX_PNL_SETSIZE(txn->tw.spilled.list,
MDBX_PNL_GETSIZE(txn->tw.spilled.list) -
(idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list)));
MDBX_PNL_GETSIZE(txn->tw.spilled.list) - (idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list)));
while (unlikely(npages > 1)) {
const pgno_t pgno = (txn->tw.spilled.list[idx] >> 1) + 1;
if (MDBX_PNL_ASCENDING) {
if (++idx > MDBX_PNL_GETSIZE(txn->tw.spilled.list) ||
(txn->tw.spilled.list[idx] >> 1) != pgno)
if (++idx > MDBX_PNL_GETSIZE(txn->tw.spilled.list) || (txn->tw.spilled.list[idx] >> 1) != pgno)
return;
} else {
if (--idx < 1 || (txn->tw.spilled.list[idx] >> 1) != pgno)
return;
txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed)
? idx
: txn->tw.spilled.least_removed;
txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed) ? idx : txn->tw.spilled.least_removed;
}
txn->tw.spilled.list[idx] |= 1;
MDBX_PNL_SETSIZE(txn->tw.spilled.list,
MDBX_PNL_GETSIZE(txn->tw.spilled.list) -
(idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list)));
MDBX_PNL_GETSIZE(txn->tw.spilled.list) - (idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list)));
--npages;
}
}
@ -57,8 +49,7 @@ pnl_t spill_purge(MDBX_txn *txn) {
/*----------------------------------------------------------------------------*/
static int spill_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp,
const size_t npages) {
static int spill_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, const size_t npages) {
tASSERT(txn, !(txn->flags & MDBX_WRITEMAP));
#if MDBX_ENABLE_PGOP_STAT
txn->env->lck->pgops.spill.weak += npages;
@ -72,8 +63,7 @@ static int spill_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp,
/* Set unspillable LRU-label for dirty pages watched by txn.
* Returns the number of pages marked as unspillable. */
static size_t spill_cursor_keep(const MDBX_txn *const txn,
const MDBX_cursor *mc) {
static size_t spill_cursor_keep(const MDBX_txn *const txn, const MDBX_cursor *mc) {
tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0);
size_t keep = 0;
while (!is_poor(mc)) {
@ -87,8 +77,7 @@ static size_t spill_cursor_keep(const MDBX_txn *const txn,
size_t const n = dpl_search(txn, mp->pgno);
if (txn->tw.dirtylist->items[n].pgno == mp->pgno &&
/* не считаем дважды */ dpl_age(txn, n)) {
size_t *const ptr = ptr_disp(txn->tw.dirtylist->items[n].ptr,
-(ptrdiff_t)sizeof(size_t));
size_t *const ptr = ptr_disp(txn->tw.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t));
*ptr = txn->tw.dirtylru;
tASSERT(txn, dpl_age(txn, n) == 0);
++keep;
@ -112,8 +101,7 @@ static size_t spill_txn_keep(MDBX_txn *txn, MDBX_cursor *m0) {
size_t keep = m0 ? spill_cursor_keep(txn, m0) : 0;
TXN_FOREACH_DBI_ALL(txn, dbi) {
if (F_ISSET(txn->dbi_state[dbi], DBI_DIRTY | DBI_VALID) &&
txn->dbs[dbi].root != P_INVALID)
if (F_ISSET(txn->dbi_state[dbi], DBI_DIRTY | DBI_VALID) && txn->dbs[dbi].root != P_INVALID)
for (MDBX_cursor *mc = txn->cursors[dbi]; mc; mc = mc->next)
if (mc != m0)
keep += spill_cursor_keep(txn, mc);
@ -126,8 +114,7 @@ static size_t spill_txn_keep(MDBX_txn *txn, MDBX_cursor *m0) {
* 0 = should be spilled;
* ...
* > 255 = must not be spilled. */
MDBX_NOTHROW_PURE_FUNCTION static unsigned
spill_prio(const MDBX_txn *txn, const size_t i, const uint32_t reciprocal) {
MDBX_NOTHROW_PURE_FUNCTION static unsigned spill_prio(const MDBX_txn *txn, const size_t i, const uint32_t reciprocal) {
dpl_t *const dl = txn->tw.dirtylist;
const uint32_t age = dpl_age(txn, i);
const size_t npages = dpl_npages(dl, i);
@ -139,8 +126,7 @@ spill_prio(const MDBX_txn *txn, const size_t i, const uint32_t reciprocal) {
page_t *const dp = dl->items[i].ptr;
if (dp->flags & (P_LOOSE | P_SPILLED)) {
DEBUG("skip %s %zu page %" PRIaPGNO,
(dp->flags & P_LOOSE) ? "loose" : "parent-spilled", npages, pgno);
DEBUG("skip %s %zu page %" PRIaPGNO, (dp->flags & P_LOOSE) ? "loose" : "parent-spilled", npages, pgno);
return 256;
}
@ -175,67 +161,49 @@ spill_prio(const MDBX_txn *txn, const size_t i, const uint32_t reciprocal) {
return prio = (unsigned)factor;
}
static size_t spill_gate(const MDBX_env *env, intptr_t part,
const size_t total) {
const intptr_t spill_min =
env->options.spill_min_denominator
? (total + env->options.spill_min_denominator - 1) /
env->options.spill_min_denominator
: 1;
static size_t spill_gate(const MDBX_env *env, intptr_t part, const size_t total) {
const intptr_t spill_min = env->options.spill_min_denominator
? (total + env->options.spill_min_denominator - 1) / env->options.spill_min_denominator
: 1;
const intptr_t spill_max =
total - (env->options.spill_max_denominator
? total / env->options.spill_max_denominator
: 0);
total - (env->options.spill_max_denominator ? total / env->options.spill_max_denominator : 0);
part = (part < spill_max) ? part : spill_max;
part = (part > spill_min) ? part : spill_min;
eASSERT(env, part >= 0 && (size_t)part <= total);
return (size_t)part;
}
__cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0,
const intptr_t wanna_spill_entries,
const intptr_t wanna_spill_npages,
const size_t need) {
__cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intptr_t wanna_spill_entries,
const intptr_t wanna_spill_npages, const size_t need) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
int rc = MDBX_SUCCESS;
if (unlikely(txn->tw.loose_count >=
(txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose
: txn->tw.writemap_dirty_npages)))
(txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages)))
goto done;
const size_t dirty_entries =
txn->tw.dirtylist ? (txn->tw.dirtylist->length - txn->tw.loose_count) : 1;
const size_t dirty_entries = txn->tw.dirtylist ? (txn->tw.dirtylist->length - txn->tw.loose_count) : 1;
const size_t dirty_npages =
(txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose
: txn->tw.writemap_dirty_npages) -
(txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages) -
txn->tw.loose_count;
const size_t need_spill_entries =
spill_gate(txn->env, wanna_spill_entries, dirty_entries);
const size_t need_spill_npages =
spill_gate(txn->env, wanna_spill_npages, dirty_npages);
const size_t need_spill_entries = spill_gate(txn->env, wanna_spill_entries, dirty_entries);
const size_t need_spill_npages = spill_gate(txn->env, wanna_spill_npages, dirty_npages);
const size_t need_spill = (need_spill_entries > need_spill_npages)
? need_spill_entries
: need_spill_npages;
const size_t need_spill = (need_spill_entries > need_spill_npages) ? need_spill_entries : need_spill_npages;
if (!need_spill)
goto done;
if (txn->flags & MDBX_WRITEMAP) {
NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "msync",
dirty_entries, dirty_npages);
NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "msync", dirty_entries, dirty_npages);
const MDBX_env *env = txn->env;
tASSERT(txn, txn->tw.spilled.list == nullptr);
rc = osal_msync(&txn->env->dxb_mmap, 0,
pgno_align2os_bytes(env, txn->geo.first_unallocated),
MDBX_SYNC_KICK);
rc = osal_msync(&txn->env->dxb_mmap, 0, pgno_align2os_bytes(env, txn->geo.first_unallocated), MDBX_SYNC_KICK);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
#if MDBX_AVOID_MSYNC
MDBX_ANALYSIS_ASSUME(txn->tw.dirtylist != nullptr);
tASSERT(txn, dpl_check(txn));
env->lck->unsynced_pages.weak +=
txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count;
env->lck->unsynced_pages.weak += txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count;
dpl_clear(txn->tw.dirtylist);
txn->tw.dirtyroom = env->options.dp_limit - txn->tw.loose_count;
for (page_t *lp = txn->tw.loose_pages; lp != nullptr; lp = page_next(lp)) {
@ -256,12 +224,10 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0,
goto done;
}
NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "write",
need_spill_entries, need_spill_npages);
NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "write", need_spill_entries, need_spill_npages);
MDBX_ANALYSIS_ASSUME(txn->tw.dirtylist != nullptr);
tASSERT(txn, txn->tw.dirtylist->length - txn->tw.loose_count >= 1);
tASSERT(txn, txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count >=
need_spill_npages);
tASSERT(txn, txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count >= need_spill_npages);
if (!txn->tw.spilled.list) {
txn->tw.spilled.least_removed = INT_MAX;
txn->tw.spilled.list = pnl_alloc(need_spill);
@ -338,10 +304,8 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0,
for (size_t i = 1; i <= dl->length; ++i) {
const unsigned prio = spill_prio(txn, i, reciprocal);
size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t));
TRACE("page %" PRIaPGNO
", lru %zu, is_multi %c, npages %u, age %u of %u, prio %u",
dl->items[i].pgno, *ptr, (dl->items[i].npages > 1) ? 'Y' : 'N',
dpl_npages(dl, i), dpl_age(txn, i), age_max, prio);
TRACE("page %" PRIaPGNO ", lru %zu, is_multi %c, npages %u, age %u of %u, prio %u", dl->items[i].pgno, *ptr,
(dl->items[i].npages > 1) ? 'Y' : 'N', dpl_npages(dl, i), dpl_age(txn, i), age_max, prio);
if (prio < 256) {
radix_entries[prio] += 1;
spillable_entries += 1;
@ -354,20 +318,16 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0,
tASSERT(txn, spillable_npages >= spillable_entries);
pgno_t spilled_entries = 0, spilled_npages = 0;
if (likely(spillable_entries > 0)) {
size_t prio2spill = 0, prio2adjacent = 128,
amount_entries = radix_entries[0], amount_npages = radix_npages[0];
size_t prio2spill = 0, prio2adjacent = 128, amount_entries = radix_entries[0], amount_npages = radix_npages[0];
for (size_t i = 1; i < 256; i++) {
if (amount_entries < need_spill_entries ||
amount_npages < need_spill_npages) {
if (amount_entries < need_spill_entries || amount_npages < need_spill_npages) {
prio2spill = i;
prio2adjacent = i + (257 - i) / 2;
amount_entries += radix_entries[i];
amount_npages += radix_npages[i];
} else if (amount_entries + amount_entries <
spillable_entries + need_spill_entries
} else if (amount_entries + amount_entries < spillable_entries + need_spill_entries
/* РАВНОЗНАЧНО: amount - need_spill < spillable - amount */
|| amount_npages + amount_npages <
spillable_npages + need_spill_npages) {
|| amount_npages + amount_npages < spillable_npages + need_spill_npages) {
prio2adjacent = i;
amount_entries += radix_entries[i];
amount_npages += radix_npages[i];
@ -377,44 +337,38 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0,
VERBOSE("prio2spill %zu, prio2adjacent %zu, spillable %zu/%zu,"
" wanna-spill %zu/%zu, amount %zu/%zu",
prio2spill, prio2adjacent, spillable_entries, spillable_npages,
need_spill_entries, need_spill_npages, amount_entries,
amount_npages);
prio2spill, prio2adjacent, spillable_entries, spillable_npages, need_spill_entries, need_spill_npages,
amount_entries, amount_npages);
tASSERT(txn, prio2spill < prio2adjacent && prio2adjacent <= 256);
iov_ctx_t ctx;
rc = iov_init(
txn, &ctx, amount_entries, amount_npages,
rc = iov_init(txn, &ctx, amount_entries, amount_npages,
#if defined(_WIN32) || defined(_WIN64)
txn->env->ioring.overlapped_fd ? txn->env->ioring.overlapped_fd :
txn->env->ioring.overlapped_fd ? txn->env->ioring.overlapped_fd :
#endif
txn->env->lazy_fd,
true);
txn->env->lazy_fd,
true);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
size_t r = 0, w = 0;
pgno_t last = 0;
while (r < dl->length && (spilled_entries < need_spill_entries ||
spilled_npages < need_spill_npages)) {
while (r < dl->length && (spilled_entries < need_spill_entries || spilled_npages < need_spill_npages)) {
dl->items[++w] = dl->items[++r];
unsigned prio = spill_prio(txn, w, reciprocal);
if (prio > prio2spill &&
(prio >= prio2adjacent || last != dl->items[w].pgno))
if (prio > prio2spill && (prio >= prio2adjacent || last != dl->items[w].pgno))
continue;
const size_t e = w;
last = dpl_endpgno(dl, w);
while (--w && dpl_endpgno(dl, w) == dl->items[w + 1].pgno &&
spill_prio(txn, w, reciprocal) < prio2adjacent)
while (--w && dpl_endpgno(dl, w) == dl->items[w + 1].pgno && spill_prio(txn, w, reciprocal) < prio2adjacent)
;
for (size_t i = w; ++i <= e;) {
const unsigned npages = dpl_npages(dl, i);
prio = spill_prio(txn, i, reciprocal);
DEBUG("%sspill[%zu] %u page %" PRIaPGNO " (age %d, prio %u)",
(prio > prio2spill) ? "co-" : "", i, npages, dl->items[i].pgno,
dpl_age(txn, i), prio);
DEBUG("%sspill[%zu] %u page %" PRIaPGNO " (age %d, prio %u)", (prio > prio2spill) ? "co-" : "", i, npages,
dl->items[i].pgno, dpl_age(txn, i), prio);
tASSERT(txn, prio < 256);
++spilled_entries;
spilled_npages += npages;
@ -424,8 +378,7 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0,
}
}
VERBOSE("spilled entries %u, spilled npages %u", spilled_entries,
spilled_npages);
VERBOSE("spilled entries %u, spilled npages %u", spilled_entries, spilled_npages);
tASSERT(txn, spillable_entries == 0 || spilled_entries > 0);
tASSERT(txn, spilled_npages >= spilled_entries);
@ -449,16 +402,14 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0,
txn->env->lck->unsynced_pages.weak += spilled_npages;
pnl_sort(txn->tw.spilled.list, (size_t)txn->geo.first_unallocated << 1);
txn->flags |= MDBX_TXN_SPILLS;
NOTICE("spilled %u dirty-entries, %u dirty-npages, now have %zu dirty-room",
spilled_entries, spilled_npages, txn->tw.dirtyroom);
NOTICE("spilled %u dirty-entries, %u dirty-npages, now have %zu dirty-room", spilled_entries, spilled_npages,
txn->tw.dirtyroom);
} else {
tASSERT(txn, rc == MDBX_SUCCESS);
for (size_t i = 1; i <= dl->length; ++i) {
page_t *dp = dl->items[i].ptr;
VERBOSE(
"unspillable[%zu]: pgno %u, npages %u, flags 0x%04X, age %u, prio %u",
i, dp->pgno, dpl_npages(dl, i), dp->flags, dpl_age(txn, i),
spill_prio(txn, i, reciprocal));
VERBOSE("unspillable[%zu]: pgno %u, npages %u, flags 0x%04X, age %u, prio %u", i, dp->pgno, dpl_npages(dl, i),
dp->flags, dpl_age(txn, i), spill_prio(txn, i, reciprocal));
}
}
@ -468,17 +419,13 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0,
"needed %zu, spillable %zu; "
"spilled %u dirty-entries, now have %zu dirty-room",
dl->length + spilled_entries, dl->length,
(txn->parent && txn->parent->tw.dirtylist)
? (intptr_t)txn->parent->tw.dirtylist->length
: -1,
txn->tw.loose_count, need, spillable_entries, spilled_entries,
txn->tw.dirtyroom);
(txn->parent && txn->parent->tw.dirtylist) ? (intptr_t)txn->parent->tw.dirtylist->length : -1,
txn->tw.loose_count, need, spillable_entries, spilled_entries, txn->tw.dirtyroom);
ENSURE(txn->env, txn->tw.loose_count + txn->tw.dirtyroom > need / 2);
#endif /* xMDBX_DEBUG_SPILLING */
done:
return likely(txn->tw.dirtyroom + txn->tw.loose_count >
((need > CURSOR_STACK_SIZE) ? CURSOR_STACK_SIZE : need))
return likely(txn->tw.dirtyroom + txn->tw.loose_count > ((need > CURSOR_STACK_SIZE) ? CURSOR_STACK_SIZE : need))
? MDBX_SUCCESS
: MDBX_TXN_FULL;
}

34
src/spill.h
View File

@ -7,10 +7,8 @@
MDBX_INTERNAL void spill_remove(MDBX_txn *txn, size_t idx, size_t npages);
MDBX_INTERNAL pnl_t spill_purge(MDBX_txn *txn);
MDBX_INTERNAL int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0,
const intptr_t wanna_spill_entries,
const intptr_t wanna_spill_npages,
const size_t need);
MDBX_INTERNAL int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intptr_t wanna_spill_entries,
const intptr_t wanna_spill_npages, const size_t need);
/*----------------------------------------------------------------------------*/
static inline size_t spill_search(const MDBX_txn *txn, pgno_t pgno) {
@ -23,8 +21,7 @@ static inline size_t spill_search(const MDBX_txn *txn, pgno_t pgno) {
return (n <= MDBX_PNL_GETSIZE(pnl) && pnl[n] == pgno) ? n : 0;
}
static inline bool spill_intersect(const MDBX_txn *txn, pgno_t pgno,
size_t npages) {
static inline bool spill_intersect(const MDBX_txn *txn, pgno_t pgno, size_t npages) {
const pnl_t pnl = txn->tw.spilled.list;
if (likely(!pnl))
return false;
@ -32,23 +29,18 @@ static inline bool spill_intersect(const MDBX_txn *txn, pgno_t pgno,
if (LOG_ENABLED(MDBX_LOG_EXTRA)) {
DEBUG_EXTRA("PNL len %zu [", len);
for (size_t i = 1; i <= len; ++i)
DEBUG_EXTRA_PRINT(" %li", (pnl[i] & 1) ? -(long)(pnl[i] >> 1)
: (long)(pnl[i] >> 1));
DEBUG_EXTRA_PRINT(" %li", (pnl[i] & 1) ? -(long)(pnl[i] >> 1) : (long)(pnl[i] >> 1));
DEBUG_EXTRA_PRINT("%s\n", "]");
}
const pgno_t spilled_range_begin = pgno << 1;
const pgno_t spilled_range_last = ((pgno + (pgno_t)npages) << 1) - 1;
#if MDBX_PNL_ASCENDING
const size_t n =
pnl_search(pnl, spilled_range_begin, (size_t)(MAX_PAGENO + 1) << 1);
tASSERT(txn, n && (n == MDBX_PNL_GETSIZE(pnl) + 1 ||
spilled_range_begin <= pnl[n]));
const size_t n = pnl_search(pnl, spilled_range_begin, (size_t)(MAX_PAGENO + 1) << 1);
tASSERT(txn, n && (n == MDBX_PNL_GETSIZE(pnl) + 1 || spilled_range_begin <= pnl[n]));
const bool rc = n <= MDBX_PNL_GETSIZE(pnl) && pnl[n] <= spilled_range_last;
#else
const size_t n =
pnl_search(pnl, spilled_range_last, (size_t)MAX_PAGENO + MAX_PAGENO + 1);
tASSERT(txn, n && (n == MDBX_PNL_GETSIZE(pnl) + 1 ||
spilled_range_last >= pnl[n]));
const size_t n = pnl_search(pnl, spilled_range_last, (size_t)MAX_PAGENO + MAX_PAGENO + 1);
tASSERT(txn, n && (n == MDBX_PNL_GETSIZE(pnl) + 1 || spilled_range_last >= pnl[n]));
const bool rc = n <= MDBX_PNL_GETSIZE(pnl) && pnl[n] >= spilled_range_begin;
#endif
if (ASSERT_ENABLED()) {
@ -60,17 +52,13 @@ static inline bool spill_intersect(const MDBX_txn *txn, pgno_t pgno,
return rc;
}
static inline int txn_spill(MDBX_txn *const txn, MDBX_cursor *const m0,
const size_t need) {
static inline int txn_spill(MDBX_txn *const txn, MDBX_cursor *const m0, const size_t need) {
tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0);
tASSERT(txn, !m0 || cursor_is_tracked(m0));
const intptr_t wanna_spill_entries =
txn->tw.dirtylist ? (need - txn->tw.dirtyroom - txn->tw.loose_count) : 0;
const intptr_t wanna_spill_entries = txn->tw.dirtylist ? (need - txn->tw.dirtyroom - txn->tw.loose_count) : 0;
const intptr_t wanna_spill_npages =
need +
(txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose
: txn->tw.writemap_dirty_npages) -
need + (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages) -
txn->tw.loose_count - txn->env->options.dp_limit;
/* production mode */

33
src/table.c
View File

@ -19,11 +19,8 @@ int tbl_setup(const MDBX_env *env, kvx_t *const kvx, const tree_t *const db) {
kvx->clc.v.lmax = env_valsize_max(env, db->flags);
if ((db->flags & (MDBX_DUPFIXED | MDBX_INTEGERDUP)) != 0 && db->dupfix_size) {
if (!MDBX_DISABLE_VALIDATION &&
unlikely(db->dupfix_size < kvx->clc.v.lmin ||
db->dupfix_size > kvx->clc.v.lmax)) {
ERROR("db.dupfix_size (%u) <> min/max value-length (%zu/%zu)",
db->dupfix_size, kvx->clc.v.lmin, kvx->clc.v.lmax);
if (!MDBX_DISABLE_VALIDATION && unlikely(db->dupfix_size < kvx->clc.v.lmin || db->dupfix_size > kvx->clc.v.lmax)) {
ERROR("db.dupfix_size (%u) <> min/max value-length (%zu/%zu)", db->dupfix_size, kvx->clc.v.lmin, kvx->clc.v.lmax);
return MDBX_CORRUPTED;
}
kvx->clc.v.lmin = kvx->clc.v.lmax = db->dupfix_size;
@ -41,10 +38,8 @@ int tbl_fetch(MDBX_txn *txn, size_t dbi) {
rc = tree_search(&couple.outer, &kvx->name, 0);
if (unlikely(rc != MDBX_SUCCESS)) {
bailout:
NOTICE("dbi %zu refs to inaccessible table `%*s` for txn %" PRIaTXN
" (err %d)",
dbi, (int)kvx->name.iov_len, (const char *)kvx->name.iov_base,
txn->txnid, rc);
NOTICE("dbi %zu refs to inaccessible table `%*s` for txn %" PRIaTXN " (err %d)", dbi, (int)kvx->name.iov_len,
(const char *)kvx->name.iov_base, txn->txnid, rc);
return (rc == MDBX_NOTFOUND) ? MDBX_BAD_DBI : rc;
}
@ -55,21 +50,18 @@ int tbl_fetch(MDBX_txn *txn, size_t dbi) {
goto bailout;
}
if (unlikely((node_flags(nsr.node) & (N_DUP | N_TREE)) != N_TREE)) {
NOTICE("dbi %zu refs to not a named table `%*s` for txn %" PRIaTXN " (%s)",
dbi, (int)kvx->name.iov_len, (const char *)kvx->name.iov_base,
txn->txnid, "wrong flags");
NOTICE("dbi %zu refs to not a named table `%*s` for txn %" PRIaTXN " (%s)", dbi, (int)kvx->name.iov_len,
(const char *)kvx->name.iov_base, txn->txnid, "wrong flags");
return MDBX_INCOMPATIBLE; /* not a named DB */
}
rc = node_read(&couple.outer, nsr.node, &data,
couple.outer.pg[couple.outer.top]);
rc = node_read(&couple.outer, nsr.node, &data, couple.outer.pg[couple.outer.top]);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
if (unlikely(data.iov_len != sizeof(tree_t))) {
NOTICE("dbi %zu refs to not a named table `%*s` for txn %" PRIaTXN " (%s)",
dbi, (int)kvx->name.iov_len, (const char *)kvx->name.iov_base,
txn->txnid, "wrong rec-size");
NOTICE("dbi %zu refs to not a named table `%*s` for txn %" PRIaTXN " (%s)", dbi, (int)kvx->name.iov_len,
(const char *)kvx->name.iov_base, txn->txnid, "wrong rec-size");
return MDBX_INCOMPATIBLE; /* not a named DB */
}
@ -80,8 +72,8 @@ int tbl_fetch(MDBX_txn *txn, size_t dbi) {
if (unlikely((db->flags & DB_PERSISTENT_FLAGS) != flags)) {
NOTICE("dbi %zu refs to the re-created table `%*s` for txn %" PRIaTXN
" with different flags (present 0x%X != wanna 0x%X)",
dbi, (int)kvx->name.iov_len, (const char *)kvx->name.iov_base,
txn->txnid, db->flags & DB_PERSISTENT_FLAGS, flags);
dbi, (int)kvx->name.iov_len, (const char *)kvx->name.iov_base, txn->txnid, db->flags & DB_PERSISTENT_FLAGS,
flags);
return MDBX_INCOMPATIBLE;
}
@ -90,8 +82,7 @@ int tbl_fetch(MDBX_txn *txn, size_t dbi) {
const txnid_t pp_txnid = couple.outer.pg[couple.outer.top]->txnid;
tASSERT(txn, txn->front_txnid >= pp_txnid);
if (unlikely(db->mod_txnid > pp_txnid)) {
ERROR("db.mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")",
db->mod_txnid, pp_txnid);
ERROR("db.mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")", db->mod_txnid, pp_txnid);
return MDBX_CORRUPTED;
}
#endif /* !MDBX_DISABLE_VALIDATION */

204
src/tls.c
View File

@ -29,21 +29,17 @@ static int uniq_peek(const osal_mmap_t *pending, osal_mmap_t *scan) {
bait = 0 /* hush MSVC warning */;
rc = osal_msync(scan, 0, sizeof(lck_t), MDBX_SYNC_DATA);
if (rc == MDBX_SUCCESS)
rc = osal_pread(pending->fd, &bait, sizeof(scan_lck->bait_uniqueness),
offsetof(lck_t, bait_uniqueness));
rc = osal_pread(pending->fd, &bait, sizeof(scan_lck->bait_uniqueness), offsetof(lck_t, bait_uniqueness));
}
if (likely(rc == MDBX_SUCCESS) &&
bait == atomic_load64(&scan_lck->bait_uniqueness, mo_AcquireRelease))
if (likely(rc == MDBX_SUCCESS) && bait == atomic_load64(&scan_lck->bait_uniqueness, mo_AcquireRelease))
rc = MDBX_RESULT_TRUE;
TRACE("uniq-peek: %s, bait 0x%016" PRIx64 ",%s rc %d",
pending_lck ? "mem" : "file", bait,
TRACE("uniq-peek: %s, bait 0x%016" PRIx64 ",%s rc %d", pending_lck ? "mem" : "file", bait,
(rc == MDBX_RESULT_TRUE) ? " found," : (rc ? " FAILED," : ""), rc);
return rc;
}
static int uniq_poke(const osal_mmap_t *pending, osal_mmap_t *scan,
uint64_t *abra) {
static int uniq_poke(const osal_mmap_t *pending, osal_mmap_t *scan, uint64_t *abra) {
if (*abra == 0) {
const uintptr_t tid = osal_thread_self();
uintptr_t uit = 0;
@ -51,9 +47,7 @@ static int uniq_poke(const osal_mmap_t *pending, osal_mmap_t *scan,
*abra = rrxmrrxmsx_0(osal_monotime() + UINT64_C(5873865991930747) * uit);
}
const uint64_t cadabra =
rrxmrrxmsx_0(*abra + UINT64_C(7680760450171793) * (unsigned)osal_getpid())
<< 24 |
*abra >> 40;
rrxmrrxmsx_0(*abra + UINT64_C(7680760450171793) * (unsigned)osal_getpid()) << 24 | *abra >> 40;
lck_t *const scan_lck = scan->lck;
atomic_store64(&scan_lck->bait_uniqueness, cadabra, mo_AcquireRelease);
*abra = *abra * UINT64_C(6364136223846793005) + 1;
@ -67,14 +61,12 @@ __cold int rthc_uniq_check(const osal_mmap_t *pending, MDBX_env **found) {
MDBX_env *const scan = rthc_table[i].env;
if (!scan->lck_mmap.lck || &scan->lck_mmap == pending)
continue;
int err =
atomic_load64(&scan->lck_mmap.lck->bait_uniqueness, mo_AcquireRelease)
? uniq_peek(pending, &scan->lck_mmap)
: uniq_poke(pending, &scan->lck_mmap, &salt);
int err = atomic_load64(&scan->lck_mmap.lck->bait_uniqueness, mo_AcquireRelease)
? uniq_peek(pending, &scan->lck_mmap)
: uniq_poke(pending, &scan->lck_mmap, &salt);
if (err == MDBX_ENODATA) {
uint64_t length = 0;
if (likely(osal_filesize(pending->fd, &length) == MDBX_SUCCESS &&
length == 0)) {
if (likely(osal_filesize(pending->fd, &length) == MDBX_SUCCESS && length == 0)) {
/* LY: skip checking since LCK-file is empty, i.e. just created. */
DEBUG("%s", "unique (new/empty lck)");
return MDBX_SUCCESS;
@ -114,8 +106,7 @@ static osal_thread_key_t rthc_key;
static mdbx_atomic_uint32_t rthc_pending;
static inline uint64_t rthc_signature(const void *addr, uint8_t kind) {
uint64_t salt = osal_thread_self() * UINT64_C(0xA2F0EEC059629A17) ^
UINT64_C(0x01E07C6FDB596497) * (uintptr_t)(addr);
uint64_t salt = osal_thread_self() * UINT64_C(0xA2F0EEC059629A17) ^ UINT64_C(0x01E07C6FDB596497) * (uintptr_t)(addr);
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
return salt << 8 | kind;
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
@ -128,45 +119,36 @@ static inline uint64_t rthc_signature(const void *addr, uint8_t kind) {
#define MDBX_THREAD_RTHC_REGISTERED(addr) rthc_signature(addr, 0x0D)
#define MDBX_THREAD_RTHC_COUNTED(addr) rthc_signature(addr, 0xC0)
static __thread uint64_t rthc_thread_state
#if __has_attribute(tls_model) && \
(defined(__PIC__) || defined(__pic__) || MDBX_BUILD_SHARED_LIBRARY)
#if __has_attribute(tls_model) && (defined(__PIC__) || defined(__pic__) || MDBX_BUILD_SHARED_LIBRARY)
__attribute__((tls_model("local-dynamic")))
#endif
;
#if defined(__APPLE__) && defined(__SANITIZE_ADDRESS__) && \
!defined(MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS)
#if defined(__APPLE__) && defined(__SANITIZE_ADDRESS__) && !defined(MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS)
/* Avoid ASAN-trap due the target TLS-variable feed by Darwin's tlv_free() */
#define MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS \
__attribute__((__no_sanitize_address__, __noinline__))
#define MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((__no_sanitize_address__, __noinline__))
#else
#define MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS inline
#endif
MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS static uint64_t rthc_read(const void *rthc) {
return *(volatile uint64_t *)rthc;
}
MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS static uint64_t rthc_read(const void *rthc) { return *(volatile uint64_t *)rthc; }
MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS static uint64_t
rthc_compare_and_clean(const void *rthc, const uint64_t signature) {
MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS static uint64_t rthc_compare_and_clean(const void *rthc, const uint64_t signature) {
#if MDBX_64BIT_CAS
return atomic_cas64((mdbx_atomic_uint64_t *)rthc, signature, 0);
#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
return atomic_cas32((mdbx_atomic_uint32_t *)rthc, (uint32_t)signature, 0);
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
return atomic_cas32((mdbx_atomic_uint32_t *)rthc, (uint32_t)(signature >> 32),
0);
return atomic_cas32((mdbx_atomic_uint32_t *)rthc, (uint32_t)(signature >> 32), 0);
#else
#error "FIXME: Unsupported byte order"
#endif
}
static inline int rthc_atexit(void (*dtor)(void *), void *obj,
void *dso_symbol) {
static inline int rthc_atexit(void (*dtor)(void *), void *obj, void *dso_symbol) {
#ifndef MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL
#if defined(LIBCXXABI_HAS_CXA_THREAD_ATEXIT_IMPL) || \
defined(HAVE___CXA_THREAD_ATEXIT_IMPL) || __GLIBC_PREREQ(2, 18) || \
defined(BIONIC)
#if defined(LIBCXXABI_HAS_CXA_THREAD_ATEXIT_IMPL) || defined(HAVE___CXA_THREAD_ATEXIT_IMPL) || \
__GLIBC_PREREQ(2, 18) || defined(BIONIC)
#define MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL 1
#else
#define MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL 0
@ -174,11 +156,9 @@ static inline int rthc_atexit(void (*dtor)(void *), void *obj,
#endif /* MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL */
#ifndef MDBX_HAVE_CXA_THREAD_ATEXIT
#if defined(LIBCXXABI_HAS_CXA_THREAD_ATEXIT) || \
defined(HAVE___CXA_THREAD_ATEXIT)
#if defined(LIBCXXABI_HAS_CXA_THREAD_ATEXIT) || defined(HAVE___CXA_THREAD_ATEXIT)
#define MDBX_HAVE_CXA_THREAD_ATEXIT 1
#elif !MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL && \
(defined(__linux__) || defined(__gnu_linux__))
#elif !MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL && (defined(__linux__) || defined(__gnu_linux__))
#define MDBX_HAVE_CXA_THREAD_ATEXIT 1
#else
#define MDBX_HAVE_CXA_THREAD_ATEXIT 0
@ -190,13 +170,11 @@ static inline int rthc_atexit(void (*dtor)(void *), void *obj,
#define __cxa_thread_atexit __cxa_thread_atexit_impl
#endif
#if MDBX_HAVE_CXA_THREAD_ATEXIT || defined(__cxa_thread_atexit)
extern int __cxa_thread_atexit(void (*dtor)(void *), void *obj,
void *dso_symbol) MDBX_WEAK_IMPORT_ATTRIBUTE;
extern int __cxa_thread_atexit(void (*dtor)(void *), void *obj, void *dso_symbol) MDBX_WEAK_IMPORT_ATTRIBUTE;
if (&__cxa_thread_atexit)
rc = __cxa_thread_atexit(dtor, obj, dso_symbol);
#elif defined(__APPLE__) || defined(_DARWIN_C_SOURCE)
extern void _tlv_atexit(void (*termfunc)(void *objAddr), void *objAddr)
MDBX_WEAK_IMPORT_ATTRIBUTE;
extern void _tlv_atexit(void (*termfunc)(void *objAddr), void *objAddr) MDBX_WEAK_IMPORT_ATTRIBUTE;
if (&_tlv_atexit) {
(void)dso_symbol;
_tlv_atexit(dtor, obj);
@ -250,8 +228,7 @@ static inline int thread_key_create(osal_thread_key_t *key) {
#else
rc = pthread_key_create(key, nullptr);
#endif
TRACE("&key = %p, value %" PRIuPTR ", rc %d", __Wpedantic_format_voidptr(key),
(uintptr_t)*key, rc);
TRACE("&key = %p, value %" PRIuPTR ", rc %d", __Wpedantic_format_voidptr(key), (uintptr_t)*key, rc);
return rc;
}
@ -259,21 +236,17 @@ void thread_rthc_set(osal_thread_key_t key, const void *value) {
#if defined(_WIN32) || defined(_WIN64)
ENSURE(nullptr, TlsSetValue(key, (void *)value));
#else
const uint64_t sign_registered =
MDBX_THREAD_RTHC_REGISTERED(&rthc_thread_state);
const uint64_t sign_registered = MDBX_THREAD_RTHC_REGISTERED(&rthc_thread_state);
const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(&rthc_thread_state);
if (value && unlikely(rthc_thread_state != sign_registered &&
rthc_thread_state != sign_counted)) {
if (value && unlikely(rthc_thread_state != sign_registered && rthc_thread_state != sign_counted)) {
rthc_thread_state = sign_registered;
TRACE("thread registered 0x%" PRIxPTR, osal_thread_self());
if (rthc_atexit(rthc_thread_dtor, &rthc_thread_state,
(void *)&mdbx_version /* dso_anchor */)) {
if (rthc_atexit(rthc_thread_dtor, &rthc_thread_state, (void *)&mdbx_version /* dso_anchor */)) {
ENSURE(nullptr, pthread_setspecific(rthc_key, &rthc_thread_state) == 0);
rthc_thread_state = sign_counted;
const unsigned count_before = atomic_add32(&rthc_pending, 1);
ENSURE(nullptr, count_before < INT_MAX);
NOTICE("fallback to pthreads' tsd, key %" PRIuPTR ", count %u",
(uintptr_t)rthc_key, count_before);
NOTICE("fallback to pthreads' tsd, key %" PRIuPTR ", count %u", (uintptr_t)rthc_key, count_before);
(void)count_before;
}
}
@ -286,11 +259,9 @@ __cold void rthc_thread_dtor(void *rthc) {
rthc_lock();
const uint32_t current_pid = osal_getpid();
#if defined(_WIN32) || defined(_WIN64)
TRACE(">> pid %d, thread 0x%" PRIxPTR ", module %p", current_pid,
osal_thread_self(), rthc);
TRACE(">> pid %d, thread 0x%" PRIxPTR ", module %p", current_pid, osal_thread_self(), rthc);
#else
TRACE(">> pid %d, thread 0x%" PRIxPTR ", rthc %p", current_pid,
osal_thread_self(), rthc);
TRACE(">> pid %d, thread 0x%" PRIxPTR ", rthc %p", current_pid, osal_thread_self(), rthc);
#endif
for (size_t i = 0; i < rthc_count; ++i) {
@ -306,22 +277,18 @@ __cold void rthc_thread_dtor(void *rthc) {
continue;
#if !defined(_WIN32) && !defined(_WIN64)
if (pthread_setspecific(env->me_txkey, nullptr) != 0) {
TRACE("== thread 0x%" PRIxPTR
", rthc %p: ignore race with tsd-key deletion",
osal_thread_self(), __Wpedantic_format_voidptr(reader));
TRACE("== thread 0x%" PRIxPTR ", rthc %p: ignore race with tsd-key deletion", osal_thread_self(),
__Wpedantic_format_voidptr(reader));
continue /* ignore race with tsd-key deletion by mdbx_env_close() */;
}
#endif
TRACE("== thread 0x%" PRIxPTR
", rthc %p, [%zi], %p ... %p (%+i), rtch-pid %i, "
TRACE("== thread 0x%" PRIxPTR ", rthc %p, [%zi], %p ... %p (%+i), rtch-pid %i, "
"current-pid %i",
osal_thread_self(), __Wpedantic_format_voidptr(reader), i,
__Wpedantic_format_voidptr(begin), __Wpedantic_format_voidptr(end),
(int)(reader - begin), reader->pid.weak, current_pid);
osal_thread_self(), __Wpedantic_format_voidptr(reader), i, __Wpedantic_format_voidptr(begin),
__Wpedantic_format_voidptr(end), (int)(reader - begin), reader->pid.weak, current_pid);
if (atomic_load32(&reader->pid, mo_Relaxed) == current_pid) {
TRACE("==== thread 0x%" PRIxPTR ", rthc %p, cleanup", osal_thread_self(),
__Wpedantic_format_voidptr(reader));
TRACE("==== thread 0x%" PRIxPTR ", rthc %p, cleanup", osal_thread_self(), __Wpedantic_format_voidptr(reader));
(void)atomic_cas32(&reader->pid, current_pid, 0);
atomic_store32(&env->lck->rdt_refresh_flag, true, mo_Relaxed);
}
@ -334,26 +301,20 @@ __cold void rthc_thread_dtor(void *rthc) {
const uint64_t sign_registered = MDBX_THREAD_RTHC_REGISTERED(rthc);
const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(rthc);
const uint64_t state = rthc_read(rthc);
if (state == sign_registered &&
rthc_compare_and_clean(rthc, sign_registered)) {
TRACE("== thread 0x%" PRIxPTR
", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")",
osal_thread_self(), rthc, osal_getpid(), "registered", state);
} else if (state == sign_counted &&
rthc_compare_and_clean(rthc, sign_counted)) {
TRACE("== thread 0x%" PRIxPTR
", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")",
osal_thread_self(), rthc, osal_getpid(), "counted", state);
if (state == sign_registered && rthc_compare_and_clean(rthc, sign_registered)) {
TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), rthc,
osal_getpid(), "registered", state);
} else if (state == sign_counted && rthc_compare_and_clean(rthc, sign_counted)) {
TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), rthc,
osal_getpid(), "counted", state);
ENSURE(nullptr, atomic_sub32(&rthc_pending, 1) > 0);
} else {
WARNING("thread 0x%" PRIxPTR
", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")",
osal_thread_self(), rthc, osal_getpid(), "wrong", state);
WARNING("thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), rthc,
osal_getpid(), "wrong", state);
}
if (atomic_load32(&rthc_pending, mo_AcquireRelease) == 0) {
TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, wake", osal_thread_self(),
rthc, osal_getpid());
TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, wake", osal_thread_self(), rthc, osal_getpid());
ENSURE(nullptr, pthread_cond_broadcast(&rthc_cond) == 0);
}
@ -367,8 +328,7 @@ __cold void rthc_thread_dtor(void *rthc) {
}
__cold int rthc_register(MDBX_env *const env) {
TRACE(">> env %p, rthc_count %u, rthc_limit %u",
__Wpedantic_format_voidptr(env), rthc_count, rthc_limit);
TRACE(">> env %p, rthc_count %u, rthc_limit %u", __Wpedantic_format_voidptr(env), rthc_count, rthc_limit);
int rc = MDBX_SUCCESS;
for (size_t i = 0; i < rthc_count; ++i)
@ -380,8 +340,7 @@ __cold int rthc_register(MDBX_env *const env) {
env->me_txkey = 0;
if (unlikely(rthc_count == rthc_limit)) {
rthc_entry_t *new_table =
osal_realloc((rthc_table == rthc_table_static) ? nullptr : rthc_table,
sizeof(rthc_entry_t) * rthc_limit * 2);
osal_realloc((rthc_table == rthc_table_static) ? nullptr : rthc_table, sizeof(rthc_entry_t) * rthc_limit * 2);
if (unlikely(new_table == nullptr)) {
rc = MDBX_ENOMEM;
goto bailout;
@ -400,14 +359,12 @@ __cold int rthc_register(MDBX_env *const env) {
}
rthc_table[rthc_count].env = env;
TRACE("== [%i] = env %p, key %" PRIuPTR, rthc_count,
__Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey);
TRACE("== [%i] = env %p, key %" PRIuPTR, rthc_count, __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey);
++rthc_count;
bailout:
TRACE("<< env %p, key %" PRIuPTR ", rthc_count %u, rthc_limit %u, rc %d",
__Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey, rthc_count,
rthc_limit, rc);
TRACE("<< env %p, key %" PRIuPTR ", rthc_count %u, rthc_limit %u, rc %d", __Wpedantic_format_voidptr(env),
(uintptr_t)env->me_txkey, rthc_count, rthc_limit, rc);
return rc;
}
@ -418,10 +375,8 @@ __cold static int rthc_drown(MDBX_env *const env) {
if (likely(env->lck_mmap.lck && current_pid == env->pid)) {
reader_slot_t *const begin = &env->lck_mmap.lck->rdt[0];
reader_slot_t *const end = &env->lck_mmap.lck->rdt[env->max_readers];
TRACE("== %s env %p pid %d, readers %p ...%p, current-pid %d",
(current_pid == env->pid) ? "cleanup" : "skip",
__Wpedantic_format_voidptr(env), env->pid,
__Wpedantic_format_voidptr(begin), __Wpedantic_format_voidptr(end),
TRACE("== %s env %p pid %d, readers %p ...%p, current-pid %d", (current_pid == env->pid) ? "cleanup" : "skip",
__Wpedantic_format_voidptr(env), env->pid, __Wpedantic_format_voidptr(begin), __Wpedantic_format_voidptr(end),
current_pid);
bool cleaned = false;
for (reader_slot_t *r = begin; r < end; ++r) {
@ -434,8 +389,7 @@ __cold static int rthc_drown(MDBX_env *const env) {
if (cleaned)
atomic_store32(&env->lck_mmap.lck->rdt_refresh_flag, true, mo_Relaxed);
rc = rthc_uniq_check(&env->lck_mmap, &inprocess_neighbor);
if (!inprocess_neighbor && env->registered_reader_pid &&
env->lck_mmap.fd != INVALID_HANDLE_VALUE) {
if (!inprocess_neighbor && env->registered_reader_pid && env->lck_mmap.fd != INVALID_HANDLE_VALUE) {
int err = lck_rpid_clear(env);
rc = rc ? rc : err;
}
@ -446,9 +400,8 @@ __cold static int rthc_drown(MDBX_env *const env) {
}
__cold int rthc_remove(MDBX_env *const env) {
TRACE(">>> env %p, key %zu, rthc_count %u, rthc_limit %u",
__Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey, rthc_count,
rthc_limit);
TRACE(">>> env %p, key %zu, rthc_count %u, rthc_limit %u", __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey,
rthc_count, rthc_limit);
int rc = MDBX_SUCCESS;
if (likely(env->pid))
@ -469,9 +422,8 @@ __cold int rthc_remove(MDBX_env *const env) {
}
}
TRACE("<<< %p, key %zu, rthc_count %u, rthc_limit %u",
__Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey, rthc_count,
rthc_limit);
TRACE("<<< %p, key %zu, rthc_count %u, rthc_limit %u", __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey,
rthc_count, rthc_limit);
return rc;
}
@ -508,8 +460,8 @@ __cold void rthc_ctor(void) {
#else
ENSURE(nullptr, pthread_atfork(nullptr, nullptr, rthc_afterfork) == 0);
ENSURE(nullptr, pthread_key_create(&rthc_key, rthc_thread_dtor) == 0);
TRACE("pid %d, &mdbx_rthc_key = %p, value 0x%x", osal_getpid(),
__Wpedantic_format_voidptr(&rthc_key), (unsigned)rthc_key);
TRACE("pid %d, &mdbx_rthc_key = %p, value 0x%x", osal_getpid(), __Wpedantic_format_voidptr(&rthc_key),
(unsigned)rthc_key);
#endif
}
@ -517,33 +469,23 @@ __cold void rthc_dtor(const uint32_t current_pid) {
rthc_lock();
#if !defined(_WIN32) && !defined(_WIN64)
uint64_t *rthc = pthread_getspecific(rthc_key);
TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status 0x%08" PRIx64
", left %d",
osal_thread_self(), __Wpedantic_format_voidptr(rthc), current_pid,
rthc ? rthc_read(rthc) : ~UINT64_C(0),
TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status 0x%08" PRIx64 ", left %d", osal_thread_self(),
__Wpedantic_format_voidptr(rthc), current_pid, rthc ? rthc_read(rthc) : ~UINT64_C(0),
atomic_load32(&rthc_pending, mo_Relaxed));
if (rthc) {
const uint64_t sign_registered = MDBX_THREAD_RTHC_REGISTERED(rthc);
const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(rthc);
const uint64_t state = rthc_read(rthc);
if (state == sign_registered &&
rthc_compare_and_clean(rthc, sign_registered)) {
TRACE("== thread 0x%" PRIxPTR
", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")",
osal_thread_self(), __Wpedantic_format_voidptr(rthc), current_pid,
"registered", state);
} else if (state == sign_counted &&
rthc_compare_and_clean(rthc, sign_counted)) {
TRACE("== thread 0x%" PRIxPTR
", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")",
osal_thread_self(), __Wpedantic_format_voidptr(rthc), current_pid,
"counted", state);
if (state == sign_registered && rthc_compare_and_clean(rthc, sign_registered)) {
TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(),
__Wpedantic_format_voidptr(rthc), current_pid, "registered", state);
} else if (state == sign_counted && rthc_compare_and_clean(rthc, sign_counted)) {
TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(),
__Wpedantic_format_voidptr(rthc), current_pid, "counted", state);
ENSURE(nullptr, atomic_sub32(&rthc_pending, 1) > 0);
} else {
WARNING("thread 0x%" PRIxPTR
", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")",
osal_thread_self(), __Wpedantic_format_voidptr(rthc), current_pid,
"wrong", state);
WARNING("thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(),
__Wpedantic_format_voidptr(rthc), current_pid, "wrong", state);
}
}
@ -558,8 +500,7 @@ __cold void rthc_dtor(const uint32_t current_pid) {
abstime.tv_sec += 600;
#endif
for (unsigned left;
(left = atomic_load32(&rthc_pending, mo_AcquireRelease)) > 0;) {
for (unsigned left; (left = atomic_load32(&rthc_pending, mo_AcquireRelease)) > 0;) {
NOTICE("tls-cleanup: pid %d, pending %u, wait for...", current_pid, left);
const int rc = pthread_cond_timedwait(&rthc_cond, &rthc_mutex, &abstime);
if (rc && rc != EINTR)
@ -581,9 +522,8 @@ __cold void rthc_dtor(const uint32_t current_pid) {
for (reader_slot_t *reader = begin; reader < end; ++reader) {
TRACE("== [%zi] = key %" PRIuPTR ", %p ... %p, rthc %p (%+i), "
"rthc-pid %i, current-pid %i",
i, (uintptr_t)env->me_txkey, __Wpedantic_format_voidptr(begin),
__Wpedantic_format_voidptr(end), __Wpedantic_format_voidptr(reader),
(int)(reader - begin), reader->pid.weak, current_pid);
i, (uintptr_t)env->me_txkey, __Wpedantic_format_voidptr(begin), __Wpedantic_format_voidptr(end),
__Wpedantic_format_voidptr(reader), (int)(reader - begin), reader->pid.weak, current_pid);
if (atomic_load32(&reader->pid, mo_Relaxed) == current_pid) {
(void)atomic_cas32(&reader->pid, current_pid, 0);
TRACE("== cleanup %p", __Wpedantic_format_voidptr(reader));

209
src/tools/chk.c
View File

@ -30,8 +30,7 @@ static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
static uint64_t GetMilliseconds(void) {
LARGE_INTEGER Counter, Frequency;
return (QueryPerformanceFrequency(&Frequency) &&
QueryPerformanceCounter(&Counter))
return (QueryPerformanceFrequency(&Frequency) && QueryPerformanceCounter(&Counter))
? Counter.QuadPart * 1000ul / Frequency.QuadPart
: 0;
}
@ -93,9 +92,8 @@ static void lf_flush(void) {
}
static bool silently(enum MDBX_chk_severity severity) {
int cutoff =
chk.scope ? chk.scope->verbosity >> MDBX_chk_severity_prio_shift
: verbose + (MDBX_chk_result >> MDBX_chk_severity_prio_shift);
int cutoff = chk.scope ? chk.scope->verbosity >> MDBX_chk_severity_prio_shift
: verbose + (MDBX_chk_result >> MDBX_chk_severity_prio_shift);
int prio = (severity >> MDBX_chk_severity_prio_shift);
if (chk.scope && chk.scope->stage == MDBX_chk_tables && verbose < 2)
prio += 1;
@ -125,11 +123,9 @@ static FILE *prefix(enum MDBX_chk_severity severity) {
" ////// " // F +2
};
const bool nl =
line_struct.scope_depth != chk.scope_nesting ||
(line_struct.severity != severity &&
(line_struct.severity != MDBX_chk_processing ||
severity < MDBX_chk_result || severity > MDBX_chk_resolution));
const bool nl = line_struct.scope_depth != chk.scope_nesting ||
(line_struct.severity != severity && (line_struct.severity != MDBX_chk_processing ||
severity < MDBX_chk_result || severity > MDBX_chk_resolution));
if (nl)
lf();
if (severity < MDBX_chk_warning)
@ -157,8 +153,7 @@ static void suffix(size_t cookie, const char *str) {
}
}
static size_t MDBX_PRINTF_ARGS(2, 3)
print(enum MDBX_chk_severity severity, const char *msg, ...) {
static size_t MDBX_PRINTF_ARGS(2, 3) print(enum MDBX_chk_severity severity, const char *msg, ...) {
FILE *out = prefix(severity);
if (out) {
va_list args;
@ -171,8 +166,7 @@ static size_t MDBX_PRINTF_ARGS(2, 3)
return 0;
}
static FILE *MDBX_PRINTF_ARGS(2, 3)
print_ln(enum MDBX_chk_severity severity, const char *msg, ...) {
static FILE *MDBX_PRINTF_ARGS(2, 3) print_ln(enum MDBX_chk_severity severity, const char *msg, ...) {
FILE *out = prefix(severity);
if (out) {
va_list args;
@ -185,15 +179,12 @@ static FILE *MDBX_PRINTF_ARGS(2, 3)
return out;
}
static void logger(MDBX_log_level_t level, const char *function, int line,
const char *fmt, va_list args) {
static void logger(MDBX_log_level_t level, const char *function, int line, const char *fmt, va_list args) {
if (level <= MDBX_LOG_ERROR)
mdbx_env_chk_encount_problem(&chk);
const unsigned kind = (level > MDBX_LOG_NOTICE)
? level - MDBX_LOG_NOTICE +
(MDBX_chk_extra & MDBX_chk_severity_kind_mask)
: level;
const unsigned kind =
(level > MDBX_LOG_NOTICE) ? level - MDBX_LOG_NOTICE + (MDBX_chk_extra & MDBX_chk_severity_kind_mask) : level;
const unsigned prio = kind << MDBX_chk_severity_prio_shift;
enum MDBX_chk_severity severity = prio + kind;
FILE *out = prefix(severity);
@ -204,8 +195,8 @@ static void logger(MDBX_log_level_t level, const char *function, int line,
if (have_lf)
for (size_t i = 0; i < line_struct.scope_depth; ++i)
fputs(" ", out);
fprintf(out, have_lf ? " %s(), %u" : " (%s:%u)",
function + (strncmp(function, "mdbx_", 5) ? 0 : 5), line);
fprintf(out, have_lf ? " %s(), %u" : " (%s:%u)", function + (strncmp(function, "mdbx_", 5) ? 0 : 5),
line);
lf();
} else if (have_lf) {
line_struct.empty = true;
@ -249,8 +240,8 @@ static bool check_break(MDBX_chk_context_t *ctx) {
return true;
}
static int scope_push(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *scope,
MDBX_chk_scope_t *inner, const char *fmt, va_list args) {
static int scope_push(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *scope, MDBX_chk_scope_t *inner, const char *fmt,
va_list args) {
(void)scope;
if (fmt && *fmt) {
FILE *out = prefix(MDBX_chk_processing);
@ -264,22 +255,19 @@ static int scope_push(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *scope,
return MDBX_SUCCESS;
}
static void scope_pop(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *scope,
MDBX_chk_scope_t *inner) {
static void scope_pop(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *scope, MDBX_chk_scope_t *inner) {
(void)ctx;
(void)scope;
suffix(inner->usr_o.number, inner->subtotal_issues ? "error(s)" : "done");
flush();
}
static MDBX_chk_user_table_cookie_t *table_filter(MDBX_chk_context_t *ctx,
const MDBX_val *name,
static MDBX_chk_user_table_cookie_t *table_filter(MDBX_chk_context_t *ctx, const MDBX_val *name,
MDBX_db_flags_t flags) {
(void)ctx;
(void)flags;
return (!only_table.iov_base ||
(only_table.iov_len == name->iov_len &&
memcmp(only_table.iov_base, name->iov_base, name->iov_len) == 0))
(only_table.iov_len == name->iov_len && memcmp(only_table.iov_base, name->iov_base, name->iov_len) == 0))
? (void *)(intptr_t)-1
: nullptr;
}
@ -293,8 +281,7 @@ static int stage_begin(MDBX_chk_context_t *ctx, enum MDBX_chk_stage stage) {
}
static int conclude(MDBX_chk_context_t *ctx);
static int stage_end(MDBX_chk_context_t *ctx, enum MDBX_chk_stage stage,
int err) {
static int stage_end(MDBX_chk_context_t *ctx, enum MDBX_chk_stage stage, int err) {
if (stage == MDBX_chk_conclude && !err)
err = conclude(ctx);
suffix(anchor_lineno, err ? "error(s)" : "done");
@ -303,14 +290,12 @@ static int stage_end(MDBX_chk_context_t *ctx, enum MDBX_chk_stage stage,
return err;
}
static MDBX_chk_line_t *print_begin(MDBX_chk_context_t *ctx,
enum MDBX_chk_severity severity) {
static MDBX_chk_line_t *print_begin(MDBX_chk_context_t *ctx, enum MDBX_chk_severity severity) {
(void)ctx;
if (silently(severity))
return nullptr;
if (line_struct.ctx) {
if (line_struct.severity == MDBX_chk_processing &&
severity >= MDBX_chk_result && severity <= MDBX_chk_resolution &&
if (line_struct.severity == MDBX_chk_processing && severity >= MDBX_chk_result && severity <= MDBX_chk_resolution &&
line_output)
fputc(' ', line_output);
else
@ -356,39 +341,36 @@ static const MDBX_chk_callbacks_t cb = {.check_break = check_break,
.print_format = print_format};
static void usage(char *prog) {
fprintf(
stderr,
"usage: %s "
"[-V] [-v] [-q] [-c] [-0|1|2] [-w] [-d] [-i] [-s table] [-u|U] dbpath\n"
" -V\t\tprint version and exit\n"
" -v\t\tmore verbose, could be repeated upto 9 times for extra details\n"
" -q\t\tbe quiet\n"
" -c\t\tforce cooperative mode (don't try exclusive)\n"
" -w\t\twrite-mode checking\n"
" -d\t\tdisable page-by-page traversal of B-tree\n"
" -i\t\tignore wrong order errors (for custom comparators case)\n"
" -s table\tprocess a specific subdatabase only\n"
" -u\t\twarmup database before checking\n"
" -U\t\twarmup and try lock database pages in memory before checking\n"
" -0|1|2\tforce using specific meta-page 0, or 2 for checking\n"
" -t\t\tturn to a specified meta-page on successful check\n"
" -T\t\tturn to a specified meta-page EVEN ON UNSUCCESSFUL CHECK!\n",
prog);
fprintf(stderr,
"usage: %s "
"[-V] [-v] [-q] [-c] [-0|1|2] [-w] [-d] [-i] [-s table] [-u|U] dbpath\n"
" -V\t\tprint version and exit\n"
" -v\t\tmore verbose, could be repeated upto 9 times for extra details\n"
" -q\t\tbe quiet\n"
" -c\t\tforce cooperative mode (don't try exclusive)\n"
" -w\t\twrite-mode checking\n"
" -d\t\tdisable page-by-page traversal of B-tree\n"
" -i\t\tignore wrong order errors (for custom comparators case)\n"
" -s table\tprocess a specific subdatabase only\n"
" -u\t\twarmup database before checking\n"
" -U\t\twarmup and try lock database pages in memory before checking\n"
" -0|1|2\tforce using specific meta-page 0, or 2 for checking\n"
" -t\t\tturn to a specified meta-page on successful check\n"
" -T\t\tturn to a specified meta-page EVEN ON UNSUCCESSFUL CHECK!\n",
prog);
exit(EXIT_INTERRUPTED);
}
static int conclude(MDBX_chk_context_t *ctx) {
int err = MDBX_SUCCESS;
if (ctx->result.total_problems == 1 && ctx->result.problems_meta == 1 &&
(chk_flags &
(MDBX_CHK_SKIP_BTREE_TRAVERSAL | MDBX_CHK_SKIP_KV_TRAVERSAL)) == 0 &&
(env_flags & MDBX_RDONLY) == 0 && !only_table.iov_base &&
stuck_meta < 0 && ctx->result.steady_txnid < ctx->result.recent_txnid) {
const size_t step_lineno =
print(MDBX_chk_resolution,
"Perform sync-to-disk for make steady checkpoint"
" at txn-id #%" PRIi64 "...",
ctx->result.recent_txnid);
(chk_flags & (MDBX_CHK_SKIP_BTREE_TRAVERSAL | MDBX_CHK_SKIP_KV_TRAVERSAL)) == 0 &&
(env_flags & MDBX_RDONLY) == 0 && !only_table.iov_base && stuck_meta < 0 &&
ctx->result.steady_txnid < ctx->result.recent_txnid) {
const size_t step_lineno = print(MDBX_chk_resolution,
"Perform sync-to-disk for make steady checkpoint"
" at txn-id #%" PRIi64 "...",
ctx->result.recent_txnid);
flush();
err = error_fn("walk_pages", mdbx_env_sync_ex(ctx->env, true, false));
if (err == MDBX_SUCCESS) {
@ -398,19 +380,13 @@ static int conclude(MDBX_chk_context_t *ctx) {
}
}
if (turn_meta && stuck_meta >= 0 &&
(chk_flags &
(MDBX_CHK_SKIP_BTREE_TRAVERSAL | MDBX_CHK_SKIP_KV_TRAVERSAL)) == 0 &&
!only_table.iov_base &&
(env_flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == MDBX_EXCLUSIVE) {
const bool successful_check =
(err | ctx->result.total_problems | ctx->result.problems_meta) == 0;
if (turn_meta && stuck_meta >= 0 && (chk_flags & (MDBX_CHK_SKIP_BTREE_TRAVERSAL | MDBX_CHK_SKIP_KV_TRAVERSAL)) == 0 &&
!only_table.iov_base && (env_flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == MDBX_EXCLUSIVE) {
const bool successful_check = (err | ctx->result.total_problems | ctx->result.problems_meta) == 0;
if (successful_check || force_turn_meta) {
const size_t step_lineno = print(
MDBX_chk_resolution,
"Performing turn to the specified meta-page (%d) due to %s!",
stuck_meta,
successful_check ? "successful check" : "the -T option was given");
const size_t step_lineno =
print(MDBX_chk_resolution, "Performing turn to the specified meta-page (%d) due to %s!", stuck_meta,
successful_check ? "successful check" : "the -T option was given");
flush();
err = mdbx_env_turn_for_recovery(ctx->env, stuck_meta);
if (err != MDBX_SUCCESS)
@ -475,12 +451,9 @@ int main(int argc, char *argv[]) {
" - build: %s for %s by %s\n"
" - flags: %s\n"
" - options: %s\n",
mdbx_version.major, mdbx_version.minor, mdbx_version.patch,
mdbx_version.tweak, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit,
mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
mdbx_build.options);
mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor,
mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options);
return EXIT_SUCCESS;
case 'v':
if (verbose >= 9 && 0)
@ -546,8 +519,7 @@ int main(int argc, char *argv[]) {
break;
case 'U':
warmup = true;
warmup_flags =
MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock;
warmup_flags = MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock;
break;
default:
usage(prog);
@ -566,21 +538,17 @@ int main(int argc, char *argv[]) {
}
if (turn_meta) {
if (stuck_meta < 0) {
error_fmt(
"meta-page must be specified (by -0, -1 or -2 options) to turn to "
"it.");
error_fmt("meta-page must be specified (by -0, -1 or -2 options) to turn to "
"it.");
rc = EXIT_INTERRUPTED;
}
if (env_flags & MDBX_RDONLY) {
error_fmt(
"write-mode must be enabled to turn to the specified meta-page.");
error_fmt("write-mode must be enabled to turn to the specified meta-page.");
rc = EXIT_INTERRUPTED;
}
if (only_table.iov_base || (chk_flags & (MDBX_CHK_SKIP_BTREE_TRAVERSAL |
MDBX_CHK_SKIP_KV_TRAVERSAL))) {
error_fmt(
"whole database checking with b-tree traversal are required to turn "
"to the specified meta-page.");
if (only_table.iov_base || (chk_flags & (MDBX_CHK_SKIP_BTREE_TRAVERSAL | MDBX_CHK_SKIP_KV_TRAVERSAL))) {
error_fmt("whole database checking with b-tree traversal are required to turn "
"to the specified meta-page.");
rc = EXIT_INTERRUPTED;
}
}
@ -604,20 +572,15 @@ int main(int argc, char *argv[]) {
print(MDBX_chk_result,
"mdbx_chk %s (%s, T-%s)\nRunning for %s in 'read-%s' mode with "
"verbosity level %u (%s)...",
mdbx_version.git.describe, mdbx_version.git.datetime,
mdbx_version.git.tree, envname,
mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_version.git.tree, envname,
(env_flags & MDBX_RDONLY) ? "only" : "write", verbose,
(verbose > 8)
? (MDBX_DEBUG ? "extra details for debugging"
: "same as 8 for non-debug builds with MDBX_DEBUG=0")
? (MDBX_DEBUG ? "extra details for debugging" : "same as 8 for non-debug builds with MDBX_DEBUG=0")
: "of 0..9");
lf_flush();
mdbx_setup_debug((verbose + MDBX_LOG_WARN < MDBX_LOG_TRACE)
? (MDBX_log_level_t)(verbose + MDBX_LOG_WARN)
: MDBX_LOG_TRACE,
MDBX_DBG_DUMP | MDBX_DBG_ASSERT | MDBX_DBG_AUDIT |
MDBX_DBG_LEGACY_OVERLAP | MDBX_DBG_DONT_UPGRADE,
logger);
mdbx_setup_debug(
(verbose + MDBX_LOG_WARN < MDBX_LOG_TRACE) ? (MDBX_log_level_t)(verbose + MDBX_LOG_WARN) : MDBX_LOG_TRACE,
MDBX_DBG_DUMP | MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_LEGACY_OVERLAP | MDBX_DBG_DONT_UPGRADE, logger);
rc = mdbx_env_create(&env);
if (rc) {
@ -632,18 +595,16 @@ int main(int argc, char *argv[]) {
}
if (stuck_meta >= 0) {
rc = mdbx_env_open_for_recovery(env, envname, stuck_meta,
(env_flags & MDBX_RDONLY) ? false : true);
rc = mdbx_env_open_for_recovery(env, envname, stuck_meta, (env_flags & MDBX_RDONLY) ? false : true);
} else {
rc = mdbx_env_open(env, envname, env_flags, 0);
if ((env_flags & MDBX_EXCLUSIVE) &&
(rc == MDBX_BUSY ||
if ((env_flags & MDBX_EXCLUSIVE) && (rc == MDBX_BUSY ||
#if defined(_WIN32) || defined(_WIN64)
rc == ERROR_LOCK_VIOLATION || rc == ERROR_SHARING_VIOLATION
rc == ERROR_LOCK_VIOLATION || rc == ERROR_SHARING_VIOLATION
#else
rc == EBUSY || rc == EAGAIN
rc == EBUSY || rc == EAGAIN
#endif
)) {
)) {
env_flags &= ~MDBX_EXCLUSIVE;
rc = mdbx_env_open(env, envname, env_flags | MDBX_ACCEDE, 0);
}
@ -652,13 +613,10 @@ int main(int argc, char *argv[]) {
if (rc) {
error_fn("mdbx_env_open", rc);
if (rc == MDBX_WANNA_RECOVERY && (env_flags & MDBX_RDONLY))
print_ln(MDBX_chk_result,
"Please run %s in the read-write mode (with '-w' option).",
prog);
print_ln(MDBX_chk_result, "Please run %s in the read-write mode (with '-w' option).", prog);
goto bailout;
}
print_ln(MDBX_chk_verbose, "%s mode",
(env_flags & MDBX_EXCLUSIVE) ? "monopolistic" : "cooperative");
print_ln(MDBX_chk_verbose, "%s mode", (env_flags & MDBX_EXCLUSIVE) ? "monopolistic" : "cooperative");
if (warmup) {
anchor_lineno = print(MDBX_chk_verbose, "warming up...");
@ -671,9 +629,7 @@ int main(int argc, char *argv[]) {
suffix(anchor_lineno, rc ? "timeout" : "done");
}
rc = mdbx_env_chk(env, &cb, &chk, chk_flags,
MDBX_chk_result + (verbose << MDBX_chk_severity_prio_shift),
0);
rc = mdbx_env_chk(env, &cb, &chk, chk_flags, MDBX_chk_result + (verbose << MDBX_chk_severity_prio_shift), 0);
if (rc) {
if (chk.result.total_problems == 0)
error_fn("mdbx_env_chk", rc);
@ -683,8 +639,7 @@ int main(int argc, char *argv[]) {
bailout:
if (env) {
const bool dont_sync = rc != 0 || chk.result.total_problems ||
(chk_flags & MDBX_CHK_READWRITE) == 0;
const bool dont_sync = rc != 0 || chk.result.total_problems || (chk_flags & MDBX_CHK_READWRITE) == 0;
mdbx_env_close_ex(env, dont_sync);
}
flush();
@ -702,21 +657,17 @@ bailout:
error_fn("clock_gettime", errno);
return EXIT_FAILURE_SYS;
}
elapsed = timestamp_finish.tv_sec - timestamp_start.tv_sec +
(timestamp_finish.tv_nsec - timestamp_start.tv_nsec) * 1e-9;
elapsed =
timestamp_finish.tv_sec - timestamp_start.tv_sec + (timestamp_finish.tv_nsec - timestamp_start.tv_nsec) * 1e-9;
#endif /* !WINDOWS */
if (chk.result.total_problems) {
print_ln(MDBX_chk_result,
"Total %" PRIuSIZE " error%s detected, elapsed %.3f seconds.",
chk.result.total_problems,
print_ln(MDBX_chk_result, "Total %" PRIuSIZE " error%s detected, elapsed %.3f seconds.", chk.result.total_problems,
(chk.result.total_problems > 1) ? "s are" : " is", elapsed);
if (chk.result.problems_meta || chk.result.problems_kv ||
chk.result.problems_gc)
if (chk.result.problems_meta || chk.result.problems_kv || chk.result.problems_gc)
return EXIT_FAILURE_CHECK_MAJOR;
return EXIT_FAILURE_CHECK_MINOR;
}
print_ln(MDBX_chk_result, "No error is detected, elapsed %.3f seconds.",
elapsed);
print_ln(MDBX_chk_result, "No error is detected, elapsed %.3f seconds.", elapsed);
return EXIT_SUCCESS;
}

52
src/tools/copy.c
View File

@ -37,20 +37,19 @@ static void signal_handler(int sig) {
#endif /* !WINDOWS */
static void usage(const char *prog) {
fprintf(
stderr,
"usage: %s [-V] [-q] [-c] [-d] [-p] [-u|U] src_path [dest_path]\n"
" -V\t\tprint version and exit\n"
" -q\t\tbe quiet\n"
" -c\t\tenable compactification (skip unused pages)\n"
" -d\t\tenforce copy to be a dynamic size DB\n"
" -p\t\tusing transaction parking/ousting during copying MVCC-snapshot\n"
" \t\tto avoid stopping recycling and overflowing the DB\n"
" -u\t\twarmup database before copying\n"
" -U\t\twarmup and try lock database pages in memory before copying\n"
" src_path\tsource database\n"
" dest_path\tdestination (stdout if not specified)\n",
prog);
fprintf(stderr,
"usage: %s [-V] [-q] [-c] [-d] [-p] [-u|U] src_path [dest_path]\n"
" -V\t\tprint version and exit\n"
" -q\t\tbe quiet\n"
" -c\t\tenable compactification (skip unused pages)\n"
" -d\t\tenforce copy to be a dynamic size DB\n"
" -p\t\tusing transaction parking/ousting during copying MVCC-snapshot\n"
" \t\tto avoid stopping recycling and overflowing the DB\n"
" -u\t\twarmup database before copying\n"
" -U\t\twarmup and try lock database pages in memory before copying\n"
" src_path\tsource database\n"
" dest_path\tdestination (stdout if not specified)\n",
prog);
exit(EXIT_FAILURE);
}
@ -79,10 +78,8 @@ int main(int argc, char *argv[]) {
warmup = true;
else if (argv[1][1] == 'U' && argv[1][2] == '\0') {
warmup = true;
warmup_flags =
MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock;
} else if ((argv[1][1] == 'h' && argv[1][2] == '\0') ||
strcmp(argv[1], "--help") == 0)
warmup_flags = MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock;
} else if ((argv[1][1] == 'h' && argv[1][2] == '\0') || strcmp(argv[1], "--help") == 0)
usage(progname);
else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
printf("mdbx_copy version %d.%d.%d.%d\n"
@ -91,12 +88,9 @@ int main(int argc, char *argv[]) {
" - build: %s for %s by %s\n"
" - flags: %s\n"
" - options: %s\n",
mdbx_version.major, mdbx_version.minor, mdbx_version.patch,
mdbx_version.tweak, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit,
mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
mdbx_build.options);
mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor,
mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options);
return EXIT_SUCCESS;
} else
argc = 0;
@ -119,10 +113,9 @@ int main(int argc, char *argv[]) {
#endif /* !WINDOWS */
if (!quiet) {
fprintf((argc == 2) ? stderr : stdout,
"mdbx_copy %s (%s, T-%s)\nRunning for copy %s to %s...\n",
mdbx_version.git.describe, mdbx_version.git.datetime,
mdbx_version.git.tree, argv[1], (argc == 2) ? "stdout" : argv[2]);
fprintf((argc == 2) ? stderr : stdout, "mdbx_copy %s (%s, T-%s)\nRunning for copy %s to %s...\n",
mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_version.git.tree, argv[1],
(argc == 2) ? "stdout" : argv[2]);
fflush(nullptr);
}
@ -150,8 +143,7 @@ int main(int argc, char *argv[]) {
rc = mdbx_env_copy(env, argv[2], cpflags);
}
if (rc)
fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc,
mdbx_strerror(rc));
fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc, mdbx_strerror(rc));
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;

15
src/tools/drop.c
View File

@ -54,8 +54,7 @@ static void usage(void) {
static void error(const char *func, int rc) {
if (!quiet)
fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc,
mdbx_strerror(rc));
fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc));
}
int main(int argc, char *argv[]) {
@ -86,12 +85,9 @@ int main(int argc, char *argv[]) {
" - build: %s for %s by %s\n"
" - flags: %s\n"
" - options: %s\n",
mdbx_version.major, mdbx_version.minor, mdbx_version.patch,
mdbx_version.tweak, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit,
mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
mdbx_build.options);
mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor,
mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options);
return EXIT_SUCCESS;
case 'q':
quiet = true;
@ -127,8 +123,7 @@ int main(int argc, char *argv[]) {
envname = argv[optind];
if (!quiet) {
printf("mdbx_drop %s (%s, T-%s)\nRunning for %s/%s...\n",
mdbx_version.git.describe, mdbx_version.git.datetime,
printf("mdbx_drop %s (%s, T-%s)\nRunning for %s/%s...\n", mdbx_version.git.describe, mdbx_version.git.datetime,
mdbx_version.git.tree, envname, subname ? subname : "@MAIN");
fflush(nullptr);
}

97
src/tools/dump.c
View File

@ -95,8 +95,7 @@ bool quiet = false, rescue = false;
const char *prog;
static void error(const char *func, int rc) {
if (!quiet)
fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc,
mdbx_strerror(rc));
fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc));
}
/* Dump in BDB-compatible format */
@ -126,10 +125,8 @@ static int dump_tbl(MDBX_txn *txn, MDBX_dbi dbi, char *name) {
if (mode & GLOBAL) {
mode -= GLOBAL;
if (info.mi_geo.upper != info.mi_geo.lower)
printf("geometry=l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64
",g%" PRIu64 "\n",
info.mi_geo.lower, info.mi_geo.current, info.mi_geo.upper,
info.mi_geo.shrink, info.mi_geo.grow);
printf("geometry=l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64 ",g%" PRIu64 "\n", info.mi_geo.lower,
info.mi_geo.current, info.mi_geo.upper, info.mi_geo.shrink, info.mi_geo.grow);
printf("mapsize=%" PRIu64 "\n", info.mi_geo.upper);
printf("maxreaders=%u\n", info.mi_maxreaders);
@ -140,8 +137,7 @@ static int dump_tbl(MDBX_txn *txn, MDBX_dbi dbi, char *name) {
return rc;
}
if (canary.v)
printf("canary=v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64 "\n",
canary.v, canary.x, canary.y, canary.z);
printf("canary=v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64 "\n", canary.v, canary.x, canary.y, canary.z);
}
printf("format=%s\n", mode & PRINT ? "print" : "bytevalue");
if (name)
@ -153,10 +149,7 @@ static int dump_tbl(MDBX_txn *txn, MDBX_dbi dbi, char *name) {
else if (!name)
printf("txnid=%" PRIaTXN "\n", mdbx_txn_id(txn)); */
printf("duplicates=%d\n", (flags & (MDBX_DUPSORT | MDBX_DUPFIXED |
MDBX_INTEGERDUP | MDBX_REVERSEDUP))
? 1
: 0);
printf("duplicates=%d\n", (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP)) ? 1 : 0);
for (int i = 0; dbflags[i].bit; i++)
if (flags & dbflags[i].bit)
printf("%s=1\n", dbflags[i].name);
@ -187,8 +180,7 @@ static int dump_tbl(MDBX_txn *txn, MDBX_dbi dbi, char *name) {
}
}
while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) ==
MDBX_SUCCESS) {
while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == MDBX_SUCCESS) {
if (user_break) {
rc = MDBX_EINTR;
break;
@ -212,31 +204,27 @@ static int dump_tbl(MDBX_txn *txn, MDBX_dbi dbi, char *name) {
}
static void usage(void) {
fprintf(
stderr,
"usage: %s "
"[-V] [-q] [-f file] [-l] [-p] [-r] [-a|-s table] [-u|U] "
"dbpath\n"
" -V\t\tprint version and exit\n"
" -q\t\tbe quiet\n"
" -f\t\twrite to file instead of stdout\n"
" -l\t\tlist tables and exit\n"
" -p\t\tuse printable characters\n"
" -r\t\trescue mode (ignore errors to dump corrupted DB)\n"
" -a\t\tdump main DB and all tables\n"
" -s name\tdump only the specified named table\n"
" -u\t\twarmup database before dumping\n"
" -U\t\twarmup and try lock database pages in memory before dumping\n"
" \t\tby default dump only the main DB\n",
prog);
fprintf(stderr,
"usage: %s "
"[-V] [-q] [-f file] [-l] [-p] [-r] [-a|-s table] [-u|U] "
"dbpath\n"
" -V\t\tprint version and exit\n"
" -q\t\tbe quiet\n"
" -f\t\twrite to file instead of stdout\n"
" -l\t\tlist tables and exit\n"
" -p\t\tuse printable characters\n"
" -r\t\trescue mode (ignore errors to dump corrupted DB)\n"
" -a\t\tdump main DB and all tables\n"
" -s name\tdump only the specified named table\n"
" -u\t\twarmup database before dumping\n"
" -U\t\twarmup and try lock database pages in memory before dumping\n"
" \t\tby default dump only the main DB\n",
prog);
exit(EXIT_FAILURE);
}
static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) {
return (a->iov_len == b->iov_len &&
memcmp(a->iov_base, b->iov_base, a->iov_len) == 0)
? 0
: 1;
return (a->iov_len == b->iov_len && memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) ? 0 : 1;
}
int main(int argc, char *argv[]) {
@ -274,12 +262,9 @@ int main(int argc, char *argv[]) {
" - build: %s for %s by %s\n"
" - flags: %s\n"
" - options: %s\n",
mdbx_version.major, mdbx_version.minor, mdbx_version.patch,
mdbx_version.tweak, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit,
mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
mdbx_build.options);
mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor,
mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options);
return EXIT_SUCCESS;
case 'l':
list = true;
@ -292,8 +277,7 @@ int main(int argc, char *argv[]) {
break;
case 'f':
if (freopen(optarg, "w", stdout) == nullptr) {
fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg,
mdbx_strerror(errno));
fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, mdbx_strerror(errno));
exit(EXIT_FAILURE);
}
break;
@ -318,8 +302,7 @@ int main(int argc, char *argv[]) {
break;
case 'U':
warmup = true;
warmup_flags =
MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock;
warmup_flags = MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock;
break;
default:
usage();
@ -344,9 +327,8 @@ int main(int argc, char *argv[]) {
envname = argv[optind];
if (!quiet) {
fprintf(stderr, "mdbx_dump %s (%s, T-%s)\nRunning for %s...\n",
mdbx_version.git.describe, mdbx_version.git.datetime,
mdbx_version.git.tree, envname);
fprintf(stderr, "mdbx_dump %s (%s, T-%s)\nRunning for %s...\n", mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.tree, envname);
fflush(nullptr);
}
@ -364,11 +346,8 @@ int main(int argc, char *argv[]) {
}
}
err = mdbx_env_open(
env, envname,
envflags | (rescue ? MDBX_RDONLY | MDBX_EXCLUSIVE | MDBX_VALIDATION
: MDBX_RDONLY),
0);
err = mdbx_env_open(env, envname, envflags | (rescue ? MDBX_RDONLY | MDBX_EXCLUSIVE | MDBX_VALIDATION : MDBX_RDONLY),
0);
if (unlikely(err != MDBX_SUCCESS)) {
error("mdbx_env_open", err);
goto env_close;
@ -414,8 +393,7 @@ int main(int argc, char *argv[]) {
bool have_raw = false;
int count = 0;
MDBX_val key;
while (MDBX_SUCCESS ==
(err = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) {
while (MDBX_SUCCESS == (err = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) {
if (user_break) {
err = MDBX_EINTR;
break;
@ -434,8 +412,7 @@ int main(int argc, char *argv[]) {
subname[key.iov_len] = '\0';
MDBX_dbi sub_dbi;
err = mdbx_dbi_open_ex(txn, subname, MDBX_DB_ACCEDE, &sub_dbi,
rescue ? equal_or_greater : nullptr,
err = mdbx_dbi_open_ex(txn, subname, MDBX_DB_ACCEDE, &sub_dbi, rescue ? equal_or_greater : nullptr,
rescue ? equal_or_greater : nullptr);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_INCOMPATIBLE) {
@ -455,8 +432,7 @@ int main(int argc, char *argv[]) {
if (!rescue)
break;
if (!quiet)
fprintf(stderr, "%s: %s: ignore %s for `%s` and continue\n", prog,
envname, mdbx_strerror(err), subname);
fprintf(stderr, "%s: %s: ignore %s for `%s` and continue\n", prog, envname, mdbx_strerror(err), subname);
/* Here is a hack for rescue mode, don't do that:
* - we should restart transaction in case error due
* database corruption;
@ -491,8 +467,7 @@ int main(int argc, char *argv[]) {
err = dump_tbl(txn, MAIN_DBI, nullptr);
else if (!count) {
if (!quiet)
fprintf(stderr, "%s: %s does not contain multiple databases\n", prog,
envname);
fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, envname);
err = MDBX_NOTFOUND;
}
} else {

188
src/tools/load.c
View File

@ -44,11 +44,10 @@ static size_t lineno;
static void error(const char *func, int rc) {
if (!quiet) {
if (lineno)
fprintf(stderr, "%s: at input line %" PRIiSIZE ": %s() error %d, %s\n",
prog, lineno, func, rc, mdbx_strerror(rc));
else
fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc,
fprintf(stderr, "%s: at input line %" PRIiSIZE ": %s() error %d, %s\n", prog, lineno, func, rc,
mdbx_strerror(rc));
else
fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc));
}
}
@ -60,9 +59,7 @@ static char *valstr(char *line, const char *item) {
if (line[len] > ' ')
return nullptr;
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", prog,
lineno, item);
fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", prog, lineno, item);
exit(EXIT_FAILURE);
}
char *ptr = strchr(line, '\n');
@ -80,9 +77,7 @@ static bool valnum(char *line, const char *item, uint64_t *value) {
*value = strtoull(str, &end, 0);
if (end && *end) {
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": unexpected number format for '%s'\n",
prog, lineno, item);
fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected number format for '%s'\n", prog, lineno, item);
exit(EXIT_FAILURE);
}
return true;
@ -95,8 +90,7 @@ static bool valbool(char *line, const char *item, bool *value) {
if (u64 > 1) {
if (!quiet)
fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected value for '%s'\n",
prog, lineno, item);
fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected value for '%s'\n", prog, lineno, item);
exit(EXIT_FAILURE);
}
*value = u64 != 0;
@ -129,11 +123,10 @@ typedef struct flagbit {
#define S(s) STRLENOF(s), s
flagbit dbflags[] = {
{MDBX_REVERSEKEY, S("reversekey")}, {MDBX_DUPSORT, S("duplicates")},
{MDBX_DUPSORT, S("dupsort")}, {MDBX_INTEGERKEY, S("integerkey")},
{MDBX_DUPFIXED, S("dupfix")}, {MDBX_INTEGERDUP, S("integerdup")},
{MDBX_REVERSEDUP, S("reversedup")}, {0, 0, nullptr}};
flagbit dbflags[] = {{MDBX_REVERSEKEY, S("reversekey")}, {MDBX_DUPSORT, S("duplicates")},
{MDBX_DUPSORT, S("dupsort")}, {MDBX_INTEGERKEY, S("integerkey")},
{MDBX_DUPFIXED, S("dupfix")}, {MDBX_INTEGERDUP, S("integerdup")},
{MDBX_REVERSEDUP, S("reversedup")}, {0, 0, nullptr}};
static int readhdr(void) {
/* reset parameters */
@ -158,10 +151,8 @@ static int readhdr(void) {
if (valnum(dbuf.iov_base, "VERSION", &u64)) {
if (u64 != 3) {
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": unsupported value %" PRIu64
" for %s\n",
prog, lineno, u64, "VERSION");
fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported value %" PRIu64 " for %s\n", prog, lineno, u64,
"VERSION");
exit(EXIT_FAILURE);
}
continue;
@ -170,16 +161,12 @@ static int readhdr(void) {
if (valnum(dbuf.iov_base, "db_pagesize", &u64)) {
if (!(mode & GLOBAL) && envinfo.mi_dxb_pagesize != u64) {
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": ignore value %" PRIu64
" for '%s' in non-global context\n",
prog, lineno, u64, "db_pagesize");
fprintf(stderr, "%s: line %" PRIiSIZE ": ignore value %" PRIu64 " for '%s' in non-global context\n", prog,
lineno, u64, "db_pagesize");
} else if (u64 < MDBX_MIN_PAGESIZE || u64 > MDBX_MAX_PAGESIZE) {
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": ignore unsupported value %" PRIu64
" for %s\n",
prog, lineno, u64, "db_pagesize");
fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value %" PRIu64 " for %s\n", prog, lineno, u64,
"db_pagesize");
} else
envinfo.mi_dxb_pagesize = (uint32_t)u64;
continue;
@ -196,9 +183,7 @@ static int readhdr(void) {
continue;
}
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", prog,
lineno, str, "format");
fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", prog, lineno, str, "format");
exit(EXIT_FAILURE);
}
@ -220,9 +205,7 @@ static int readhdr(void) {
if (str) {
if (strcmp(str, "btree") != 0) {
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n",
prog, lineno, str, "type");
fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", prog, lineno, str, "type");
free(subname);
exit(EXIT_FAILURE);
}
@ -232,10 +215,8 @@ static int readhdr(void) {
if (valnum(dbuf.iov_base, "mapaddr", &u64)) {
if (u64) {
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64
" for %s\n",
prog, lineno, u64, "mapaddr");
fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 " for %s\n", prog, lineno, u64,
"mapaddr");
}
continue;
}
@ -243,16 +224,12 @@ static int readhdr(void) {
if (valnum(dbuf.iov_base, "mapsize", &u64)) {
if (!(mode & GLOBAL)) {
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": ignore value %" PRIu64
" for '%s' in non-global context\n",
prog, lineno, u64, "mapsize");
fprintf(stderr, "%s: line %" PRIiSIZE ": ignore value %" PRIu64 " for '%s' in non-global context\n", prog,
lineno, u64, "mapsize");
} else if (u64 < MIN_MAPSIZE || u64 > MAX_MAPSIZE64) {
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64
" for %s\n",
prog, lineno, u64, "mapsize");
fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 " for %s\n", prog, lineno, u64,
"mapsize");
} else
envinfo.mi_mapsize = (size_t)u64;
continue;
@ -261,16 +238,12 @@ static int readhdr(void) {
if (valnum(dbuf.iov_base, "maxreaders", &u64)) {
if (!(mode & GLOBAL)) {
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": ignore value %" PRIu64
" for '%s' in non-global context\n",
prog, lineno, u64, "maxreaders");
fprintf(stderr, "%s: line %" PRIiSIZE ": ignore value %" PRIu64 " for '%s' in non-global context\n", prog,
lineno, u64, "maxreaders");
} else if (u64 < 1 || u64 > MDBX_READERS_LIMIT) {
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64
" for %s\n",
prog, lineno, u64, "maxreaders");
fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 " for %s\n", prog, lineno, u64,
"maxreaders");
} else
envinfo.mi_maxreaders = (int)u64;
continue;
@ -279,10 +252,8 @@ static int readhdr(void) {
if (valnum(dbuf.iov_base, "txnid", &u64)) {
if (u64 < MIN_TXNID || u64 > MAX_TXNID) {
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64
" for %s\n",
prog, lineno, u64, "txnid");
fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 " for %s\n", prog, lineno, u64,
"txnid");
} else
txnid = u64;
continue;
@ -301,16 +272,11 @@ static int readhdr(void) {
"%s: line %" PRIiSIZE ": ignore values %s"
" for '%s' in non-global context\n",
prog, lineno, str, "geometry");
} else if (sscanf(str,
"l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64
",g%" PRIu64,
&envinfo.mi_geo.lower, &envinfo.mi_geo.current,
&envinfo.mi_geo.upper, &envinfo.mi_geo.shrink,
} else if (sscanf(str, "l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64 ",g%" PRIu64, &envinfo.mi_geo.lower,
&envinfo.mi_geo.current, &envinfo.mi_geo.upper, &envinfo.mi_geo.shrink,
&envinfo.mi_geo.grow) != 5) {
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": unexpected line format for '%s'\n",
prog, lineno, "geometry");
fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", prog, lineno, "geometry");
exit(EXIT_FAILURE);
}
continue;
@ -324,12 +290,10 @@ static int readhdr(void) {
"%s: line %" PRIiSIZE ": ignore values %s"
" for '%s' in non-global context\n",
prog, lineno, str, "canary");
} else if (sscanf(str, "v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64,
&canary.v, &canary.x, &canary.y, &canary.z) != 4) {
} else if (sscanf(str, "v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64, &canary.v, &canary.x, &canary.y,
&canary.z) != 4) {
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": unexpected line format for '%s'\n",
prog, lineno, "canary");
fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", prog, lineno, "canary");
exit(EXIT_FAILURE);
}
continue;
@ -353,9 +317,8 @@ static int readhdr(void) {
}
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n",
prog, lineno, (char *)dbuf.iov_base);
fprintf(stderr, "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n", prog, lineno,
(char *)dbuf.iov_base);
next:;
}
return EOF;
@ -363,8 +326,7 @@ static int readhdr(void) {
static int badend(void) {
if (!quiet)
fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog,
lineno);
fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog, lineno);
return errno ? errno : MDBX_ENODATA;
}
@ -416,9 +378,7 @@ __hot static int readline(MDBX_val *out, MDBX_val *buf) {
buf->iov_base = osal_realloc(buf->iov_base, buf->iov_len * 2);
if (!buf->iov_base) {
if (!quiet)
fprintf(stderr,
"%s: line %" PRIiSIZE ": out of memory, line too long\n", prog,
lineno);
fprintf(stderr, "%s: line %" PRIiSIZE ": out of memory, line too long\n", prog, lineno);
return MDBX_ENOMEM;
}
c1 = buf->iov_base;
@ -490,10 +450,7 @@ static void usage(void) {
}
static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) {
return (a->iov_len == b->iov_len &&
memcmp(a->iov_base, b->iov_base, a->iov_len) == 0)
? 0
: 1;
return (a->iov_len == b->iov_len && memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) ? 0 : 1;
}
int main(int argc, char *argv[]) {
@ -530,12 +487,9 @@ int main(int argc, char *argv[]) {
" - build: %s for %s by %s\n"
" - flags: %s\n"
" - options: %s\n",
mdbx_version.major, mdbx_version.minor, mdbx_version.patch,
mdbx_version.tweak, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit,
mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
mdbx_build.options);
mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor,
mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options);
return EXIT_SUCCESS;
case 'a':
putflags |= MDBX_APPEND;
@ -543,8 +497,7 @@ int main(int argc, char *argv[]) {
case 'f':
if (freopen(optarg, "r", stdin) == nullptr) {
if (!quiet)
fprintf(stderr, "%s: %s: open: %s\n", prog, optarg,
mdbx_strerror(errno));
fprintf(stderr, "%s: %s: open: %s\n", prog, optarg, mdbx_strerror(errno));
exit(EXIT_FAILURE);
}
break;
@ -592,8 +545,7 @@ int main(int argc, char *argv[]) {
envname = argv[optind];
if (!quiet)
printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n",
mdbx_version.git.describe, mdbx_version.git.datetime,
printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n", mdbx_version.git.describe, mdbx_version.git.datetime,
mdbx_version.git.tree, envname);
fflush(nullptr);
@ -638,25 +590,22 @@ int main(int argc, char *argv[]) {
if (envinfo.mi_geo.current | envinfo.mi_mapsize) {
if (envinfo.mi_geo.current) {
err = mdbx_env_set_geometry(
env, (intptr_t)envinfo.mi_geo.lower, (intptr_t)envinfo.mi_geo.current,
(intptr_t)envinfo.mi_geo.upper, (intptr_t)envinfo.mi_geo.shrink,
(intptr_t)envinfo.mi_geo.grow,
envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1);
err = mdbx_env_set_geometry(env, (intptr_t)envinfo.mi_geo.lower, (intptr_t)envinfo.mi_geo.current,
(intptr_t)envinfo.mi_geo.upper, (intptr_t)envinfo.mi_geo.shrink,
(intptr_t)envinfo.mi_geo.grow,
envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1);
} else {
if (envinfo.mi_mapsize > MAX_MAPSIZE) {
if (!quiet)
fprintf(
stderr,
"Database size is too large for current system (mapsize=%" PRIu64
" is great than system-limit %zu)\n",
envinfo.mi_mapsize, (size_t)MAX_MAPSIZE);
fprintf(stderr,
"Database size is too large for current system (mapsize=%" PRIu64
" is great than system-limit %zu)\n",
envinfo.mi_mapsize, (size_t)MAX_MAPSIZE);
goto bailout;
}
err = mdbx_env_set_geometry(
env, (intptr_t)envinfo.mi_mapsize, (intptr_t)envinfo.mi_mapsize,
(intptr_t)envinfo.mi_mapsize, 0, 0,
envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1);
err = mdbx_env_set_geometry(env, (intptr_t)envinfo.mi_mapsize, (intptr_t)envinfo.mi_mapsize,
(intptr_t)envinfo.mi_mapsize, 0, 0,
envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1);
}
if (unlikely(err != MDBX_SUCCESS)) {
error("mdbx_env_set_geometry", err);
@ -673,8 +622,7 @@ int main(int argc, char *argv[]) {
kbuf.iov_len = mdbx_env_get_maxvalsize_ex(env, 0) + (size_t)1;
if (kbuf.iov_len >= INTPTR_MAX / 2) {
if (!quiet)
fprintf(stderr, "mdbx_env_get_maxkeysize() failed, returns %zu\n",
kbuf.iov_len);
fprintf(stderr, "mdbx_env_get_maxkeysize() failed, returns %zu\n", kbuf.iov_len);
goto bailout;
}
@ -709,10 +657,9 @@ int main(int argc, char *argv[]) {
}
const char *const dbi_name = subname ? subname : "@MAIN";
err =
mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi,
(putflags & MDBX_APPEND) ? equal_or_greater : nullptr,
(putflags & MDBX_APPEND) ? equal_or_greater : nullptr);
err = mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi,
(putflags & MDBX_APPEND) ? equal_or_greater : nullptr,
(putflags & MDBX_APPEND) ? equal_or_greater : nullptr);
if (unlikely(err != MDBX_SUCCESS)) {
error("mdbx_dbi_open_ex", err);
goto bailout;
@ -726,9 +673,7 @@ int main(int argc, char *argv[]) {
}
if (present_sequence > sequence) {
if (!quiet)
fprintf(stderr,
"present sequence for '%s' value (%" PRIu64
") is greater than loaded (%" PRIu64 ")\n",
fprintf(stderr, "present sequence for '%s' value (%" PRIu64 ") is greater than loaded (%" PRIu64 ")\n",
dbi_name, present_sequence, sequence);
err = MDBX_RESULT_TRUE;
goto bailout;
@ -750,8 +695,7 @@ int main(int argc, char *argv[]) {
}
if (putflags & MDBX_APPEND)
putflags = (dbi_flags & MDBX_DUPSORT) ? putflags | MDBX_APPENDDUP
: putflags & ~MDBX_APPENDDUP;
putflags = (dbi_flags & MDBX_DUPSORT) ? putflags | MDBX_APPENDDUP : putflags & ~MDBX_APPENDDUP;
err = mdbx_cursor_open(txn, dbi, &mc);
if (unlikely(err != MDBX_SUCCESS)) {
@ -770,8 +714,7 @@ int main(int argc, char *argv[]) {
err = readline(&data, &dbuf);
if (err) {
if (!quiet)
fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n",
prog, lineno);
fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n", prog, lineno);
goto bailout;
}
@ -780,8 +723,7 @@ int main(int argc, char *argv[]) {
continue;
if (err == MDBX_BAD_VALSIZE && rescue) {
if (!quiet)
fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno,
mdbx_strerror(err));
fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno, mdbx_strerror(err));
continue;
}
if (unlikely(err != MDBX_SUCCESS)) {

117
src/tools/stat.c
View File

@ -61,27 +61,24 @@ static void usage(const char *prog) {
exit(EXIT_FAILURE);
}
static int reader_list_func(void *ctx, int num, int slot, mdbx_pid_t pid,
mdbx_tid_t thread, uint64_t txnid, uint64_t lag,
size_t bytes_used, size_t bytes_retained) {
static int reader_list_func(void *ctx, int num, int slot, mdbx_pid_t pid, mdbx_tid_t thread, uint64_t txnid,
uint64_t lag, size_t bytes_used, size_t bytes_retained) {
(void)ctx;
if (num == 1)
printf("Reader Table\n"
" #\tslot\t%6s %*s %20s %10s %13s %13s\n",
"pid", (int)sizeof(size_t) * 2, "thread", "txnid", "lag", "used",
"retained");
"pid", (int)sizeof(size_t) * 2, "thread", "txnid", "lag", "used", "retained");
if (thread < (mdbx_tid_t)((intptr_t)MDBX_TID_TXN_OUSTED))
printf(" %3d)\t[%d]\t%6" PRIdSIZE " %*" PRIxPTR, num, slot, (size_t)pid,
(int)sizeof(size_t) * 2, (uintptr_t)thread);
printf(" %3d)\t[%d]\t%6" PRIdSIZE " %*" PRIxPTR, num, slot, (size_t)pid, (int)sizeof(size_t) * 2,
(uintptr_t)thread);
else
printf(" %3d)\t[%d]\t%6" PRIdSIZE " %sed", num, slot, (size_t)pid,
(thread == (mdbx_tid_t)((uintptr_t)MDBX_TID_TXN_PARKED)) ? "park"
: "oust");
(thread == (mdbx_tid_t)((uintptr_t)MDBX_TID_TXN_PARKED)) ? "park" : "oust");
if (txnid)
printf(" %20" PRIu64 " %10" PRIu64 " %12.1fM %12.1fM\n", txnid, lag,
bytes_used / 1048576.0, bytes_retained / 1048576.0);
printf(" %20" PRIu64 " %10" PRIu64 " %12.1fM %12.1fM\n", txnid, lag, bytes_used / 1048576.0,
bytes_retained / 1048576.0);
else
printf(" %20s %10s %13s %13s\n", "-", "0", "0", "0");
@ -92,8 +89,7 @@ const char *prog;
bool quiet = false;
static void error(const char *func, int rc) {
if (!quiet)
fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc,
mdbx_strerror(rc));
fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc));
}
int main(int argc, char *argv[]) {
@ -129,12 +125,9 @@ int main(int argc, char *argv[]) {
" - build: %s for %s by %s\n"
" - flags: %s\n"
" - options: %s\n",
mdbx_version.major, mdbx_version.minor, mdbx_version.patch,
mdbx_version.tweak, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit,
mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
mdbx_build.options);
mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor,
mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options);
return EXIT_SUCCESS;
case 'q':
quiet = true;
@ -187,8 +180,7 @@ int main(int argc, char *argv[]) {
envname = argv[optind];
envname = argv[optind];
if (!quiet) {
printf("mdbx_stat %s (%s, T-%s)\nRunning for %s...\n",
mdbx_version.git.describe, mdbx_version.git.datetime,
printf("mdbx_stat %s (%s, T-%s)\nRunning for %s...\n", mdbx_version.git.describe, mdbx_version.git.datetime,
mdbx_version.git.tree, envname);
fflush(nullptr);
}
@ -232,39 +224,27 @@ int main(int argc, char *argv[]) {
if (pgop) {
printf("Page Operations (for current session):\n");
printf(" New: %8" PRIu64 "\t// quantity of a new pages added\n",
mei.mi_pgop_stat.newly);
printf(" CoW: %8" PRIu64
"\t// quantity of pages copied for altering\n",
mei.mi_pgop_stat.cow);
printf(" New: %8" PRIu64 "\t// quantity of a new pages added\n", mei.mi_pgop_stat.newly);
printf(" CoW: %8" PRIu64 "\t// quantity of pages copied for altering\n", mei.mi_pgop_stat.cow);
printf(" Clone: %8" PRIu64 "\t// quantity of parent's dirty pages "
"clones for nested transactions\n",
mei.mi_pgop_stat.clone);
printf(" Split: %8" PRIu64
"\t// page splits during insertions or updates\n",
mei.mi_pgop_stat.split);
printf(" Merge: %8" PRIu64
"\t// page merges during deletions or updates\n",
mei.mi_pgop_stat.merge);
printf(" Split: %8" PRIu64 "\t// page splits during insertions or updates\n", mei.mi_pgop_stat.split);
printf(" Merge: %8" PRIu64 "\t// page merges during deletions or updates\n", mei.mi_pgop_stat.merge);
printf(" Spill: %8" PRIu64 "\t// quantity of spilled/ousted `dirty` "
"pages during large transactions\n",
mei.mi_pgop_stat.spill);
printf(" Unspill: %8" PRIu64 "\t// quantity of unspilled/redone `dirty` "
"pages during large transactions\n",
mei.mi_pgop_stat.unspill);
printf(" WOP: %8" PRIu64
"\t// number of explicit write operations (not a pages) to a disk\n",
printf(" WOP: %8" PRIu64 "\t// number of explicit write operations (not a pages) to a disk\n",
mei.mi_pgop_stat.wops);
printf(" PreFault: %8" PRIu64
"\t// number of prefault write operations (not a pages)\n",
printf(" PreFault: %8" PRIu64 "\t// number of prefault write operations (not a pages)\n",
mei.mi_pgop_stat.prefault);
printf(" mInCore: %8" PRIu64 "\t// number of mincore() calls\n",
mei.mi_pgop_stat.mincore);
printf(" mSync: %8" PRIu64
"\t// number of explicit msync-to-disk operations (not a pages)\n",
printf(" mInCore: %8" PRIu64 "\t// number of mincore() calls\n", mei.mi_pgop_stat.mincore);
printf(" mSync: %8" PRIu64 "\t// number of explicit msync-to-disk operations (not a pages)\n",
mei.mi_pgop_stat.msync);
printf(" fSync: %8" PRIu64
"\t// number of explicit fsync-to-disk operations (not a pages)\n",
printf(" fSync: %8" PRIu64 "\t// number of explicit fsync-to-disk operations (not a pages)\n",
mei.mi_pgop_stat.fsync);
}
@ -272,18 +252,15 @@ int main(int argc, char *argv[]) {
printf("Environment Info\n");
printf(" Pagesize: %u\n", mei.mi_dxb_pagesize);
if (mei.mi_geo.lower != mei.mi_geo.upper) {
printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64
"/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64
"/-%" PRIu64 ")\n",
mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow,
mei.mi_geo.shrink, mei.mi_geo.lower / mei.mi_dxb_pagesize,
mei.mi_geo.upper / mei.mi_dxb_pagesize,
mei.mi_geo.grow / mei.mi_dxb_pagesize,
mei.mi_geo.shrink / mei.mi_dxb_pagesize);
printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n",
mei.mi_mapsize, mei.mi_mapsize / mei.mi_dxb_pagesize);
printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
mei.mi_geo.current, mei.mi_geo.current / mei.mi_dxb_pagesize);
printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64 "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64
" pages (+%" PRIu64 "/-%" PRIu64 ")\n",
mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow, mei.mi_geo.shrink,
mei.mi_geo.lower / mei.mi_dxb_pagesize, mei.mi_geo.upper / mei.mi_dxb_pagesize,
mei.mi_geo.grow / mei.mi_dxb_pagesize, mei.mi_geo.shrink / mei.mi_dxb_pagesize);
printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n", mei.mi_mapsize,
mei.mi_mapsize / mei.mi_dxb_pagesize);
printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", mei.mi_geo.current,
mei.mi_geo.current / mei.mi_dxb_pagesize);
#if defined(_WIN32) || defined(_WIN64)
if (mei.mi_geo.shrink && mei.mi_geo.current != mei.mi_geo.upper)
printf(" WARNING: Due Windows system limitations a "
@ -293,12 +270,11 @@ int main(int argc, char *argv[]) {
"until it will be closed or reopened in read-write mode.\n");
#endif
} else {
printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
mei.mi_geo.current, mei.mi_geo.current / mei.mi_dxb_pagesize);
printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", mei.mi_geo.current,
mei.mi_geo.current / mei.mi_dxb_pagesize);
}
printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid);
printf(" Latter reader transaction ID: %" PRIu64 " (%" PRIi64 ")\n",
mei.mi_latter_reader_txnid,
printf(" Latter reader transaction ID: %" PRIu64 " (%" PRIi64 ")\n", mei.mi_latter_reader_txnid,
mei.mi_latter_reader_txnid - mei.mi_recent_txnid);
printf(" Max readers: %u\n", mei.mi_maxreaders);
printf(" Number of reader slots uses: %u\n", mei.mi_numreaders);
@ -352,8 +328,7 @@ int main(int argc, char *argv[]) {
pgno_t pages = 0, *iptr;
pgno_t reclaimable = 0;
MDBX_val key, data;
while (MDBX_SUCCESS ==
(rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT))) {
while (MDBX_SUCCESS == (rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT))) {
if (user_break) {
rc = MDBX_EINTR;
break;
@ -367,29 +342,23 @@ int main(int argc, char *argv[]) {
if (freinfo > 1) {
char *bad = "";
pgno_t prev =
MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)mei.mi_last_pgno + 1;
pgno_t prev = MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)mei.mi_last_pgno + 1;
pgno_t span = 1;
for (unsigned i = 0; i < number; ++i) {
pgno_t pg = iptr[i];
if (MDBX_PNL_DISORDERED(prev, pg))
bad = " [bad sequence]";
prev = pg;
while (i + span < number &&
iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span)
: pgno_sub(pg, span)))
while (i + span < number && iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) : pgno_sub(pg, span)))
++span;
}
printf(" Transaction %" PRIaTXN ", %" PRIaPGNO
" pages, maxspan %" PRIaPGNO "%s\n",
*(txnid_t *)key.iov_base, number, span, bad);
printf(" Transaction %" PRIaTXN ", %" PRIaPGNO " pages, maxspan %" PRIaPGNO "%s\n", *(txnid_t *)key.iov_base,
number, span, bad);
if (freinfo > 2) {
for (unsigned i = 0; i < number; i += span) {
const pgno_t pg = iptr[i];
for (span = 1;
i + span < number &&
iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span)
: pgno_sub(pg, span));
i + span < number && iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) : pgno_sub(pg, span));
++span)
;
if (span > 1)
@ -443,8 +412,7 @@ int main(int argc, char *argv[]) {
value = reclaimable;
printf(" Reclaimable: %" PRIu64 " %.1f%%\n", value, value / percent);
value = mei.mi_mapsize / mei.mi_dxb_pagesize - (mei.mi_last_pgno + 1) +
reclaimable;
value = mei.mi_mapsize / mei.mi_dxb_pagesize - (mei.mi_last_pgno + 1) + reclaimable;
printf(" Available: %" PRIu64 " %.1f%%\n", value, value / percent);
} else
printf(" GC: %" PRIaPGNO " pages\n", pages);
@ -474,8 +442,7 @@ int main(int argc, char *argv[]) {
}
MDBX_val key;
while (MDBX_SUCCESS ==
(rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) {
while (MDBX_SUCCESS == (rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) {
MDBX_dbi xdbi;
if (memchr(key.iov_base, '\0', key.iov_len))
continue;

11
src/tools/wingetopt.c
View File

@ -11,12 +11,12 @@
#ifdef _MSC_VER
#pragma warning(push, 1)
#pragma warning(disable : 4548) /* expression before comma has no effect; \
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
* semantics are not enabled. Specify /EHsc */
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
* mode specified; termination on exception is \
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
* mode specified; termination on exception is \
* not guaranteed. Specify /EHsc */
#if !defined(_CRT_SECURE_NO_WARNINGS)
#define _CRT_SECURE_NO_WARNINGS
@ -70,8 +70,7 @@ int getopt(int argc, char *const argv[], const char *opts) {
if (argv[optind][sp + 1] != '\0')
optarg = &argv[optind++][sp + 1];
else if (++optind >= argc) {
fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument",
c);
fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", c);
sp = 1;
return '?';
} else

286
src/tree.c
View File

@ -5,8 +5,7 @@
#include "internals.h"
static MDBX_cursor *cursor_clone(const MDBX_cursor *csrc,
cursor_couple_t *couple) {
static MDBX_cursor *cursor_clone(const MDBX_cursor *csrc, cursor_couple_t *couple) {
cASSERT(csrc, csrc->txn->txnid >= csrc->txn->env->lck->cached_oldest.weak);
couple->outer.next = nullptr;
couple->outer.backup = nullptr;
@ -40,13 +39,10 @@ static MDBX_cursor *cursor_clone(const MDBX_cursor *csrc,
void recalculate_merge_thresholds(MDBX_env *env) {
const size_t bytes = page_space(env);
env->merge_threshold =
(uint16_t)(bytes -
(bytes * env->options.merge_threshold_16dot16_percent >> 16));
env->merge_threshold = (uint16_t)(bytes - (bytes * env->options.merge_threshold_16dot16_percent >> 16));
env->merge_threshold_gc =
(uint16_t)(bytes - ((env->options.merge_threshold_16dot16_percent > 19005)
? bytes / 3 /* 33 % */
: bytes / 4 /* 25 % */));
(uint16_t)(bytes - ((env->options.merge_threshold_16dot16_percent > 19005) ? bytes / 3 /* 33 % */
: bytes / 4 /* 25 % */));
}
int tree_drop(MDBX_cursor *mc, const bool may_have_tables) {
@ -60,9 +56,8 @@ int tree_drop(MDBX_cursor *mc, const bool may_have_tables) {
if (!(may_have_tables | mc->tree->large_pages))
cursor_pop(mc);
rc = pnl_need(&txn->tw.retired_pages, (size_t)mc->tree->branch_pages +
(size_t)mc->tree->leaf_pages +
(size_t)mc->tree->large_pages);
rc = pnl_need(&txn->tw.retired_pages,
(size_t)mc->tree->branch_pages + (size_t)mc->tree->leaf_pages + (size_t)mc->tree->large_pages);
if (unlikely(rc != MDBX_SUCCESS))
goto bailout;
@ -100,9 +95,7 @@ int tree_drop(MDBX_cursor *mc, const bool may_have_tables) {
cASSERT(mc, mc->top + 1 < mc->tree->height);
mc->checking |= z_retiring;
const unsigned pagetype = (is_frozen(txn, mp) ? P_FROZEN : 0) +
((mc->top + 2 == mc->tree->height)
? (mc->checking & (P_LEAF | P_DUPFIX))
: P_BRANCH);
((mc->top + 2 == mc->tree->height) ? (mc->checking & (P_LEAF | P_DUPFIX)) : P_BRANCH);
for (size_t i = 0; i < nkeys; i++) {
node_t *node = page_node(mp, i);
tASSERT(txn, (node_flags(node) & (N_BIG | N_TREE | N_DUP)) == 0);
@ -153,8 +146,7 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) {
cASSERT(csrc, csrc->top == cdst->top);
if (unlikely(page_type(psrc) != page_type(pdst))) {
bailout:
ERROR("Wrong or mismatch pages's types (src %d, dst %d) to move node",
page_type(psrc), page_type(pdst));
ERROR("Wrong or mismatch pages's types (src %d, dst %d) to move node", page_type(psrc), page_type(pdst));
csrc->txn->flags |= MDBX_TXN_ERROR;
return MDBX_PROBLEM;
}
@ -225,8 +217,7 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) {
mn->top = top;
mn->ki[mn->top] = 0;
const intptr_t delta = EVEN_CEIL(key.iov_len) -
EVEN_CEIL(node_ks(page_node(mn->pg[mn->top], 0)));
const intptr_t delta = EVEN_CEIL(key.iov_len) - EVEN_CEIL(node_ks(page_node(mn->pg[mn->top], 0)));
const intptr_t needed = branch_size(cdst->txn->env, &key4move) + delta;
const intptr_t have = page_room(pdst);
if (unlikely(needed > have))
@ -255,10 +246,8 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) {
pdst = cdst->pg[cdst->top];
}
DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO
" to node %u on page %" PRIaPGNO,
"branch", csrc->ki[csrc->top], DKEY_DEBUG(&key4move), psrc->pgno,
cdst->ki[cdst->top], pdst->pgno);
DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO " to node %u on page %" PRIaPGNO, "branch", csrc->ki[csrc->top],
DKEY_DEBUG(&key4move), psrc->pgno, cdst->ki[cdst->top], pdst->pgno);
/* Add the node to the destination page. */
rc = node_add_branch(cdst, cdst->ki[cdst->top], &key4move, srcpg);
} break;
@ -275,13 +264,10 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) {
data.iov_base = node_data(srcnode);
key4move.iov_len = node_ks(srcnode);
key4move.iov_base = node_key(srcnode);
DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO
" to node %u on page %" PRIaPGNO,
"leaf", csrc->ki[csrc->top], DKEY_DEBUG(&key4move), psrc->pgno,
cdst->ki[cdst->top], pdst->pgno);
DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO " to node %u on page %" PRIaPGNO, "leaf", csrc->ki[csrc->top],
DKEY_DEBUG(&key4move), psrc->pgno, cdst->ki[cdst->top], pdst->pgno);
/* Add the node to the destination page. */
rc = node_add_leaf(cdst, cdst->ki[cdst->top], &key4move, &data,
node_flags(srcnode));
rc = node_add_leaf(cdst, cdst->ki[cdst->top], &key4move, &data, node_flags(srcnode));
} break;
case P_LEAF | P_DUPFIX: {
@ -290,12 +276,9 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) {
return rc;
psrc = csrc->pg[csrc->top];
pdst = cdst->pg[cdst->top];
key4move =
page_dupfix_key(psrc, csrc->ki[csrc->top], csrc->tree->dupfix_size);
DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO
" to node %u on page %" PRIaPGNO,
"leaf2", csrc->ki[csrc->top], DKEY_DEBUG(&key4move), psrc->pgno,
cdst->ki[cdst->top], pdst->pgno);
key4move = page_dupfix_key(psrc, csrc->ki[csrc->top], csrc->tree->dupfix_size);
DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO " to node %u on page %" PRIaPGNO, "leaf2", csrc->ki[csrc->top],
DKEY_DEBUG(&key4move), psrc->pgno, cdst->ki[cdst->top], pdst->pgno);
/* Add the node to the destination page. */
rc = node_add_dupfix(cdst, cdst->ki[cdst->top], &key4move);
} break;
@ -329,13 +312,11 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) {
if (!is_related(csrc, m3))
continue;
if (m3 != cdst && m3->pg[csrc->top] == pdst &&
m3->ki[csrc->top] >= cdst->ki[csrc->top]) {
if (m3 != cdst && m3->pg[csrc->top] == pdst && m3->ki[csrc->top] >= cdst->ki[csrc->top]) {
m3->ki[csrc->top] += 1;
}
if (/* m3 != csrc && */ m3->pg[csrc->top] == psrc &&
m3->ki[csrc->top] == csrc->ki[csrc->top]) {
if (/* m3 != csrc && */ m3->pg[csrc->top] == psrc && m3->ki[csrc->top] == csrc->ki[csrc->top]) {
m3->pg[csrc->top] = pdst;
m3->ki[csrc->top] = cdst->ki[cdst->top];
cASSERT(csrc, csrc->top > 0);
@ -387,8 +368,7 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) {
key.iov_len = node_ks(srcnode);
key.iov_base = node_key(srcnode);
}
DEBUG("update separator for source page %" PRIaPGNO " to [%s]",
psrc->pgno, DKEY_DEBUG(&key));
DEBUG("update separator for source page %" PRIaPGNO " to [%s]", psrc->pgno, DKEY_DEBUG(&key));
cursor_couple_t couple;
MDBX_cursor *const mn = cursor_clone(csrc, &couple);
@ -423,8 +403,7 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) {
key.iov_len = node_ks(srcnode);
key.iov_base = node_key(srcnode);
}
DEBUG("update separator for destination page %" PRIaPGNO " to [%s]",
pdst->pgno, DKEY_DEBUG(&key));
DEBUG("update separator for destination page %" PRIaPGNO " to [%s]", pdst->pgno, DKEY_DEBUG(&key));
cursor_couple_t couple;
MDBX_cursor *const mn = cursor_clone(cdst, &couple);
cASSERT(cdst, mn->top > 0);
@ -465,12 +444,10 @@ static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) {
cASSERT(csrc, csrc->clc == cdst->clc && csrc->tree == cdst->tree);
cASSERT(csrc, csrc->top > 0); /* can't merge root page */
cASSERT(cdst, cdst->top > 0);
cASSERT(cdst, cdst->top + 1 < cdst->tree->height ||
is_leaf(cdst->pg[cdst->tree->height - 1]));
cASSERT(csrc, csrc->top + 1 < csrc->tree->height ||
is_leaf(csrc->pg[csrc->tree->height - 1]));
cASSERT(cdst, csrc->txn->env->options.prefer_waf_insteadof_balance ||
page_room(pdst) >= page_used(cdst->txn->env, psrc));
cASSERT(cdst, cdst->top + 1 < cdst->tree->height || is_leaf(cdst->pg[cdst->tree->height - 1]));
cASSERT(csrc, csrc->top + 1 < csrc->tree->height || is_leaf(csrc->pg[csrc->tree->height - 1]));
cASSERT(cdst,
csrc->txn->env->options.prefer_waf_insteadof_balance || page_room(pdst) >= page_used(cdst->txn->env, psrc));
const int pagetype = page_type(psrc);
/* Move all nodes from src to dst */
@ -560,10 +537,8 @@ static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) {
}
pdst = cdst->pg[cdst->top];
DEBUG("dst page %" PRIaPGNO " now has %zu keys (%u.%u%% filled)",
pdst->pgno, page_numkeys(pdst),
page_fill_percentum_x10(cdst->txn->env, pdst) / 10,
page_fill_percentum_x10(cdst->txn->env, pdst) % 10);
DEBUG("dst page %" PRIaPGNO " now has %zu keys (%u.%u%% filled)", pdst->pgno, page_numkeys(pdst),
page_fill_percentum_x10(cdst->txn->env, pdst) / 10, page_fill_percentum_x10(cdst->txn->env, pdst) % 10);
cASSERT(csrc, psrc == csrc->pg[csrc->top]);
cASSERT(cdst, pdst == cdst->pg[cdst->top]);
@ -598,11 +573,8 @@ static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) {
m3->pg[csrc->top] = pdst;
m3->ki[csrc->top] += (indx_t)dst_nkeys;
m3->ki[csrc->top - 1] = cdst->ki[csrc->top - 1];
} else if (m3->pg[csrc->top - 1] == csrc->pg[csrc->top - 1] &&
m3->ki[csrc->top - 1] > csrc->ki[csrc->top - 1]) {
cASSERT(m3, m3->ki[csrc->top - 1] > 0 &&
m3->ki[csrc->top - 1] <=
page_numkeys(m3->pg[csrc->top - 1]));
} else if (m3->pg[csrc->top - 1] == csrc->pg[csrc->top - 1] && m3->ki[csrc->top - 1] > csrc->ki[csrc->top - 1]) {
cASSERT(m3, m3->ki[csrc->top - 1] > 0 && m3->ki[csrc->top - 1] <= page_numkeys(m3->pg[csrc->top - 1]));
m3->ki[csrc->top - 1] -= 1;
}
@ -641,8 +613,7 @@ static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) {
if (is_leaf(cdst->pg[cdst->top])) {
/* LY: don't touch cursor if top-page is a LEAF */
cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) ||
page_type(cdst->pg[cdst->top]) == pagetype);
cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype);
return MDBX_SUCCESS;
}
@ -656,8 +627,7 @@ static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) {
if (top_page == cdst->pg[cdst->top]) {
/* LY: don't touch cursor if prev top-page already on the top */
cASSERT(cdst, cdst->ki[cdst->top] == top_indx);
cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) ||
page_type(cdst->pg[cdst->top]) == pagetype);
cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype);
return MDBX_SUCCESS;
}
@ -671,18 +641,15 @@ static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) {
cASSERT(cdst, cdst->ki[new_top] == top_indx);
/* LY: restore cursor stack */
cdst->top = (int8_t)new_top;
cASSERT(cdst, cdst->top + 1 < cdst->tree->height ||
is_leaf(cdst->pg[cdst->tree->height - 1]));
cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) ||
page_type(cdst->pg[cdst->top]) == pagetype);
cASSERT(cdst, cdst->top + 1 < cdst->tree->height || is_leaf(cdst->pg[cdst->tree->height - 1]));
cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype);
return MDBX_SUCCESS;
}
page_t *const stub_page = (page_t *)(~(uintptr_t)top_page);
const indx_t stub_indx = top_indx;
if (save_height > cdst->tree->height &&
((cdst->pg[save_top] == top_page && cdst->ki[save_top] == top_indx) ||
(cdst->pg[save_top] == stub_page && cdst->ki[save_top] == stub_indx))) {
if (save_height > cdst->tree->height && ((cdst->pg[save_top] == top_page && cdst->ki[save_top] == top_indx) ||
(cdst->pg[save_top] == stub_page && cdst->ki[save_top] == stub_indx))) {
/* LY: restore cursor stack */
cdst->pg[new_top] = top_page;
cdst->ki[new_top] = top_indx;
@ -691,10 +658,8 @@ static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) {
cdst->ki[new_top + 1] = INT16_MAX;
#endif
cdst->top = (int8_t)new_top;
cASSERT(cdst, cdst->top + 1 < cdst->tree->height ||
is_leaf(cdst->pg[cdst->tree->height - 1]));
cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) ||
page_type(cdst->pg[cdst->top]) == pagetype);
cASSERT(cdst, cdst->top + 1 < cdst->tree->height || is_leaf(cdst->pg[cdst->tree->height - 1]));
cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype);
return MDBX_SUCCESS;
}
@ -707,8 +672,7 @@ bailout:
int tree_rebalance(MDBX_cursor *mc) {
cASSERT(mc, cursor_is_tracked(mc));
cASSERT(mc, mc->top >= 0);
cASSERT(mc, mc->top + 1 < mc->tree->height ||
is_leaf(mc->pg[mc->tree->height - 1]));
cASSERT(mc, mc->top + 1 < mc->tree->height || is_leaf(mc->pg[mc->tree->height - 1]));
const page_t *const tp = mc->pg[mc->top];
const uint8_t pagetype = page_type(tp);
@ -716,29 +680,22 @@ int tree_rebalance(MDBX_cursor *mc) {
const size_t minkeys = (pagetype & P_BRANCH) + (size_t)1;
/* Pages emptier than this are candidates for merging. */
size_t room_threshold = likely(mc->tree != &mc->txn->dbs[FREE_DBI])
? mc->txn->env->merge_threshold
: mc->txn->env->merge_threshold_gc;
size_t room_threshold =
likely(mc->tree != &mc->txn->dbs[FREE_DBI]) ? mc->txn->env->merge_threshold : mc->txn->env->merge_threshold_gc;
const size_t numkeys = page_numkeys(tp);
const size_t room = page_room(tp);
DEBUG("rebalancing %s page %" PRIaPGNO
" (has %zu keys, fill %u.%u%%, used %zu, room %zu bytes)",
is_leaf(tp) ? "leaf" : "branch", tp->pgno, numkeys,
page_fill_percentum_x10(mc->txn->env, tp) / 10,
page_fill_percentum_x10(mc->txn->env, tp) % 10,
page_used(mc->txn->env, tp), room);
DEBUG("rebalancing %s page %" PRIaPGNO " (has %zu keys, fill %u.%u%%, used %zu, room %zu bytes)",
is_leaf(tp) ? "leaf" : "branch", tp->pgno, numkeys, page_fill_percentum_x10(mc->txn->env, tp) / 10,
page_fill_percentum_x10(mc->txn->env, tp) % 10, page_used(mc->txn->env, tp), room);
cASSERT(mc, is_modifable(mc->txn, tp));
if (unlikely(numkeys < minkeys)) {
DEBUG("page %" PRIaPGNO " must be merged due keys < %zu threshold",
tp->pgno, minkeys);
DEBUG("page %" PRIaPGNO " must be merged due keys < %zu threshold", tp->pgno, minkeys);
} else if (unlikely(room > room_threshold)) {
DEBUG("page %" PRIaPGNO " should be merged due room %zu > %zu threshold",
tp->pgno, room, room_threshold);
DEBUG("page %" PRIaPGNO " should be merged due room %zu > %zu threshold", tp->pgno, room, room_threshold);
} else {
DEBUG("no need to rebalance page %" PRIaPGNO ", room %zu < %zu threshold",
tp->pgno, room, room_threshold);
DEBUG("no need to rebalance page %" PRIaPGNO ", room %zu < %zu threshold", tp->pgno, room, room_threshold);
cASSERT(mc, mc->tree->items > 0);
return MDBX_SUCCESS;
}
@ -752,11 +709,9 @@ int tree_rebalance(MDBX_cursor *mc) {
DEBUG("%s", "tree is completely empty");
cASSERT(mc, is_leaf(mp));
cASSERT(mc, (*cursor_dbi_state(mc) & DBI_DIRTY) != 0);
cASSERT(mc, mc->tree->branch_pages == 0 && mc->tree->large_pages == 0 &&
mc->tree->leaf_pages == 1);
cASSERT(mc, mc->tree->branch_pages == 0 && mc->tree->large_pages == 0 && mc->tree->leaf_pages == 1);
/* Adjust cursors pointing to mp */
for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2;
m2 = m2->next) {
for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) {
MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2;
if (!is_poor(m3) && m3->pg[0] == mp) {
be_poor(m3);
@ -790,8 +745,7 @@ int tree_rebalance(MDBX_cursor *mc) {
}
/* Adjust other cursors pointing to mp */
for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2;
m2 = m2->next) {
for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) {
MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2;
if (is_related(mc, m3) && m3->pg[0] == mp) {
for (intptr_t i = 0; i < mc->tree->height; i++) {
@ -801,14 +755,11 @@ int tree_rebalance(MDBX_cursor *mc) {
m3->top -= 1;
}
}
cASSERT(mc, is_leaf(mc->pg[mc->top]) ||
page_type(mc->pg[mc->top]) == pagetype);
cASSERT(mc, mc->top + 1 < mc->tree->height ||
is_leaf(mc->pg[mc->tree->height - 1]));
cASSERT(mc, is_leaf(mc->pg[mc->top]) || page_type(mc->pg[mc->top]) == pagetype);
cASSERT(mc, mc->top + 1 < mc->tree->height || is_leaf(mc->pg[mc->tree->height - 1]));
return page_retire(mc, mp);
}
DEBUG("root page %" PRIaPGNO " doesn't need rebalancing (flags 0x%x)",
mp->pgno, mp->flags);
DEBUG("root page %" PRIaPGNO " doesn't need rebalancing (flags 0x%x)", mp->pgno, mp->flags);
return MDBX_SUCCESS;
}
@ -829,17 +780,14 @@ int tree_rebalance(MDBX_cursor *mc) {
page_t *left = nullptr, *right = nullptr;
if (mn->ki[pre_top] > 0) {
rc =
page_get(mn, node_pgno(page_node(mn->pg[pre_top], mn->ki[pre_top] - 1)),
&left, mc->pg[mc->top]->txnid);
rc = page_get(mn, node_pgno(page_node(mn->pg[pre_top], mn->ki[pre_top] - 1)), &left, mc->pg[mc->top]->txnid);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
cASSERT(mc, page_type(left) == page_type(mc->pg[mc->top]));
}
if (mn->ki[pre_top] + (size_t)1 < page_numkeys(mn->pg[pre_top])) {
rc = page_get(
mn, node_pgno(page_node(mn->pg[pre_top], mn->ki[pre_top] + (size_t)1)),
&right, mc->pg[mc->top]->txnid);
rc = page_get(mn, node_pgno(page_node(mn->pg[pre_top], mn->ki[pre_top] + (size_t)1)), &right,
mc->pg[mc->top]->txnid);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
cASSERT(mc, page_type(right) == page_type(mc->pg[mc->top]));
@ -857,8 +805,7 @@ int tree_rebalance(MDBX_cursor *mc) {
bool involve = !(left && right);
retry:
cASSERT(mc, mc->top > 0);
if (left_room > room_threshold && left_room >= right_room &&
(is_modifable(mc->txn, left) || involve)) {
if (left_room > room_threshold && left_room >= right_room && (is_modifable(mc->txn, left) || involve)) {
/* try merge with left */
cASSERT(mc, left_nkeys >= minkeys);
mn->pg[mn->top] = left;
@ -878,8 +825,7 @@ retry:
return rc;
}
}
if (right_room > room_threshold &&
(is_modifable(mc->txn, right) || involve)) {
if (right_room > room_threshold && (is_modifable(mc->txn, right) || involve)) {
/* try merge with right */
cASSERT(mc, right_nkeys >= minkeys);
mn->pg[mn->top] = right;
@ -897,8 +843,7 @@ retry:
}
}
if (left_nkeys > minkeys &&
(right_nkeys <= left_nkeys || right_room >= left_room) &&
if (left_nkeys > minkeys && (right_nkeys <= left_nkeys || right_room >= left_room) &&
(is_modifable(mc->txn, left) || involve)) {
/* try move from left */
mn->pg[mn->top] = left;
@ -939,16 +884,13 @@ retry:
return MDBX_SUCCESS;
}
if (mc->txn->env->options.prefer_waf_insteadof_balance &&
likely(room_threshold > 0)) {
if (mc->txn->env->options.prefer_waf_insteadof_balance && likely(room_threshold > 0)) {
room_threshold = 0;
goto retry;
}
if (likely(!involve) &&
(likely(mc->tree != &mc->txn->dbs[FREE_DBI]) || mc->txn->tw.loose_pages ||
MDBX_PNL_GETSIZE(mc->txn->tw.relist) ||
(mc->flags & z_gcu_preparation) || (mc->txn->flags & txn_gc_drained) ||
room_threshold)) {
(likely(mc->tree != &mc->txn->dbs[FREE_DBI]) || mc->txn->tw.loose_pages || MDBX_PNL_GETSIZE(mc->txn->tw.relist) ||
(mc->flags & z_gcu_preparation) || (mc->txn->flags & txn_gc_drained) || room_threshold)) {
involve = true;
goto retry;
}
@ -957,17 +899,14 @@ retry:
goto retry;
}
ERROR("Unable to merge/rebalance %s page %" PRIaPGNO
" (has %zu keys, fill %u.%u%%, used %zu, room %zu bytes)",
is_leaf(tp) ? "leaf" : "branch", tp->pgno, numkeys,
page_fill_percentum_x10(mc->txn->env, tp) / 10,
page_fill_percentum_x10(mc->txn->env, tp) % 10,
page_used(mc->txn->env, tp), room);
ERROR("Unable to merge/rebalance %s page %" PRIaPGNO " (has %zu keys, fill %u.%u%%, used %zu, room %zu bytes)",
is_leaf(tp) ? "leaf" : "branch", tp->pgno, numkeys, page_fill_percentum_x10(mc->txn->env, tp) / 10,
page_fill_percentum_x10(mc->txn->env, tp) % 10, page_used(mc->txn->env, tp), room);
return MDBX_PROBLEM;
}
int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
MDBX_val *const newdata, pgno_t newpgno, const unsigned naf) {
int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, MDBX_val *const newdata, pgno_t newpgno,
const unsigned naf) {
unsigned flags;
int rc = MDBX_SUCCESS, foliage = 0;
MDBX_env *const env = mc->txn->env;
@ -988,11 +927,8 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
STATIC_ASSERT(P_BRANCH == 1);
const size_t minkeys = (mp->flags & P_BRANCH) + (size_t)1;
DEBUG(">> splitting %s-page %" PRIaPGNO
" and adding %zu+%zu [%s] at %i, nkeys %zi",
is_leaf(mp) ? "leaf" : "branch", mp->pgno, newkey->iov_len,
newdata ? newdata->iov_len : 0, DKEY_DEBUG(newkey), mc->ki[mc->top],
nkeys);
DEBUG(">> splitting %s-page %" PRIaPGNO " and adding %zu+%zu [%s] at %i, nkeys %zi", is_leaf(mp) ? "leaf" : "branch",
mp->pgno, newkey->iov_len, newdata ? newdata->iov_len : 0, DKEY_DEBUG(newkey), mc->ki[mc->top], nkeys);
cASSERT(mc, nkeys + 1 >= minkeys * 2);
/* Create a new sibling page. */
@ -1057,10 +993,8 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
mn->ki[mn->top] = 0;
mn->ki[prev_top] = mc->ki[prev_top] + 1;
size_t split_indx =
(newindx < nkeys)
? /* split at the middle */ (nkeys + 1) >> 1
: /* split at the end (i.e. like append-mode ) */ nkeys - minkeys + 1;
size_t split_indx = (newindx < nkeys) ? /* split at the middle */ (nkeys + 1) >> 1
: /* split at the end (i.e. like append-mode ) */ nkeys - minkeys + 1;
eASSERT(env, split_indx >= minkeys && split_indx <= nkeys - minkeys + 1);
cASSERT(mc, !is_branch(mp) || newindx > 0);
@ -1094,11 +1028,9 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
if (foliage) {
TRACE("pure-left: foliage %u, top %i, ptop %zu, split_indx %zi, "
"minkeys %zi, sepkey %s, parent-room %zu, need4split %zu",
foliage, mc->top, prev_top, split_indx, minkeys,
DKEY_DEBUG(&sepkey), page_room(mc->pg[prev_top]),
foliage, mc->top, prev_top, split_indx, minkeys, DKEY_DEBUG(&sepkey), page_room(mc->pg[prev_top]),
branch_size(env, &sepkey));
TRACE("pure-left: newkey %s, newdata %s, newindx %zu",
DKEY_DEBUG(newkey), DVAL_DEBUG(newdata), newindx);
TRACE("pure-left: newkey %s, newdata %s, newindx %zu", DKEY_DEBUG(newkey), DVAL_DEBUG(newdata), newindx);
}
}
}
@ -1112,8 +1044,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
sepkey = *newkey;
} else if (unlikely(pure_left)) {
/* newindx == split_indx == 0 */
TRACE("pure-left: no-split, but add new pure page at the %s",
"left/before");
TRACE("pure-left: no-split, but add new pure page at the %s", "left/before");
cASSERT(mc, newindx == 0 && split_indx == 0 && minkeys == 1);
TRACE("pure-left: old-first-key is %s", DKEY_DEBUG(&sepkey));
} else {
@ -1139,8 +1070,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
void *const ins = page_dupfix_ptr(mp, mc->ki[mc->top], ksize);
memcpy(sister->entries, split, rsize);
sepkey.iov_base = sister->entries;
memmove(ptr_disp(ins, ksize), ins,
(split_indx - mc->ki[mc->top]) * ksize);
memmove(ptr_disp(ins, ksize), ins, (split_indx - mc->ki[mc->top]) * ksize);
memcpy(ins, newkey->iov_base, ksize);
cASSERT(mc, UINT16_MAX - mp->lower >= (int)sizeof(indx_t));
mp->lower += sizeof(indx_t);
@ -1151,16 +1081,14 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
memcpy(sister->entries, split, distance * ksize);
void *const ins = page_dupfix_ptr(sister, distance, ksize);
memcpy(ins, newkey->iov_base, ksize);
memcpy(ptr_disp(ins, ksize), ptr_disp(split, distance * ksize),
rsize - distance * ksize);
memcpy(ptr_disp(ins, ksize), ptr_disp(split, distance * ksize), rsize - distance * ksize);
cASSERT(mc, UINT16_MAX - sister->lower >= (int)sizeof(indx_t));
sister->lower += sizeof(indx_t);
cASSERT(mc, sister->upper >= ksize - sizeof(indx_t));
sister->upper -= (indx_t)(ksize - sizeof(indx_t));
cASSERT(mc, distance <= (int)UINT16_MAX);
mc->ki[mc->top] = (indx_t)distance;
cASSERT(mc,
(((ksize & page_numkeys(sister)) ^ sister->upper) & 1) == 0);
cASSERT(mc, (((ksize & page_numkeys(sister)) ^ sister->upper) & 1) == 0);
}
if (AUDIT_ENABLED()) {
@ -1180,8 +1108,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
}
const size_t max_space = page_space(env);
const size_t new_size = is_leaf(mp) ? leaf_size(env, newkey, newdata)
: branch_size(env, newkey);
const size_t new_size = is_leaf(mp) ? leaf_size(env, newkey, newdata) : branch_size(env, newkey);
/* prepare to insert */
size_t i = 0;
@ -1218,8 +1145,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
split_indx += mp->flags & P_BRANCH;
}
eASSERT(env, split_indx >= minkeys && split_indx <= nkeys + 1 - minkeys);
const size_t dim_nodes =
(newindx >= split_indx) ? split_indx : nkeys - split_indx;
const size_t dim_nodes = (newindx >= split_indx) ? split_indx : nkeys - split_indx;
const size_t dim_used = (sizeof(indx_t) + NODESIZE + 1) * dim_nodes;
if (new_size >= dim_used) {
/* Search for best acceptable split point */
@ -1239,15 +1165,13 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
node_t *node = ptr_disp(mp, tmp_ki_copy->entries[i] + PAGEHDRSZ);
size = NODESIZE + node_ks(node) + sizeof(indx_t);
if (is_leaf(mp))
size +=
(node_flags(node) & N_BIG) ? sizeof(pgno_t) : node_ds(node);
size += (node_flags(node) & N_BIG) ? sizeof(pgno_t) : node_ds(node);
size = EVEN_CEIL(size);
}
before += size;
after -= size;
TRACE("step %zu, size %zu, before %zu, after %zu, max %zu", i, size,
before, after, max_space);
TRACE("step %zu, size %zu, before %zu, after %zu, max %zu", i, size, before, after, max_space);
if (before <= max_space && after <= max_space) {
const size_t split = i + (dir > 0);
@ -1271,8 +1195,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
sepkey = *newkey;
if (split_indx != newindx) {
node_t *node =
ptr_disp(mp, tmp_ki_copy->entries[split_indx] + PAGEHDRSZ);
node_t *node = ptr_disp(mp, tmp_ki_copy->entries[split_indx] + PAGEHDRSZ);
sepkey.iov_len = node_ks(node);
sepkey.iov_base = node_key(node);
}
@ -1308,8 +1231,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
/* Right page might now have changed parent.
* Check if left page also changed parent. */
if (mn->pg[prev_top] != mc->pg[prev_top] &&
mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) {
if (mn->pg[prev_top] != mc->pg[prev_top] && mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) {
for (intptr_t i = 0; i < prev_top; i++) {
mc->pg[i] = mn->pg[i];
mc->ki[i] = mn->ki[i];
@ -1334,14 +1256,11 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
page_t *ptop_page = mc->pg[prev_top];
TRACE("pure-left: adding to parent page %u node[%u] left-leaf page #%u key "
"%s",
ptop_page->pgno, mc->ki[prev_top], sister->pgno,
DKEY(mc->ki[prev_top] ? newkey : nullptr));
ptop_page->pgno, mc->ki[prev_top], sister->pgno, DKEY(mc->ki[prev_top] ? newkey : nullptr));
assert(mc->top == prev_top + 1);
mc->top = (uint8_t)prev_top;
rc = node_add_branch(mc, mc->ki[prev_top],
mc->ki[prev_top] ? newkey : nullptr, sister->pgno);
cASSERT(mc, mp == mc->pg[prev_top + 1] && newindx == mc->ki[prev_top + 1] &&
prev_top == mc->top);
rc = node_add_branch(mc, mc->ki[prev_top], mc->ki[prev_top] ? newkey : nullptr, sister->pgno);
cASSERT(mc, mp == mc->pg[prev_top + 1] && newindx == mc->ki[prev_top + 1] && prev_top == mc->top);
if (likely(rc == MDBX_SUCCESS) && mc->ki[prev_top] == 0) {
node_t *node = page_node(mc->pg[prev_top], 1);
@ -1351,12 +1270,10 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
mc->ki[prev_top] = 1;
rc = tree_propagate_key(mc, &sepkey);
cASSERT(mc, mc->top == prev_top && mc->ki[prev_top] == 1);
cASSERT(mc,
mp == mc->pg[prev_top + 1] && newindx == mc->ki[prev_top + 1]);
cASSERT(mc, mp == mc->pg[prev_top + 1] && newindx == mc->ki[prev_top + 1]);
mc->ki[prev_top] = 0;
} else {
TRACE("pure-left: no-need-update prev-first key on parent %s",
DKEY(&sepkey));
TRACE("pure-left: no-need-update prev-first key on parent %s", DKEY(&sepkey));
}
mc->top++;
@ -1367,8 +1284,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
cASSERT(mc, node_pgno(node) == mp->pgno && mc->pg[prev_top] == ptop_page);
} else {
mn->top -= 1;
TRACE("add-to-parent the right-entry[%u] for new sibling-page",
mn->ki[prev_top]);
TRACE("add-to-parent the right-entry[%u] for new sibling-page", mn->ki[prev_top]);
rc = node_add_branch(mn, mn->ki[prev_top], &sepkey, sister->pgno);
mn->top += 1;
if (unlikely(rc != MDBX_SUCCESS))
@ -1403,8 +1319,8 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
sepkey = get_key(page_node(mc->pg[mc->top - i], mc->ki[mc->top - i]));
if (mc->clc->k.cmp(newkey, &sepkey) < 0) {
mc->top -= (int8_t)i;
DEBUG("pure-left: update new-first on parent [%i] page %u key %s",
mc->ki[mc->top], mc->pg[mc->top]->pgno, DKEY(newkey));
DEBUG("pure-left: update new-first on parent [%i] page %u key %s", mc->ki[mc->top], mc->pg[mc->top]->pgno,
DKEY(newkey));
rc = tree_propagate_key(mc, newkey);
mc->top += (int8_t)i;
if (unlikely(rc != MDBX_SUCCESS))
@ -1474,16 +1390,14 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
}
} while (ii != split_indx);
TRACE("ii %zu, nkeys %zu, n %zu, pgno #%u", ii, nkeys, n,
mc->pg[mc->top]->pgno);
TRACE("ii %zu, nkeys %zu, n %zu, pgno #%u", ii, nkeys, n, mc->pg[mc->top]->pgno);
nkeys = page_numkeys(tmp_ki_copy);
for (size_t i = 0; i < nkeys; i++)
mp->entries[i] = tmp_ki_copy->entries[i];
mp->lower = tmp_ki_copy->lower;
mp->upper = tmp_ki_copy->upper;
memcpy(page_node(mp, nkeys - 1), page_node(tmp_ki_copy, nkeys - 1),
env->ps - tmp_ki_copy->upper - PAGEHDRSZ);
memcpy(page_node(mp, nkeys - 1), page_node(tmp_ki_copy, nkeys - 1), env->ps - tmp_ki_copy->upper - PAGEHDRSZ);
/* reset back to original page */
if (newindx < split_indx) {
@ -1492,8 +1406,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
mc->pg[mc->top] = sister;
mc->ki[prev_top]++;
/* Make sure ki is still valid. */
if (mn->pg[prev_top] != mc->pg[prev_top] &&
mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) {
if (mn->pg[prev_top] != mc->pg[prev_top] && mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) {
for (intptr_t i = 0; i <= prev_top; i++) {
mc->pg[i] = mn->pg[i];
mc->ki[i] = mn->ki[i];
@ -1504,8 +1417,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
mc->pg[mc->top] = sister;
mc->ki[prev_top]++;
/* Make sure ki is still valid. */
if (mn->pg[prev_top] != mc->pg[prev_top] &&
mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) {
if (mn->pg[prev_top] != mc->pg[prev_top] && mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) {
for (intptr_t i = 0; i <= prev_top; i++) {
mc->pg[i] = mn->pg[i];
mc->ki[i] = mn->ki[i];
@ -1545,16 +1457,14 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
m3->pg[i] = mn->pg[i];
}
}
} else if (!did_split_parent && m3->top >= prev_top &&
m3->pg[prev_top] == mc->pg[prev_top] &&
} else if (!did_split_parent && m3->top >= prev_top && m3->pg[prev_top] == mc->pg[prev_top] &&
m3->ki[prev_top] >= mc->ki[prev_top]) {
m3->ki[prev_top]++; /* also for the `pure-left` case */
}
if (inner_pointed(m3) && is_leaf(mp))
cursor_inner_refresh(m3, m3->pg[mc->top], m3->ki[mc->top]);
}
TRACE("mp #%u left: %zd, sister #%u left: %zd", mp->pgno, page_room(mp),
sister->pgno, page_room(sister));
TRACE("mp #%u left: %zd, sister #%u left: %zd", mp->pgno, page_room(mp), sister->pgno, page_room(sister));
done:
if (tmp_ki_copy)
@ -1596,8 +1506,8 @@ int tree_propagate_key(MDBX_cursor *mc, const MDBX_val *key) {
MDBX_val k2;
k2.iov_base = node_key(node);
k2.iov_len = node_ks(node);
DEBUG("update key %zi (offset %zu) [%s] to [%s] on page %" PRIaPGNO, indx,
ptr, DVAL_DEBUG(&k2), DKEY_DEBUG(key), mp->pgno);
DEBUG("update key %zi (offset %zu) [%s] to [%s] on page %" PRIaPGNO, indx, ptr, DVAL_DEBUG(&k2), DKEY_DEBUG(key),
mp->pgno);
#endif /* MDBX_DEBUG */
/* Sizes must be 2-byte aligned. */

29
src/txl.c
View File

@ -6,8 +6,7 @@
static inline size_t txl_size2bytes(const size_t size) {
assert(size > 0 && size <= txl_max * 2);
size_t bytes =
ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(txnid_t) * (size + 2),
txl_granulate * sizeof(txnid_t)) -
ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(txnid_t) * (size + 2), txl_granulate * sizeof(txnid_t)) -
MDBX_ASSUME_MALLOC_OVERHEAD;
return bytes;
}
@ -38,11 +37,9 @@ MDBX_INTERNAL void txl_free(txl_t txl) {
osal_free(txl - 1);
}
MDBX_INTERNAL int txl_reserve(txl_t __restrict *__restrict ptxl,
const size_t wanna) {
MDBX_INTERNAL int txl_reserve(txl_t __restrict *__restrict ptxl, const size_t wanna) {
const size_t allocated = (size_t)MDBX_PNL_ALLOCLEN(*ptxl);
assert(MDBX_PNL_GETSIZE(*ptxl) <= txl_max &&
MDBX_PNL_ALLOCLEN(*ptxl) >= MDBX_PNL_GETSIZE(*ptxl));
assert(MDBX_PNL_GETSIZE(*ptxl) <= txl_max && MDBX_PNL_ALLOCLEN(*ptxl) >= MDBX_PNL_GETSIZE(*ptxl));
if (likely(allocated >= wanna))
return MDBX_SUCCESS;
@ -51,9 +48,7 @@ MDBX_INTERNAL int txl_reserve(txl_t __restrict *__restrict ptxl,
return MDBX_TXN_FULL;
}
const size_t size = (wanna + wanna - allocated < txl_max)
? wanna + wanna - allocated
: txl_max;
const size_t size = (wanna + wanna - allocated < txl_max) ? wanna + wanna - allocated : txl_max;
size_t bytes = txl_size2bytes(size);
txl_t txl = osal_realloc(*ptxl - 1, bytes);
if (likely(txl)) {
@ -68,14 +63,11 @@ MDBX_INTERNAL int txl_reserve(txl_t __restrict *__restrict ptxl,
return MDBX_ENOMEM;
}
static __always_inline int __must_check_result
txl_need(txl_t __restrict *__restrict ptxl, size_t num) {
assert(MDBX_PNL_GETSIZE(*ptxl) <= txl_max &&
MDBX_PNL_ALLOCLEN(*ptxl) >= MDBX_PNL_GETSIZE(*ptxl));
static __always_inline int __must_check_result txl_need(txl_t __restrict *__restrict ptxl, size_t num) {
assert(MDBX_PNL_GETSIZE(*ptxl) <= txl_max && MDBX_PNL_ALLOCLEN(*ptxl) >= MDBX_PNL_GETSIZE(*ptxl));
assert(num <= PAGELIST_LIMIT);
const size_t wanna = (size_t)MDBX_PNL_GETSIZE(*ptxl) + num;
return likely(MDBX_PNL_ALLOCLEN(*ptxl) >= wanna) ? MDBX_SUCCESS
: txl_reserve(ptxl, wanna);
return likely(MDBX_PNL_ALLOCLEN(*ptxl) >= wanna) ? MDBX_SUCCESS : txl_reserve(ptxl, wanna);
}
static __always_inline void txl_xappend(txl_t __restrict txl, txnid_t id) {
@ -86,12 +78,9 @@ static __always_inline void txl_xappend(txl_t __restrict txl, txnid_t id) {
#define TXNID_SORT_CMP(first, last) ((first) > (last))
SORT_IMPL(txnid_sort, false, txnid_t, TXNID_SORT_CMP)
MDBX_INTERNAL void txl_sort(txl_t txl) {
txnid_sort(MDBX_PNL_BEGIN(txl), MDBX_PNL_END(txl));
}
MDBX_INTERNAL void txl_sort(txl_t txl) { txnid_sort(MDBX_PNL_BEGIN(txl), MDBX_PNL_END(txl)); }
MDBX_INTERNAL int __must_check_result txl_append(txl_t __restrict *ptxl,
txnid_t id) {
MDBX_INTERNAL int __must_check_result txl_append(txl_t __restrict *ptxl, txnid_t id) {
if (unlikely(MDBX_PNL_GETSIZE(*ptxl) == MDBX_PNL_ALLOCLEN(*ptxl))) {
int rc = txl_need(ptxl, txl_granulate);
if (unlikely(rc != MDBX_SUCCESS))

6
src/txl.h
View File

@ -11,8 +11,7 @@ typedef const txnid_t *const_txl_t;
enum txl_rules {
txl_granulate = 32,
txl_initial =
txl_granulate - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t),
txl_initial = txl_granulate - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t),
txl_max = (1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)
};
@ -20,7 +19,6 @@ MDBX_INTERNAL txl_t txl_alloc(void);
MDBX_INTERNAL void txl_free(txl_t txl);
MDBX_INTERNAL int __must_check_result txl_append(txl_t __restrict *ptxl,
txnid_t id);
MDBX_INTERNAL int __must_check_result txl_append(txl_t __restrict *ptxl, txnid_t id);
MDBX_INTERNAL void txl_sort(txl_t txl);

539
src/txn.c
View File

File diff suppressed because it is too large Load Diff

119
src/unaligned.h
View File

@ -6,22 +6,17 @@
/*------------------------------------------------------------------------------
* Unaligned access */
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t
field_alignment(size_t alignment_baseline, size_t field_offset) {
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t field_alignment(size_t alignment_baseline,
size_t field_offset) {
size_t merge = alignment_baseline | (size_t)field_offset;
return merge & -(int)merge;
}
/* read-thunk for UB-sanitizer */
MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t
peek_u8(const uint8_t *__restrict ptr) {
return *ptr;
}
MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t peek_u8(const uint8_t *__restrict ptr) { return *ptr; }
/* write-thunk for UB-sanitizer */
static inline void poke_u8(uint8_t *__restrict ptr, const uint8_t v) {
*ptr = v;
}
static inline void poke_u8(uint8_t *__restrict ptr, const uint8_t v) { *ptr = v; }
static inline void *bcopy_2(void *__restrict dst, const void *__restrict src) {
uint8_t *__restrict d = (uint8_t *)dst;
@ -31,8 +26,7 @@ static inline void *bcopy_2(void *__restrict dst, const void *__restrict src) {
return d;
}
static inline void *bcopy_4(void *const __restrict dst,
const void *const __restrict src) {
static inline void *bcopy_4(void *const __restrict dst, const void *const __restrict src) {
uint8_t *__restrict d = (uint8_t *)dst;
const uint8_t *__restrict s = (uint8_t *)src;
d[0] = s[0];
@ -42,8 +36,7 @@ static inline void *bcopy_4(void *const __restrict dst,
return d;
}
static inline void *bcopy_8(void *const __restrict dst,
const void *const __restrict src) {
static inline void *bcopy_8(void *const __restrict dst, const void *const __restrict src) {
uint8_t *__restrict d = (uint8_t *)dst;
const uint8_t *__restrict s = (uint8_t *)src;
d[0] = s[0];
@ -57,14 +50,13 @@ static inline void *bcopy_8(void *const __restrict dst,
return d;
}
MDBX_NOTHROW_PURE_FUNCTION static inline uint16_t
unaligned_peek_u16(const size_t expected_alignment, const void *const ptr) {
MDBX_NOTHROW_PURE_FUNCTION static inline uint16_t unaligned_peek_u16(const size_t expected_alignment,
const void *const ptr) {
assert((uintptr_t)ptr % expected_alignment == 0);
if (MDBX_UNALIGNED_OK >= 2 || (expected_alignment % sizeof(uint16_t)) == 0)
return *(const uint16_t *)ptr;
else {
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
defined(_M_X64) || defined(_M_IA64)
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
return *(const __unaligned uint16_t *)ptr;
#else
uint16_t v;
@ -74,15 +66,12 @@ unaligned_peek_u16(const size_t expected_alignment, const void *const ptr) {
}
}
static inline void unaligned_poke_u16(const size_t expected_alignment,
void *const __restrict ptr,
const uint16_t v) {
static inline void unaligned_poke_u16(const size_t expected_alignment, void *const __restrict ptr, const uint16_t v) {
assert((uintptr_t)ptr % expected_alignment == 0);
if (MDBX_UNALIGNED_OK >= 2 || (expected_alignment % sizeof(v)) == 0)
*(uint16_t *)ptr = v;
else {
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
defined(_M_X64) || defined(_M_IA64)
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
*((uint16_t __unaligned *)ptr) = v;
#else
bcopy_2((uint8_t *)ptr, (const uint8_t *)&v);
@ -90,21 +79,17 @@ static inline void unaligned_poke_u16(const size_t expected_alignment,
}
}
MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t
unaligned_peek_u32(const size_t expected_alignment,
const void *const __restrict ptr) {
MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t unaligned_peek_u32(const size_t expected_alignment,
const void *const __restrict ptr) {
assert((uintptr_t)ptr % expected_alignment == 0);
if (MDBX_UNALIGNED_OK >= 4 || (expected_alignment % sizeof(uint32_t)) == 0)
return *(const uint32_t *)ptr;
else if ((expected_alignment % sizeof(uint16_t)) == 0) {
const uint16_t lo =
((const uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
const uint16_t hi =
((const uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
const uint16_t lo = ((const uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
const uint16_t hi = ((const uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
return lo | (uint32_t)hi << 16;
} else {
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
defined(_M_X64) || defined(_M_IA64)
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
return *(const __unaligned uint32_t *)ptr;
#else
uint32_t v;
@ -114,19 +99,15 @@ unaligned_peek_u32(const size_t expected_alignment,
}
}
static inline void unaligned_poke_u32(const size_t expected_alignment,
void *const __restrict ptr,
const uint32_t v) {
static inline void unaligned_poke_u32(const size_t expected_alignment, void *const __restrict ptr, const uint32_t v) {
assert((uintptr_t)ptr % expected_alignment == 0);
if (MDBX_UNALIGNED_OK >= 4 || (expected_alignment % sizeof(v)) == 0)
*(uint32_t *)ptr = v;
else if ((expected_alignment % sizeof(uint16_t)) == 0) {
((uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint16_t)v;
((uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] =
(uint16_t)(v >> 16);
((uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] = (uint16_t)(v >> 16);
} else {
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
defined(_M_X64) || defined(_M_IA64)
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
*((uint32_t __unaligned *)ptr) = v;
#else
bcopy_4((uint8_t *)ptr, (const uint8_t *)&v);
@ -134,21 +115,17 @@ static inline void unaligned_poke_u32(const size_t expected_alignment,
}
}
MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t
unaligned_peek_u64(const size_t expected_alignment,
const void *const __restrict ptr) {
MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t unaligned_peek_u64(const size_t expected_alignment,
const void *const __restrict ptr) {
assert((uintptr_t)ptr % expected_alignment == 0);
if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(uint64_t)) == 0)
return *(const uint64_t *)ptr;
else if ((expected_alignment % sizeof(uint32_t)) == 0) {
const uint32_t lo =
((const uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
const uint32_t hi =
((const uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
const uint32_t lo = ((const uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
const uint32_t hi = ((const uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
return lo | (uint64_t)hi << 32;
} else {
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
defined(_M_X64) || defined(_M_IA64)
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
return *(const __unaligned uint64_t *)ptr;
#else
uint64_t v;
@ -158,40 +135,32 @@ unaligned_peek_u64(const size_t expected_alignment,
}
}
static inline uint64_t
unaligned_peek_u64_volatile(const size_t expected_alignment,
const volatile void *const __restrict ptr) {
static inline uint64_t unaligned_peek_u64_volatile(const size_t expected_alignment,
const volatile void *const __restrict ptr) {
assert((uintptr_t)ptr % expected_alignment == 0);
assert(expected_alignment % sizeof(uint32_t) == 0);
if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(uint64_t)) == 0)
return *(const volatile uint64_t *)ptr;
else {
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
defined(_M_X64) || defined(_M_IA64)
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
return *(const volatile __unaligned uint64_t *)ptr;
#else
const uint32_t lo = ((const volatile uint32_t *)
ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
const uint32_t hi = ((const volatile uint32_t *)
ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
const uint32_t lo = ((const volatile uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__];
const uint32_t hi = ((const volatile uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__];
return lo | (uint64_t)hi << 32;
#endif /* _MSC_VER || __unaligned */
}
}
static inline void unaligned_poke_u64(const size_t expected_alignment,
void *const __restrict ptr,
const uint64_t v) {
static inline void unaligned_poke_u64(const size_t expected_alignment, void *const __restrict ptr, const uint64_t v) {
assert((uintptr_t)ptr % expected_alignment == 0);
if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(v)) == 0)
*(uint64_t *)ptr = v;
else if ((expected_alignment % sizeof(uint32_t)) == 0) {
((uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint32_t)v;
((uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] =
(uint32_t)(v >> 32);
((uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] = (uint32_t)(v >> 32);
} else {
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \
defined(_M_X64) || defined(_M_IA64)
#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
*((uint64_t __unaligned *)ptr) = v;
#else
bcopy_8((uint8_t *)ptr, (const uint8_t *)&v);
@ -199,28 +168,22 @@ static inline void unaligned_poke_u64(const size_t expected_alignment,
}
}
#define UNALIGNED_PEEK_8(ptr, struct, field) \
peek_u8(ptr_disp(ptr, offsetof(struct, field)))
#define UNALIGNED_POKE_8(ptr, struct, field, value) \
poke_u8(ptr_disp(ptr, offsetof(struct, field)), value)
#define UNALIGNED_PEEK_8(ptr, struct, field) peek_u8(ptr_disp(ptr, offsetof(struct, field)))
#define UNALIGNED_POKE_8(ptr, struct, field, value) poke_u8(ptr_disp(ptr, offsetof(struct, field)), value)
#define UNALIGNED_PEEK_16(ptr, struct, field) \
unaligned_peek_u16(1, ptr_disp(ptr, offsetof(struct, field)))
#define UNALIGNED_POKE_16(ptr, struct, field, value) \
#define UNALIGNED_PEEK_16(ptr, struct, field) unaligned_peek_u16(1, ptr_disp(ptr, offsetof(struct, field)))
#define UNALIGNED_POKE_16(ptr, struct, field, value) \
unaligned_poke_u16(1, ptr_disp(ptr, offsetof(struct, field)), value)
#define UNALIGNED_PEEK_32(ptr, struct, field) \
unaligned_peek_u32(1, ptr_disp(ptr, offsetof(struct, field)))
#define UNALIGNED_POKE_32(ptr, struct, field, value) \
#define UNALIGNED_PEEK_32(ptr, struct, field) unaligned_peek_u32(1, ptr_disp(ptr, offsetof(struct, field)))
#define UNALIGNED_POKE_32(ptr, struct, field, value) \
unaligned_poke_u32(1, ptr_disp(ptr, offsetof(struct, field)), value)
#define UNALIGNED_PEEK_64(ptr, struct, field) \
unaligned_peek_u64(1, ptr_disp(ptr, offsetof(struct, field)))
#define UNALIGNED_POKE_64(ptr, struct, field, value) \
#define UNALIGNED_PEEK_64(ptr, struct, field) unaligned_peek_u64(1, ptr_disp(ptr, offsetof(struct, field)))
#define UNALIGNED_POKE_64(ptr, struct, field, value) \
unaligned_poke_u64(1, ptr_disp(ptr, offsetof(struct, field)), value)
MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t
peek_pgno(const void *const __restrict ptr) {
MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t peek_pgno(const void *const __restrict ptr) {
if (sizeof(pgno_t) == sizeof(uint32_t))
return (pgno_t)unaligned_peek_u32(1, ptr);
else if (sizeof(pgno_t) == sizeof(uint64_t))

11
src/utils.c
View File

@ -3,10 +3,8 @@
#include "internals.h"
MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL unsigned
log2n_powerof2(size_t value_uintptr) {
assert(value_uintptr > 0 && value_uintptr < INT32_MAX &&
is_powerof2(value_uintptr));
MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL unsigned log2n_powerof2(size_t value_uintptr) {
assert(value_uintptr > 0 && value_uintptr < INT32_MAX && is_powerof2(value_uintptr));
assert((value_uintptr & -(intptr_t)value_uintptr) == value_uintptr);
const uint32_t value_uint32 = (uint32_t)value_uintptr;
#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctz)
@ -18,9 +16,8 @@ log2n_powerof2(size_t value_uintptr) {
_BitScanForward(&index, value_uint32);
return index;
#else
static const uint8_t debruijn_ctz32[32] = {
0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9};
static const uint8_t debruijn_ctz32[32] = {0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9};
return debruijn_ctz32[(uint32_t)(value_uint32 * 0x077CB531ul) >> 27];
#endif
}

41
src/utils.h
View File

@ -27,46 +27,36 @@
/* Pointer distance as signed number of bytes */
#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less)))
#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \
do { \
TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
(size_t)(size), __LINE__); \
ASAN_POISON_MEMORY_REGION(addr, size); \
#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \
do { \
TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \
ASAN_POISON_MEMORY_REGION(addr, size); \
} while (0)
#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \
do { \
TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \
(size_t)(size), __LINE__); \
ASAN_UNPOISON_MEMORY_REGION(addr, size); \
#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \
do { \
TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \
ASAN_UNPOISON_MEMORY_REGION(addr, size); \
} while (0)
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t
branchless_abs(intptr_t value) {
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t branchless_abs(intptr_t value) {
assert(value > INT_MIN);
const size_t expanded_sign =
(size_t)(value >> (sizeof(value) * CHAR_BIT - 1));
const size_t expanded_sign = (size_t)(value >> (sizeof(value) * CHAR_BIT - 1));
return ((size_t)value + expanded_sign) ^ expanded_sign;
}
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline bool
is_powerof2(size_t x) {
return (x & (x - 1)) == 0;
}
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline bool is_powerof2(size_t x) { return (x & (x - 1)) == 0; }
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t
floor_powerof2(size_t value, size_t granularity) {
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t floor_powerof2(size_t value, size_t granularity) {
assert(is_powerof2(granularity));
return value & ~(granularity - 1);
}
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t
ceil_powerof2(size_t value, size_t granularity) {
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t ceil_powerof2(size_t value, size_t granularity) {
return floor_powerof2(value + granularity - 1, granularity);
}
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL unsigned
log2n_powerof2(size_t value_uintptr);
MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL unsigned log2n_powerof2(size_t value_uintptr);
MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL uint64_t rrxmrrxmsx_0(uint64_t v);
@ -75,8 +65,7 @@ struct monotime_cache {
int expire_countdown;
};
MDBX_MAYBE_UNUSED static inline uint64_t
monotime_since_cached(uint64_t begin_timestamp, struct monotime_cache *cache) {
MDBX_MAYBE_UNUSED static inline uint64_t monotime_since_cached(uint64_t begin_timestamp, struct monotime_cache *cache) {
if (cache->expire_countdown)
cache->expire_countdown -= 1;
else {

12
src/version.c.in
View File

@ -3,8 +3,7 @@
#include "internals.h"
#if MDBX_VERSION_MAJOR != ${MDBX_VERSION_MAJOR} || \
MDBX_VERSION_MINOR != ${MDBX_VERSION_MINOR}
#if MDBX_VERSION_MAJOR != ${MDBX_VERSION_MAJOR} || MDBX_VERSION_MINOR != ${MDBX_VERSION_MINOR}
#error "API version mismatch! Had `git fetch --tags` done?"
#endif
@ -18,8 +17,7 @@ __dll_export
#endif
#ifdef __attribute_externally_visible__
__attribute_externally_visible__
#elif (defined(__GNUC__) && !defined(__clang__)) || \
__has_attribute(__externally_visible__)
#elif (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__externally_visible__)
__attribute__((__externally_visible__))
#endif
const struct MDBX_version_info mdbx_version = {
@ -29,8 +27,7 @@ __dll_export
${MDBX_VERSION_TWEAK},
"@MDBX_VERSION_PRERELEASE@", /* pre-release suffix of SemVer
@MDBX_VERSION_PURE@ */
{"@MDBX_GIT_TIMESTAMP@", "@MDBX_GIT_TREE@", "@MDBX_GIT_COMMIT@",
"@MDBX_GIT_DESCRIBE@"},
{"@MDBX_GIT_TIMESTAMP@", "@MDBX_GIT_TREE@", "@MDBX_GIT_COMMIT@", "@MDBX_GIT_DESCRIBE@"},
sourcery};
__dll_export
@ -41,8 +38,7 @@ __dll_export
#endif
#ifdef __attribute_externally_visible__
__attribute_externally_visible__
#elif (defined(__GNUC__) && !defined(__clang__)) || \
__has_attribute(__externally_visible__)
#elif (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__externally_visible__)
__attribute__((__externally_visible__))
#endif
const char *const mdbx_sourcery_anchor = sourcery;

76
src/walk.c
View File

@ -41,19 +41,16 @@ static page_type_t walk_subpage_type(const page_t *sp) {
}
/* Depth-first tree traversal. */
__cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno,
txnid_t parent_txnid) {
__cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, txnid_t parent_txnid) {
assert(pgno != P_INVALID);
page_t *mp = nullptr;
int err = page_get(ctx->cursor, pgno, &mp, parent_txnid);
const page_type_t type = walk_page_type(mp);
const size_t nentries = mp ? page_numkeys(mp) : 0;
size_t header_size =
(mp && !is_dupfix_leaf(mp)) ? PAGEHDRSZ + mp->lower : PAGEHDRSZ;
size_t header_size = (mp && !is_dupfix_leaf(mp)) ? PAGEHDRSZ + mp->lower : PAGEHDRSZ;
size_t payload_size = 0;
size_t unused_size =
(mp ? page_room(mp) : ctx->txn->env->ps - header_size) - payload_size;
size_t unused_size = (mp ? page_room(mp) : ctx->txn->env->ps - header_size) - payload_size;
size_t align_bytes = 0;
for (size_t i = 0; err == MDBX_SUCCESS && i < nentries; ++i) {
@ -89,12 +86,10 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno,
assert(err == MDBX_SUCCESS);
pgr_t lp = page_get_large(ctx->cursor, large_pgno, mp->txnid);
const size_t npages =
((err = lp.err) == MDBX_SUCCESS) ? lp.page->pages : 1;
const size_t npages = ((err = lp.err) == MDBX_SUCCESS) ? lp.page->pages : 1;
const size_t pagesize = pgno2bytes(ctx->txn->env, npages);
const size_t over_unused = pagesize - over_payload - over_header;
const int rc = ctx->visitor(large_pgno, npages, ctx->userctx, ctx->deep,
tbl, pagesize, page_large, err, 1,
const int rc = ctx->visitor(large_pgno, npages, ctx->userctx, ctx->deep, tbl, pagesize, page_large, err, 1,
over_payload, over_header, over_unused);
if (unlikely(rc != MDBX_SUCCESS))
return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc;
@ -104,8 +99,7 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno,
case N_TREE /* sub-db */: {
if (unlikely(node_data_size != sizeof(tree_t))) {
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid table node size", (unsigned)node_data_size);
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid table node size", (unsigned)node_data_size);
assert(err == MDBX_CORRUPTED);
err = MDBX_CORRUPTED;
}
@ -115,8 +109,7 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno,
case N_TREE | N_DUP /* dupsorted sub-tree */:
if (unlikely(node_data_size != sizeof(tree_t))) {
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid sub-tree node size", (unsigned)node_data_size);
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-tree node size", (unsigned)node_data_size);
assert(err == MDBX_CORRUPTED);
err = MDBX_CORRUPTED;
}
@ -126,8 +119,7 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno,
case N_DUP /* short sub-page */: {
if (unlikely(node_data_size <= PAGEHDRSZ || (node_data_size & 1))) {
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid sub-page node size", (unsigned)node_data_size);
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-page node size", (unsigned)node_data_size);
assert(err == MDBX_CORRUPTED);
err = MDBX_CORRUPTED;
break;
@ -137,14 +129,12 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno,
const page_type_t subtype = walk_subpage_type(sp);
const size_t nsubkeys = page_numkeys(sp);
if (unlikely(subtype == page_sub_broken)) {
ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid sub-page flags", sp->flags);
ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-page flags", sp->flags);
assert(err == MDBX_CORRUPTED);
err = MDBX_CORRUPTED;
}
size_t subheader_size =
is_dupfix_leaf(sp) ? PAGEHDRSZ : PAGEHDRSZ + sp->lower;
size_t subheader_size = is_dupfix_leaf(sp) ? PAGEHDRSZ : PAGEHDRSZ + sp->lower;
size_t subunused_size = page_room(sp);
size_t subpayload_size = 0;
size_t subalign_bytes = 0;
@ -161,18 +151,15 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno,
subpayload_size += subnode_size;
subalign_bytes += subnode_size & 1;
if (unlikely(node_flags(subnode) != 0)) {
ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"unexpected sub-node flags", node_flags(subnode));
ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, "unexpected sub-node flags", node_flags(subnode));
assert(err == MDBX_CORRUPTED);
err = MDBX_CORRUPTED;
}
}
}
const int rc =
ctx->visitor(pgno, 0, ctx->userctx, ctx->deep + 1, tbl,
node_data_size, subtype, err, nsubkeys, subpayload_size,
subheader_size, subunused_size + subalign_bytes);
const int rc = ctx->visitor(pgno, 0, ctx->userctx, ctx->deep + 1, tbl, node_data_size, subtype, err, nsubkeys,
subpayload_size, subheader_size, subunused_size + subalign_bytes);
if (unlikely(rc != MDBX_SUCCESS))
return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc;
header_size += subheader_size;
@ -182,16 +169,14 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno,
} break;
default:
ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid node flags", node_flags(node));
ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid node flags", node_flags(node));
assert(err == MDBX_CORRUPTED);
err = MDBX_CORRUPTED;
}
}
const int rc = ctx->visitor(
pgno, 1, ctx->userctx, ctx->deep, tbl, ctx->txn->env->ps, type, err,
nentries, payload_size, header_size, unused_size + align_bytes);
const int rc = ctx->visitor(pgno, 1, ctx->userctx, ctx->deep, tbl, ctx->txn->env->ps, type, err, nentries,
payload_size, header_size, unused_size + align_bytes);
if (unlikely(rc != MDBX_SUCCESS))
return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc;
@ -220,8 +205,7 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno,
case N_TREE /* sub-db */:
if (unlikely(node_ds(node) != sizeof(tree_t))) {
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid sub-tree node size", (unsigned)node_ds(node));
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-tree node size", (unsigned)node_ds(node));
assert(err == MDBX_CORRUPTED);
err = MDBX_CORRUPTED;
} else {
@ -238,8 +222,8 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno,
case N_TREE | N_DUP /* dupsorted sub-tree */:
if (unlikely(node_ds(node) != sizeof(tree_t))) {
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED,
"invalid dupsort sub-tree node size", (unsigned)node_ds(node));
ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid dupsort sub-tree node size",
(unsigned)node_ds(node));
assert(err == MDBX_CORRUPTED);
err = MDBX_CORRUPTED;
} else {
@ -248,8 +232,7 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno,
assert(err == MDBX_SUCCESS);
err = cursor_dupsort_setup(ctx->cursor, node, mp);
if (likely(err == MDBX_SUCCESS)) {
assert(ctx->cursor->subcur ==
&container_of(ctx->cursor, cursor_couple_t, outer)->inner);
assert(ctx->cursor->subcur == &container_of(ctx->cursor, cursor_couple_t, outer)->inner);
ctx->cursor = &ctx->cursor->subcur->cursor;
ctx->deep += 1;
tbl->nested = &aligned_db;
@ -257,8 +240,7 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno,
tbl->nested = nullptr;
ctx->deep -= 1;
subcur_t *inner_xcursor = container_of(ctx->cursor, subcur_t, cursor);
cursor_couple_t *couple =
container_of(inner_xcursor, cursor_couple_t, inner);
cursor_couple_t *couple = container_of(inner_xcursor, cursor_couple_t, inner);
ctx->cursor = &couple->outer;
}
}
@ -280,30 +262,24 @@ __cold static int walk_tbl(walk_ctx_t *ctx, walk_tbl_t *tbl) {
if (unlikely(rc != MDBX_SUCCESS))
return rc;
const uint8_t cursor_checking = (ctx->options & dont_check_keys_ordering)
? z_pagecheck | z_ignord
: z_pagecheck;
const uint8_t cursor_checking = (ctx->options & dont_check_keys_ordering) ? z_pagecheck | z_ignord : z_pagecheck;
couple.outer.checking |= cursor_checking;
couple.inner.cursor.checking |= cursor_checking;
couple.outer.next = ctx->cursor;
couple.outer.top_and_flags = z_disable_tree_search_fastpath;
ctx->cursor = &couple.outer;
rc = walk_pgno(ctx, tbl, db->root,
db->mod_txnid ? db->mod_txnid : ctx->txn->txnid);
rc = walk_pgno(ctx, tbl, db->root, db->mod_txnid ? db->mod_txnid : ctx->txn->txnid);
ctx->cursor = couple.outer.next;
return rc;
}
__cold int walk_pages(MDBX_txn *txn, walk_func *visitor, void *user,
walk_options_t options) {
__cold int walk_pages(MDBX_txn *txn, walk_func *visitor, void *user, walk_options_t options) {
int rc = check_txn(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
walk_ctx_t ctx = {
.txn = txn, .userctx = user, .visitor = visitor, .options = options};
walk_tbl_t tbl = {.name = {.iov_base = MDBX_CHK_GC},
.internal = &txn->dbs[FREE_DBI]};
walk_ctx_t ctx = {.txn = txn, .userctx = user, .visitor = visitor, .options = options};
walk_tbl_t tbl = {.name = {.iov_base = MDBX_CHK_GC}, .internal = &txn->dbs[FREE_DBI]};
rc = walk_tbl(&ctx, &tbl);
if (!MDBX_IS_ERROR(rc)) {
tbl.name.iov_base = MDBX_CHK_MAIN;

13
src/walk.h
View File

@ -10,14 +10,11 @@ typedef struct walk_tbl {
tree_t *internal, *nested;
} walk_tbl_t;
typedef int walk_func(const size_t pgno, const unsigned number, void *const ctx,
const int deep, const walk_tbl_t *table,
const size_t page_size, const page_type_t page_type,
const MDBX_error_t err, const size_t nentries,
const size_t payload_bytes, const size_t header_bytes,
const size_t unused_bytes);
typedef int walk_func(const size_t pgno, const unsigned number, void *const ctx, const int deep,
const walk_tbl_t *table, const size_t page_size, const page_type_t page_type,
const MDBX_error_t err, const size_t nentries, const size_t payload_bytes,
const size_t header_bytes, const size_t unused_bytes);
typedef enum walk_options { dont_check_keys_ordering = 1 } walk_options_t;
MDBX_INTERNAL int walk_pages(MDBX_txn *txn, walk_func *visitor, void *user,
walk_options_t options);
MDBX_INTERNAL int walk_pages(MDBX_txn *txn, walk_func *visitor, void *user, walk_options_t options);

27
src/windows-import.c
View File

@ -9,9 +9,7 @@
// Stub for slim read-write lock
// Portion Copyright (C) 1995-2002 Brad Wilson
static void WINAPI stub_srwlock_Init(osal_srwlock_t *srwl) {
srwl->readerCount = srwl->writerCount = 0;
}
static void WINAPI stub_srwlock_Init(osal_srwlock_t *srwl) { srwl->readerCount = srwl->writerCount = 0; }
static void WINAPI stub_srwlock_AcquireShared(osal_srwlock_t *srwl) {
while (true) {
@ -76,8 +74,7 @@ static void WINAPI stub_srwlock_ReleaseExclusive(osal_srwlock_t *srwl) {
static uint64_t WINAPI stub_GetTickCount64(void) {
LARGE_INTEGER Counter, Frequency;
return (QueryPerformanceFrequency(&Frequency) &&
QueryPerformanceCounter(&Counter))
return (QueryPerformanceFrequency(&Frequency) && QueryPerformanceCounter(&Counter))
? Counter.QuadPart * 1000ul / Frequency.QuadPart
: 0;
}
@ -91,8 +88,7 @@ struct libmdbx_imports imports;
#pragma GCC diagnostic ignored "-Wcast-function-type"
#endif /* GCC/MINGW */
#define MDBX_IMPORT(HANDLE, ENTRY) \
imports.ENTRY = (MDBX_##ENTRY)GetProcAddress(HANDLE, #ENTRY)
#define MDBX_IMPORT(HANDLE, ENTRY) imports.ENTRY = (MDBX_##ENTRY)GetProcAddress(HANDLE, #ENTRY)
void windows_import(void) {
const HINSTANCE hNtdll = GetModuleHandleA("ntdll.dll");
@ -121,20 +117,13 @@ void windows_import(void) {
}
const osal_srwlock_t_function srwlock_init =
(osal_srwlock_t_function)(hKernel32dll
? GetProcAddress(hKernel32dll,
"InitializeSRWLock")
: nullptr);
(osal_srwlock_t_function)(hKernel32dll ? GetProcAddress(hKernel32dll, "InitializeSRWLock") : nullptr);
if (srwlock_init) {
imports.srwl_Init = srwlock_init;
imports.srwl_AcquireShared = (osal_srwlock_t_function)GetProcAddress(
hKernel32dll, "AcquireSRWLockShared");
imports.srwl_ReleaseShared = (osal_srwlock_t_function)GetProcAddress(
hKernel32dll, "ReleaseSRWLockShared");
imports.srwl_AcquireExclusive = (osal_srwlock_t_function)GetProcAddress(
hKernel32dll, "AcquireSRWLockExclusive");
imports.srwl_ReleaseExclusive = (osal_srwlock_t_function)GetProcAddress(
hKernel32dll, "ReleaseSRWLockExclusive");
imports.srwl_AcquireShared = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "AcquireSRWLockShared");
imports.srwl_ReleaseShared = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "ReleaseSRWLockShared");
imports.srwl_AcquireExclusive = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "AcquireSRWLockExclusive");
imports.srwl_ReleaseExclusive = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "ReleaseSRWLockExclusive");
} else {
imports.srwl_Init = stub_srwlock_Init;
imports.srwl_AcquireShared = stub_srwlock_AcquireShared;

55
src/windows-import.h
View File

@ -59,32 +59,27 @@ typedef struct _FILE_REMOTE_PROTOCOL_INFO {
#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */
typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(
_In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
_Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(_In_ HANDLE hFile,
_In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
_Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)(
_In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer,
_In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber,
_Out_opt_ LPDWORD lpMaximumComponentLength,
_Out_opt_ LPDWORD lpFileSystemFlags,
_Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);
_In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, _In_ DWORD nVolumeNameSize,
_Out_opt_ LPDWORD lpVolumeSerialNumber, _Out_opt_ LPDWORD lpMaximumComponentLength,
_Out_opt_ LPDWORD lpFileSystemFlags, _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);
typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile,
_Out_ LPWSTR lpszFilePath,
_In_ DWORD cchFilePath,
_In_ DWORD dwFlags);
typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, _Out_ LPWSTR lpszFilePath,
_In_ DWORD cchFilePath, _In_ DWORD dwFlags);
typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(
_In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
_Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(_In_ HANDLE hFile,
_In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
_Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(
IN HANDLE FileHandle, IN OUT HANDLE Event,
IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);
typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(IN HANDLE FileHandle, IN OUT HANDLE Event,
IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);
typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void);
@ -95,27 +90,21 @@ typedef struct _WIN32_MEMORY_RANGE_ENTRY {
} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY;
#endif /* Windows 8.x */
typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(
HANDLE hProcess, ULONG_PTR NumberOfEntries,
PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);
typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(HANDLE hProcess, ULONG_PTR NumberOfEntries,
PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);
typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT;
typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle,
IN PLARGE_INTEGER NewSectionSize);
typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, IN PLARGE_INTEGER NewSectionSize);
typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey,
LPCSTR lpValue, DWORD dwFlags,
LPDWORD pdwType, PVOID pvData,
LPDWORD pcbData);
typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, LPCSTR lpValue, DWORD dwFlags, LPDWORD pdwType,
PVOID pvData, LPDWORD pcbData);
typedef long(WINAPI *MDBX_CoCreateGuid)(bin128_t *guid);
NTSYSAPI ULONG RtlRandomEx(PULONG Seed);
typedef BOOL(WINAPI *MDBX_SetFileIoOverlappedRange)(HANDLE FileHandle,
PUCHAR OverlappedRangeStart,
ULONG Length);
typedef BOOL(WINAPI *MDBX_SetFileIoOverlappedRange)(HANDLE FileHandle, PUCHAR OverlappedRangeStart, ULONG Length);
struct libmdbx_imports {
osal_srwlock_t_function srwl_Init;

47
test/append.c++
View File

@ -5,16 +5,14 @@
class testcase_append : public testcase {
public:
testcase_append(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
testcase_append(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {}
bool run() override;
static bool review_params(actor_params &params, unsigned space_id) {
if (!testcase::review_params(params, space_id))
return false;
const bool ordered = !flipcoin_x3();
log_notice("the '%s' key-generation mode is selected",
ordered ? "ordered/linear" : "unordered/non-linear");
log_notice("the '%s' key-generation mode is selected", ordered ? "ordered/linear" : "unordered/non-linear");
if (ordered && !params.make_keygen_linear())
return false;
return true;
@ -37,13 +35,10 @@ bool testcase_append::run() {
keyvalue_maker.setup(config.params, 0 /* thread_number */);
/* LY: тест наполнения таблиц в append-режиме,
* при котором записи добавляются строго в конец (в порядке сортировки) */
const MDBX_put_flags_t flags =
reverse
? ((config.params.table_flags & MDBX_DUPSORT) ? MDBX_UPSERT
: MDBX_NOOVERWRITE)
: ((config.params.table_flags & MDBX_DUPSORT)
? (flipcoin() ? MDBX_APPEND | MDBX_APPENDDUP : MDBX_APPENDDUP)
: MDBX_APPEND);
const MDBX_put_flags_t flags = reverse ? ((config.params.table_flags & MDBX_DUPSORT) ? MDBX_UPSERT : MDBX_NOOVERWRITE)
: ((config.params.table_flags & MDBX_DUPSORT)
? (flipcoin() ? MDBX_APPEND | MDBX_APPENDDUP : MDBX_APPENDDUP)
: MDBX_APPEND);
key = keygen::alloc(config.params.keylen_max);
data = keygen::alloc(config.params.datalen_max);
@ -59,11 +54,9 @@ bool testcase_append::run() {
simple_checksum committed_inserted_checksum = inserted_checksum;
while (should_continue()) {
const keygen::serial_t serial = serial_count;
const bool turn_key = (config.params.table_flags & MDBX_DUPSORT) == 0 ||
flipcoin_n(config.params.keygen.split);
if (turn_key
? !keyvalue_maker.increment_key_part(serial_count, reverse ? -1 : 1)
: !keyvalue_maker.increment(serial_count, reverse ? -1 : 1)) {
const bool turn_key = (config.params.table_flags & MDBX_DUPSORT) == 0 || flipcoin_n(config.params.keygen.split);
if (turn_key ? !keyvalue_maker.increment_key_part(serial_count, reverse ? -1 : 1)
: !keyvalue_maker.increment(serial_count, reverse ? -1 : 1)) {
// дошли до границы пространства ключей
break;
}
@ -106,8 +99,7 @@ bool testcase_append::run() {
break;
case MDBX_APPENDDUP:
assert((config.params.table_flags & MDBX_DUPSORT) != 0);
expect_key_mismatch =
mdbx_cmp(txn_guard.get(), dbi, &key->value, &ge_key) == 0;
expect_key_mismatch = mdbx_cmp(txn_guard.get(), dbi, &key->value, &ge_key) == 0;
break;
}
} else if (err == MDBX_NOTFOUND /* all pair are less than */) {
@ -152,10 +144,9 @@ bool testcase_append::run() {
const auto insertion_result = speculum.insert(item);
if (!insertion_result.second) {
char dump_key[32], dump_value[32];
log_error(
"speculum.append: unexpected %s {%s, %s}", "MDBX_SUCCESS",
mdbx_dump_val(&key->value, dump_key, sizeof(dump_key)),
mdbx_dump_val(&data->value, dump_value, sizeof(dump_value)));
log_error("speculum.append: unexpected %s {%s, %s}", "MDBX_SUCCESS",
mdbx_dump_val(&key->value, dump_key, sizeof(dump_key)),
mdbx_dump_val(&data->value, dump_value, sizeof(dump_value)));
return false;
}
}
@ -199,8 +190,7 @@ bool testcase_append::run() {
cursor_renew();
MDBX_val check_key, check_data;
err = mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data,
reverse ? MDBX_LAST : MDBX_FIRST);
err = mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, reverse ? MDBX_LAST : MDBX_FIRST);
if (likely(inserted_number)) {
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_cursor_get(MDBX_FIRST)", err);
@ -213,19 +203,16 @@ bool testcase_append::run() {
read_checksum.push((uint32_t)read_count, check_key);
read_checksum.push(10639, check_data);
err = mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data,
reverse ? MDBX_PREV : MDBX_NEXT);
err = mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, reverse ? MDBX_PREV : MDBX_NEXT);
}
if (unlikely(err != MDBX_NOTFOUND))
failure_perror("mdbx_cursor_get(MDBX_NEXT) != EOF", err);
if (unlikely(read_count != inserted_number))
failure("read_count(%" PRIu64 ") != inserted_number(%" PRIu64 ")",
read_count, inserted_number);
failure("read_count(%" PRIu64 ") != inserted_number(%" PRIu64 ")", read_count, inserted_number);
if (unlikely(read_checksum.value != inserted_checksum.value) &&
!keyvalue_maker.is_unordered())
if (unlikely(read_checksum.value != inserted_checksum.value) && !keyvalue_maker.is_unordered())
failure("read_checksum(0x%016" PRIu64 ") "
"!= inserted_checksum(0x%016" PRIu64 ")",
read_checksum.value, inserted_checksum.value);

22
test/base.h++
View File

@ -7,12 +7,12 @@
#ifdef _MSC_VER
#pragma warning(push, 1)
#pragma warning(disable : 4548) /* expression before comma has no effect; \
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
semantics are not enabled. Specify /EHsc */
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
mode specified; termination on exception \
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
mode specified; termination on exception \
is not guaranteed. Specify /EHsc */
#endif /* _MSC_VER (warnings) */
@ -71,24 +71,22 @@
#ifdef _MSC_VER
#pragma warning(pop)
#pragma warning(disable : 4201) /* nonstandard extension used: nameless \
#pragma warning(disable : 4201) /* nonstandard extension used: nameless \
struct/union */
#pragma warning(disable : 4127) /* conditional expression is constant */
#if _MSC_VER < 1900
#pragma warning(disable : 4510) /* default constructor could \
#pragma warning(disable : 4510) /* default constructor could \
not be generated */
#pragma warning(disable : 4512) /* assignment operator could \
#pragma warning(disable : 4512) /* assignment operator could \
not be generated */
#pragma warning(disable : 4610) /* user-defined constructor required */
#ifndef snprintf
#define snprintf(buffer, buffer_size, format, ...) \
_snprintf_s(buffer, buffer_size, _TRUNCATE, format, __VA_ARGS__)
#define snprintf(buffer, buffer_size, format, ...) _snprintf_s(buffer, buffer_size, _TRUNCATE, format, __VA_ARGS__)
#endif
#ifndef vsnprintf
#define vsnprintf(buffer, buffer_size, format, args) \
_vsnprintf_s(buffer, buffer_size, _TRUNCATE, format, args)
#define vsnprintf(buffer, buffer_size, format, args) _vsnprintf_s(buffer, buffer_size, _TRUNCATE, format, args)
#endif
#pragma warning(disable : 4996) /* 'vsnprintf': This function or variable \
#pragma warning(disable : 4996) /* 'vsnprintf': This function or variable \
may be unsafe */
#endif
#endif /* _MSC_VER */

30
test/cases.c++
View File

@ -14,9 +14,7 @@ bool registry::add(const record *item) {
auto const singleton = instance();
assert(singleton->name2id.count(std::string(item->name)) == 0);
assert(singleton->id2record.count(item->id) == 0);
if (singleton->name2id.count(std::string(item->name)) +
singleton->id2record.count(item->id) ==
0) {
if (singleton->name2id.count(std::string(item->name)) + singleton->id2record.count(item->id) == 0) {
singleton->name2id[std::string(item->name)] = item;
singleton->id2record[item->id] = item;
return true;
@ -24,28 +22,24 @@ bool registry::add(const record *item) {
return false;
}
testcase *registry::create_actor(const actor_config &config,
const mdbx_pid_t pid) {
testcase *registry::create_actor(const actor_config &config, const mdbx_pid_t pid) {
return instance()->id2record.at(config.testcase)->constructor(config, pid);
}
bool registry::review_actor_params(const actor_testcase id,
actor_params &params,
const unsigned space_id) {
bool registry::review_actor_params(const actor_testcase id, actor_params &params, const unsigned space_id) {
return instance()->id2record.at(id)->review_params(params, space_id);
}
//-----------------------------------------------------------------------------
void configure_actor(unsigned &last_space_id, const actor_testcase testcase,
const char *space_id_cstr, actor_params params) {
void configure_actor(unsigned &last_space_id, const actor_testcase testcase, const char *space_id_cstr,
actor_params params) {
unsigned wait4id = 0;
if (params.waitfor_nops) {
for (auto i = global::actors.rbegin(); i != global::actors.rend(); ++i) {
if (i->is_waitable(params.waitfor_nops)) {
if (i->signal_nops && i->signal_nops != params.waitfor_nops)
failure("Previous waitable actor (id=%u) already linked on %u-ops\n",
i->actor_id, i->signal_nops);
failure("Previous waitable actor (id=%u) already linked on %u-ops\n", i->actor_id, i->signal_nops);
wait4id = i->actor_id;
i->signal_nops = params.waitfor_nops;
break;
@ -75,15 +69,12 @@ void configure_actor(unsigned &last_space_id, const actor_testcase testcase,
failure("Actor config-review failed for space-id %lu\n", space_id);
last_space_id = unsigned(space_id);
log_trace("configure_actor: space %lu for %s", space_id,
testcase2str(testcase));
global::actors.emplace_back(
actor_config(testcase, params, unsigned(space_id), wait4id));
log_trace("configure_actor: space %lu for %s", space_id, testcase2str(testcase));
global::actors.emplace_back(actor_config(testcase, params, unsigned(space_id), wait4id));
global::databases.insert(params.pathname_db);
}
void testcase_setup(const char *casename, const actor_params &params,
unsigned &last_space_id) {
void testcase_setup(const char *casename, const actor_params &params, unsigned &last_space_id) {
if (strcmp(casename, "basic") == 0) {
log_notice(">>> testcase_setup(%s)", casename);
configure_actor(last_space_id, ac_nested, nullptr, params);
@ -111,8 +102,7 @@ void keycase_setup(const char *casename, actor_params &params) {
params.keygen.keycase = kc_random;
// TODO
log_notice("<<< keycase_setup(%s): done", casename);
} else if (strcmp(casename, "dashes") == 0 ||
strcmp(casename, "aside") == 0) {
} else if (strcmp(casename, "dashes") == 0 || strcmp(casename, "aside") == 0) {
log_notice(">>> keycase_setup(%s)", casename);
params.keygen.keycase = kc_dashes;
// TODO

32
test/chrono.c++
View File

@ -18,9 +18,7 @@ uint32_t ns2fractional(uint32_t ns) {
return uint32_t((uint64_t(ns) << 32) / NSEC_PER_SEC);
}
uint32_t fractional2ns(uint32_t fractional) {
return uint32_t((fractional * uint64_t(NSEC_PER_SEC)) >> 32);
}
uint32_t fractional2ns(uint32_t fractional) { return uint32_t((fractional * uint64_t(NSEC_PER_SEC)) >> 32); }
#ifndef USEC_PER_SEC
#define USEC_PER_SEC 1000000u
@ -51,33 +49,27 @@ uint32_t ms2fractional(uint32_t ms) {
return uint32_t((uint64_t(ms) << 32) / MSEC_PER_SEC);
}
uint32_t fractional2ms(uint32_t fractional) {
return uint32_t((fractional * uint64_t(MSEC_PER_SEC)) >> 32);
}
uint32_t fractional2ms(uint32_t fractional) { return uint32_t((fractional * uint64_t(MSEC_PER_SEC)) >> 32); }
time from_ns(uint64_t ns) {
time result;
result.fixedpoint =
((ns / NSEC_PER_SEC) << 32) | ns2fractional(uint32_t(ns % NSEC_PER_SEC));
result.fixedpoint = ((ns / NSEC_PER_SEC) << 32) | ns2fractional(uint32_t(ns % NSEC_PER_SEC));
return result;
}
time from_us(uint64_t us) {
time result;
result.fixedpoint =
((us / USEC_PER_SEC) << 32) | us2fractional(uint32_t(us % USEC_PER_SEC));
result.fixedpoint = ((us / USEC_PER_SEC) << 32) | us2fractional(uint32_t(us % USEC_PER_SEC));
return result;
}
time from_ms(uint64_t ms) {
time result;
result.fixedpoint =
((ms / MSEC_PER_SEC) << 32) | ms2fractional(uint32_t(ms % MSEC_PER_SEC));
result.fixedpoint = ((ms / MSEC_PER_SEC) << 32) | ms2fractional(uint32_t(ms % MSEC_PER_SEC));
return result;
}
#if __GNUC_PREREQ(8, 0) && \
(defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
#if __GNUC_PREREQ(8, 0) && (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wcast-function-type"
#endif /* GCC/MINGW */
@ -88,16 +80,14 @@ time now_realtime() {
if (unlikely(!query_time)) {
HMODULE hModule = GetModuleHandle(TEXT("kernel32.dll"));
if (hModule)
query_time = (void(WINAPI *)(LPFILETIME))GetProcAddress(
hModule, "GetSystemTimePreciseAsFileTime");
query_time = (void(WINAPI *)(LPFILETIME))GetProcAddress(hModule, "GetSystemTimePreciseAsFileTime");
if (!query_time)
query_time = GetSystemTimeAsFileTime;
}
FILETIME filetime;
query_time(&filetime);
uint64_t ns100 =
(uint64_t)filetime.dwHighDateTime << 32 | filetime.dwLowDateTime;
uint64_t ns100 = (uint64_t)filetime.dwHighDateTime << 32 | filetime.dwLowDateTime;
return from_ns((ns100 - UINT64_C(116444736000000000)) * 100u);
#else
struct timespec ts;
@ -115,8 +105,7 @@ time now_monotonic() {
if (reciprocal == 0) {
if (!QueryPerformanceFrequency(&Frequency))
failure_perror("QueryPerformanceFrequency()", GetLastError());
reciprocal = (((UINT64_C(1) << 48) + Frequency.QuadPart / 2 + 1) /
Frequency.QuadPart);
reciprocal = (((UINT64_C(1) << 48) + Frequency.QuadPart / 2 + 1) / Frequency.QuadPart);
assert(reciprocal);
}
@ -138,8 +127,7 @@ time now_monotonic() {
#endif
}
#if __GNUC_PREREQ(8, 0) && \
(defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
#if __GNUC_PREREQ(8, 0) && (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__))
#pragma GCC diagnostic pop
#endif /* GCC/MINGW */

9
test/chrono.h++
View File

@ -63,12 +63,10 @@ inline time infinite() {
return result;
}
#if defined(HAVE_TIMESPEC_TV_NSEC) || defined(__timespec_defined) || \
defined(CLOCK_REALTIME)
#if defined(HAVE_TIMESPEC_TV_NSEC) || defined(__timespec_defined) || defined(CLOCK_REALTIME)
inline time from_timespec(const struct timespec &ts) {
time result;
result.fixedpoint =
((uint64_t)ts.tv_sec << 32) | ns2fractional((uint32_t)ts.tv_nsec);
result.fixedpoint = ((uint64_t)ts.tv_sec << 32) | ns2fractional((uint32_t)ts.tv_nsec);
return result;
}
#endif /* HAVE_TIMESPEC_TV_NSEC */
@ -76,8 +74,7 @@ inline time from_timespec(const struct timespec &ts) {
#if defined(HAVE_TIMEVAL_TV_USEC) || defined(_STRUCT_TIMEVAL)
inline time from_timeval(const struct timeval &tv) {
time result;
result.fixedpoint =
((uint64_t)tv.tv_sec << 32) | us2fractional((uint32_t)tv.tv_usec);
result.fixedpoint = ((uint64_t)tv.tv_sec << 32) | us2fractional((uint32_t)tv.tv_usec);
return result;
}
#endif /* HAVE_TIMEVAL_TV_USEC */

335
test/config.c++
View File

@ -9,8 +9,8 @@
namespace config {
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
const char **value, const char *default_value) {
bool parse_option(int argc, char *const argv[], int &narg, const char *option, const char **value,
const char *default_value) {
assert(narg < argc);
const char *current = argv[narg];
const size_t optlen = strlen(option);
@ -49,14 +49,11 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
failure("No value given for '--%s' option\n", option);
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty) {
return parse_option(argc, argv, narg, option, value, allow_empty,
allow_empty ? "" : nullptr);
bool parse_option(int argc, char *const argv[], int &narg, const char *option, std::string &value, bool allow_empty) {
return parse_option(argc, argv, narg, option, value, allow_empty, allow_empty ? "" : nullptr);
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty,
bool parse_option(int argc, char *const argv[], int &narg, const char *option, std::string &value, bool allow_empty,
const char *default_value) {
const char *value_cstr;
if (!parse_option(argc, argv, narg, option, &value_cstr, default_value))
@ -70,8 +67,7 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
}
template <>
bool parse_option<unsigned>(int argc, char *const argv[], int &narg,
const char *option, unsigned &mask,
bool parse_option<unsigned>(int argc, char *const argv[], int &narg, const char *option, unsigned &mask,
const option_verb *verbs) {
const char *list;
if (!parse_option(argc, argv, narg, option, &list))
@ -95,8 +91,7 @@ bool parse_option<unsigned>(int argc, char *const argv[], int &narg,
while (true) {
if (!scan->verb)
failure("Unknown verb '%.*s', for option '--%s'\n", (int)len, list,
option);
failure("Unknown verb '%.*s', for option '--%s'\n", (int)len, list, option);
if (strlen(scan->verb) == len && strncmp(list, scan->verb, len) == 0) {
mask = strikethrough ? mask & ~scan->mask : mask | scan->mask;
clear = strikethrough ? clear & ~scan->mask : clear | scan->mask;
@ -110,10 +105,8 @@ bool parse_option<unsigned>(int argc, char *const argv[], int &narg,
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint64_t &value, const scale_mode scale,
const uint64_t minval, const uint64_t maxval,
const uint64_t default_value) {
bool parse_option(int argc, char *const argv[], int &narg, const char *option, uint64_t &value, const scale_mode scale,
const uint64_t minval, const uint64_t maxval, const uint64_t default_value) {
const char *value_cstr;
if (!parse_option(argc, argv, narg, option, &value_cstr))
@ -134,17 +127,13 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
return true;
}
if (strcmp(value_cstr, "rnd") == 0 || strcmp(value_cstr, "rand") == 0 ||
strcmp(value_cstr, "random") == 0) {
if (strcmp(value_cstr, "rnd") == 0 || strcmp(value_cstr, "rand") == 0 || strcmp(value_cstr, "random") == 0) {
value = minval;
if (maxval > minval) {
uint64_t salt = (scale != entropy)
? prng64() ^ UINT64_C(44263400549519813)
: (chrono::now_monotonic().fixedpoint ^
UINT64_C(0xD85794512ED321FD)) *
UINT64_C(0x9120038359EAF3) ^
chrono::now_realtime().fixedpoint *
UINT64_C(0x2FE5232BDC8E5F);
uint64_t salt = (scale != entropy) ? prng64() ^ UINT64_C(44263400549519813)
: (chrono::now_monotonic().fixedpoint ^ UINT64_C(0xD85794512ED321FD)) *
UINT64_C(0x9120038359EAF3) ^
chrono::now_realtime().fixedpoint * UINT64_C(0x2FE5232BDC8E5F);
value += salt % (maxval - minval);
}
if (scale == intkey)
@ -161,43 +150,32 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
raw = strtoull(value_cstr, &suffix, 10);
}
if (errno)
failure("Option '--%s' expects a numeric value (%s)\n", option,
test_strerror(errno));
failure("Option '--%s' expects a numeric value (%s)\n", option, test_strerror(errno));
uint64_t multiplier = 1;
if (suffix && *suffix) {
if (scale == no_scale || scale == intkey)
failure("Option '--%s' doesn't accepts suffixes, so '%s' is unexpected\n",
option, suffix);
failure("Option '--%s' doesn't accepts suffixes, so '%s' is unexpected\n", option, suffix);
if (strcmp(suffix, "K") == 0 || strcasecmp(suffix, "Kilo") == 0)
multiplier = (scale == decimal) ? UINT64_C(1000) : UINT64_C(1024);
else if (strcmp(suffix, "M") == 0 || strcasecmp(suffix, "Mega") == 0)
multiplier =
(scale == decimal) ? UINT64_C(1000) * 1000 : UINT64_C(1024) * 1024;
multiplier = (scale == decimal) ? UINT64_C(1000) * 1000 : UINT64_C(1024) * 1024;
else if (strcmp(suffix, "G") == 0 || strcasecmp(suffix, "Giga") == 0)
multiplier = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000
: UINT64_C(1024) * 1024 * 1024;
multiplier = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 : UINT64_C(1024) * 1024 * 1024;
else if (strcmp(suffix, "T") == 0 || strcasecmp(suffix, "Tera") == 0)
multiplier = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 * 1000
: UINT64_C(1024) * 1024 * 1024 * 1024;
else if (scale == duration &&
(strcmp(suffix, "s") == 0 || strcasecmp(suffix, "Seconds") == 0))
multiplier = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 * 1000 : UINT64_C(1024) * 1024 * 1024 * 1024;
else if (scale == duration && (strcmp(suffix, "s") == 0 || strcasecmp(suffix, "Seconds") == 0))
multiplier = 1;
else if (scale == duration &&
(strcmp(suffix, "m") == 0 || strcasecmp(suffix, "Minutes") == 0))
else if (scale == duration && (strcmp(suffix, "m") == 0 || strcasecmp(suffix, "Minutes") == 0))
multiplier = 60;
else if (scale == duration &&
(strcmp(suffix, "h") == 0 || strcasecmp(suffix, "Hours") == 0))
else if (scale == duration && (strcmp(suffix, "h") == 0 || strcasecmp(suffix, "Hours") == 0))
multiplier = 3600;
else if (scale == duration &&
(strcmp(suffix, "d") == 0 || strcasecmp(suffix, "Days") == 0))
else if (scale == duration && (strcmp(suffix, "d") == 0 || strcasecmp(suffix, "Days") == 0))
multiplier = 3600 * 24;
else
failure(
"Option '--%s' expects a numeric value with Kilo/Mega/Giga/Tera %s"
"suffixes, but '%s' is unexpected\n",
option, (scale == duration) ? "or Seconds/Minutes/Hours/Days " : "",
suffix);
failure("Option '--%s' expects a numeric value with Kilo/Mega/Giga/Tera %s"
"suffixes, but '%s' is unexpected\n",
option, (scale == duration) ? "or Seconds/Minutes/Hours/Days " : "", suffix);
}
if (raw >= UINT64_MAX / multiplier)
@ -205,47 +183,38 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
value = raw * multiplier;
if (maxval && value > maxval)
failure("The maximal value for option '--%s' is %" PRIu64 "\n", option,
maxval);
failure("The maximal value for option '--%s' is %" PRIu64 "\n", option, maxval);
if (value < minval)
failure("The minimal value for option '--%s' is %" PRIu64 "\n", option,
minval);
failure("The minimal value for option '--%s' is %" PRIu64 "\n", option, minval);
if (scale == intkey)
value &= ~3u;
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &value, const scale_mode scale,
const unsigned minval, const unsigned maxval,
const unsigned default_value) {
bool parse_option(int argc, char *const argv[], int &narg, const char *option, unsigned &value, const scale_mode scale,
const unsigned minval, const unsigned maxval, const unsigned default_value) {
uint64_t huge;
if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval,
default_value))
if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval, default_value))
return false;
value = unsigned(huge);
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint8_t &value, const uint8_t minval, const uint8_t maxval,
const uint8_t default_value) {
bool parse_option(int argc, char *const argv[], int &narg, const char *option, uint8_t &value, const uint8_t minval,
const uint8_t maxval, const uint8_t default_value) {
uint64_t huge;
if (!parse_option(argc, argv, narg, option, huge, no_scale, minval, maxval,
default_value))
if (!parse_option(argc, argv, narg, option, huge, no_scale, minval, maxval, default_value))
return false;
value = uint8_t(huge);
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int64_t &value, const int64_t minval, const int64_t maxval,
const int64_t default_value) {
bool parse_option(int argc, char *const argv[], int &narg, const char *option, int64_t &value, const int64_t minval,
const int64_t maxval, const int64_t default_value) {
uint64_t proxy = uint64_t(value);
if (parse_option(argc, argv, narg, option, proxy, config::binary,
uint64_t(minval), uint64_t(maxval),
if (parse_option(argc, argv, narg, option, proxy, config::binary, uint64_t(minval), uint64_t(maxval),
uint64_t(default_value))) {
value = int64_t(proxy);
return true;
@ -253,12 +222,10 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
return false;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int32_t &value, const int32_t minval, const int32_t maxval,
const int32_t default_value) {
bool parse_option(int argc, char *const argv[], int &narg, const char *option, int32_t &value, const int32_t minval,
const int32_t maxval, const int32_t default_value) {
uint64_t proxy = uint64_t(value);
if (parse_option(argc, argv, narg, option, proxy, config::binary,
uint64_t(minval), uint64_t(maxval),
if (parse_option(argc, argv, narg, option, proxy, config::binary, uint64_t(minval), uint64_t(maxval),
uint64_t(default_value))) {
value = int32_t(proxy);
return true;
@ -266,14 +233,12 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
return false;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
logging::loglevel &loglevel) {
bool parse_option(int argc, char *const argv[], int &narg, const char *option, logging::loglevel &loglevel) {
const char *value_cstr;
if (!parse_option(argc, argv, narg, option, &value_cstr))
return false;
if (strcmp(value_cstr, "min") == 0 || strcmp(value_cstr, "minimal") == 0 ||
strcmp(value_cstr, "fatal") == 0) {
if (strcmp(value_cstr, "min") == 0 || strcmp(value_cstr, "minimal") == 0 || strcmp(value_cstr, "fatal") == 0) {
loglevel = logging::failure;
return true;
}
@ -308,8 +273,7 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
return true;
}
if (strcmp(value_cstr, "max") == 0 || strcmp(value_cstr, "maximal") == 0 ||
strcmp(value_cstr, "extra") == 0) {
if (strcmp(value_cstr, "max") == 0 || strcmp(value_cstr, "maximal") == 0 || strcmp(value_cstr, "extra") == 0) {
loglevel = logging::extra;
return true;
}
@ -329,8 +293,7 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
failure("Unknown log-level '%s', for option '--%s'\n", value_cstr, option);
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
bool &value) {
bool parse_option(int argc, char *const argv[], int &narg, const char *option, bool &value) {
const char *value_cstr = nullptr;
if (!parse_option(argc, argv, narg, option, &value_cstr, "yes")) {
const char *current = argv[narg];
@ -338,8 +301,7 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
value = false;
return true;
}
if (strncmp(current, "--dont-", 7) == 0 &&
strcmp(current + 7, option) == 0) {
if (strncmp(current, "--dont-", 7) == 0 && strcmp(current + 7, option) == 0) {
value = false;
return true;
}
@ -361,41 +323,36 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
return true;
}
failure(
"Option '--%s' expects a 'boolean' value Yes/No, so '%s' is unexpected\n",
option, value_cstr);
failure("Option '--%s' expects a 'boolean' value Yes/No, so '%s' is unexpected\n", option, value_cstr);
}
//-----------------------------------------------------------------------------
const struct option_verb mode_bits[] = {
{"rdonly", unsigned(MDBX_RDONLY)},
{"nosync-utterly", unsigned(MDBX_UTTERLY_NOSYNC)},
{"nosubdir", unsigned(MDBX_NOSUBDIR)},
{"nosync-safe", unsigned(MDBX_SAFE_NOSYNC)},
{"nometasync", unsigned(MDBX_NOMETASYNC)},
{"writemap", unsigned(MDBX_WRITEMAP)},
{"nostickythreads", unsigned(MDBX_NOSTICKYTHREADS)},
{"no-sticky-threads", unsigned(MDBX_NOSTICKYTHREADS)},
{"nordahead", unsigned(MDBX_NORDAHEAD)},
{"nomeminit", unsigned(MDBX_NOMEMINIT)},
{"lifo", unsigned(MDBX_LIFORECLAIM)},
{"perturb", unsigned(MDBX_PAGEPERTURB)},
{"accede", unsigned(MDBX_ACCEDE)},
{"exclusive", unsigned(MDBX_EXCLUSIVE)},
{nullptr, 0}};
const struct option_verb mode_bits[] = {{"rdonly", unsigned(MDBX_RDONLY)},
{"nosync-utterly", unsigned(MDBX_UTTERLY_NOSYNC)},
{"nosubdir", unsigned(MDBX_NOSUBDIR)},
{"nosync-safe", unsigned(MDBX_SAFE_NOSYNC)},
{"nometasync", unsigned(MDBX_NOMETASYNC)},
{"writemap", unsigned(MDBX_WRITEMAP)},
{"nostickythreads", unsigned(MDBX_NOSTICKYTHREADS)},
{"no-sticky-threads", unsigned(MDBX_NOSTICKYTHREADS)},
{"nordahead", unsigned(MDBX_NORDAHEAD)},
{"nomeminit", unsigned(MDBX_NOMEMINIT)},
{"lifo", unsigned(MDBX_LIFORECLAIM)},
{"perturb", unsigned(MDBX_PAGEPERTURB)},
{"accede", unsigned(MDBX_ACCEDE)},
{"exclusive", unsigned(MDBX_EXCLUSIVE)},
{nullptr, 0}};
const struct option_verb table_bits[] = {
{"key.reverse", unsigned(MDBX_REVERSEKEY)},
{"key.integer", unsigned(MDBX_INTEGERKEY)},
{"data.integer", unsigned(MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT)},
{"data.fixed", unsigned(MDBX_DUPFIXED | MDBX_DUPSORT)},
{"data.reverse", unsigned(MDBX_REVERSEDUP | MDBX_DUPSORT)},
{"data.dups", unsigned(MDBX_DUPSORT)},
{nullptr, 0}};
const struct option_verb table_bits[] = {{"key.reverse", unsigned(MDBX_REVERSEKEY)},
{"key.integer", unsigned(MDBX_INTEGERKEY)},
{"data.integer", unsigned(MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT)},
{"data.fixed", unsigned(MDBX_DUPFIXED | MDBX_DUPSORT)},
{"data.reverse", unsigned(MDBX_REVERSEDUP | MDBX_DUPSORT)},
{"data.dups", unsigned(MDBX_DUPSORT)},
{nullptr, 0}};
static void dump_verbs(const char *caption, size_t bits,
const struct option_verb *verbs) {
static void dump_verbs(const char *caption, size_t bits, const struct option_verb *verbs) {
log_verbose("%s: 0x%" PRIx64 " = ", caption, (uint64_t)bits);
const char *comma = "";
@ -429,28 +386,21 @@ void dump(const char *title) {
logging::local_suffix indent(title);
for (auto i = global::actors.begin(); i != global::actors.end(); ++i) {
log_verbose("#%u, testcase %s, space_id/table %u\n", i->actor_id,
testcase2str(i->testcase), i->space_id);
log_verbose("#%u, testcase %s, space_id/table %u\n", i->actor_id, testcase2str(i->testcase), i->space_id);
indent.push();
log_verbose("prng-seed: %u\n", i->params.prng_seed);
if (i->params.loglevel) {
log_verbose("log: level %u, %s\n", i->params.loglevel,
i->params.pathname_log.empty()
? "console"
: i->params.pathname_log.c_str());
i->params.pathname_log.empty() ? "console" : i->params.pathname_log.c_str());
}
log_verbose("database: %s, size %" PRIuPTR "[%" PRIiPTR "..%" PRIiPTR
", %i %i, %i]\n",
i->params.pathname_db.c_str(), i->params.size_now,
i->params.size_lower, i->params.size_upper,
i->params.shrink_threshold, i->params.growth_step,
i->params.pagesize);
log_verbose("database: %s, size %" PRIuPTR "[%" PRIiPTR "..%" PRIiPTR ", %i %i, %i]\n",
i->params.pathname_db.c_str(), i->params.size_now, i->params.size_lower, i->params.size_upper,
i->params.shrink_threshold, i->params.growth_step, i->params.pagesize);
dump_verbs("mode", i->params.mode_flags, mode_bits);
log_verbose("random-writemap: %s\n",
i->params.random_writemap ? "Yes" : "No");
log_verbose("random-writemap: %s\n", i->params.random_writemap ? "Yes" : "No");
dump_verbs("table", i->params.table_flags, table_bits);
if (i->params.test_nops)
@ -465,62 +415,46 @@ void dump(const char *title) {
log_verbose("threads %u\n", i->params.nthreads);
log_verbose(
"keygen.params: case %s, width %u, mesh %u, rotate %u, offset %" PRIu64
", split %u/%u\n",
keygencase2str(i->params.keygen.keycase), i->params.keygen.width,
i->params.keygen.mesh, i->params.keygen.rotate, i->params.keygen.offset,
i->params.keygen.split,
i->params.keygen.width - i->params.keygen.split);
log_verbose("keygen.zerofill: %s\n",
i->params.keygen.zero_fill ? "Yes" : "No");
log_verbose("key: minlen %u, maxlen %u\n", i->params.keylen_min,
i->params.keylen_max);
log_verbose("data: minlen %u, maxlen %u\n", i->params.datalen_min,
i->params.datalen_max);
log_verbose("keygen.params: case %s, width %u, mesh %u, rotate %u, offset %" PRIu64 ", split %u/%u\n",
keygencase2str(i->params.keygen.keycase), i->params.keygen.width, i->params.keygen.mesh,
i->params.keygen.rotate, i->params.keygen.offset, i->params.keygen.split,
i->params.keygen.width - i->params.keygen.split);
log_verbose("keygen.zerofill: %s\n", i->params.keygen.zero_fill ? "Yes" : "No");
log_verbose("key: minlen %u, maxlen %u\n", i->params.keylen_min, i->params.keylen_max);
log_verbose("data: minlen %u, maxlen %u\n", i->params.datalen_min, i->params.datalen_max);
log_verbose("batch: read %u, write %u\n", i->params.batch_read,
i->params.batch_write);
log_verbose("batch: read %u, write %u\n", i->params.batch_read, i->params.batch_write);
if (i->params.waitfor_nops)
log_verbose("wait: actor %u for %u ops\n", i->wait4id,
i->params.waitfor_nops);
log_verbose("wait: actor %u for %u ops\n", i->wait4id, i->params.waitfor_nops);
else if (i->params.delaystart)
dump_duration("delay", i->params.delaystart);
else
log_verbose("no-delay\n");
if (i->params.inject_writefaultn)
log_verbose("inject-writefault on %u ops\n",
i->params.inject_writefaultn);
log_verbose("inject-writefault on %u ops\n", i->params.inject_writefaultn);
else
log_verbose("no-inject-writefault\n");
log_verbose("limits: readers %u, tables %u, txn-bytes %zu\n",
i->params.max_readers, i->params.max_tables,
log_verbose("limits: readers %u, tables %u, txn-bytes %zu\n", i->params.max_readers, i->params.max_tables,
mdbx_limits_txnsize_max(i->params.pagesize));
log_verbose("drop table: %s\n", i->params.drop_table ? "Yes" : "No");
log_verbose("ignore MDBX_MAP_FULL error: %s\n",
i->params.ignore_dbfull ? "Yes" : "No");
log_verbose("verifying by speculum: %s\n",
i->params.speculum ? "Yes" : "No");
log_verbose("ignore MDBX_MAP_FULL error: %s\n", i->params.ignore_dbfull ? "Yes" : "No");
log_verbose("verifying by speculum: %s\n", i->params.speculum ? "Yes" : "No");
indent.pop();
}
dump_duration("timeout", global::config::timeout_duration_seconds);
log_verbose("cleanup: before %s, after %s\n",
global::config::cleanup_before ? "Yes" : "No",
log_verbose("cleanup: before %s, after %s\n", global::config::cleanup_before ? "Yes" : "No",
global::config::cleanup_after ? "Yes" : "No");
log_verbose("failfast: %s\n", global::config::failfast ? "Yes" : "No");
log_verbose("progress indicator: %s\n",
global::config::progress_indicator ? "Yes" : "No");
log_verbose("console mode: %s\n",
global::config::console_mode ? "Yes" : "No");
log_verbose("geometry jitter: %s\n",
global::config::geometry_jitter ? "Yes" : "No");
log_verbose("progress indicator: %s\n", global::config::progress_indicator ? "Yes" : "No");
log_verbose("console mode: %s\n", global::config::console_mode ? "Yes" : "No");
log_verbose("geometry jitter: %s\n", global::config::geometry_jitter ? "Yes" : "No");
}
} /* namespace config */
@ -529,11 +463,8 @@ void dump(const char *title) {
using namespace config;
actor_config::actor_config(actor_testcase testcase, const actor_params &params,
unsigned space_id, unsigned wait4id)
: actor_config_pod(1 + unsigned(global::actors.size()), testcase, space_id,
wait4id),
params(params) {}
actor_config::actor_config(actor_testcase testcase, const actor_params &params, unsigned space_id, unsigned wait4id)
: actor_config_pod(1 + unsigned(global::actors.size()), testcase, space_id, wait4id), params(params) {}
const std::string actor_config::serialize(const char *prefix) const {
simple_checksum checksum;
@ -551,25 +482,19 @@ const std::string actor_config::serialize(const char *prefix) const {
result.push_back('|');
#if __cplusplus > 201400
static_assert(std::is_trivially_copyable<actor_params_pod>::value,
"actor_params_pod should by POD");
static_assert(std::is_trivially_copyable<actor_params_pod>::value, "actor_params_pod should by POD");
#else
static_assert(std::is_standard_layout<actor_params_pod>::value,
"actor_params_pod should by POD");
static_assert(std::is_standard_layout<actor_params_pod>::value, "actor_params_pod should by POD");
#endif
result.append(data2hex(static_cast<const actor_params_pod *>(&params),
sizeof(actor_params_pod), checksum));
result.append(data2hex(static_cast<const actor_params_pod *>(&params), sizeof(actor_params_pod), checksum));
result.push_back('|');
#if __cplusplus > 201400
static_assert(std::is_trivially_copyable<actor_config_pod>::value,
"actor_config_pod should by POD");
static_assert(std::is_trivially_copyable<actor_config_pod>::value, "actor_config_pod should by POD");
#else
static_assert(std::is_standard_layout<actor_config_pod>::value,
"actor_config_pod should by POD");
static_assert(std::is_standard_layout<actor_config_pod>::value, "actor_config_pod should by POD");
#endif
result.append(data2hex(static_cast<const actor_config_pod *>(this),
sizeof(actor_config_pod), checksum));
result.append(data2hex(static_cast<const actor_config_pod *>(this), sizeof(actor_config_pod), checksum));
result.push_back('|');
result.push_back(global::config::progress_indicator ? 'Y' : 'N');
checksum.push(global::config::progress_indicator);
@ -615,16 +540,12 @@ bool actor_config::deserialize(const char *str, actor_config &config) {
return false;
}
#if __cplusplus > 201400
static_assert(std::is_trivially_copyable<actor_params_pod>::value,
"actor_params_pod should by POD");
static_assert(std::is_trivially_copyable<actor_params_pod>::value, "actor_params_pod should by POD");
#else
static_assert(std::is_standard_layout<actor_params_pod>::value,
"actor_params_pod should by POD");
static_assert(std::is_standard_layout<actor_params_pod>::value, "actor_params_pod should by POD");
#endif
if (!hex2data(str, slash, static_cast<actor_params_pod *>(&config.params),
sizeof(actor_params_pod), checksum)) {
TRACE("<< actor_config::deserialize: actor_params_pod(%.*s)\n",
(int)(slash - str), str);
if (!hex2data(str, slash, static_cast<actor_params_pod *>(&config.params), sizeof(actor_params_pod), checksum)) {
TRACE("<< actor_config::deserialize: actor_params_pod(%.*s)\n", (int)(slash - str), str);
return false;
}
str = slash + 1;
@ -635,16 +556,12 @@ bool actor_config::deserialize(const char *str, actor_config &config) {
return false;
}
#if __cplusplus > 201400
static_assert(std::is_trivially_copyable<actor_config_pod>::value,
"actor_config_pod should by POD");
static_assert(std::is_trivially_copyable<actor_config_pod>::value, "actor_config_pod should by POD");
#else
static_assert(std::is_standard_layout<actor_config_pod>::value,
"actor_config_pod should by POD");
static_assert(std::is_standard_layout<actor_config_pod>::value, "actor_config_pod should by POD");
#endif
if (!hex2data(str, slash, static_cast<actor_config_pod *>(&config),
sizeof(actor_config_pod), checksum)) {
TRACE("<< actor_config::deserialize: actor_config_pod(%.*s)\n",
(int)(slash - str), str);
if (!hex2data(str, slash, static_cast<actor_config_pod *>(&config), sizeof(actor_config_pod), checksum)) {
TRACE("<< actor_config::deserialize: actor_config_pod(%.*s)\n", (int)(slash - str), str);
return false;
}
str = slash + 1;
@ -654,8 +571,7 @@ bool actor_config::deserialize(const char *str, actor_config &config) {
TRACE("<< actor_config::deserialize: slash-5\n");
return false;
}
if ((str[0] == 'Y' || str[0] == 'N') && (str[1] == 'Y' || str[1] == 'N') &&
(str[2] == 'Y' || str[2] == 'N')) {
if ((str[0] == 'Y' || str[0] == 'N') && (str[1] == 'Y' || str[1] == 'N') && (str[2] == 'Y' || str[2] == 'N')) {
global::config::progress_indicator = str[0] == 'Y';
checksum.push(global::config::progress_indicator);
global::config::console_mode = str[1] == 'Y';
@ -690,21 +606,14 @@ bool actor_config::deserialize(const char *str, actor_config &config) {
return true;
}
unsigned actor_params::mdbx_keylen_min() const {
return unsigned(mdbx_limits_keysize_min(table_flags));
}
unsigned actor_params::mdbx_keylen_min() const { return unsigned(mdbx_limits_keysize_min(table_flags)); }
unsigned actor_params::mdbx_keylen_max() const {
return unsigned(mdbx_limits_keysize_max(pagesize, table_flags));
}
unsigned actor_params::mdbx_keylen_max() const { return unsigned(mdbx_limits_keysize_max(pagesize, table_flags)); }
unsigned actor_params::mdbx_datalen_min() const {
return unsigned(mdbx_limits_valsize_min(table_flags));
}
unsigned actor_params::mdbx_datalen_min() const { return unsigned(mdbx_limits_valsize_min(table_flags)); }
unsigned actor_params::mdbx_datalen_max() const {
return std::min(unsigned(UINT16_MAX),
unsigned(mdbx_limits_valsize_max(pagesize, table_flags)));
return std::min(unsigned(UINT16_MAX), unsigned(mdbx_limits_valsize_max(pagesize, table_flags)));
}
bool actor_params::make_keygen_linear() {
@ -713,26 +622,18 @@ bool actor_params::make_keygen_linear() {
keygen.rotate = 0;
keygen.offset = 0;
const auto max_serial = serial_mask(keygen.width) + base;
const auto max_key_serial = (keygen.split && (table_flags & MDBX_DUPSORT))
? max_serial >> keygen.split
: max_serial;
const auto max_value_serial = (keygen.split && (table_flags & MDBX_DUPSORT))
? serial_mask(keygen.split)
: 0;
const auto max_key_serial = (keygen.split && (table_flags & MDBX_DUPSORT)) ? max_serial >> keygen.split : max_serial;
const auto max_value_serial = (keygen.split && (table_flags & MDBX_DUPSORT)) ? serial_mask(keygen.split) : 0;
while (keylen_min < 8 &&
(keylen_min == 0 || serial_mask(keylen_min * 8) < max_key_serial)) {
while (keylen_min < 8 && (keylen_min == 0 || serial_mask(keylen_min * 8) < max_key_serial)) {
keylen_min += (table_flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) ? 4 : 1;
if (keylen_max < keylen_min)
keylen_max = keylen_min;
}
if (table_flags & MDBX_DUPSORT)
while (
datalen_min < 8 &&
(datalen_min == 0 || serial_mask(datalen_min * 8) < max_value_serial)) {
datalen_min +=
(table_flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) ? 4 : 1;
while (datalen_min < 8 && (datalen_min == 0 || serial_mask(datalen_min * 8) < max_value_serial)) {
datalen_min += (table_flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) ? 4 : 1;
if (datalen_max < datalen_min)
datalen_max = datalen_min;
}

69
test/config.h++
View File

@ -54,18 +54,16 @@ namespace config {
enum scale_mode { no_scale, decimal, binary, duration, intkey, entropy };
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
const char **value, const char *default_value = nullptr);
bool parse_option(int argc, char *const argv[], int &narg, const char *option, const char **value,
const char *default_value = nullptr);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty = false);
bool parse_option(int argc, char *const argv[], int &narg, const char *option, std::string &value,
bool allow_empty = false);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty,
bool parse_option(int argc, char *const argv[], int &narg, const char *option, std::string &value, bool allow_empty,
const char *default_value);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
bool &value);
bool parse_option(int argc, char *const argv[], int &narg, const char *option, bool &value);
struct option_verb {
const char *const verb;
@ -73,8 +71,7 @@ struct option_verb {
};
template <typename MASK>
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
MASK &mask, const option_verb *verbs) {
bool parse_option(int argc, char *const argv[], int &narg, const char *option, MASK &mask, const option_verb *verbs) {
static_assert(sizeof(MASK) <= sizeof(unsigned), "WTF?");
unsigned u = unsigned(mask);
if (parse_option<unsigned>(argc, argv, narg, option, u, verbs)) {
@ -85,49 +82,36 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
}
template <>
bool parse_option<unsigned>(int argc, char *const argv[], int &narg,
const char *option, unsigned &mask,
bool parse_option<unsigned>(int argc, char *const argv[], int &narg, const char *option, unsigned &mask,
const option_verb *verbs);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint64_t &value, const scale_mode scale,
const uint64_t minval = 0, const uint64_t maxval = INT64_MAX,
const uint64_t default_value = 0);
bool parse_option(int argc, char *const argv[], int &narg, const char *option, uint64_t &value, const scale_mode scale,
const uint64_t minval = 0, const uint64_t maxval = INT64_MAX, const uint64_t default_value = 0);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &value, const scale_mode scale,
const unsigned minval = 0, const unsigned maxval = INT32_MAX,
const unsigned default_value = 0);
bool parse_option(int argc, char *const argv[], int &narg, const char *option, unsigned &value, const scale_mode scale,
const unsigned minval = 0, const unsigned maxval = INT32_MAX, const unsigned default_value = 0);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint8_t &value, const uint8_t minval = 0,
bool parse_option(int argc, char *const argv[], int &narg, const char *option, uint8_t &value, const uint8_t minval = 0,
const uint8_t maxval = 255, const uint8_t default_value = 0);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int64_t &value, const int64_t minval, const int64_t maxval,
const int64_t default_value = -1);
bool parse_option(int argc, char *const argv[], int &narg, const char *option, int64_t &value, const int64_t minval,
const int64_t maxval, const int64_t default_value = -1);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int32_t &value, const int32_t minval, const int32_t maxval,
const int32_t default_value = -1);
bool parse_option(int argc, char *const argv[], int &narg, const char *option, int32_t &value, const int32_t minval,
const int32_t maxval, const int32_t default_value = -1);
inline bool parse_option_intptr(int argc, char *const argv[], int &narg,
const char *option, intptr_t &value,
const intptr_t minval, const intptr_t maxval,
const intptr_t default_value = -1) {
inline bool parse_option_intptr(int argc, char *const argv[], int &narg, const char *option, intptr_t &value,
const intptr_t minval, const intptr_t maxval, const intptr_t default_value = -1) {
static_assert(sizeof(intptr_t) == 4 || sizeof(intptr_t) == 8, "WTF?");
if (sizeof(intptr_t) == 8)
return parse_option(argc, argv, narg, option,
*reinterpret_cast<int64_t *>(&value), int64_t(minval),
return parse_option(argc, argv, narg, option, *reinterpret_cast<int64_t *>(&value), int64_t(minval),
int64_t(maxval), int64_t(default_value));
else
return parse_option(argc, argv, narg, option,
*reinterpret_cast<int32_t *>(&value), int32_t(minval),
return parse_option(argc, argv, narg, option, *reinterpret_cast<int32_t *>(&value), int32_t(minval),
int32_t(maxval), int32_t(default_value));
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
logging::loglevel &);
bool parse_option(int argc, char *const argv[], int &narg, const char *option, logging::loglevel &);
//-----------------------------------------------------------------------------
struct keygen_params_pod {
@ -295,10 +279,8 @@ struct actor_config_pod {
unsigned signal_nops{0};
actor_config_pod() = default;
actor_config_pod(unsigned actor_id, actor_testcase testcase,
unsigned space_id, unsigned wait4id)
: actor_id(actor_id), space_id(space_id), testcase(testcase),
wait4id(wait4id) {}
actor_config_pod(unsigned actor_id, actor_testcase testcase, unsigned space_id, unsigned wait4id)
: actor_id(actor_id), space_id(space_id), testcase(testcase), wait4id(wait4id) {}
};
extern const struct option_verb mode_bits[];
@ -326,8 +308,7 @@ struct actor_config : public config::actor_config_pod {
bool wanna_event4signalling() const { return true /* TODO ? */; }
actor_config() = default;
actor_config(actor_testcase testcase, const actor_params &params,
unsigned space_id, unsigned wait4id);
actor_config(actor_testcase testcase, const actor_params &params, unsigned space_id, unsigned wait4id);
actor_config(const char *str) : actor_config() {
if (!deserialize(str, *this))

41
test/copy.c++
View File

@ -9,8 +9,7 @@ class testcase_copy : public testcase {
public:
testcase_copy(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid),
copy_pathname(config.params.pathname_db + "-copy") {}
: testcase(config, pid), copy_pathname(config.params.pathname_db + "-copy") {}
bool run() override;
};
REGISTER_TESTCASE(copy);
@ -21,14 +20,10 @@ void testcase_copy::copy_db(const bool with_compaction) {
failure_perror("osal_removefile()", err);
if (flipcoin()) {
err = mdbx_env_copy(db_guard.get(), copy_pathname.c_str(),
with_compaction ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS);
log_verbose("mdbx_env_copy(%s), err %d", with_compaction ? "true" : "false",
err);
err = mdbx_env_copy(db_guard.get(), copy_pathname.c_str(), with_compaction ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS);
log_verbose("mdbx_env_copy(%s), err %d", with_compaction ? "true" : "false", err);
if (unlikely(err != MDBX_SUCCESS))
failure_perror(with_compaction ? "mdbx_env_copy(MDBX_CP_COMPACT)"
: "mdbx_env_copy(MDBX_CP_ASIS)",
err);
failure_perror(with_compaction ? "mdbx_env_copy(MDBX_CP_COMPACT)" : "mdbx_env_copy(MDBX_CP_ASIS)", err);
} else {
do {
const bool ro = mode_readonly() || flipcoin();
@ -36,26 +31,20 @@ void testcase_copy::copy_db(const bool with_compaction) {
const bool dynsize = flipcoin();
const bool flush = flipcoin();
const bool enable_renew = flipcoin();
const MDBX_copy_flags_t flags =
(with_compaction ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
(dynsize ? MDBX_CP_FORCE_DYNAMIC_SIZE : MDBX_CP_DEFAULTS) |
(throttle ? MDBX_CP_THROTTLE_MVCC : MDBX_CP_DEFAULTS) |
(flush ? MDBX_CP_DEFAULTS : MDBX_CP_DONT_FLUSH) |
(enable_renew ? MDBX_CP_RENEW_TXN : MDBX_CP_DEFAULTS);
const MDBX_copy_flags_t flags = (with_compaction ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
(dynsize ? MDBX_CP_FORCE_DYNAMIC_SIZE : MDBX_CP_DEFAULTS) |
(throttle ? MDBX_CP_THROTTLE_MVCC : MDBX_CP_DEFAULTS) |
(flush ? MDBX_CP_DEFAULTS : MDBX_CP_DONT_FLUSH) |
(enable_renew ? MDBX_CP_RENEW_TXN : MDBX_CP_DEFAULTS);
txn_begin(ro);
err =
mdbx_txn_copy2pathname(txn_guard.get(), copy_pathname.c_str(), flags);
err = mdbx_txn_copy2pathname(txn_guard.get(), copy_pathname.c_str(), flags);
log_verbose("mdbx_txn_copy2pathname(flags=0x%X), err %d", flags, err);
txn_end(err != MDBX_SUCCESS || flipcoin());
if (unlikely(
err != MDBX_SUCCESS && !(throttle && err == MDBX_OUSTED) &&
!(!enable_renew && err == MDBX_MVCC_RETARDED) &&
!(err == MDBX_EINVAL && !ro &&
(flags & (MDBX_CP_THROTTLE_MVCC | MDBX_CP_RENEW_TXN)) != 0)))
failure_perror(with_compaction
? "mdbx_txn_copy2pathname(MDBX_CP_COMPACT)"
: "mdbx_txn_copy2pathname(MDBX_CP_ASIS)",
err);
if (unlikely(err != MDBX_SUCCESS && !(throttle && err == MDBX_OUSTED) &&
!(!enable_renew && err == MDBX_MVCC_RETARDED) &&
!(err == MDBX_EINVAL && !ro && (flags & (MDBX_CP_THROTTLE_MVCC | MDBX_CP_RENEW_TXN)) != 0)))
failure_perror(
with_compaction ? "mdbx_txn_copy2pathname(MDBX_CP_COMPACT)" : "mdbx_txn_copy2pathname(MDBX_CP_ASIS)", err);
} while (err != MDBX_SUCCESS);
}
}

6
test/dead.c++
View File

@ -5,8 +5,7 @@
class testcase_deadread : public testcase {
public:
testcase_deadread(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
testcase_deadread(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {}
bool run() override;
};
REGISTER_TESTCASE(deadread);
@ -24,8 +23,7 @@ bool testcase_deadread::run() {
class testcase_deadwrite : public testcase {
public:
testcase_deadwrite(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
testcase_deadwrite(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {}
bool run() override;
};

76
test/extra/crunched_delete.c++
View File

@ -26,8 +26,7 @@ std::string format_va(const char *fmt, va_list ap) {
result.reserve(size_t(needed + 1));
result.resize(size_t(needed), '\0');
assert(int(result.capacity()) > needed);
int actual = vsnprintf(const_cast<char *>(result.data()), result.capacity(),
fmt, ones);
int actual = vsnprintf(const_cast<char *>(result.data()), result.capacity(), fmt, ones);
assert(actual == needed);
(void)actual;
va_end(ones);
@ -47,10 +46,8 @@ struct acase {
unsigned vlen_min, vlen_max;
unsigned dupmax_log2;
acase(unsigned klen_min, unsigned klen_max, unsigned vlen_min,
unsigned vlen_max, unsigned dupmax_log2)
: klen_min(klen_min), klen_max(klen_max), vlen_min(vlen_min),
vlen_max(vlen_max), dupmax_log2(dupmax_log2) {}
acase(unsigned klen_min, unsigned klen_max, unsigned vlen_min, unsigned vlen_max, unsigned dupmax_log2)
: klen_min(klen_min), klen_max(klen_max), vlen_min(vlen_min), vlen_max(vlen_max), dupmax_log2(dupmax_log2) {}
};
// std::random_device rd;
@ -80,33 +77,26 @@ static mdbx::slice mk_val(mdbx::default_buffer &buf, const acase &thecase) {
static std::string name(unsigned n) { return format("Commitment_%05u", n); }
static mdbx::map_handle create_and_fill(mdbx::txn txn, const acase &thecase,
const unsigned n) {
static mdbx::map_handle create_and_fill(mdbx::txn txn, const acase &thecase, const unsigned n) {
auto map = txn.create_map(name(n),
(thecase.klen_min == thecase.klen_max &&
(thecase.klen_min == 4 || thecase.klen_max == 8))
(thecase.klen_min == thecase.klen_max && (thecase.klen_min == 4 || thecase.klen_max == 8))
? mdbx::key_mode::ordinal
: mdbx::key_mode::usual,
(thecase.vlen_min == thecase.vlen_max)
? mdbx::value_mode::multi_samelength
: mdbx::value_mode::multi);
(thecase.vlen_min == thecase.vlen_max) ? mdbx::value_mode::multi_samelength
: mdbx::value_mode::multi);
if (txn.get_map_stat(map).ms_entries < NN) {
mdbx::default_buffer k, v;
for (auto i = 0u; i < NN; i++) {
mk_key(k, thecase);
for (auto ii = thecase.dupmax_log2
? 1u + (rnd() & ((2u << thecase.dupmax_log2) - 1u))
: 1u;
ii > 0; --ii)
for (auto ii = thecase.dupmax_log2 ? 1u + (rnd() & ((2u << thecase.dupmax_log2) - 1u)) : 1u; ii > 0; --ii)
txn.upsert(map, k, mk_val(v, thecase));
}
}
return map;
}
static void chunched_delete(mdbx::txn txn, const acase &thecase,
const unsigned n) {
static void chunched_delete(mdbx::txn txn, const acase &thecase, const unsigned n) {
// printf(">> %s, case #%i\n", __FUNCTION__, n);
mdbx::default_buffer k, v;
auto map = txn.open_map_accede(name(n));
@ -121,27 +111,20 @@ static void chunched_delete(mdbx::txn txn, const acase &thecase,
bool last_r;
if (true == ((last_op = "MDBX_GET_BOTH"),
(last_r = cursor.find_multivalue(
mk_key(k, thecase), mk_val(v, thecase), false))) ||
(last_r = cursor.find_multivalue(mk_key(k, thecase), mk_val(v, thecase), false))) ||
rnd() % 3 == 0 ||
true == ((last_op = "MDBX_SET_RANGE"),
(last_r = cursor.lower_bound(mk_key(k, thecase), false)))) {
true == ((last_op = "MDBX_SET_RANGE"), (last_r = cursor.lower_bound(mk_key(k, thecase), false)))) {
int i = int(rnd() % 7) - 3;
// if (i)
// printf(" %s -> %s\n", last_op, last_r ? "true" : "false");
// printf("== shift multi %i\n", i);
try {
while (i < 0 &&
true == ((last_op = "MDBX_PREV_DUP"),
(last_r = cursor.to_current_prev_multi(false))))
while (i < 0 && true == ((last_op = "MDBX_PREV_DUP"), (last_r = cursor.to_current_prev_multi(false))))
++i;
while (i > 0 &&
true == ((last_op = "MDBX_NEXT_DUP"),
(last_r = cursor.to_current_next_multi(false))))
while (i > 0 && true == ((last_op = "MDBX_NEXT_DUP"), (last_r = cursor.to_current_next_multi(false))))
--i;
} catch (const mdbx::no_data &) {
printf("cursor_del() -> exception, last %s %s\n", last_op,
last_r ? "true" : "false");
printf("cursor_del() -> exception, last %s %s\n", last_op, last_r ? "true" : "false");
continue;
}
}
@ -159,8 +142,7 @@ static void chunched_delete(mdbx::txn txn, const acase &thecase,
// printf(" cursor_del() -> %s\n", last_r ? "true" : "false");
} while (cursor.to_next(false) && --i > 0);
} catch (const mdbx::no_data &) {
printf("cursor_del() -> exception, last %s %s\n", last_op,
last_r ? "true" : "false");
printf("cursor_del() -> exception, last %s %s\n", last_op, last_r ? "true" : "false");
}
// (void) last_op;
@ -178,8 +160,8 @@ static void chunched_delete(mdbx::txn txn, const acase &thecase,
static char log_buffer[1024];
static void logger_nofmt(MDBX_log_level_t loglevel, const char *function,
int line, const char *msg, unsigned length) noexcept {
static void logger_nofmt(MDBX_log_level_t loglevel, const char *function, int line, const char *msg,
unsigned length) noexcept {
(void)length;
(void)loglevel;
fprintf(stdout, "%s:%u %s", function, line, msg);
@ -187,12 +169,10 @@ static void logger_nofmt(MDBX_log_level_t loglevel, const char *function,
bool outofrange_prev(mdbx::env env) {
mdbx::cursor_managed cursor;
const std::array<mdbx::pair, 4> items = {
{{"k1", "v1"}, {"k1", "v2"}, {"k2", "v1"}, {"k2", "v2"}}};
const std::array<mdbx::pair, 4> items = {{{"k1", "v1"}, {"k1", "v2"}, {"k2", "v1"}, {"k2", "v2"}}};
auto txn = env.start_write();
auto multi =
txn.create_map("multi", mdbx::key_mode::usual, mdbx::value_mode::multi);
auto multi = txn.create_map("multi", mdbx::key_mode::usual, mdbx::value_mode::multi);
auto simple = txn.create_map("simple");
txn.clear_map(multi);
txn.clear_map(simple);
@ -242,12 +222,10 @@ bool outofrange_prev(mdbx::env env) {
}
bool next_prev_current(mdbx::env env) {
const std::array<mdbx::pair, 4> items = {
{{"k1", "v1"}, {"k1", "v2"}, {"k2", "v1"}, {"k2", "v2"}}};
const std::array<mdbx::pair, 4> items = {{{"k1", "v1"}, {"k1", "v2"}, {"k2", "v1"}, {"k2", "v2"}}};
auto txn = env.start_write();
auto map =
txn.create_map("multi", mdbx::key_mode::usual, mdbx::value_mode::multi);
auto map = txn.create_map("multi", mdbx::key_mode::usual, mdbx::value_mode::multi);
txn.clear_map(map);
for (const auto &i : items)
txn.upsert(map, i);
@ -309,8 +287,7 @@ bool next_prev_current(mdbx::env env) {
}
bool simple(mdbx::env env) {
const std::array<mdbx::pair, 3> items = {
{{"k0", "v0"}, {"k1", "v1"}, {"k2", "v2"}}};
const std::array<mdbx::pair, 3> items = {{{"k0", "v0"}, {"k1", "v1"}, {"k2", "v2"}}};
auto txn = env.start_write();
auto map = txn.create_map("simple");
@ -376,14 +353,12 @@ int main(int argc, const char *argv[]) {
(void)argc;
(void)argv;
mdbx_setup_debug_nofmt(MDBX_LOG_NOTICE, MDBX_DBG_ASSERT, logger_nofmt,
log_buffer, sizeof(log_buffer));
mdbx_setup_debug_nofmt(MDBX_LOG_NOTICE, MDBX_DBG_ASSERT, logger_nofmt, log_buffer, sizeof(log_buffer));
mdbx::path db_filename = "test-crunched-del";
mdbx::env::remove(db_filename);
mdbx::env_managed env(db_filename, mdbx::env_managed::create_parameters(),
mdbx::env::operate_parameters(42));
mdbx::env_managed env(db_filename, mdbx::env_managed::create_parameters(), mdbx::env::operate_parameters(42));
if (!simple(env) || !next_prev_current(env) || !outofrange_prev(env))
return EXIT_FAILURE;
@ -392,8 +367,7 @@ int main(int argc, const char *argv[]) {
// Значения разной длины от 100 до 1000 байт.
testset.emplace_back(/* keylen_min */ 1, /* keylen_max */ 64,
/* datalen_min */ 100, /* datalen_max */
mdbx_env_get_valsize4page_max(
env, MDBX_db_flags_t(mdbx::value_mode::multi)),
mdbx_env_get_valsize4page_max(env, MDBX_db_flags_t(mdbx::value_mode::multi)),
/* dups_log2 */ 6);
// В одной таблице DupSort: path -> version_u64+data
// path - это префикс в дереве. Самые частые длины: 1-5 байт и 32-36 байт.

15
test/extra/cursor_closing.c++
View File

@ -2,8 +2,8 @@
#include <iostream>
static void logger_nofmt(MDBX_log_level_t loglevel, const char *function,
int line, const char *msg, unsigned length) noexcept {
static void logger_nofmt(MDBX_log_level_t loglevel, const char *function, int line, const char *msg,
unsigned length) noexcept {
(void)length;
(void)loglevel;
std::cout << function << ":" << line << " " << msg;
@ -15,20 +15,17 @@ int main(int argc, const char *argv[]) {
(void)argc;
(void)argv;
mdbx_setup_debug_nofmt(MDBX_LOG_NOTICE, MDBX_DBG_ASSERT, logger_nofmt,
log_buffer, sizeof(log_buffer));
mdbx_setup_debug_nofmt(MDBX_LOG_NOTICE, MDBX_DBG_ASSERT, logger_nofmt, log_buffer, sizeof(log_buffer));
mdbx::path db_filename = "test-cursor-closing";
mdbx::env::remove(db_filename);
mdbx::env_managed env(
db_filename, mdbx::env_managed::create_parameters(),
mdbx::env::operate_parameters(42, 0, mdbx::env::nested_transactions));
mdbx::env_managed env(db_filename, mdbx::env_managed::create_parameters(),
mdbx::env::operate_parameters(42, 0, mdbx::env::nested_transactions));
{
auto txn = env.start_write();
auto table = txn.create_map("dummy", mdbx::key_mode::usual,
mdbx::value_mode::single);
auto table = txn.create_map("dummy", mdbx::key_mode::usual, mdbx::value_mode::single);
auto cursor_1 = txn.open_cursor(table);
auto cursor_2 = cursor_1.clone();

13
test/extra/dbi.c++
View File

@ -4,8 +4,8 @@
static char log_buffer[1024];
static void logger_nofmt(MDBX_log_level_t loglevel, const char *function,
int line, const char *msg, unsigned length) noexcept {
static void logger_nofmt(MDBX_log_level_t loglevel, const char *function, int line, const char *msg,
unsigned length) noexcept {
(void)length;
(void)loglevel;
fprintf(stdout, "%s:%u %s", function, line, msg);
@ -15,8 +15,7 @@ int main(int argc, const char *argv[]) {
(void)argc;
(void)argv;
mdbx_setup_debug_nofmt(MDBX_LOG_NOTICE, MDBX_DBG_ASSERT, logger_nofmt,
log_buffer, sizeof(log_buffer));
mdbx_setup_debug_nofmt(MDBX_LOG_NOTICE, MDBX_DBG_ASSERT, logger_nofmt, log_buffer, sizeof(log_buffer));
mdbx::path db_filename = "test-dbi";
mdbx::env::remove(db_filename);
@ -26,14 +25,12 @@ int main(int argc, const char *argv[]) {
{
mdbx::env_managed env2(db_filename, createParameters, operateParameters);
mdbx::txn_managed txn2 = env2.start_write(false);
/* mdbx::map_handle testHandle2 = */ txn2.create_map(
"fap1", mdbx::key_mode::reverse, mdbx::value_mode::single);
/* mdbx::map_handle testHandle2 = */ txn2.create_map("fap1", mdbx::key_mode::reverse, mdbx::value_mode::single);
txn2.commit();
}
mdbx::env_managed env(db_filename, createParameters, operateParameters);
mdbx::txn_managed txn = env.start_write(false);
/* mdbx::map_handle testHandle = */ txn.create_map(
"fap1", mdbx::key_mode::usual, mdbx::value_mode::single);
/* mdbx::map_handle testHandle = */ txn.create_map("fap1", mdbx::key_mode::usual, mdbx::value_mode::single);
txn.commit();
std::cout << "OK\n";

Some files were not shown because too many files have changed in this diff Show More