mdbx-test: add --ignore-dbfull option (major).

Change-Id: I252f9c3679a371722a780913ba994ca3dee9b90a
This commit is contained in:
Leonid Yuriev 2019-06-23 15:55:13 +03:00
parent 728f98d3de
commit 2d5a3ebd8f
8 changed files with 308 additions and 85 deletions

View File

@ -15,11 +15,12 @@
#include "test.h"
bool testcase_append::run() {
db_open();
txn_begin(false);
MDBX_dbi dbi = db_table_open(true);
db_table_clear(dbi);
MDBX_dbi dbi;
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-prepare due '%s'", mdbx_strerror(err));
return true;
}
keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
/* LY: тест наполнения таблиц в append-режиме,
@ -41,7 +42,10 @@ bool testcase_append::run() {
simple_checksum inserted_checksum;
uint64_t inserted_number = 0;
uint64_t serial_count = 0;
unsigned txn_nops = 0;
uint64_t commited_inserted_number = inserted_number;
simple_checksum commited_inserted_checksum = inserted_checksum;
while (should_continue()) {
const keygen::serial_t serial = serial_count;
if (!keyvalue_maker.increment(serial_count, 1)) {
@ -57,10 +61,19 @@ bool testcase_append::run() {
if (cmp == 0 && (config.params.table_flags & MDBX_DUPSORT))
cmp = mdbx_dcmp(txn_guard.get(), dbi, &data->value, &last_data->value);
int err = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value, flags);
err = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value, flags);
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("append: bailout-insert due '%s'", mdbx_strerror(err));
txn_end(true);
inserted_number = commited_inserted_number;
inserted_checksum = commited_inserted_checksum;
break;
}
if (cmp > 0) {
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_put(appenda-a)", err);
memcpy(last_key->value.iov_base, key->value.iov_base,
last_key->value.iov_len = key->value.iov_len);
memcpy(last_data->value.iov_base, data->value.iov_base,
@ -74,22 +87,40 @@ bool testcase_append::run() {
}
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-commit due '%s'", mdbx_strerror(err));
inserted_number = commited_inserted_number;
inserted_checksum = commited_inserted_checksum;
break;
}
commited_inserted_number = inserted_number;
commited_inserted_checksum = inserted_checksum;
txn_nops = 0;
}
report(1);
}
txn_restart(false, true);
if (txn_guard) {
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-commit due '%s'", mdbx_strerror(err));
inserted_number = commited_inserted_number;
inserted_checksum = commited_inserted_checksum;
}
}
//----------------------------------------------------------------------------
txn_begin(true);
cursor_open(dbi);
MDBX_val check_key, check_data;
int err =
err =
mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_FIRST);
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_cursor_get(MDBX_FIRST)", err);
if (likely(inserted_number)) {
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_cursor_get(MDBX_FIRST)", err);
}
simple_checksum read_checksum;
uint64_t read_count = 0;
@ -115,15 +146,18 @@ bool testcase_append::run() {
read_checksum.value, inserted_checksum.value);
cursor_close();
txn_end(true);
//----------------------------------------------------------------------------
if (txn_guard)
txn_end(false);
if (dbi) {
if (config.params.drop_table && !mode_readonly()) {
txn_begin(false);
db_table_drop(dbi);
txn_end(false);
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-clean due '%s'", mdbx_strerror(err));
return true;
}
} else
db_table_close(dbi);
}

View File

@ -1,4 +1,4 @@
/*
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
@ -412,6 +412,8 @@ void dump(const char *title) {
i->params.max_tables);
log_info("drop table: %s\n", i->params.drop_table ? "Yes" : "No");
log_info("ignore MDBX_MAP_FULL error: %s\n",
i->params.ignore_dbfull ? "Yes" : "No");
indent.pop();
}

View File

@ -248,6 +248,7 @@ struct actor_params_pod {
keygen_params_pod keygen;
bool drop_table;
bool ignore_dbfull;
};
struct actor_config_pod {

View File

@ -15,11 +15,12 @@
#include "test.h"
bool testcase_hill::run() {
db_open();
txn_begin(false);
MDBX_dbi dbi = db_table_open(true);
txn_end(false);
MDBX_dbi dbi;
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("hill: bailout-prepare due '%s'", mdbx_strerror(err));
return true;
}
/* LY: тест "холмиком":
* - сначала наполняем таблицу циклическими CRUD-манипуляциями,
@ -59,9 +60,8 @@ bool testcase_hill::run() {
: MDBX_NODUPDATA;
uint64_t serial_count = 0;
uint64_t commited_serial = serial_count;
unsigned txn_nops = 0;
if (!txn_guard)
txn_begin(false);
while (should_continue()) {
const keygen::serial_t a_serial = serial_count;
@ -76,26 +76,52 @@ bool testcase_hill::run() {
log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift,
a_serial);
generate_pair(a_serial, a_key, a_data_1, age_shift);
int rc = mdbx_put(txn_guard.get(), dbi, &a_key->value, &a_data_1->value,
insert_flags);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_put(insert-a.1)", rc);
err = mdbx_put(txn_guard.get(), dbi, &a_key->value, &a_data_1->value,
insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at insert-a due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
break;
}
failure_perror("mdbx_put(insert-a.1)", err);
}
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
break;
}
commited_serial = a_serial;
txn_nops = 0;
}
// создаем вторую запись из пары
log_trace("uphill: insert-b %" PRIu64, b_serial);
generate_pair(b_serial, b_key, b_data, 0);
rc = mdbx_put(txn_guard.get(), dbi, &b_key->value, &b_data->value,
insert_flags);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_put(insert-b)", rc);
err = mdbx_put(txn_guard.get(), dbi, &b_key->value, &b_data->value,
insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at insert-b due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
break;
}
failure_perror("mdbx_put(insert-b)", err);
}
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
break;
}
commited_serial = a_serial;
txn_nops = 0;
}
@ -104,25 +130,51 @@ bool testcase_hill::run() {
a_serial);
generate_pair(a_serial, a_key, a_data_0, 0);
checkdata("uphill: update-a", dbi, a_key->value, a_data_1->value);
rc = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_0->value,
&a_data_1->value, update_flags);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_replace(update-a: 1->0)", rc);
err = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_0->value,
&a_data_1->value, update_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at update-a due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
break;
}
failure_perror("mdbx_replace(update-a: 1->0)", err);
}
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
break;
}
commited_serial = a_serial;
txn_nops = 0;
}
// удаляем вторую запись
log_trace("uphill: delete-b %" PRIu64, b_serial);
checkdata("uphill: delete-b", dbi, b_key->value, b_data->value);
rc = mdbx_del(txn_guard.get(), dbi, &b_key->value, &b_data->value);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_del(b)", rc);
err = mdbx_del(txn_guard.get(), dbi, &b_key->value, &b_data->value);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at delete-b due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
break;
}
failure_perror("mdbx_del(b)", err);
}
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
break;
}
commited_serial = a_serial;
txn_nops = 0;
}
@ -150,26 +202,48 @@ bool testcase_hill::run() {
generate_pair(a_serial, a_key, a_data_0, 0);
generate_pair(a_serial, a_key, a_data_1, age_shift);
checkdata("downhill: update-a", dbi, a_key->value, a_data_0->value);
int rc = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_1->value,
&a_data_0->value, update_flags);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_put(update-a: 0->1)", rc);
err = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_1->value,
&a_data_0->value, update_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at update-a due '%s'",
mdbx_strerror(err));
txn_end(true);
break;
}
failure_perror("mdbx_put(update-a: 0->1)", err);
}
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
break;
}
txn_nops = 0;
}
// создаем вторую запись из пары
log_trace("downhill: insert-b %" PRIu64, b_serial);
generate_pair(b_serial, b_key, b_data, 0);
rc = mdbx_put(txn_guard.get(), dbi, &b_key->value, &b_data->value,
insert_flags);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_put(insert-b)", rc);
err = mdbx_put(txn_guard.get(), dbi, &b_key->value, &b_data->value,
insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at insert-a due '%s'",
mdbx_strerror(err));
txn_end(true);
break;
}
failure_perror("mdbx_put(insert-b)", err);
}
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
break;
}
txn_nops = 0;
}
@ -177,24 +251,46 @@ bool testcase_hill::run() {
log_trace("downhill: delete-a (age %" PRIu64 ") %" PRIu64, age_shift,
a_serial);
checkdata("downhill: delete-a", dbi, a_key->value, a_data_1->value);
rc = mdbx_del(txn_guard.get(), dbi, &a_key->value, &a_data_1->value);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_del(a)", rc);
err = mdbx_del(txn_guard.get(), dbi, &a_key->value, &a_data_1->value);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at delete-a due '%s'",
mdbx_strerror(err));
txn_end(true);
break;
}
failure_perror("mdbx_del(a)", err);
}
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
break;
}
txn_nops = 0;
}
// удаляем вторую запись
log_trace("downhill: delete-b %" PRIu64, b_serial);
checkdata("downhill: delete-b", dbi, b_key->value, b_data->value);
rc = mdbx_del(txn_guard.get(), dbi, &b_key->value, &b_data->value);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_del(b)", rc);
err = mdbx_del(txn_guard.get(), dbi, &b_key->value, &b_data->value);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at delete-b due '%s'",
mdbx_strerror(err));
txn_end(true);
break;
}
failure_perror("mdbx_del(b)", err);
}
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
break;
}
txn_nops = 0;
}
@ -208,7 +304,11 @@ bool testcase_hill::run() {
if (config.params.drop_table && !mode_readonly()) {
txn_begin(false);
db_table_drop(dbi);
txn_end(false);
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("hill: bailout-clean due '%s'", mdbx_strerror(err));
return true;
}
} else
db_table_close(dbi);
}

View File

@ -70,6 +70,7 @@ void actor_params::set_defaults(const std::string &tmpdir) {
inject_writefaultn = 0;
drop_table = false;
ignore_dbfull = false;
max_readers = 42;
max_tables = 42;
@ -288,6 +289,9 @@ int main(int argc, char *const argv[]) {
continue;
if (config::parse_option(argc, argv, narg, "drop", params.drop_table))
continue;
if (config::parse_option(argc, argv, narg, "ignore-dbfull",
params.ignore_dbfull))
continue;
if (config::parse_option(argc, argv, narg, "dump-config",
global::config::dump_config))
continue;

View File

@ -170,20 +170,42 @@ void testcase::txn_begin(bool readonly, unsigned flags) {
flags);
}
int testcase::breakable_commit() {
int rc = MDBX_SUCCESS;
log_trace(">> txn_commit");
assert(txn_guard);
MDBX_txn *txn = txn_guard.release();
txn_inject_writefault(txn);
int err = mdbx_txn_commit(txn);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
rc = err;
err = mdbx_txn_abort(txn);
if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH))
failure_perror("mdbx_txn_abort()", err);
} else
failure_perror("mdbx_txn_commit()", err);
}
log_trace("<< txn_commit: %s", rc ? "failed" : "Ok");
return rc;
}
void testcase::txn_end(bool abort) {
log_trace(">> txn_end(%s)", abort ? "abort" : "commit");
assert(txn_guard);
MDBX_txn *txn = txn_guard.release();
if (abort) {
int rc = mdbx_txn_abort(txn);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_txn_abort()", rc);
int err = mdbx_txn_abort(txn);
if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH))
failure_perror("mdbx_txn_abort()", err);
} else {
txn_inject_writefault(txn);
int rc = mdbx_txn_commit(txn);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_txn_commit()", rc);
int err = mdbx_txn_commit(txn);
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_txn_commit()", err);
}
log_trace("<< txn_end(%s)", abort ? "abort" : "commit");
@ -211,6 +233,16 @@ void testcase::cursor_close() {
log_trace("<< cursor_close()");
}
int testcase::breakable_restart() {
int rc = MDBX_SUCCESS;
if (txn_guard)
rc = breakable_commit();
if (cursor_guard)
cursor_close();
txn_begin(false, 0);
return rc;
}
void testcase::txn_restart(bool abort, bool readonly, unsigned flags) {
if (txn_guard)
txn_end(abort);
@ -394,6 +426,28 @@ void testcase::update_canary(uint64_t increment) {
log_trace("<< update_canary: sequence = %" PRIu64, canary_now.y);
}
int testcase::db_open__begin__table_create_open_clean(MDBX_dbi &dbi) {
db_open();
int err, retry_left = 42;
for (;;) {
txn_begin(false);
dbi = db_table_open(true);
db_table_clear(dbi);
err = breakable_commit();
if (likely(err == MDBX_SUCCESS)) {
txn_begin(false);
return MDBX_SUCCESS;
}
if (--retry_left == 0)
break;
jitter_delay(true);
}
log_notice("db_begin_table_create_open_clean: bailout due '%s'",
mdbx_strerror(err));
return err;
}
MDBX_dbi testcase::db_table_open(bool create) {
log_trace(">> testcase::db_table_create");

View File

@ -105,7 +105,9 @@ protected:
void db_open();
void db_close();
void txn_begin(bool readonly, unsigned flags = 0);
int breakable_commit();
void txn_end(bool abort);
int breakable_restart();
void txn_restart(bool abort, bool readonly, unsigned flags = 0);
void cursor_open(unsigned dbi);
void cursor_close();
@ -121,6 +123,7 @@ protected:
void db_table_drop(MDBX_dbi handle);
void db_table_clear(MDBX_dbi handle);
void db_table_close(MDBX_dbi handle);
int db_open__begin__table_create_open_clean(MDBX_dbi &dbi);
bool wait4start();
void report(size_t nops_done);

View File

@ -29,12 +29,12 @@ static unsigned edge2count(uint64_t edge, unsigned count_max) {
}
bool testcase_ttl::run() {
db_open();
txn_begin(false);
MDBX_dbi dbi = db_table_open(true);
db_table_clear(dbi);
txn_end(false);
MDBX_dbi dbi;
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("ttl: bailout-prepare due '%s'", mdbx_strerror(err));
return true;
}
/* LY: тест "эмуляцией time-to-live":
* - организуется "скользящее окно", которое двигается вперед вдоль
@ -73,12 +73,10 @@ bool testcase_ttl::run() {
std::deque<std::pair<uint64_t, unsigned>> fifo;
uint64_t serial = 0;
while (should_continue()) {
if (!txn_guard)
txn_begin(false);
const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */;
const unsigned window_width = edge2window(salt, window_max);
const unsigned head_count = edge2count(salt, count_max);
unsigned head_count = edge2count(salt, count_max);
log_info("ttl: step #%zu (serial %" PRIu64
", window %u, count %u) salt %" PRIu64,
nops_completed, serial, window_width, head_count, salt);
@ -93,9 +91,14 @@ bool testcase_ttl::run() {
for (unsigned n = 0; n < tail_count; ++n) {
log_trace("ttl: remove-tail %" PRIu64, serial);
generate_pair(tail_serial);
int err = mdbx_del(txn_guard.get(), dbi, &key->value, &data->value);
if (unlikely(err != MDBX_SUCCESS))
err = mdbx_del(txn_guard.get(), dbi, &key->value, &data->value);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("ttl: tail-bailout due '%s'", mdbx_strerror(err));
goto bailout;
}
failure_perror("mdbx_del(tail)", err);
}
if (unlikely(!keyvalue_maker.increment(tail_serial, 1)))
failure("ttl: unexpected key-space overflow on the tail");
}
@ -106,30 +109,52 @@ bool testcase_ttl::run() {
fifo.clear();
}
txn_restart(false, false);
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("ttl: bailout at commit due '%s'", mdbx_strerror(err));
break;
}
fifo.push_front(std::make_pair(serial, head_count));
retry:
for (unsigned n = 0; n < head_count; ++n) {
log_trace("ttl: insert-head %" PRIu64, serial);
generate_pair(serial);
int err = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value,
insert_flags);
if (unlikely(err != MDBX_SUCCESS))
err = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value,
insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("ttl: head-insert skip due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial = fifo.front().first;
fifo.front().second = head_count = n;
goto retry;
}
failure_perror("mdbx_put(head)", err);
}
if (unlikely(!keyvalue_maker.increment(serial, 1)))
failure("uphill: unexpected key-space overflow");
}
txn_end(false);
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("ttl: head-commit skip due '%s'", mdbx_strerror(err));
serial = fifo.front().first;
fifo.pop_front();
}
report(1);
}
bailout:
txn_end(true);
if (dbi) {
if (config.params.drop_table && !mode_readonly()) {
txn_begin(false);
db_table_drop(dbi);
txn_end(false);
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("ttl: bailout-clean due '%s'", mdbx_strerror(err));
return true;
}
} else
db_table_close(dbi);
}