mdbx-test: add verifying by speculum option.

Change-Id: I3d689234d0214f93f27dd2581bd9d0658692c0c9
This commit is contained in:
Leonid Yuriev 2019-10-09 17:56:59 +03:00
parent 1573d110f6
commit 66430fd10d
7 changed files with 223 additions and 210 deletions

View File

@ -416,6 +416,9 @@ void dump(const char *title) {
log_verbose("drop table: %s\n", i->params.drop_table ? "Yes" : "No");
log_verbose("ignore MDBX_MAP_FULL error: %s\n",
i->params.ignore_dbfull ? "Yes" : "No");
log_verbose("verifying by speculum: %s\n",
i->params.speculum ? "Yes" : "No");
indent.pop();
}

View File

@ -266,6 +266,7 @@ struct actor_params_pod {
bool drop_table;
bool ignore_dbfull;
bool speculum;
};
struct actor_config_pod {

View File

@ -14,131 +14,14 @@
#include "test.h"
int testcase_hill::insert(const keygen::buffer &akey,
const keygen::buffer &adata, unsigned flags) {
int err = mdbx_put(txn_guard.get(), dbi, &akey->value, &adata->value, flags);
if (err == MDBX_SUCCESS) {
const auto S_key = S(akey);
const auto S_data = S(adata);
const bool inserted = mirror.emplace(S_key, S_data).second;
assert(inserted);
(void)inserted;
}
return err;
}
int testcase_hill::replace(const keygen::buffer &akey,
const keygen::buffer &new_data,
const keygen::buffer &old_data, unsigned flags) {
const auto S_key = S(akey);
const auto S_old = S(old_data);
const auto S_new = S(new_data);
const auto removed = mirror.erase(set::key_type(S_key, S_old));
assert(removed == 1);
(void)removed;
const bool inserted = mirror.emplace(S_key, S_new).second;
assert(inserted);
(void)inserted;
return mdbx_replace(txn_guard.get(), dbi, &akey->value, &new_data->value,
&old_data->value, flags);
}
int testcase_hill::remove(const keygen::buffer &akey,
const keygen::buffer &adata) {
const auto S_key = S(akey);
const auto S_data = S(adata);
const auto removed = mirror.erase(set::key_type(S_key, S_data));
assert(removed == 1);
(void)removed;
return mdbx_del(txn_guard.get(), dbi, &akey->value, &adata->value);
}
bool testcase_hill::verify() const {
char dump_key[128], dump_value[128];
char dump_mkey[128], dump_mvalue[128];
MDBX_cursor *cursor;
int err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor);
if (err != MDBX_SUCCESS)
failure_perror("mdbx_cursor_open()", err);
bool rc = true;
MDBX_val akey, avalue;
MDBX_val mkey, mvalue;
err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_FIRST);
assert(std::is_sorted(mirror.cbegin(), mirror.cend(), ItemCompare(this)));
auto it = mirror.cbegin();
while (true) {
if (err != MDBX_SUCCESS) {
akey.iov_len = avalue.iov_len = 0;
akey.iov_base = avalue.iov_base = nullptr;
}
const auto S_key = S(akey);
const auto S_data = S(avalue);
if (it != mirror.cend()) {
mkey.iov_base = (void *)it->first.c_str();
mkey.iov_len = it->first.size();
mvalue.iov_base = (void *)it->second.c_str();
mvalue.iov_len = it->second.size();
}
if (err == MDBX_SUCCESS && it != mirror.cend() && S_key == it->first &&
S_data == it->second) {
++it;
err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT);
} else if (err == MDBX_SUCCESS &&
(it == mirror.cend() || S_key < it->first ||
(S_key == it->first && S_data < it->second))) {
if (it != mirror.cend()) {
log_error("extra pair: db{%s, %s} < mi{%s, %s}",
mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)),
mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
} else {
log_error("extra pair: db{%s, %s} < mi.END",
mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)));
}
err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT);
rc = false;
} else if (it != mirror.cend() &&
(err == MDBX_NOTFOUND || S_key > it->first ||
(S_key == it->first && S_data > it->second))) {
if (err == MDBX_NOTFOUND) {
log_error("lost pair: db.END > mi{%s, %s}",
mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
} else {
log_error("lost pair: db{%s, %s} > mi{%s, %s}",
mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)),
mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
}
++it;
rc = false;
} else if (err == MDBX_NOTFOUND && it == mirror.cend()) {
break;
} else if (err != MDBX_SUCCESS) {
failure_perror("mdbx_cursor_get()", err);
} else {
assert(!"WTF?");
}
}
mdbx_cursor_close(cursor);
return rc;
}
bool testcase_hill::run() {
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("hill: bailout-prepare due '%s'", mdbx_strerror(err));
return true;
}
mirror.clear();
mirror_commited.clear();
speculum.clear();
speculum_commited.clear();
/* LY: тест "холмиком":
* - сначала наполняем таблицу циклическими CRUD-манипуляциями,
@ -204,12 +87,12 @@ bool testcase_hill::run() {
log_notice("uphill: bailout at insert-a due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_put(insert-a.1)", err);
}
if (!verify()) {
if (!speculum_verify()) {
log_notice("uphill: bailout after insert-a, before commit");
goto bailout;
}
@ -219,13 +102,13 @@ bool testcase_hill::run() {
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
mirror_commited = mirror;
speculum_commited = speculum;
commited_serial = a_serial;
txn_nops = 0;
if (!verify()) {
if (!speculum_verify()) {
log_notice("uphill: bailout after insert-a, after commit");
goto bailout;
}
@ -240,12 +123,12 @@ bool testcase_hill::run() {
log_notice("uphill: bailout at insert-b due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_put(insert-b)", err);
}
if (!verify()) {
if (!speculum_verify()) {
log_notice("uphill: bailout after insert-b, before commit");
goto bailout;
}
@ -255,13 +138,13 @@ bool testcase_hill::run() {
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
mirror_commited = mirror;
speculum_commited = speculum;
commited_serial = a_serial;
txn_nops = 0;
if (!verify()) {
if (!speculum_verify()) {
log_notice("uphill: bailout after insert-b, after commit");
goto bailout;
}
@ -278,12 +161,12 @@ bool testcase_hill::run() {
log_notice("uphill: bailout at update-a due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_replace(update-a: 1->0)", err);
}
if (!verify()) {
if (!speculum_verify()) {
log_notice("uphill: bailout after update-a, before commit");
goto bailout;
}
@ -293,13 +176,13 @@ bool testcase_hill::run() {
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
mirror_commited = mirror;
speculum_commited = speculum;
commited_serial = a_serial;
txn_nops = 0;
if (!verify()) {
if (!speculum_verify()) {
log_notice("uphill: bailout after update-a, after commit");
goto bailout;
}
@ -314,12 +197,12 @@ bool testcase_hill::run() {
log_notice("uphill: bailout at delete-b due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_del(b)", err);
}
if (!verify()) {
if (!speculum_verify()) {
log_notice("uphill: bailout after delete-b, before commit");
goto bailout;
}
@ -329,13 +212,13 @@ bool testcase_hill::run() {
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
mirror_commited = mirror;
speculum_commited = speculum;
commited_serial = a_serial;
txn_nops = 0;
if (!verify()) {
if (!speculum_verify()) {
log_notice("uphill: bailout after delete-b, after commit");
goto bailout;
}
@ -371,12 +254,12 @@ bool testcase_hill::run() {
log_notice("downhill: bailout at update-a due '%s'",
mdbx_strerror(err));
txn_end(true);
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_put(update-a: 0->1)", err);
}
if (!verify()) {
if (!speculum_verify()) {
log_notice("downhill: bailout after update-a, before commit");
break;
}
@ -385,12 +268,12 @@ bool testcase_hill::run() {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
mirror_commited = mirror;
speculum_commited = speculum;
txn_nops = 0;
if (!verify()) {
if (!speculum_verify()) {
log_notice("downhill: bailout after update-a, after commit");
break;
}
@ -405,12 +288,12 @@ bool testcase_hill::run() {
log_notice("downhill: bailout at insert-a due '%s'",
mdbx_strerror(err));
txn_end(true);
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_put(insert-b)", err);
}
if (!verify()) {
if (!speculum_verify()) {
log_notice("downhill: bailout after insert-b, before commit");
break;
}
@ -419,12 +302,12 @@ bool testcase_hill::run() {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
mirror_commited = mirror;
speculum_commited = speculum;
txn_nops = 0;
if (!verify()) {
if (!speculum_verify()) {
log_notice("downhill: bailout after insert-b, after commit");
break;
}
@ -440,12 +323,12 @@ bool testcase_hill::run() {
log_notice("downhill: bailout at delete-a due '%s'",
mdbx_strerror(err));
txn_end(true);
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_del(a)", err);
}
if (!verify()) {
if (!speculum_verify()) {
log_notice("downhill: bailout after delete-a, before commit");
break;
}
@ -454,12 +337,12 @@ bool testcase_hill::run() {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
mirror_commited = mirror;
speculum_commited = speculum;
txn_nops = 0;
if (!verify()) {
if (!speculum_verify()) {
log_notice("downhill: bailout after delete-a, after commit");
break;
}
@ -474,12 +357,12 @@ bool testcase_hill::run() {
log_notice("downhill: bailout at delete-b due '%s'",
mdbx_strerror(err));
txn_end(true);
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_del(b)", err);
}
if (!verify()) {
if (!speculum_verify()) {
log_notice("downhill: bailout after delete-b, before commit");
break;
}
@ -488,12 +371,12 @@ bool testcase_hill::run() {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
mirror = mirror_commited;
speculum = speculum_commited;
break;
}
mirror_commited = mirror;
speculum_commited = speculum;
txn_nops = 0;
if (!verify()) {
if (!speculum_verify()) {
log_notice("downhill: bailout after delete-b, after commit");
goto bailout;
}
@ -502,7 +385,7 @@ bool testcase_hill::run() {
report(1);
}
rc = verify();
rc = speculum_verify();
bailout:
if (txn_guard) {
err = breakable_commit();

View File

@ -71,6 +71,7 @@ void actor_params::set_defaults(const std::string &tmpdir) {
drop_table = false;
ignore_dbfull = false;
speculum = false;
max_readers = 42;
max_tables = 42;
@ -296,6 +297,8 @@ int main(int argc, char *const argv[]) {
if (config::parse_option(argc, argv, narg, "ignore-dbfull",
params.ignore_dbfull))
continue;
if (config::parse_option(argc, argv, narg, "speculum", params.speculum))
continue;
if (config::parse_option(argc, argv, narg, "dump-config",
global::config::dump_config))
continue;

View File

@ -588,3 +588,128 @@ bool test_execute(const actor_config &config_const) {
return false;
}
}
//-----------------------------------------------------------------------------
int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata,
unsigned flags) {
int err = mdbx_put(txn_guard.get(), dbi, &akey->value, &adata->value, flags);
if (err == MDBX_SUCCESS && config.params.speculum) {
const auto S_key = S(akey);
const auto S_data = S(adata);
const bool inserted = speculum.emplace(S_key, S_data).second;
assert(inserted);
(void)inserted;
}
return err;
}
int testcase::replace(const keygen::buffer &akey,
const keygen::buffer &new_data,
const keygen::buffer &old_data, unsigned flags) {
if (config.params.speculum) {
const auto S_key = S(akey);
const auto S_old = S(old_data);
const auto S_new = S(new_data);
const auto removed = speculum.erase(SET::key_type(S_key, S_old));
assert(removed == 1);
(void)removed;
const bool inserted = speculum.emplace(S_key, S_new).second;
assert(inserted);
(void)inserted;
}
return mdbx_replace(txn_guard.get(), dbi, &akey->value, &new_data->value,
&old_data->value, flags);
}
int testcase::remove(const keygen::buffer &akey, const keygen::buffer &adata) {
if (config.params.speculum) {
const auto S_key = S(akey);
const auto S_data = S(adata);
const auto removed = speculum.erase(SET::key_type(S_key, S_data));
assert(removed == 1);
(void)removed;
}
return mdbx_del(txn_guard.get(), dbi, &akey->value, &adata->value);
}
bool testcase::speculum_verify() const {
if (!config.params.speculum)
return true;
char dump_key[128], dump_value[128];
char dump_mkey[128], dump_mvalue[128];
MDBX_cursor *cursor;
int err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor);
if (err != MDBX_SUCCESS)
failure_perror("mdbx_cursor_open()", err);
bool rc = true;
MDBX_val akey, avalue;
MDBX_val mkey, mvalue;
err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_FIRST);
assert(std::is_sorted(speculum.cbegin(), speculum.cend(), ItemCompare(this)));
auto it = speculum.cbegin();
while (true) {
if (err != MDBX_SUCCESS) {
akey.iov_len = avalue.iov_len = 0;
akey.iov_base = avalue.iov_base = nullptr;
}
const auto S_key = S(akey);
const auto S_data = S(avalue);
if (it != speculum.cend()) {
mkey.iov_base = (void *)it->first.c_str();
mkey.iov_len = it->first.size();
mvalue.iov_base = (void *)it->second.c_str();
mvalue.iov_len = it->second.size();
}
if (err == MDBX_SUCCESS && it != speculum.cend() && S_key == it->first &&
S_data == it->second) {
++it;
err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT);
} else if (err == MDBX_SUCCESS &&
(it == speculum.cend() || S_key < it->first ||
(S_key == it->first && S_data < it->second))) {
if (it != speculum.cend()) {
log_error("extra pair: db{%s, %s} < mi{%s, %s}",
mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)),
mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
} else {
log_error("extra pair: db{%s, %s} < mi.END",
mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)));
}
err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT);
rc = false;
} else if (it != speculum.cend() &&
(err == MDBX_NOTFOUND || S_key > it->first ||
(S_key == it->first && S_data > it->second))) {
if (err == MDBX_NOTFOUND) {
log_error("lost pair: db.END > mi{%s, %s}",
mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
} else {
log_error("lost pair: db{%s, %s} > mi{%s, %s}",
mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)),
mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
}
++it;
rc = false;
} else if (err == MDBX_NOTFOUND && it == speculum.cend()) {
break;
} else if (err != MDBX_SUCCESS) {
failure_perror("mdbx_cursor_get()", err);
} else {
assert(!"WTF?");
}
}
mdbx_cursor_close(cursor);
return rc;
}

View File

@ -95,9 +95,45 @@ typedef std::unique_ptr<MDBX_cursor, cursor_deleter> scoped_cursor_guard;
class testcase {
protected:
#if HAVE_cxx17_std_string_view
using data_view = std::string_view;
#else
using data_view = std::string;
#endif
static inline data_view S(const MDBX_val &v) {
return data_view(static_cast<const char *>(v.iov_base), v.iov_len);
}
static inline data_view S(const keygen::buffer &b) { return S(b->value); }
using Item = std::pair<std::string, std::string>;
struct ItemCompare {
const testcase *context;
ItemCompare(const testcase *owner) : context(owner) {}
bool operator()(const Item &a, const Item &b) const {
MDBX_val va, vb;
va.iov_base = (void *)a.first.data();
va.iov_len = a.first.size();
vb.iov_base = (void *)b.first.data();
vb.iov_len = b.first.size();
int cmp = mdbx_cmp(context->txn_guard.get(), context->dbi, &va, &vb);
if (cmp == 0 &&
(context->config.params.table_flags & MDBX_DUPSORT) != 0) {
va.iov_base = (void *)a.second.data();
va.iov_len = a.second.size();
vb.iov_base = (void *)b.second.data();
vb.iov_len = b.second.size();
cmp = mdbx_dcmp(context->txn_guard.get(), context->dbi, &va, &vb);
}
return cmp < 0;
}
};
using SET = std::set<Item, ItemCompare>;
const actor_config &config;
const mdbx_pid_t pid;
MDBX_dbi dbi;
scoped_db_guard db_guard;
scoped_txn_guard txn_guard;
scoped_cursor_guard cursor_guard;
@ -113,6 +149,14 @@ protected:
mdbx_canary canary;
} last;
SET speculum;
bool speculum_verify() const;
int insert(const keygen::buffer &akey, const keygen::buffer &adata,
unsigned flags);
int replace(const keygen::buffer &akey, const keygen::buffer &new_value,
const keygen::buffer &old_value, unsigned flags);
int remove(const keygen::buffer &akey, const keygen::buffer &adata);
static int oom_callback(MDBX_env *env, mdbx_pid_t pid, mdbx_tid_t tid,
uint64_t txn, unsigned gap, size_t space, int retry);
@ -160,7 +204,8 @@ protected:
public:
testcase(const actor_config &config, const mdbx_pid_t pid)
: config(config), pid(pid), signalled(false), nops_completed(0) {
: config(config), pid(pid), signalled(false), nops_completed(0),
speculum(ItemCompare(this)) {
start_timestamp.reset();
memset(&last, 0, sizeof(last));
}
@ -181,57 +226,11 @@ public:
class testcase_hill : public testcase {
using inherited = testcase;
#if HAVE_cxx17_std_string_view
using data_view = std::string_view;
#else
using data_view = std::string;
#endif
MDBX_dbi dbi;
using Item = std::pair<std::string, std::string>;
struct ItemCompare {
const testcase_hill *context;
ItemCompare(const testcase_hill *owner) : context(owner) {}
bool operator()(const Item &a, const Item &b) const {
MDBX_val va, vb;
va.iov_base = (void *)a.first.data();
va.iov_len = a.first.size();
vb.iov_base = (void *)b.first.data();
vb.iov_len = b.first.size();
int cmp = mdbx_cmp(context->txn_guard.get(), context->dbi, &va, &vb);
if (cmp == 0 &&
(context->config.params.table_flags & MDBX_DUPSORT) != 0) {
va.iov_base = (void *)a.second.data();
va.iov_len = a.second.size();
vb.iov_base = (void *)b.second.data();
vb.iov_len = b.second.size();
cmp = mdbx_dcmp(context->txn_guard.get(), context->dbi, &va, &vb);
}
return cmp < 0;
}
};
using set = std::set<Item, ItemCompare>;
set mirror, mirror_commited;
bool verify() const;
int insert(const keygen::buffer &akey, const keygen::buffer &adata,
unsigned flags);
int replace(const keygen::buffer &akey, const keygen::buffer &new_value,
const keygen::buffer &old_value, unsigned flags);
int remove(const keygen::buffer &akey, const keygen::buffer &adata);
static inline data_view S(const MDBX_val &v) {
return data_view(static_cast<const char *>(v.iov_base), v.iov_len);
}
static inline data_view S(const keygen::buffer &b) { return S(b->value); }
SET speculum_commited;
public:
testcase_hill(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid), mirror(ItemCompare(this)),
mirror_commited(ItemCompare(this)) {}
: testcase(config, pid), speculum_commited(ItemCompare(this)) {}
bool run() override;
};

View File

@ -29,7 +29,6 @@ static unsigned edge2count(uint64_t edge, unsigned count_max) {
}
bool testcase_ttl::run() {
MDBX_dbi dbi;
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("ttl: bailout-prepare due '%s'", mdbx_strerror(err));