mdbx-test: extending speculum mode for cursors tracking verification.

Change-Id: I44786efcee6feb1c7d414c925717d08ed9d94e20
This commit is contained in:
Leonid Yuriev 2020-11-16 06:52:48 +03:00
parent 149b3d09e6
commit 7cdbe1badb
4 changed files with 392 additions and 34 deletions

View File

@ -238,6 +238,7 @@ datalen
DATANAME
DATASIGN
datasync
dataview
datetime
DBC
dbenv

View File

@ -645,29 +645,295 @@ bool test_execute(const actor_config &config_const) {
//-----------------------------------------------------------------------------
enum speculum_cursors : int {
lowerbound = 0,
prev = 1,
prev_prev = 2,
next = 3,
next_next = 4
};
bool testcase::is_same(const Item &a, const Item &b) const {
if (!is_samedata(dataview2iov(a.first), dataview2iov(b.first)))
return false;
if ((config.params.table_flags & MDBX_DUPSORT) &&
!is_samedata(dataview2iov(a.second), dataview2iov(b.second)))
return false;
return true;
}
bool testcase::is_same(const testcase::SET::const_iterator &it,
const MDBX_val &k, const MDBX_val &v) const {
return is_samedata(dataview2iov(it->first), k) &&
is_samedata(dataview2iov(it->second), v);
}
void testcase::verbose(const char *where, const char *stage,
const testcase::SET::const_iterator &it) const {
if (it == speculum.end())
log_verbose("speculum-%s: %s expect END", where, stage);
else {
char dump_key[32], dump_value[32];
MDBX_val it_key = dataview2iov(it->first);
MDBX_val it_data = dataview2iov(it->second);
log_verbose("speculum-%s: %s expect {%s, %s}", where, stage,
mdbx_dump_val(&it_key, dump_key, sizeof(dump_key)),
mdbx_dump_val(&it_data, dump_value, sizeof(dump_value)));
}
}
void testcase::verbose(const char *where, const char *stage,
const MDBX_val &key, const MDBX_val &data,
int err) const {
char dump_key[32], dump_value[32];
if (err != MDBX_SUCCESS)
log_verbose("speculum-%s: %s cursor {%d, %s}", where, stage, err,
mdbx_strerror(err));
else
log_verbose("speculum-%s: %s cursor {%s, %s}", where, stage,
mdbx_dump_val(&key, dump_key, sizeof(dump_key)),
mdbx_dump_val(&data, dump_value, sizeof(dump_value)));
}
void testcase::speculum_check_iterator(const char *where, const char *stage,
const testcase::SET::const_iterator &it,
const MDBX_val &key,
const MDBX_val &data) const {
char dump_key[32], dump_value[32];
MDBX_val it_key = dataview2iov(it->first);
MDBX_val it_data = dataview2iov(it->second);
// log_verbose("speculum-%s: %s expect {%s, %s}", where, stage,
// mdbx_dump_val(&it_key, dump_key, sizeof(dump_key)),
// mdbx_dump_val(&it_data, dump_value, sizeof(dump_value)));
if (!is_samedata(it_key, key))
failure("speculum-%s: %s key mismatch %s (must) != %s", where, stage,
mdbx_dump_val(&it_key, dump_key, sizeof(dump_key)),
mdbx_dump_val(&key, dump_value, sizeof(dump_value)));
if (!is_samedata(it_data, data))
failure("speculum-%s: %s data mismatch %s (must) != %s", where, stage,
mdbx_dump_val(&it_data, dump_key, sizeof(dump_key)),
mdbx_dump_val(&data, dump_value, sizeof(dump_value)));
}
void testcase::speculum_check_cursor(const char *where, const char *stage,
const testcase::SET::const_iterator &it,
int cursor_err, const MDBX_val &cursor_key,
const MDBX_val &cursor_data) const {
// verbose(where, stage, cursor_key, cursor_data, cursor_err);
// verbose(where, stage, it);
if (cursor_err != MDBX_SUCCESS && cursor_err != MDBX_NOTFOUND)
failure("speculum-%s: %s %s %d %s", where, stage, "cursor-get", cursor_err,
mdbx_strerror(cursor_err));
char dump_key[32], dump_value[32];
if (it == speculum.end() && cursor_err != MDBX_NOTFOUND)
failure("speculum-%s: %s extra pair {%s, %s}", where, stage,
mdbx_dump_val(&cursor_key, dump_key, sizeof(dump_key)),
mdbx_dump_val(&cursor_data, dump_value, sizeof(dump_value)));
else if (it != speculum.end() && cursor_err == MDBX_NOTFOUND) {
MDBX_val it_key = dataview2iov(it->first);
MDBX_val it_data = dataview2iov(it->second);
failure("speculum-%s: %s lack pair {%s, %s}", where, stage,
mdbx_dump_val(&it_key, dump_key, sizeof(dump_key)),
mdbx_dump_val(&it_data, dump_value, sizeof(dump_value)));
} else if (cursor_err == MDBX_SUCCESS)
speculum_check_iterator(where, stage, it, cursor_key, cursor_data);
}
void testcase::speculum_check_cursor(const char *where, const char *stage,
const testcase::SET::const_iterator &it,
MDBX_cursor *cursor,
const MDBX_cursor_op op) const {
MDBX_val cursor_key = {};
MDBX_val cursor_data = {};
int err;
if (std::next(it) == speculum.end() && op == MDBX_PREV &&
(config.params.table_flags & MDBX_DUPSORT)) {
/* Workaround for MDBX/LMDB flaw */
err = mdbx_cursor_get(cursor, &cursor_key, &cursor_data, MDBX_LAST);
if (err == MDBX_SUCCESS)
err = mdbx_cursor_get(cursor, &cursor_key, &cursor_data, MDBX_LAST_DUP);
} else
err = mdbx_cursor_get(cursor, &cursor_key, &cursor_data, op);
return speculum_check_cursor(where, stage, it, err, cursor_key, cursor_data);
}
void testcase::speculum_prepare_cursors(const Item &item) {
int err;
assert(config.params.speculum);
if (speculum_cursors[lowerbound])
for (auto &guard : speculum_cursors) {
if (txn_guard.get() != mdbx_cursor_txn(guard.get()) ||
dbi != mdbx_cursor_dbi(guard.get())) {
err = mdbx_cursor_bind(txn_guard.get(), guard.get(), dbi);
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_cursor_bind()", err);
}
}
else
for (auto &guard : speculum_cursors) {
MDBX_cursor *cursor = nullptr;
err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor);
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_cursor_open()", err);
guard.reset(cursor);
}
const auto cursor_lowerbound = speculum_cursors[lowerbound].get();
const MDBX_val item_key = dataview2iov(item.first),
item_data = dataview2iov(item.second);
MDBX_val lowerbound_key = item_key;
MDBX_val lowerbound_data = item_data;
// verbose("prepare-cursors", "item", item_key, item_data);
err = mdbx_cursor_get(cursor_lowerbound, &lowerbound_key, &lowerbound_data,
MDBX_SET_RANGE);
if (err == MDBX_SUCCESS && (config.params.table_flags & MDBX_DUPSORT) &&
mdbx_cmp(txn_guard.get(), dbi, &lowerbound_key, &item_key) == 0) {
lowerbound_data = item_data;
err = mdbx_cursor_get(cursor_lowerbound, &lowerbound_key, &lowerbound_data,
MDBX_GET_BOTH_RANGE);
if (err == MDBX_NOTFOUND)
err = mdbx_cursor_get(cursor_lowerbound, &lowerbound_key,
&lowerbound_data, MDBX_NEXT_NODUP);
}
// verbose("prepare-cursors", "lowerbound", lowerbound_key, lowerbound_data,
// err);
auto it_lowerbound = speculum.lower_bound(item);
// verbose("prepare-cursors", "lowerbound", it_lowerbound);
speculum_check_cursor("prepare-cursors", "lowerbound", it_lowerbound, err,
lowerbound_key, lowerbound_data);
const auto cursor_prev = speculum_cursors[prev].get();
err = mdbx_cursor_copy(cursor_lowerbound, cursor_prev);
if (unlikely(err != MDBX_SUCCESS))
failure("speculum-%s: %s %s %d %s", "prepare-cursors", "prev",
"cursor-copy", err, mdbx_strerror(err));
auto it_prev = it_lowerbound;
if (it_prev != speculum.begin()) {
speculum_check_cursor("prepare-cursors", "prev", --it_prev, cursor_prev,
MDBX_PREV);
} else if ((err = mdbx_cursor_on_first(cursor_prev)) != MDBX_RESULT_TRUE)
failure("speculum-%s: %s on-first %d %s", "prepare-cursors", "prev", err,
mdbx_strerror(err));
const auto cursor_prev_prev = speculum_cursors[prev_prev].get();
err = mdbx_cursor_copy(cursor_prev, cursor_prev_prev);
if (unlikely(err != MDBX_SUCCESS))
failure("speculum-%s: %s %s %d %s", "prepare-cursors", "prev-prev",
"cursor-copy", err, mdbx_strerror(err));
auto it_prev_prev = it_prev;
if (it_prev_prev != speculum.begin()) {
speculum_check_cursor("prepare-cursors", "prev-prev", --it_prev_prev,
cursor_prev_prev, MDBX_PREV);
} else if ((err = mdbx_cursor_on_first(cursor_prev_prev)) != MDBX_RESULT_TRUE)
failure("speculum-%s: %s on-first %d %s", "prepare-cursors", "prev-prev",
err, mdbx_strerror(err));
const auto cursor_next = speculum_cursors[next].get();
err = mdbx_cursor_copy(cursor_lowerbound, cursor_next);
if (unlikely(err != MDBX_SUCCESS))
failure("speculum-%s: %s %s %d %s", "prepare-cursors", "next",
"cursor-copy", err, mdbx_strerror(err));
auto it_next = it_lowerbound;
if (it_next != speculum.end()) {
speculum_check_cursor("prepare-cursors", "next", ++it_next, cursor_next,
MDBX_NEXT);
} else if ((err = mdbx_cursor_on_last(cursor_next)) != MDBX_RESULT_TRUE)
failure("speculum-%s: %s on-last %d %s", "prepare-cursors", "next", err,
mdbx_strerror(err));
const auto cursor_next_next = speculum_cursors[next_next].get();
err = mdbx_cursor_copy(cursor_next, cursor_next_next);
if (unlikely(err != MDBX_SUCCESS))
failure("speculum-%s: %s %s %d %s", "prepare-cursors", "next-next",
"cursor-copy", err, mdbx_strerror(err));
auto it_next_next = it_next;
if (it_next_next != speculum.end()) {
speculum_check_cursor("prepare-cursors", "next-next", ++it_next_next,
cursor_next_next, MDBX_NEXT);
} else if ((err = mdbx_cursor_on_last(cursor_next_next)) != MDBX_RESULT_TRUE)
failure("speculum-%s: %s on-last %d %s", "prepare-cursors", "next-next",
err, mdbx_strerror(err));
}
int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata,
MDBX_put_flags_t flags) {
int err = mdbx_put(txn_guard.get(), dbi, &akey->value, &adata->value, flags);
if (err == MDBX_SUCCESS && config.params.speculum) {
const auto S_key = S(akey);
const auto S_data = S(adata);
if (unlikely(!speculum.emplace(S_key, S_data).second)) {
char dump_key[128], dump_value[128];
log_error("speculum-insert: pair not inserted {%s, %s}",
int err;
bool rc = true;
Item item;
if (config.params.speculum) {
item.first = iov2dataview(akey);
item.second = iov2dataview(adata);
speculum_prepare_cursors(item);
}
err = mdbx_put(txn_guard.get(), dbi, &akey->value, &adata->value, flags);
if (err != MDBX_SUCCESS && err != MDBX_KEYEXIST)
return err;
if (config.params.speculum) {
char dump_key[32], dump_value[32];
const auto insertion_result = speculum.insert(item);
if (err == MDBX_KEYEXIST && insertion_result.second) {
log_error("speculum.insert: unexpected %s {%s, %s}", "MDBX_KEYEXIST",
mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)),
mdbx_dump_val(&adata->value, dump_value, sizeof(dump_value)));
rc = false;
}
if (err == MDBX_SUCCESS && !insertion_result.second) {
log_error("speculum.insert: unexpected %s {%s, %s}", "MDBX_SUCCESS",
mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)),
mdbx_dump_val(&adata->value, dump_value, sizeof(dump_value)));
rc = false;
}
if (insertion_result.first != speculum.begin()) {
const auto cursor_prev = speculum_cursors[prev].get();
auto it_prev = insertion_result.first;
speculum_check_cursor("after-insert", "prev", --it_prev, cursor_prev,
MDBX_GET_CURRENT);
if (it_prev != speculum.begin()) {
const auto cursor_prev_prev = speculum_cursors[prev_prev].get();
auto it_prev_prev = it_prev;
speculum_check_cursor("after-insert", "prev-prev", --it_prev_prev,
cursor_prev_prev, MDBX_GET_CURRENT);
}
}
auto it_lowerbound = insertion_result.first;
if (++it_lowerbound != speculum.end()) {
const auto cursor_lowerbound = speculum_cursors[lowerbound].get();
speculum_check_cursor("after-insert", "lowerbound", it_lowerbound,
cursor_lowerbound, MDBX_GET_CURRENT);
auto it_next = it_lowerbound;
if (++it_next != speculum.end()) {
const auto cursor_next = speculum_cursors[next].get();
speculum_check_cursor("after-insert", "next", it_next, cursor_next,
MDBX_GET_CURRENT);
auto it_next_next = it_next;
if (++it_next_next != speculum.end()) {
const auto cursor_next_next = speculum_cursors[next_next].get();
speculum_check_cursor("after-insert", "next-next", it_next_next,
cursor_next_next, MDBX_GET_CURRENT);
}
}
}
}
return err;
return rc ? MDBX_SUCCESS : MDBX_RESULT_TRUE;
}
int testcase::replace(const keygen::buffer &akey,
const keygen::buffer &new_data,
const keygen::buffer &old_data, MDBX_put_flags_t flags) {
if (config.params.speculum) {
const auto S_key = S(akey);
const auto S_old = S(old_data);
const auto S_new = S(new_data);
const auto S_key = iov2dataview(akey);
const auto S_old = iov2dataview(old_data);
const auto S_new = iov2dataview(new_data);
const auto removed = speculum.erase(SET::key_type(S_key, S_old));
if (unlikely(removed != 1)) {
char dump_key[128], dump_value[128];
@ -690,19 +956,82 @@ int testcase::replace(const keygen::buffer &akey,
}
int testcase::remove(const keygen::buffer &akey, const keygen::buffer &adata) {
int err;
bool rc = true;
Item item;
if (config.params.speculum) {
const auto S_key = S(akey);
const auto S_data = S(adata);
const auto removed = speculum.erase(SET::key_type(S_key, S_data));
if (unlikely(removed != 1)) {
char dump_key[128], dump_value[128];
log_error("speculum-%s: %s old value {%s, %s}", "remove",
(removed > 1) ? "multi" : "no",
mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)),
mdbx_dump_val(&adata->value, dump_value, sizeof(dump_value)));
item.first = iov2dataview(akey);
item.second = iov2dataview(adata);
speculum_prepare_cursors(item);
}
err = mdbx_del(txn_guard.get(), dbi, &akey->value, &adata->value);
if (err != MDBX_NOTFOUND && err != MDBX_SUCCESS)
return err;
if (config.params.speculum) {
char dump_key[32], dump_value[32];
const auto it_found = speculum.find(item);
if (it_found == speculum.end()) {
if (err != MDBX_NOTFOUND) {
log_error("speculum.remove: unexpected %s {%s, %s}", "MDBX_SUCCESS",
mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)),
mdbx_dump_val(&adata->value, dump_value, sizeof(dump_value)));
rc = false;
}
} else {
if (err != MDBX_SUCCESS) {
log_error("speculum.remove: unexpected %s {%s, %s}", "MDBX_NOTFOUND",
mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)),
mdbx_dump_val(&adata->value, dump_value, sizeof(dump_value)));
rc = false;
}
if (it_found != speculum.begin()) {
const auto cursor_prev = speculum_cursors[prev].get();
auto it_prev = it_found;
speculum_check_cursor("after-remove", "prev", --it_prev, cursor_prev,
MDBX_GET_CURRENT);
if (it_prev != speculum.begin()) {
const auto cursor_prev_prev = speculum_cursors[prev_prev].get();
auto it_prev_prev = it_prev;
speculum_check_cursor("after-remove", "prev-prev", --it_prev_prev,
cursor_prev_prev, MDBX_GET_CURRENT);
}
}
auto it_next = it_found;
const auto cursor_next = speculum_cursors[next].get();
const auto cursor_lowerbound = speculum_cursors[lowerbound].get();
if (++it_next != speculum.end()) {
speculum_check_cursor("after-remove", "next", it_next, cursor_next,
MDBX_GET_CURRENT);
speculum_check_cursor("after-remove", "lowerbound", it_next,
cursor_lowerbound, MDBX_NEXT);
auto it_next_next = it_next;
const auto cursor_next_next = speculum_cursors[next_next].get();
if (++it_next_next != speculum.end()) {
speculum_check_cursor("after-remove", "next-next", it_next_next,
cursor_next_next, MDBX_GET_CURRENT);
} else if ((err = mdbx_cursor_on_last(cursor_next_next)) !=
MDBX_RESULT_TRUE)
failure("speculum-%s: %s on-last %d %s", "after-remove", "next-next",
err, mdbx_strerror(err));
} else {
if ((err = mdbx_cursor_on_last(cursor_next)) != MDBX_RESULT_TRUE)
failure("speculum-%s: %s on-last %d %s", "after-remove", "next", err,
mdbx_strerror(err));
if ((err = mdbx_cursor_on_last(cursor_lowerbound)) != MDBX_RESULT_TRUE)
failure("speculum-%s: %s on-last %d %s", "after-remove", "lowerbound",
err, mdbx_strerror(err));
}
speculum.erase(it_found);
}
}
return mdbx_del(txn_guard.get(), dbi, &akey->value, &adata->value);
return rc ? MDBX_SUCCESS : MDBX_RESULT_TRUE;
}
bool testcase::speculum_verify() {
@ -733,8 +1062,8 @@ bool testcase::speculum_verify() {
akey.iov_len = avalue.iov_len = 0;
akey.iov_base = avalue.iov_base = nullptr;
}
const auto S_key = S(akey);
const auto S_data = S(avalue);
const auto S_key = iov2dataview(akey);
const auto S_data = iov2dataview(avalue);
if (it != speculum.cend()) {
mkey.iov_base = (void *)it->first.c_str();
mkey.iov_len = it->first.size();

View File

@ -103,31 +103,33 @@ protected:
#else
using data_view = std::string;
#endif
static inline data_view S(const MDBX_val &v) {
static inline data_view iov2dataview(const MDBX_val &v) {
return (v.iov_base && v.iov_len)
? data_view(static_cast<const char *>(v.iov_base), v.iov_len)
: data_view();
}
static inline data_view S(const keygen::buffer &b) { return S(b->value); }
static inline data_view iov2dataview(const keygen::buffer &b) {
return iov2dataview(b->value);
}
using Item = std::pair<std::string, std::string>;
static MDBX_val dataview2iov(const data_view &v) {
MDBX_val r;
r.iov_base = (void *)v.data();
r.iov_len = v.size();
return r;
}
struct ItemCompare {
const testcase *context;
ItemCompare(const testcase *owner) : context(owner) {}
bool operator()(const Item &a, const Item &b) const {
MDBX_val va, vb;
va.iov_base = (void *)a.first.data();
va.iov_len = a.first.size();
vb.iov_base = (void *)b.first.data();
vb.iov_len = b.first.size();
MDBX_val va = dataview2iov(a.first), vb = dataview2iov(b.first);
int cmp = mdbx_cmp(context->txn_guard.get(), context->dbi, &va, &vb);
if (cmp == 0 &&
(context->config.params.table_flags & MDBX_DUPSORT) != 0) {
va.iov_base = (void *)a.second.data();
va.iov_len = a.second.size();
vb.iov_base = (void *)b.second.data();
vb.iov_len = b.second.size();
va = dataview2iov(a.second);
vb = dataview2iov(b.second);
cmp = mdbx_dcmp(context->txn_guard.get(), context->dbi, &va, &vb);
}
return cmp < 0;
@ -159,6 +161,29 @@ protected:
} last;
SET speculum{ItemCompare(this)}, speculum_committed{ItemCompare(this)};
scoped_cursor_guard speculum_cursors[5];
void speculum_prepare_cursors(const Item &item);
void speculum_check_iterator(const char *where, const char *stage,
const testcase::SET::const_iterator &it,
const MDBX_val &key, const MDBX_val &data) const;
void speculum_check_cursor(const char *where, const char *stage,
const testcase::SET::const_iterator &it,
int cursor_err, const MDBX_val &cursor_key,
const MDBX_val &cursor_data) const;
void speculum_check_cursor(const char *where, const char *stage,
const testcase::SET::const_iterator &it,
MDBX_cursor *cursor,
const MDBX_cursor_op op) const;
void verbose(const char *where, const char *stage,
const testcase::SET::const_iterator &it) const;
void verbose(const char *where, const char *stage, const MDBX_val &key,
const MDBX_val &data, int err = MDBX_SUCCESS) const;
bool is_same(const Item &a, const Item &b) const;
bool is_same(const SET::const_iterator &it, const MDBX_val &k,
const MDBX_val &v) const;
bool speculum_verify();
int insert(const keygen::buffer &akey, const keygen::buffer &adata,
MDBX_put_flags_t flags);

View File

@ -287,6 +287,9 @@ std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum);
bool hex2data(const char *hex_begin, const char *hex_end, void *ptr,
size_t bytes, simple_checksum &checksum);
bool is_samedata(const MDBX_val *a, const MDBX_val *b);
inline bool is_samedata(const MDBX_val &a, const MDBX_val &b) {
return is_samedata(&a, &b);
}
std::string format(const char *fmt, ...);
uint64_t entropy_ticks(void);