ISAR/libmdbx
3
0
Fork 0
mirror of https://github.com/isar/libmdbx.git synced 2024-12-30 01:54:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

mdbx-test: add dataset verification to hill testcase.

Browse Source 
Change-Id: I8b781e98a02a8c32eeb82e54821b0941556d9f8c
This commit is contained in:
Leonid Yuriev 2019-10-06 12:07:14 +03:00
parent 440bfec193
commit f629914217
2 changed files with 287 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

237
test/hill.cc
View File

@ -14,13 +14,131 @@
#include "test.h"
int testcase_hill::insert(const keygen::buffer &akey,
const keygen::buffer &adata, unsigned flags) {
int err = mdbx_put(txn_guard.get(), dbi, &akey->value, &adata->value, flags);
if (err == MDBX_SUCCESS) {
const auto S_key = S(akey);
const auto S_data = S(adata);
const bool inserted = mirror.emplace(S_key, S_data).second;
assert(inserted);
(void)inserted;
}
return err;
}
int testcase_hill::replace(const keygen::buffer &akey,
const keygen::buffer &new_data,
const keygen::buffer &old_data, unsigned flags) {
const auto S_key = S(akey);
const auto S_old = S(old_data);
const auto S_new = S(new_data);
const auto removed = mirror.erase(set::key_type(S_key, S_old));
assert(removed == 1);
(void)removed;
const bool inserted = mirror.emplace(S_key, S_new).second;
assert(inserted);
(void)inserted;
return mdbx_replace(txn_guard.get(), dbi, &akey->value, &new_data->value,
&old_data->value, flags);
}
int testcase_hill::remove(const keygen::buffer &akey,
const keygen::buffer &adata) {
const auto S_key = S(akey);
const auto S_data = S(adata);
const auto removed = mirror.erase(set::key_type(S_key, S_data));
assert(removed == 1);
(void)removed;
return mdbx_del(txn_guard.get(), dbi, &akey->value, &adata->value);
}
bool testcase_hill::verify() const {
char dump_key[128], dump_value[128];
char dump_mkey[128], dump_mvalue[128];
MDBX_cursor *cursor;
int err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor);
if (err != MDBX_SUCCESS)
failure_perror("mdbx_cursor_open()", err);
bool rc = true;
MDBX_val akey, avalue;
MDBX_val mkey, mvalue;
err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_FIRST);
assert(std::is_sorted(mirror.cbegin(), mirror.cend(), ItemCompare(this)));
auto it = mirror.cbegin();
while (true) {
if (err != MDBX_SUCCESS) {
akey.iov_len = avalue.iov_len = 0;
akey.iov_base = avalue.iov_base = nullptr;
}
const auto S_key = S(akey);
const auto S_data = S(avalue);
if (it != mirror.cend()) {
mkey.iov_base = (void *)it->first.c_str();
mkey.iov_len = it->first.size();
mvalue.iov_base = (void *)it->second.c_str();
mvalue.iov_len = it->second.size();
}
if (err == MDBX_SUCCESS && it != mirror.cend() && S_key == it->first &&
S_data == it->second) {
++it;
err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT);
} else if (err == MDBX_SUCCESS &&
(it == mirror.cend() || S_key < it->first ||
(S_key == it->first && S_data < it->second))) {
if (it != mirror.cend()) {
log_error("extra pair: db{%s, %s} < mi{%s, %s}",
mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)),
mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
} else {
log_error("extra pair: db{%s, %s} < mi.END",
mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)));
}
err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT);
rc = false;
} else if (it != mirror.cend() &&
(err == MDBX_NOTFOUND || S_key > it->first ||
(S_key == it->first && S_data > it->second))) {
if (err == MDBX_NOTFOUND) {
log_error("lost pair: db.END > mi{%s, %s}",
mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
} else {
log_error("lost pair: db{%s, %s} > mi{%s, %s}",
mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)),
mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
}
++it;
rc = false;
} else if (err == MDBX_NOTFOUND && it == mirror.cend()) {
break;
} else if (err != MDBX_SUCCESS) {
failure_perror("mdbx_cursor_get()", err);
} else {
assert(!"WTF?");
}
}
mdbx_cursor_close(cursor);
return rc;
}
bool testcase_hill::run() {
MDBX_dbi dbi;
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("hill: bailout-prepare due '%s'", mdbx_strerror(err));
return true;
}
mirror.clear();
mirror_commited.clear();
/* LY: тест "холмиком":
* - сначала наполняем таблицу циклическими CRUD-манипуляциями,
@ -63,6 +181,7 @@ bool testcase_hill::run() {
uint64_t commited_serial = serial_count;
unsigned txn_nops = 0;
bool rc = false;
while (should_continue()) {
const keygen::serial_t a_serial = serial_count;
if (unlikely(!keyvalue_maker.increment(serial_count, 1))) {
@ -78,53 +197,74 @@ bool testcase_hill::run() {
log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift,
a_serial);
generate_pair(a_serial, a_key, a_data_1, age_shift);
err = mdbx_put(txn_guard.get(), dbi, &a_key->value, &a_data_1->value,
insert_flags);
err = insert(a_key, a_data_1, insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at insert-a due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
mirror = mirror_commited;
break;
}
failure_perror("mdbx_put(insert-a.1)", err);
}
if (!verify()) {
log_notice("uphill: bailout after insert-a, before commit");
goto bailout;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
mirror = mirror_commited;
break;
}
mirror_commited = mirror;
commited_serial = a_serial;
txn_nops = 0;
if (!verify()) {
log_notice("uphill: bailout after insert-a, after commit");
goto bailout;
}
}
// создаем вторую запись из пары
log_trace("uphill: insert-b %" PRIu64, b_serial);
generate_pair(b_serial, b_key, b_data, 0);
err = mdbx_put(txn_guard.get(), dbi, &b_key->value, &b_data->value,
insert_flags);
err = insert(b_key, b_data, insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at insert-b due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
mirror = mirror_commited;
break;
}
failure_perror("mdbx_put(insert-b)", err);
}
if (!verify()) {
log_notice("uphill: bailout after insert-b, before commit");
goto bailout;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
mirror = mirror_commited;
break;
}
mirror_commited = mirror;
commited_serial = a_serial;
txn_nops = 0;
if (!verify()) {
log_notice("uphill: bailout after insert-b, after commit");
goto bailout;
}
}
// обновляем данные в первой записи
@ -132,52 +272,73 @@ bool testcase_hill::run() {
a_serial);
generate_pair(a_serial, a_key, a_data_0, 0);
checkdata("uphill: update-a", dbi, a_key->value, a_data_1->value);
err = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_0->value,
&a_data_1->value, update_flags);
err = replace(a_key, a_data_0, a_data_1, update_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at update-a due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
mirror = mirror_commited;
break;
}
failure_perror("mdbx_replace(update-a: 1->0)", err);
}
if (!verify()) {
log_notice("uphill: bailout after update-a, before commit");
goto bailout;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
mirror = mirror_commited;
break;
}
mirror_commited = mirror;
commited_serial = a_serial;
txn_nops = 0;
if (!verify()) {
log_notice("uphill: bailout after update-a, after commit");
goto bailout;
}
}
// удаляем вторую запись
log_trace("uphill: delete-b %" PRIu64, b_serial);
checkdata("uphill: delete-b", dbi, b_key->value, b_data->value);
err = mdbx_del(txn_guard.get(), dbi, &b_key->value, &b_data->value);
err = remove(b_key, b_data);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at delete-b due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
mirror = mirror_commited;
break;
}
failure_perror("mdbx_del(b)", err);
}
if (!verify()) {
log_notice("uphill: bailout after delete-b, before commit");
goto bailout;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
mirror = mirror_commited;
break;
}
mirror_commited = mirror;
commited_serial = a_serial;
txn_nops = 0;
if (!verify()) {
log_notice("uphill: bailout after delete-b, after commit");
goto bailout;
}
}
report(1);
@ -204,101 +365,145 @@ bool testcase_hill::run() {
generate_pair(a_serial, a_key, a_data_0, 0);
generate_pair(a_serial, a_key, a_data_1, age_shift);
checkdata("downhill: update-a", dbi, a_key->value, a_data_0->value);
err = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_1->value,
&a_data_0->value, update_flags);
err = replace(a_key, a_data_1, a_data_0, update_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at update-a due '%s'",
mdbx_strerror(err));
txn_end(true);
mirror = mirror_commited;
break;
}
failure_perror("mdbx_put(update-a: 0->1)", err);
}
if (!verify()) {
log_notice("downhill: bailout after update-a, before commit");
break;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
mirror = mirror_commited;
break;
}
mirror_commited = mirror;
txn_nops = 0;
if (!verify()) {
log_notice("downhill: bailout after update-a, after commit");
break;
}
}
// создаем вторую запись из пары
log_trace("downhill: insert-b %" PRIu64, b_serial);
generate_pair(b_serial, b_key, b_data, 0);
err = mdbx_put(txn_guard.get(), dbi, &b_key->value, &b_data->value,
insert_flags);
err = insert(b_key, b_data, insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at insert-a due '%s'",
mdbx_strerror(err));
txn_end(true);
mirror = mirror_commited;
break;
}
failure_perror("mdbx_put(insert-b)", err);
}
if (!verify()) {
log_notice("downhill: bailout after insert-b, before commit");
break;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
mirror = mirror_commited;
break;
}
mirror_commited = mirror;
txn_nops = 0;
if (!verify()) {
log_notice("downhill: bailout after insert-b, after commit");
break;
}
}
// удаляем первую запись
log_trace("downhill: delete-a (age %" PRIu64 ") %" PRIu64, age_shift,
a_serial);
checkdata("downhill: delete-a", dbi, a_key->value, a_data_1->value);
err = mdbx_del(txn_guard.get(), dbi, &a_key->value, &a_data_1->value);
err = remove(a_key, a_data_1);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at delete-a due '%s'",
mdbx_strerror(err));
txn_end(true);
mirror = mirror_commited;
break;
}
failure_perror("mdbx_del(a)", err);
}
if (!verify()) {
log_notice("downhill: bailout after delete-a, before commit");
break;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
mirror = mirror_commited;
break;
}
mirror_commited = mirror;
txn_nops = 0;
if (!verify()) {
log_notice("downhill: bailout after delete-a, after commit");
break;
}
}
// удаляем вторую запись
log_trace("downhill: delete-b %" PRIu64, b_serial);
checkdata("downhill: delete-b", dbi, b_key->value, b_data->value);
err = mdbx_del(txn_guard.get(), dbi, &b_key->value, &b_data->value);
err = remove(b_key, b_data);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at delete-b due '%s'",
mdbx_strerror(err));
txn_end(true);
mirror = mirror_commited;
break;
}
failure_perror("mdbx_del(b)", err);
}
if (!verify()) {
log_notice("downhill: bailout after delete-b, before commit");
break;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
mirror = mirror_commited;
break;
}
mirror_commited = mirror;
txn_nops = 0;
if (!verify()) {
log_notice("downhill: bailout after delete-b, after commit");
goto bailout;
}
}
report(1);
}
rc = verify();
bailout:
if (txn_guard) {
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS))
@ -312,10 +517,10 @@ bool testcase_hill::run() {
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("hill: bailout-clean due '%s'", mdbx_strerror(err));
return true;
return rc;
}
} else
db_table_close(dbi);
}
return true;
return rc;
}

68
test/test.h
View File

@ -22,6 +22,21 @@
#include "osal.h"
#include "utils.h"
#include <set>
#ifndef HAVE_cxx17_std_string_view
#if __cplusplus >= 201703L && __has_include(<string_view>)
#include <string_view>
#define HAVE_cxx17_std_string_view 1
#else
#define HAVE_cxx17_std_string_view 0
#endif
#endif /* HAVE_cxx17_std_string_view */
#if HAVE_cxx17_std_string_view
#include <string_view>
#endif
bool test_execute(const actor_config &config);
std::string thunk_param(const actor_config &config);
void testcase_setup(const char *casename, actor_params &params,
@ -165,10 +180,59 @@ public:
};
class testcase_hill : public testcase {
using inherited = testcase;
#if HAVE_cxx17_std_string_view
using data_view = std::string_view;
#else
using data_view = std::string;
#endif
MDBX_dbi dbi;
using Item = std::pair<std::string, std::string>;
struct ItemCompare {
const testcase_hill *context;
ItemCompare(const testcase_hill *owner) : context(owner) {}
bool operator()(const Item &a, const Item &b) const {
MDBX_val va, vb;
va.iov_base = (void *)a.first.data();
va.iov_len = a.first.size();
vb.iov_base = (void *)b.first.data();
vb.iov_len = b.first.size();
int cmp = mdbx_cmp(context->txn_guard.get(), context->dbi, &va, &vb);
if (cmp == 0 &&
(context->config.params.table_flags & MDBX_DUPSORT) != 0) {
va.iov_base = (void *)a.second.data();
va.iov_len = a.second.size();
vb.iov_base = (void *)b.second.data();
vb.iov_len = b.second.size();
cmp = mdbx_dcmp(context->txn_guard.get(), context->dbi, &va, &vb);
}
return cmp < 0;
}
};
using set = std::set<Item, ItemCompare>;
set mirror, mirror_commited;
bool verify() const;
int insert(const keygen::buffer &akey, const keygen::buffer &adata,
unsigned flags);
int replace(const keygen::buffer &akey, const keygen::buffer &new_value,
const keygen::buffer &old_value, unsigned flags);
int remove(const keygen::buffer &akey, const keygen::buffer &adata);
static inline data_view S(const MDBX_val &v) {
return data_view(static_cast<const char *>(v.iov_base), v.iov_len);
}
static inline data_view S(const keygen::buffer &b) { return S(b->value); }
public:
testcase_hill(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool run();
: testcase(config, pid), mirror(ItemCompare(this)),
mirror_commited(ItemCompare(this)) {}
bool run() override;
};
class testcase_append : public testcase {