diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1de75560..75db969f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -29,6 +29,7 @@ add_executable(mdbx_test utils.h append.cc ttl.cc + nested.cc ) set_target_properties(mdbx_test PROPERTIES diff --git a/test/cases.cc b/test/cases.cc index 02177140..5442c08d 100644 --- a/test/cases.cc +++ b/test/cases.cc @@ -70,6 +70,7 @@ void testcase_setup(const char *casename, actor_params ¶ms, configure_actor(last_space_id, ac_try, nullptr, params); configure_actor(last_space_id, ac_copy, nullptr, params); configure_actor(last_space_id, ac_append, nullptr, params); + configure_actor(last_space_id, ac_nested, nullptr, params); log_notice("<<< testcase_setup(%s): done", casename); } else { failure("unknown testcase `%s`", casename); diff --git a/test/config.h b/test/config.h index 50e00382..3efba627 100644 --- a/test/config.h +++ b/test/config.h @@ -29,7 +29,8 @@ enum actor_testcase { ac_try, ac_copy, ac_append, - ac_ttl + ac_ttl, + ac_nested }; enum actor_status { diff --git a/test/hill.cc b/test/hill.cc index 89a802ec..9d989095 100644 --- a/test/hill.cc +++ b/test/hill.cc @@ -18,7 +18,7 @@ bool testcase_hill::run() { int err = db_open__begin__table_create_open_clean(dbi); if (unlikely(err != MDBX_SUCCESS)) { log_notice("hill: bailout-prepare due '%s'", mdbx_strerror(err)); - return true; + return false; } speculum.clear(); speculum_commited.clear(); diff --git a/test/main.cc b/test/main.cc index e34f914f..4a90605f 100644 --- a/test/main.cc +++ b/test/main.cc @@ -361,6 +361,10 @@ int main(int argc, char *const argv[]) { configure_actor(last_space_id, ac_ttl, value, params); continue; } + if (config::parse_option(argc, argv, narg, "nested", nullptr)) { + configure_actor(last_space_id, ac_nested, value, params); + continue; + } if (config::parse_option(argc, argv, narg, "failfast", global::config::failfast)) continue; diff --git a/test/nested.cc b/test/nested.cc new file mode 100644 index 00000000..e6202464 --- /dev/null +++ b/test/nested.cc @@ -0,0 +1,278 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" +#include + +bool testcase_nested::setup() { + if (!inherited::setup()) + return false; + int err = db_open__begin__table_create_open_clean(dbi); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("nested: bailout-prepare due '%s'", mdbx_strerror(err)); + return false; + } + + keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */); + key = keygen::alloc(config.params.keylen_max); + data = keygen::alloc(config.params.datalen_max); + serial = 0; + fifo.clear(); + speculum.clear(); + assert(stack.empty()); + stack.emplace(nullptr, serial, fifo, speculum); + return true; +} + +bool testcase_nested::teardown() { + while (!stack.empty()) + pop_txn(true); + + bool ok = true; + if (dbi) { + if (config.params.drop_table && !mode_readonly()) { + txn_begin(false); + db_table_drop(dbi); + int err = breakable_commit(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("nested: bailout-clean due '%s'", mdbx_strerror(err)); + ok = false; + } + } else + db_table_close(dbi); + dbi = 0; + } + return inherited::teardown() && ok; +} + +static unsigned edge2window(uint64_t edge, unsigned window_max) { + const double rnd = u64_to_double1(bleach64(edge)); + const unsigned window = window_max - std::lrint(std::pow(window_max, rnd)); + return window; +} + +static unsigned edge2count(uint64_t edge, unsigned count_max) { + const double rnd = u64_to_double1(prng64_map1_white(edge)); + const unsigned count = std::lrint(std::pow(count_max, rnd)); + return count; +} + +void testcase_nested::push_txn() { + MDBX_txn *txn; + int err = mdbx_txn_begin( + db_guard.get(), txn_guard.get(), + prng32() & (MDBX_NOSYNC | MDBX_NOMETASYNC | MDBX_MAPASYNC), &txn); + if (unlikely(err != MDBX_SUCCESS)) + failure_perror("mdbx_txn_begin(nested)", err); + stack.emplace(txn, serial, fifo, speculum); + std::swap(txn_guard, std::get<0>(stack.top())); + log_verbose("begin level#%zu txn, serial %" PRIu64, stack.size(), serial); +} + +bool testcase_nested::pop_txn(bool abort) { + assert(txn_guard && !stack.empty()); + bool should_continue = true; + MDBX_txn *txn = txn_guard.release(); + bool commited = false; + if (abort) { + log_verbose("abort level#%zu txn, undo serial %" PRIu64 " <- %" PRIu64, + stack.size(), serial, std::get<1>(stack.top())); + int err = mdbx_txn_abort(txn); + if (unlikely(err != MDBX_SUCCESS)) + failure_perror("mdbx_txn_abort()", err); + } else { + log_verbose("commit level#%zu txn, nested serial %" PRIu64 " -> %" PRIu64, + stack.size(), serial, std::get<1>(stack.top())); + int err = mdbx_txn_commit(txn); + if (likely(err == MDBX_SUCCESS)) + commited = true; + else { + should_continue = false; + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + err = mdbx_txn_abort(txn); + if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH)) + failure_perror("mdbx_txn_abort()", err); + } else + failure_perror("mdbx_txn_commit()", err); + } + } + + std::swap(txn_guard, std::get<0>(stack.top())); + if (!commited) { + serial = std::get<1>(stack.top()); + std::swap(fifo, std::get<2>(stack.top())); + std::swap(speculum, std::get<3>(stack.top())); + } + stack.pop(); + return should_continue; +} + +bool testcase_nested::stochastic_breakable_restart_with_nested( + bool force_restart) { + log_trace(">> stochastic_breakable_restart_with_nested%s", + force_restart ? ": force_restart" : ""); + + if (force_restart) + while (txn_guard) + pop_txn(true); + + bool should_continue = true; + while (!stack.empty() && + (flipcoin() || txn_underutilization_x256(txn_guard.get()) < 42)) + should_continue &= pop_txn(); + + if (should_continue) + while (stack.empty() || + (is_nested_txn_available() && flipcoin() && stack.size() < 5)) + push_txn(); + + log_trace("<< stochastic_breakable_restart_with_nested: should_continue=%s", + should_continue ? "yes" : "no"); + return should_continue; +} + +bool testcase_nested::trim_tail(unsigned window_width) { + if (window_width) { + while (fifo.size() > window_width) { + uint64_t tail_serial = fifo.back().first; + const unsigned tail_count = fifo.back().second; + log_trace("nested: pop-tail (serial %" PRIu64 ", count %u)", tail_serial, + tail_count); + fifo.pop_back(); + for (unsigned n = 0; n < tail_count; ++n) { + log_trace("nested: remove-tail %" PRIu64, tail_serial); + generate_pair(tail_serial); + int err = remove(key, data); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("nested: tail-bailout due '%s'", mdbx_strerror(err)); + return false; + } + failure_perror("mdbx_del(tail)", err); + } + if (unlikely(!keyvalue_maker.increment(tail_serial, 1))) + failure("nested: unexpected key-space overflow on the tail"); + } + } + } else { + log_trace("nested: purge state"); + db_table_clear(dbi, txn_guard.get()); + fifo.clear(); + speculum.clear(); + } + return true; +} + +bool testcase_nested::grow_head(unsigned head_count) { + const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT) + ? MDBX_NODUPDATA + : MDBX_NODUPDATA | MDBX_NOOVERWRITE; +retry: + fifo.push_front(std::make_pair(serial, head_count)); + for (unsigned n = 0; n < head_count; ++n) { + log_trace("nested: insert-head %" PRIu64, serial); + generate_pair(serial); + int err = insert(key, data, insert_flags); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("nested: head-insert skip due '%s'", mdbx_strerror(err)); + head_count = n; + stochastic_breakable_restart_with_nested(true); + goto retry; + } + failure_perror("mdbx_put(head)", err); + } + + if (unlikely(!keyvalue_maker.increment(serial, 1))) { + log_notice("nested: unexpected key-space overflow"); + return false; + } + } + + return true; +} + +bool testcase_nested::run() { + /* LY: тест "эмуляцией time-to-live" с вложенными транзакциями: + * - организуется "скользящее окно", которое каждую транзакцию сдвигается + * вперед вдоль числовой оси. + * - по переднему краю "скользящего окна" записи добавляются в таблицу, + * а по заднему удаляются. + * - количество добавляемых/удаляемых записей псевдослучайно зависит + * от номера транзакции, но с экспоненциальным распределением. + * - размер "скользящего окна" также псевдослучайно зависит от номера + * транзакции с "отрицательным" экспоненциальным распределением + * MAX_WIDTH - exp(rnd(N)), при уменьшении окна сдвигается задний + * край и удаляются записи позади него. + * - групповое добавление данных в начало окна и групповое уделение в конце, + * в половине случаев выполняются во вложенных транзакциях. + * - половина запускаемых вложенных транзакций отменяется, последуюим + * повтором групповой операции. + * + * Таким образом имитируется поведение таблицы с TTL: записи стохастически + * добавляются и удаляются, но изредка происходят массивные удаления. */ + + /* LY: для параметризации используем подходящие параметры, которые не имеют + * здесь смысла в первоначальном значении. */ + const unsigned window_max_lower = 333; + const unsigned count_max_lower = 333; + + const unsigned window_max = (config.params.batch_read > window_max_lower) + ? config.params.batch_read + : window_max_lower; + const unsigned count_max = (config.params.batch_write > count_max_lower) + ? config.params.batch_write + : count_max_lower; + log_verbose("nested: using `batch_read` value %u for window_max", window_max); + log_verbose("nested: using `batch_write` value %u for count_max", count_max); + + uint64_t seed = + prng64_map2_white(config.params.keygen.seed) + config.actor_id; + + while (should_continue()) { + const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */; + const unsigned window_width = edge2window(salt, window_max); + const unsigned head_count = edge2count(salt, count_max); + log_debug("nested: step #%zu (serial %" PRIu64 + ", window %u, count %u) salt %" PRIu64, + nops_completed, serial, window_width, head_count, salt); + + if (!trim_tail(window_width)) + return false; + if (!stochastic_breakable_restart_with_nested()) { + log_notice("nested: bailout at commit/restart after tail-trim"); + return false; + } + if (!speculum_verify()) { + log_notice("nested: bailout after tail-trim"); + return false; + } + + if (!grow_head(head_count)) + return false; + if (!stochastic_breakable_restart_with_nested()) + log_notice("nested: skip commit/restart after head-grow"); + if (!speculum_verify()) { + log_notice("nested: bailout after head-grow"); + return false; + } + + report(1); + } + + while (!stack.empty()) + pop_txn(false); + + return true; +} diff --git a/test/test.cc b/test/test.cc index 479fdf60..f4fd8a8a 100644 --- a/test/test.cc +++ b/test/test.cc @@ -37,6 +37,8 @@ const char *testcase2str(const actor_testcase testcase) { return "append"; case ac_ttl: return "ttl"; + case ac_nested: + return "nested"; } } @@ -197,6 +199,20 @@ int testcase::breakable_commit() { return rc; } +unsigned testcase::txn_underutilization_x256(MDBX_txn *txn) const { + if (txn) { + MDBX_txn_info info; + int err = mdbx_txn_info(txn, &info, false); + if (unlikely(err != MDBX_SUCCESS)) + failure_perror("mdbx_txn_info()", err); + const size_t left = size_t(info.txn_space_leftover); + const size_t total = + size_t(info.txn_space_leftover) + size_t(info.txn_space_dirty); + return (unsigned)(left / (total >> 8)); + } + return 0; +} + void testcase::txn_end(bool abort) { log_trace(">> txn_end(%s)", abort ? "abort" : "commit"); assert(txn_guard); @@ -479,9 +495,9 @@ void testcase::db_table_drop(MDBX_dbi handle) { } } -void testcase::db_table_clear(MDBX_dbi handle) { +void testcase::db_table_clear(MDBX_dbi handle, MDBX_txn *txn) { log_trace(">> testcase::db_table_clear, handle %u", handle); - int rc = mdbx_drop(txn_guard.get(), handle, false); + int rc = mdbx_drop(txn ? txn : txn_guard.get(), handle, false); if (unlikely(rc != MDBX_SUCCESS)) failure_perror("mdbx_drop(delete=false)", rc); log_trace("<< testcase::db_table_clear"); @@ -549,6 +565,9 @@ bool test_execute(const actor_config &config_const) { case ac_ttl: test.reset(new testcase_ttl(config, pid)); break; + case ac_nested: + test.reset(new testcase_nested(config, pid)); + break; default: test.reset(new testcase(config, pid)); break; diff --git a/test/test.h b/test/test.h index b65d51aa..0ca71754 100644 --- a/test/test.h +++ b/test/test.h @@ -22,7 +22,10 @@ #include "osal.h" #include "utils.h" +#include #include +#include +#include #ifndef HAVE_cxx17_std_string_view #if __cplusplus >= 201703L && __has_include() @@ -160,6 +163,10 @@ protected: static int oom_callback(MDBX_env *env, mdbx_pid_t pid, mdbx_tid_t tid, uint64_t txn, unsigned gap, size_t space, int retry); + bool is_nested_txn_available() const { + return (config.params.mode_flags & MDBX_WRITEMAP) == 0; + } + void kick_progress(bool active) const; void db_prepare(); void db_open(); void db_close(); @@ -176,10 +183,11 @@ protected: void update_canary(uint64_t increment); void checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check, MDBX_val expected_valued); + unsigned txn_underutilization_x256(MDBX_txn *txn) const; MDBX_dbi db_table_open(bool create); void db_table_drop(MDBX_dbi handle); - void db_table_clear(MDBX_dbi handle); + void db_table_clear(MDBX_dbi handle, MDBX_txn *txn = nullptr); void db_table_close(MDBX_dbi handle); int db_open__begin__table_create_open_clean(MDBX_dbi &dbi); @@ -214,14 +222,13 @@ public: virtual bool run() { return true; } virtual bool teardown(); virtual ~testcase() {} - void kick_progress(bool active) const; }; class testcase_ttl : public testcase { public: testcase_ttl(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} - bool run(); + bool run() override; }; class testcase_hill : public testcase { @@ -238,35 +245,35 @@ class testcase_append : public testcase { public: testcase_append(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} - bool run(); + bool run() override; }; class testcase_deadread : public testcase { public: testcase_deadread(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} - bool run(); + bool run() override; }; class testcase_deadwrite : public testcase { public: testcase_deadwrite(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} - bool run(); + bool run() override; }; class testcase_jitter : public testcase { public: testcase_jitter(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} - bool run(); + bool run() override; }; class testcase_try : public testcase { public: testcase_try(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} - bool run(); + bool run() override; }; class testcase_copy : public testcase { @@ -277,5 +284,28 @@ public: testcase_copy(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid), copy_pathname(config.params.pathname_db + "-copy") {} - bool run(); + bool run() override; +}; + +class testcase_nested : public testcase { + using inherited = testcase; + using FIFO = std::deque>; + + uint64_t serial; + FIFO fifo; + std::stack> stack; + + bool trim_tail(unsigned window_width); + bool grow_head(unsigned head_count); + bool pop_txn(bool abort); + bool pop_txn() { return pop_txn(flipcoin_x2()); } + void push_txn(); + bool stochastic_breakable_restart_with_nested(bool force_restart = false); + +public: + testcase_nested(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool setup() override; + bool run() override; + bool teardown() override; }; diff --git a/test/ttl.cc b/test/ttl.cc index 5782460d..f7ae4220 100644 --- a/test/ttl.cc +++ b/test/ttl.cc @@ -32,7 +32,7 @@ bool testcase_ttl::run() { int err = db_open__begin__table_create_open_clean(dbi); if (unlikely(err != MDBX_SUCCESS)) { log_notice("ttl: bailout-prepare due '%s'", mdbx_strerror(err)); - return true; + return false; } /* LY: тест "эмуляцией time-to-live": @@ -53,18 +53,8 @@ bool testcase_ttl::run() { /* LY: для параметризации используем подходящие параметры, которые не имеют * здесь смысла в первоначальном значении. */ - const unsigned window_max_lower = -#ifdef __APPLE__ - 333; -#else - 999; -#endif - const unsigned count_max_lower = -#ifdef __APPLE__ - 333; -#else - 999; -#endif + const unsigned window_max_lower = 333; + const unsigned count_max_lower = 333; const unsigned window_max = (config.params.batch_read > window_max_lower) ? config.params.batch_read @@ -86,6 +76,7 @@ bool testcase_ttl::run() { std::deque> fifo; uint64_t serial = 0; + bool rc = false; while (should_continue()) { const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */; @@ -103,7 +94,7 @@ bool testcase_ttl::run() { tail_count); fifo.pop_back(); for (unsigned n = 0; n < tail_count; ++n) { - log_trace("ttl: remove-tail %" PRIu64, serial); + log_trace("ttl: remove-tail %" PRIu64, tail_serial); generate_pair(tail_serial); err = mdbx_del(txn_guard.get(), dbi, &key->value, &data->value); if (unlikely(err != MDBX_SUCCESS)) { @@ -157,7 +148,9 @@ bool testcase_ttl::run() { serial = fifo.front().first; fifo.pop_front(); } + report(1); + rc = true; } bailout: @@ -169,10 +162,10 @@ bailout: err = breakable_commit(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("ttl: bailout-clean due '%s'", mdbx_strerror(err)); - return true; + return false; } } else db_table_close(dbi); } - return true; + return rc; } diff --git a/test/utils.cc b/test/utils.cc index 4372fcf0..c807d0e5 100644 --- a/test/utils.cc +++ b/test/utils.cc @@ -331,6 +331,7 @@ double double_from_upper(uint64_t salt) { } bool flipcoin() { return bleach32((uint32_t)entropy_ticks()) & 1; } +bool flipcoin_x2() { return (bleach32((uint32_t)entropy_ticks()) & 3) == 0; } bool jitter(unsigned probability_percent) { const uint32_t top = UINT32_MAX - UINT32_MAX % 100; diff --git a/test/utils.h b/test/utils.h index cdce1de4..35a54d30 100644 --- a/test/utils.h +++ b/test/utils.h @@ -355,5 +355,6 @@ uint64_t prng64(void); void prng_fill(void *ptr, size_t bytes); bool flipcoin(); +bool flipcoin_x2(); bool jitter(unsigned probability_percent); void jitter_delay(bool extra = false);