diff --git a/test/config.cc b/test/config.cc index da918400..e900d4a8 100644 --- a/test/config.cc +++ b/test/config.cc @@ -416,6 +416,9 @@ void dump(const char *title) { log_verbose("drop table: %s\n", i->params.drop_table ? "Yes" : "No"); log_verbose("ignore MDBX_MAP_FULL error: %s\n", i->params.ignore_dbfull ? "Yes" : "No"); + log_verbose("verifying by speculum: %s\n", + i->params.speculum ? "Yes" : "No"); + indent.pop(); } diff --git a/test/config.h b/test/config.h index 6582f1d8..50e00382 100644 --- a/test/config.h +++ b/test/config.h @@ -266,6 +266,7 @@ struct actor_params_pod { bool drop_table; bool ignore_dbfull; + bool speculum; }; struct actor_config_pod { diff --git a/test/hill.cc b/test/hill.cc index f65ec060..89a802ec 100644 --- a/test/hill.cc +++ b/test/hill.cc @@ -14,131 +14,14 @@ #include "test.h" -int testcase_hill::insert(const keygen::buffer &akey, - const keygen::buffer &adata, unsigned flags) { - int err = mdbx_put(txn_guard.get(), dbi, &akey->value, &adata->value, flags); - if (err == MDBX_SUCCESS) { - const auto S_key = S(akey); - const auto S_data = S(adata); - const bool inserted = mirror.emplace(S_key, S_data).second; - assert(inserted); - (void)inserted; - } - return err; -} - -int testcase_hill::replace(const keygen::buffer &akey, - const keygen::buffer &new_data, - const keygen::buffer &old_data, unsigned flags) { - const auto S_key = S(akey); - const auto S_old = S(old_data); - const auto S_new = S(new_data); - const auto removed = mirror.erase(set::key_type(S_key, S_old)); - assert(removed == 1); - (void)removed; - const bool inserted = mirror.emplace(S_key, S_new).second; - assert(inserted); - (void)inserted; - return mdbx_replace(txn_guard.get(), dbi, &akey->value, &new_data->value, - &old_data->value, flags); -} - -int testcase_hill::remove(const keygen::buffer &akey, - const keygen::buffer &adata) { - const auto S_key = S(akey); - const auto S_data = S(adata); - const auto removed = mirror.erase(set::key_type(S_key, S_data)); - assert(removed == 1); - (void)removed; - return mdbx_del(txn_guard.get(), dbi, &akey->value, &adata->value); -} - -bool testcase_hill::verify() const { - char dump_key[128], dump_value[128]; - char dump_mkey[128], dump_mvalue[128]; - - MDBX_cursor *cursor; - int err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor); - if (err != MDBX_SUCCESS) - failure_perror("mdbx_cursor_open()", err); - - bool rc = true; - MDBX_val akey, avalue; - MDBX_val mkey, mvalue; - err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_FIRST); - - assert(std::is_sorted(mirror.cbegin(), mirror.cend(), ItemCompare(this))); - auto it = mirror.cbegin(); - while (true) { - if (err != MDBX_SUCCESS) { - akey.iov_len = avalue.iov_len = 0; - akey.iov_base = avalue.iov_base = nullptr; - } - const auto S_key = S(akey); - const auto S_data = S(avalue); - if (it != mirror.cend()) { - mkey.iov_base = (void *)it->first.c_str(); - mkey.iov_len = it->first.size(); - mvalue.iov_base = (void *)it->second.c_str(); - mvalue.iov_len = it->second.size(); - } - if (err == MDBX_SUCCESS && it != mirror.cend() && S_key == it->first && - S_data == it->second) { - ++it; - err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT); - } else if (err == MDBX_SUCCESS && - (it == mirror.cend() || S_key < it->first || - (S_key == it->first && S_data < it->second))) { - if (it != mirror.cend()) { - log_error("extra pair: db{%s, %s} < mi{%s, %s}", - mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), - mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), - mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), - mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); - } else { - log_error("extra pair: db{%s, %s} < mi.END", - mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), - mdbx_dump_val(&avalue, dump_value, sizeof(dump_value))); - } - err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT); - rc = false; - } else if (it != mirror.cend() && - (err == MDBX_NOTFOUND || S_key > it->first || - (S_key == it->first && S_data > it->second))) { - if (err == MDBX_NOTFOUND) { - log_error("lost pair: db.END > mi{%s, %s}", - mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), - mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); - } else { - log_error("lost pair: db{%s, %s} > mi{%s, %s}", - mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), - mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), - mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), - mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); - } - ++it; - rc = false; - } else if (err == MDBX_NOTFOUND && it == mirror.cend()) { - break; - } else if (err != MDBX_SUCCESS) { - failure_perror("mdbx_cursor_get()", err); - } else { - assert(!"WTF?"); - } - } - - mdbx_cursor_close(cursor); - return rc; -} - bool testcase_hill::run() { int err = db_open__begin__table_create_open_clean(dbi); if (unlikely(err != MDBX_SUCCESS)) { log_notice("hill: bailout-prepare due '%s'", mdbx_strerror(err)); return true; } - mirror.clear(); - mirror_commited.clear(); + speculum.clear(); + speculum_commited.clear(); /* LY: тест "холмиком": * - сначала наполняем таблицу циклическими CRUD-манипуляциями, @@ -204,12 +87,12 @@ bool testcase_hill::run() { log_notice("uphill: bailout at insert-a due '%s'", mdbx_strerror(err)); txn_restart(true, false); serial_count = commited_serial; - mirror = mirror_commited; + speculum = speculum_commited; break; } failure_perror("mdbx_put(insert-a.1)", err); } - if (!verify()) { + if (!speculum_verify()) { log_notice("uphill: bailout after insert-a, before commit"); goto bailout; } @@ -219,13 +102,13 @@ bool testcase_hill::run() { if (unlikely(err != MDBX_SUCCESS)) { log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); serial_count = commited_serial; - mirror = mirror_commited; + speculum = speculum_commited; break; } - mirror_commited = mirror; + speculum_commited = speculum; commited_serial = a_serial; txn_nops = 0; - if (!verify()) { + if (!speculum_verify()) { log_notice("uphill: bailout after insert-a, after commit"); goto bailout; } @@ -240,12 +123,12 @@ bool testcase_hill::run() { log_notice("uphill: bailout at insert-b due '%s'", mdbx_strerror(err)); txn_restart(true, false); serial_count = commited_serial; - mirror = mirror_commited; + speculum = speculum_commited; break; } failure_perror("mdbx_put(insert-b)", err); } - if (!verify()) { + if (!speculum_verify()) { log_notice("uphill: bailout after insert-b, before commit"); goto bailout; } @@ -255,13 +138,13 @@ bool testcase_hill::run() { if (unlikely(err != MDBX_SUCCESS)) { log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); serial_count = commited_serial; - mirror = mirror_commited; + speculum = speculum_commited; break; } - mirror_commited = mirror; + speculum_commited = speculum; commited_serial = a_serial; txn_nops = 0; - if (!verify()) { + if (!speculum_verify()) { log_notice("uphill: bailout after insert-b, after commit"); goto bailout; } @@ -278,12 +161,12 @@ bool testcase_hill::run() { log_notice("uphill: bailout at update-a due '%s'", mdbx_strerror(err)); txn_restart(true, false); serial_count = commited_serial; - mirror = mirror_commited; + speculum = speculum_commited; break; } failure_perror("mdbx_replace(update-a: 1->0)", err); } - if (!verify()) { + if (!speculum_verify()) { log_notice("uphill: bailout after update-a, before commit"); goto bailout; } @@ -293,13 +176,13 @@ bool testcase_hill::run() { if (unlikely(err != MDBX_SUCCESS)) { log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); serial_count = commited_serial; - mirror = mirror_commited; + speculum = speculum_commited; break; } - mirror_commited = mirror; + speculum_commited = speculum; commited_serial = a_serial; txn_nops = 0; - if (!verify()) { + if (!speculum_verify()) { log_notice("uphill: bailout after update-a, after commit"); goto bailout; } @@ -314,12 +197,12 @@ bool testcase_hill::run() { log_notice("uphill: bailout at delete-b due '%s'", mdbx_strerror(err)); txn_restart(true, false); serial_count = commited_serial; - mirror = mirror_commited; + speculum = speculum_commited; break; } failure_perror("mdbx_del(b)", err); } - if (!verify()) { + if (!speculum_verify()) { log_notice("uphill: bailout after delete-b, before commit"); goto bailout; } @@ -329,13 +212,13 @@ bool testcase_hill::run() { if (unlikely(err != MDBX_SUCCESS)) { log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); serial_count = commited_serial; - mirror = mirror_commited; + speculum = speculum_commited; break; } - mirror_commited = mirror; + speculum_commited = speculum; commited_serial = a_serial; txn_nops = 0; - if (!verify()) { + if (!speculum_verify()) { log_notice("uphill: bailout after delete-b, after commit"); goto bailout; } @@ -371,12 +254,12 @@ bool testcase_hill::run() { log_notice("downhill: bailout at update-a due '%s'", mdbx_strerror(err)); txn_end(true); - mirror = mirror_commited; + speculum = speculum_commited; break; } failure_perror("mdbx_put(update-a: 0->1)", err); } - if (!verify()) { + if (!speculum_verify()) { log_notice("downhill: bailout after update-a, before commit"); break; } @@ -385,12 +268,12 @@ bool testcase_hill::run() { err = breakable_restart(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); - mirror = mirror_commited; + speculum = speculum_commited; break; } - mirror_commited = mirror; + speculum_commited = speculum; txn_nops = 0; - if (!verify()) { + if (!speculum_verify()) { log_notice("downhill: bailout after update-a, after commit"); break; } @@ -405,12 +288,12 @@ bool testcase_hill::run() { log_notice("downhill: bailout at insert-a due '%s'", mdbx_strerror(err)); txn_end(true); - mirror = mirror_commited; + speculum = speculum_commited; break; } failure_perror("mdbx_put(insert-b)", err); } - if (!verify()) { + if (!speculum_verify()) { log_notice("downhill: bailout after insert-b, before commit"); break; } @@ -419,12 +302,12 @@ bool testcase_hill::run() { err = breakable_restart(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); - mirror = mirror_commited; + speculum = speculum_commited; break; } - mirror_commited = mirror; + speculum_commited = speculum; txn_nops = 0; - if (!verify()) { + if (!speculum_verify()) { log_notice("downhill: bailout after insert-b, after commit"); break; } @@ -440,12 +323,12 @@ bool testcase_hill::run() { log_notice("downhill: bailout at delete-a due '%s'", mdbx_strerror(err)); txn_end(true); - mirror = mirror_commited; + speculum = speculum_commited; break; } failure_perror("mdbx_del(a)", err); } - if (!verify()) { + if (!speculum_verify()) { log_notice("downhill: bailout after delete-a, before commit"); break; } @@ -454,12 +337,12 @@ bool testcase_hill::run() { err = breakable_restart(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); - mirror = mirror_commited; + speculum = speculum_commited; break; } - mirror_commited = mirror; + speculum_commited = speculum; txn_nops = 0; - if (!verify()) { + if (!speculum_verify()) { log_notice("downhill: bailout after delete-a, after commit"); break; } @@ -474,12 +357,12 @@ bool testcase_hill::run() { log_notice("downhill: bailout at delete-b due '%s'", mdbx_strerror(err)); txn_end(true); - mirror = mirror_commited; + speculum = speculum_commited; break; } failure_perror("mdbx_del(b)", err); } - if (!verify()) { + if (!speculum_verify()) { log_notice("downhill: bailout after delete-b, before commit"); break; } @@ -488,12 +371,12 @@ bool testcase_hill::run() { err = breakable_restart(); if (unlikely(err != MDBX_SUCCESS)) { log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); - mirror = mirror_commited; + speculum = speculum_commited; break; } - mirror_commited = mirror; + speculum_commited = speculum; txn_nops = 0; - if (!verify()) { + if (!speculum_verify()) { log_notice("downhill: bailout after delete-b, after commit"); goto bailout; } @@ -502,7 +385,7 @@ bool testcase_hill::run() { report(1); } - rc = verify(); + rc = speculum_verify(); bailout: if (txn_guard) { err = breakable_commit(); diff --git a/test/main.cc b/test/main.cc index 1d9f51d5..e34f914f 100644 --- a/test/main.cc +++ b/test/main.cc @@ -71,6 +71,7 @@ void actor_params::set_defaults(const std::string &tmpdir) { drop_table = false; ignore_dbfull = false; + speculum = false; max_readers = 42; max_tables = 42; @@ -296,6 +297,8 @@ int main(int argc, char *const argv[]) { if (config::parse_option(argc, argv, narg, "ignore-dbfull", params.ignore_dbfull)) continue; + if (config::parse_option(argc, argv, narg, "speculum", params.speculum)) + continue; if (config::parse_option(argc, argv, narg, "dump-config", global::config::dump_config)) continue; diff --git a/test/test.cc b/test/test.cc index 747f8328..479fdf60 100644 --- a/test/test.cc +++ b/test/test.cc @@ -588,3 +588,128 @@ bool test_execute(const actor_config &config_const) { return false; } } + +//----------------------------------------------------------------------------- + +int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata, + unsigned flags) { + int err = mdbx_put(txn_guard.get(), dbi, &akey->value, &adata->value, flags); + if (err == MDBX_SUCCESS && config.params.speculum) { + const auto S_key = S(akey); + const auto S_data = S(adata); + const bool inserted = speculum.emplace(S_key, S_data).second; + assert(inserted); + (void)inserted; + } + return err; +} + +int testcase::replace(const keygen::buffer &akey, + const keygen::buffer &new_data, + const keygen::buffer &old_data, unsigned flags) { + if (config.params.speculum) { + const auto S_key = S(akey); + const auto S_old = S(old_data); + const auto S_new = S(new_data); + const auto removed = speculum.erase(SET::key_type(S_key, S_old)); + assert(removed == 1); + (void)removed; + const bool inserted = speculum.emplace(S_key, S_new).second; + assert(inserted); + (void)inserted; + } + return mdbx_replace(txn_guard.get(), dbi, &akey->value, &new_data->value, + &old_data->value, flags); +} + +int testcase::remove(const keygen::buffer &akey, const keygen::buffer &adata) { + if (config.params.speculum) { + const auto S_key = S(akey); + const auto S_data = S(adata); + const auto removed = speculum.erase(SET::key_type(S_key, S_data)); + assert(removed == 1); + (void)removed; + } + return mdbx_del(txn_guard.get(), dbi, &akey->value, &adata->value); +} + +bool testcase::speculum_verify() const { + if (!config.params.speculum) + return true; + + char dump_key[128], dump_value[128]; + char dump_mkey[128], dump_mvalue[128]; + + MDBX_cursor *cursor; + int err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor); + if (err != MDBX_SUCCESS) + failure_perror("mdbx_cursor_open()", err); + + bool rc = true; + MDBX_val akey, avalue; + MDBX_val mkey, mvalue; + err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_FIRST); + + assert(std::is_sorted(speculum.cbegin(), speculum.cend(), ItemCompare(this))); + auto it = speculum.cbegin(); + while (true) { + if (err != MDBX_SUCCESS) { + akey.iov_len = avalue.iov_len = 0; + akey.iov_base = avalue.iov_base = nullptr; + } + const auto S_key = S(akey); + const auto S_data = S(avalue); + if (it != speculum.cend()) { + mkey.iov_base = (void *)it->first.c_str(); + mkey.iov_len = it->first.size(); + mvalue.iov_base = (void *)it->second.c_str(); + mvalue.iov_len = it->second.size(); + } + if (err == MDBX_SUCCESS && it != speculum.cend() && S_key == it->first && + S_data == it->second) { + ++it; + err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT); + } else if (err == MDBX_SUCCESS && + (it == speculum.cend() || S_key < it->first || + (S_key == it->first && S_data < it->second))) { + if (it != speculum.cend()) { + log_error("extra pair: db{%s, %s} < mi{%s, %s}", + mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), + mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), + mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), + mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); + } else { + log_error("extra pair: db{%s, %s} < mi.END", + mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), + mdbx_dump_val(&avalue, dump_value, sizeof(dump_value))); + } + err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT); + rc = false; + } else if (it != speculum.cend() && + (err == MDBX_NOTFOUND || S_key > it->first || + (S_key == it->first && S_data > it->second))) { + if (err == MDBX_NOTFOUND) { + log_error("lost pair: db.END > mi{%s, %s}", + mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), + mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); + } else { + log_error("lost pair: db{%s, %s} > mi{%s, %s}", + mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), + mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), + mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), + mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); + } + ++it; + rc = false; + } else if (err == MDBX_NOTFOUND && it == speculum.cend()) { + break; + } else if (err != MDBX_SUCCESS) { + failure_perror("mdbx_cursor_get()", err); + } else { + assert(!"WTF?"); + } + } + + mdbx_cursor_close(cursor); + return rc; +} diff --git a/test/test.h b/test/test.h index 795a9992..b65d51aa 100644 --- a/test/test.h +++ b/test/test.h @@ -95,9 +95,45 @@ typedef std::unique_ptr scoped_cursor_guard; class testcase { protected: +#if HAVE_cxx17_std_string_view + using data_view = std::string_view; +#else + using data_view = std::string; +#endif + static inline data_view S(const MDBX_val &v) { + return data_view(static_cast(v.iov_base), v.iov_len); + } + static inline data_view S(const keygen::buffer &b) { return S(b->value); } + + using Item = std::pair; + struct ItemCompare { + const testcase *context; + ItemCompare(const testcase *owner) : context(owner) {} + + bool operator()(const Item &a, const Item &b) const { + MDBX_val va, vb; + va.iov_base = (void *)a.first.data(); + va.iov_len = a.first.size(); + vb.iov_base = (void *)b.first.data(); + vb.iov_len = b.first.size(); + int cmp = mdbx_cmp(context->txn_guard.get(), context->dbi, &va, &vb); + if (cmp == 0 && + (context->config.params.table_flags & MDBX_DUPSORT) != 0) { + va.iov_base = (void *)a.second.data(); + va.iov_len = a.second.size(); + vb.iov_base = (void *)b.second.data(); + vb.iov_len = b.second.size(); + cmp = mdbx_dcmp(context->txn_guard.get(), context->dbi, &va, &vb); + } + return cmp < 0; + } + }; + using SET = std::set; + const actor_config &config; const mdbx_pid_t pid; + MDBX_dbi dbi; scoped_db_guard db_guard; scoped_txn_guard txn_guard; scoped_cursor_guard cursor_guard; @@ -113,6 +149,14 @@ protected: mdbx_canary canary; } last; + SET speculum; + bool speculum_verify() const; + int insert(const keygen::buffer &akey, const keygen::buffer &adata, + unsigned flags); + int replace(const keygen::buffer &akey, const keygen::buffer &new_value, + const keygen::buffer &old_value, unsigned flags); + int remove(const keygen::buffer &akey, const keygen::buffer &adata); + static int oom_callback(MDBX_env *env, mdbx_pid_t pid, mdbx_tid_t tid, uint64_t txn, unsigned gap, size_t space, int retry); @@ -160,7 +204,8 @@ protected: public: testcase(const actor_config &config, const mdbx_pid_t pid) - : config(config), pid(pid), signalled(false), nops_completed(0) { + : config(config), pid(pid), signalled(false), nops_completed(0), + speculum(ItemCompare(this)) { start_timestamp.reset(); memset(&last, 0, sizeof(last)); } @@ -181,57 +226,11 @@ public: class testcase_hill : public testcase { using inherited = testcase; - -#if HAVE_cxx17_std_string_view - using data_view = std::string_view; -#else - using data_view = std::string; -#endif - - MDBX_dbi dbi; - using Item = std::pair; - struct ItemCompare { - const testcase_hill *context; - ItemCompare(const testcase_hill *owner) : context(owner) {} - - bool operator()(const Item &a, const Item &b) const { - MDBX_val va, vb; - va.iov_base = (void *)a.first.data(); - va.iov_len = a.first.size(); - vb.iov_base = (void *)b.first.data(); - vb.iov_len = b.first.size(); - int cmp = mdbx_cmp(context->txn_guard.get(), context->dbi, &va, &vb); - if (cmp == 0 && - (context->config.params.table_flags & MDBX_DUPSORT) != 0) { - va.iov_base = (void *)a.second.data(); - va.iov_len = a.second.size(); - vb.iov_base = (void *)b.second.data(); - vb.iov_len = b.second.size(); - cmp = mdbx_dcmp(context->txn_guard.get(), context->dbi, &va, &vb); - } - return cmp < 0; - } - }; - - using set = std::set; - set mirror, mirror_commited; - - bool verify() const; - int insert(const keygen::buffer &akey, const keygen::buffer &adata, - unsigned flags); - int replace(const keygen::buffer &akey, const keygen::buffer &new_value, - const keygen::buffer &old_value, unsigned flags); - int remove(const keygen::buffer &akey, const keygen::buffer &adata); - - static inline data_view S(const MDBX_val &v) { - return data_view(static_cast(v.iov_base), v.iov_len); - } - static inline data_view S(const keygen::buffer &b) { return S(b->value); } + SET speculum_commited; public: testcase_hill(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid), mirror(ItemCompare(this)), - mirror_commited(ItemCompare(this)) {} + : testcase(config, pid), speculum_commited(ItemCompare(this)) {} bool run() override; }; diff --git a/test/ttl.cc b/test/ttl.cc index e5631045..5782460d 100644 --- a/test/ttl.cc +++ b/test/ttl.cc @@ -29,7 +29,6 @@ static unsigned edge2count(uint64_t edge, unsigned count_max) { } bool testcase_ttl::run() { - MDBX_dbi dbi; int err = db_open__begin__table_create_open_clean(dbi); if (unlikely(err != MDBX_SUCCESS)) { log_notice("ttl: bailout-prepare due '%s'", mdbx_strerror(err));