From 0639f54280f8e698c6810afd06598d4c94bc7c5b Mon Sep 17 00:00:00 2001 From: Leonid Yuriev Date: Sun, 3 Feb 2019 22:37:57 +0300 Subject: [PATCH] mdbx-test: add 'append' testcase. Change-Id: I71620ea1a019e16b8e3d84a81dcc042961eae5b5 --- libmdbx.files | 1 + src/tools/mdbx_load.c | 2 +- test/CMakeLists.txt | 1 + test/append.cc | 132 ++++++++++++++++++++++++++++++++++++++++++ test/cases.cc | 1 + test/config.h | 3 +- test/keygen.cc | 9 +++ test/keygen.h | 2 + test/main.cc | 4 ++ test/test.cc | 31 +++++++++- test/test.h | 9 +++ test/test.vcxproj | 1 + test/utils.h | 9 ++- 13 files changed, 200 insertions(+), 5 deletions(-) create mode 100644 test/append.cc diff --git a/libmdbx.files b/libmdbx.files index 653b0397..76ea29fb 100644 --- a/libmdbx.files +++ b/libmdbx.files @@ -3,6 +3,7 @@ README-RU.md pcrf_test/CMakeLists.txt src/tools/CMakeLists.txt test/CMakeLists.txt +test/append.cc test/copy.cc tutorial/CMakeLists.txt tutorial/sample-mdbx.c diff --git a/src/tools/mdbx_load.c b/src/tools/mdbx_load.c index 1535b192..2458b0e2 100644 --- a/src/tools/mdbx_load.c +++ b/src/tools/mdbx_load.c @@ -487,7 +487,7 @@ int main(int argc, char *argv[]) { if (dbi_flags & MDBX_DUPSORT) { if (prevk.iov_len == key.iov_len && memcmp(prevk.iov_base, key.iov_base, key.iov_len) == 0) - appflag = MDBX_CURRENT | MDBX_APPENDDUP; + appflag = MDBX_APPEND | MDBX_APPENDDUP; else memcpy(prevk.iov_base, key.iov_base, prevk.iov_len = key.iov_len); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ca7dd794..88fd09e0 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -27,6 +27,7 @@ add_executable(${TARGET} try.cc utils.cc utils.h + append.cc ) target_link_libraries(${TARGET} diff --git a/test/append.cc b/test/append.cc new file mode 100644 index 00000000..3ce53eb2 --- /dev/null +++ b/test/append.cc @@ -0,0 +1,132 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +bool testcase_append::run() { + db_open(); + + txn_begin(false); + MDBX_dbi dbi = db_table_open(true); + int rc = mdbx_drop(txn_guard.get(), dbi, false); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_drop(delete=false)", rc); + + keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */); + /* LY: тест наполнения таблиц в append-режиме, + * при котором записи добавляются строго в конец (в порядке сортировки) */ + const unsigned flags = (config.params.table_flags & MDBX_DUPSORT) + ? MDBX_APPEND | MDBX_APPENDDUP + : MDBX_APPEND; + keyvalue_maker.make_ordered(); + + key = keygen::alloc(config.params.keylen_max); + data = keygen::alloc(config.params.datalen_max); + keygen::buffer last_key = keygen::alloc(config.params.keylen_max); + keygen::buffer last_data = keygen::alloc(config.params.datalen_max); + last_key->value.iov_base = last_key->bytes; + last_key->value.iov_len = 0; + last_data->value.iov_base = last_data->bytes; + last_data->value.iov_len = 0; + + simple_checksum inserted_checksum; + uint64_t inserted_number = 0; + uint64_t serial_count = 0; + unsigned txn_nops = 0; + while (should_continue()) { + const keygen::serial_t serial = serial_count; + if (!keyvalue_maker.increment(serial_count, 1)) { + // дошли до границы пространства ключей + break; + } + + log_trace("append: append-a %" PRIu64, serial); + generate_pair(serial, key, data); + int cmp = inserted_number ? mdbx_cmp(txn_guard.get(), dbi, &key->value, + &last_key->value) + : 1; + if (cmp == 0 && (config.params.table_flags & MDBX_DUPSORT)) + cmp = mdbx_dcmp(txn_guard.get(), dbi, &data->value, &last_data->value); + + rc = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value, flags); + if (cmp > 0) { + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_put(appenda-a)", rc); + memcpy(last_key->value.iov_base, key->value.iov_base, + last_key->value.iov_len = key->value.iov_len); + memcpy(last_data->value.iov_base, data->value.iov_base, + last_data->value.iov_len = data->value.iov_len); + ++inserted_number; + inserted_checksum.push((uint32_t)inserted_number, key->value); + inserted_checksum.push(10639, data->value); + } else { + if (unlikely(rc != MDBX_EKEYMISMATCH)) + failure_perror("mdbx_put(appenda-a) != MDBX_EKEYMISMATCH", rc); + } + + if (++txn_nops >= config.params.batch_write) { + txn_restart(false, false); + txn_nops = 0; + } + + report(1); + } + + txn_restart(false, true); + //---------------------------------------------------------------------------- + cursor_open(dbi); + + MDBX_val check_key, check_data; + rc = mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_FIRST); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_cursor_get(MDBX_FIRST)", rc); + + simple_checksum read_checksum; + uint64_t read_count = 0; + while (rc == MDBX_SUCCESS) { + ++read_count; + read_checksum.push((uint32_t)read_count, check_key); + read_checksum.push(10639, check_data); + + rc = + mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_NEXT); + } + + if (unlikely(rc != MDBX_NOTFOUND)) + failure_perror("mdbx_cursor_get(MDBX_NEXT) != EOF", rc); + + if (unlikely(read_count != inserted_number)) + failure("read_count(%" PRIu64 ") != inserted_number(%" PRIu64 ")", + read_count, inserted_number); + + if (unlikely(read_checksum.value != inserted_checksum.value)) + failure("read_checksum(0x%016" PRIu64 ") " + "!= inserted_checksum(0x%016" PRIu64 ")", + read_checksum.value, inserted_checksum.value); + + cursor_close(); + //---------------------------------------------------------------------------- + if (txn_guard) + txn_end(false); + + if (dbi) { + if (config.params.drop_table && !mode_readonly()) { + txn_begin(false); + db_table_drop(dbi); + txn_end(false); + } else + db_table_close(dbi); + } + return true; +} diff --git a/test/cases.cc b/test/cases.cc index 62dadcec..1d41efc8 100644 --- a/test/cases.cc +++ b/test/cases.cc @@ -69,6 +69,7 @@ void testcase_setup(const char *casename, actor_params ¶ms, configure_actor(last_space_id, ac_hill, nullptr, params); configure_actor(last_space_id, ac_try, nullptr, params); configure_actor(last_space_id, ac_copy, nullptr, params); + configure_actor(last_space_id, ac_append, nullptr, params); log_notice("<<< testcase_setup(%s): done", casename); } else { failure("unknown testcase `%s`", casename); diff --git a/test/config.h b/test/config.h index c96f4b12..d6eaea2e 100644 --- a/test/config.h +++ b/test/config.h @@ -27,7 +27,8 @@ enum actor_testcase { ac_deadwrite, ac_jitter, ac_try, - ac_copy + ac_copy, + ac_append }; enum actor_status { diff --git a/test/keygen.cc b/test/keygen.cc index 4753899e..5876fd8c 100644 --- a/test/keygen.cc +++ b/test/keygen.cc @@ -167,6 +167,15 @@ void maker::setup(const config::actor_params_pod &actor, unsigned actor_id, base = 0; } +void maker::make_ordered() { + mapping.mesh = 0; + mapping.rotate = 0; +} + +bool maker::is_unordered() const { + return (mapping.mesh >= serial_minwith || mapping.rotate) != 0; +} + bool maker::increment(serial_t &serial, int delta) { if (serial > mask(mapping.width)) { log_extra("keygen-increment: %" PRIu64 " > %" PRIu64 ", overflow", serial, diff --git a/test/keygen.h b/test/keygen.h index 1466720d..890397b8 100644 --- a/test/keygen.h +++ b/test/keygen.h @@ -121,6 +121,8 @@ public: serial_t value_age); void setup(const config::actor_params_pod &actor, unsigned actor_id, unsigned thread_number); + void make_ordered(); + bool is_unordered() const; bool increment(serial_t &serial, int delta); }; diff --git a/test/main.cc b/test/main.cc index 41868ccf..f3ee76b6 100644 --- a/test/main.cc +++ b/test/main.cc @@ -341,6 +341,10 @@ int main(int argc, char *const argv[]) { configure_actor(last_space_id, ac_copy, value, params); continue; } + if (config::parse_option(argc, argv, narg, "append", nullptr)) { + configure_actor(last_space_id, ac_append, value, params); + continue; + } if (config::parse_option(argc, argv, narg, "failfast", global::config::failfast)) continue; diff --git a/test/test.cc b/test/test.cc index b81aecb2..e34bd7f0 100644 --- a/test/test.cc +++ b/test/test.cc @@ -33,6 +33,8 @@ const char *testcase2str(const actor_testcase testcase) { return "try"; case ac_copy: return "copy"; + case ac_append: + return "append"; } } @@ -185,9 +187,33 @@ void testcase::txn_end(bool abort) { log_trace("<< txn_end(%s)", abort ? "abort" : "commit"); } +void testcase::cursor_open(unsigned dbi) { + log_trace(">> cursor_open(%u)", dbi); + assert(!cursor_guard); + assert(txn_guard); + + MDBX_cursor *cursor = nullptr; + int rc = mdbx_cursor_open(txn_guard.get(), dbi, &cursor); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_cursor_open()", rc); + cursor_guard.reset(cursor); + + log_trace("<< cursor_open(%u)", dbi); +} + +void testcase::cursor_close() { + log_trace(">> cursor_close()"); + assert(cursor_guard); + MDBX_cursor *cursor = cursor_guard.release(); + mdbx_cursor_close(cursor); + log_trace("<< cursor_close()"); +} + void testcase::txn_restart(bool abort, bool readonly, unsigned flags) { if (txn_guard) txn_end(abort); + if (cursor_guard) + cursor_close(); txn_begin(readonly, flags); } @@ -396,7 +422,7 @@ void testcase::db_table_drop(MDBX_dbi handle) { if (config.params.drop_table) { int rc = mdbx_drop(txn_guard.get(), handle, true); if (unlikely(rc != MDBX_SUCCESS)) - failure_perror("mdbx_drop()", rc); + failure_perror("mdbx_drop(delete=true)", rc); log_trace("<< testcase::db_table_drop"); } else { log_trace("<< testcase::db_table_drop: not needed"); @@ -458,6 +484,9 @@ bool test_execute(const actor_config &config) { case ac_copy: test.reset(new testcase_copy(config, pid)); break; + case ac_append: + test.reset(new testcase_append(config, pid)); + break; default: test.reset(new testcase(config, pid)); break; diff --git a/test/test.h b/test/test.h index e6f4b046..e7260232 100644 --- a/test/test.h +++ b/test/test.h @@ -107,6 +107,8 @@ protected: void txn_begin(bool readonly, unsigned flags = 0); void txn_end(bool abort); void txn_restart(bool abort, bool readonly, unsigned flags = 0); + void cursor_open(unsigned dbi); + void cursor_close(); void txn_inject_writefault(void); void txn_inject_writefault(MDBX_txn *txn); void fetch_canary(); @@ -158,6 +160,13 @@ public: bool run(); }; +class testcase_append : public testcase { +public: + testcase_append(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool run(); +}; + class testcase_deadread : public testcase { public: testcase_deadread(const actor_config &config, const mdbx_pid_t pid) diff --git a/test/test.vcxproj b/test/test.vcxproj index b47bc7a8..9eb62cdc 100644 --- a/test/test.vcxproj +++ b/test/test.vcxproj @@ -181,6 +181,7 @@ + diff --git a/test/utils.h b/test/utils.h index 1427cba5..7bf3abd3 100644 --- a/test/utils.h +++ b/test/utils.h @@ -288,6 +288,7 @@ struct simple_checksum { void push(uint32_t data) { value += data * UINT64_C(9386433910765580089) + 1; value ^= value >> 41; + value *= UINT64_C(0xBD9CACC22C6E9571); } void push(uint64_t data) { @@ -304,11 +305,15 @@ struct simple_checksum { } void push(const double &data) { push(&data, sizeof(double)); } - void push(const char *cstr) { push(cstr, strlen(cstr)); } - void push(const std::string &str) { push(str.data(), str.size()); } + void push(unsigned salt, const MDBX_val &val) { + push(val.iov_len); + push(salt); + push(val.iov_base, val.iov_len); + } + #if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) void push(const HANDLE &handle) { push(&handle, sizeof(handle)); } #endif /* _WINDOWS */