From 9cf9d6eac2a60ed64a02df557477f91609837cc3 Mon Sep 17 00:00:00 2001 From: Leonid Yuriev Date: Fri, 21 Jun 2019 22:42:45 +0300 Subject: [PATCH] mdbx-test: add `ttl` testcase. Change-Id: Ia5d164fde250e959226a53c63fcaf024ffe965a2 --- libmdbx.files | 1 + test/CMakeLists.txt | 1 + test/cases.cc | 1 + test/config.h | 3 +- test/keygen.cc | 4 +- test/keygen.h | 4 +- test/test.cc | 5 ++ test/test.h | 7 +++ test/ttl.cc | 128 ++++++++++++++++++++++++++++++++++++++++++++ test/utils.cc | 15 +----- test/utils.h | 44 ++++++++++++++- 11 files changed, 192 insertions(+), 21 deletions(-) create mode 100644 test/ttl.cc diff --git a/libmdbx.files b/libmdbx.files index 76ea29fb..f147554a 100644 --- a/libmdbx.files +++ b/libmdbx.files @@ -5,6 +5,7 @@ src/tools/CMakeLists.txt test/CMakeLists.txt test/append.cc test/copy.cc +test/ttl.cc tutorial/CMakeLists.txt tutorial/sample-mdbx.c AUTHORS diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 88fd09e0..bb4abd5d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -28,6 +28,7 @@ add_executable(${TARGET} utils.cc utils.h append.cc + ttl.cc ) target_link_libraries(${TARGET} diff --git a/test/cases.cc b/test/cases.cc index 1d41efc8..023a8002 100644 --- a/test/cases.cc +++ b/test/cases.cc @@ -67,6 +67,7 @@ void testcase_setup(const char *casename, actor_params ¶ms, configure_actor(last_space_id, ac_hill, nullptr, params); configure_actor(last_space_id, ac_jitter, nullptr, params); configure_actor(last_space_id, ac_hill, nullptr, params); + configure_actor(last_space_id, ac_ttl, nullptr, params); configure_actor(last_space_id, ac_try, nullptr, params); configure_actor(last_space_id, ac_copy, nullptr, params); configure_actor(last_space_id, ac_append, nullptr, params); diff --git a/test/config.h b/test/config.h index d6eaea2e..b8a4b682 100644 --- a/test/config.h +++ b/test/config.h @@ -28,7 +28,8 @@ enum actor_testcase { ac_jitter, ac_try, ac_copy, - ac_append + ac_append, + ac_ttl }; enum actor_status { diff --git a/test/keygen.cc b/test/keygen.cc index 5876fd8c..30cdf7a5 100644 --- a/test/keygen.cc +++ b/test/keygen.cc @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -176,7 +176,7 @@ bool maker::is_unordered() const { return (mapping.mesh >= serial_minwith || mapping.rotate) != 0; } -bool maker::increment(serial_t &serial, int delta) { +bool maker::increment(serial_t &serial, int delta) const { if (serial > mask(mapping.width)) { log_extra("keygen-increment: %" PRIu64 " > %" PRIu64 ", overflow", serial, mask(mapping.width)); diff --git a/test/keygen.h b/test/keygen.h index 890397b8..d0299e1e 100644 --- a/test/keygen.h +++ b/test/keygen.h @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -124,7 +124,7 @@ public: void make_ordered(); bool is_unordered() const; - bool increment(serial_t &serial, int delta); + bool increment(serial_t &serial, int delta) const; }; } /* namespace keygen */ diff --git a/test/test.cc b/test/test.cc index bfb9341d..ab1cd0d0 100644 --- a/test/test.cc +++ b/test/test.cc @@ -35,6 +35,8 @@ const char *testcase2str(const actor_testcase testcase) { return "copy"; case ac_append: return "append"; + case ac_ttl: + return "ttl"; } } @@ -488,6 +490,9 @@ bool test_execute(const actor_config &config) { case ac_append: test.reset(new testcase_append(config, pid)); break; + case ac_ttl: + test.reset(new testcase_ttl(config, pid)); + break; default: test.reset(new testcase(config, pid)); break; diff --git a/test/test.h b/test/test.h index e7260232..1dd6ed8f 100644 --- a/test/test.h +++ b/test/test.h @@ -153,6 +153,13 @@ public: virtual ~testcase() {} }; +class testcase_ttl : public testcase { +public: + testcase_ttl(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool run(); +}; + class testcase_hill : public testcase { public: testcase_hill(const actor_config &config, const mdbx_pid_t pid) diff --git a/test/ttl.cc b/test/ttl.cc new file mode 100644 index 00000000..dbc00146 --- /dev/null +++ b/test/ttl.cc @@ -0,0 +1,128 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" +#include +#include + +static unsigned edge2count(uint64_t edge, unsigned count_max) { + const double rnd = u64_to_double1(prng64_map1_white(edge)); + const unsigned count = std::lrint(std::pow(count_max, rnd)); + return count; +} + +static unsigned edge2window(uint64_t edge, unsigned window_max) { + const double rnd = u64_to_double1(prng64_map2_white(edge)); + const unsigned window = window_max - std::lrint(std::pow(window_max, rnd)); + return window - (window > 0); +} + +bool testcase_ttl::run() { + db_open(); + + txn_begin(false); + MDBX_dbi dbi = db_table_open(true); + int rc = mdbx_drop(txn_guard.get(), dbi, false); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_drop(delete=false)", rc); + txn_end(false); + + /* LY: тест "эмуляцией time-to-live": + * - организуется "скользящее окно", которое двигается вперед вдоль + * числовой оси каждую транзакцию. + * - по переднему краю "скользящего окна" записи добавляются в таблицу, + * а по заднему удаляются. + * - количество добавляемых/удаляемых записей псевдослучайно зависит + * от номера транзакции, но с экспоненциальным распределением. + * - размер "скользящего окна" также псевдослучайно зависит от номера + * транзакции с "отрицательным" экспоненциальным распределением + * MAX_WIDTH - exp(rnd(N)), при уменьшении окна сдвигается задний + * край и удаляются записи позади него. + * + * Таким образом имитируется поведение таблицы с TTL: записи стохастически + * добавляются и удаляются, но изредка происходят массивные удаления. + */ + + /* LY: для параметризации используем подходящие параметры, которые не имеют + * здесь смысла в первоначальном значении */ + const unsigned window_max = config.params.batch_read; + log_info("ttl: using `batch_read` value %u for window_max", window_max); + const unsigned count_max = config.params.batch_write; + log_info("ttl: using `batch_write` value %u for count_max", count_max); + + keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */); + keygen::buffer key = keygen::alloc(config.params.keylen_max); + keygen::buffer data = keygen::alloc(config.params.datalen_max); + const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT) + ? MDBX_NODUPDATA + : MDBX_NODUPDATA | MDBX_NOOVERWRITE; + + std::queue> fifo; + uint64_t serial = 0; + while (should_continue()) { + if (!txn_guard) + txn_begin(false); + const uint64_t salt = mdbx_txn_id(txn_guard.get()); + + const unsigned window = edge2window(salt, window_max); + log_trace("ttl: window %u at %" PRIu64, window, salt); + while (fifo.size() > window) { + uint64_t tail_serial = fifo.front().first; + const unsigned tail_count = fifo.front().second; + log_trace("ttl: pop-tail (serial %" PRIu64 ", count %u)", tail_serial, + tail_count); + fifo.pop(); + for (unsigned n = 0; n < tail_count; ++n) { + log_trace("ttl: remove-tail %" PRIu64, serial); + generate_pair(tail_serial, key, data, 0); + int err = mdbx_del(txn_guard.get(), dbi, &key->value, &data->value); + if (unlikely(err != MDBX_SUCCESS)) + failure_perror("mdbx_del(tail)", err); + if (unlikely(!keyvalue_maker.increment(tail_serial, 1))) + failure("ttl: unexpected key-space overflow on the tail"); + } + } + + txn_restart(false, false); + const unsigned head_count = edge2count(salt, count_max); + fifo.push(std::make_pair(serial, head_count)); + log_trace("ttl: push-head (serial %" PRIu64 ", count %u)", serial, + head_count); + + for (unsigned n = 0; n < head_count; ++n) { + log_trace("ttl: insert-head %" PRIu64, serial); + generate_pair(serial, key, data, 0); + int err = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value, + insert_flags); + if (unlikely(err != MDBX_SUCCESS)) + failure_perror("mdbx_put(head)", err); + + if (unlikely(!keyvalue_maker.increment(serial, 1))) + failure("uphill: unexpected key-space overflow"); + } + + txn_end(false); + report(1); + } + + if (dbi) { + if (config.params.drop_table && !mode_readonly()) { + txn_begin(false); + db_table_drop(dbi); + txn_end(false); + } else + db_table_close(dbi); + } + return true; +} diff --git a/test/utils.cc b/test/utils.cc index 326455a6..ddf47a4c 100644 --- a/test/utils.cc +++ b/test/utils.cc @@ -252,21 +252,8 @@ uint64_t entropy_ticks(void) { //----------------------------------------------------------------------------- -static __inline uint64_t bleach64(uint64_t dirty) { - return mul_64x64_high(bswap64(dirty), UINT64_C(17048867929148541611)); -} - -static __inline uint32_t bleach32(uint32_t dirty) { - return (uint32_t)((bswap32(dirty) * UINT64_C(2175734609)) >> 32); -} - -uint64_t prng64_careless(uint64_t &state) { - state = state * UINT64_C(6364136223846793005) + 1; - return state; -} - uint64_t prng64_white(uint64_t &state) { - state = state * UINT64_C(6364136223846793005) + UINT64_C(1442695040888963407); + state = prng64_map2_careless(state); return bleach64(state); } diff --git a/test/utils.h b/test/utils.h index 7bf3abd3..9ebfc9b9 100644 --- a/test/utils.h +++ b/test/utils.h @@ -1,4 +1,4 @@ -/* +/* * Copyright 2017-2019 Leonid Yuriev * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -327,7 +327,47 @@ std::string format(const char *fmt, ...); uint64_t entropy_ticks(void); uint64_t entropy_white(void); -uint64_t prng64_careless(uint64_t &state); +static inline uint64_t bleach64(uint64_t dirty) { + return mul_64x64_high(bswap64(dirty), UINT64_C(17048867929148541611)); +} + +static inline uint32_t bleach32(uint32_t dirty) { + return (uint32_t)((bswap32(dirty) * UINT64_C(2175734609)) >> 32); +} + +static inline uint64_t prng64_map1_careless(uint64_t state) { + return state * UINT64_C(6364136223846793005) + 1; +} + +static inline uint64_t prng64_map2_careless(uint64_t state) { + return (state + UINT64_C(1442695040888963407)) * + UINT64_C(6364136223846793005); +} + +static inline uint64_t prng64_map1_white(uint64_t state) { + return bleach64(prng64_map1_careless(state)); +} + +static inline uint64_t prng64_map2_white(uint64_t state) { + return bleach64(prng64_map2_careless(state)); +} + +static inline uint64_t prng64_careless(uint64_t &state) { + state = prng64_map1_careless(state); + return state; +} + +static inline double u64_to_double1(uint64_t v) { + union { + uint64_t u64; + double d; + } casting; + + casting.u64 = UINT64_C(0x3ff) << 52 | (v >> 12); + assert(casting.d >= 1.0 && casting.d < 2.0); + return casting.d - 1.0; +} + uint64_t prng64_white(uint64_t &state); uint32_t prng32(uint64_t &state); void prng_fill(uint64_t &state, void *ptr, size_t bytes);