mdbx-test: add ttl testcase.

Change-Id: Ia5d164fde250e959226a53c63fcaf024ffe965a2

libmdbx.files

@@ -5,6 +5,7 @@ src/tools/CMakeLists.txt
 test/CMakeLists.txt
 test/append.cc
 test/copy.cc
+test/ttl.cc
 tutorial/CMakeLists.txt
 tutorial/sample-mdbx.c
 AUTHORS

test/CMakeLists.txt

@@ -28,6 +28,7 @@ add_executable(${TARGET}
   utils.cc
   utils.h
   append.cc
+  ttl.cc
   )
 
 target_link_libraries(${TARGET}

test/cases.cc

@@ -67,6 +67,7 @@ void testcase_setup(const char *casename, actor_params &params,
     configure_actor(last_space_id, ac_hill, nullptr, params);
     configure_actor(last_space_id, ac_jitter, nullptr, params);
     configure_actor(last_space_id, ac_hill, nullptr, params);
+    configure_actor(last_space_id, ac_ttl, nullptr, params);
     configure_actor(last_space_id, ac_try, nullptr, params);
     configure_actor(last_space_id, ac_copy, nullptr, params);
     configure_actor(last_space_id, ac_append, nullptr, params);

test/config.h

@@ -28,7 +28,8 @@ enum actor_testcase {
   ac_jitter,
   ac_try,
   ac_copy,
-  ac_append
+  ac_append,
+  ac_ttl
 };
 
 enum actor_status {

test/keygen.cc

@@ -1,4 +1,4 @@
-/*
+/*
  * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
  * and other libmdbx authors: please see AUTHORS file.
  * All rights reserved.
@@ -176,7 +176,7 @@ bool maker::is_unordered() const {
   return (mapping.mesh >= serial_minwith || mapping.rotate) != 0;
 }
 
-bool maker::increment(serial_t &serial, int delta) {
+bool maker::increment(serial_t &serial, int delta) const {
   if (serial > mask(mapping.width)) {
     log_extra("keygen-increment: %" PRIu64 " > %" PRIu64 ", overflow", serial,
               mask(mapping.width));

test/keygen.h

@@ -1,4 +1,4 @@
-/*
+/*
  * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
  * and other libmdbx authors: please see AUTHORS file.
  * All rights reserved.
@@ -124,7 +124,7 @@ public:
   void make_ordered();
   bool is_unordered() const;
 
-  bool increment(serial_t &serial, int delta);
+  bool increment(serial_t &serial, int delta) const;
 };
 
 } /* namespace keygen */

test/test.cc

@@ -35,6 +35,8 @@ const char *testcase2str(const actor_testcase testcase) {
     return "copy";
   case ac_append:
     return "append";
+  case ac_ttl:
+    return "ttl";
   }
 }
 
@@ -488,6 +490,9 @@ bool test_execute(const actor_config &config) {
     case ac_append:
       test.reset(new testcase_append(config, pid));
       break;
+    case ac_ttl:
+      test.reset(new testcase_ttl(config, pid));
+      break;
     default:
       test.reset(new testcase(config, pid));
       break;

test/test.h

@@ -153,6 +153,13 @@ public:
   virtual ~testcase() {}
 };
 
+class testcase_ttl : public testcase {
+public:
+  testcase_ttl(const actor_config &config, const mdbx_pid_t pid)
+      : testcase(config, pid) {}
+  bool run();
+};
+
 class testcase_hill : public testcase {
 public:
   testcase_hill(const actor_config &config, const mdbx_pid_t pid)

test/ttl.cc

@@ -0,0 +1,128 @@
+/*
+ * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+#include "test.h"
+#include <cmath>
+#include <queue>
+
+static unsigned edge2count(uint64_t edge, unsigned count_max) {
+  const double rnd = u64_to_double1(prng64_map1_white(edge));
+  const unsigned count = std::lrint(std::pow(count_max, rnd));
+  return count;
+}
+
+static unsigned edge2window(uint64_t edge, unsigned window_max) {
+  const double rnd = u64_to_double1(prng64_map2_white(edge));
+  const unsigned window = window_max - std::lrint(std::pow(window_max, rnd));
+  return window - (window > 0);
+}
+
+bool testcase_ttl::run() {
+  db_open();
+
+  txn_begin(false);
+  MDBX_dbi dbi = db_table_open(true);
+  int rc = mdbx_drop(txn_guard.get(), dbi, false);
+  if (unlikely(rc != MDBX_SUCCESS))
+    failure_perror("mdbx_drop(delete=false)", rc);
+  txn_end(false);
+
+  /* LY: тест "эмуляцией time-to-live":
+   *  - организуется "скользящее окно", которое двигается вперед вдоль
+   *    числовой оси каждую транзакцию.
+   *  - по переднему краю "скользящего окна" записи добавляются в таблицу,
+   *    а по заднему удаляются.
+   *  - количество добавляемых/удаляемых записей псевдослучайно зависит
+   *    от номера транзакции, но с экспоненциальным распределением.
+   *  - размер "скользящего окна" также псевдослучайно зависит от номера
+   *    транзакции с "отрицательным" экспоненциальным распределением
+   *    MAX_WIDTH - exp(rnd(N)), при уменьшении окна сдвигается задний
+   *    край и удаляются записи позади него.
+   *
+   *  Таким образом имитируется поведение таблицы с TTL: записи стохастически
+   *  добавляются и удаляются, но изредка происходят массивные удаления.
+   */
+
+  /* LY: для параметризации используем подходящие параметры, которые не имеют
+   * здесь смысла в первоначальном значении */
+  const unsigned window_max = config.params.batch_read;
+  log_info("ttl: using `batch_read` value %u for window_max", window_max);
+  const unsigned count_max = config.params.batch_write;
+  log_info("ttl: using `batch_write` value %u for count_max", count_max);
+
+  keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
+  keygen::buffer key = keygen::alloc(config.params.keylen_max);
+  keygen::buffer data = keygen::alloc(config.params.datalen_max);
+  const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT)
+                                    ? MDBX_NODUPDATA
+                                    : MDBX_NODUPDATA | MDBX_NOOVERWRITE;
+
+  std::queue<std::pair<uint64_t, unsigned>> fifo;
+  uint64_t serial = 0;
+  while (should_continue()) {
+    if (!txn_guard)
+      txn_begin(false);
+    const uint64_t salt = mdbx_txn_id(txn_guard.get());
+
+    const unsigned window = edge2window(salt, window_max);
+    log_trace("ttl: window %u at %" PRIu64, window, salt);
+    while (fifo.size() > window) {
+      uint64_t tail_serial = fifo.front().first;
+      const unsigned tail_count = fifo.front().second;
+      log_trace("ttl: pop-tail (serial %" PRIu64 ", count %u)", tail_serial,
+                tail_count);
+      fifo.pop();
+      for (unsigned n = 0; n < tail_count; ++n) {
+        log_trace("ttl: remove-tail %" PRIu64, serial);
+        generate_pair(tail_serial, key, data, 0);
+        int err = mdbx_del(txn_guard.get(), dbi, &key->value, &data->value);
+        if (unlikely(err != MDBX_SUCCESS))
+          failure_perror("mdbx_del(tail)", err);
+        if (unlikely(!keyvalue_maker.increment(tail_serial, 1)))
+          failure("ttl: unexpected key-space overflow on the tail");
+      }
+    }
+
+    txn_restart(false, false);
+    const unsigned head_count = edge2count(salt, count_max);
+    fifo.push(std::make_pair(serial, head_count));
+    log_trace("ttl: push-head (serial %" PRIu64 ", count %u)", serial,
+              head_count);
+
+    for (unsigned n = 0; n < head_count; ++n) {
+      log_trace("ttl: insert-head %" PRIu64, serial);
+      generate_pair(serial, key, data, 0);
+      int err = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value,
+                         insert_flags);
+      if (unlikely(err != MDBX_SUCCESS))
+        failure_perror("mdbx_put(head)", err);
+
+      if (unlikely(!keyvalue_maker.increment(serial, 1)))
+        failure("uphill: unexpected key-space overflow");
+    }
+
+    txn_end(false);
+    report(1);
+  }
+
+  if (dbi) {
+    if (config.params.drop_table && !mode_readonly()) {
+      txn_begin(false);
+      db_table_drop(dbi);
+      txn_end(false);
+    } else
+      db_table_close(dbi);
+  }
+  return true;
+}

test/utils.cc

@@ -252,21 +252,8 @@ uint64_t entropy_ticks(void) {
 
 //-----------------------------------------------------------------------------
 
-static __inline uint64_t bleach64(uint64_t dirty) {
-  return mul_64x64_high(bswap64(dirty), UINT64_C(17048867929148541611));
-}
-
-static __inline uint32_t bleach32(uint32_t dirty) {
-  return (uint32_t)((bswap32(dirty) * UINT64_C(2175734609)) >> 32);
-}
-
-uint64_t prng64_careless(uint64_t &state) {
-  state = state * UINT64_C(6364136223846793005) + 1;
-  return state;
-}
-
 uint64_t prng64_white(uint64_t &state) {
-  state = state * UINT64_C(6364136223846793005) + UINT64_C(1442695040888963407);
+  state = prng64_map2_careless(state);
   return bleach64(state);
 }
 

test/utils.h

@@ -1,4 +1,4 @@
-/*
+/*
  * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
  * and other libmdbx authors: please see AUTHORS file.
  * All rights reserved.
@@ -327,7 +327,47 @@ std::string format(const char *fmt, ...);
 
 uint64_t entropy_ticks(void);
 uint64_t entropy_white(void);
-uint64_t prng64_careless(uint64_t &state);
+static inline uint64_t bleach64(uint64_t dirty) {
+  return mul_64x64_high(bswap64(dirty), UINT64_C(17048867929148541611));
+}
+
+static inline uint32_t bleach32(uint32_t dirty) {
+  return (uint32_t)((bswap32(dirty) * UINT64_C(2175734609)) >> 32);
+}
+
+static inline uint64_t prng64_map1_careless(uint64_t state) {
+  return state * UINT64_C(6364136223846793005) + 1;
+}
+
+static inline uint64_t prng64_map2_careless(uint64_t state) {
+  return (state + UINT64_C(1442695040888963407)) *
+         UINT64_C(6364136223846793005);
+}
+
+static inline uint64_t prng64_map1_white(uint64_t state) {
+  return bleach64(prng64_map1_careless(state));
+}
+
+static inline uint64_t prng64_map2_white(uint64_t state) {
+  return bleach64(prng64_map2_careless(state));
+}
+
+static inline uint64_t prng64_careless(uint64_t &state) {
+  state = prng64_map1_careless(state);
+  return state;
+}
+
+static inline double u64_to_double1(uint64_t v) {
+  union {
+    uint64_t u64;
+    double d;
+  } casting;
+
+  casting.u64 = UINT64_C(0x3ff) << 52 | (v >> 12);
+  assert(casting.d >= 1.0 && casting.d < 2.0);
+  return casting.d - 1.0;
+}
+
 uint64_t prng64_white(uint64_t &state);
 uint32_t prng32(uint64_t &state);
 void prng_fill(uint64_t &state, void *ptr, size_t bytes);