mdbx-test: rework cycling for ttl & nested testcases.

Change-Id: If2f83187bd7998c2ddc7e2487a17d13648241b9c

test/nested.cc

@@ -24,20 +24,6 @@ bool testcase_nested::setup() {
     return false;
   }
 
-  constexpr unsigned reduce_nops_threshold = 5000;
-  if (nops_target > reduce_nops_threshold) {
-    unsigned batching = config.params.batch_read + config.params.batch_write;
-    unsigned reduce_nops =
-        reduce_nops_threshold +
-        5 * unsigned(sqrt(nops_target - reduce_nops_threshold +
-                          nops_target / (batching ? batching : 1)));
-    if (reduce_nops >= config.signal_nops) {
-      log_notice("nested: return target-nops from %u to %u", nops_target,
-                 reduce_nops);
-      nops_target = reduce_nops;
-    }
-  }
-
   keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
   key = keygen::alloc(config.params.keylen_max);
   data = keygen::alloc(config.params.datalen_max);
@@ -97,7 +83,6 @@ void testcase_nested::push_txn() {
   std::swap(txn_guard, std::get<0>(stack.top()));
   log_verbose("begin level#%zu txn #%" PRIu64 ", flags 0x%x, serial %" PRIu64,
               stack.size(), mdbx_txn_id(txn), flags, serial);
-  report(1);
 }
 
 bool testcase_nested::pop_txn(bool abort) {
@@ -137,7 +122,6 @@ bool testcase_nested::pop_txn(bool abort) {
     std::swap(speculum, std::get<3>(stack.top()));
   }
   stack.pop();
-  report(1);
   return should_continue;
 }
 
@@ -166,7 +150,8 @@ bool testcase_nested::stochastic_breakable_restart_with_nested(
 }
 
 bool testcase_nested::trim_tail(unsigned window_width) {
-  if (window_width) {
+  if (window_width || flipcoin()) {
+    clear_stepbystep_passed += window_width == 0;
     while (fifo.size() > window_width) {
       uint64_t tail_serial = fifo.back().first;
       const unsigned tail_count = fifo.back().second;
@@ -195,6 +180,7 @@ bool testcase_nested::trim_tail(unsigned window_width) {
                 fifo.size());
     db_table_clear(dbi, txn_guard.get());
     fifo.clear();
+    clear_wholetable_passed += 1;
     report(1);
   }
   return true;
@@ -215,6 +201,7 @@ retry:
         log_notice("nested: head-insert skip due '%s'", mdbx_strerror(err));
         head_count = n;
         stochastic_breakable_restart_with_nested(true);
+        dbfull_passed += 1;
         goto retry;
       }
       failure_perror("mdbx_put(head)", err);
@@ -226,7 +213,6 @@ retry:
     }
   }
 
-  report(1);
   return true;
 }
 
@@ -267,10 +253,15 @@ bool testcase_nested::run() {
   uint64_t seed =
       prng64_map2_white(config.params.keygen.seed) + config.actor_id;
 
-  while (should_continue()) {
+  clear_wholetable_passed = 0;
+  clear_stepbystep_passed = 0;
+  dbfull_passed = 0;
+  unsigned loops = 0;
+  while (true) {
     const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */;
-    const unsigned window_width =
+    const unsigned window_width = (!should_continue() || flipcoin_x4())
-        flipcoin_x4() ? 0 : edge2window(salt, window_max);
+                                      ? 0
+                                      : edge2window(salt, window_max);
     const unsigned head_count = edge2count(salt, count_max);
     log_debug("nested: step #%zu (serial %" PRIu64
               ", window %u, count %u) salt %" PRIu64,
@@ -287,13 +278,30 @@ bool testcase_nested::run() {
       return false;
     }
 
-    if (!grow_head(head_count))
+    if (should_continue() || !clear_wholetable_passed ||
-      return false;
+        !clear_stepbystep_passed) {
-    if (!stochastic_breakable_restart_with_nested())
+      unsigned underutilization_x256 =
-      log_notice("nested: skip commit/restart after head-grow");
+          txn_underutilization_x256(txn_guard.get());
-    if (!speculum_verify()) {
+      if (dbfull_passed > underutilization_x256) {
-      log_notice("nested: bailout after head-grow");
+        log_notice("nested: skip head-grow to avoid one more dbfull (was %u, "
-      return false;
+                   "underutilization %.2f%%)",
+                   dbfull_passed, underutilization_x256 / 2.560);
+        continue;
+      }
+      if (!grow_head(head_count))
+        return false;
+      if (!stochastic_breakable_restart_with_nested())
+        log_notice("nested: skip commit/restart after head-grow");
+      if (!speculum_verify()) {
+        log_notice("nested: bailout after head-grow");
+        return false;
+      }
+      loops += 1;
+    } else if (fifo.empty()) {
+      log_notice("nested: done %u whole loops", loops);
+      break;
+    } else {
+      log_notice("nested: done, wait for empty, skip head-grow");
     }
   }
 

test/test.h

@@ -236,7 +236,6 @@ public:
   testcase_ttl(const actor_config &config, const mdbx_pid_t pid)
       : testcase(config, pid) {}
   bool run() override;
-  bool setup() override;
 };
 
 class testcase_hill : public testcase {
@@ -299,6 +298,9 @@ class testcase_nested : public testcase {
   using FIFO = std::deque<std::pair<uint64_t, unsigned>>;
 
   uint64_t serial{0};
+  unsigned clear_wholetable_passed{0};
+  unsigned clear_stepbystep_passed{0};
+  unsigned dbfull_passed{0};
   FIFO fifo;
   std::stack<std::tuple<scoped_txn_guard, uint64_t, FIFO, SET>> stack;
 

test/ttl.cc

@@ -16,27 +16,6 @@
 #include <cmath>
 #include <deque>
 
-bool testcase_ttl::setup() {
-  if (!inherited::setup())
-    return false;
-
-  constexpr unsigned reduce_nops_threshold = 10000;
-  if (nops_target > reduce_nops_threshold) {
-    unsigned batching = config.params.batch_read + config.params.batch_write;
-    unsigned reduce_nops =
-        reduce_nops_threshold + (nops_target - reduce_nops_threshold +
-                                 nops_target / (batching ? batching : 1)) /
-                                    5;
-    if (reduce_nops >= config.signal_nops) {
-      log_notice("ttl: return target-nops from %u to %u", nops_target,
-                 reduce_nops);
-      nops_target = reduce_nops;
-    }
-  }
-
-  return true;
-}
-
 static unsigned edge2window(uint64_t edge, unsigned window_max) {
   const double rnd = u64_to_double1(bleach64(edge));
   const unsigned window = window_max - std::lrint(std::pow(window_max, rnd));
@@ -98,17 +77,23 @@ bool testcase_ttl::run() {
   std::deque<std::pair<uint64_t, unsigned>> fifo;
   uint64_t serial = 0;
   bool rc = false;
-  while (should_continue()) {
+  unsigned clear_wholetable_passed = 0;
+  unsigned clear_stepbystep_passed = 0;
+  unsigned dbfull_passed = 0;
+  unsigned loops = 0;
+  while (true) {
     const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */;
 
-    const unsigned window_width =
+    const unsigned window_width = (!should_continue() || flipcoin_x4())
-        flipcoin_x4() ? 0 : edge2window(salt, window_max);
+                                      ? 0
+                                      : edge2window(salt, window_max);
     unsigned head_count = edge2count(salt, count_max);
     log_debug("ttl: step #%zu (serial %" PRIu64
               ", window %u, count %u) salt %" PRIu64,
               nops_completed, serial, window_width, head_count, salt);
 
-    if (window_width) {
+    if (window_width || flipcoin()) {
+      clear_stepbystep_passed += window_width == 0;
       while (fifo.size() > window_width) {
         uint64_t tail_serial = fifo.back().first;
         const unsigned tail_count = fifo.back().second;
@@ -135,6 +120,7 @@ bool testcase_ttl::run() {
       log_trace("ttl: purge state");
       db_table_clear(dbi);
       fifo.clear();
+      clear_wholetable_passed += 1;
       report(1);
     }
 
@@ -148,41 +134,58 @@ bool testcase_ttl::run() {
       return false;
     }
 
-    fifo.push_front(std::make_pair(serial, head_count));
+    if (should_continue() || !clear_wholetable_passed ||
-  retry:
+        !clear_stepbystep_passed) {
-    for (unsigned n = 0; n < head_count; ++n) {
+      unsigned underutilization_x256 =
-      log_trace("ttl: insert-head %" PRIu64, serial);
+          txn_underutilization_x256(txn_guard.get());
-      generate_pair(serial);
+      if (dbfull_passed > underutilization_x256) {
-      err = insert(key, data, insert_flags);
+        log_notice("ttl: skip head-grow to avoid one more dbfull (was %u, "
-      if (unlikely(err != MDBX_SUCCESS)) {
+                   "underutilization %.2f%%)",
-        if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
+                   dbfull_passed, underutilization_x256 / 2.560);
-          log_notice("ttl: head-insert skip due '%s'", mdbx_strerror(err));
+        continue;
-          txn_restart(true, false);
+      }
-          serial = fifo.front().first;
+      fifo.push_front(std::make_pair(serial, head_count));
-          fifo.front().second = head_count = n;
+    retry:
-          goto retry;
+      for (unsigned n = 0; n < head_count; ++n) {
+        log_trace("ttl: insert-head %" PRIu64, serial);
+        generate_pair(serial);
+        err = insert(key, data, insert_flags);
+        if (unlikely(err != MDBX_SUCCESS)) {
+          if ((err == MDBX_TXN_FULL || err == MDBX_MAP_FULL) &&
+              config.params.ignore_dbfull) {
+            log_notice("ttl: head-insert skip due '%s'", mdbx_strerror(err));
+            txn_restart(true, false);
+            serial = fifo.front().first;
+            fifo.front().second = head_count = n;
+            dbfull_passed += 1;
+            goto retry;
+          }
+          failure_perror("mdbx_put(head)", err);
         }
-        failure_perror("mdbx_put(head)", err);
-      }
 
-      if (unlikely(!keyvalue_maker.increment(serial, 1))) {
+        if (unlikely(!keyvalue_maker.increment(serial, 1))) {
-        log_notice("ttl: unexpected key-space overflow");
+          log_notice("ttl: unexpected key-space overflow");
-        goto bailout;
+          goto bailout;
+        }
       }
+      err = breakable_restart();
+      if (unlikely(err != MDBX_SUCCESS)) {
+        log_notice("ttl: head-commit skip due '%s'", mdbx_strerror(err));
+        serial = fifo.front().first;
+        fifo.pop_front();
+      }
+      if (!speculum_verify()) {
+        log_notice("ttl: bailout after head-grow");
+        return false;
+      }
+      loops += 1;
+    } else if (fifo.empty()) {
+      log_notice("ttl: done %u whole loops", loops);
+      rc = true;
+      break;
+    } else {
+      log_notice("ttl: done, wait for empty, skip head-grow");
     }
-    err = breakable_restart();
-    if (unlikely(err != MDBX_SUCCESS)) {
-      log_notice("ttl: head-commit skip due '%s'", mdbx_strerror(err));
-      serial = fifo.front().first;
-      fifo.pop_front();
-    }
-    if (!speculum_verify()) {
-      log_notice("ttl: bailout after head-grow");
-      return false;
-    }
-
-    report(1);
-    rc = true;
   }
 
 bailout: