mdbx-test: add dataset verification to hill testcase.

Change-Id: I8b781e98a02a8c32eeb82e54821b0941556d9f8c

test/hill.cc

@@ -14,13 +14,131 @@
 
 #include "test.h"
 
+int testcase_hill::insert(const keygen::buffer &akey,
+                          const keygen::buffer &adata, unsigned flags) {
+  int err = mdbx_put(txn_guard.get(), dbi, &akey->value, &adata->value, flags);
+  if (err == MDBX_SUCCESS) {
+    const auto S_key = S(akey);
+    const auto S_data = S(adata);
+    const bool inserted = mirror.emplace(S_key, S_data).second;
+    assert(inserted);
+    (void)inserted;
+  }
+  return err;
+}
+
+int testcase_hill::replace(const keygen::buffer &akey,
+                           const keygen::buffer &new_data,
+                           const keygen::buffer &old_data, unsigned flags) {
+  const auto S_key = S(akey);
+  const auto S_old = S(old_data);
+  const auto S_new = S(new_data);
+  const auto removed = mirror.erase(set::key_type(S_key, S_old));
+  assert(removed == 1);
+  (void)removed;
+  const bool inserted = mirror.emplace(S_key, S_new).second;
+  assert(inserted);
+  (void)inserted;
+  return mdbx_replace(txn_guard.get(), dbi, &akey->value, &new_data->value,
+                      &old_data->value, flags);
+}
+
+int testcase_hill::remove(const keygen::buffer &akey,
+                          const keygen::buffer &adata) {
+  const auto S_key = S(akey);
+  const auto S_data = S(adata);
+  const auto removed = mirror.erase(set::key_type(S_key, S_data));
+  assert(removed == 1);
+  (void)removed;
+  return mdbx_del(txn_guard.get(), dbi, &akey->value, &adata->value);
+}
+
+bool testcase_hill::verify() const {
+  char dump_key[128], dump_value[128];
+  char dump_mkey[128], dump_mvalue[128];
+
+  MDBX_cursor *cursor;
+  int err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor);
+  if (err != MDBX_SUCCESS)
+    failure_perror("mdbx_cursor_open()", err);
+
+  bool rc = true;
+  MDBX_val akey, avalue;
+  MDBX_val mkey, mvalue;
+  err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_FIRST);
+
+  assert(std::is_sorted(mirror.cbegin(), mirror.cend(), ItemCompare(this)));
+  auto it = mirror.cbegin();
+  while (true) {
+    if (err != MDBX_SUCCESS) {
+      akey.iov_len = avalue.iov_len = 0;
+      akey.iov_base = avalue.iov_base = nullptr;
+    }
+    const auto S_key = S(akey);
+    const auto S_data = S(avalue);
+    if (it != mirror.cend()) {
+      mkey.iov_base = (void *)it->first.c_str();
+      mkey.iov_len = it->first.size();
+      mvalue.iov_base = (void *)it->second.c_str();
+      mvalue.iov_len = it->second.size();
+    }
+    if (err == MDBX_SUCCESS && it != mirror.cend() && S_key == it->first &&
+        S_data == it->second) {
+      ++it;
+      err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT);
+    } else if (err == MDBX_SUCCESS &&
+               (it == mirror.cend() || S_key < it->first ||
+                (S_key == it->first && S_data < it->second))) {
+      if (it != mirror.cend()) {
+        log_error("extra pair: db{%s, %s} < mi{%s, %s}",
+                  mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
+                  mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)),
+                  mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
+                  mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
+      } else {
+        log_error("extra pair: db{%s, %s} < mi.END",
+                  mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
+                  mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)));
+      }
+      err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT);
+      rc = false;
+    } else if (it != mirror.cend() &&
+               (err == MDBX_NOTFOUND || S_key > it->first ||
+                (S_key == it->first && S_data > it->second))) {
+      if (err == MDBX_NOTFOUND) {
+        log_error("lost pair: db.END > mi{%s, %s}",
+                  mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
+                  mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
+      } else {
+        log_error("lost pair: db{%s, %s} > mi{%s, %s}",
+                  mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
+                  mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)),
+                  mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
+                  mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
+      }
+      ++it;
+      rc = false;
+    } else if (err == MDBX_NOTFOUND && it == mirror.cend()) {
+      break;
+    } else if (err != MDBX_SUCCESS) {
+      failure_perror("mdbx_cursor_get()", err);
+    } else {
+      assert(!"WTF?");
+    }
+  }
+
+  mdbx_cursor_close(cursor);
+  return rc;
+}
+
 bool testcase_hill::run() {
-  MDBX_dbi dbi;
   int err = db_open__begin__table_create_open_clean(dbi);
   if (unlikely(err != MDBX_SUCCESS)) {
     log_notice("hill: bailout-prepare due '%s'", mdbx_strerror(err));
     return true;
   }
+  mirror.clear();
+  mirror_commited.clear();
 
   /* LY: тест "холмиком":
    *  - сначала наполняем таблицу циклическими CRUD-манипуляциями,
@@ -63,6 +181,7 @@ bool testcase_hill::run() {
   uint64_t commited_serial = serial_count;
   unsigned txn_nops = 0;
 
+  bool rc = false;
   while (should_continue()) {
     const keygen::serial_t a_serial = serial_count;
     if (unlikely(!keyvalue_maker.increment(serial_count, 1))) {
@@ -78,53 +197,74 @@ bool testcase_hill::run() {
     log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift,
               a_serial);
     generate_pair(a_serial, a_key, a_data_1, age_shift);
-    err = mdbx_put(txn_guard.get(), dbi, &a_key->value, &a_data_1->value,
+
-                   insert_flags);
+    err = insert(a_key, a_data_1, insert_flags);
     if (unlikely(err != MDBX_SUCCESS)) {
       if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
         log_notice("uphill: bailout at insert-a due '%s'", mdbx_strerror(err));
         txn_restart(true, false);
         serial_count = commited_serial;
+        mirror = mirror_commited;
         break;
       }
       failure_perror("mdbx_put(insert-a.1)", err);
     }
+    if (!verify()) {
+      log_notice("uphill: bailout after insert-a, before commit");
+      goto bailout;
+    }
 
     if (++txn_nops >= config.params.batch_write) {
       err = breakable_restart();
       if (unlikely(err != MDBX_SUCCESS)) {
         log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
         serial_count = commited_serial;
+        mirror = mirror_commited;
         break;
       }
+      mirror_commited = mirror;
       commited_serial = a_serial;
       txn_nops = 0;
+      if (!verify()) {
+        log_notice("uphill: bailout after insert-a, after commit");
+        goto bailout;
+      }
     }
 
     // создаем вторую запись из пары
     log_trace("uphill: insert-b %" PRIu64, b_serial);
     generate_pair(b_serial, b_key, b_data, 0);
-    err = mdbx_put(txn_guard.get(), dbi, &b_key->value, &b_data->value,
+    err = insert(b_key, b_data, insert_flags);
-                   insert_flags);
     if (unlikely(err != MDBX_SUCCESS)) {
       if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
         log_notice("uphill: bailout at insert-b due '%s'", mdbx_strerror(err));
         txn_restart(true, false);
         serial_count = commited_serial;
+        mirror = mirror_commited;
         break;
       }
       failure_perror("mdbx_put(insert-b)", err);
     }
+    if (!verify()) {
+      log_notice("uphill: bailout after insert-b, before commit");
+      goto bailout;
+    }
 
     if (++txn_nops >= config.params.batch_write) {
       err = breakable_restart();
       if (unlikely(err != MDBX_SUCCESS)) {
         log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
         serial_count = commited_serial;
+        mirror = mirror_commited;
         break;
       }
+      mirror_commited = mirror;
       commited_serial = a_serial;
       txn_nops = 0;
+      if (!verify()) {
+        log_notice("uphill: bailout after insert-b, after commit");
+        goto bailout;
+      }
     }
 
     // обновляем данные в первой записи
@@ -132,52 +272,73 @@ bool testcase_hill::run() {
               a_serial);
     generate_pair(a_serial, a_key, a_data_0, 0);
     checkdata("uphill: update-a", dbi, a_key->value, a_data_1->value);
-    err = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_0->value,
+    err = replace(a_key, a_data_0, a_data_1, update_flags);
-                       &a_data_1->value, update_flags);
     if (unlikely(err != MDBX_SUCCESS)) {
       if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
         log_notice("uphill: bailout at update-a due '%s'", mdbx_strerror(err));
         txn_restart(true, false);
         serial_count = commited_serial;
+        mirror = mirror_commited;
         break;
       }
       failure_perror("mdbx_replace(update-a: 1->0)", err);
     }
+    if (!verify()) {
+      log_notice("uphill: bailout after update-a, before commit");
+      goto bailout;
+    }
 
     if (++txn_nops >= config.params.batch_write) {
       err = breakable_restart();
       if (unlikely(err != MDBX_SUCCESS)) {
         log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
         serial_count = commited_serial;
+        mirror = mirror_commited;
         break;
       }
+      mirror_commited = mirror;
       commited_serial = a_serial;
       txn_nops = 0;
+      if (!verify()) {
+        log_notice("uphill: bailout after update-a, after commit");
+        goto bailout;
+      }
     }
 
     // удаляем вторую запись
     log_trace("uphill: delete-b %" PRIu64, b_serial);
     checkdata("uphill: delete-b", dbi, b_key->value, b_data->value);
-    err = mdbx_del(txn_guard.get(), dbi, &b_key->value, &b_data->value);
+    err = remove(b_key, b_data);
     if (unlikely(err != MDBX_SUCCESS)) {
       if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
         log_notice("uphill: bailout at delete-b due '%s'", mdbx_strerror(err));
         txn_restart(true, false);
         serial_count = commited_serial;
+        mirror = mirror_commited;
         break;
       }
       failure_perror("mdbx_del(b)", err);
     }
+    if (!verify()) {
+      log_notice("uphill: bailout after delete-b, before commit");
+      goto bailout;
+    }
 
     if (++txn_nops >= config.params.batch_write) {
       err = breakable_restart();
       if (unlikely(err != MDBX_SUCCESS)) {
         log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
         serial_count = commited_serial;
+        mirror = mirror_commited;
         break;
       }
+      mirror_commited = mirror;
       commited_serial = a_serial;
       txn_nops = 0;
+      if (!verify()) {
+        log_notice("uphill: bailout after delete-b, after commit");
+        goto bailout;
+      }
     }
 
     report(1);
@@ -204,101 +365,145 @@ bool testcase_hill::run() {
     generate_pair(a_serial, a_key, a_data_0, 0);
     generate_pair(a_serial, a_key, a_data_1, age_shift);
     checkdata("downhill: update-a", dbi, a_key->value, a_data_0->value);
-    err = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_1->value,
+    err = replace(a_key, a_data_1, a_data_0, update_flags);
-                       &a_data_0->value, update_flags);
     if (unlikely(err != MDBX_SUCCESS)) {
       if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
         log_notice("downhill: bailout at update-a due '%s'",
                    mdbx_strerror(err));
         txn_end(true);
+        mirror = mirror_commited;
         break;
       }
       failure_perror("mdbx_put(update-a: 0->1)", err);
     }
+    if (!verify()) {
+      log_notice("downhill: bailout after update-a, before commit");
+      break;
+    }
 
     if (++txn_nops >= config.params.batch_write) {
       err = breakable_restart();
       if (unlikely(err != MDBX_SUCCESS)) {
         log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
+        mirror = mirror_commited;
         break;
       }
+      mirror_commited = mirror;
       txn_nops = 0;
+      if (!verify()) {
+        log_notice("downhill: bailout after update-a, after commit");
+        break;
+      }
     }
 
     // создаем вторую запись из пары
     log_trace("downhill: insert-b %" PRIu64, b_serial);
     generate_pair(b_serial, b_key, b_data, 0);
-    err = mdbx_put(txn_guard.get(), dbi, &b_key->value, &b_data->value,
+    err = insert(b_key, b_data, insert_flags);
-                   insert_flags);
     if (unlikely(err != MDBX_SUCCESS)) {
       if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
         log_notice("downhill: bailout at insert-a due '%s'",
                    mdbx_strerror(err));
         txn_end(true);
+        mirror = mirror_commited;
         break;
       }
       failure_perror("mdbx_put(insert-b)", err);
     }
+    if (!verify()) {
+      log_notice("downhill: bailout after insert-b, before commit");
+      break;
+    }
 
     if (++txn_nops >= config.params.batch_write) {
       err = breakable_restart();
       if (unlikely(err != MDBX_SUCCESS)) {
         log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
+        mirror = mirror_commited;
         break;
       }
+      mirror_commited = mirror;
       txn_nops = 0;
+      if (!verify()) {
+        log_notice("downhill: bailout after insert-b, after commit");
+        break;
+      }
     }
 
     // удаляем первую запись
     log_trace("downhill: delete-a (age %" PRIu64 ") %" PRIu64, age_shift,
               a_serial);
     checkdata("downhill: delete-a", dbi, a_key->value, a_data_1->value);
-    err = mdbx_del(txn_guard.get(), dbi, &a_key->value, &a_data_1->value);
+    err = remove(a_key, a_data_1);
     if (unlikely(err != MDBX_SUCCESS)) {
       if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
         log_notice("downhill: bailout at delete-a due '%s'",
                    mdbx_strerror(err));
         txn_end(true);
+        mirror = mirror_commited;
         break;
       }
       failure_perror("mdbx_del(a)", err);
     }
+    if (!verify()) {
+      log_notice("downhill: bailout after delete-a, before commit");
+      break;
+    }
 
     if (++txn_nops >= config.params.batch_write) {
       err = breakable_restart();
       if (unlikely(err != MDBX_SUCCESS)) {
         log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
+        mirror = mirror_commited;
         break;
       }
+      mirror_commited = mirror;
       txn_nops = 0;
+      if (!verify()) {
+        log_notice("downhill: bailout after delete-a, after commit");
+        break;
+      }
     }
 
     // удаляем вторую запись
     log_trace("downhill: delete-b %" PRIu64, b_serial);
     checkdata("downhill: delete-b", dbi, b_key->value, b_data->value);
-    err = mdbx_del(txn_guard.get(), dbi, &b_key->value, &b_data->value);
+    err = remove(b_key, b_data);
     if (unlikely(err != MDBX_SUCCESS)) {
       if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
         log_notice("downhill: bailout at delete-b due '%s'",
                    mdbx_strerror(err));
         txn_end(true);
+        mirror = mirror_commited;
         break;
       }
       failure_perror("mdbx_del(b)", err);
     }
+    if (!verify()) {
+      log_notice("downhill: bailout after delete-b, before commit");
+      break;
+    }
 
     if (++txn_nops >= config.params.batch_write) {
       err = breakable_restart();
       if (unlikely(err != MDBX_SUCCESS)) {
         log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
+        mirror = mirror_commited;
         break;
       }
+      mirror_commited = mirror;
       txn_nops = 0;
+      if (!verify()) {
+        log_notice("downhill: bailout after delete-b, after commit");
+        goto bailout;
+      }
     }
 
     report(1);
   }
 
+  rc = verify();
+bailout:
   if (txn_guard) {
     err = breakable_commit();
     if (unlikely(err != MDBX_SUCCESS))
@@ -312,10 +517,10 @@ bool testcase_hill::run() {
       err = breakable_commit();
       if (unlikely(err != MDBX_SUCCESS)) {
         log_notice("hill: bailout-clean due '%s'", mdbx_strerror(err));
-        return true;
+        return rc;
       }
     } else
       db_table_close(dbi);
   }
-  return true;
+  return rc;
 }

test/test.h

@@ -22,6 +22,21 @@
 #include "osal.h"
 #include "utils.h"
 
+#include <set>
+
+#ifndef HAVE_cxx17_std_string_view
+#if __cplusplus >= 201703L && __has_include(<string_view>)
+#include <string_view>
+#define HAVE_cxx17_std_string_view 1
+#else
+#define HAVE_cxx17_std_string_view 0
+#endif
+#endif /* HAVE_cxx17_std_string_view */
+
+#if HAVE_cxx17_std_string_view
+#include <string_view>
+#endif
+
 bool test_execute(const actor_config &config);
 std::string thunk_param(const actor_config &config);
 void testcase_setup(const char *casename, actor_params &params,
@@ -165,10 +180,59 @@ public:
 };
 
 class testcase_hill : public testcase {
+  using inherited = testcase;
+
+#if HAVE_cxx17_std_string_view
+  using data_view = std::string_view;
+#else
+  using data_view = std::string;
+#endif
+
+  MDBX_dbi dbi;
+  using Item = std::pair<std::string, std::string>;
+  struct ItemCompare {
+    const testcase_hill *context;
+    ItemCompare(const testcase_hill *owner) : context(owner) {}
+
+    bool operator()(const Item &a, const Item &b) const {
+      MDBX_val va, vb;
+      va.iov_base = (void *)a.first.data();
+      va.iov_len = a.first.size();
+      vb.iov_base = (void *)b.first.data();
+      vb.iov_len = b.first.size();
+      int cmp = mdbx_cmp(context->txn_guard.get(), context->dbi, &va, &vb);
+      if (cmp == 0 &&
+          (context->config.params.table_flags & MDBX_DUPSORT) != 0) {
+        va.iov_base = (void *)a.second.data();
+        va.iov_len = a.second.size();
+        vb.iov_base = (void *)b.second.data();
+        vb.iov_len = b.second.size();
+        cmp = mdbx_dcmp(context->txn_guard.get(), context->dbi, &va, &vb);
+      }
+      return cmp < 0;
+    }
+  };
+
+  using set = std::set<Item, ItemCompare>;
+  set mirror, mirror_commited;
+
+  bool verify() const;
+  int insert(const keygen::buffer &akey, const keygen::buffer &adata,
+             unsigned flags);
+  int replace(const keygen::buffer &akey, const keygen::buffer &new_value,
+              const keygen::buffer &old_value, unsigned flags);
+  int remove(const keygen::buffer &akey, const keygen::buffer &adata);
+
+  static inline data_view S(const MDBX_val &v) {
+    return data_view(static_cast<const char *>(v.iov_base), v.iov_len);
+  }
+  static inline data_view S(const keygen::buffer &b) { return S(b->value); }
+
 public:
   testcase_hill(const actor_config &config, const mdbx_pid_t pid)
-      : testcase(config, pid) {}
+      : testcase(config, pid), mirror(ItemCompare(this)),
-  bool run();
+        mirror_commited(ItemCompare(this)) {}
+  bool run() override;
 };
 
 class testcase_append : public testcase {