mdbx-test: add 'append' testcase.

Change-Id: I71620ea1a019e16b8e3d84a81dcc042961eae5b5

libmdbx.files

@@ -3,6 +3,7 @@ README-RU.md
 pcrf_test/CMakeLists.txt
 src/tools/CMakeLists.txt
 test/CMakeLists.txt
+test/append.cc
 test/copy.cc
 tutorial/CMakeLists.txt
 tutorial/sample-mdbx.c

src/tools/mdbx_load.c

@@ -487,7 +487,7 @@ int main(int argc, char *argv[]) {
         if (dbi_flags & MDBX_DUPSORT) {
           if (prevk.iov_len == key.iov_len &&
               memcmp(prevk.iov_base, key.iov_base, key.iov_len) == 0)
-            appflag = MDBX_CURRENT | MDBX_APPENDDUP;
+            appflag = MDBX_APPEND | MDBX_APPENDDUP;
           else
             memcpy(prevk.iov_base, key.iov_base, prevk.iov_len = key.iov_len);
         }

test/CMakeLists.txt

@@ -27,6 +27,7 @@ add_executable(${TARGET}
   try.cc
   utils.cc
   utils.h
+  append.cc
   )
 
 target_link_libraries(${TARGET}

test/append.cc

@@ -0,0 +1,132 @@
+/*
+ * Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+#include "test.h"
+
+bool testcase_append::run() {
+  db_open();
+
+  txn_begin(false);
+  MDBX_dbi dbi = db_table_open(true);
+  int rc = mdbx_drop(txn_guard.get(), dbi, false);
+  if (unlikely(rc != MDBX_SUCCESS))
+    failure_perror("mdbx_drop(delete=false)", rc);
+
+  keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
+  /* LY: тест наполнения таблиц в append-режиме,
+   * при котором записи добавляются строго в конец (в порядке сортировки) */
+  const unsigned flags = (config.params.table_flags & MDBX_DUPSORT)
+                             ? MDBX_APPEND | MDBX_APPENDDUP
+                             : MDBX_APPEND;
+  keyvalue_maker.make_ordered();
+
+  key = keygen::alloc(config.params.keylen_max);
+  data = keygen::alloc(config.params.datalen_max);
+  keygen::buffer last_key = keygen::alloc(config.params.keylen_max);
+  keygen::buffer last_data = keygen::alloc(config.params.datalen_max);
+  last_key->value.iov_base = last_key->bytes;
+  last_key->value.iov_len = 0;
+  last_data->value.iov_base = last_data->bytes;
+  last_data->value.iov_len = 0;
+
+  simple_checksum inserted_checksum;
+  uint64_t inserted_number = 0;
+  uint64_t serial_count = 0;
+  unsigned txn_nops = 0;
+  while (should_continue()) {
+    const keygen::serial_t serial = serial_count;
+    if (!keyvalue_maker.increment(serial_count, 1)) {
+      // дошли до границы пространства ключей
+      break;
+    }
+
+    log_trace("append: append-a %" PRIu64, serial);
+    generate_pair(serial, key, data);
+    int cmp = inserted_number ? mdbx_cmp(txn_guard.get(), dbi, &key->value,
+                                         &last_key->value)
+                              : 1;
+    if (cmp == 0 && (config.params.table_flags & MDBX_DUPSORT))
+      cmp = mdbx_dcmp(txn_guard.get(), dbi, &data->value, &last_data->value);
+
+    rc = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value, flags);
+    if (cmp > 0) {
+      if (unlikely(rc != MDBX_SUCCESS))
+        failure_perror("mdbx_put(appenda-a)", rc);
+      memcpy(last_key->value.iov_base, key->value.iov_base,
+             last_key->value.iov_len = key->value.iov_len);
+      memcpy(last_data->value.iov_base, data->value.iov_base,
+             last_data->value.iov_len = data->value.iov_len);
+      ++inserted_number;
+      inserted_checksum.push((uint32_t)inserted_number, key->value);
+      inserted_checksum.push(10639, data->value);
+    } else {
+      if (unlikely(rc != MDBX_EKEYMISMATCH))
+        failure_perror("mdbx_put(appenda-a) != MDBX_EKEYMISMATCH", rc);
+    }
+
+    if (++txn_nops >= config.params.batch_write) {
+      txn_restart(false, false);
+      txn_nops = 0;
+    }
+
+    report(1);
+  }
+
+  txn_restart(false, true);
+  //----------------------------------------------------------------------------
+  cursor_open(dbi);
+
+  MDBX_val check_key, check_data;
+  rc = mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_FIRST);
+  if (unlikely(rc != MDBX_SUCCESS))
+    failure_perror("mdbx_cursor_get(MDBX_FIRST)", rc);
+
+  simple_checksum read_checksum;
+  uint64_t read_count = 0;
+  while (rc == MDBX_SUCCESS) {
+    ++read_count;
+    read_checksum.push((uint32_t)read_count, check_key);
+    read_checksum.push(10639, check_data);
+
+    rc =
+        mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_NEXT);
+  }
+
+  if (unlikely(rc != MDBX_NOTFOUND))
+    failure_perror("mdbx_cursor_get(MDBX_NEXT) != EOF", rc);
+
+  if (unlikely(read_count != inserted_number))
+    failure("read_count(%" PRIu64 ") != inserted_number(%" PRIu64 ")",
+            read_count, inserted_number);
+
+  if (unlikely(read_checksum.value != inserted_checksum.value))
+    failure("read_checksum(0x%016" PRIu64 ") "
+            "!= inserted_checksum(0x%016" PRIu64 ")",
+            read_checksum.value, inserted_checksum.value);
+
+  cursor_close();
+  //----------------------------------------------------------------------------
+  if (txn_guard)
+    txn_end(false);
+
+  if (dbi) {
+    if (config.params.drop_table && !mode_readonly()) {
+      txn_begin(false);
+      db_table_drop(dbi);
+      txn_end(false);
+    } else
+      db_table_close(dbi);
+  }
+  return true;
+}

test/cases.cc

@@ -69,6 +69,7 @@ void testcase_setup(const char *casename, actor_params &params,
     configure_actor(last_space_id, ac_hill, nullptr, params);
     configure_actor(last_space_id, ac_try, nullptr, params);
     configure_actor(last_space_id, ac_copy, nullptr, params);
+    configure_actor(last_space_id, ac_append, nullptr, params);
     log_notice("<<< testcase_setup(%s): done", casename);
   } else {
     failure("unknown testcase `%s`", casename);

test/config.h

@@ -27,7 +27,8 @@ enum actor_testcase {
   ac_deadwrite,
   ac_jitter,
   ac_try,
-  ac_copy
+  ac_copy,
+  ac_append
 };
 
 enum actor_status {

test/keygen.cc

@@ -167,6 +167,15 @@ void maker::setup(const config::actor_params_pod &actor, unsigned actor_id,
   base = 0;
 }
 
+void maker::make_ordered() {
+  mapping.mesh = 0;
+  mapping.rotate = 0;
+}
+
+bool maker::is_unordered() const {
+  return (mapping.mesh >= serial_minwith || mapping.rotate) != 0;
+}
+
 bool maker::increment(serial_t &serial, int delta) {
   if (serial > mask(mapping.width)) {
     log_extra("keygen-increment: %" PRIu64 " > %" PRIu64 ", overflow", serial,

test/keygen.h

@@ -121,6 +121,8 @@ public:
             serial_t value_age);
   void setup(const config::actor_params_pod &actor, unsigned actor_id,
              unsigned thread_number);
+  void make_ordered();
+  bool is_unordered() const;
 
   bool increment(serial_t &serial, int delta);
 };

test/main.cc

@@ -341,6 +341,10 @@ int main(int argc, char *const argv[]) {
       configure_actor(last_space_id, ac_copy, value, params);
       continue;
     }
+    if (config::parse_option(argc, argv, narg, "append", nullptr)) {
+      configure_actor(last_space_id, ac_append, value, params);
+      continue;
+    }
     if (config::parse_option(argc, argv, narg, "failfast",
                              global::config::failfast))
       continue;

test/test.cc

@@ -33,6 +33,8 @@ const char *testcase2str(const actor_testcase testcase) {
     return "try";
   case ac_copy:
     return "copy";
+  case ac_append:
+    return "append";
   }
 }
 
@@ -185,9 +187,33 @@ void testcase::txn_end(bool abort) {
   log_trace("<< txn_end(%s)", abort ? "abort" : "commit");
 }
 
+void testcase::cursor_open(unsigned dbi) {
+  log_trace(">> cursor_open(%u)", dbi);
+  assert(!cursor_guard);
+  assert(txn_guard);
+
+  MDBX_cursor *cursor = nullptr;
+  int rc = mdbx_cursor_open(txn_guard.get(), dbi, &cursor);
+  if (unlikely(rc != MDBX_SUCCESS))
+    failure_perror("mdbx_cursor_open()", rc);
+  cursor_guard.reset(cursor);
+
+  log_trace("<< cursor_open(%u)", dbi);
+}
+
+void testcase::cursor_close() {
+  log_trace(">> cursor_close()");
+  assert(cursor_guard);
+  MDBX_cursor *cursor = cursor_guard.release();
+  mdbx_cursor_close(cursor);
+  log_trace("<< cursor_close()");
+}
+
 void testcase::txn_restart(bool abort, bool readonly, unsigned flags) {
   if (txn_guard)
     txn_end(abort);
+  if (cursor_guard)
+    cursor_close();
   txn_begin(readonly, flags);
 }
 
@@ -396,7 +422,7 @@ void testcase::db_table_drop(MDBX_dbi handle) {
   if (config.params.drop_table) {
     int rc = mdbx_drop(txn_guard.get(), handle, true);
     if (unlikely(rc != MDBX_SUCCESS))
-      failure_perror("mdbx_drop()", rc);
+      failure_perror("mdbx_drop(delete=true)", rc);
     log_trace("<< testcase::db_table_drop");
   } else {
     log_trace("<< testcase::db_table_drop: not needed");
@@ -458,6 +484,9 @@ bool test_execute(const actor_config &config) {
     case ac_copy:
       test.reset(new testcase_copy(config, pid));
       break;
+    case ac_append:
+      test.reset(new testcase_append(config, pid));
+      break;
     default:
       test.reset(new testcase(config, pid));
       break;

test/test.h

@@ -107,6 +107,8 @@ protected:
   void txn_begin(bool readonly, unsigned flags = 0);
   void txn_end(bool abort);
   void txn_restart(bool abort, bool readonly, unsigned flags = 0);
+  void cursor_open(unsigned dbi);
+  void cursor_close();
   void txn_inject_writefault(void);
   void txn_inject_writefault(MDBX_txn *txn);
   void fetch_canary();
@@ -158,6 +160,13 @@ public:
   bool run();
 };
 
+class testcase_append : public testcase {
+public:
+  testcase_append(const actor_config &config, const mdbx_pid_t pid)
+      : testcase(config, pid) {}
+  bool run();
+};
+
 class testcase_deadread : public testcase {
 public:
   testcase_deadread(const actor_config &config, const mdbx_pid_t pid)

test/test.vcxproj

@@ -181,6 +181,7 @@
     <ClInclude Include="utils.h" />
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="append.cc" />
     <ClCompile Include="cases.cc" />
     <ClCompile Include="chrono.cc" />
     <ClCompile Include="config.cc" />

test/utils.h

@@ -288,6 +288,7 @@ struct simple_checksum {
   void push(uint32_t data) {
     value += data * UINT64_C(9386433910765580089) + 1;
     value ^= value >> 41;
+    value *= UINT64_C(0xBD9CACC22C6E9571);
   }
 
   void push(uint64_t data) {
@@ -304,11 +305,15 @@ struct simple_checksum {
   }
 
   void push(const double &data) { push(&data, sizeof(double)); }
-
   void push(const char *cstr) { push(cstr, strlen(cstr)); }
-
   void push(const std::string &str) { push(str.data(), str.size()); }
 
+  void push(unsigned salt, const MDBX_val &val) {
+    push(val.iov_len);
+    push(salt);
+    push(val.iov_base, val.iov_len);
+  }
+
 #if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
   void push(const HANDLE &handle) { push(&handle, sizeof(handle)); }
 #endif /* _WINDOWS */