mdbx: adds test7 by Klaus Malorny <klaus.malorny@knipp.de>

AUTHORS

@@ -12,6 +12,7 @@ Howard Chu <hyc@openldap.org>, <hyc@symas.com>
 Ignacio Casal Quinteiro <ignacio.casal@nice-software.com>
 Jean-Christophe DUBOIS <jcd@tribudubois.net>
 John Hewson <john@jahewson.com>
+Klaus Malorny <klaus.malorny@knipp.de>
 Kurt Zeilenga <kurt.zeilenga@isode.com>
 Leonid Yuriev <leo@yuriev.ru>, <lyuryev@ptsecurity.com>
 Lorenz Bauer <lmb@cloudflare.com>

Makefile

@@ -41,8 +41,8 @@ HEADERS		:= mdbx.h
 LIBRARIES	:= libmdbx.a libmdbx.so
 TOOLS		:= mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_chk
 MANPAGES	:= mdbx_stat.1 mdbx_copy.1 mdbx_dump.1 mdbx_load.1
-TESTS		:= test0 test1 test2 test3 test4 test5 test6 test_bench \
+TESTS		:= test0 test1 test2 test3 test4 test5 test6 test7 \
-		   test_yota1 test_yota2
+		   test_bench test_yota1 test_yota2
 
 MDBX_SRC	:= mdbx.h $(addprefix src/, mdbx.c osal.c lck-posix.c defs.h bits.h osal.h midl.h)
 

libmdbx.files

@@ -1,4 +1,6 @@
 AUTHORS
+LICENSE
+Makefile
 README.md
 mdbx.h
 src/bits.h
@@ -25,6 +27,7 @@ test/test3.c
 test/test4.c
 test/test5.c
 test/test6.c
+test/test7.c
 test/test_bench.c
 test/test_yota1.c
 test/test_yota2.c

test/test7.c

@@ -0,0 +1,246 @@
+/*
+ * Copyright 2017 Klaus Malorny <klaus.malorny@knipp.de>
+ * and other libmdbx authors: please see AUTHORS file.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+
+#include <endian.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "../mdbx.h"
+
+static const char *fileName = "/dev/shm/test.mdbx";
+static const char *dbName = "test";
+static long size = 1500000000;
+static int recordCount = 33000000;
+static int majorIdCount = 6000;
+static int minorIdCount = 1000000;
+static unsigned int seed = 1;
+static long *majorIds;
+
+typedef struct {
+  long majorId;
+  long minorId;
+} KeyType;
+
+typedef struct { long refId; } DataType;
+
+typedef struct {
+  KeyType key;
+  DataType data;
+} KeyDataType;
+
+void check(const char *op, int error) {
+  if (error != 0) {
+    fprintf(stderr, "%s: unexpected error %d: %s\n", op, error,
+            mdbx_strerror(error));
+    exit(1);
+  }
+}
+
+void shuffle(void *data, int recordSize, int recordCount) {
+  char *ptr = (char *)data;
+  char *swapBuf = malloc(recordSize);
+
+  for (int i = recordCount - 2; i >= 0; i--) {
+    int j = (int)(random() % (recordCount - i));
+
+    if (j > 0) {
+      char *ptr1 = ptr + i * recordSize;
+      char *ptr2 = ptr + (i + j) * recordSize;
+
+      memcpy(swapBuf, ptr1, recordSize);
+      memcpy(ptr1, ptr2, recordSize);
+      memcpy(ptr2, swapBuf, recordSize);
+    }
+  }
+
+  free(swapBuf);
+}
+
+void fill(MDB_env *env, MDB_dbi dbi) {
+  KeyType key;
+  DataType data;
+
+  MDB_val keyRef;
+  MDB_val dataRef;
+  MDB_txn *txn;
+
+  printf("generating data\n");
+
+  srandom(seed);
+
+  majorIds = (long *)malloc(majorIdCount * sizeof(long));
+
+  if (!majorIds) {
+    fprintf(stderr, "out of memory\n");
+    exit(1);
+  }
+
+  for (int i = 0; i < majorIdCount; i++)
+    majorIds[i] = i;
+
+  // now shuffle (for later deletion test)
+  shuffle((void *)majorIds, sizeof(long), majorIdCount);
+
+  KeyDataType *records = malloc(sizeof(KeyDataType) * recordCount);
+  KeyDataType *ptr = records;
+  int remaining = recordCount;
+  long refId = 0;
+
+  for (int i = 0; i < minorIdCount; i++) {
+    long majorId = random() % majorIdCount;
+    long minorId = i;
+
+    int max = remaining / (minorIdCount - i + 1);
+    int use;
+
+    if (i == minorIdCount - 1 || max < 2) {
+      use = max;
+
+    } else {
+      long rand1 = random() % max;
+      long rand2 = random() % max;
+      use = (int)((rand1 * rand2 / (max - 1))) + 1; // non-linear distribution
+    }
+
+    //    printf ("%d %d %d\n", i, max, use);
+
+    while (use-- > 0) {
+      ptr->key.majorId = majorId;
+      ptr->key.minorId = minorId;
+      ptr->data.refId = ++refId;
+      ptr++;
+      remaining--;
+    }
+  }
+
+  shuffle((void *)records, sizeof(KeyDataType), recordCount);
+
+  printf("writing data\n");
+
+  check("txn_begin", mdbx_txn_begin(env, NULL, 0, &txn));
+
+  ptr = records;
+
+  for (int i = recordCount; i > 0; i--) {
+
+    key.majorId = htobe64(ptr->key.majorId);
+    key.minorId = htobe64(ptr->key.minorId);
+    data.refId = htobe64(ptr->data.refId);
+
+    keyRef.mv_size = sizeof(key);
+    keyRef.mv_data = (void *)&key;
+    dataRef.mv_size = sizeof(data);
+    dataRef.mv_data = (void *)&data;
+
+    check("mdbx_put", mdbx_put(txn, dbi, &keyRef, &dataRef, 0));
+
+    ptr++;
+  }
+
+  check("txn_commit", mdbx_txn_commit(txn));
+
+  printf("%d records written\n", recordCount);
+}
+
+void deleteRange(MDB_env *env, MDB_dbi dbi, MDB_txn *txn, KeyType *startKey,
+                 KeyType *endKey, int endIsInclusive) {
+  MDB_cursor *cursor;
+  MDB_val curKeyRef;
+  MDB_val endKeyRef;
+  MDB_val curDataRef;
+  (void)env;
+
+  check("cursor_open", mdbx_cursor_open(txn, dbi, &cursor));
+
+  curKeyRef.mv_size = sizeof(KeyType);
+  curKeyRef.mv_data = (void *)startKey;
+  endKeyRef.mv_size = sizeof(KeyType);
+  endKeyRef.mv_data = (void *)endKey;
+  curDataRef.mv_size = 0;
+  curDataRef.mv_data = NULL;
+
+  int error = mdbx_cursor_get(cursor, &curKeyRef, &curDataRef, MDB_SET_RANGE);
+
+  while (error != MDB_NOTFOUND) {
+    check("mdbx_cursor_get", error);
+
+    int compResult = mdbx_cmp(txn, dbi, &curKeyRef, &endKeyRef);
+
+    if (compResult > 0 || (!compResult && !endIsInclusive))
+      break;
+
+    check("mdbx_cursor_del", mdbx_cursor_del(cursor, MDB_NODUPDATA));
+
+    error = mdbx_cursor_get(cursor, &curKeyRef, &curDataRef, MDB_NEXT);
+  }
+
+  mdbx_cursor_close(cursor);
+}
+
+void testDelete(MDB_env *env, MDB_dbi dbi) {
+  MDB_txn *txn;
+  KeyType startKey;
+  KeyType endKey;
+
+  printf("testing\n");
+
+  check("txn_begin", mdbx_txn_begin(env, NULL, 0, &txn));
+
+  long majorId;
+
+  for (int i = 0; i < majorIdCount; i++) {
+    majorId = majorIds[i];
+    startKey.majorId = htobe64(majorId);
+    startKey.minorId = htobe64(1);
+    endKey.majorId = htobe64(majorId);
+    endKey.minorId = htobe64((long)(~0UL >> 1));
+
+    deleteRange(env, dbi, txn, &startKey, &endKey, 1);
+  }
+
+  check("txn_commit", mdbx_txn_commit(txn));
+}
+
+int main(int argc, char *argv[]) {
+  MDB_env *env;
+  MDB_dbi dbi;
+  MDB_txn *txn;
+  (void)argc;
+  (void)argv;
+
+  printf("LMDB version: %s\n", MDBX_VERSION_STRING);
+
+  unlink(fileName);
+  check("env_create", mdbx_env_create(&env));
+  check("env_set_mapsize", mdbx_env_set_mapsize(env, size));
+  check("env_set_maxdbs", mdbx_env_set_maxdbs(env, 2));
+
+  check("env_open",
+        mdbx_env_open(env, fileName, MDB_NOSUBDIR | MDB_WRITEMAP, 0666));
+
+  check("txn_begin", mdbx_txn_begin(env, NULL, 0, &txn));
+
+  check("dbi_open", mdbx_dbi_open(txn, dbName, MDB_CREATE | MDB_DUPSORT, &dbi));
+
+  check("txn_commit", mdbx_txn_commit(txn));
+
+  fill(env, dbi);
+  testDelete(env, dbi);
+
+  mdbx_env_close(env);
+
+  printf("done.\n");
+}