From e9ea16a54e1591c308017ddf65b14fd7bcba36bf Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Wed, 22 Mar 2017 16:03:45 +0300 Subject: [PATCH] mdbx: adds test7 by Klaus Malorny --- AUTHORS | 1 + Makefile | 4 +- libmdbx.files | 3 + test/test7.c | 246 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 252 insertions(+), 2 deletions(-) create mode 100644 test/test7.c diff --git a/AUTHORS b/AUTHORS index b65b8080..0c7ca23f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -12,6 +12,7 @@ Howard Chu , Ignacio Casal Quinteiro Jean-Christophe DUBOIS John Hewson +Klaus Malorny Kurt Zeilenga Leonid Yuriev , Lorenz Bauer diff --git a/Makefile b/Makefile index 2dbfbeea..9da7d70e 100644 --- a/Makefile +++ b/Makefile @@ -41,8 +41,8 @@ HEADERS := mdbx.h LIBRARIES := libmdbx.a libmdbx.so TOOLS := mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_chk MANPAGES := mdbx_stat.1 mdbx_copy.1 mdbx_dump.1 mdbx_load.1 -TESTS := test0 test1 test2 test3 test4 test5 test6 test_bench \ - test_yota1 test_yota2 +TESTS := test0 test1 test2 test3 test4 test5 test6 test7 \ + test_bench test_yota1 test_yota2 MDBX_SRC := mdbx.h $(addprefix src/, mdbx.c osal.c lck-posix.c defs.h bits.h osal.h midl.h) diff --git a/libmdbx.files b/libmdbx.files index b7fa32d3..1cb076a5 100644 --- a/libmdbx.files +++ b/libmdbx.files @@ -1,4 +1,6 @@ AUTHORS +LICENSE +Makefile README.md mdbx.h src/bits.h @@ -25,6 +27,7 @@ test/test3.c test/test4.c test/test5.c test/test6.c +test/test7.c test/test_bench.c test/test_yota1.c test/test_yota2.c diff --git a/test/test7.c b/test/test7.c new file mode 100644 index 00000000..61110d5c --- /dev/null +++ b/test/test7.c @@ -0,0 +1,246 @@ +/* + * Copyright 2017 Klaus Malorny + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include +#include +#include +#include +#include + +#include "../mdbx.h" + +static const char *fileName = "/dev/shm/test.mdbx"; +static const char *dbName = "test"; +static long size = 1500000000; +static int recordCount = 33000000; +static int majorIdCount = 6000; +static int minorIdCount = 1000000; +static unsigned int seed = 1; +static long *majorIds; + +typedef struct { + long majorId; + long minorId; +} KeyType; + +typedef struct { long refId; } DataType; + +typedef struct { + KeyType key; + DataType data; +} KeyDataType; + +void check(const char *op, int error) { + if (error != 0) { + fprintf(stderr, "%s: unexpected error %d: %s\n", op, error, + mdbx_strerror(error)); + exit(1); + } +} + +void shuffle(void *data, int recordSize, int recordCount) { + char *ptr = (char *)data; + char *swapBuf = malloc(recordSize); + + for (int i = recordCount - 2; i >= 0; i--) { + int j = (int)(random() % (recordCount - i)); + + if (j > 0) { + char *ptr1 = ptr + i * recordSize; + char *ptr2 = ptr + (i + j) * recordSize; + + memcpy(swapBuf, ptr1, recordSize); + memcpy(ptr1, ptr2, recordSize); + memcpy(ptr2, swapBuf, recordSize); + } + } + + free(swapBuf); +} + +void fill(MDB_env *env, MDB_dbi dbi) { + KeyType key; + DataType data; + + MDB_val keyRef; + MDB_val dataRef; + MDB_txn *txn; + + printf("generating data\n"); + + srandom(seed); + + majorIds = (long *)malloc(majorIdCount * sizeof(long)); + + if (!majorIds) { + fprintf(stderr, "out of memory\n"); + exit(1); + } + + for (int i = 0; i < majorIdCount; i++) + majorIds[i] = i; + + // now shuffle (for later deletion test) + shuffle((void *)majorIds, sizeof(long), majorIdCount); + + KeyDataType *records = malloc(sizeof(KeyDataType) * recordCount); + KeyDataType *ptr = records; + int remaining = recordCount; + long refId = 0; + + for (int i = 0; i < minorIdCount; i++) { + long majorId = random() % majorIdCount; + long minorId = i; + + int max = remaining / (minorIdCount - i + 1); + int use; + + if (i == minorIdCount - 1 || max < 2) { + use = max; + + } else { + long rand1 = random() % max; + long rand2 = random() % max; + use = (int)((rand1 * rand2 / (max - 1))) + 1; // non-linear distribution + } + + // printf ("%d %d %d\n", i, max, use); + + while (use-- > 0) { + ptr->key.majorId = majorId; + ptr->key.minorId = minorId; + ptr->data.refId = ++refId; + ptr++; + remaining--; + } + } + + shuffle((void *)records, sizeof(KeyDataType), recordCount); + + printf("writing data\n"); + + check("txn_begin", mdbx_txn_begin(env, NULL, 0, &txn)); + + ptr = records; + + for (int i = recordCount; i > 0; i--) { + + key.majorId = htobe64(ptr->key.majorId); + key.minorId = htobe64(ptr->key.minorId); + data.refId = htobe64(ptr->data.refId); + + keyRef.mv_size = sizeof(key); + keyRef.mv_data = (void *)&key; + dataRef.mv_size = sizeof(data); + dataRef.mv_data = (void *)&data; + + check("mdbx_put", mdbx_put(txn, dbi, &keyRef, &dataRef, 0)); + + ptr++; + } + + check("txn_commit", mdbx_txn_commit(txn)); + + printf("%d records written\n", recordCount); +} + +void deleteRange(MDB_env *env, MDB_dbi dbi, MDB_txn *txn, KeyType *startKey, + KeyType *endKey, int endIsInclusive) { + MDB_cursor *cursor; + MDB_val curKeyRef; + MDB_val endKeyRef; + MDB_val curDataRef; + (void)env; + + check("cursor_open", mdbx_cursor_open(txn, dbi, &cursor)); + + curKeyRef.mv_size = sizeof(KeyType); + curKeyRef.mv_data = (void *)startKey; + endKeyRef.mv_size = sizeof(KeyType); + endKeyRef.mv_data = (void *)endKey; + curDataRef.mv_size = 0; + curDataRef.mv_data = NULL; + + int error = mdbx_cursor_get(cursor, &curKeyRef, &curDataRef, MDB_SET_RANGE); + + while (error != MDB_NOTFOUND) { + check("mdbx_cursor_get", error); + + int compResult = mdbx_cmp(txn, dbi, &curKeyRef, &endKeyRef); + + if (compResult > 0 || (!compResult && !endIsInclusive)) + break; + + check("mdbx_cursor_del", mdbx_cursor_del(cursor, MDB_NODUPDATA)); + + error = mdbx_cursor_get(cursor, &curKeyRef, &curDataRef, MDB_NEXT); + } + + mdbx_cursor_close(cursor); +} + +void testDelete(MDB_env *env, MDB_dbi dbi) { + MDB_txn *txn; + KeyType startKey; + KeyType endKey; + + printf("testing\n"); + + check("txn_begin", mdbx_txn_begin(env, NULL, 0, &txn)); + + long majorId; + + for (int i = 0; i < majorIdCount; i++) { + majorId = majorIds[i]; + startKey.majorId = htobe64(majorId); + startKey.minorId = htobe64(1); + endKey.majorId = htobe64(majorId); + endKey.minorId = htobe64((long)(~0UL >> 1)); + + deleteRange(env, dbi, txn, &startKey, &endKey, 1); + } + + check("txn_commit", mdbx_txn_commit(txn)); +} + +int main(int argc, char *argv[]) { + MDB_env *env; + MDB_dbi dbi; + MDB_txn *txn; + (void)argc; + (void)argv; + + printf("LMDB version: %s\n", MDBX_VERSION_STRING); + + unlink(fileName); + check("env_create", mdbx_env_create(&env)); + check("env_set_mapsize", mdbx_env_set_mapsize(env, size)); + check("env_set_maxdbs", mdbx_env_set_maxdbs(env, 2)); + + check("env_open", + mdbx_env_open(env, fileName, MDB_NOSUBDIR | MDB_WRITEMAP, 0666)); + + check("txn_begin", mdbx_txn_begin(env, NULL, 0, &txn)); + + check("dbi_open", mdbx_dbi_open(txn, dbName, MDB_CREATE | MDB_DUPSORT, &dbi)); + + check("txn_commit", mdbx_txn_commit(txn)); + + fill(env, dbi); + testDelete(env, dbi); + + mdbx_env_close(env); + + printf("done.\n"); +}