From a3644aa6d02fc677920ec67465056aeba2c1d2aa Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Thu, 30 Mar 2017 18:54:57 +0300 Subject: [PATCH] mdbx: new testset (initial, stub). Initial stub for https://github.com/ReOpen/libmdbx/issues/8 --- .appveyor.yml | 7 +- .gitignore | 45 ++-- Makefile | 42 +-- mdbx-dll.vcxproj => dll.vcxproj | 0 libmdbx.files | 31 ++- mdbx.h | 2 +- mdbx.sln | 12 +- src/bits.h | 8 +- src/defs.h | 7 + src/lck-windows.c | 2 + src/mdbx.c | 52 ++-- src/osal.c | 8 +- src/osal.h | 5 +- test/base.h | 61 +++++ test/cases.cc | 69 +++++ test/config.cc | 446 ++++++++++++++++++++++++++++++++ test/config.h | 149 +++++++++++ test/dead.cc | 61 +++++ test/hill.cc | 37 +++ test/jitter.cc | 33 +++ test/keygen.cc | 72 ++++++ test/keygen.h | 123 +++++++++ test/log.cc | 129 +++++++++ test/log.h | 61 +++++ test/main.cc | 311 ++++++++++++++++++++++ test/osal-unix.cc | 230 ++++++++++++++++ test/osal-windows.cc | 262 +++++++++++++++++++ test/osal.h | 28 ++ test/test.cc | 192 ++++++++++++++ test/test.h | 145 +++++++++++ test/test.vcxproj | 182 +++++++++++++ test/test0.c | 220 ---------------- test/test1.c | 199 -------------- test/test2.c | 153 ----------- test/test3.c | 162 ------------ test/test4.c | 196 -------------- test/test5.c | 164 ------------ test/test6.c | 175 ------------- test/test7.c | 246 ------------------ test/test_bench.c | 260 ------------------- test/test_yota1.c | 277 -------------------- test/test_yota2.c | 335 ------------------------ test/utils.cc | 90 +++++++ test/utils.h | 312 ++++++++++++++++++++++ 44 files changed, 3120 insertions(+), 2481 deletions(-) rename mdbx-dll.vcxproj => dll.vcxproj (100%) create mode 100644 test/base.h create mode 100644 test/cases.cc create mode 100644 test/config.cc create mode 100644 test/config.h create mode 100644 test/dead.cc create mode 100644 test/hill.cc create mode 100644 test/jitter.cc create mode 100644 test/keygen.cc create mode 100644 test/keygen.h create mode 100644 test/log.cc create mode 100644 test/log.h create mode 100644 test/main.cc create mode 100644 test/osal-unix.cc create mode 100644 test/osal-windows.cc create mode 100644 test/osal.h create mode 100644 test/test.cc create mode 100644 test/test.h create mode 100644 test/test.vcxproj delete mode 100644 test/test0.c delete mode 100644 test/test1.c delete mode 100644 test/test2.c delete mode 100644 test/test3.c delete mode 100644 test/test4.c delete mode 100644 test/test5.c delete mode 100644 test/test6.c delete mode 100644 test/test7.c delete mode 100644 test/test_bench.c delete mode 100644 test/test_yota1.c delete mode 100644 test/test_yota2.c create mode 100644 test/utils.cc create mode 100644 test/utils.h diff --git a/.appveyor.yml b/.appveyor.yml index aa374141..a164b1c8 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,4 +1,3 @@ -version: '{build}' max_jobs: 1 image: Visual Studio 2015 @@ -23,6 +22,6 @@ build: verbosity: minimal project: mdbx.sln -# test_script: -# - ps: | -# & "C:\projects\mdbx\$env:PLATFORM\$env:CONFIGURATION\test\test.exe" +test_script: +- ps: | + & "C:\projects\mdbx\$env:PLATFORM\$env:CONFIGURATION\test\test.exe" --pathname=tmp.db --basic --dont-cleanup-after diff --git a/.gitignore b/.gitignore index 7a7d1485..84d48914 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,33 @@ -mtest[0123456] -wbench -testdb -mdbx_copy -mdbx_stat -mdbx_dump -mdbx_load -mdbx_chk -*.lo -*.[ao] -*.so -*.exe *[~#] +*.[ao] *.bak -*.orig -*.rej -*.gcov -*.gcda -*.gcno core core.* -valgrind.* -yota_test* +*.exe +*.gcda +*.gcno +*.gcov +libmdbx.creator.user +*.lo +mdbx_chk +mdbx_copy mdbx-dll.VC.db mdbx-dll.VC.VC.opendb mdbx-dll.vcxproj.filters +mdbx_dump +mdbx_load +mdbx_stat +*.orig +*.rej +*.so +/test/test +test/test.vcxproj.user +test/tmp.db +test/tmp.db-lock +tmp.db +tmp.db-lock +valgrind.* +.vs/ +Win32/ +x64/ +x86/ diff --git a/Makefile b/Makefile index 9da7d70e..e88007b1 100644 --- a/Makefile +++ b/Makefile @@ -23,10 +23,13 @@ mandir ?= $(prefix)/man suffix ?= CC ?= gcc +CXX ?= g++ XCFLAGS ?= -DNDEBUG=1 -DMDB_DEBUG=0 -DMDBX_EXPORTS=1 CFLAGS ?= -O2 -g3 -Wall -Werror -Wextra -ffunction-sections -fPIC -fvisibility=hidden CFLAGS += -D_GNU_SOURCE=1 -std=gnu99 -pthread $(XCFLAGS) -COVER ?= -coverage -fprofile-arcs -ftest-coverage -O0 +# COVER ?= -coverage -fprofile-arcs -ftest-coverage -Og + +CXXFLAGS = -std=c++11 $(filter-out -std=gnu99,$(CFLAGS)) # LY: for ability to built with modern glibc, # but then run with the old @@ -41,14 +44,12 @@ HEADERS := mdbx.h LIBRARIES := libmdbx.a libmdbx.so TOOLS := mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_chk MANPAGES := mdbx_stat.1 mdbx_copy.1 mdbx_dump.1 mdbx_load.1 -TESTS := test0 test1 test2 test3 test4 test5 test6 test7 \ - test_bench test_yota1 test_yota2 MDBX_SRC := mdbx.h $(addprefix src/, mdbx.c osal.c lck-posix.c defs.h bits.h osal.h midl.h) -.PHONY: mdbx all install clean check tests coverage +.PHONY: mdbx all install clean check coverage -all: $(LIBRARIES) $(TOOLS) +all: $(LIBRARIES) $(TOOLS) test/test mdbx: libmdbx.a libmdbx.so @@ -65,20 +66,10 @@ install: $(LIBRARIES) $(TOOLS) $(HEADERS) && cp -t $(SANDBOX)$(mandir)/man1 $(MANPAGES) clean: - rm -rf $(TOOLS) $(TESTS) @* *.[ao] *.[ls]o *~ tmp.db/* *.gcov *.log *.err + rm -rf $(TOOLS) test/test @* *.[ao] *.[ls]o *~ tmp.db/* *.gcov *.log *.err -tests: $(TESTS) - -check: tests - [ -d tmp.db ] || mkdir tmp.db && rm -f tmp.db/* \ - && echo "*** LMDB-TEST-0" && ./test0 && ./mdbx_chk -v tmp.db \ - && echo "*** LMDB-TEST-1" && ./test1 && ./mdbx_chk -v tmp.db \ - && echo "*** LMDB-TEST-2" && ./test2 && ./mdbx_chk -v tmp.db \ - && echo "*** LMDB-TEST-3" && ./test3 && ./mdbx_chk -v tmp.db \ - && echo "*** LMDB-TEST-4" && ./test4 && ./mdbx_chk -v tmp.db \ - && echo "*** LMDB-TEST-5" && ./test5 && ./mdbx_chk -v tmp.db \ - && echo "*** LMDB-TEST-6" && ./test6 && ./mdbx_chk -v tmp.db \ - && echo "*** LMDB-TESTs - all done" +check: test/test + test/test --pathname=tmp.db --basic --dont-cleanup-after && ./mdbx_chk -vn tmp.db mdbx.o: $(MDBX_SRC) Makefile $(CC) $(CFLAGS) -c src/mdbx.c -o $@ @@ -98,19 +89,8 @@ libmdbx.so: mdbx.o osal.o lck-posix.o mdbx_%: src/tools/mdbx_%.c libmdbx.a $(CC) $(CFLAGS) $(LDFLAGS) $(LDOPS) -o $@ $^ -test%: test/test%.c libmdbx.a - $(CC) $(CFLAGS) $(LDFLAGS) -Isrc -o $@ $^ - -gcov-mdbx.o: $(MDBX_SRC) Makefile - $(CC) $(CFLAGS) $(COVER) -c src/mdbx.c -o $@ - -# Seem this useless :( -coverage: gcov-mdbx.o - for t in test/test[0-9]*.c; do x=`basename \$$t .c`; \ - $(CC) $(CFLAGS) $(COVER) -Isrc $$t -o gcov-$$x $^; \ - rm -rf tmp.db; mkdir tmp.db; ./gcov-$$x; \ - done - gcov *.gcno +test/test: $(wildcard test/*.h) $(filter-out test/osal-windows.cc, $(wildcard test/*.cc)) libmdbx.a + $(CXX) $(CXXFLAGS) $(LDFLAGS) -Isrc -o $@ $(filter-out %.h, $^) ifneq ($(wildcard $(IOARENA)),) diff --git a/mdbx-dll.vcxproj b/dll.vcxproj similarity index 100% rename from mdbx-dll.vcxproj rename to dll.vcxproj diff --git a/libmdbx.files b/libmdbx.files index 1cb076a5..c0b049fa 100644 --- a/libmdbx.files +++ b/libmdbx.files @@ -20,17 +20,26 @@ src/tools/mdbx_load.1 src/tools/mdbx_load.c src/tools/mdbx_stat.1 src/tools/mdbx_stat.c -test/test0.c -test/test1.c -test/test2.c -test/test3.c -test/test4.c -test/test5.c -test/test6.c -test/test7.c -test/test_bench.c -test/test_yota1.c -test/test_yota2.c +test/actor.cc +test/base.h +test/config.h +test/dead.cc +test/hill.cc +test/jitter.cc +test/keygen.cc +test/keygen.h +test/log.cc +test/log.h +test/main.cc +test/config.cc +test/cases.cc +test/osal-unix.cc +test/osal-windows.cc +test/osal.h +test/test.cc +test/test.h +test/utils.cc +test/utils.h tutorial/README.md tutorial/sample-bdb.txt tutorial/sample-mdb.txt diff --git a/mdbx.h b/mdbx.h index 2d30b2b8..75688438 100644 --- a/mdbx.h +++ b/mdbx.h @@ -90,7 +90,7 @@ #if defined(LIBMDBX_EXPORTS) # define LIBMDBX_API __dll_export -#elif defined(MDBX_IMPORTS) +#elif defined(LIBMDBX_IMPORTS) # define LIBMDBX_API __dll_import #else # define LIBMDBX_API diff --git a/mdbx.sln b/mdbx.sln index b94111d6..aa2025d8 100644 --- a/mdbx.sln +++ b/mdbx.sln @@ -3,7 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 14 VisualStudioVersion = 14.0.25420.1 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mdbx-dll", "mdbx-dll.vcxproj", "{6D19209B-ECE7-4B9C-941C-0AA2B484F199}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test", "test\test.vcxproj", "{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dll", "dll.vcxproj", "{6D19209B-ECE7-4B9C-941C-0AA2B484F199}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -21,6 +23,14 @@ Global {6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Release|x64.Build.0 = Release|x64 {6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Release|x86.ActiveCfg = Release|Win32 {6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Release|x86.Build.0 = Release|Win32 + {30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Debug|x64.ActiveCfg = Debug|x64 + {30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Debug|x64.Build.0 = Debug|x64 + {30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Debug|x86.ActiveCfg = Debug|Win32 + {30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Debug|x86.Build.0 = Debug|Win32 + {30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Release|x64.ActiveCfg = Release|x64 + {30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Release|x64.Build.0 = Release|x64 + {30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Release|x86.ActiveCfg = Release|Win32 + {30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Release|x86.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/bits.h b/src/bits.h index 57551a48..70cfaa48 100644 --- a/src/bits.h +++ b/src/bits.h @@ -96,11 +96,11 @@ #endif #if defined(__i386) || defined(__x86_64) || defined(_M_IX86) -# define MISALIGNED_OK 1 /* TODO */ +# define UNALIGNED_OK 1 /* TODO */ #endif -#ifndef MISALIGNED_OK -# define MISALIGNED_OK 0 -#endif /* MISALIGNED_OK */ +#ifndef UNALIGNED_OK +# define UNALIGNED_OK 0 +#endif /* UNALIGNED_OK */ #if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF # error "Sanity checking failed: Two's complement, reasonably sized integer types" diff --git a/src/defs.h b/src/defs.h index a00fce4a..24c67cc2 100644 --- a/src/defs.h +++ b/src/defs.h @@ -283,6 +283,13 @@ # endif #endif /* unlikely */ +#if !defined(__noop) && !defined(_MSC_VER) + static __inline int __do_noop(void* crutch, ...) { + (void) crutch; return 0; + } +# define __noop(...) __do_noop(0, __VA_ARGS__) +#endif /* __noop */ + /*----------------------------------------------------------------------------*/ /* Wrapper around __func__, which is a C99 feature */ diff --git a/src/lck-windows.c b/src/lck-windows.c index e2fd071c..a037dbda 100644 --- a/src/lck-windows.c +++ b/src/lck-windows.c @@ -287,12 +287,14 @@ void mdbx_lck_destroy(MDB_env *env) { /* double `unlock` for robustly remove overlapped shared/exclusive locks */ while (funlock(env->me_lfd, LCK_LOWER)) ; + rc = GetLastError(); assert(rc == ERROR_NOT_LOCKED); (void)rc; SetLastError(ERROR_SUCCESS); while (funlock(env->me_lfd, LCK_UPPER)) ; + rc = GetLastError(); assert(rc == ERROR_NOT_LOCKED); (void)rc; SetLastError(ERROR_SUCCESS); diff --git a/src/mdbx.c b/src/mdbx.c index 1af8aec8..19424aba 100644 --- a/src/mdbx.c +++ b/src/mdbx.c @@ -538,7 +538,7 @@ static __inline MDB_node *NODEPTR(MDB_page *p, unsigned i) { #define NODEKSZ(node) ((node)->mn_ksize) /** Copy a page number from src to dst */ -#if MISALIGNED_OK +#if UNALIGNED_OK #define COPY_PGNO(dst, src) dst = src #elif SIZE_MAX > 4294967295UL #define COPY_PGNO(dst, src) \ @@ -560,7 +560,7 @@ static __inline MDB_node *NODEPTR(MDB_page *p, unsigned i) { *d++ = *s++; \ *d = *s; \ } while (0) -#endif /* MISALIGNED_OK */ +#endif /* UNALIGNED_OK */ /** The address of a key in a LEAF2 page. * LEAF2 pages are used for #MDB_DUPFIXED sorted-duplicate @@ -754,16 +754,31 @@ static const char *__mdbx_strerr(int errnum) { } } -const char *mdbx_strerror_r(int errnum, char *buf, size_t buflen) { +const char *__cold mdbx_strerror_r(int errnum, char *buf, size_t buflen) { const char *msg = __mdbx_strerr(errnum); if (!msg) { -#if defined(_WIN32) || defined(_WIN64) - (void)errnum; - (void)buf; - (void)buflen; - msg = FIXME; -#else + if (!buflen) + return NULL; +#ifdef _MSC_VER + int rc = strerror_s(buf, buflen, errnum); + assert(rc == 0); + (void)rc; + return buf; +#elif defined(_GNU_SOURCE) + /* GNU-specific */ msg = strerror_r(errnum, buf, buflen); +#elif (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) + /* XSI-compliant */ + int rc = strerror_r(errnum, buf, buflen); + if (rc) { + rc = snprintf(buf, buflen, "error %d", errnum); + assert(rc > 0); + } + return buf; +#else + strncpy(buf, strerror(errnum), buflen); + buf[buflen - 1] = '\0'; + return buf; #endif } return msg; @@ -772,9 +787,12 @@ const char *mdbx_strerror_r(int errnum, char *buf, size_t buflen) { const char *__cold mdbx_strerror(int errnum) { const char *msg = __mdbx_strerr(errnum); if (!msg) { -#if defined(_WIN32) || defined(_WIN64) - (void)errnum; - msg = FIXME; +#ifdef _MSC_VER + static __thread char buffer[1024]; + int rc = strerror_s(buffer, sizeof(buffer), errnum); + assert(rc == 0); + (void)rc; + msg = buffer; #else msg = strerror(errnum); #endif @@ -3776,7 +3794,7 @@ int __cold mdbx_env_set_maxreaders(MDB_env *env, unsigned readers) { if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) return MDBX_EBADSIGN; - if (unlikely(env->me_map)) + if (unlikely(env->me_map || readers > INT16_MAX)) return EINVAL; env->me_maxreaders = readers; @@ -4247,7 +4265,7 @@ static int __hot mdbx_cmp_int_a2(const MDB_val *a, const MDB_val *b) { mdbx_assert(NULL, a->mv_size == b->mv_size); mdbx_assert(NULL, 0 == (uintptr_t)a->mv_data % sizeof(uint16_t) && 0 == (uintptr_t)b->mv_data % sizeof(uint16_t)); -#if MISALIGNED_OK +#if UNALIGNED_OK switch (a->mv_size) { case 4: return mdbx_cmp2int(*(uint32_t *)a->mv_data, *(uint32_t *)b->mv_data); @@ -4282,7 +4300,7 @@ static int __hot mdbx_cmp_int_a2(const MDB_val *a, const MDB_val *b) { } while (pa != end); return diff; } -#endif /* MISALIGNED_OK */ +#endif /* UNALIGNED_OK */ } /** Compare two items pointing at unsigneds of unknown alignment. @@ -4291,7 +4309,7 @@ static int __hot mdbx_cmp_int_a2(const MDB_val *a, const MDB_val *b) { */ static int __hot mdbx_cmp_int_ua(const MDB_val *a, const MDB_val *b) { mdbx_assert(NULL, a->mv_size == b->mv_size); -#if MISALIGNED_OK +#if UNALIGNED_OK switch (a->mv_size) { case 4: return mdbx_cmp2int(*(uint32_t *)a->mv_data, *(uint32_t *)b->mv_data); @@ -4322,7 +4340,7 @@ static int __hot mdbx_cmp_int_ua(const MDB_val *a, const MDB_val *b) { #else /* __BYTE_ORDER__ */ return memcmp(a->mv_data, b->mv_data, a->mv_size); #endif /* __BYTE_ORDER__ */ -#endif /* MISALIGNED_OK */ +#endif /* UNALIGNED_OK */ } /** Compare two items lexically */ diff --git a/src/osal.c b/src/osal.c index 40e93178..12cf7782 100644 --- a/src/osal.c +++ b/src/osal.c @@ -17,7 +17,7 @@ #include "./bits.h" #if defined(_WIN32) || defined(_WIN64) -static int waitfor2errcode(DWORD result) { +static int waitstatus2errcode(DWORD result) { switch (result) { case WAIT_OBJECT_0: return MDB_SUCCESS; @@ -183,7 +183,7 @@ int mdbx_mutex_destroy(mdbx_mutex_t *mutex) { int mdbx_mutex_lock(mdbx_mutex_t *mutex) { #if defined(_WIN32) || defined(_WIN64) DWORD code = WaitForSingleObject(*mutex, INFINITE); - return waitfor2errcode(code); + return waitstatus2errcode(code); #else return pthread_mutex_lock(mutex); #endif @@ -231,7 +231,7 @@ int mdbx_cond_wait(mdbx_cond_t *cond, mdbx_mutex_t *mutex) { DWORD code = SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); if (code == WAIT_OBJECT_0) code = WaitForSingleObject(*mutex, INFINITE); - return waitfor2errcode(code); + return waitstatus2errcode(code); #else return pthread_cond_wait(cond, mutex); #endif @@ -555,7 +555,7 @@ int mdbx_thread_create(mdbx_thread_t *thread, int mdbx_thread_join(mdbx_thread_t thread) { #if defined(_WIN32) || defined(_WIN64) DWORD code = WaitForSingleObject(thread, INFINITE); - return waitfor2errcode(code); + return waitstatus2errcode(code); #else void *unused_retval = &unused_retval; return pthread_join(thread, &unused_retval); diff --git a/src/osal.h b/src/osal.h index 4000475b..e0dc9244 100644 --- a/src/osal.h +++ b/src/osal.h @@ -172,11 +172,12 @@ typedef pthread_key_t mdbx_thread_key_t; defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ defined(__i386) || defined(__x86_64__) || defined(_M_IX86) || \ defined(_M_X64) || defined(i386) || defined(_X86_) || defined(__i386__) || \ - defined(_X86_64_) || defined(_M_ARM) || defined(__e2k__) + defined(_X86_64_) || defined(_M_ARM) || defined(_M_ARM64) || \ + defined(__e2k__) #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ #elif defined(__BIG_ENDIAN__) || defined(_BIG_ENDIAN) || defined(__ARMEB__) || \ defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || \ - defined(_MIPSEB) || defined(__MIPSEB) + defined(_MIPSEB) || defined(__MIPSEB) || defined(_M_IA64) #define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ #else #error __BYTE_ORDER__ should be defined. diff --git a/test/base.h b/test/base.h new file mode 100644 index 00000000..155cd98f --- /dev/null +++ b/test/base.h @@ -0,0 +1,61 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once + +#ifndef NOMINMAX +#define NOMINMAX +#endif + +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) +/* If you wish to build your application for a previous Windows platform, + * include WinSDKVer.h and set the _WIN32_WINNT macro to the platform you + * wish to support before including SDKDDKVer.h. + * + * TODO: #define _WIN32_WINNT WIN32_MUSTDIE */ +#include +#endif /* WINDOWS */ + +#include +#include +#include +#include +#include +#include + +#ifdef _BSD_SOURCE +#include +#endif + +#include +#include +#include // for PRId64, PRIu64 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _MSC_VER +#include +#endif + +#include "../mdbx.h" +#include "../src/defs.h" diff --git a/test/cases.cc b/test/cases.cc new file mode 100644 index 00000000..a24838c6 --- /dev/null +++ b/test/cases.cc @@ -0,0 +1,69 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +void configure_actor(unsigned &lastid, const actor_testcase testcase, + const char *id_cstr, const actor_params ¶ms) { + unsigned wait4id = 0; + + if (params.waitfor_nops) { + for (auto i = global::actors.rbegin(); i != global::actors.rend(); ++i) { + if (i->is_waitable(params.waitfor_nops)) { + if (i->signal_nops && i->signal_nops != params.waitfor_nops) + failure("Previous waitable actor (id=%u) already linked on %u-ops\n", + i->id, i->signal_nops); + wait4id = i->id; + i->signal_nops = params.waitfor_nops; + break; + } + } + if (!wait4id) + failure("No previous waitable actor for %u-ops\n", params.waitfor_nops); + } + + unsigned long id = 0; + if (!id_cstr || strcmp(id_cstr, "auto") == 0) + id = lastid + 1; + else { + char *end = nullptr; + errno = 0; + id = strtoul(id_cstr, &end, 0); + if (errno) + failure_perror("Expects an integer value for actor-id\n", errno); + if (end && *end) + failure("The '%s' is unexpected for actor-id\n", end); + } + + if (id < 1 || id > ACTOR_ID_MAX) + failure("Invalid actor-id %lu\n", id); + lastid = id; + + global::actors.emplace_back(actor_config(testcase, params, id, wait4id)); + global::databases.insert(params.pathname_db); +} + +bool testcase_setup(const char *casename, const actor_params ¶ms, + unsigned &lastid) { + log_notice("testcase_setup(%s): TODO", casename); + + if (strcmp(casename, "basic") == 0) { + configure_actor(lastid, ac_hill, nullptr, params); + return true; + } + + return false; +} + +/* TODO */ diff --git a/test/config.cc b/test/config.cc new file mode 100644 index 00000000..fd9bc732 --- /dev/null +++ b/test/config.cc @@ -0,0 +1,446 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +#if defined(_MSC_VER) && !defined(strcasecmp) +#define strcasecmp(str, len) _stricmp(str, len) +#endif /* _MSC_VER && strcasecmp() */ + +namespace config { + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + const char **value, const char *default_value) { + assert(narg < argc); + const char *current = argv[narg]; + const size_t optlen = strlen(option); + + if (strncmp(current, "--", 2) || strncmp(current + 2, option, optlen)) + return false; + + if (!value) { + if (current[optlen + 2] == '=') + failure("Option '--%s' doen't accept any value\n", option); + narg += 1; + return true; + } + + *value = nullptr; + if (current[optlen + 2] == '=') { + *value = ¤t[optlen + 3]; + narg += 1; + return true; + } + + if (narg + 1 < argc && strncmp("--", argv[narg + 1], 2)) { + *value = argv[narg + 1]; + narg += 2; + return true; + } + + if (default_value) { + *value = default_value; + narg += 1; + return true; + } + + failure("No value given for '--%s' option\n", option); +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + std::string &value, bool allow_empty) { + const char *value_cstr; + if (!parse_option(argc, argv, narg, option, &value_cstr, + allow_empty ? "" : nullptr)) + return false; + + if (!allow_empty && strlen(value_cstr) == 0) + failure("Value for option '--%s' could't be empty\n", option); + + value = value_cstr; + return true; +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + size_t &mask, const option_verb *verbs) { + const char *list; + if (!parse_option(argc, argv, narg, option, &list)) + return false; + + mask = 0; + while (*list) { + if (*list == ',' || *list == ' ' || *list == '\t') { + ++list; + continue; + } + + const char *const comma = strchr(list, ','); + const size_t len = (comma) ? comma - list : strlen(list); + const option_verb *scan = verbs; + while (true) { + if (!scan->verb) + failure("Unknown verb '%.*s', for option '==%s'\n", (int)len, list, + option); + if (strlen(scan->verb) == len && strncmp(list, scan->verb, len) == 0) { + mask |= scan->mask; + list += len; + break; + } + ++scan; + } + } + + return true; +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + uint64_t &value, const scale_mode scale, + const uint64_t minval, const uint64_t maxval) { + + const char *value_cstr; + if (!parse_option(argc, argv, narg, option, &value_cstr)) + return false; + + char *suffix = nullptr; + errno = 0; + unsigned long raw = strtoul(value_cstr, &suffix, 0); + if (errno) + failure("Option '--%s' expects a numeric value (%s)\n", option, + test_strerror(errno)); + + uint64_t multipler = 1; + if (suffix && *suffix) { + if (scale == no_scale) + failure("Option '--%s' doen't accepts suffixes, so '%s' is unexpected\n", + option, suffix); + if (strcmp(suffix, "K") == 0 || strcasecmp(suffix, "Kilo") == 0) + multipler = (scale == decimal) ? UINT64_C(1000) : UINT64_C(1024); + else if (strcmp(suffix, "M") == 0 || strcasecmp(suffix, "Mega") == 0) + multipler = + (scale == decimal) ? UINT64_C(1000) * 1000 : UINT64_C(1024) * 1024; + else if (strcmp(suffix, "G") == 0 || strcasecmp(suffix, "Giga") == 0) + multipler = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 + : UINT64_C(1024) * 1024 * 1024; + else if (strcmp(suffix, "T") == 0 || strcasecmp(suffix, "Tera") == 0) + multipler = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 * 1000 + : UINT64_C(1024) * 1024 * 1024 * 1024; + else if (scale == duration && + (strcmp(suffix, "s") == 0 || strcasecmp(suffix, "Seconds") == 0)) + multipler = 1; + else if (scale == duration && + (strcmp(suffix, "m") == 0 || strcasecmp(suffix, "Minutes") == 0)) + multipler = 60; + else if (scale == duration && + (strcmp(suffix, "h") == 0 || strcasecmp(suffix, "Hours") == 0)) + multipler = 3600; + else if (scale == duration && + (strcmp(suffix, "d") == 0 || strcasecmp(suffix, "Days") == 0)) + multipler = 3600 * 24; + else + failure( + "Option '--%s' expects a numeric value with Kilo/Mega/Giga/Tera %s" + "suffixes, but '%s' is unexpected\n", + option, (scale == duration) ? "or Seconds/Minutes/Hours/Days " : "", + suffix); + } + + if (raw >= UINT64_MAX / multipler) + failure("The value for option '--%s' is too huge\n", option); + + value = raw * multipler; + if (maxval && value > maxval) + failure("The maximal value for option '--%s' is %" PRIu64 "\n", option, + maxval); + if (value < minval) + failure("The minimal value for option '--%s' is %" PRIu64 "\n", option, + minval); + return true; +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + unsigned &value, const scale_mode scale, + const unsigned minval, const unsigned maxval) { + + uint64_t huge; + if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval)) + return false; + value = (unsigned)huge; + return true; +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + bool &value) { + const char *value_cstr = NULL; + if (!parse_option(argc, argv, narg, option, &value_cstr, "yes")) { + const char *current = argv[narg]; + if (strncmp(current, "--no-", 5) || strcmp(current + 5, option)) + return false; + value = false; + narg += 1; + return true; + } + + if (!value_cstr) { + value = true; + return true; + } + + if (strcasecmp(value_cstr, "yes") == 0 || strcasecmp(value_cstr, "1") == 0) { + value = true; + return true; + } + + if (strcasecmp(value_cstr, "no") == 0 || strcasecmp(value_cstr, "0") == 0) { + value = false; + return true; + } + + failure( + "Option '--%s' expects a 'boolean' value Yes/No, so '%s' is unexpected\n", + option, value_cstr); +} + +//----------------------------------------------------------------------------- + +const struct option_verb mode_bits[] = { + {"rdonly", MDB_RDONLY}, {"mapasync", MDB_MAPASYNC}, + {"utterly", MDBX_UTTERLY_NOSYNC}, {"nosubdir", MDB_NOSUBDIR}, + {"nosync", MDB_NOSYNC}, {"nometasync", MDB_NOMETASYNC}, + {"writemap", MDB_WRITEMAP}, {"notls", MDB_NOTLS}, + {"nordahead", MDB_NORDAHEAD}, {"nomeminit", MDB_NOMEMINIT}, + {"coasesce", MDBX_COALESCE}, {"lifo", MDBX_LIFORECLAIM}, + {"parturb", MDBX_PAGEPERTURB}, {nullptr, 0}}; + +const struct option_verb table_bits[] = { + {"key.reverse", MDB_REVERSEKEY}, + {"key.integer", MDB_INTEGERKEY}, + {"data.integer", MDB_INTEGERDUP | MDB_DUPFIXED | MDB_DUPSORT}, + {"data.fixed", MDB_DUPFIXED | MDB_DUPSORT}, + {"data.reverse", MDB_REVERSEDUP | MDB_DUPSORT}, + {"data.dups", MDB_DUPSORT}, + {nullptr, 0}}; + +static void dump_verbs(FILE *out, const char *caption, size_t bits, + const struct option_verb *verbs) { + fprintf(out, "%s: (%" PRIu64 ")", caption, (uint64_t)bits); + + while (verbs->mask && bits) { + if ((bits & verbs->mask) == verbs->mask) { + fprintf(out, ", %s", verbs->verb); + bits -= verbs->mask; + } + ++verbs; + } + + fprintf(out, "\n"); +} + +static void dump_duration(FILE *out, const char *caption, unsigned duration) { + fprintf(out, "%s: ", caption); + if (duration) { + if (duration > 24 * 3600) + fprintf(out, "%u_", duration / (24 * 3600)); + if (duration > 3600) + fprintf(out, "%02u:", (duration % (24 * 3600)) / 3600); + fprintf(out, "%02u:%02u", (duration % 3600) / 60, duration % 60); + } else + fprintf(out, "INFINITE"); + fprintf(out, "\n"); +} + +void dump(FILE *out) { + for (auto i = global::actors.begin(); i != global::actors.end(); ++i) { + fprintf(out, "testcase %s\n", testcase2str(i->testcase)); + if (i->id) + fprintf(out, "\tid/table %u\n", i->id); + + if (i->params.loglevel) { + fprintf(out, "\tlog: level %u, %s\n", i->params.loglevel, + i->params.pathname_log.empty() ? "console" + : i->params.pathname_log.c_str()); + } + + fprintf(out, "\tdatabase: %s, size %" PRIu64 "\n", + i->params.pathname_db.c_str(), i->params.size); + + dump_verbs(out, "\tmode", i->params.mode_flags, mode_bits); + dump_verbs(out, "\ttable", i->params.table_flags, table_bits); + + fprintf(out, "\tseed %u\n", i->params.seed); + + if (i->params.test_nrecords) + fprintf(out, "\trecords %u\n", i->params.test_nrecords); + else + dump_duration(out, "\tduration", i->params.test_duration); + + if (i->params.nrepeat) + fprintf(out, "\trepeat %u\n", i->params.nrepeat); + else + fprintf(out, "\trepeat ETERNALLY\n"); + + fprintf(out, "\tthreads %u\n", i->params.nthreads); + + fprintf(out, "\tkey: minlen %u, maxlen %u\n", i->params.keylen_min, + i->params.keylen_max); + fprintf(out, "\tdata: minlen %u, maxlen %u\n", i->params.datalen_min, + i->params.datalen_max); + + fprintf(out, "\tbatch: read %u, write %u\n", i->params.batch_read, + i->params.batch_write); + + if (i->params.waitfor_nops) + fprintf(out, "\twait: actor %u for %u ops\n", i->wait4id, + i->params.waitfor_nops); + else if (i->params.delaystart) + dump_duration(out, "\tdelay", i->params.delaystart); + else + fprintf(out, "\tno-delay\n"); + + fprintf(out, "\tlimits: readers %u, tables %u\n", i->params.max_readers, + i->params.max_tables); + + fprintf(out, "\tdrop table: %s\n", i->params.drop_table ? "Yes" : "No"); + + fprintf(out, "\t#---\n"); + } + + dump_duration(out, "timeout", global::config::timeout); + fprintf(out, "cleanup: before %s, after %s\n", + global::config::dont_cleanup_before ? "No" : "Yes", + global::config::dont_cleanup_after ? "No" : "Yes"); +} + +} /* namespace config */ + +//----------------------------------------------------------------------------- + +using namespace config; + +actor_config::actor_config(actor_testcase testcase, const actor_params ¶ms, + unsigned id, unsigned wait4id) + : params(params) { + this->id = id; + this->order = (unsigned)global::actors.size(); + this->testcase = testcase; + this->wait4id = wait4id; + signal_nops = 0; +} + +const std::string actor_config::serialize(const char *prefix) const { + simple_checksum checksum; + + std::string result; + if (prefix) + result.append(prefix); + + checksum.push(params.pathname_db); + result.append(params.pathname_db); + result.append("|"); + + checksum.push(params.pathname_log); + result.append(params.pathname_log); + result.append("|"); + + static_assert(std::is_pod::value, + "actor_params_pod should by POD"); + result.append(data2hex(static_cast(¶ms), + sizeof(actor_params_pod), checksum)); + result.append("|"); + + static_assert(std::is_pod::value, + "actor_config_pod should by POD"); + result.append(data2hex(static_cast(this), + sizeof(actor_config_pod), checksum)); + result.append("|"); + + result.append(osal_serialize(checksum)); + result.append("|"); + + result.append(std::to_string(checksum.value)); + return result; +} + +bool actor_config::deserialize(const char *str, actor_config &config) { + simple_checksum checksum; + + TRACE(">> actor_config::deserialize: %s\n", str); + + const char *slash = strchr(str, '|'); + if (!slash) { + TRACE("<< actor_config::deserialize: slash-1\n"); + return false; + } + config.params.pathname_db.assign(str, slash - str); + checksum.push(config.params.pathname_db); + str = slash + 1; + + slash = strchr(str, '|'); + if (!slash) { + TRACE("<< actor_config::deserialize: slash-2\n"); + return false; + } + config.params.pathname_log.assign(str, slash - str); + checksum.push(config.params.pathname_log); + str = slash + 1; + + slash = strchr(str, '|'); + if (!slash) { + TRACE("<< actor_config::deserialize: slash-3\n"); + return false; + } + static_assert(std::is_pod::value, + "actor_params_pod should by POD"); + if (!hex2data(str, slash, static_cast(&config.params), + sizeof(actor_params_pod), checksum)) { + TRACE("<< actor_config::deserialize: actor_params_pod(%.*s)\n", + (int)(slash - str), str); + return false; + } + str = slash + 1; + + slash = strchr(str, '|'); + if (!slash) { + TRACE("<< actor_config::deserialize: slash-4\n"); + return false; + } + static_assert(std::is_pod::value, + "actor_config_pod should by POD"); + if (!hex2data(str, slash, static_cast(&config), + sizeof(actor_config_pod), checksum)) { + TRACE("<< actor_config::deserialize: actor_config_pod(%.*s)\n", + (int)(slash - str), str); + return false; + } + str = slash + 1; + + slash = strchr(str, '|'); + if (!slash) { + TRACE("<< actor_config::deserialize: slash-5\n"); + return false; + } + if (!config.osal_deserialize(str, slash, checksum)) { + TRACE("<< actor_config::deserialize: osal\n"); + return false; + } + str = slash + 1; + + uint64_t verify = std::stoull(std::string(str)); + if (checksum.value != verify) { + TRACE("<< actor_config::deserialize: checksum mismatch\n"); + return false; + } + + TRACE("<< actor_config::deserialize: OK\n"); + return true; +} diff --git a/test/config.h b/test/config.h new file mode 100644 index 00000000..c16eca9a --- /dev/null +++ b/test/config.h @@ -0,0 +1,149 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once + +#include "base.h" +#include "log.h" +#include "utils.h" + +#define ACTOR_ID_MAX INT16_MAX + +enum actor_testcase { ac_none, ac_hill, ac_deadread, ac_deadwrite, ac_jitter }; + +enum actor_status { + as_unknown, + as_debuging, + as_running, + as_successful, + as_killed, + as_failed +}; + +const char *testcase2str(const actor_testcase); +const char *status2str(actor_status status); + +//----------------------------------------------------------------------------- + +namespace config { + +enum scale_mode { no_scale, decimal, binary, duration }; + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + const char **value, const char *default_value = nullptr); + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + std::string &value, bool allow_empty = false); + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + bool &value); + +struct option_verb { + const char *const verb; + unsigned mask; +}; + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + size_t &mask, const option_verb *verbs); + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + uint64_t &value, const scale_mode scale, + const uint64_t minval = 0, const uint64_t maxval = INT64_MAX); + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + unsigned &value, const scale_mode scale, + const unsigned minval = 0, const unsigned maxval = INT32_MAX); + +//----------------------------------------------------------------------------- + +#pragma pack(push, 1) + +struct actor_params_pod { + unsigned loglevel; + + size_t mode_flags; + size_t table_flags; + uint64_t size; + unsigned seed; + + unsigned test_duration; + unsigned test_nrecords; + unsigned nrepeat; + unsigned nthreads; + + unsigned keylen_min, keylen_max; + unsigned datalen_min, datalen_max; + + unsigned batch_read; + unsigned batch_write; + + unsigned delaystart; + unsigned waitfor_nops; + + bool drop_table; + + unsigned max_readers; + unsigned max_tables; +}; + +struct actor_config_pod { + unsigned id, order; + actor_testcase testcase; + unsigned wait4id; + unsigned signal_nops; +}; + +#pragma pack(pop) + +extern const struct option_verb mode_bits[]; +extern const struct option_verb table_bits[]; +void dump(FILE *out); + +} /* namespace config */ + +struct actor_params : public config::actor_params_pod { + std::string pathname_log; + std::string pathname_db; + void set_defaults(void); +}; + +struct actor_config : public config::actor_config_pod { + actor_params params; + + bool wanna_event4signalling() const { return true /* TODO ? */; } + + actor_config(actor_testcase testcase, const actor_params ¶ms, unsigned id, + unsigned wait4id); + + actor_config(const char *str) { + if (!deserialize(str, *this)) + failure("Invalid internal parameter '%s'\n", str); + } + + const std::string osal_serialize(simple_checksum &) const; + bool osal_deserialize(const char *str, const char *end, simple_checksum &); + + const std::string serialize(const char *prefix) const; + static bool deserialize(const char *str, actor_config &config); + + bool is_waitable(size_t nops) const { + switch (testcase) { + case ac_hill: + if (!params.test_nrecords || params.test_nrecords >= nops) + return true; + default: + return false; + } + } +}; diff --git a/test/dead.cc b/test/dead.cc new file mode 100644 index 00000000..7afa042d --- /dev/null +++ b/test/dead.cc @@ -0,0 +1,61 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +bool testcase_deadread::setup() { + log_trace(">> setup"); + if (!inherited::setup()) + return false; + + log_trace("<< setup"); + return true; +} + +bool testcase_deadread::run() { + /* TODO */ + return true; +} + +bool testcase_deadread::teardown() { + log_trace(">> teardown"); + cursor_guard.release(); + txn_guard.release(); + db_guard.release(); + return true; +} + +//----------------------------------------------------------------------------- + +bool testcase_deadwrite::setup() { + log_trace(">> setup"); + if (!inherited::setup()) + return false; + + log_trace("<< setup"); + return true; +} + +bool testcase_deadwrite::run() { + /* TODO */ + return true; +} + +bool testcase_deadwrite::teardown() { + log_trace(">> teardown"); + cursor_guard.release(); + txn_guard.release(); + db_guard.release(); + return true; +} diff --git a/test/hill.cc b/test/hill.cc new file mode 100644 index 00000000..98a7b82c --- /dev/null +++ b/test/hill.cc @@ -0,0 +1,37 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +bool testcase_hill::setup() { + log_trace(">> setup"); + if (!inherited::setup()) + return false; + + /* TODO */ + + log_trace("<< setup"); + return true; +} + +bool testcase_hill::run() { + mdbx_open(); + /* TODO */ + return true; +} + +bool testcase_hill::teardown() { + log_trace(">> teardown"); + return inherited::teardown(); +} diff --git a/test/jitter.cc b/test/jitter.cc new file mode 100644 index 00000000..00385362 --- /dev/null +++ b/test/jitter.cc @@ -0,0 +1,33 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +bool testcase_jitter::setup() { + log_trace(">> setup"); + if (!inherited::setup()) + return false; + + /* TODO */ + + log_trace("<< setup"); + return true; +} + +bool testcase_jitter::run() { return true; } + +bool testcase_jitter::teardown() { + log_trace(">> teardown"); + return inherited::teardown(); +} diff --git a/test/keygen.cc b/test/keygen.cc new file mode 100644 index 00000000..20c80a2a --- /dev/null +++ b/test/keygen.cc @@ -0,0 +1,72 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +namespace keygen { + +size_t ffs_fallback(serial_t serial) { + size_t bit = sizeof(serial_t) * 8 - 1; + auto mask = (serial_t)1u << bit; + do { + if (serial & mask) + return bit; + --bit; + } while (mask >>= 1); + return 0; +} + +void __hot make(const serial_t serial, const params_t ¶ms, result_t &out) { + assert(out.limit >= params.maxlen); + assert(params.maxlen >= params.minlen); + assert(params.maxlen >= length(serial)); + + out.value.mv_data = out.bytes; + out.value.mv_size = params.minlen; + + if (params.flags & (MDB_INTEGERKEY | MDB_INTEGERDUP)) { + assert(params.maxlen == params.minlen); + assert(params.minlen == 4 || params.minlen == 8); + if (is_byteorder_le() || params.minlen == 8) + out.u64 = serial; + else + out.u32 = (uint32_t)serial; + } else if (params.flags & (MDB_REVERSEKEY | MDB_REVERSEDUP)) { + if (out.value.mv_size > 8) { + memset(out.bytes, '\0', out.value.mv_size - 8); + unaligned::store(out.bytes + out.value.mv_size - 8, htobe64(serial)); + } else { + out.u64 = htobe64(serial); + if (out.value.mv_size < 8) { + out.value.mv_size = std::max(length(serial), out.value.mv_size); + out.value.mv_data = out.bytes + 8 - out.value.mv_size; + } + } + } else { + out.u64 = htole64(serial); + if (out.value.mv_size > 8) + memset(out.bytes + 8, '\0', out.value.mv_size - 8); + else + out.value.mv_size = std::max(length(serial), out.value.mv_size); + } + + assert(out.value.mv_size >= params.minlen); + assert(out.value.mv_size <= params.maxlen); + assert(out.value.mv_size >= length(serial)); + assert(out.value.mv_data >= out.bytes); + assert((uint8_t *)out.value.mv_data + out.value.mv_size <= + out.bytes + out.limit); +} + +} /* namespace keygen */ diff --git a/test/keygen.h b/test/keygen.h new file mode 100644 index 00000000..58db2633 --- /dev/null +++ b/test/keygen.h @@ -0,0 +1,123 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once + +#include "base.h" +#include "log.h" +#include "utils.h" + +namespace keygen { + +/* Под "генерацией ключей" здесь понимается генерация обоих значений для + * пар key-value, т.е. не только ключей, но и ассоциированных с ними данных. + */ + +/* Генерацию ключей нельзя отнести к простым задачам, так как требования + * примерно следующие: + * - генерация разного количества уникальных ключей различной длины + * в задаваемом диапазоне; + * - возможность выбора как псевдо-случайного порядка ключей, + * так и по некоторым специфическим законам (ограниченными упорядоченными + * последовательностями, в шахматном порядке по граница диапазона и т.д.); + * - возможность генерации дубликатов с задаваемым законом распределения; + * - возможность генерации непересекающимися кластерами для параллельного + * использования в нескольких потоках; + * - использовать минимум ресурсов, как CPU, так и RAM, в том числе + * включая cache pollution и ram bandwidth. + * + * При этом заведомо известно, что для MDBX не имеет значения: + * - используемый алфавит (значения байтов); + * - частотное распределение по алфавиту; + * - абсолютное значение ключей или разность между отдельными значениями; + * + * Соответственно, схема генерации следующая: + * - для ключей вводится плоская одномерная "координата" uint64_t; + * - все преобразования (назначение диапазонов, переупорядочивание, + * коррекция распределения) выполняются только над "координатой"; + * - итоговая "координата" преобразуется в 8-байтное суррогатное значение + * ключа, при этом опционально суррогат может усекаться до ненулевых байт; + * - для получения ключей длиной более 8 байт суррогат дополняется + * фиксированной последовательностью; + */ + +typedef uint64_t serial_t; + +struct params_t { + uint8_t minlen; + uint8_t flags; + uint16_t maxlen; +}; + +struct result_t { + MDB_val value; + size_t limit; + union { + uint8_t bytes[sizeof(uint64_t)]; + uint32_t u32; + uint64_t u64; + }; +}; + +void make(const serial_t serial, const params_t ¶ms, result_t &out); + +static __inline void make(const serial_t serial, const params_t ¶ms, + result_t &out, size_t limit) { + out.limit = limit; + make(serial, params, out); +} + +size_t ffs_fallback(serial_t serial); + +static __inline size_t ffs(serial_t serial) { + size_t rc; +#ifdef __GNUC__ + if (sizeof(serial) <= sizeof(int)) + rc = __builtin_ffs((int)serial); + else if (sizeof(serial) == sizeof(long)) + rc = __builtin_ffsl((long)serial); + else if (sizeof(serial) == sizeof(long long)) + rc = __builtin_ffsll((long long)serial); + else + return ffs_fallback(serial); +#elif defined(_MSC_VER) + unsigned long index; + if (sizeof(serial) <= sizeof(unsigned long)) + rc = _BitScanReverse(&index, (unsigned long)serial) ? index : 0; + else if (sizeof(serial) <= sizeof(unsigned __int64)) { +#if defined(_M_ARM64) || defined(_M_X64) + rc = _BitScanReverse64(&index, (unsigned __int64)serial) ? index : 0; +#else + size_t base = 0; + unsigned long value = (unsigned long)serial; + if ((unsigned __int64)serial > ULONG_MAX) { + base = 32; + value = (unsigned long)(serial >> 32); + } + rc = (_BitScanReverse(&index, value) ? index : 0) + base; +#endif /* _M_ARM64 || _M_X64 */ + } else + return ffs_fallback(serial); +#else + return ffs_fallback(serial); +#endif + assert(rc == ffs_fallback(serial)); + return rc; +} + +static __inline size_t length(const serial_t serial) { + return (ffs(serial) + 7) >> 3; +} + +} /* namespace keygen */ diff --git a/test/log.cc b/test/log.cc new file mode 100644 index 00000000..d149393e --- /dev/null +++ b/test/log.cc @@ -0,0 +1,129 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +void failure(const char *fmt, ...) { + va_list ap; + fflush(NULL); + va_start(ap, fmt); + loggging::output(loggging::failure, fmt, ap); + va_end(ap); + fflush(NULL); + exit(EXIT_FAILURE); +} + +const char *test_strerror(int errnum) { + static __thread char buf[1024]; + return mdbx_strerror_r(errnum, buf, sizeof(buf)); +} + +void __noreturn failure_perror(const char *what, int errnum) { + failure("%s failed: %s (%d)\n", what, test_strerror(errnum), errnum); +} + +//----------------------------------------------------------------------------- + +namespace loggging { + +static std::string prefix; +static loglevel level; + +void setup(loglevel _level, const std::string &_prefix) { + level = (_level > error) ? failure : _level; + prefix = _prefix; +} + +void setup(const std::string &_prefix) { prefix = _prefix; } + +const char *level2str(const loglevel level) { + switch (level) { + default: + return "invalid/unknown"; + case trace: + return "trace"; + case info: + return "info"; + case notice: + return "notice"; + case warning: + return "warning"; + case error: + return "error"; + case failure: + return "failure"; + } +} + +void output(loglevel priority, const char *format, va_list ap) { + if (priority >= level) { + fprintf(stderr, "[ %u %-10s %6s ] " /* TODO */, osal_getpid(), + prefix.c_str(), level2str(priority)); + vfprintf(stderr, format, ap); + size_t len = strlen(format); + if (len && format[len - 1] != '\n') + putc('\n', stderr); + } +} + +} /* namespace log */ + +void log_trace(const char *msg, ...) { + if (loggging::trace >= loggging::level) { + va_list ap; + va_start(ap, msg); + loggging::output(loggging::trace, msg, ap); + va_end(ap); + } +} + +void log_info(const char *msg, ...) { + if (loggging::info >= loggging::level) { + va_list ap; + va_start(ap, msg); + loggging::output(loggging::info, msg, ap); + va_end(ap); + } +} + +void log_notice(const char *msg, ...) { + if (loggging::notice >= loggging::level) { + va_list ap; + va_start(ap, msg); + loggging::output(loggging::notice, msg, ap); + va_end(ap); + } +} + +void log_warning(const char *msg, ...) { + if (loggging::warning >= loggging::level) { + va_list ap; + va_start(ap, msg); + loggging::output(loggging::warning, msg, ap); + va_end(ap); + } +} + +void log_error(const char *msg, ...) { + if (loggging::error >= loggging::level) { + va_list ap; + va_start(ap, msg); + loggging::output(loggging::error, msg, ap); + va_end(ap); + } +} + +void log_touble(const char *where, const char *what, int errnum) { + log_error("%s: %s %s", where, what, test_strerror(errnum)); +} diff --git a/test/log.h b/test/log.h new file mode 100644 index 00000000..627a11a0 --- /dev/null +++ b/test/log.h @@ -0,0 +1,61 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once + +#include "base.h" + +void __noreturn usage(void); + +void __noreturn +#ifdef __GNUC__ + __attribute__((format(printf, 1, 2))) +#endif + failure(const char *fmt, ...); + +void __noreturn failure_perror(const char *what, int errnum); +const char *test_strerror(int errnum); + +namespace loggging { + +enum loglevel { + trace, + info, + notice, + warning, + error, + failure, +}; + +const char *level2str(const loglevel level); +void setup(loglevel level, const std::string &prefix); +void setup(const std::string &prefix); + +void output(loglevel priority, const char *format, va_list ap); + +} /* namespace log */ + +void log_trace(const char *msg, ...); +void log_info(const char *msg, ...); +void log_notice(const char *msg, ...); +void log_warning(const char *msg, ...); +void log_error(const char *msg, ...); + +void log_touble(const char *where, const char *what, int errnum); + +#ifdef _DEBUG +#define TRACE(...) log_trace(__VA_ARGS__) +#else +#define TRACE(...) __noop(__VA_ARGS__) +#endif diff --git a/test/main.cc b/test/main.cc new file mode 100644 index 00000000..929f2094 --- /dev/null +++ b/test/main.cc @@ -0,0 +1,311 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +void __noreturn usage(void) { + printf("usage:\n" + "\tFIXME\n"); + exit(EXIT_FAILURE); +} + +//----------------------------------------------------------------------------- + +void actor_params::set_defaults(void) { + pathname_log = ""; + loglevel = +#ifdef NDEBUG + loggging::notice; +#else + loggging::trace; +#endif + + pathname_db = +#ifdef __linux__ + "/dev/shm/test_tmpdb.mdbx"; +#else + "test_tmpdb.mdbx"; +#endif + mode_flags = MDB_NOSUBDIR | MDB_WRITEMAP | MDB_MAPASYNC | MDB_NORDAHEAD | + MDB_NOMEMINIT | MDBX_COALESCE | MDBX_LIFORECLAIM; + table_flags = MDB_DUPSORT; + size = 1024 * 1024; + seed = 1; + + test_duration = 0; + test_nrecords = 1000; + nrepeat = 1; + nthreads = 1; + + keylen_min = 0; + keylen_max = 42; + datalen_min = 0; + datalen_max = 256; + + batch_read = 4; + batch_write = 4; + + delaystart = 0; + waitfor_nops = 0; + + drop_table = false; + + max_readers = 42; + max_tables = 42; +} + +namespace global { + +std::vector actors; +std::unordered_map events; +std::unordered_map pid2actor; +std::set databases; +unsigned nactors; + +namespace config { +unsigned timeout; +bool dump_config; +bool dont_cleanup_before; +bool dont_cleanup_after; +} /* namespace config */ + +} /* namespace global */ + +//----------------------------------------------------------------------------- + +const char global::thunk_param_prefix[] = "--execute="; + +std::string thunk_param(const actor_config &config) { + return config.serialize(global::thunk_param_prefix); +} + +void cleanup() { + log_trace(">> osal_setup"); + /* TODO: remove each database */ + log_trace("<< osal_setup"); +} + +int main(int argc, char *const argv[]) { + +#ifdef _DEBUG + log_trace("#argc = %d", argc); + for (int i = 0; i < argc; ++i) + log_trace("#argv[%d] = %s", i, argv[i]); +#endif /* _DEBUG */ + + if (argc < 2) + failure("No parameters given\n"); + + if (argc == 2 && + strncmp(argv[1], global::thunk_param_prefix, + strlen(global::thunk_param_prefix)) == 0) + return test_execute( + actor_config(argv[1] + strlen(global::thunk_param_prefix))) + ? EXIT_SUCCESS + : EXIT_FAILURE; + + actor_params params; + params.set_defaults(); + global::config::dump_config = true; + loggging::setup((loggging::loglevel)params.loglevel, "main"); + unsigned lastid = 0; + + if (argc == 2 && strncmp(argv[1], "--case=", 7) == 0) { + const char *casename = argv[1] + 7; + if (!testcase_setup(casename, params, lastid)) + failure("unknown testcase `%s`", casename); + } else { + for (int i = 1; i < argc;) { + const char *value = nullptr; + if (config::parse_option(argc, argv, i, "basic", nullptr)) { + bool ok = testcase_setup("basic", params, lastid); + assert(ok); + (void)ok; + } else if (config::parse_option(argc, argv, i, "race", nullptr)) { + bool ok = testcase_setup("race", params, lastid); + assert(ok); + (void)ok; + } else if (config::parse_option(argc, argv, i, "bench", nullptr)) { + bool ok = testcase_setup("bench", params, lastid); + assert(ok); + (void)ok; + } else if (config::parse_option(argc, argv, i, "pathname", + params.pathname_db) || + config::parse_option(argc, argv, i, "mode", params.mode_flags, + config::mode_bits) || + config::parse_option(argc, argv, i, "table", + params.table_flags, config::table_bits) || + config::parse_option(argc, argv, i, "size", params.size, + config::binary, 4096 * 4) || + config::parse_option(argc, argv, i, "seed", params.seed, + config::no_scale) || + config::parse_option(argc, argv, i, "repeat", params.nrepeat, + config::no_scale) || + config::parse_option(argc, argv, i, "threads", params.nthreads, + config::no_scale, 1, 64) || + config::parse_option(argc, argv, i, "timeout", + global::config::timeout, config::duration, + 1) || + config::parse_option(argc, argv, i, "keylen.min", + params.keylen_min, config::no_scale, 0, + params.keylen_max) || + config::parse_option(argc, argv, i, "keylen.max", + params.keylen_max, config::no_scale, + params.keylen_min, + mdbx_get_maxkeysize(0)) || + config::parse_option(argc, argv, i, "datalen.min", + params.datalen_min, config::no_scale, 0, + params.datalen_max) || + config::parse_option(argc, argv, i, "datalen.max", + params.datalen_max, config::no_scale, + params.datalen_min, MDBX_MAXDATASIZE) || + config::parse_option(argc, argv, i, "batch.read", + params.batch_read, config::no_scale, 1) || + config::parse_option(argc, argv, i, "batch.write", + params.batch_write, config::no_scale, + 1) || + config::parse_option(argc, argv, i, "delay", params.delaystart, + config::duration) || + config::parse_option(argc, argv, i, "wait4ops", + params.waitfor_nops, config::decimal) || + config::parse_option(argc, argv, i, "drop", + params.drop_table) || + config::parse_option(argc, argv, i, "dump-config", + global::config::dump_config) || + config::parse_option(argc, argv, i, "dont-cleanup-before", + global::config::dont_cleanup_before) || + config::parse_option(argc, argv, i, "dont-cleanup-after", + global::config::dont_cleanup_after) || + config::parse_option(argc, argv, i, "max-readers", + params.max_readers, config::no_scale, 1, + 255) || + config::parse_option(argc, argv, i, "max-tables", + params.max_tables, config::no_scale, 1, + INT16_MAX) || + false) { + continue; + } else if (config::parse_option(argc, argv, i, "no-delay", nullptr)) { + params.delaystart = 0; + } else if (config::parse_option(argc, argv, i, "no-wait", nullptr)) { + params.waitfor_nops = 0; + } else if (config::parse_option(argc, argv, i, "duration", + params.test_duration, config::duration, + 1)) { + params.test_nrecords = 0; + continue; + } else if (config::parse_option(argc, argv, i, "records", + params.test_nrecords, config::decimal, + 1)) { + params.test_duration = 0; + continue; + } else if (config::parse_option(argc, argv, i, "hill", &value)) { + configure_actor(lastid, ac_hill, value, params); + continue; + } else if (config::parse_option(argc, argv, i, "jitter", nullptr)) { + configure_actor(lastid, ac_jitter, value, params); + continue; + } else if (config::parse_option(argc, argv, i, "dead.reader", nullptr)) { + configure_actor(lastid, ac_deadread, value, params); + continue; + } else if (config::parse_option(argc, argv, i, "dead.writer", nullptr)) { + configure_actor(lastid, ac_deadwrite, value, params); + continue; + } else { + failure("Unknown option '%s'\n", argv[i]); + } + } + } + + if (global::config::dump_config) + config::dump(stdout); + + bool failed = false; + if (global::actors.size()) { + loggging::setup("overlord"); + + if (!global::config::dont_cleanup_before) + cleanup(); + + log_trace(">> osal_setup"); + osal_setup(global::actors); + log_trace("<< osal_setup"); + + for (auto &a : global::actors) { + mdbx_pid_t pid; + log_trace(">> actor_start"); + int rc = osal_actor_start(a, pid); + log_trace("<< actor_start"); + if (rc) { + log_trace(">> killall_actors"); + osal_killall_actors(); + log_trace("<< killall_actors"); + failure("Failed to start actor #%u (%s)\n", a.order, test_strerror(rc)); + } + global::pid2actor[pid] = &a; + } + + atexit(osal_killall_actors); + log_trace(">> wait4barrier"); + osal_wait4barrier(); + log_trace("<< wait4barrier"); + } + + time_t timestamp_start = time(nullptr); + size_t left = global::actors.size(); + + while (left > 0) { + unsigned timeout = INT_MAX; + if (global::config::timeout) { + time_t timestamp_now = time(nullptr); + if (timestamp_now - timestamp_start > global::config::timeout) + timeout = 0; + else + timeout = global::config::timeout - + (unsigned)(timestamp_now - timestamp_start); + } + + mdbx_pid_t pid; + int rc = osal_actor_poll(pid, timeout); + if (rc) + failure("Poll error: %s (%d)\n", test_strerror(rc), rc); + + if (pid) { + actor_status status = osal_actor_info(pid); + actor_config *actor = global::pid2actor.at(pid); + if (!actor) + continue; + + log_info("actor #%u, id %d, pid %u: %s\n", actor->order, actor->id, pid, + status2str(status)); + if (status > as_running) { + left -= 1; + if (status != as_successful) + failed = true; + } + } else { + if (global::config::timeout && + time(nullptr) - timestamp_start > global::config::timeout) + failure("Timeout\n"); + } + } + + log_notice("OVERALL: %s\n", failed ? "Failed" : "Successful"); + if (!global::config::dont_cleanup_before) { + if (failed) + log_info("skip cleanup"); + else + cleanup(); + } + return failed ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/test/osal-unix.cc b/test/osal-unix.cc new file mode 100644 index 00000000..5131181c --- /dev/null +++ b/test/osal-unix.cc @@ -0,0 +1,230 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +#include +#include +#include +#include +#include +#include + +struct shared_t { + pthread_barrier_t barrier; + pthread_mutex_t mutex; + pthread_cond_t conds[0]; +}; + +static shared_t *shared; +static std::unordered_map events; + +void osal_wait4barrier(void) { + assert(shared != nullptr && shared != MAP_FAILED); + int rc = pthread_barrier_wait(&shared->barrier); + if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { + failure_perror("pthread_barrier_wait(shared)", rc); + } +} + +void osal_setup(const std::vector &actors) { + assert(shared == nullptr); + + pthread_mutexattr_t mutexattr; + int rc = pthread_mutexattr_init(&mutexattr); + if (rc) + failure_perror("pthread_mutexattr_init()", rc); + rc = pthread_mutexattr_setpshared(&mutexattr, PTHREAD_PROCESS_SHARED); + if (rc) + failure_perror("pthread_mutexattr_setpshared()", rc); + + pthread_barrierattr_t barrierattr; + rc = pthread_barrierattr_init(&barrierattr); + if (rc) + failure_perror("pthread_barrierattr_init()", rc); + rc = pthread_barrierattr_setpshared(&barrierattr, PTHREAD_PROCESS_SHARED); + if (rc) + failure_perror("pthread_barrierattr_setpshared()", rc); + + pthread_condattr_t condattr; + rc = pthread_condattr_init(&condattr); + if (rc) + failure_perror("pthread_condattr_init()", rc); + rc = pthread_condattr_setpshared(&condattr, PTHREAD_PROCESS_SHARED); + if (rc) + failure_perror("pthread_condattr_setpshared()", rc); + + size_t n = 0; + for (const auto &a : actors) + if (a.wanna_event4signalling()) + ++n; + + shared = (shared_t *)mmap( + nullptr, sizeof(shared_t) + n * sizeof(pthread_cond_t), + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (MAP_FAILED == (void *)shared) + failure_perror("mmap(shared_conds)", errno); + + rc = pthread_mutex_init(&shared->mutex, &mutexattr); + if (rc) + failure_perror("pthread_mutex_init(shared)", rc); + + rc = pthread_barrier_init(&shared->barrier, &barrierattr, actors.size() + 1); + if (rc) + failure_perror("pthread_barrier_init(shared)", rc); + + auto a = actors.begin(); + for (size_t i = 0; i < n; ++i) { + pthread_cond_t *event = &shared->conds[i]; + rc = pthread_cond_init(event, &condattr); + if (rc) + failure_perror("pthread_cond_init(shared)", rc); + + unsigned id = 0; + while (a != actors.end()) { + if (a->wanna_event4signalling()) { + id = a->id; + break; + } + ++a; + } + assert(id != 0); + events[id] = event; + } + + pthread_barrierattr_destroy(&barrierattr); + pthread_condattr_destroy(&condattr); + pthread_mutexattr_destroy(&mutexattr); +} + +void osal_broadcast(unsigned id) { + assert(shared != nullptr && shared != MAP_FAILED); + int rc = pthread_cond_broadcast(events.at(id)); + if (rc) + failure_perror("sem_post(shared)", rc); +} + +int osal_waitfor(unsigned id) { + assert(shared != nullptr && shared != MAP_FAILED); + + int rc = pthread_mutex_lock(&shared->mutex); + if (rc != 0) + failure_perror("pthread_mutex_lock(shared)", rc); + + rc = pthread_cond_wait(events.at(id), &shared->mutex); + if (rc && rc != EINTR) + failure_perror("pthread_cond_wait(shared)", rc); + + rc = pthread_mutex_unlock(&shared->mutex); + if (rc != 0) + failure_perror("pthread_mutex_unlock(shared)", rc); + + return (rc == 0) ? true : false; +} + +//----------------------------------------------------------------------------- + +const std::string +actor_config::osal_serialize(simple_checksum &checksum) const { + (void)checksum; + /* not used in workload, but just for testing */ + return "unix.fork"; +} + +bool actor_config::osal_deserialize(const char *str, const char *end, + simple_checksum &checksum) { + (void)checksum; + /* not used in workload, but just for testing */ + return strncmp(str, "unix.fork", 9) == 0 && str + 9 == end; +} + +//----------------------------------------------------------------------------- + +static std::unordered_map childs; + +static void handler_SIGCHLD(int unused) { (void)unused; } + +mdbx_pid_t osal_getpid(void) { return getpid(); } + +int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) { + if (childs.empty()) + signal(SIGCHLD, handler_SIGCHLD); + + pid = fork(); + + if (pid == 0) { + const bool result = test_execute(config); + exit(result ? EXIT_SUCCESS : EXIT_FAILURE); + } + + if (pid < 0) + return errno; + + childs[pid] = as_running; + return 0; +} + +actor_status osal_actor_info(const mdbx_pid_t pid) { return childs.at(pid); } + +void osal_killall_actors(void) { + for (auto &pair : childs) { + kill(pair.first, SIGKILL); + pair.second = as_killed; + } +} + +int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) { +retry: + int status, options = WNOHANG; +#ifdef WUNTRACED + options |= WUNTRACED; +#endif +#ifdef WCONTINUED + options |= WCONTINUED; +#endif + pid = waitpid(0, &status, options); + + if (pid > 0) { + if (WIFEXITED(status)) + childs[pid] = + (WEXITSTATUS(status) == EXIT_SUCCESS) ? as_successful : as_failed; + else if (WIFSIGNALED(status) || WCOREDUMP(status)) + childs[pid] = as_killed; + else if (WIFSTOPPED(status)) + childs[pid] = as_debuging; + else if (WIFCONTINUED(status)) + childs[pid] = as_running; + else { + assert(false); + } + return 0; + } + + if (pid == 0) { + if (timeout && sleep(timeout)) + goto retry; + return 0; + } + + switch (errno) { + case EINTR: + pid = 0; + return 0; + + case ECHILD: + default: + pid = 0; + return errno; + } +} diff --git a/test/osal-windows.cc b/test/osal-windows.cc new file mode 100644 index 00000000..d4083c81 --- /dev/null +++ b/test/osal-windows.cc @@ -0,0 +1,262 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +static std::unordered_map events; +static HANDLE hBarrierSemaphore, hBarrierEvent; + +static int waitstatus2errcode(DWORD result) { + switch (result) { + case WAIT_OBJECT_0: + return MDB_SUCCESS; + case WAIT_FAILED: + return GetLastError(); + case WAIT_ABANDONED: + return ERROR_ABANDONED_WAIT_0; + case WAIT_IO_COMPLETION: + return ERROR_USER_APC; + case WAIT_TIMEOUT: + return ERROR_TIMEOUT; + default: + return ERROR_UNHANDLED_ERROR; + } +} + +void osal_wait4barrier(void) { + DWORD rc = WaitForSingleObject(hBarrierSemaphore, 0); + switch (rc) { + default: + failure_perror("WaitForSingleObject(BarrierSemaphore)", + waitstatus2errcode(rc)); + case WAIT_OBJECT_0: + rc = WaitForSingleObject(hBarrierEvent, INFINITE); + if (rc != WAIT_OBJECT_0) + failure_perror("WaitForSingleObject(BarrierEvent)", + waitstatus2errcode(rc)); + break; + case WAIT_TIMEOUT: + if (!SetEvent(hBarrierEvent)) + failure_perror("SetEvent(BarrierEvent)", GetLastError()); + break; + } +} + +static HANDLE make_inharitable(HANDLE hHandle) { + assert(hHandle != NULL && hHandle != INVALID_HANDLE_VALUE); + if (!DuplicateHandle(GetCurrentProcess(), hHandle, GetCurrentProcess(), + &hHandle, 0, TRUE, + DUPLICATE_CLOSE_SOURCE | DUPLICATE_SAME_ACCESS)) + failure_perror("DuplicateHandle()", GetLastError()); + return hHandle; +} + +void osal_setup(const std::vector &actors) { + size_t n = 0; + for (const auto &a : actors) { + if (a.wanna_event4signalling()) { + HANDLE hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + if (!hEvent) + failure_perror("CreateEvent()", GetLastError()); + hEvent = make_inharitable(hEvent); + events[a.id] = hEvent; + } + } + + hBarrierSemaphore = CreateSemaphore(NULL, 0, (LONG)actors.size(), NULL); + if (!hBarrierSemaphore) + failure_perror("CreateSemaphore(BarrierSemaphore)", GetLastError()); + hBarrierSemaphore = make_inharitable(hBarrierSemaphore); + + hBarrierEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + if (!hBarrierEvent) + failure_perror("CreateEvent(BarrierEvent)", GetLastError()); + hBarrierEvent = make_inharitable(hBarrierEvent); +} + +void osal_broadcast(unsigned id) { + if (!SetEvent(events.at(id))) + failure_perror("SetEvent()", GetLastError()); +} + +int osal_waitfor(unsigned id) { + DWORD rc = WaitForSingleObject(events.at(id), INFINITE); + return waitstatus2errcode(rc); +} + +mdbx_pid_t osal_getpid(void) { return GetCurrentProcessId(); } + +//----------------------------------------------------------------------------- + +const std::string +actor_config::osal_serialize(simple_checksum &checksum) const { + checksum.push(hBarrierSemaphore); + checksum.push(hBarrierEvent); + + HANDLE hWait = INVALID_HANDLE_VALUE; + if (wait4id) { + hWait = events.at(wait4id); + checksum.push(hWait); + } + + HANDLE hSignal = INVALID_HANDLE_VALUE; + if (wanna_event4signalling()) { + hSignal = events.at(id); + checksum.push(hSignal); + } + + return format("%p.%p.%p.%p", hBarrierSemaphore, hBarrierEvent, hWait, + hSignal); +} + +bool actor_config::osal_deserialize(const char *str, const char *end, + simple_checksum &checksum) { + + std::string copy(str, end - str); + TRACE(">> osal_deserialize(%s)\n", copy.c_str()); + + assert(hBarrierSemaphore == 0); + assert(hBarrierEvent == 0); + assert(events.empty()); + + HANDLE hWait, hSignal; + if (sscanf_s(copy.c_str(), "%p.%p.%p.%p", &hBarrierSemaphore, &hBarrierEvent, + &hWait, &hSignal) != 4) { + TRACE("<< osal_deserialize: failed\n"); + return false; + } + + checksum.push(hBarrierSemaphore); + checksum.push(hBarrierEvent); + + if (wait4id) { + checksum.push(hWait); + events[wait4id] = hWait; + } + + if (wanna_event4signalling()) { + checksum.push(hSignal); + events[id] = hSignal; + } + + TRACE("<< osal_deserialize: OK\n"); + return true; +} + +//----------------------------------------------------------------------------- + +typedef std::pair child; +static std::unordered_map childs; + +int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) { + if (childs.size() == MAXIMUM_WAIT_OBJECTS) + failure("Could't manage more that %u actors on Windows\n", + MAXIMUM_WAIT_OBJECTS); + + _flushall(); + + STARTUPINFOA StartupInfo; + GetStartupInfoA(&StartupInfo); + + char exename[_MAX_PATH]; + DWORD exename_size = sizeof(exename); + if (!QueryFullProcessImageNameA(GetCurrentProcess(), 0, exename, + &exename_size)) + failure_perror("QueryFullProcessImageName()", GetLastError()); + + std::string cmdline = "test_mdbx.child " + thunk_param(config); + + PROCESS_INFORMATION ProcessInformation; + if (!CreateProcessA(exename, const_cast(cmdline.c_str()), + NULL, // Retuned process handle is not inheritable. + NULL, // Retuned thread handle is not inheritable. + TRUE, // Child inherits all inheritable handles. + NORMAL_PRIORITY_CLASS | INHERIT_PARENT_AFFINITY, + NULL, // Inherit the parent's environment. + NULL, // Inherit the parent's current directory. + &StartupInfo, &ProcessInformation)) + return GetLastError(); + + CloseHandle(ProcessInformation.hThread); + pid = ProcessInformation.dwProcessId; + childs[pid] = std::make_pair(ProcessInformation.hProcess, as_running); + return 0; +} + +actor_status osal_actor_info(const mdbx_pid_t pid) { + actor_status status = childs.at(pid).second; + if (status > as_running) + return status; + + DWORD ExitCode; + if (!GetExitCodeProcess(childs.at(pid).first, &ExitCode)) + failure_perror("GetExitCodeProcess()", GetLastError()); + + switch (ExitCode) { + case STILL_ACTIVE: + return as_running; + case EXIT_SUCCESS: + status = as_successful; + break; + // case EXCEPTION_BREAKPOINT: + case EXCEPTION_SINGLE_STEP: + status = as_debuging; + break; + case STATUS_CONTROL_C_EXIT: + case EXCEPTION_NONCONTINUABLE_EXCEPTION: + status = as_killed; + break; + default: + status = as_failed; + break; + } + + childs.at(pid).second = status; + return status; +} + +void osal_killall_actors(void) { + for (auto &pair : childs) + TerminateProcess(pair.second.first, STATUS_CONTROL_C_EXIT); +} + +int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) { + std::vector handles; + handles.reserve(childs.size()); + for (const auto &pair : childs) + if (pair.second.second <= as_running) + handles.push_back(pair.second.first); + + DWORD rc = + MsgWaitForMultipleObjectsEx((DWORD)handles.size(), &handles[0], + (timeout > 60) ? 60 * 1000 : timeout * 1000, + QS_ALLINPUT | QS_ALLPOSTMESSAGE, 0); + + if (rc >= WAIT_OBJECT_0 && rc < WAIT_OBJECT_0 + handles.size()) { + pid = 0; + for (const auto &pair : childs) + if (pair.second.first == handles[rc - WAIT_OBJECT_0]) { + pid = pair.first; + break; + } + return 0; + } + + if (rc == WAIT_TIMEOUT) { + pid = 0; + return 0; + } + + return waitstatus2errcode(rc); +} diff --git a/test/osal.h b/test/osal.h new file mode 100644 index 00000000..1e5de123 --- /dev/null +++ b/test/osal.h @@ -0,0 +1,28 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once + +#include "base.h" + +void osal_setup(const std::vector &actors); +void osal_broadcast(unsigned id); +int osal_waitfor(unsigned id); + +mdbx_pid_t osal_getpid(void); +int osal_actor_start(const actor_config &config, mdbx_pid_t &pid); +actor_status osal_actor_info(const mdbx_pid_t pid); +void osal_killall_actors(void); +int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout); +void osal_wait4barrier(void); diff --git a/test/test.cc b/test/test.cc new file mode 100644 index 00000000..104fe93f --- /dev/null +++ b/test/test.cc @@ -0,0 +1,192 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +const char *testcase2str(const actor_testcase testcase) { + switch (testcase) { + default: + return "?!"; + case ac_none: + return "none"; + case ac_hill: + return "hill"; + case ac_deadread: + return "deadread"; + case ac_deadwrite: + return "deadwrite"; + case ac_jitter: + return "jitter"; + } +} + +const char *status2str(actor_status status) { + switch (status) { + default: + assert(false); + return "?!"; + case as_debuging: + return "debuging"; + case as_running: + return "running"; + case as_successful: + return "successful"; + case as_killed: + return "killed"; + case as_failed: + return "failed"; + } +} + +//----------------------------------------------------------------------------- + +void testcase::mdbx_prepare() { + log_trace(">> mdbx_prepare"); + + MDB_env *env = nullptr; + int rc = mdbx_env_create(&env); + if (rc != MDB_SUCCESS) + failure_perror("mdbx_env_create()", rc); + + assert(env != nullptr); + db_guard.reset(env); + + rc = mdbx_env_set_userctx(env, this); + if (rc != MDB_SUCCESS) + failure_perror("mdbx_env_set_userctx()", rc); + + rc = mdbx_env_set_maxreaders(env, config.params.max_readers); + if (rc != MDB_SUCCESS) + failure_perror("mdbx_env_set_maxreaders()", rc); + + rc = mdbx_env_set_maxdbs(env, config.params.max_tables); + if (rc != MDB_SUCCESS) + failure_perror("mdbx_env_set_maxdbs()", rc); + + rc = mdbx_env_set_mapsize(env, (size_t)config.params.size); + if (rc != MDB_SUCCESS) + failure_perror("mdbx_env_set_mapsize()", rc); + + log_trace("<< mdbx_prepare"); +} + +void testcase::mdbx_open() { + log_trace(">> mdbx_open"); + int rc = mdbx_env_open(db_guard.get(), config.params.pathname_db.c_str(), + (unsigned)config.params.mode_flags, 0640); + if (rc != MDB_SUCCESS) + failure_perror("mdbx_env_open()", rc); + log_trace("<< mdbx_open"); +} + +void testcase::mdbx_close() { + log_trace(">> mdbx_close"); + cursor_guard.reset(); + txn_guard.reset(); + db_guard.reset(); + log_trace("<< mdbx_close"); +} + +bool testcase::wait4start() { + if (config.wait4id) { + log_trace(">> wait4start(%u)", config.wait4id); + int rc = osal_waitfor(config.wait4id); + if (rc) { + log_trace("<< wait4start(%u), failed %s", test_strerror(rc)); + return false; + } + return true; + } else { + log_trace("== wait4start(not needed)"); + return true; + } +} + +void testcase::report(size_t nops_done) { + if (config.signal_nops && !signalled && config.signal_nops <= nops_done) { + log_trace(">> signal(n-ops %zu)", nops_done); + osal_broadcast(config.id); + signalled = true; + log_trace("<< signal(n-ops %zu)", nops_done); + } +} + +void testcase::signal() { + if (!signalled) { + log_trace(">> signal(forced)"); + osal_broadcast(config.id); + signalled = true; + log_trace("<< signal(forced)"); + } +} + +bool testcase::setup() { + mdbx_prepare(); + return wait4start(); +} + +bool testcase::teardown() { + log_trace(">> testcase::teardown"); + signal(); + mdbx_close(); + log_trace("<< testcase::teardown"); + return true; +} + +//----------------------------------------------------------------------------- + +bool test_execute(const actor_config &config) { + const mdbx_pid_t pid = osal_getpid(); + loggging::setup((loggging::loglevel)config.params.loglevel, + format("child_%u.%u", config.order, config.id)); + + log_trace(">> wait4barrier"); + osal_wait4barrier(); + log_trace("<< wait4barrier"); + + try { + std::unique_ptr test; + switch (config.testcase) { + case ac_hill: + test.reset(new testcase_hill(config, pid)); + break; + case ac_deadread: + test.reset(new testcase_deadread(config, pid)); + break; + case ac_deadwrite: + test.reset(new testcase_deadwrite(config, pid)); + break; + case ac_jitter: + test.reset(new testcase_jitter(config, pid)); + break; + default: + test.reset(new testcase(config, pid)); + break; + } + + if (!test->setup()) + log_notice("test setup failed"); + else if (!test->run()) + log_notice("test failed"); + else if (!test->teardown()) + log_notice("test teardown failed"); + else { + log_info("test successed"); + return true; + } + } catch (const std::exception &pipets) { + failure("Exception: %s", pipets.what()); + } + return false; +} diff --git a/test/test.h b/test/test.h new file mode 100644 index 00000000..b1ce82af --- /dev/null +++ b/test/test.h @@ -0,0 +1,145 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once + +#include "base.h" +#include "config.h" +#include "keygen.h" +#include "log.h" +#include "osal.h" +#include "utils.h" + +bool test_execute(const actor_config &config); +std::string thunk_param(const actor_config &config); +bool testcase_setup(const char *casename, const actor_params ¶ms, + unsigned &lastid); +void configure_actor(unsigned &lastid, const actor_testcase testcase, + const char *id_cstr, const actor_params ¶ms); + +namespace global { + +extern const char thunk_param_prefix[]; +extern std::vector actors; +extern std::unordered_map events; +extern std::unordered_map pid2actor; +extern std::set databases; + +namespace config { +extern unsigned timeout; +extern bool dump_config; +extern bool dont_cleanup_before; +extern bool dont_cleanup_after; +} /* namespace config */ + +} /* namespace global */ + +//----------------------------------------------------------------------------- + +struct db_deleter : public std::unary_function { + void operator()(MDB_env *env) const { mdbx_env_close(env); } +}; + +struct txn_deleter : public std::unary_function { + void operator()(MDB_txn *txn) const { + int rc = mdbx_txn_abort(txn); + if (rc) + log_touble(__func__, "mdbx_txn_abort()", rc); + } +}; + +struct cursor_deleter : public std::unary_function { + void operator()(MDB_cursor *cursor) const { mdbx_cursor_close(cursor); } +}; + +typedef std::unique_ptr scoped_db_guard; +typedef std::unique_ptr scoped_txn_guard; +typedef std::unique_ptr scoped_cursor_guard; + +//----------------------------------------------------------------------------- + +class testcase { +protected: + const actor_config &config; + const mdbx_pid_t pid; + + scoped_db_guard db_guard; + scoped_txn_guard txn_guard; + scoped_cursor_guard cursor_guard; + bool signalled; + + void mdbx_prepare(); + void mdbx_open(); + void mdbx_close(); + + bool wait4start(); + void report(size_t nops_done); + void signal(); + +public: + testcase(const actor_config &config, const mdbx_pid_t pid) + : config(config), pid(pid) { + loggging::setup(format("%s_%u.%u", testcase2str(config.testcase), + config.order, config.id)); + } + + virtual bool setup(); + virtual bool run() { return true; } + virtual bool teardown(); + virtual ~testcase() {} +}; + +class testcase_hill : public testcase { + typedef testcase inherited; + +public: + testcase_hill(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool setup(); + bool run(); + bool teardown(); +}; + +class testcase_deadread : public testcase { + typedef testcase inherited; + +public: + testcase_deadread(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool setup(); + bool run(); + bool teardown(); +}; + +class testcase_deadwrite : public testcase { + typedef testcase inherited; + +public: + testcase_deadwrite(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool setup(); + bool run(); + bool teardown(); +}; + +class testcase_jitter : public testcase { + typedef testcase inherited; + +public: + testcase_jitter(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool setup(); + bool run(); + bool teardown(); +}; diff --git a/test/test.vcxproj b/test/test.vcxproj new file mode 100644 index 00000000..331963a8 --- /dev/null +++ b/test/test.vcxproj @@ -0,0 +1,182 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + {6d19209b-ece7-4b9c-941c-0aa2b484f199} + + + + {30E29CE6-E6FC-4D32-AA07-46A55FAF3A31} + Win32Proj + mdbxtest + 8.1 + + + + Application + true + v140 + Unicode + + + Application + false + v140 + true + Unicode + + + Application + true + v140 + Unicode + + + Application + false + v140 + true + Unicode + + + + + + + + + + + + + + + + + + + + + true + + + true + + + false + + + false + + + + Use + Level3 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + test.h + + + Console + true + + + + + Use + Level3 + Disabled + _DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + test.h + + + Console + true + + + + + Level3 + Use + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + test.h + + + Console + true + true + true + + + + + Level3 + Use + MaxSpeed + true + true + NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + test.h + + + Console + true + true + true + + + + + + + + + + + + + + + + + + + + + Create + Create + Create + Create + + + + + + + + diff --git a/test/test0.c b/test/test0.c deleted file mode 100644 index 68919b22..00000000 --- a/test/test0.c +++ /dev/null @@ -1,220 +0,0 @@ -/* mtest.c - memory-mapped database tester/toy */ - -/* - * Copyright 2015-2017 Leonid Yuriev . - * Copyright 2011-2017 Howard Chu, Symas Corp. - * Copyright 2015,2016 Peter-Service R&D LLC. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -#include "../mdbx.h" -#include -#include -#include -#include -#include -#include - -#include - -#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) -#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) -#define CHECK(test, msg) \ - ((test) ? (void)0 : ((void)fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, \ - __LINE__, msg, mdbx_strerror(rc)), \ - abort())) - -#ifndef DBPATH -#define DBPATH "./tmp.db" -#endif - -void *thread_entry(void *ctx) { - MDB_env *env = ctx; - MDB_txn *txn; - int rc; - - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - mdbx_txn_abort(txn); - - return NULL; -} - -int main(int argc, char *argv[]) { - int i = 0, j = 0, rc; - MDB_env *env; - MDB_dbi dbi; - MDB_val key, data; - MDB_txn *txn; - MDBX_stat mst; - MDB_cursor *cursor, *cur2; - MDB_cursor_op op; - int count; - int *values; - char sval[32] = ""; - int env_oflags; - struct stat db_stat, exe_stat; - - (void)argc; - (void)argv; - srand(time(NULL)); - - count = (rand() % 384) + 64; - values = (int *)malloc(count * sizeof(int)); - - for (i = 0; i < count; i++) { - values[i] = rand() % 1024; - } - - E(mdbx_env_create(&env)); - E(mdbx_env_set_maxreaders(env, 42)); - E(mdbx_env_set_mapsize(env, 10485760)); - - E(stat("/proc/self/exe", &exe_stat) ? errno : 0); - E(stat(DBPATH "/.", &db_stat) ? errno : 0); - env_oflags = MDB_NOSYNC; - if (major(db_stat.st_dev) != major(exe_stat.st_dev)) { - /* LY: Assume running inside a CI-environment: - * 1) don't use FIXEDMAP to avoid EBUSY in case collision, - * which could be inspired by address space randomisation feature. - * 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some - * dedicated storage. - */ - env_oflags = 0; - } - E(mdbx_env_open(env, DBPATH, env_oflags, 0664)); - - E(mdbx_txn_begin(env, NULL, 0, &txn)); - E(mdbx_dbi_open(txn, NULL, 0, &dbi)); - - key.mv_size = sizeof(int); - key.mv_data = sval; - - printf("Adding %d values\n", count); - for (i = 0; i < count; i++) { - sprintf(sval, "%03x %d foo bar", values[i], values[i]); - /* Set in each iteration, since MDB_NOOVERWRITE may modify it */ - data.mv_size = sizeof(sval); - data.mv_data = sval; - if (RES(MDB_KEYEXIST, mdbx_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) { - j++; - data.mv_size = sizeof(sval); - data.mv_data = sval; - } - } - if (j) - printf("%d duplicates skipped\n", j); - E(mdbx_txn_commit(txn)); - E(mdbx_env_stat(env, &mst, sizeof(mst))); - - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int)key.mv_size, - (char *)key.mv_data, data.mv_data, (int)data.mv_size, - (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - - j = 0; - key.mv_data = sval; - for (i = count - 1; i > -1; i -= (rand() % 5)) { - j++; - txn = NULL; - E(mdbx_txn_begin(env, NULL, 0, &txn)); - sprintf(sval, "%03x ", values[i]); - if (RES(MDB_NOTFOUND, mdbx_del(txn, dbi, &key, NULL))) { - j--; - mdbx_txn_abort(txn); - } else { - E(mdbx_txn_commit(txn)); - } - } - free(values); - printf("Deleted %d values\n", j); - - E(mdbx_env_stat(env, &mst, sizeof(mst))); - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - printf("Cursor next\n"); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - printf("Cursor last\n"); - E(mdbx_cursor_get(cursor, &key, &data, MDB_LAST)); - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - printf("Cursor prev\n"); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - printf("Cursor last/prev\n"); - E(mdbx_cursor_get(cursor, &key, &data, MDB_LAST)); - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - E(mdbx_cursor_get(cursor, &key, &data, MDB_PREV)); - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - - printf("Deleting with cursor\n"); - E(mdbx_txn_begin(env, NULL, 0, &txn)); - E(mdbx_cursor_open(txn, dbi, &cur2)); - for (i = 0; i < 50; i++) { - if (RES(MDB_NOTFOUND, mdbx_cursor_get(cur2, &key, &data, MDB_NEXT))) - break; - printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int)key.mv_size, - (char *)key.mv_data, data.mv_data, (int)data.mv_size, - (char *)data.mv_data); - E(mdbx_del(txn, dbi, &key, NULL)); - } - - printf("Restarting cursor in txn\n"); - for (op = MDB_FIRST, i = 0; i <= 32; op = MDB_NEXT, i++) { - if (RES(MDB_NOTFOUND, mdbx_cursor_get(cur2, &key, &data, op))) - break; - printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int)key.mv_size, - (char *)key.mv_data, data.mv_data, (int)data.mv_size, - (char *)data.mv_data); - } - mdbx_cursor_close(cur2); - E(mdbx_txn_commit(txn)); - - for (i = 0; i < 41; ++i) { - pthread_t thread; - pthread_create(&thread, NULL, thread_entry, env); - } - - printf("Restarting cursor outside txn\n"); - E(mdbx_txn_begin(env, NULL, 0, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - for (op = MDB_FIRST, i = 0; i <= 32; op = MDB_NEXT, i++) { - if (RES(MDB_NOTFOUND, mdbx_cursor_get(cursor, &key, &data, op))) - break; - printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int)key.mv_size, - (char *)key.mv_data, data.mv_data, (int)data.mv_size, - (char *)data.mv_data); - } - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - - mdbx_dbi_close(env, dbi); - mdbx_env_close(env); - - return 0; -} diff --git a/test/test1.c b/test/test1.c deleted file mode 100644 index e0c8f10c..00000000 --- a/test/test1.c +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright 2015-2017 Leonid Yuriev . - * Copyright 2015,2016 Peter-Service R&D LLC. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -/* Based on mtest2.c - memory-mapped database tester/toy */ - -#include "../mdbx.h" -#include -#include -#include -#include -#include -#include - -#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) -#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) -#define CHECK(test, msg) \ - ((test) ? (void)0 : ((void)fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, \ - __LINE__, msg, mdbx_strerror(rc)), \ - abort())) - -#ifndef DBPATH -#define DBPATH "./tmp.db" -#endif - -int main(int argc, char *argv[]) { - int i = 0, j = 0, rc; - MDB_env *env; - MDB_dbi dbi; - MDB_val key, data; - MDB_txn *txn; - MDBX_stat mst; - MDB_cursor *cursor; - int count; - int *values; - char sval[32] = ""; - int env_oflags; - struct stat db_stat, exe_stat; - - (void)argc; - (void)argv; - srand(time(NULL)); - - count = (rand() % 384) + 64; - values = (int *)malloc(count * sizeof(int)); - - for (i = 0; i < count; i++) { - values[i] = rand() % 1024; - } - - E(mdbx_env_create(&env)); - E(mdbx_env_set_maxreaders(env, 1)); - E(mdbx_env_set_mapsize(env, 10485760)); - E(mdbx_env_set_maxdbs(env, 4)); - - E(stat("/proc/self/exe", &exe_stat) ? errno : 0); - E(stat(DBPATH "/.", &db_stat) ? errno : 0); - env_oflags = MDB_NOSYNC; - if (major(db_stat.st_dev) != major(exe_stat.st_dev)) { - /* LY: Assume running inside a CI-environment: - * 1) don't use FIXEDMAP to avoid EBUSY in case collision, - * which could be inspired by address space randomisation feature. - * 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some - * dedicated storage. - */ - env_oflags = 0; - } - /* LY: especially here we always needs MDB_NOSYNC - * for testing mdbx_env_close_ex() and "redo-to-steady" on open. */ - env_oflags |= MDB_NOSYNC; - E(mdbx_env_open(env, DBPATH, env_oflags, 0664)); - - E(mdbx_txn_begin(env, NULL, 0, &txn)); - if (mdbx_dbi_open(txn, "id1", MDB_CREATE, &dbi) == MDB_SUCCESS) - E(mdbx_drop(txn, dbi, 1)); - E(mdbx_dbi_open(txn, "id1", MDB_CREATE, &dbi)); - - key.mv_size = sizeof(int); - key.mv_data = sval; - data.mv_size = sizeof(sval); - data.mv_data = sval; - - printf("Adding %d values\n", count); - for (i = 0; i < count; i++) { - sprintf(sval, "%03x %d foo bar", values[i], values[i]); - if (RES(MDB_KEYEXIST, mdbx_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) - j++; - } - if (j) - printf("%d duplicates skipped\n", j); - E(mdbx_txn_commit(txn)); - E(mdbx_env_stat(env, &mst, sizeof(mst))); - - printf("check-preset-a\n"); - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - int present_a = 0; - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int)key.mv_size, - (char *)key.mv_data, data.mv_data, (int)data.mv_size, - (char *)data.mv_data); - ++present_a; - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - CHECK(present_a == count - j, "mismatch"); - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - mdbx_env_sync(env, 1); - - int deleted = 0; - key.mv_data = sval; - for (i = count - 1; i > -1; i -= (rand() % 5)) { - txn = NULL; - E(mdbx_txn_begin(env, NULL, 0, &txn)); - sprintf(sval, "%03x ", values[i]); - if (RES(MDB_NOTFOUND, mdbx_del(txn, dbi, &key, NULL))) { - mdbx_txn_abort(txn); - } else { - E(mdbx_txn_commit(txn)); - deleted++; - } - } - free(values); - printf("Deleted %d values\n", deleted); - - printf("check-preset-b.cursor-next\n"); - E(mdbx_env_stat(env, &mst, sizeof(mst))); - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - int present_b = 0; - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - ++present_b; - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - CHECK(present_b == present_a - deleted, "mismatch"); - - printf("check-preset-b.cursor-prev\n"); - j = 1; - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - ++j; - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - CHECK(present_b == j, "mismatch"); - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - - mdbx_dbi_close(env, dbi); - /********************* LY: kept DB dirty ****************/ - mdbx_env_close_ex(env, 1); - E(mdbx_env_create(&env)); - E(mdbx_env_set_maxdbs(env, 4)); - E(mdbx_env_open(env, DBPATH, env_oflags, 0664)); - - printf("check-preset-c.cursor-next\n"); - E(mdbx_env_stat(env, &mst, sizeof(mst))); - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - E(mdbx_dbi_open(txn, "id1", 0, &dbi)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - int present_c = 0; - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - ++present_c; - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - printf("Rolled back %d deletion(s)\n", present_c - (present_a - deleted)); - CHECK(present_c > present_a - deleted, "mismatch"); - - printf("check-preset-d.cursor-prev\n"); - j = 1; - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - ++j; - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - CHECK(present_c == j, "mismatch"); - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - - mdbx_dbi_close(env, dbi); - mdbx_env_close_ex(env, 0); - - return 0; -} diff --git a/test/test2.c b/test/test2.c deleted file mode 100644 index f35bca9b..00000000 --- a/test/test2.c +++ /dev/null @@ -1,153 +0,0 @@ -/* mtest2.c - memory-mapped database tester/toy */ - -/* - * Copyright 2015-2017 Leonid Yuriev . - * Copyright 2011-2017 Howard Chu, Symas Corp. - * Copyright 2015,2016 Peter-Service R&D LLC. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -/* Just like mtest.c, but using a subDB instead of the main DB */ - -#include "../mdbx.h" -#include -#include -#include -#include -#include -#include - -#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) -#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) -#define CHECK(test, msg) \ - ((test) ? (void)0 : ((void)fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, \ - __LINE__, msg, mdbx_strerror(rc)), \ - abort())) - -#ifndef DBPATH -#define DBPATH "./tmp.db" -#endif - -int main(int argc, char *argv[]) { - int i = 0, j = 0, rc; - MDB_env *env; - MDB_dbi dbi; - MDB_val key, data; - MDB_txn *txn; - MDBX_stat mst; - MDB_cursor *cursor; - int count; - int *values; - char sval[32] = ""; - int env_oflags; - struct stat db_stat, exe_stat; - - (void)argc; - (void)argv; - srand(time(NULL)); - - count = (rand() % 384) + 64; - values = (int *)malloc(count * sizeof(int)); - - for (i = 0; i < count; i++) { - values[i] = rand() % 1024; - } - - E(mdbx_env_create(&env)); - E(mdbx_env_set_maxreaders(env, 1)); - E(mdbx_env_set_mapsize(env, 10485760)); - E(mdbx_env_set_maxdbs(env, 4)); - - E(stat("/proc/self/exe", &exe_stat) ? errno : 0); - E(stat(DBPATH "/.", &db_stat) ? errno : 0); - env_oflags = MDB_NOSYNC; - if (major(db_stat.st_dev) != major(exe_stat.st_dev)) { - /* LY: Assume running inside a CI-environment: - * 1) don't use FIXEDMAP to avoid EBUSY in case collision, - * which could be inspired by address space randomisation feature. - * 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some - * dedicated storage. - */ - env_oflags = 0; - } - E(mdbx_env_open(env, DBPATH, env_oflags, 0664)); - - E(mdbx_txn_begin(env, NULL, 0, &txn)); - if (mdbx_dbi_open(txn, "id2", MDB_CREATE, &dbi) == MDB_SUCCESS) - E(mdbx_drop(txn, dbi, 1)); - E(mdbx_dbi_open(txn, "id2", MDB_CREATE, &dbi)); - - key.mv_size = sizeof(int); - key.mv_data = sval; - - printf("Adding %d values\n", count); - for (i = 0; i < count; i++) { - sprintf(sval, "%03x %d foo bar", values[i], values[i]); - data.mv_size = sizeof(sval); - data.mv_data = sval; - if (RES(MDB_KEYEXIST, mdbx_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) - j++; - } - if (j) - printf("%d duplicates skipped\n", j); - E(mdbx_txn_commit(txn)); - E(mdbx_env_stat(env, &mst, sizeof(mst))); - - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int)key.mv_size, - (char *)key.mv_data, data.mv_data, (int)data.mv_size, - (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - - j = 0; - key.mv_data = sval; - for (i = count - 1; i > -1; i -= (rand() % 5)) { - j++; - txn = NULL; - E(mdbx_txn_begin(env, NULL, 0, &txn)); - sprintf(sval, "%03x ", values[i]); - if (RES(MDB_NOTFOUND, mdbx_del(txn, dbi, &key, NULL))) { - j--; - mdbx_txn_abort(txn); - } else { - E(mdbx_txn_commit(txn)); - } - } - free(values); - printf("Deleted %d values\n", j); - - E(mdbx_env_stat(env, &mst, sizeof(mst))); - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - printf("Cursor next\n"); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - printf("Cursor prev\n"); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - - mdbx_dbi_close(env, dbi); - mdbx_env_close(env); - return 0; -} diff --git a/test/test3.c b/test/test3.c deleted file mode 100644 index 2dac03d0..00000000 --- a/test/test3.c +++ /dev/null @@ -1,162 +0,0 @@ -/* mtest3.c - memory-mapped database tester/toy */ - -/* - * Copyright 2015-2017 Leonid Yuriev . - * Copyright 2011-2017 Howard Chu, Symas Corp. - * Copyright 2015,2016 Peter-Service R&D LLC. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -/* Tests for sorted duplicate DBs */ -#include "../mdbx.h" -#include -#include -#include -#include -#include -#include -#include - -#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) -#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) -#define CHECK(test, msg) \ - ((test) ? (void)0 : ((void)fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, \ - __LINE__, msg, mdbx_strerror(rc)), \ - abort())) - -#ifndef DBPATH -#define DBPATH "./tmp.db" -#endif - -int main(int argc, char *argv[]) { - int i = 0, j = 0, rc; - MDB_env *env; - MDB_dbi dbi; - MDB_val key, data; - MDB_txn *txn; - MDBX_stat mst; - MDB_cursor *cursor; - int count; - int *values; - char sval[32]; - char kval[sizeof(int)]; - int env_oflags; - struct stat db_stat, exe_stat; - - (void)argc; - (void)argv; - srand(time(NULL)); - - memset(sval, 0, sizeof(sval)); - - count = (rand() % 384) + 64; - values = (int *)malloc(count * sizeof(int)); - - for (i = 0; i < count; i++) { - values[i] = rand() % 1024; - } - - E(mdbx_env_create(&env)); - E(mdbx_env_set_mapsize(env, 10485760)); - E(mdbx_env_set_maxdbs(env, 4)); - - E(stat("/proc/self/exe", &exe_stat) ? errno : 0); - E(stat(DBPATH "/.", &db_stat) ? errno : 0); - env_oflags = MDB_NOSYNC; - if (major(db_stat.st_dev) != major(exe_stat.st_dev)) { - /* LY: Assume running inside a CI-environment: - * 1) don't use FIXEDMAP to avoid EBUSY in case collision, - * which could be inspired by address space randomisation feature. - * 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some - * dedicated storage. - */ - env_oflags = 0; - } - E(mdbx_env_open(env, DBPATH, env_oflags, 0664)); - - E(mdbx_txn_begin(env, NULL, 0, &txn)); - if (mdbx_dbi_open(txn, "id3", MDB_CREATE, &dbi) == MDB_SUCCESS) - E(mdbx_drop(txn, dbi, 1)); - E(mdbx_dbi_open(txn, "id3", MDB_CREATE | MDB_DUPSORT, &dbi)); - - key.mv_size = sizeof(int); - key.mv_data = kval; - data.mv_size = sizeof(sval); - data.mv_data = sval; - - printf("Adding %d values\n", count); - for (i = 0; i < count; i++) { - if (!(i & 0x0f)) - sprintf(kval, "%03x", values[i]); - sprintf(sval, "%03x %d foo bar", values[i], values[i]); - if (RES(MDB_KEYEXIST, mdbx_put(txn, dbi, &key, &data, MDB_NODUPDATA))) - j++; - } - if (j) - printf("%d duplicates skipped\n", j); - E(mdbx_txn_commit(txn)); - E(mdbx_env_stat(env, &mst, sizeof(mst))); - - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int)key.mv_size, - (char *)key.mv_data, data.mv_data, (int)data.mv_size, - (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - - j = 0; - - for (i = count - 1; i > -1; i -= (rand() % 5)) { - j++; - txn = NULL; - E(mdbx_txn_begin(env, NULL, 0, &txn)); - sprintf(kval, "%03x", values[i & ~0x0f]); - sprintf(sval, "%03x %d foo bar", values[i], values[i]); - key.mv_size = sizeof(int); - key.mv_data = kval; - data.mv_size = sizeof(sval); - data.mv_data = sval; - if (RES(MDB_NOTFOUND, mdbx_del(txn, dbi, &key, &data))) { - j--; - mdbx_txn_abort(txn); - } else { - E(mdbx_txn_commit(txn)); - } - } - free(values); - printf("Deleted %d values\n", j); - - E(mdbx_env_stat(env, &mst, sizeof(mst))); - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - printf("Cursor next\n"); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - printf("Cursor prev\n"); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - - mdbx_dbi_close(env, dbi); - mdbx_env_close(env); - return 0; -} diff --git a/test/test4.c b/test/test4.c deleted file mode 100644 index aedec134..00000000 --- a/test/test4.c +++ /dev/null @@ -1,196 +0,0 @@ -/* mtest4.c - memory-mapped database tester/toy */ - -/* - * Copyright 2015-2017 Leonid Yuriev . - * Copyright 2011-2017 Howard Chu, Symas Corp. - * Copyright 2015,2016 Peter-Service R&D LLC. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -/* Tests for sorted duplicate DBs with fixed-size keys */ -#include "../mdbx.h" -#include -#include -#include -#include -#include -#include -#include - -#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) -#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) -#define CHECK(test, msg) \ - ((test) ? (void)0 : ((void)fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, \ - __LINE__, msg, mdbx_strerror(rc)), \ - abort())) - -#ifndef DBPATH -#define DBPATH "./tmp.db" -#endif - -int main(int argc, char *argv[]) { - int i = 0, j = 0, rc; - MDB_env *env; - MDB_dbi dbi; - MDB_val key, data; - MDB_txn *txn; - MDBX_stat mst; - MDB_cursor *cursor; - int count; - int *values; - char sval[8]; - char kval[sizeof(int)]; - int env_oflags; - struct stat db_stat, exe_stat; - - (void)argc; - (void)argv; - memset(sval, 0, sizeof(sval)); - - count = 510; - values = (int *)malloc(count * sizeof(int)); - - for (i = 0; i < count; i++) { - values[i] = i * 5; - } - - E(mdbx_env_create(&env)); - E(mdbx_env_set_mapsize(env, 10485760)); - E(mdbx_env_set_maxdbs(env, 4)); - - E(stat("/proc/self/exe", &exe_stat) ? errno : 0); - E(stat(DBPATH "/.", &db_stat) ? errno : 0); - env_oflags = MDB_NOSYNC; - if (major(db_stat.st_dev) != major(exe_stat.st_dev)) { - /* LY: Assume running inside a CI-environment: - * 1) don't use FIXEDMAP to avoid EBUSY in case collision, - * which could be inspired by address space randomisation feature. - * 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some - * dedicated storage. - */ - env_oflags = 0; - } - E(mdbx_env_open(env, DBPATH, env_oflags, 0664)); - - E(mdbx_txn_begin(env, NULL, 0, &txn)); - if (mdbx_dbi_open(txn, "id4", MDB_CREATE, &dbi) == MDB_SUCCESS) - E(mdbx_drop(txn, dbi, 1)); - E(mdbx_dbi_open(txn, "id4", MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, &dbi)); - - key.mv_size = sizeof(int); - key.mv_data = kval; - data.mv_size = sizeof(sval); - data.mv_data = sval; - - printf("Adding %d values\n", count); - strcpy(kval, "001"); - for (i = 0; i < count; i++) { - sprintf(sval, "%07x", values[i]); - if (RES(MDB_KEYEXIST, mdbx_put(txn, dbi, &key, &data, MDB_NODUPDATA))) - j++; - } - if (j) - printf("%d duplicates skipped\n", j); - E(mdbx_txn_commit(txn)); - E(mdbx_env_stat(env, &mst, sizeof(mst))); - - /* there should be one full page of dups now. - */ - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int)key.mv_size, - (char *)key.mv_data, data.mv_data, (int)data.mv_size, - (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - - /* test all 3 branches of split code: - * 1: new key in lower half - * 2: new key at split point - * 3: new key in upper half - */ - - key.mv_size = sizeof(int); - key.mv_data = kval; - data.mv_size = sizeof(sval); - data.mv_data = sval; - - sprintf(sval, "%07x", values[3] + 1); - E(mdbx_txn_begin(env, NULL, 0, &txn)); - (void)RES(MDB_KEYEXIST, mdbx_put(txn, dbi, &key, &data, MDB_NODUPDATA)); - mdbx_txn_abort(txn); - - sprintf(sval, "%07x", values[255] + 1); - E(mdbx_txn_begin(env, NULL, 0, &txn)); - (void)RES(MDB_KEYEXIST, mdbx_put(txn, dbi, &key, &data, MDB_NODUPDATA)); - mdbx_txn_abort(txn); - - sprintf(sval, "%07x", values[500] + 1); - E(mdbx_txn_begin(env, NULL, 0, &txn)); - (void)RES(MDB_KEYEXIST, mdbx_put(txn, dbi, &key, &data, MDB_NODUPDATA)); - E(mdbx_txn_commit(txn)); - - /* Try MDB_NEXT_MULTIPLE */ - E(mdbx_txn_begin(env, NULL, 0, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT_MULTIPLE)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - j = 0; - - for (i = count - 1; i > -1; i -= (rand() % 3)) { - j++; - txn = NULL; - E(mdbx_txn_begin(env, NULL, 0, &txn)); - sprintf(sval, "%07x", values[i]); - key.mv_size = sizeof(int); - key.mv_data = kval; - data.mv_size = sizeof(sval); - data.mv_data = sval; - if (RES(MDB_NOTFOUND, mdbx_del(txn, dbi, &key, &data))) { - j--; - mdbx_txn_abort(txn); - } else { - E(mdbx_txn_commit(txn)); - } - } - free(values); - printf("Deleted %d values\n", j); - - E(mdbx_env_stat(env, &mst, sizeof(mst))); - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - printf("Cursor next\n"); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - printf("Cursor prev\n"); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - - mdbx_dbi_close(env, dbi); - mdbx_env_close(env); - return 0; -} diff --git a/test/test5.c b/test/test5.c deleted file mode 100644 index c1018c64..00000000 --- a/test/test5.c +++ /dev/null @@ -1,164 +0,0 @@ -/* mtest5.c - memory-mapped database tester/toy */ - -/* - * Copyright 2015-2017 Leonid Yuriev . - * Copyright 2011-2017 Howard Chu, Symas Corp. - * Copyright 2015,2016 Peter-Service R&D LLC. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -/* Tests for sorted duplicate DBs using cursor_put */ -#include "../mdbx.h" -#include -#include -#include -#include -#include -#include -#include - -#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) -#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) -#define CHECK(test, msg) \ - ((test) ? (void)0 : ((void)fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, \ - __LINE__, msg, mdbx_strerror(rc)), \ - abort())) - -#ifndef DBPATH -#define DBPATH "./tmp.db" -#endif - -int main(int argc, char *argv[]) { - int i = 0, j = 0, rc; - MDB_env *env; - MDB_dbi dbi; - MDB_val key, data; - MDB_txn *txn; - MDBX_stat mst; - MDB_cursor *cursor; - int count; - int *values; - char sval[32]; - char kval[sizeof(int)]; - int env_oflags; - struct stat db_stat, exe_stat; - - (void)argc; - (void)argv; - srand(time(NULL)); - - memset(sval, 0, sizeof(sval)); - - count = (rand() % 384) + 64; - values = (int *)malloc(count * sizeof(int)); - - for (i = 0; i < count; i++) { - values[i] = rand() % 1024; - } - - E(mdbx_env_create(&env)); - E(mdbx_env_set_mapsize(env, 10485760)); - E(mdbx_env_set_maxdbs(env, 4)); - - E(stat("/proc/self/exe", &exe_stat) ? errno : 0); - E(stat(DBPATH "/.", &db_stat) ? errno : 0); - env_oflags = MDB_NOSYNC; - if (major(db_stat.st_dev) != major(exe_stat.st_dev)) { - /* LY: Assume running inside a CI-environment: - * 1) don't use FIXEDMAP to avoid EBUSY in case collision, - * which could be inspired by address space randomisation feature. - * 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some - * dedicated storage. - */ - env_oflags = 0; - } - E(mdbx_env_open(env, DBPATH, env_oflags, 0664)); - - E(mdbx_txn_begin(env, NULL, 0, &txn)); - if (mdbx_dbi_open(txn, "id5", MDB_CREATE, &dbi) == MDB_SUCCESS) - E(mdbx_drop(txn, dbi, 1)); - E(mdbx_dbi_open(txn, "id5", MDB_CREATE | MDB_DUPSORT, &dbi)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - - key.mv_size = sizeof(int); - key.mv_data = kval; - data.mv_size = sizeof(sval); - data.mv_data = sval; - - printf("Adding %d values\n", count); - for (i = 0; i < count; i++) { - if (!(i & 0x0f)) - sprintf(kval, "%03x", values[i]); - sprintf(sval, "%03x %d foo bar", values[i], values[i]); - if (RES(MDB_KEYEXIST, mdbx_cursor_put(cursor, &key, &data, MDB_NODUPDATA))) - j++; - } - if (j) - printf("%d duplicates skipped\n", j); - mdbx_cursor_close(cursor); - E(mdbx_txn_commit(txn)); - E(mdbx_env_stat(env, &mst, sizeof(mst))); - - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %p %.*s, data: %p %.*s\n", key.mv_data, (int)key.mv_size, - (char *)key.mv_data, data.mv_data, (int)data.mv_size, - (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - - j = 0; - - for (i = count - 1; i > -1; i -= (rand() % 5)) { - j++; - txn = NULL; - E(mdbx_txn_begin(env, NULL, 0, &txn)); - sprintf(kval, "%03x", values[i & ~0x0f]); - sprintf(sval, "%03x %d foo bar", values[i], values[i]); - key.mv_size = sizeof(int); - key.mv_data = kval; - data.mv_size = sizeof(sval); - data.mv_data = sval; - if (RES(MDB_NOTFOUND, mdbx_del(txn, dbi, &key, &data))) { - j--; - mdbx_txn_abort(txn); - } else { - E(mdbx_txn_commit(txn)); - } - } - free(values); - printf("Deleted %d values\n", j); - - E(mdbx_env_stat(env, &mst, sizeof(mst))); - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - printf("Cursor next\n"); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - printf("Cursor prev\n"); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { - printf("key: %.*s, data: %.*s\n", (int)key.mv_size, (char *)key.mv_data, - (int)data.mv_size, (char *)data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - - mdbx_dbi_close(env, dbi); - mdbx_env_close(env); - return 0; -} diff --git a/test/test6.c b/test/test6.c deleted file mode 100644 index 03b6f7d1..00000000 --- a/test/test6.c +++ /dev/null @@ -1,175 +0,0 @@ -/* mtest6.c - memory-mapped database tester/toy */ - -/* - * Copyright 2015-2017 Leonid Yuriev . - * Copyright 2011-2017 Howard Chu, Symas Corp. - * Copyright 2015,2016 Peter-Service R&D LLC. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -/* Tests for DB splits and merges */ -#include "../mdbx.h" -#include -#include -#include -#include -#include -#include -#include - -#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) -#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) -#define CHECK(test, msg) \ - ((test) ? (void)0 : ((void)fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, \ - __LINE__, msg, mdbx_strerror(rc)), \ - abort())) - -#ifndef DBPATH -#define DBPATH "./tmp.db" -#endif - -char dkbuf[1024]; - -int main(int argc, char *argv[]) { - int i = 0, rc; - MDB_env *env; - MDB_dbi dbi; - MDB_val key, data, sdata; - MDB_txn *txn; - MDBX_stat mst; - MDB_cursor *cursor; - long kval; - char *sval; - int env_oflags; - struct stat db_stat, exe_stat; - - (void)argc; - (void)argv; - srand(time(NULL)); - - E(mdbx_env_create(&env)); - E(mdbx_env_set_mapsize(env, 10485760)); - E(mdbx_env_set_maxdbs(env, 4)); - - E(stat("/proc/self/exe", &exe_stat) ? errno : 0); - E(stat(DBPATH "/.", &db_stat) ? errno : 0); - env_oflags = MDB_NOSYNC; - if (major(db_stat.st_dev) != major(exe_stat.st_dev)) { - /* LY: Assume running inside a CI-environment: - * 1) don't use FIXEDMAP to avoid EBUSY in case collision, - * which could be inspired by address space randomisation feature. - * 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some - * dedicated storage. - */ - env_oflags = 0; - } - E(mdbx_env_open(env, DBPATH, env_oflags, 0664)); - - E(mdbx_txn_begin(env, NULL, 0, &txn)); - if (mdbx_dbi_open(txn, "id6", MDB_CREATE, &dbi) == MDB_SUCCESS) - E(mdbx_drop(txn, dbi, 1)); - E(mdbx_dbi_open(txn, "id6", MDB_CREATE | MDB_INTEGERKEY, &dbi)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - E(mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst))); - - sval = calloc(1, mst.ms_psize / 4); - key.mv_size = sizeof(long); - key.mv_data = &kval; - sdata.mv_size = mst.ms_psize / 4 - 30; - sdata.mv_data = sval; - - printf("Adding 12 values, should yield 3 splits\n"); - for (i = 0; i < 12; i++) { - kval = i * 5; - sprintf(sval, "%08lx", kval); - data = sdata; - (void)RES(MDB_KEYEXIST, - mdbx_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); - } - printf("Adding 12 more values, should yield 3 splits\n"); - for (i = 0; i < 12; i++) { - kval = i * 5 + 4; - sprintf(sval, "%08lx", kval); - data = sdata; - (void)RES(MDB_KEYEXIST, - mdbx_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); - } - printf("Adding 12 more values, should yield 3 splits\n"); - for (i = 0; i < 12; i++) { - kval = i * 5 + 1; - sprintf(sval, "%08lx", kval); - data = sdata; - (void)RES(MDB_KEYEXIST, - mdbx_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); - } - E(mdbx_cursor_get(cursor, &key, &data, MDB_FIRST)); - - do { - printf("key: %p %s, data: %p %.*s\n", key.mv_data, - mdbx_dkey(&key, dkbuf, sizeof(dkbuf)), data.mv_data, - (int)data.mv_size, (char *)data.mv_data); - } while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0); - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - mdbx_cursor_close(cursor); - mdbx_txn_commit(txn); - -#if 0 - int j=0; - int count = 333; - int *values = alloca(sizeof(int) * count); - - for (i= count - 1; i > -1; i-= (rand()%5)) { - j++; - txn=NULL; - E(mdbx_txn_begin(env, NULL, 0, &txn)); - sprintf(kval, "%03x", values[i & ~0x0f]); - sprintf(sval, "%03x %d foo bar", values[i], values[i]); - key.mv_size = sizeof(int); - key.mv_data = kval; - data.mv_size = sizeof(sval); - data.mv_data = sval; - if (RES(MDB_NOTFOUND, mdbx_del(txn, dbi, &key, &data))) { - j--; - mdbx_txn_abort(txn); - } else { - E(mdbx_txn_commit(txn)); - } - } - free(values); - printf("Deleted %d values\n", j); - - E(mdbx_env_stat(env, &mst, sizeof(mst))); - E(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - E(mdbx_cursor_open(txn, dbi, &cursor)); - printf("Cursor next\n"); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - printf("Cursor prev\n"); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { - printf("key: %.*s, data: %.*s\n", - (int) key.mv_size, (char *) key.mv_data, - (int) data.mv_size, (char *) data.mv_data); - } - CHECK(rc == MDB_NOTFOUND, "mdbx_cursor_get"); - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); - - mdbx_dbi_close(env, dbi); -#endif - mdbx_env_close(env); - free(sval); - - return 0; -} diff --git a/test/test7.c b/test/test7.c deleted file mode 100644 index 61110d5c..00000000 --- a/test/test7.c +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright 2017 Klaus Malorny - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -#include -#include -#include -#include -#include - -#include "../mdbx.h" - -static const char *fileName = "/dev/shm/test.mdbx"; -static const char *dbName = "test"; -static long size = 1500000000; -static int recordCount = 33000000; -static int majorIdCount = 6000; -static int minorIdCount = 1000000; -static unsigned int seed = 1; -static long *majorIds; - -typedef struct { - long majorId; - long minorId; -} KeyType; - -typedef struct { long refId; } DataType; - -typedef struct { - KeyType key; - DataType data; -} KeyDataType; - -void check(const char *op, int error) { - if (error != 0) { - fprintf(stderr, "%s: unexpected error %d: %s\n", op, error, - mdbx_strerror(error)); - exit(1); - } -} - -void shuffle(void *data, int recordSize, int recordCount) { - char *ptr = (char *)data; - char *swapBuf = malloc(recordSize); - - for (int i = recordCount - 2; i >= 0; i--) { - int j = (int)(random() % (recordCount - i)); - - if (j > 0) { - char *ptr1 = ptr + i * recordSize; - char *ptr2 = ptr + (i + j) * recordSize; - - memcpy(swapBuf, ptr1, recordSize); - memcpy(ptr1, ptr2, recordSize); - memcpy(ptr2, swapBuf, recordSize); - } - } - - free(swapBuf); -} - -void fill(MDB_env *env, MDB_dbi dbi) { - KeyType key; - DataType data; - - MDB_val keyRef; - MDB_val dataRef; - MDB_txn *txn; - - printf("generating data\n"); - - srandom(seed); - - majorIds = (long *)malloc(majorIdCount * sizeof(long)); - - if (!majorIds) { - fprintf(stderr, "out of memory\n"); - exit(1); - } - - for (int i = 0; i < majorIdCount; i++) - majorIds[i] = i; - - // now shuffle (for later deletion test) - shuffle((void *)majorIds, sizeof(long), majorIdCount); - - KeyDataType *records = malloc(sizeof(KeyDataType) * recordCount); - KeyDataType *ptr = records; - int remaining = recordCount; - long refId = 0; - - for (int i = 0; i < minorIdCount; i++) { - long majorId = random() % majorIdCount; - long minorId = i; - - int max = remaining / (minorIdCount - i + 1); - int use; - - if (i == minorIdCount - 1 || max < 2) { - use = max; - - } else { - long rand1 = random() % max; - long rand2 = random() % max; - use = (int)((rand1 * rand2 / (max - 1))) + 1; // non-linear distribution - } - - // printf ("%d %d %d\n", i, max, use); - - while (use-- > 0) { - ptr->key.majorId = majorId; - ptr->key.minorId = minorId; - ptr->data.refId = ++refId; - ptr++; - remaining--; - } - } - - shuffle((void *)records, sizeof(KeyDataType), recordCount); - - printf("writing data\n"); - - check("txn_begin", mdbx_txn_begin(env, NULL, 0, &txn)); - - ptr = records; - - for (int i = recordCount; i > 0; i--) { - - key.majorId = htobe64(ptr->key.majorId); - key.minorId = htobe64(ptr->key.minorId); - data.refId = htobe64(ptr->data.refId); - - keyRef.mv_size = sizeof(key); - keyRef.mv_data = (void *)&key; - dataRef.mv_size = sizeof(data); - dataRef.mv_data = (void *)&data; - - check("mdbx_put", mdbx_put(txn, dbi, &keyRef, &dataRef, 0)); - - ptr++; - } - - check("txn_commit", mdbx_txn_commit(txn)); - - printf("%d records written\n", recordCount); -} - -void deleteRange(MDB_env *env, MDB_dbi dbi, MDB_txn *txn, KeyType *startKey, - KeyType *endKey, int endIsInclusive) { - MDB_cursor *cursor; - MDB_val curKeyRef; - MDB_val endKeyRef; - MDB_val curDataRef; - (void)env; - - check("cursor_open", mdbx_cursor_open(txn, dbi, &cursor)); - - curKeyRef.mv_size = sizeof(KeyType); - curKeyRef.mv_data = (void *)startKey; - endKeyRef.mv_size = sizeof(KeyType); - endKeyRef.mv_data = (void *)endKey; - curDataRef.mv_size = 0; - curDataRef.mv_data = NULL; - - int error = mdbx_cursor_get(cursor, &curKeyRef, &curDataRef, MDB_SET_RANGE); - - while (error != MDB_NOTFOUND) { - check("mdbx_cursor_get", error); - - int compResult = mdbx_cmp(txn, dbi, &curKeyRef, &endKeyRef); - - if (compResult > 0 || (!compResult && !endIsInclusive)) - break; - - check("mdbx_cursor_del", mdbx_cursor_del(cursor, MDB_NODUPDATA)); - - error = mdbx_cursor_get(cursor, &curKeyRef, &curDataRef, MDB_NEXT); - } - - mdbx_cursor_close(cursor); -} - -void testDelete(MDB_env *env, MDB_dbi dbi) { - MDB_txn *txn; - KeyType startKey; - KeyType endKey; - - printf("testing\n"); - - check("txn_begin", mdbx_txn_begin(env, NULL, 0, &txn)); - - long majorId; - - for (int i = 0; i < majorIdCount; i++) { - majorId = majorIds[i]; - startKey.majorId = htobe64(majorId); - startKey.minorId = htobe64(1); - endKey.majorId = htobe64(majorId); - endKey.minorId = htobe64((long)(~0UL >> 1)); - - deleteRange(env, dbi, txn, &startKey, &endKey, 1); - } - - check("txn_commit", mdbx_txn_commit(txn)); -} - -int main(int argc, char *argv[]) { - MDB_env *env; - MDB_dbi dbi; - MDB_txn *txn; - (void)argc; - (void)argv; - - printf("LMDB version: %s\n", MDBX_VERSION_STRING); - - unlink(fileName); - check("env_create", mdbx_env_create(&env)); - check("env_set_mapsize", mdbx_env_set_mapsize(env, size)); - check("env_set_maxdbs", mdbx_env_set_maxdbs(env, 2)); - - check("env_open", - mdbx_env_open(env, fileName, MDB_NOSUBDIR | MDB_WRITEMAP, 0666)); - - check("txn_begin", mdbx_txn_begin(env, NULL, 0, &txn)); - - check("dbi_open", mdbx_dbi_open(txn, dbName, MDB_CREATE | MDB_DUPSORT, &dbi)); - - check("txn_commit", mdbx_txn_commit(txn)); - - fill(env, dbi); - testDelete(env, dbi); - - mdbx_env_close(env); - - printf("done.\n"); -} diff --git a/test/test_bench.c b/test/test_bench.c deleted file mode 100644 index 377fbf70..00000000 --- a/test/test_bench.c +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright 2015-2017 Leonid Yuriev . - * Copyright 2015,2016 Peter-Service R&D LLC. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../mdbx.h" - -#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) -#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) -#define CHECK(test, msg) \ - ((test) ? (void)0 : ((void)fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, \ - __LINE__, msg, mdbx_strerror(rc)), \ - abort())) - -#ifndef DBPATH -#define DBPATH "./tmp.db" -#endif - -struct t0 { - struct rusage ru; - struct timespec ts; -}; - -void t0(struct t0 *t0) { - int rc; - E(getrusage(RUSAGE_SELF, &t0->ru)); - E(clock_gettime(CLOCK_MONOTONIC_RAW, &t0->ts)); -} - -struct info { - double wall_s, cpu_sys_s, cpu_user_s; - long iops_r, iops_w, iops_pf; -}; - -double delta_s(const struct timeval *begin, const struct timeval *end) { - return end->tv_sec - begin->tv_sec + - (end->tv_usec - begin->tv_usec) / 1000000.0; -} - -double delta2_s(const struct timespec *begin, const struct timespec *end) { - return end->tv_sec - begin->tv_sec + - (end->tv_nsec - begin->tv_nsec) / 1000000000.0; -} - -void measure(const struct t0 *t0, struct info *i) { - struct t0 t1; - int rc; - - E(clock_gettime(CLOCK_MONOTONIC_RAW, &t1.ts)); - E(getrusage(RUSAGE_SELF, &t1.ru)); - - i->wall_s = delta2_s(&t0->ts, &t1.ts); - i->cpu_user_s = delta_s(&t0->ru.ru_utime, &t1.ru.ru_utime); - i->cpu_sys_s = delta_s(&t0->ru.ru_stime, &t1.ru.ru_stime); - i->iops_r = t1.ru.ru_inblock - t0->ru.ru_inblock; - i->iops_w = t1.ru.ru_oublock - t0->ru.ru_oublock; - i->iops_pf = - t1.ru.ru_majflt - t0->ru.ru_majflt + t1.ru.ru_minflt - t0->ru.ru_minflt; -} - -void print(struct info *i) { - printf("wall-clock %.3f, iops: %lu reads, %lu writes, %lu page-faults, " - "cpu: %.3f user, %.3f sys\n", - i->wall_s, i->iops_r, i->iops_w, i->iops_pf, i->cpu_user_s, - i->cpu_sys_s); -} - -static void wbench(int flags, int mb, int count, int salt) { - MDB_env *env; - MDB_dbi dbi; - MDB_txn *txn; - MDB_val key, data; - unsigned key_value = salt; - char data_value[777]; - int i, rc; - struct t0 start; - struct info ra, rd, rs, rt; - - mkdir(DBPATH, 0755); - unlink(DBPATH "/data.mdb"); - unlink(DBPATH "/lock.mdb"); - - printf("\nProbing %d Mb, %d items, flags:", mb, count); - if (flags & MDB_NOSYNC) - printf(" NOSYNC"); - if (flags & MDB_NOMETASYNC) - printf(" NOMETASYNC"); - if (flags & MDB_WRITEMAP) - printf(" WRITEMAP"); - if (flags & MDB_MAPASYNC) - printf(" MAPASYNC"); -#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM) - if (flags & MDBX_COALESCE) - printf(" COALESCE"); - if (flags & MDBX_LIFORECLAIM) - printf(" LIFO"); -#endif - printf(" 0x%X\n", flags); - - E(mdbx_env_create(&env)); - E(mdbx_env_set_mapsize(env, (1ull << 20) * mb)); - E(mdbx_env_open(env, DBPATH, flags, 0664)); - - key.mv_size = sizeof(key_value); - key.mv_data = &key_value; - data.mv_size = sizeof(data_value); - data.mv_data = &data_value; - - printf("\tAdding %d values...", count); - fflush(stdout); - key_value = salt; - t0(&start); - for (i = 0; i < count; ++i) { - E(mdbx_txn_begin(env, NULL, 0, &txn)); - E(mdbx_dbi_open(txn, NULL, 0, &dbi)); - - snprintf(data_value, sizeof(data_value), "value=%u", key_value); - E(mdbx_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)); - E(mdbx_txn_commit(txn)); - - key_value = key_value * 1664525 + 1013904223; - } - measure(&start, &ra); - print(&ra); - - printf("\tDeleting %d values...", count); - fflush(stdout); - key_value = salt; - t0(&start); - for (i = 0; i < count; ++i) { - E(mdbx_txn_begin(env, NULL, 0, &txn)); - E(mdbx_dbi_open(txn, NULL, 0, &dbi)); - - E(mdbx_del(txn, dbi, &key, NULL)); - E(mdbx_txn_commit(txn)); - - key_value = key_value * 1664525 + 1013904223; - } - measure(&start, &rd); - print(&rd); - - printf("\tCheckpoint..."); - fflush(stdout); - t0(&start); - mdbx_env_sync(env, 1); - measure(&start, &rs); - print(&rs); - - mdbx_env_close(env); - rt.wall_s = ra.wall_s + rd.wall_s + rs.wall_s; - rt.cpu_sys_s = ra.cpu_sys_s + rd.cpu_sys_s + rs.cpu_sys_s; - rt.cpu_user_s = ra.cpu_user_s + rd.cpu_user_s + rs.cpu_user_s; - rt.iops_r = ra.iops_r + rd.iops_r + rs.iops_r; - rt.iops_w = ra.iops_w + rd.iops_w + rs.iops_w; - rt.iops_pf = ra.iops_pf + rd.iops_pf + rs.iops_pf; - printf("Total "); - print(&rt); - - fprintf(stderr, "flags: "); - if (flags & MDB_NOSYNC) - fprintf(stderr, " NOSYNC"); - if (flags & MDB_NOMETASYNC) - fprintf(stderr, " NOMETASYNC"); - if (flags & MDB_WRITEMAP) - fprintf(stderr, " WRITEMAP"); - if (flags & MDB_MAPASYNC) - fprintf(stderr, " MAPASYNC"); -#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM) - if (flags & MDBX_COALESCE) - fprintf(stderr, " COALESCE"); - if (flags & MDBX_LIFORECLAIM) - fprintf(stderr, " LIFO"); -#endif - fprintf(stderr, "\t%.3f\t%.3f\t%.3f\t%.3f\n", rt.iops_w / 1000.0, - rt.cpu_user_s, rt.cpu_sys_s, rt.wall_s); -} - -int main(int argc, char *argv[]) { - (void)argc; - (void)argv; - -#define SALT 1 -#define COUNT 10000 -#define SIZE 12 - - printf("\nDefault 'sync' mode..."); - wbench(0, SIZE, COUNT, SALT); -#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM) - // wbench(MDBX_COALESCE, SIZE, COUNT, SALT); - wbench(MDBX_COALESCE | MDBX_LIFORECLAIM, SIZE, COUNT, SALT); -// wbench(MDBX_LIFORECLAIM, SIZE, COUNT, SALT); -#endif - - printf("\nno-meta-sync hack..."); - wbench(MDB_NOMETASYNC, SIZE, COUNT, SALT); -#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM) - // wbench(MDB_NOMETASYNC | MDBX_COALESCE, SIZE, COUNT, SALT); - wbench(MDB_NOMETASYNC | MDBX_COALESCE | MDBX_LIFORECLAIM, SIZE, COUNT, SALT); -// wbench(MDB_NOMETASYNC | MDBX_LIFORECLAIM, SIZE, COUNT, SALT); -#endif - - printf("\nno-sync..."); - wbench(MDB_NOSYNC, SIZE, COUNT, SALT); -#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM) -// wbench(MDB_NOSYNC | MDBX_COALESCE, SIZE, COUNT, SALT); -// wbench(MDB_NOSYNC | MDBX_COALESCE | MDBX_LIFORECLAIM, SIZE, COUNT, -// SALT); -// wbench(MDB_NOSYNC | MDBX_LIFORECLAIM, SIZE, COUNT, SALT); -#endif - - printf("\nr/w-map..."); - wbench(MDB_WRITEMAP, SIZE, COUNT, SALT); -#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM) - // wbench(MDB_WRITEMAP | MDBX_COALESCE, SIZE, COUNT, SALT); - wbench(MDB_WRITEMAP | MDBX_COALESCE | MDBX_LIFORECLAIM, SIZE, COUNT, SALT); -// wbench(MDB_WRITEMAP | MDBX_LIFORECLAIM, SIZE, COUNT, SALT); -#endif - - printf("\nasync..."); - wbench(MDB_WRITEMAP | MDB_MAPASYNC, SIZE, COUNT, SALT); -#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM) - // wbench(MDB_WRITEMAP | MDB_MAPASYNC | MDBX_COALESCE, SIZE, COUNT, - // SALT); - wbench(MDB_WRITEMAP | MDB_MAPASYNC | MDBX_COALESCE | MDBX_LIFORECLAIM, SIZE, - COUNT, SALT); -// wbench(MDB_WRITEMAP | MDB_MAPASYNC | MDBX_LIFORECLAIM, SIZE, COUNT, -// SALT); -#endif - - printf("\nr/w-map + no-sync..."); - wbench(MDB_NOSYNC | MDB_WRITEMAP, SIZE, COUNT, SALT); -#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM) - // wbench(MDB_NOSYNC | MDB_WRITEMAP | MDBX_COALESCE, SIZE, COUNT, SALT); - wbench(MDB_NOSYNC | MDB_WRITEMAP | MDBX_COALESCE | MDBX_LIFORECLAIM, SIZE, - COUNT, SALT); -// wbench(MDB_NOSYNC | MDB_WRITEMAP | MDBX_LIFORECLAIM, SIZE, COUNT, -// SALT); -#endif - - return 0; -} diff --git a/test/test_yota1.c b/test/test_yota1.c deleted file mode 100644 index 7d036f0f..00000000 --- a/test/test_yota1.c +++ /dev/null @@ -1,277 +0,0 @@ -/* - * Copyright 2016-2017 Leonid Yuriev . - * Copyright 2015 Vladimir Romanov - * , Yota Lab. - * - * This file is part of libmdbx. - * - * libmdbx is free software; you can redistribute it and/or modify it under - * the terms of the GNU Affero General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * libmdbx is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include - -#include "../mdbx.h" -#include -#include -#include -#include -#include -#include -#include - -#define IP_PRINTF_ARG_HOST(addr) \ - (int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), \ - (int)((addr)&0xff) - -char opt_db_path[PATH_MAX] = "/dev/shm/x_bench1"; -static MDB_env *env; -#define REC_COUNT 1000000 -int64_t ids[REC_COUNT + REC_COUNT / 10]; -int32_t ids_count = 0; - -int64_t x_add = 0; -int64_t x_del = 0; -int64_t obj_id = 0; - -static void add_id_to_pool(int64_t id) { - ids[ids_count] = id; - ids_count++; -} - -static inline int64_t getTimeMicroseconds(void) { - struct timeval val; - gettimeofday(&val, NULL); - return val.tv_sec * ((int64_t)1000000) + val.tv_usec; -} - -static int64_t get_id_from_pool() { - if (ids_count == 0) { - return -1; - } - int32_t index = rand() % ids_count; - int64_t id = ids[index]; - ids[index] = ids[ids_count - 1]; - ids_count--; - return id; -} - -#define LMDB_CHECK(x) \ - do { \ - const int rc = (x); \ - if (rc != MDB_SUCCESS) { \ - printf("Error [%d] %s in %s at %s:%d\n", rc, mdbx_strerror(rc), #x, \ - __FILE__, __LINE__); \ - exit(EXIT_FAILURE); \ - } \ - } while (0) - -static void db_connect() { - LMDB_CHECK(mdbx_env_create(&env)); - LMDB_CHECK(mdbx_env_set_mapsize(env, 3L * 1024L * 1024L * 1024L)); - LMDB_CHECK(mdbx_env_set_maxdbs(env, 30)); -#if defined(MDBX_LIFORECLAIM) - LMDB_CHECK(mdbx_env_open( - env, opt_db_path, - MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP | MDBX_LIFORECLAIM, 0664)); -#else - LMDB_CHECK(mdbx_env_open(env, opt_db_path, - MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP, 0664)); -#endif - printf("Connection open\n"); -} - -typedef struct { - char session_id1[100]; - char session_id2[100]; - char ip[20]; - uint8_t fill[100]; -} session_data_t; - -typedef struct { - int64_t obj_id; - int8_t event_type; -} __attribute__((__packed__)) event_data_t; - -static void create_record(int64_t record_id) { - MDB_dbi dbi_session; - MDB_dbi dbi_session_id; - MDB_dbi dbi_event; - MDB_dbi dbi_ip; - event_data_t event; - MDB_txn *txn; - session_data_t data; - // transaction init - snprintf(data.session_id1, sizeof(data.session_id1), - "mskugw%02ld_%02ld.gx.yota.ru;3800464060;4152;%ld", - record_id % 3 + 1, record_id % 9 + 1, record_id); - snprintf(data.session_id2, sizeof(data.session_id2), - "gx_service;%ld;%ld;node@spb-jsm1", record_id, - record_id % 1000000000 + 99999); - snprintf(data.ip, sizeof(data.ip), "%d.%d.%d.%d", - IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF)); - event.obj_id = record_id; - event.event_type = 1; - - MDB_val _session_id1_rec = {data.session_id1, strlen(data.session_id1)}; - MDB_val _session_id2_rec = {data.session_id2, strlen(data.session_id2)}; - MDB_val _ip_rec = {data.ip, strlen(data.ip)}; - MDB_val _obj_id_rec = {&record_id, sizeof(record_id)}; - MDB_val _data_rec = {&data, offsetof(session_data_t, fill) + - (rand() % sizeof(data.fill))}; - MDB_val _event_rec = {&event, sizeof(event)}; - - LMDB_CHECK(mdbx_txn_begin(env, NULL, 0, &txn)); - LMDB_CHECK(mdbx_dbi_open(txn, "session", MDB_CREATE, &dbi_session)); - LMDB_CHECK(mdbx_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id)); - LMDB_CHECK(mdbx_dbi_open(txn, "event", MDB_CREATE, &dbi_event)); - LMDB_CHECK(mdbx_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip)); - LMDB_CHECK(mdbx_put(txn, dbi_session, &_obj_id_rec, &_data_rec, - MDB_NOOVERWRITE | MDB_NODUPDATA)); - LMDB_CHECK(mdbx_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec, - MDB_NOOVERWRITE | MDB_NODUPDATA)); - LMDB_CHECK(mdbx_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec, - MDB_NOOVERWRITE | MDB_NODUPDATA)); - LMDB_CHECK(mdbx_put(txn, dbi_ip, &_ip_rec, &_obj_id_rec, 0)); - LMDB_CHECK(mdbx_put(txn, dbi_event, &_event_rec, &_obj_id_rec, 0)); - - // transaction commit - LMDB_CHECK(mdbx_txn_commit(txn)); - x_add++; -} - -static void delete_record(int64_t record_id) { - MDB_dbi dbi_session; - MDB_dbi dbi_session_id; - MDB_dbi dbi_event; - MDB_dbi dbi_ip; - event_data_t event; - MDB_txn *txn; - - // transaction init - LMDB_CHECK(mdbx_txn_begin(env, NULL, 0, &txn)); - // open database in read-write mode - LMDB_CHECK(mdbx_dbi_open(txn, "session", MDB_CREATE, &dbi_session)); - LMDB_CHECK(mdbx_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id)); - LMDB_CHECK(mdbx_dbi_open(txn, "event", MDB_CREATE, &dbi_event)); - LMDB_CHECK(mdbx_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip)); - // put data - MDB_val _obj_id_rec = {&record_id, sizeof(record_id)}; - MDB_val v_rec; - // get data - LMDB_CHECK(mdbx_get(txn, dbi_session, &_obj_id_rec, &v_rec)); - session_data_t *data = (session_data_t *)v_rec.mv_data; - - MDB_val _session_id1_rec = {data->session_id1, strlen(data->session_id1)}; - MDB_val _session_id2_rec = {data->session_id2, strlen(data->session_id2)}; - MDB_val _ip_rec = {data->ip, strlen(data->ip)}; - LMDB_CHECK(mdbx_del(txn, dbi_session_id, &_session_id1_rec, NULL)); - LMDB_CHECK(mdbx_del(txn, dbi_session_id, &_session_id2_rec, NULL)); - LMDB_CHECK(mdbx_del(txn, dbi_ip, &_ip_rec, NULL)); - event.obj_id = record_id; - event.event_type = 1; - MDB_val _event_rec = {&event, sizeof(event)}; - LMDB_CHECK(mdbx_del(txn, dbi_event, &_event_rec, NULL)); - LMDB_CHECK(mdbx_del(txn, dbi_session, &_obj_id_rec, NULL)); - - // transaction commit - LMDB_CHECK(mdbx_txn_commit(txn)); - x_del++; -} - -static void db_disconnect() { - mdbx_env_close(env); - printf("Connection closed\n"); -} - -static void get_db_stat(const char *db, int64_t *ms_branch_pages, - int64_t *ms_leaf_pages) { - MDB_txn *txn; - MDBX_stat stat; - MDB_dbi dbi; - - LMDB_CHECK(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - LMDB_CHECK(mdbx_dbi_open(txn, db, MDB_CREATE, &dbi)); - LMDB_CHECK(mdbx_dbi_stat(txn, dbi, &stat, sizeof(stat))); - mdbx_txn_abort(txn); - printf("%15s | %15ld | %5u | %10ld | %10ld | %11ld |\n", db, - stat.ms_branch_pages, stat.ms_depth, stat.ms_entries, - stat.ms_leaf_pages, stat.ms_overflow_pages); - (*ms_branch_pages) += stat.ms_branch_pages; - (*ms_leaf_pages) += stat.ms_leaf_pages; -} - -static void periodic_stat(void) { - int64_t ms_branch_pages = 0; - int64_t ms_leaf_pages = 0; - printf(" Name | ms_branch_pages | depth | entries | " - "leaf_pages | overf_pages |\n"); - get_db_stat("session", &ms_branch_pages, &ms_leaf_pages); - get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages); - get_db_stat("event", &ms_branch_pages, &ms_leaf_pages); - get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages); - printf("%15s | %15ld | %5s | %10s | %10ld | %11s |\n", "", ms_branch_pages, - "", "", ms_leaf_pages, ""); - static int64_t prev_add; - static int64_t prev_del; - static int64_t t = -1; - if (t > 0) { - int64_t delta = getTimeMicroseconds() - t; - printf("CPS: add %ld, delete %ld, items processed - %ld\n", - (x_add - prev_add) * 1000000 / delta, - (x_del - prev_del) * 1000000 / delta, obj_id); - } - t = getTimeMicroseconds(); - prev_add = x_add; - prev_del = x_del; -} - -static void periodic_add_rec() { - int i; - for (i = 0; i < 10000; i++) { - if (ids_count <= REC_COUNT) { - int64_t id = obj_id++; - create_record(id); - add_id_to_pool(id); - } - if (ids_count > REC_COUNT) { - int64_t id = get_id_from_pool(); - delete_record(id); - } - } - periodic_stat(); -} - -int main(int argc, char **argv) { - (void)argc; - (void)argv; - - char filename[PATH_MAX]; - mkdir(opt_db_path, 0775); - - strcpy(filename, opt_db_path); - strcat(filename, "/data.mdb"); - remove(filename); - - strcpy(filename, opt_db_path); - strcat(filename, "/lock.mdb"); - remove(filename); - - db_connect(); - while (1) { - periodic_add_rec(); - } - db_disconnect(); - return 0; -} diff --git a/test/test_yota2.c b/test/test_yota2.c deleted file mode 100644 index 79c72880..00000000 --- a/test/test_yota2.c +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright 2016-2017 Leonid Yuriev . - * Copyright 2015 Vladimir Romanov - * , Yota Lab. - * - * This file is part of libmdbx. - * - * libmdbx is free software; you can redistribute it and/or modify it under - * the terms of the GNU Affero General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * libmdbx is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include - -#include "../mdbx.h" -#include -#include -#include -#include -#include -#include -#include - -#define IP_PRINTF_ARG_HOST(addr) \ - (int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), \ - (int)((addr)&0xff) - -char opt_db_path[PATH_MAX] = "/dev/shm/x_bench2"; -static MDB_env *env; -#define REC_COUNT 1024000 -int64_t ids[REC_COUNT * 10]; -int32_t ids_count = 0; - -int64_t x_add = 0; -int64_t x_del = 0; -int64_t obj_id = 0; -int64_t x_data_size = 0; -int64_t x_key_size = 0; - -static void add_id_to_pool(int64_t id) { - ids[ids_count] = id; - ids_count++; -} - -static inline int64_t getTimeMicroseconds(void) { - struct timeval val; - gettimeofday(&val, NULL); - return val.tv_sec * ((int64_t)1000000) + val.tv_usec; -} - -static int64_t get_id_from_pool() { - if (ids_count == 0) { - return -1; - } - int32_t index = rand() % ids_count; - int64_t id = ids[index]; - ids[index] = ids[ids_count - 1]; - ids_count--; - return id; -} - -#define LMDB_CHECK(x) \ - do { \ - const int rc = (x); \ - if (rc != MDB_SUCCESS) { \ - printf("Error [%d] %s in %s at %s:%d\n", rc, mdbx_strerror(rc), #x, \ - __FILE__, __LINE__); \ - exit(EXIT_FAILURE); \ - } \ - } while (0) - -static void db_connect() { - MDB_dbi dbi_session; - MDB_dbi dbi_session_id; - MDB_dbi dbi_event; - MDB_dbi dbi_ip; - - LMDB_CHECK(mdbx_env_create(&env)); - LMDB_CHECK(mdbx_env_set_mapsize(env, 300000L * 4096L)); - LMDB_CHECK(mdbx_env_set_maxdbs(env, 30)); -#if defined(MDBX_LIFORECLAIM) - LMDB_CHECK(mdbx_env_open( - env, opt_db_path, - MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP | MDBX_LIFORECLAIM, 0664)); -#else - LMDB_CHECK(mdbx_env_open(env, opt_db_path, - MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP, 0664)); -#endif - MDB_txn *txn; - - // transaction init - LMDB_CHECK(mdbx_txn_begin(env, NULL, 0, &txn)); - // open database in read-write mode - LMDB_CHECK(mdbx_dbi_open(txn, "session", MDB_CREATE, &dbi_session)); - LMDB_CHECK(mdbx_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id)); - LMDB_CHECK(mdbx_dbi_open(txn, "event", MDB_CREATE, &dbi_event)); - LMDB_CHECK(mdbx_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip)); - // transaction commit - LMDB_CHECK(mdbx_txn_commit(txn)); - printf("Connection open\n"); -} - -typedef struct { - char session_id1[100]; - char session_id2[100]; - char ip[20]; - uint8_t fill[100]; -} session_data_t; - -typedef struct { - int64_t obj_id; - int8_t event_type; -} __attribute__((__packed__)) event_data_t; - -static void create_record(int64_t record_id) { - MDB_dbi dbi_session; - MDB_dbi dbi_session_id; - MDB_dbi dbi_event; - MDB_dbi dbi_ip; - event_data_t event; - MDB_txn *txn; - session_data_t data; - // transaction init - snprintf(data.session_id1, sizeof(data.session_id1), - "mskugw%02ld_%02ld.gx.yota.ru;3800464060;4152;%ld", - record_id % 3 + 1, record_id % 9 + 1, record_id); - snprintf(data.session_id2, sizeof(data.session_id2), - "gx_service;%ld;%ld;node@spb-jsm1", record_id, - record_id % 1000000000 + 99999); - snprintf(data.ip, sizeof(data.ip), "%d.%d.%d.%d", - IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF)); - event.obj_id = record_id; - event.event_type = 1; - - MDB_val _session_id1_rec = {data.session_id1, strlen(data.session_id1)}; - MDB_val _session_id2_rec = {data.session_id2, strlen(data.session_id2)}; - MDB_val _ip_rec = {data.ip, strlen(data.ip)}; - MDB_val _obj_id_rec = {&record_id, sizeof(record_id)}; - MDB_val _data_rec = {&data, offsetof(session_data_t, fill) + - (rand() % sizeof(data.fill))}; - MDB_val _event_rec = {&event, sizeof(event)}; - - LMDB_CHECK(mdbx_txn_begin(env, NULL, 0, &txn)); - LMDB_CHECK(mdbx_dbi_open(txn, "session", MDB_CREATE, &dbi_session)); - LMDB_CHECK(mdbx_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id)); - LMDB_CHECK(mdbx_dbi_open(txn, "event", MDB_CREATE, &dbi_event)); - LMDB_CHECK(mdbx_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip)); - LMDB_CHECK(mdbx_put(txn, dbi_session, &_obj_id_rec, &_data_rec, - MDB_NOOVERWRITE | MDB_NODUPDATA)); - LMDB_CHECK(mdbx_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec, - MDB_NOOVERWRITE | MDB_NODUPDATA)); - LMDB_CHECK(mdbx_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec, - MDB_NOOVERWRITE | MDB_NODUPDATA)); - LMDB_CHECK(mdbx_put(txn, dbi_ip, &_ip_rec, &_obj_id_rec, 0)); - LMDB_CHECK(mdbx_put(txn, dbi_event, &_event_rec, &_obj_id_rec, 0)); - x_data_size += (_data_rec.mv_size + _obj_id_rec.mv_size * 4); - x_key_size += - (_obj_id_rec.mv_size + _session_id1_rec.mv_size + - _session_id2_rec.mv_size + _ip_rec.mv_size + _event_rec.mv_size); - - // transaction commit - LMDB_CHECK(mdbx_txn_commit(txn)); - x_add++; -} - -static void delete_record(int64_t record_id) { - MDB_dbi dbi_session; - MDB_dbi dbi_session_id; - MDB_dbi dbi_event; - MDB_dbi dbi_ip; - event_data_t event; - MDB_txn *txn; - - // transaction init - LMDB_CHECK(mdbx_txn_begin(env, NULL, 0, &txn)); - // open database in read-write mode - LMDB_CHECK(mdbx_dbi_open(txn, "session", MDB_CREATE, &dbi_session)); - LMDB_CHECK(mdbx_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id)); - LMDB_CHECK(mdbx_dbi_open(txn, "event", MDB_CREATE, &dbi_event)); - LMDB_CHECK(mdbx_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip)); - // put data - MDB_val _obj_id_rec = {&record_id, sizeof(record_id)}; - MDB_val _data_rec; - // get data - LMDB_CHECK(mdbx_get(txn, dbi_session, &_obj_id_rec, &_data_rec)); - session_data_t *data = (session_data_t *)_data_rec.mv_data; - - MDB_val _session_id1_rec = {data->session_id1, strlen(data->session_id1)}; - MDB_val _session_id2_rec = {data->session_id2, strlen(data->session_id2)}; - MDB_val _ip_rec = {data->ip, strlen(data->ip)}; - LMDB_CHECK(mdbx_del(txn, dbi_session_id, &_session_id1_rec, NULL)); - LMDB_CHECK(mdbx_del(txn, dbi_session_id, &_session_id2_rec, NULL)); - LMDB_CHECK(mdbx_del(txn, dbi_ip, &_ip_rec, NULL)); - event.obj_id = record_id; - event.event_type = 1; - MDB_val _event_rec = {&event, sizeof(event)}; - LMDB_CHECK(mdbx_del(txn, dbi_event, &_event_rec, NULL)); - LMDB_CHECK(mdbx_del(txn, dbi_session, &_obj_id_rec, NULL)); - - x_data_size -= (_data_rec.mv_size + _obj_id_rec.mv_size * 4); - x_key_size -= - (_obj_id_rec.mv_size + _session_id1_rec.mv_size + - _session_id2_rec.mv_size + _ip_rec.mv_size + _event_rec.mv_size); - - // transaction commit - LMDB_CHECK(mdbx_txn_commit(txn)); - x_del++; -} - -static void db_disconnect() { - mdbx_env_close(env); - printf("Connection closed\n"); -} - -static void get_db_stat(const char *db, int64_t *ms_branch_pages, - int64_t *ms_leaf_pages) { - MDB_txn *txn; - MDBX_stat stat; - MDB_dbi dbi; - - LMDB_CHECK(mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn)); - LMDB_CHECK(mdbx_dbi_open(txn, db, MDB_CREATE, &dbi)); - LMDB_CHECK(mdbx_dbi_stat(txn, dbi, &stat, sizeof(stat))); - mdbx_txn_abort(txn); - printf("%15s | %15ld | %5u | %10ld | %10ld | %11ld |\n", db, - stat.ms_branch_pages, stat.ms_depth, stat.ms_entries, - stat.ms_leaf_pages, stat.ms_overflow_pages); - (*ms_branch_pages) += stat.ms_branch_pages; - (*ms_leaf_pages) += stat.ms_leaf_pages; -} - -static void periodic_stat(void) { - int64_t ms_branch_pages = 0; - int64_t ms_leaf_pages = 0; - printf(" Name | ms_branch_pages | depth | entries | " - "leaf_pages | overf_pages |\n"); - get_db_stat("session", &ms_branch_pages, &ms_leaf_pages); - get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages); - get_db_stat("event", &ms_branch_pages, &ms_leaf_pages); - get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages); - printf("%15s | %15ld | %5s | %10s | %10ld | %11s |\n", "", ms_branch_pages, - "", "", ms_leaf_pages, ""); - static int64_t prev_add; - static int64_t prev_del; - static int64_t t = -1; - if (t > 0) { - int64_t delta = getTimeMicroseconds() - t; - printf("CPS: add %ld, delete %ld, items processed - %ldK data=%ldK " - "key=%ldK\n", - (x_add - prev_add) * 1000000 / delta, - (x_del - prev_del) * 1000000 / delta, obj_id / 1024, - x_data_size / 1024, x_key_size / 1024); - printf("usage data=%ld%%\n", - ((x_data_size + x_key_size) * 100) / - ((ms_leaf_pages + ms_branch_pages) * 4096)); - } - t = getTimeMicroseconds(); - prev_add = x_add; - prev_del = x_del; -} - -// static void periodic_add_rec() { -// for (int i = 0; i < 10240; i++) { -// if (ids_count <= REC_COUNT) { -// int64_t id = obj_id++; -// create_record(id); -// add_id_to_pool(id); -// } -// if (ids_count > REC_COUNT) { -// int64_t id = get_id_from_pool(); -// delete_record(id); -// } -// } -// periodic_stat(); -//} - -int main(int argc, char **argv) { - (void)argc; - (void)argv; - - char filename[PATH_MAX]; - int i; - int64_t t; - - mkdir(opt_db_path, 0775); - - strcpy(filename, opt_db_path); - strcat(filename, "/data.mdb"); - remove(filename); - - strcpy(filename, opt_db_path); - strcat(filename, "/lock.mdb"); - remove(filename); - - db_connect(); - periodic_stat(); - for (i = 0; i < 1024000; i++) { - int64_t id = obj_id++; - create_record(id); - add_id_to_pool(id); - } - periodic_stat(); - t = getTimeMicroseconds(); - while (1) { - int i; - int64_t now; - for (i = 0; i < 100; i++) { - int64_t id = obj_id++; - create_record(id); - add_id_to_pool(id); - id = get_id_from_pool(); - delete_record(id); - } - // int64_t id = obj_id++; - // create_record(id); - // add_id_to_pool(id); - now = getTimeMicroseconds(); - if ((now - t) > 100000) { - periodic_stat(); - t = now; - } - } - db_disconnect(); - return 0; -} diff --git a/test/utils.cc b/test/utils.cc new file mode 100644 index 00000000..c3be0ec0 --- /dev/null +++ b/test/utils.cc @@ -0,0 +1,90 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +std::string format(const char *fmt, ...) { + va_list ap, ones; + va_start(ap, fmt); + va_copy(ones, ap); +#ifdef _MSC_VER + int needed = _vscprintf(fmt, ap); +#else + int needed = vsnprintf(nullptr, 0, fmt, ap); +#endif + assert(needed >= 0); + va_end(ap); + std::string result; + result.reserve((size_t)needed + 1); + result.resize((size_t)needed, '\0'); + int actual = vsnprintf((char *)result.data(), result.capacity(), fmt, ones); + assert(actual == needed); + (void)actual; + va_end(ones); + return result; +} + +std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum) { + std::string result; + if (bytes > 0) { + const uint8_t *data = (const uint8_t *)ptr; + checksum.push(data, bytes); + result.reserve(bytes * 2); + const uint8_t *const end = data + bytes; + do { + char h = *data >> 4; + char l = *data & 15; + result.push_back((l < 10) ? l + '0' : l - 10 + 'a'); + result.push_back((h < 10) ? h + '0' : h - 10 + 'a'); + } while (++data < end); + } + assert(result.size() == bytes * 2); + return result; +} + +bool hex2data(const char *hex_begin, const char *hex_end, void *ptr, + size_t bytes, simple_checksum &checksum) { + if (bytes * 2 != (size_t)(hex_end - hex_begin)) + return false; + + uint8_t *data = (uint8_t *)ptr; + for (const char *hex = hex_begin; hex != hex_end; hex += 2, ++data) { + unsigned l = hex[0], h = hex[1]; + + if (l >= '0' && l <= '9') + l = l - '0'; + else if (l >= 'A' && l <= 'F') + l = l - 'A' + 10; + else if (l >= 'a' && l <= 'f') + l = l - 'a' + 10; + else + return false; + + if (h >= '0' && h <= '9') + h = h - '0'; + else if (h >= 'A' && h <= 'F') + h = h - 'A' + 10; + else if (h >= 'a' && h <= 'f') + h = h - 'a' + 10; + else + return false; + + uint32_t c = l + (h << 4); + checksum.push(c); + *data = c; + } + return true; +} + +//----------------------------------------------------------------------------- diff --git a/test/utils.h b/test/utils.h new file mode 100644 index 00000000..55d8f6fd --- /dev/null +++ b/test/utils.h @@ -0,0 +1,312 @@ +/* + * Copyright 2017 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once +#include "base.h" + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ + !defined(__ORDER_BIG_ENDIAN__) +#ifndef _MSC_VER +#include /* for endianness */ +#endif +#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN +#define __BYTE_ORDER__ __BYTE_ORDER +#else +#define __ORDER_LITTLE_ENDIAN__ 1234 +#define __ORDER_BIG_ENDIAN__ 4321 +#if defined(__LITTLE_ENDIAN__) || defined(_LITTLE_ENDIAN) || \ + defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ + defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(__i386) || defined(__x86_64__) || defined(_M_IX86) || \ + defined(_M_X64) || defined(i386) || defined(_X86_) || defined(__i386__) || \ + defined(_X86_64_) || defined(_M_ARM) || defined(_M_ARM64) || \ + defined(__e2k__) +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ +#elif defined(__BIG_ENDIAN__) || defined(_BIG_ENDIAN) || defined(__ARMEB__) || \ + defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || \ + defined(_MIPSEB) || defined(__MIPSEB) || defined(_M_IA64) +#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ +#else +#error __BYTE_ORDER__ should be defined. +#endif +#endif +#endif + +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \ + __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ +#error Unsupported byte order. +#endif + +#if __GNUC_PREREQ(4, 4) || defined(__clang__) +#if __GNUC_PREREQ(4, 5) || defined(__clang__) +#define unreachable() __builtin_unreachable() +#endif +#define bswap64(v) __builtin_bswap64(v) +#define bswap32(v) __builtin_bswap32(v) +#if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16) +#define bswap16(v) __builtin_bswap16(v) +#endif + +#elif defined(_MSC_VER) + +#if _MSC_FULL_VER < 190024215 +#pragma message( \ + "It is recommended to use Visual Studio 2015 (MSC 19.0) or newer.") +#endif + +#define unreachable() __assume(0) +#define bswap64(v) _byteswap_uint64(v) +#define bswap32(v) _byteswap_ulong(v) +#define bswap16(v) _byteswap_ushort(v) +#define rot64(v, s) _rotr64(v, s) +#define rot32(v, s) _rotr(v, s) + +#if defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) +#pragma intrinsic(_umul128) +#define mul_64x64_128(a, b, ph) _umul128(a, b, ph) +#pragma intrinsic(__umulh) +#define mul_64x64_high(a, b) __umulh(a, b) +#endif + +#if defined(_M_IX86) +#pragma intrinsic(__emulu) +#define mul_32x32_64(a, b) __emulu(a, b) +#elif defined(_M_ARM) +#define mul_32x32_64(a, b) _arm_umull(a, b) +#endif + +#endif /* compiler */ + +#ifndef unreachable +#define unreachable() \ + do { \ + } while (1) +#endif + +#ifndef bswap64 +#ifdef __bswap_64 +#define bswap64(v) __bswap_64(v) +#else +static __inline uint64_t bswap64(uint64_t v) { + return v << 56 | v >> 56 | ((v << 40) & 0x00ff000000000000ull) | + ((v << 24) & 0x0000ff0000000000ull) | + ((v << 8) & 0x000000ff00000000ull) | + ((v >> 8) & 0x00000000ff000000ull) | + ((v >> 24) & 0x0000000000ff0000ull) | + ((v >> 40) & 0x000000000000ff00ull); +} +#endif +#endif /* bswap64 */ + +#ifndef bswap32 +#ifdef __bswap_32 +#define bswap32(v) __bswap_32(v) +#else +static __inline uint32_t bswap32(uint32_t v) { + return v << 24 | v >> 24 | ((v << 8) & 0x00ff0000) | ((v >> 8) & 0x0000ff00); +} +#endif +#endif /* bswap32 */ + +#ifndef bswap16 +#ifdef __bswap_16 +#define bswap16(v) __bswap_16(v) +#else +static __inline uint16_t bswap16(uint16_t v) { return v << 8 | v >> 8; } +#endif +#endif /* bswap16 */ + +#define is_byteorder_le() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +#define is_byteorder_be() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + +#ifndef htole16 +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define htobe16(v) bswap16(v) +#define htole16(v) (v) +#define be16toh(v) bswap16(v) +#define le16toh(v) (v) +#else +#define htobe16(v) (v) +#define htole16(v) bswap16(v) +#define be16toh(v) (v) +#define le16toh(v) bswap16(v) +#endif +#endif /* htole16 */ + +#ifndef htole32 +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define htobe32(v) bswap32(v) +#define htole32(v) (v) +#define be32toh(v) bswap32(v) +#define le32toh(v) (v) +#else +#define htobe32(v) (v) +#define htole32(v) bswap32(v) +#define be32toh(v) (v) +#define le32toh(v) bswap32(v) +#endif +#endif /* htole32 */ + +#ifndef htole64 +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define htobe64(v) bswap64(v) +#define htole64(v) (v) +#define be64toh(v) bswap64(v) +#define le64toh(v) (v) +#else +#define htobe64(v) (v) +#define htole64(v) bswap_64(v) +#define be64toh(v) (v) +#define le64toh(v) bswap_64(v) +#endif +#endif /* htole64 */ + +namespace unaligned { + +template static __inline T load(const void *ptr) { +#if defined(_MSC_VER) && \ + (defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)) + return *(const T __unaligned *)ptr; +#elif UNALIGNED_OK + return *(const T *)ptr; +#else + T local; +#if defined(__GNUC__) || defined(__clang__) + __builtin_memcpy(&local, (const T *)ptr, sizeof(T)); +#else + memcpy(&local, (const T *)ptr, sizeof(T)); +#endif /* __GNUC__ || __clang__ */ + return local; +#endif /* UNALIGNED_OK */ +} + +template static __inline void store(void *ptr, const T &value) { +#if defined(_MSC_VER) && \ + (defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)) + *((T __unaligned *)ptr) = value; +#elif UNALIGNED_OK + *(volatile T *)ptr = value; +#else +#if defined(__GNUC__) || defined(__clang__) + __builtin_memcpy(ptr, &value, sizeof(T)); +#else + memcpy(ptr, &value, sizeof(T)); +#endif /* __GNUC__ || __clang__ */ +#endif /* UNALIGNED_OK */ +} + +} /* namespace unaligned */ + +//----------------------------------------------------------------------------- + +#ifndef rot64 +static __inline uint64_t rot64(uint64_t v, unsigned s) { + return (v >> s) | (v << (64 - s)); +} +#endif /* rot64 */ + +#ifndef mul_32x32_64 +static __inline uint64_t mul_32x32_64(uint32_t a, uint32_t b) { + return a * (uint64_t)b; +} +#endif /* mul_32x32_64 */ + +#ifndef mul_64x64_128 + +static __inline unsigned add_with_carry(uint64_t *sum, uint64_t addend) { + *sum += addend; + return *sum < addend; +} + +static __inline uint64_t mul_64x64_128(uint64_t a, uint64_t b, uint64_t *h) { +#if defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + __uint128_t r = (__uint128_t)a * (__uint128_t)b; + /* modern GCC could nicely optimize this */ + *h = r >> 64; + return r; +#elif defined(mul_64x64_high) + *h = mul_64x64_high(a, b); + return a * b; +#else + /* performs 64x64 to 128 bit multiplication */ + uint64_t ll = mul_32x32_64((uint32_t)a, (uint32_t)b); + uint64_t lh = mul_32x32_64(a >> 32, (uint32_t)b); + uint64_t hl = mul_32x32_64((uint32_t)a, b >> 32); + *h = mul_32x32_64(a >> 32, b >> 32) + (lh >> 32) + (hl >> 32) + + add_with_carry(&ll, lh << 32) + add_with_carry(&ll, hl << 32); + return ll; +#endif +} + +#endif /* mul_64x64_128() */ + +#ifndef mul_64x64_high +static __inline uint64_t mul_64x64_high(uint64_t a, uint64_t b) { + uint64_t h; + mul_64x64_128(a, b, &h); + return h; +} +#endif /* mul_64x64_high */ + +static __inline bool is_power2(size_t x) { return (x & (x - 1)) == 0; } + +static __inline size_t roundup2(size_t value, size_t granularity) { + assert(is_power2(granularity)); + return (value + granularity - 1) & ~(granularity - 1); +} + +//----------------------------------------------------------------------------- + +struct simple_checksum { + uint64_t value; + + simple_checksum() : value(0) {} + + void push(uint32_t data) { + value += data * UINT64_C(9386433910765580089) + 1; + value ^= value >> 41; + } + + void push(uint64_t data) { + push((uint32_t)data); + push((uint32_t)(data >> 32)); + } + + void push(bool data) { push(data ? UINT32_C(0x780E) : UINT32_C(0xFA18E)); } + + void push(const void *ptr, size_t bytes) { + const uint8_t *data = (const uint8_t *)ptr; + for (size_t i = 0; i < bytes; ++i) + push((uint32_t)data[i]); + } + + void push(const double &data) { push(&data, sizeof(double)); } + + void push(const char *cstr) { push(cstr, strlen(cstr)); } + + void push(const std::string &str) { push(str.data(), str.size()); } + +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) + void push(const HANDLE &handle) { push(&handle, sizeof(handle)); } +#endif /* _WINDOWS */ +}; + +std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum); +bool hex2data(const char *hex_begin, const char *hex_end, void *ptr, + size_t bytes, simple_checksum &checksum); + +std::string format(const char *fmt, ...);