ISAR/libmdbx
3
0
Fork 0
mirror of https://github.com/isar/libmdbx.git synced 2024-10-29 23:19:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

mdbx: Merge branch 'devel' (brave new world, forever lost compatibility with LMDB).

Browse Source 
Change-Id: Icc359e01c8a00be6b3ac34a3aaad6f73201e39b3
This commit is contained in:
Leo Yuriev 2017-07-21 17:09:33 +03:00
commit 621bf74e55
94 changed files with 26311 additions and 21608 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.clang-format Normal file
View File

@ -0,0 +1,3 @@
BasedOnStyle: LLVM
Standard: Cpp11
ReflowComments: true

50
.gitignore vendored
View File

@ -1,25 +1,35 @@
mtest[0123456]
wbench
testdb
mdbx_copy
mdbx_stat
mdbx_dump
mdbx_load
mdbx_chk
*.lo
*.[ao]
*.so
*.exe
*[~#]
*.[ao]
*.bak
*.orig
*.rej
*.gcov
*.gcda
*.gcno
.le.ini
core
core.*
*.exe
*.gcda
*.gcno
*.gcov
libmdbx.creator.user
*.lo
mdbx_chk
mdbx_copy
mdbx-dll.VC.db
mdbx-dll.VC.VC.opendb
mdbx-dll.vcxproj.filters
mdbx_dump
mdbx_load
mdbx_stat
*.orig
*.rej
*.so
/test/test
test/test.vcxproj.user
test/tmp.db
test/tmp.db-lck
tmp.db
tmp.db-lck
valgrind.*
man/
html/
yota_test*
.vs/
Win32/
x64/
x86/
test.log

31
.travis.yml
View File

@ -1,15 +1,30 @@
language: c
sudo: false
sudo: required
dist: trusty
cache: bundler
notifications:
email: false
compiler:
- gcc
- clang
- gcc
- clang
os:
- linux
- linux
script: if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make all lmdb check; fi
script: if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make all check; fi
env:
global:
- secure: "M+W+heGGyRQJoBq2W0uqWVrpL4KBXmL0MFL7FSs7f9vmAaDyEgziUXeZRj3GOKzW4kTef3LpIeiu9SmvqSMoQivGGiomZShqPVl045o/OUgRCAT7Al1RLzEZ0efSHpIPf0PZ6byEf6GR2ML76OfuL6JxTVdnz8iVyO2sgLE1HbX1VeB+wgd/jfMeOBhCCXskfK6MLyZihfMYsiYZYSaV98ZDhDLSlzuuRIgzb0bMi8aL6AErs0WLW0NelRBeHkKPYfAUc85pdQHscgrJw6Rh/zT6+8BQ/q5f4IgWhiu4xoRg3Ngl7SNoedRQh93ADM3UG2iGl6HDFpVORaXcFWKAtuYY+kHQ0HB84BRYpQmeBuXNpltsfxQ3d1Q3u0RlE45zRvmr2+X1mFnkcNUAWISLPbsOUlriDQM8irGwRpho77/uYnRC00bJsHW//s6+uPf9zrAw1nI4f0y3PAWukGF/xs6HAI3FZPsuSSnx18Tj3Opgbc9Spop+V3hkhdiJoPGpNKTkFX4ZRXfkPgoRVJmtp4PpbpH0Ps/mCriKjMEfGGi0HcVCi0pEGLXiecdqJ5KPg5+22zNycEujQBJcNTKd9shN+R3glrbmhAxTEzGdGwxXXJ2ybwJ2PWJLMYZ7g98nLyX+uQPaA3BlsbYJHNeS5283/9pJsd9DzfHKsN2nFSc="
before_install:
- echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca-
addons:
coverity_scan:
project:
name: "ReOpen/libmdbx"
version: 0.1
description: "Build submitted via Travis CI"
notification_email: leo@yuriev.ru
build_command_prepend: "make clean"
build_command: "make all -j 4"
branch_pattern: coverity_scan

28
AUTHORS Normal file
View File

@ -0,0 +1,28 @@
Contributors
============
Alexey Naumov <alexey.naumov@gmail.com>
Chris Mikkelson <cmikk@qwest.net>
Claude Brisson <claude.brisson@gmail.com>
David Barbour <dmbarbour@gmail.com>
David Wilson <dw@botanicus.net>
Hallvard Furuseth <hallvard@openldap.org>, <h.b.furuseth@usit.uio.no>
Heiko Becker <heirecka@exherbo.org>
Howard Chu <hyc@openldap.org>, <hyc@symas.com>
Ignacio Casal Quinteiro <ignacio.casal@nice-software.com>
Jean-Christophe DUBOIS <jcd@tribudubois.net>
John Hewson <john@jahewson.com>
Klaus Malorny <klaus.malorny@knipp.de>
Kurt Zeilenga <kurt.zeilenga@isode.com>
Leonid Yuriev <leo@yuriev.ru>, <lyuryev@ptsecurity.com>
Lorenz Bauer <lmb@cloudflare.com>
Luke Yeager <lyeager@nvidia.com>
Martin Hedenfalk <martin@bzero.se>
Ondrej Kuznik <ondrej.kuznik@acision.com>
Orivej Desh <orivej@gmx.fr>
Oskari Timperi <oskari.timperi@iki.fi>
Pavel Medvedev <pmedvedev@gmail.com>
Philipp Storz <philipp.storz@bareos.com>
Quanah Gibson-Mount <quanah@openldap.org>
Salvador Ortiz <sog@msg.com.mx>
Sebastien Launay <sebastien@slaunay.fr>

223
CHANGES
View File

@ -1,223 +0,0 @@
MDBX
Add MDB_PREV_MULTIPLE
Add error MDB_PROBLEM, replace some MDB_CORRUPTED
Workarounds for glibc bugs: #21031 and 21032.
LMDB 0.9.20 Release Engineering
Fix mdb_load with escaped plaintext (ITS#8558)
Fix mdb_cursor_last / mdb_put interaction (ITS#8557)
LMDB 0.9.19 Release (2016/12/28)
Fix mdb_env_cwalk cursor init (ITS#8424)
Fix robust mutexes on Solaris 10/11 (ITS#8339)
Fix MDB_GET_BOTH on non-dup record (ITS#8393)
Optimize mdb_drop
Fix xcursors after mdb_cursor_del (ITS#8406)
Fix MDB_NEXT_DUP after mdb_cursor_del (ITS#8412)
Fix mdb_cursor_put resetting C_EOF (ITS#8489)
Fix mdb_env_copyfd2 to return EPIPE on SIGPIPE (ITS#8504)
Fix mdb_env_copy with empty DB (ITS#8209)
Fix behaviors with fork (ITS#8505)
Fix mdb_dbi_open with mainDB cursors (ITS#8542)
Fix F_NOCACHE on MacOS, error is non-fatal (ITS#7682)
Documentation
Cleanup doxygen nits
Note reserved vs actual mem/disk usage
LMDB 0.9.18 Release (2016/02/05)
already done for mdbx - Fix robust mutex detection on glibc 2.10-11 (ITS#8330)
Fix page_search_root assert on FreeDB (ITS#8336)
Fix MDB_APPENDDUP vs. rewrite(single item) (ITS#8334)
n/a for mdbx - Fix mdb_copy of large files on Windows
Fix subcursor move after delete (ITS#8355)
Fix mdb_midl_shrink off-by-one (ITS#8363)
n/a for mdbx - Check for utf8_to_utf16 failures (ITS#7992)
Catch strdup failure in mdb_dbi_open
Build
already done for mdbx - Additional makefile var tweaks (ITS#8169)
Documentation
Add Getting Started page
Update WRITEMAP description
LMDB 0.9.17 Release (2015/11/30)
Fix ITS#7377 catch calloc failure
Fix ITS#8237 regression from ITS#7589
Fix ITS#8238 page_split for DUPFIXED pages
Fix ITS#8221 MDB_PAGE_FULL on delete/rebalance
Fix ITS#8258 rebalance/split assert
Fix ITS#8263 cursor_put cursor tracking
Fix ITS#8264 cursor_del cursor tracking
Fix ITS#8310 cursor_del cursor tracking
Fix ITS#8299 mdb_del cursor tracking
Fix ITS#8300 mdb_del cursor tracking
Fix ITS#8304 mdb_del cursor tracking
Fix ITS#7771 fakepage cursor tracking
Fix ITS#7789 ensure mapsize >= pages in use
Fix ITS#7971 mdb_txn_renew0() new reader slots
already done for mdbx - Fix ITS#7969 use __sync_synchronize on non-x86
Fix ITS#8311 page_split from update_key
Fix ITS#8312 loose pages in nested txn
Fix ITS#8313 mdb_rebalance dummy cursor
Fix ITS#8315 dirty_room in nested txn
Fix ITS#8323 dirty_list in nested txn
Fix ITS#8316 page_merge cursor tracking
Fix ITS#8319 mdb_load error messages
Fix ITS#8320 mdb_load plaintext input
Fix ITS#8321 cursor tracking
Added mdb_txn_id() (ITS#7994)
Added robust mutex support
Miscellaneous cleanup/simplification
Build
Create install dirs if needed (ITS#8256)
not affected mdbx - Fix ThreadProc decl on Win32/MSVC (ITS#8270)
not affected mdbx - Added ssize_t typedef for MSVC (ITS#8067)
not affected mdbx - Use ANSI apis on Windows (ITS#8069)
already done for mdbx - Use O_SYNC if O_DSYNC,MDB_DSYNC are not defined (ITS#7209)
already done for mdbx - Allow passing AR to make (ITS#8168)
Allow passing mandir to make install (ITS#8169)
LMDB 0.9.16 Release (2015/08/14)
Fix cursor EOF bug (ITS#8190)
Fix handling of subDB records (ITS#8181)
Fix mdb_midl_shrink() usage (ITS#8200)
not affected mdbx - fix reference to EINTR on WIN32 from ITS#8106 (ITS#8192)
LMDB 0.9.15 Release (2015/06/19)
Fix txn init (ITS#7961,#7987)
Fix MDB_PREV_DUP (ITS#7955,#7671)
Fix compact of empty env (ITS#7956)
Fix mdb_copy file mode
Fix mdb_env_close() after failed mdb_env_open()
Fix mdb_rebalance collapsing root (ITS#8062)
Fix mdb_load with large values (ITS#8066)
Fix to retry writes on EINTR (ITS#8106)
Fix mdb_cursor_del on empty DB (ITS#8109)
Fix and Rework comparison for MDB_INTEGERKEY/MDB_INTEGERDUP (ITS#8117)
Fix error handling (ITS#7959,#8157,etc.)
Fix race conditions (ITS#7969,7970)
Added workaround for fdatasync bug in ext3fs
Build
Don't use -fPIC for static lib
Update .gitignore (ITS#7952,#7953)
Cleanup for "make test" (ITS#7841), "make clean", mtest*.c
Misc. Android/Windows cleanup
Documentation
Fix MDB_APPEND doc
Fix MDB_MAXKEYSIZE doc (ITS#8156)
Fix mdb_cursor_put,mdb_cursor_del EACCES description
Fix mdb_env_sync(MDB_RDONLY env) doc (ITS#8021)
Clarify MDB_WRITEMAP doc (ITS#8021)
Clarify mdb_env_open doc
Clarify mdb_dbi_open doc
LMDB 0.9.14 Release (2014/09/20)
Fix to support 64K page size (ITS#7713)
Fix to persist decreased as well as increased mapsizes (ITS#7789)
Fix cursor bug when deleting last node of a DUPSORT key
Fix mdb_env_info to return FIXEDMAP address
Fix ambiguous error code from writing to closed DBI (ITS#7825)
Fix mdb_copy copying past end of file (ITS#7886)
Fix cursor bugs from page_merge/rebalance
Fix to dirty fewer pages in deletes (mdb_page_loose())
Fix mdb_dbi_open creating subDBs (ITS#7917)
Fix mdb_cursor_get(_DUP) with single value (ITS#7913)
Fix Windows compat issues in mtests (ITS#7879)
Add compacting variant of mdb_copy
Add BigEndian integer key compare code
Add mdb_dump/mdb_load utilities
LMDB 0.9.13 Release (2014/06/18)
Fix mdb_page_alloc unlimited overflow page search
Documentation
Re-fix MDB_CURRENT doc (ITS#7793)
Fix MDB_GET_MULTIPLE/MDB_NEXT_MULTIPLE doc
LMDB 0.9.12 Release (2014/06/13)
Fix MDB_GET_BOTH regression (ITS#7875,#7681)
Fix MDB_MULTIPLE writing multiple keys (ITS#7834)
Fix mdb_rebalance (ITS#7829)
Fix mdb_page_split (ITS#7815)
Fix md_entries count (ITS#7861,#7828,#7793)
Fix MDB_CURRENT (ITS#7793)
Fix possible crash on Windows DLL detach
Misc code cleanup
Documentation
mdb_cursor_put: cursor moves on error (ITS#7771)
LMDB 0.9.11 Release (2014/01/15)
Add mdb_env_set_assert() (ITS#7775)
Fix: invalidate txn on page allocation errors (ITS#7377)
Fix xcursor tracking in mdb_cursor_del0() (ITS#7771)
Fix corruption from deletes (ITS#7756)
Fix Windows/MSVC build issues
Raise safe limit of max MDB_MAXKEYSIZE
Misc code cleanup
Documentation
Remove spurious note about non-overlapping flags (ITS#7665)
LMDB 0.9.10 Release (2013/11/12)
Add MDB_NOMEMINIT option
Fix mdb_page_split() again (ITS#7589)
Fix MDB_NORDAHEAD definition (ITS#7734)
Fix mdb_cursor_del() positioning (ITS#7733)
Partial fix for larger page sizes (ITS#7713)
Fix Windows64/MSVC build issues
LMDB 0.9.9 Release (2013/10/24)
Add mdb_env_get_fd()
Add MDB_NORDAHEAD option
Add MDB_NOLOCK option
Avoid wasting space in mdb_page_split() (ITS#7589)
Fix mdb_page_merge() cursor fixup (ITS#7722)
Fix mdb_cursor_del() on last delete (ITS#7718)
Fix adding WRITEMAP on existing env (ITS#7715)
Fix nested txns (ITS#7515)
Fix mdb_env_copy() O_DIRECT bug (ITS#7682)
Fix mdb_cursor_set(SET_RANGE) return code (ITS#7681)
Fix mdb_rebalance() cursor fixup (ITS#7701)
Misc code cleanup
Documentation
Note that by default, readers need write access
LMDB 0.9.8 Release (2013/09/09)
Allow mdb_env_set_mapsize() on an open environment
Fix mdb_dbi_flags() (ITS#7672)
Fix mdb_page_unspill() in nested txns
Fix mdb_cursor_get(CURRENT|NEXT) after a delete
Fix mdb_cursor_get(DUP) to always return key (ITS#7671)
Fix mdb_cursor_del() to always advance to next item (ITS#7670)
Fix mdb_cursor_set(SET_RANGE) for tree with single page (ITS#7681)
Fix mdb_env_copy() retry open if O_DIRECT fails (ITS#7682)
Tweak mdb_page_spill() to be less aggressive
Documentation
Update caveats since mdb_reader_check() added in 0.9.7
LMDB 0.9.7 Release (2013/08/17)
Don't leave stale lockfile on failed RDONLY open (ITS#7664)
Fix mdb_page_split() ref beyond cursor depth
Fix read txn data race (ITS#7635)
Fix mdb_rebalance (ITS#7536, #7538)
Fix mdb_drop() (ITS#7561)
Misc DEBUG macro fixes
Add MDB_NOTLS envflag
Add mdb_env_copyfd()
Add mdb_txn_env() (ITS#7660)
Add mdb_dbi_flags() (ITS#7661)
Add mdb_env_get_maxkeysize()
Add mdb_env_reader_list()/mdb_env_reader_check()
Add mdb_page_spill/unspill, remove hard txn size limit
Use shorter names for semaphores (ITS#7615)
Build
Fix install target (ITS#7656)
Documentation
Misc updates for cursors, DB handles, data lifetime
LMDB 0.9.6 Release (2013/02/25)
Many fixes/enhancements
LMDB 0.9.5 Release (2012/11/30)
Renamed from libmdb to liblmdb
Many fixes/enhancements

2
COPYRIGHT
View File

@ -1,5 +1,5 @@
Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
Copyright 2011-2017 Howard Chu, Symas Corp.
Copyright 2011-2015 Howard Chu, Symas Corp.
Copyright 2015,2016 Peter-Service R&D LLC.
All rights reserved.

1631
Doxyfile
View File

File diff suppressed because it is too large Load Diff

164
Makefile
View File

@ -1,5 +1,4 @@
# GNU Makefile for libmdbx (reliable lightning memory-mapped DB library for Linux).
# https://github.com/ReOpen/libmdbx
# GNU Makefile for libmdbx, https://github.com/ReOpen/libmdbx
########################################################################
# Configuration. The compiler options must enable threaded compilation.
@ -23,37 +22,40 @@ mandir ?= $(prefix)/man
suffix ?=
CC ?= gcc
XCFLAGS ?= -DNDEBUG=1 -DMDB_DEBUG=0
CFLAGS ?= -O2 -g3 -Wall -Werror -Wextra -ffunction-sections
CFLAGS += -std=gnu99 -pthread $(XCFLAGS)
CXX ?= g++
XCFLAGS ?= -DNDEBUG=1 -DMDBX_DEBUG=0 -DLIBMDBX_EXPORTS=1
CFLAGS ?= -O2 -g3 -Wall -Wno-constant-logical-operand -Werror -Wextra -ffunction-sections -fPIC -fvisibility=hidden
CFLAGS += -D_GNU_SOURCE=1 -std=gnu11 -pthread $(XCFLAGS)
CXXFLAGS = -std=c++11 $(filter-out -std=gnu11,$(CFLAGS))
TESTDB ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-check.db
# LY: for ability to built with modern glibc,
# but then run with the old
LDOPS ?= -Wl,--no-as-needed,-lrt
# LY: '--no-as-needed,-lrt' for ability to built with modern glibc, but then run with the old
LDFLAGS ?= -Wl,--gc-sections,-z,relro,-O,--no-as-needed,-lrt
# LY: just for benchmarking
IOARENA ?= ../ioarena.git/@BUILD/src/ioarena
########################################################################
HEADERS := lmdb.h mdbx.h
HEADERS := mdbx.h
LIBRARIES := libmdbx.a libmdbx.so
TOOLS := mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_chk
MANPAGES := mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1
TESTS := mtest0 mtest1 mtest2 mtest3 mtest4 mtest5 mtest6 wbench \
yota_test1 yota_test2 mtest7 mtest8
MANPAGES := mdbx_stat.1 mdbx_copy.1 mdbx_dump.1 mdbx_load.1
SHELL := /bin/bash
SRC_LMDB := mdb.c midl.c lmdb.h midl.h defs.h barriers.h
SRC_MDBX := $(SRC_LMDB) mdbx.c mdbx.h
CORE_SRC := $(filter-out src/lck-windows.c, $(wildcard src/*.c))
CORE_INC := $(wildcard src/*.h)
CORE_OBJ := $(patsubst %.c,%.o,$(CORE_SRC))
TEST_SRC := $(filter-out test/osal-windows.cc, $(wildcard test/*.cc))
TEST_INC := $(wildcard test/*.h)
TEST_OBJ := $(patsubst %.cc,%.o,$(TEST_SRC))
.PHONY: mdbx lmdb all install clean check tests coverage
.PHONY: mdbx all install clean check coverage
all: $(LIBRARIES) $(TOOLS)
all: $(LIBRARIES) $(TOOLS) test/test
mdbx: libmdbx.a libmdbx.so
lmdb: liblmdb.a liblmdb.so
tools: $(TOOLS)
install: $(LIBRARIES) $(TOOLS) $(HEADERS)
@ -67,114 +69,36 @@ install: $(LIBRARIES) $(TOOLS) $(HEADERS)
&& cp -t $(SANDBOX)$(mandir)/man1 $(MANPAGES)
clean:
rm -rf $(TOOLS) $(TESTS) @* *.[ao] *.[ls]o *~ testdb/* *.gcov
rm -rf $(TOOLS) test/test @* *.[ao] *.[ls]o *~ tmp.db/* *.gcov *.log *.err src/*.o test/*.o
tests: $(TESTS)
check: all
rm -f $(TESTDB) test.log && (set -o pipefail; test/test --pathname=$(TESTDB) --dont-cleanup-after basic | tee -a test.log | tail -n 42) && ./mdbx_chk -vn $(TESTDB)
check: tests
[ -d testdb ] || mkdir testdb && rm -f testdb/* \
&& echo "*** LMDB-TEST-0" && ./mtest0 && ./mdbx_chk -v testdb \
&& echo "*** LMDB-TEST-1" && ./mtest1 && ./mdbx_chk -v testdb \
&& echo "*** LMDB-TEST-2" && ./mtest2 && ./mdbx_chk -v testdb \
&& echo "*** LMDB-TEST-3" && ./mtest3 && ./mdbx_chk -v testdb \
&& echo "*** LMDB-TEST-4" && ./mtest4 && ./mdbx_chk -v testdb \
&& echo "*** LMDB-TEST-5" && ./mtest5 && ./mdbx_chk -v testdb \
&& echo "*** LMDB-TEST-6" && ./mtest6 && ./mdbx_chk -v testdb \
&& echo "*** LMDB-TEST-7" && ./mtest7 && ./mdbx_chk -v testdb \
&& echo "*** LMDB-TEST-8" && ./mtest8 && ./mdbx_chk -v testdb \
&& echo "*** LMDB-TESTs - all done"
define core-rule
$(patsubst %.c,%.o,$(1)): $(1) $(CORE_INC) mdbx.h Makefile
$(CC) $(CFLAGS) -c $(1) -o $$@
libmdbx.a: mdbx.o
$(AR) rs $@ $^
endef
$(foreach file,$(CORE_SRC),$(eval $(call core-rule,$(file))))
libmdbx.so: mdbx.lo
$(CC) $(CFLAGS) $(LDFLAGS) -save-temps -pthread -shared $(LDOPS) -o $@ $^
define test-rule
$(patsubst %.cc,%.o,$(1)): $(1) $(TEST_INC) mdbx.h Makefile
$(CXX) $(CXXFLAGS) -c $(1) -o $$@
liblmdb.a: lmdb.o
$(AR) rs $@ $^
endef
$(foreach file,$(TEST_SRC),$(eval $(call test-rule,$(file))))
liblmdb.so: lmdb.lo
$(CC) $(CFLAGS) $(LDFLAGS) -pthread -shared $(LDOPS) -o $@ $^
libmdbx.a: $(CORE_OBJ)
$(AR) rs $@ $?
mdbx_stat: mdb_stat.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) $(LDOPS) -o $@ $^
libmdbx.so: $(CORE_OBJ)
$(CC) $(CFLAGS) -save-temps $^ -pthread -shared $(LDFLAGS) -o $@
mdbx_copy: mdb_copy.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) $(LDOPS) -o $@ $^
mdbx_%: src/tools/mdbx_%.c libmdbx.a
$(CC) $(CFLAGS) $^ $(LDFLAGS) -o $@
mdbx_dump: mdb_dump.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) $(LDOPS) -o $@ $^
mdbx_load: mdb_load.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) $(LDOPS) -o $@ $^
mdbx_chk: mdb_chk.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) $(LDOPS) -o $@ $^
mtest0: mtest0.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
mtest1: mtest1.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
mtest2: mtest2.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
mtest3: mtest3.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
mtest4: mtest4.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
mtest5: mtest5.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
mtest6: mtest6.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
mtest7: mtest7.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
mtest8: mtest8.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
yota_test1: yota_test1.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
yota_test2: yota_test2.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
wbench: wbench.o mdbx.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
mdbx.o: $(SRC_MDBX)
$(CC) $(CFLAGS) -c mdbx.c -o $@
mdbx.lo: $(SRC_MDBX)
$(CC) $(CFLAGS) -fPIC -c mdbx.c -o $@
lmdb.o: $(SRC_LMDB)
$(CC) $(CFLAGS) -c mdb.c -o $@
lmdb.lo: $(SRC_LMDB)
$(CC) $(CFLAGS) -fPIC -c mdb.c -o $@
%: %.o
$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@
%.o: %.c lmdb.h mdbx.h
$(CC) $(CFLAGS) -c $<
COFLAGS = -fprofile-arcs -ftest-coverage
@gcov-mdb.o: $(SRC_MDBX)
$(CC) $(CFLAGS) $(COFLAGS) -O0 -c mdbx.c -o $@
coverage: @gcov-mdb.o
for t in mtest*.c; do x=`basename \$$t .c`; $(MAKE) $$x.o; \
gcc -o @gcov-$$x $$x.o $^ -pthread $(COFLAGS); \
rm -rf testdb; mkdir testdb; ./@gcov-$$x; done
gcov @gcov-mdb
test/test: $(TEST_OBJ) libmdbx.a
$(CXX) $(CXXFLAGS) $^ $(LDFLAGS) -o $@
ifneq ($(wildcard $(IOARENA)),)
@ -196,20 +120,18 @@ endef
$(eval $(call bench-rule,mdbx,$(NN),libmdbx.so))
$(eval $(call bench-rule,lmdb,$(NN)))
$(eval $(call bench-rule,dummy,$(NN)))
$(eval $(call bench-rule,debug,10))
bench: bench-lmdb.txt bench-mdbx.txt
bench: bench-mdbx.txt
endif
ci-rule = ( CC=$$(which $1); if [ -n "$$CC" ]; then \
echo -n "probe by $2 ($$(readlink -f $$(which $$CC))): " && \
$(MAKE) clean >$1.log 2>$1.err && \
$(MAKE) CC=$$(readlink -f $$CC) XCFLAGS="-UNDEBUG -DMDB_DEBUG=2" all check 1>$1.log 2>$1.err && echo "OK" \
$(MAKE) CC=$$(readlink -f $$CC) XCFLAGS="-UNDEBUG -DMDBX_DEBUG=2" check 1>$1.log 2>$1.err && echo "OK" \
|| ( echo "FAILED"; cat $1.err >&2; exit 1 ); \
else echo "no $2 ($1) for probe"; fi; )
ci:

424
README.md
View File

@ -4,27 +4,39 @@ Extended LMDB, aka "Расширенная LMDB".
*The Future will Positive. Всё будет хорошо.*
[![Build Status](https://travis-ci.org/ReOpen/libmdbx.svg?branch=master)](https://travis-ci.org/ReOpen/libmdbx)
[![Build status](https://ci.appveyor.com/api/projects/status/v21jlh5kfmk85r7t/branch/master?svg=true)](https://ci.appveyor.com/project/leo-yuriev/libmdbx/branch/master)
[![Coverity Scan Status](https://scan.coverity.com/projects/12915/badge.svg)](https://scan.coverity.com/projects/reopen-libmdbx)
English version by Google [is here](https://translate.googleusercontent.com/translate_c?act=url&ie=UTF8&sl=ru&tl=en&u=https://github.com/ReOpen/libmdbx/tree/master).
English version [by Google](https://translate.googleusercontent.com/translate_c?act=url&ie=UTF8&sl=ru&tl=en&u=https://github.com/ReOpen/libmdbx/tree/master)
and [by Yandex](https://translate.yandex.ru/translate?url=https%3A%2F%2Fgithub.com%2FReOpen%2Flibmdbx%2Ftree%2Fmaster&lang=ru-en).
## Кратко
_libmdbx_ - это встраиваемый key-value движок хранения со специфическим
набором возможностей, которые при правильном применении позволяют
создавать уникальные решения с чемпионской производительностью, идеально
сочетаясь с технологией [MRAM](https://en.wikipedia.org/wiki/Magnetoresistive_random-access_memory).
набором свойств и возможностей, ориентированный на создание уникальных
легковесных решений с предельной производительностью.
_libmdbx_ позволяет множеству процессов совместно читать и обновлять
несколько key-value таблиц с соблюдением [ACID](https://ru.wikipedia.org/wiki/ACID),
при минимальных накладных расходах и амортизационной стоимости любых операций Olog(N).
_libmdbx_ обеспечивает
[serializability](https://en.wikipedia.org/wiki/Serializability)
изменений и согласованность данных после аварий. При этом транзакции
изменяющие данные никак не мешают операциям чтения и выполняются строго
последовательно с использованием единственного
[мьютекса](https://en.wikipedia.org/wiki/Mutual_exclusion).
_libmdbx_ позволяет выполнять операции чтения с гарантиями
[wait-free](https://en.wikipedia.org/wiki/Non-blocking_algorithm#Wait-freedom),
параллельно на каждом ядре CPU, без использования атомарных операций
и/или примитивов синхронизации.
_libmdbx_ обновляет совместно используемый набор данных, никак не мешая
при этом параллельным операциям чтения, не применяя атомарных операций к
самим данным, и обеспечивая согласованность при аварийной остановке в
любой момент. Поэтому _libmdbx_ позволяя строить системы с линейным
масштабированием производительности чтения/поиска по ядрам CPU и
амортизационной стоимостью любых операций Olog(N).
### История
_libmdbx_ является потомком "Lightning Memory-Mapped Database",
_libmdbx_ является развитием "Lightning Memory-Mapped Database",
известной под аббревиатурой
[LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database).
Изначально доработка производилась в составе проекта
@ -34,22 +46,26 @@ _libmdbx_ является потомком "Lightning Memory-Mapped Database",
[представлен на конференции Highload++
2015](http://www.highload.ru/2015/abstracts/1831.html).
В начале 2017 года движок _libmdbx_ получил новый импульс развития,
благодаря использованию в [Fast Positive
Tables](https://github.com/leo-yuriev/libfpta), aka ["Позитивные
Таблицы"](https://github.com/leo-yuriev/libfpta) by [Positive
Technologies](https://www.ptsecurity.ru).
Характеристики и ключевые особенности
=====================================
_libmdbx_ наследует все ключевые возможности и особенности от
своего прародителя [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database),
с устранением описанных далее проблем и архитектурных недочетов.
### Общее для оригинальной _LMDB_ и _libmdbx_
но с устранением ряда описываемых далее проблем и архитектурных недочетов.
1. Данные хранятся в упорядоченном отображении (ordered map), ключи всегда
отсортированы, поддерживается выборка диапазонов (range lookups).
2. Данные отображается в память каждого работающего с БД процесса.
Ключам и данным обеспечивается прямой доступ без необходимости их
копирования, так как они защищены транзакцией чтения и не изменяются.
К данным и ключам обеспечивается прямой доступ в памяти без необходимости их
копирования.
3. Транзакции согласно
[ACID](https://ru.wikipedia.org/wiki/ACID), посредством
@ -57,20 +73,26 @@ _libmdbx_ наследует все ключевые возможности и
[COW](https://ru.wikipedia.org/wiki/%D0%9A%D0%BE%D0%BF%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%BF%D1%80%D0%B8_%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8).
Изменения строго последовательны и не блокируются чтением,
конфликты между транзакциями не возможны.
При этом гарантируется чтение только зафиксированных данных, см [relaxing serializability](https://en.wikipedia.org/wiki/Serializability).
4. Чтение и поиск [без блокировок](https://ru.wikipedia.org/wiki/%D0%9D%D0%B5%D0%B1%D0%BB%D0%BE%D0%BA%D0%B8%D1%80%D1%83%D1%8E%D1%89%D0%B0%D1%8F_%D1%81%D0%B8%D0%BD%D1%85%D1%80%D0%BE%D0%BD%D0%B8%D0%B7%D0%B0%D1%86%D0%B8%D1%8F),
без [атомарных операций](https://ru.wikipedia.org/wiki/%D0%90%D1%82%D0%BE%D0%BC%D0%B0%D1%80%D0%BD%D0%B0%D1%8F_%D0%BE%D0%BF%D0%B5%D1%80%D0%B0%D1%86%D0%B8%D1%8F).
Читатели не блокируются операциями записи и не конкурируют
между собой, чтение масштабируется линейно по ядрам CPU.
> Для точности следует отметить, что "подключение к БД" (старт первой
> читающей транзакции в потоке) и "отключение от БД" (закрытие БД или
> завершение потока) требуют краткосрочного захвата блокировки для
> регистрации/дерегистрации текущего потока в "таблице читателей".
5. Эффективное хранение дубликатов (ключей с несколькими
значениями), без дублирования ключей, с сортировкой значений, в
том числе целочисленных (для вторичных индексов).
6. Эффективная поддержка ключей фиксированной длины, в том числе целочисленных.
6. Эффективная поддержка коротких ключей фиксированной длины, в том числе целочисленных.
7. Амортизационная стоимость любой операции Olog(N),
[WAF](https://en.wikipedia.org/wiki/Write_amplification) и RAF также Olog(N).
[WAF](https://en.wikipedia.org/wiki/Write_amplification) (Write
Amplification Factor) и RAF (Read Amplification Factor) также Olog(N).
8. Нет [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) и журнала
транзакций, после сбоев не требуется восстановление. Не требуется компактификация
@ -78,45 +100,231 @@ _libmdbx_ наследует все ключевые возможности и
"по горячему", на работающей БД без приостановки изменения данных.
9. Отсутствует какое-либо внутреннее управление памятью или кэшированием. Всё
необходимое штатно выполняет ядро ОС.
необходимое штатно выполняет ядро ОС!
### Недостатки и Компромиссы
Сравнение производительности
============================
Все представленные ниже данные получены многократным прогоном тестов на
ноутбуке Lenovo Carbon-2, i7-4600U 2.1 ГГц, 8 Гб ОЗУ, с SSD-диском
SAMSUNG MZNTD512HAGL-000L1 (DXT23L0Q) 512 Гб.
Исходный код бенчмарка [_IOArena_](https://github.com/pmwkaa/ioarena) и
сценарии тестирования [доступны на
github](https://github.com/pmwkaa/ioarena/tree/HL%2B%2B2015).
--------------------------------------------------------------------------------
### Интегральная производительность
![Comparison #1: Integral Performance](https://raw.githubusercontent.com/wiki/ReOpen/libmdbx/img/perf-slide-1.png)
Показана соотнесенная сумма ключевых показателей производительности в трёх
бенчмарках:
- Чтение/Поиск на машине с 4-мя процессорами;
- Транзакции с [CRUD](https://ru.wikipedia.org/wiki/CRUD)-операциями
(вставка, чтение, обновление, удаление) в режиме **синхронной фиксации**
данных (fdatasync при завершении каждой транзакции или аналог);
- Транзакции с [CRUD](https://ru.wikipedia.org/wiki/CRUD)-операциями
(вставка, чтение, обновление, удаление) в режиме **отложенной фиксации**
данных (отложенная запись посредством файловой систем или аналог);
*Бенчмарк в режиме асинхронной записи не включен по двум причинам:*
1. Такое сравнение не совсем правомочно, его следует делать с движками
ориентированными на хранение данных в памяти ([Tarantool](https://tarantool.io/), [Redis](https://redis.io/)).
2. Превосходство libmdbx становится еще более подавляющем, что мешает
восприятию информации.
--------------------------------------------------------------------------------
### Масштабируемость чтения
![Comparison #2: Read Scalability](https://raw.githubusercontent.com/wiki/ReOpen/libmdbx/img/perf-slide-2.png)
Для каждого движка показана суммарная производительность при
одновременном выполнении запросов чтения/поиска в 1-2-4-8 потоков на
машине с 4-мя физическими процессорами.
--------------------------------------------------------------------------------
### Синхронная фиксация
![Comparison #3: Sync-write mode](https://raw.githubusercontent.com/wiki/ReOpen/libmdbx/img/perf-slide-3.png)
- Линейная шкала слева и темные прямоугольники соответствуют количеству
транзакций в секунду, усредненному за всё время теста.
- Логарифмическая шкала справа и желтые интервальные отрезки
соответствуют времени выполнения транзакций. При этом каждый отрезок
показывает минимальное и максимальное время затраченное на выполнение
транзакций, а крестиком отмечено среднеквадратичное значение.
Выполняется **10.000 транзакций в режиме синхронной фиксации данных** на
диске. При этом требуется гарантия, что при аварийном выключении питания
(или другом подобном сбое) все данные будут консистентны и полностью
соответствовать последней завершенной транзакции. В _libmdbx_ в этом
режиме при фиксации каждой транзакции выполняется системный вызов
[fdatasync](https://linux.die.net/man/2/fdatasync).
В каждой транзакции выполняется комбинированная CRUD-операция (две
вставки, одно чтение, одно обновление, одно удаление). Бенчмарк стартует
на пустой базе, а при завершении, в результате выполняемых действий, в
базе насчитывается 10.000 небольших key-value записей.
--------------------------------------------------------------------------------
### Отложенная фиксация
![Comparison #4: Lazy-write mode](https://raw.githubusercontent.com/wiki/ReOpen/libmdbx/img/perf-slide-4.png)
- Линейная шкала слева и темные прямоугольники соответствуют количеству
транзакций в секунду, усредненному за всё время теста.
- Логарифмическая шкала справа и желтые интервальные отрезки
соответствуют времени выполнения транзакций. При этом каждый отрезок
показывает минимальное и максимальное время затраченное на выполнение
транзакций, а крестиком отмечено среднеквадратичное значение.
Выполняется **100.000 транзакций в режиме отложенной фиксации данных**
на диске. При этом требуется гарантия, что при аварийном выключении
питания (или другом подобном сбое) все данные будут консистентны на
момент завершения одной из транзакций, но допускается потеря изменений
из некоторого количества последних транзакций, что для многих движков
предполагает включение
[WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) (write-ahead
logging) либо журнала транзакций, который в свою очередь опирается на
гарантию упорядоченности данных в журналируемой файловой системе.
_libmdbx_ при этом не ведет WAL, а передает весь контроль файловой
системе и ядру ОС.
В каждой транзакции выполняется комбинированная CRUD-операция (две
вставки, одно чтение, одно обновление, одно удаление). Бенчмарк стартует
на пустой базе, а при завершении, в результате выполняемых действий, в
базе насчитывается 100.000 небольших key-value записей.
--------------------------------------------------------------------------------
### Асинхронная фиксация
![Comparison #5: Async-write mode](https://raw.githubusercontent.com/wiki/ReOpen/libmdbx/img/perf-slide-5.png)
- Линейная шкала слева и темные прямоугольники соответствуют количеству
транзакций в секунду, усредненному за всё время теста.
- Логарифмическая шкала справа и желтые интервальные отрезки
соответствуют времени выполнения транзакций. При этом каждый отрезок
показывает минимальное и максимальное время затраченное на выполнение
транзакций, а крестиком отмечено среднеквадратичное значение.
Выполняется **1.000.000 транзакций в режиме асинхронной фиксации
данных** на диске. При этом требуется гарантия, что при аварийном
выключении питания (или другом подобном сбое) все данные будут
консистентны на момент завершения одной из транзакций, но допускается
потеря изменений из значительного количества последних транзакций. Во
всех движках при этом включался режим предполагающий минимальную
нагрузку на диск по-записи, и соответственно минимальную гарантию
сохранности данных. В _libmdbx_ при этом используется режим асинхронной
записи измененных страниц на диск посредством ядра ОС и системного
вызова [msync(MS_ASYNC)](https://linux.die.net/man/2/msync).
В каждой транзакции выполняется комбинированная CRUD-операция (две
вставки, одно чтение, одно обновление, одно удаление). Бенчмарк стартует
на пустой базе, а при завершении, в результате выполняемых действий, в
базе насчитывается 10.000 небольших key-value записей.
--------------------------------------------------------------------------------
### Стоимость как потребление ресурсов
![Comparison #6: Cost comparison](https://raw.githubusercontent.com/wiki/ReOpen/libmdbx/img/perf-slide-6.png)
Показана соотнесенная сумма использованных ресурсов в ходе бенчмарка в
режиме отложенной фиксации:
- суммарное количество операций ввода-вывода (IOPS), как записи, так и
чтения.
- суммарное затраченное время процессора, как в режиме пользовательских процессов,
так и в режиме ядра ОС.
- использованное место на диске при завершении теста, после закрытия БД из тестирующего процесса,
но без ожидания всех внутренних операций обслуживания (компактификации LSM и т.п.).
Движок _ForestDB_ был исключен при оформлении результатов, так как
относительно конкурентов многократно превысил потребление каждого из
ресурсов (потратил процессорное время на генерацию IOPS для заполнения
диска), что не позволяло наглядно сравнить показатели остальных движков
на одной диаграмме.
Все данные собирались посредством системного вызова
[getrusage()](http://man7.org/linux/man-pages/man2/getrusage.2.html) и
сканированием директорий с данными.
--------------------------------------------------------------------------------
## Недостатки и Компромиссы
1. Единовременно может выполняться не более одной транзакция изменения данных
(один писатель). Зато все изменения всегда последовательны, не может быть
конфликтов или ошибок при откате транзакций.
конфликтов или логических ошибок при откате транзакций.
2. Отсутствие [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging)
обуславливает относительно большой
[WAF](https://en.wikipedia.org/wiki/Write_amplification). Поэтому фиксация
изменений на диске может быть дорогой и является главным ограничителем для
производительности по записи. В качестве компромисса предлагается несколько
режимов ленивой и/или периодической фиксации. В том числе режим `MAPASYNC`,
при котором изменения происходят только в памяти и асинхронно фиксируются на
диске ядром ОС.
[WAF](https://en.wikipedia.org/wiki/Write_amplification) (Write
Amplification Factor). Поэтому фиксация изменений на диске может быть
достаточно дорогой и являться главным ограничением производительности
при интенсивном изменении данных.
> В качестве компромисса _libmdbx_ предлагает несколько режимов ленивой
> и/или периодической фиксации. В том числе режим `MAPASYNC`, при котором
> изменения происходят только в памяти и асинхронно фиксируются на диске
> ядром ОС.
>
> Однако, следует воспринимать это свойство аккуратно и взвешенно.
> Например, полная фиксация транзакции в БД с журналом потребует минимум 2
> IOPS (скорее всего 3-4) из-за накладных расходов в файловой системе. В
> _libmdbx_ фиксация транзакции также требует от 2 IOPS. Однако, в БД с
> журналом кол-во IOPS будет меняться в зависимости от файловой системы,
> но не от кол-ва записей или их объема. Тогда как в _libmdbx_ кол-во
> будет расти логарифмически от кол-во записей/строк в БД (по высоте
> b+tree).
3. [COW](https://ru.wikipedia.org/wiki/%D0%9A%D0%BE%D0%BF%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%BF%D1%80%D0%B8_%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8)
для реализации [MVCC](https://ru.wikipedia.org/wiki/MVCC) выполняется на
уровне страниц в [B+ дереве](https://ru.wikipedia.org/wiki/B-%D0%B4%D0%B5%D1%80%D0%B5%D0%B2%D0%BE).
Поэтому изменение данных амортизационно требует копирования Olog(N) страниц,
что расходует [пропускную способность оперативной
памяти](https://en.wikipedia.org/wiki/Memory_bandwidth) и является основным
ограничителем производительности в режиме `MAPASYNC`.
уровне страниц в [B+
дереве](https://ru.wikipedia.org/wiki/B-%D0%B4%D0%B5%D1%80%D0%B5%D0%B2%D0%BE).
Поэтому изменение данных амортизационно требует копирования Olog(N)
страниц, что расходует [пропускную способность оперативной
памяти](https://en.wikipedia.org/wiki/Memory_bandwidth) и является
основным ограничителем производительности в режиме `MAPASYNC`.
> Этот недостаток неустраним, тем не менее следует дать некоторые пояснения.
> Дело в том, что фиксация изменений на диске потребует гораздо более
> значительного копирования данных в памяти и массы других затратных операций.
> Поэтому обусловленное этим недостатком падение производительности становится
> заметным только при отказе от фиксации изменений на диске.
> Соответственно, корректнее сказать что _libmdbx_ позволяет
> получить персистентность ценой минимального падения производительности.
> Если же нет необходимости оперативно сохранять данные, то логичнее
> использовать `std::map`.
4. В _LMDB_ существует проблема долгих чтений (приостановленных читателей),
которая приводит к деградации производительности и переполнению БД.
В _libmdbx_ предложены средства для предотвращения, выхода из проблемной
ситуации и устранения её последствий. Подробности ниже.
> В _libmdbx_ предложены средства для предотвращения, быстрого выхода из
> некомфортной ситуации и устранения её последствий. Подробности ниже.
5. В _LMDB_ есть вероятность разрушения БД в режиме `WRITEMAP+MAPASYNC`.
В _libmdbx_ для `WRITEMAP+MAPASYNC` гарантируется как сохранность базы,
так и согласованность данных. При этом также, в качестве альтернативы,
предложен режим `UTTERLY_NOSYNC`. Подробности ниже.
так и согласованность данных.
> Дополнительно, в качестве альтернативы, предложен режим `UTTERLY_NOSYNC`.
> Подробности ниже.
#### Проблема долгих чтений
*Следует отметить*, что проблема "сборки мусора" так или иначе
существует во всех СУБД (Vacuum в PostgreSQL). Однако в случае _libmdbx_
и LMDB она проявляется более остро, прежде всего из-за высокой
производительности, а также из-за намеренного прощения внутренних
механизмов ради производительности.
Понимание проблемы требует некоторых пояснений, которые
изложены ниже, но могут быть сложны для быстрого восприятия.
Поэтому, тезисно:
@ -141,66 +349,62 @@ _libmdbx_ наследует все ключевые возможности и
деградацию производительности.
Операции чтения выполняются в контексте снимка данных (версии
БД), который был актуальным на момент старта транзакции чтения.
Такой читаемый снимок поддерживается неизменным до завершения
операции. В свою очередь, это не позволяет повторно
использовать страницы БД в последующих версиях (снимках БД).
БД), который был актуальным на момент старта транзакции чтения. Такой
читаемый снимок поддерживается неизменным до завершения операции. В свою
очередь, это не позволяет повторно использовать страницы БД в
последующих версиях (снимках БД).
Другими словами, если обновление данных выполняется на фоне
долгой операции чтения, то вместо повторного использования
"старых" ненужных страниц будут выделяться новые, так как
"старые" страницы составляют снимок БД, который еще
используется долгой операцией чтения.
Другими словами, если обновление данных выполняется на фоне долгой
операции чтения, то вместо повторного использования "старых" ненужных
страниц будут выделяться новые, так как "старые" страницы составляют
снимок БД, который еще используется долгой операцией чтения.
В результате, при интенсивном изменении данных и достаточно
длительной операции чтения, в БД могут быть исчерпаны свободные
страницы, что не позволит создавать новые снимки/версии БД.
Такая ситуация будет сохраняться до завершения операции чтения,
которая использует старый снимок данных и препятствует
повторному использованию страниц БД.
В результате, при интенсивном изменении данных и достаточно длительной
операции чтения, в БД могут быть исчерпаны свободные страницы, что не
позволит создавать новые снимки/версии БД. Такая ситуация будет
сохраняться до завершения операции чтения, которая использует старый
снимок данных и препятствует повторному использованию страниц БД.
Однако, на этом проблемы не заканчиваются. После описанной
ситуации, все дополнительные страницы, которые были выделены
пока переработка старых была невозможна, будут участвовать в
цикле выделения/освобождения до конца жизни экземпляра БД. В
оригинальной _LMDB_ этот цикл использования страниц работает по
принципу [FIFO](https://ru.wikipedia.org/wiki/FIFO). Поэтому
увеличение количества циркулирующий страниц, с точки зрения
механизмов кэширования и/или обратной записи, выглядит как
увеличение рабочего набор данных. Проще говоря, однократное
попадание в ситуацию "уснувшего читателя" приводит к
устойчивому эффекту вымывания I/O кэша при всех последующих
изменениях данных.
Однако, на этом проблемы не заканчиваются. После описанной ситуации, все
дополнительные страницы, которые были выделены пока переработка старых
была невозможна, будут участвовать в цикле выделения/освобождения до
конца жизни экземпляра БД. В оригинальной _LMDB_ этот цикл использования
страниц работает по принципу [FIFO](https://ru.wikipedia.org/wiki/FIFO).
Поэтому увеличение количества циркулирующий страниц, с точки зрения
механизмов кэширования и/или обратной записи, выглядит как увеличение
рабочего набор данных. Проще говоря, однократное попадание в ситуацию
"уснувшего читателя" приводит к устойчивому эффекту вымывания I/O кэша
при всех последующих изменениях данных.
Для устранения описанных проблемы в _libmdbx_ сделаны
существенные доработки, подробности ниже. Иллюстрации к
проблеме "долгих чтений" можно найти в [слайдах
презентации](http://www.slideshare.net/leoyuriev/lmdb).
Там же приведен пример количественной оценки прироста
производительности за счет эффективной работы
[BBWC](https://en.wikipedia.org/wiki/BBWC) при включении `LIFO
RECLAIM` в _libmdbx_.
Для устранения описанных проблемы в _libmdbx_ сделаны существенные
доработки, подробности ниже. Иллюстрации к проблеме "долгих чтений"
можно найти в [слайдах презентации](http://www.slideshare.net/leoyuriev/lmdb).
Там же приведен пример количественной оценки прироста производительности
за счет эффективной работы [BBWC](https://en.wikipedia.org/wiki/BBWC)
при включении `LIFO RECLAIM` в _libmdbx_.
#### Вероятность разрушения БД в режиме `WRITEMAP+MAPASYNC`
При работе в режиме `WRITEMAP+MAPSYNC` запись измененных
страниц выполняется ядром ОС, что имеет ряд преимуществ. Так
например, при крахе приложения, ядро ОС сохранит все изменения.
При работе в режиме `WRITEMAP+MAPSYNC` запись измененных страниц
выполняется ядром ОС, что имеет ряд преимуществ. Так например, при крахе
приложения, ядро ОС сохранит все изменения.
Однако, при аварийном отключении питания или сбое в ядре ОС, на
диске будет сохранена только часть измененных страниц БД. При
этом с большой вероятностью может оказаться так, что будут
сохранены мета-страницы со ссылками на страницы с новыми
версиями данных, но не сами новые данные. В этом случае БД
будет безвозвратна разрушена, даже если до аварии производилась
полная синхронизация данных (посредством `mdb_env_sync()`).
Однако, при аварийном отключении питания или сбое в ядре ОС, на диске
будет сохранена только часть измененных страниц БД. При этом с большой
вероятностью может оказаться так, что будут сохранены мета-страницы со
ссылками на страницы с новыми версиями данных, но не сами новые данные.
В этом случае БД будет безвозвратна разрушена, даже если до аварии
производилась полная синхронизация данных (посредством
`mdbx_env_sync()`).
В _libmdbx_ эта проблема устранена, подробности ниже.
--------------------------------------------------------------------------------
Доработки _libmdbx_
===================
Дополнительные "фичи" _libmdbx_ относительно LMDB
=================================================
1. Режим `LIFO RECLAIM`.
@ -221,11 +425,11 @@ RECLAIM` в _libmdbx_.
Посредством `mdbx_env_set_oomfunc()` может быть установлен
внешний обработчик (callback), который будет вызван при
исчерпания свободных страниц из-за долгой операцией чтения.
Обработчику будет передан PID и pthread_id. В свою очередь
обработчик может предпринять одно из действий:
Обработчику будет передан PID и pthread_id виновника.
В свою очередь обработчик может предпринять одно из действий:
* отправить сигнал kill (#9), если долгое чтение выполняется
сторонним процессом;
* нейтрализовать виновника (отправить сигнал kill #9), если
долгое чтение выполняется сторонним процессом;
* отменить или перезапустить проблемную операцию чтения, если
операция выполняется одним из потоков текущего процесса;
@ -248,7 +452,7 @@ RECLAIM` в _libmdbx_.
сохранены мета-страницы со ссылками на страницы с новыми
версиями данных, но не сами новые данные. В этом случае БД
будет безвозвратна разрушена, даже если до аварии производилась
полная синхронизация данных (посредством `mdb_env_sync()`).
полная синхронизация данных (посредством `mdbx_env_sync()`).
В _libmdbx_ эта проблема устранена путем полной переработки
пути записи данных:
@ -264,6 +468,14 @@ RECLAIM` в _libmdbx_.
`WRITEMAP+MAPSYNC` завершаемые транзакции помечаются как
слабые, а при явной синхронизации данных как сильные.
* В _libmdbx_ поддерживается не две, а три отдельные мета-страницы.
Это позволяет выполнять фиксацию транзакций с формированием как
сильной, так и слабой точки фиксации, без потери двух предыдущих
точек фиксации (из которых одна может быть сильной, а вторая слабой).
В результате, _libmdbx_ позволяет в произвольном порядке чередовать
сильные и слабые точки фиксации без нарушения соответствующих
гарантий в случае неожиданной системной аварии во время фиксации.
* При открытии БД выполняется автоматический откат к последней
сильной фиксации. Этим обеспечивается гарантия сохранности БД.
@ -302,14 +514,14 @@ RECLAIM` в _libmdbx_.
посредством `mdbx_cursor_eof()`.
10. Возможность явно запросить обновление существующей записи, без
создания новой посредством флажка `MDB_CURRENT` для `mdbx_put()`.
создания новой посредством флажка `MDBX_CURRENT` для `mdbx_put()`.
11. Возможность обновить или удалить запись с получением предыдущего
значения данных посредством `mdbx_replace()`.
11. Возможность посредством `mdbx_replace()` обновить или удалить запись
с получением предыдущего значения данных, а также адресно изменить
конкретное multi-значение.
12. Поддержка ключей и значений нулевой длины. Включая сортированные
дубликаты, в том числе вне зависимости от порядка их добавления или
обновления.
12. Поддержка ключей и значений нулевой длины, включая сортированные
дубликаты.
13. Исправленный вариант `mdbx_cursor_count()`, возвращающий корректное
количество дубликатов для всех типов таблиц и любого положения курсора.
@ -325,7 +537,7 @@ RECLAIM` в _libmdbx_.
который используется одним из читателей.
17. Функция `mdbx_del()` не игнорирует дополнительный (уточняющий)
аргумент `data` для таблиц без дубликатов (без флажка `MDB_DUPSORT`), а
аргумент `data` для таблиц без дубликатов (без флажка `MDBX_DUPSORT`), а
при его ненулевом значении всегда использует его для сверки с удаляемой
записью.
@ -333,13 +545,14 @@ RECLAIM` в _libmdbx_.
компараторов для ключей и данных, посредством `mdbx_dbi_open_ex()`.
19. Возможность посредством `mdbx_is_dirty()` определить находятся ли
некоторый ключ или данные в "грязной" странице БД. Таким образом избегаю
лишнего копирования данных перед выполнением модифицирующих операций
(значения в размещенные "грязных" страницах могут быть перезаписаны при
изменениях, иначе они будут неизменны).
некоторый ключ или данные в "грязной" странице БД. Таким образом,
избегая лишнего копирования данных перед выполнением модифицирующих
операций (значения в размещенные "грязных" страницах могут быть
перезаписаны при изменениях, иначе они будут неизменны).
20. Корректное обновление текущей записи, в том числе сортированного
дубликата, при использовании режима `MDB_CURRENT` в `mdbx_cursor_put()`.
дубликата, при использовании режима `MDBX_CURRENT` в
`mdbx_cursor_put()`.
21. Все курсоры, как в транзакциях только для чтения, так и в пишущих,
могут быть переиспользованы посредством `mdbx_cursor_renew()` и ДОЛЖНЫ
@ -371,5 +584,14 @@ RECLAIM` в _libmdbx_.
25. При завершении читающих транзакций, открытые в них DBI-хендлы не
закрываются и не теряются при завершении таких транзакций посредством
mdb_txn_abort() или mdb_txn_reset(). Что позволяет избавится от ряда
mdbx_txn_abort() или mdbx_txn_reset(). Что позволяет избавится от ряда
сложно обнаруживаемых ошибок.
26. Генерация последовательностей посредством `mdbx_dbi_sequence()`.
27. Расширенное динамическое управление размером БД, включая выбор
размера страницы посредством `mdbx_env_set_geometry()`.
28. Три мета-страницы вместо двух, что позволяет гарантированно
консистентно обновлять слабые контрольные точки фиксации без риска
повредить крайнюю сильную точку фиксации.

89
TODO.md Normal file
View File

@ -0,0 +1,89 @@
Допеределки
===========
- [ ] Перевод mdbx-tools на С++ и сборка для Windows.
- [ ] Переход на CMake, замена заглушек mdbx_version и mdbx_build.
- [ ] Актуализация README.md
- [ ] Переход на C++11, добавление #pramga detect_mismatch().
- [ ] Убрать MDB_DEBUG (всегда: логирование важный ситуаций и ошибок, опционально: включение ассертов и трассировка).
- [ ] Заменить mdbx_debug на mdbx_trace, и почистить...
- [ ] Заметить максимум assert() на mdbx_assert(env, ...).
Качество и CI
=============
- [ ] Добавить в CI linux сборки для 32-битных таргетов.
Доработки API
=============
- [ ] Поправить/Добавить описание нового API.
- [ ] Добавить возможность "подбора" режима для mdbx_env_open().
- [ ] Переименовать в API: env->db, db->tbl.
Тесты
=====
- [ ] Тестирование поддержки lockless-режима.
- [ ] Додумать имя и размещение тестовой БД по-умолчанию.
- [ ] Реализовать cleanup в тесте.
- [ ] usage для теста.
- [ ] Логирование в файл, плюс более полный progress bar.
- [ ] Опция игнорирования (пропуска части теста) при переполнении БД.
- [ ] Базовый бенчмарк.
Развитие
========
- [ ] Отслеживание времени жизни DBI-хендлов.
- [ ] Отрефакторить mdbx_freelist_save().
- [ ] Хранить "свободный хвост" не связанный с freeDB в META.
- [ ] Возврат выделенных страниц в unallocated tail-pool.
- [ ] Валидатор страниц БД по номеру транзакции:
~0 при переработке и номер транзакции при выделении,
проверять что этот номер больше головы реклайминга и не-больше текущей транзакции.
- [ ] Размещение overflow-pages в отдельном mmap/файле с собственной геометрией.
- [ ] Зафиксировать формат БД.
- [ ] Валидатор страниц по CRC32, плюс контроль номер транзакии под модулю 2^32.
- [ ] Валидатор страниц по t1ha c контролем снимков/версий БД на основе Merkle Tree.
- [ ] Возможность хранения ключей внутри data (libfptu).
- [ ] Асинхронная фиксация (https://github.com/ReOpen/libmdbx/issues/5).
- [ ] (Пере)Выделять память под IDL-списки с учетом реального кол-ва страниц, т.е. max(MDB_IDL_UM_MAX/MDB_IDL_UM_MAX, npages).
-----------------------------------------------------------------------
Сделано
=======
- [x] разделение errno и GetLastError().
- [x] CI посредством AppVeyor.
- [x] тест конкурентного доступа.
- [x] тест основного функционала (заменить текущий треш).
- [x] uint32/uint64 в структурах.
- [x] Завершить переименование.
- [x] Макросы версионности, сделать как в fpta (cmake?).
- [x] Попробовать убрать yield (или что там с местом?).
- [x] trinity для copy/compaction.
- [x] trinity для mdbx_chk и mdbx_stat.
- [x] проверки с mdbx_meta_eq.
- [x] Не проверять режим при открытии в readonly.
- [x] Поправить выбор tail в mdbx_chk.
- [x] Там-же проверять позицию реклайминга.
- [x] поправить проблему открытия после READ-ONLY.
- [x] static-assertы на размер/выравнивание lck, meta и т.п.
- [x] Зачистить size_t.
- [x] Добавить локи вокруг dbi.
- [x] Привести в порядок volatile.
- [x] контроль meta.mapsize.
- [x] переработка формата: заголовки страниц, meta, clk...
- [x] зачистка Doxygen и бесполезных коментариев.
- [x] Добавить поле типа контрольной суммы.
- [x] Добавить поле/флаг размера pgno_t.
- [x] Поменять сигнатуры.
- [x] Добавить мета-страницы в coredump, проверить lck.
- [x] Сделать список для txnid_t, кода sizeof(txnid_t) > sizeof(pgno_t) и вернуть размер pgno_t.
- [x] Избавиться от умножения на размер страницы (заменить на сдвиг).
- [x] Устранение всех предупреждений (в том числе под Windows).
- [x] Добавить 'mti_reader_finished_flag'.
- [x] Погасить все level4-warnings от MSVC, включить /WX.
- [x] Проверка посредством Coverity с гашением всех дефектов.
- [x] Полная матрица Windows-сборок (2013/2015/2017).
- [x] Дать возможность задавать размер страницы при создании БД.
- [x] Изменение mapsize через API с блокировкой и увеличением txn.
- [x] Контроль размера страницы полного размера и кол-ва страниц при создании и обновлении.
- [x] Инкрементальный mmap.
- [x] Инкрементальное приращение размера (колбэк стратегии?).

48
appveyor.yml Normal file
View File

@ -0,0 +1,48 @@
version: 0.1.2.{build}
environment:
matrix:
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
TOOLSET: v141
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
TOOLSET: v140
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013
TOOLSET: v120
configuration:
- Debug
- Release
platform:
- x86
- x64
#- ARM
build_script:
- ps: >
msbuild "C:\projects\libmdbx\mdbx.sln" /verbosity:minimal
/logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll"
/property:PlatformToolset=$env:TOOLSET
/property:Configuration=$env:CONFIGURATION
/property:Platform=$env:PLATFORM
test_script:
- ps: |
if (($env:PLATFORM -eq "x86") -and (Test-Path "C:\projects\libmdbx\Win32\$env:CONFIGURATION\test.exe" -PathType Leaf)) {
$test = "C:\projects\libmdbx\Win32\$env:CONFIGURATION\test.exe"
$mdbx_chk = "C:\projects\libmdbx\Win32\$env:CONFIGURATION\mdbx_chk.exe"
} elseif (($env:PLATFORM -ne "ARM") -and ($env:PLATFORM -ne "ARM64")) {
$test = "C:\projects\libmdbx\$env:PLATFORM\$env:CONFIGURATION\test.exe"
$mdbx_chk = "C:\projects\libmdbx\$env:PLATFORM\$env:CONFIGURATION\mdbx_chk.exe"
} else {
$test = ""
$mdbx_chk = ""
}
if ($test -ne "") {
& "$test" --pathname=tmp.db --dont-cleanup-after basic | Tee-Object -file test.log | Select-Object -last 42
& "$mdbx_chk" -nvv tmp.db | Tee-Object -file chk.log | Select-Object -last 42
}
on_failure:
- ps: Push-AppveyorArtifact test.log chk.log

152
barriers.h
View File

@ -1,152 +0,0 @@
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/*****************************************************************************
* Properly compiler/memory/coherence barriers
* in the most portable way for ReOpenMDBX project.
*
* Feedback and comments are welcome.
* https://gist.github.com/leo-yuriev/ba186a6bf5cf3a27bae7 */
#if defined(__mips) && defined(__linux)
/* Only MIPS has explicit cache control */
# include <asm/cachectl.h>
#endif
#if defined(__GNUC__) || defined(__clang__)
# define MDBX_INLINE __inline
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
# include <intrin.h>
# if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
# pragma intrinsic(__mf)
# elif defined(__i386__) || defined(__x86_64__)
# pragma intrinsic(_mm_mfence)
# endif
# define MDBX_INLINE __inline
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
# include <mbarrier.h>
# define MDBX_INLINE inline
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) \
&& (defined(HP_IA64) || defined(__ia64))
# include <machine/sys/inline.h>
# define MDBX_INLINE
#elif defined(__IBMC__) && defined(__powerpc)
# include <atomic.h>
# define MDBX_INLINE
#elif defined(_AIX)
# include <builtins.h>
# include <sys/atomic_op.h>
# define MDBX_INLINE
#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
# include <machine/builtins.h>
# include <c_asm.h>
# define MDBX_INLINE
#elif defined(__MWERKS__)
/* CodeWarrior - troubles ? */
# pragma gcc_extensions
# define MDBX_INLINE
#elif defined(__SNC__)
/* Sony PS3 - troubles ? */
# define MDBX_INLINE
#else
# define MDBX_INLINE
#endif
#if defined(__i386__) || defined(__x86_64__) \
|| defined(_M_AMD64) || defined(_M_IX86) \
|| defined(__i386) || defined(__amd64) \
|| defined(i386) || defined(__x86_64) \
|| defined(_AMD64_) || defined(_M_X64)
# define MDB_CACHE_IS_COHERENT 1
#elif defined(__hppa) || defined(__hppa__)
# define MDB_CACHE_IS_COHERENT 1
#endif
#ifndef MDB_CACHE_IS_COHERENT
# define MDB_CACHE_IS_COHERENT 0
#endif
#define MDBX_BARRIER_COMPILER 0
#define MDBX_BARRIER_MEMORY 1
static MDBX_INLINE void mdbx_barrier(int type) {
#if defined(__clang__)
__asm__ __volatile__ ("" ::: "memory");
if (type > MDBX_BARRIER_COMPILER)
# if __has_extension(c_atomic) || __has_extension(cxx_atomic)
__c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
# else
__sync_synchronize();
# endif
#elif defined(__GNUC__)
__asm__ __volatile__ ("" ::: "memory");
if (type > MDBX_BARRIER_COMPILER)
# if defined(__ATOMIC_SEQ_CST)
__atomic_thread_fence(__ATOMIC_SEQ_CST);
# else
__sync_synchronize();
# endif
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
__memory_barrier();
if (type > MDBX_BARRIER_COMPILER)
# if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
__mf();
# elif defined(__i386__) || defined(__x86_64__)
_mm_mfence();
# else
# error "Unknown target for Intel Compiler, please report to us."
# endif
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
__compiler_barrier();
if (type > MDBX_BARRIER_COMPILER)
__machine_rw_barrier();
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) \
&& (defined(HP_IA64) || defined(__ia64))
_Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
if (type > MDBX_BARRIER_COMPILER)
_Asm_mf();
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) \
|| defined(__ppc64__) || defined(__powerpc64__)
__fence();
if (type > MDBX_BARRIER_COMPILER)
__lwsync();
#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
__PAL_DRAINA(); /* LY: excessive ? */
__MB();
#else
# error "Could not guess the kind of compiler, please report to us."
#endif
}
#define mdbx_compiler_barrier() \
mdbx_barrier(MDBX_BARRIER_COMPILER)
#define mdbx_memory_barrier() \
mdbx_barrier(MDBX_BARRIER_MEMORY)
#define mdbx_coherent_barrier() \
mdbx_barrier(MDB_CACHE_IS_COHERENT ? MDBX_BARRIER_COMPILER : MDBX_BARRIER_MEMORY)
static MDBX_INLINE void mdb_invalidate_cache(void *addr, int nbytes) {
mdbx_coherent_barrier();
#if defined(__mips) && defined(__linux)
/* MIPS has cache coherency issues.
* Note: for any nbytes >= on-chip cache size, entire is flushed. */
cacheflush(addr, nbytes, DCACHE);
#elif defined(_M_MRX000) || defined(_MIPS_)
# error "Sorry, cacheflush() for MIPS not implemented"
#else
/* LY: assume no mmap/dcache issues. */
(void) addr;
(void) nbytes;
#endif
}

4
circle.yml
View File

@ -7,8 +7,8 @@ database:
compile:
override:
- make all lmdb
- make all
test:
override:
- make check
- make check || mv test.log ${CIRCLE_ARTIFACTS}/

336
defs.h
View File

@ -1,336 +0,0 @@
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#ifndef __GNUC_PREREQ
# if defined(__GNUC__) && defined(__GNUC_MINOR__)
# define __GNUC_PREREQ(maj, min) \
((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
# else
# define __GNUC_PREREQ(maj, min) (0)
# endif
#endif /* __GNUC_PREREQ */
#ifndef __CLANG_PREREQ
# ifdef __clang__
# define __CLANG_PREREQ(maj,min) \
((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
# else
# define __CLANG_PREREQ(maj,min) (0)
# endif
#endif /* __CLANG_PREREQ */
#ifndef __GLIBC_PREREQ
# if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
# define __GLIBC_PREREQ(maj, min) \
((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
# else
# define __GLIBC_PREREQ(maj, min) (0)
# endif
#endif /* __GLIBC_PREREQ */
#ifndef __has_attribute
# define __has_attribute(x) (0)
#endif
#ifndef __has_feature
# define __has_feature(x) (0)
#endif
#ifndef __has_extension
# define __has_extension(x) (0)
#endif
#ifndef __has_builtin
# define __has_builtin(x) (0)
#endif
#if __has_feature(thread_sanitizer)
# define __SANITIZE_THREAD__ 1
#endif
#if __has_feature(address_sanitizer)
# define __SANITIZE_ADDRESS__ 1
#endif
/*----------------------------------------------------------------------------*/
#ifndef __extern_C
# ifdef __cplusplus
# define __extern_C extern "C"
# else
# define __extern_C
# endif
#endif /* __extern_C */
#ifndef __cplusplus
# ifndef bool
# define bool _Bool
# endif
# ifndef true
# define true (1)
# endif
# ifndef false
# define false (0)
# endif
#endif
#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
# define nullptr NULL
#endif
/*----------------------------------------------------------------------------*/
#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__))
# define __thread __declspec(thread)
#endif /* __thread */
#ifndef __alwaysinline
# if defined(__GNUC__) || __has_attribute(always_inline)
# define __alwaysinline __inline __attribute__((always_inline))
# elif defined(_MSC_VER)
# define __alwaysinline __forceinline
# else
# define __alwaysinline
# endif
#endif /* __alwaysinline */
#ifndef __noinline
# if defined(__GNUC__) || __has_attribute(noinline)
# define __noinline __attribute__((noinline))
# elif defined(_MSC_VER)
# define __noinline __declspec(noinline)
# elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
# define __noinline inline
# elif !defined(__INTEL_COMPILER)
# define __noinline /* FIXME ? */
# endif
#endif /* __noinline */
#ifndef __must_check_result
# if defined(__GNUC__) || __has_attribute(warn_unused_result)
# define __must_check_result __attribute__((warn_unused_result))
# else
# define __must_check_result
# endif
#endif /* __must_check_result */
#ifndef __deprecated
# if defined(__GNUC__) || __has_attribute(deprecated)
# define __deprecated __attribute__((deprecated))
# elif defined(_MSC_VER)
# define __deprecated __declspec(deprecated)
# else
# define __deprecated
# endif
#endif /* __deprecated */
#ifndef __packed
# if defined(__GNUC__) || __has_attribute(packed)
# define __packed __attribute__((packed))
# else
# define __packed
# endif
#endif /* __packed */
#ifndef __aligned
# if defined(__GNUC__) || __has_attribute(aligned)
# define __aligned(N) __attribute__((aligned(N)))
# elif defined(_MSC_VER)
# define __aligned(N) __declspec(align(N))
# else
# define __aligned(N)
# endif
#endif /* __aligned */
#ifndef __noreturn
# if defined(__GNUC__) || __has_attribute(noreturn)
# define __noreturn __attribute__((noreturn))
# elif defined(_MSC_VER)
# define __noreturn __declspec(noreturn)
# else
# define __noreturn
# endif
#endif /* __noreturn */
#ifndef __nothrow
# if defined(__GNUC__) || __has_attribute(nothrow)
# define __nothrow __attribute__((nothrow))
# elif defined(_MSC_VER) && defined(__cplusplus)
# define __nothrow __declspec(nothrow)
# else
# define __nothrow
# endif
#endif /* __nothrow */
#ifndef __pure_function
/* Many functions have no effects except the return value and their
* return value depends only on the parameters and/or global variables.
* Such a function can be subject to common subexpression elimination
* and loop optimization just as an arithmetic operator would be.
* These functions should be declared with the attribute pure. */
# if defined(__GNUC__) || __has_attribute(pure)
# define __pure_function __attribute__((pure))
# else
# define __pure_function
# endif
#endif /* __pure_function */
#ifndef __const_function
/* Many functions do not examine any values except their arguments,
* and have no effects except the return value. Basically this is just
* slightly more strict class than the PURE attribute, since function
* is not allowed to read global memory.
*
* Note that a function that has pointer arguments and examines the
* data pointed to must not be declared const. Likewise, a function
* that calls a non-const function usually must not be const.
* It does not make sense for a const function to return void. */
# if defined(__GNUC__) || __has_attribute(const)
# define __const_function __attribute__((const))
# else
# define __const_function
# endif
#endif /* __const_function */
#ifndef __dll_hidden
# if defined(__GNUC__) || __has_attribute(visibility)
# define __hidden __attribute__((visibility("hidden")))
# else
# define __hidden
# endif
#endif /* __dll_hidden */
#ifndef __optimize
# if defined(__OPTIMIZE__)
# if defined(__clang__) && !__has_attribute(optimize)
# define __optimize(ops)
# elif defined(__GNUC__) || __has_attribute(optimize)
# define __optimize(ops) __attribute__((optimize(ops)))
# else
# define __optimize(ops)
# endif
# else
# define __optimize(ops)
# endif
#endif /* __optimize */
#ifndef __hot
# if defined(__OPTIMIZE__)
# if defined(__clang__) && !__has_attribute(hot)
/* just put frequently used functions in separate section */
# define __hot __attribute__((section("text.hot"))) __optimize("O3")
# elif defined(__GNUC__) || __has_attribute(hot)
# define __hot __attribute__((hot)) __optimize("O3")
# else
# define __hot __optimize("O3")
# endif
# else
# define __hot
# endif
#endif /* __hot */
#ifndef __cold
# if defined(__OPTIMIZE__)
# if defined(__clang__) && !__has_attribute(cold)
/* just put infrequently used functions in separate section */
# define __cold __attribute__((section("text.unlikely"))) __optimize("Os")
# elif defined(__GNUC__) || __has_attribute(cold)
# define __cold __attribute__((cold)) __optimize("Os")
# else
# define __cold __optimize("Os")
# endif
# else
# define __cold
# endif
#endif /* __cold */
#ifndef __flatten
# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(flatten))
# define __flatten __attribute__((flatten))
# else
# define __flatten
# endif
#endif /* __flatten */
#ifndef likely
# if defined(__GNUC__) || defined(__clang__)
# define likely(cond) __builtin_expect(!!(cond), 1)
# else
# define likely(x) (x)
# endif
#endif /* likely */
#ifndef unlikely
# if defined(__GNUC__) || defined(__clang__)
# define unlikely(cond) __builtin_expect(!!(cond), 0)
# else
# define unlikely(x) (x)
# endif
#endif /* unlikely */
/*----------------------------------------------------------------------------*/
/* Wrapper around __func__, which is a C99 feature */
#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
# define mdbx_func_ __func__
#elif (defined(__GNUC__) && __GNUC__ >= 2) || defined(__clang__) || defined(_MSC_VER)
# define mdbx_func_ __FUNCTION__
#else
# define mdbx_func_ "<mdbx_unknown>"
#endif
/* -------------------------------------------------------------------------- */
#if defined(HAVE_VALGRIND) || defined(USE_VALGRIND)
/* Get debugging help from Valgrind */
# include <valgrind/memcheck.h>
# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE
/* LY: available since Valgrind 3.10 */
# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# endif
#else
# define VALGRIND_CREATE_MEMPOOL(h,r,z)
# define VALGRIND_DESTROY_MEMPOOL(h)
# define VALGRIND_MEMPOOL_TRIM(h,a,s)
# define VALGRIND_MEMPOOL_ALLOC(h,a,s)
# define VALGRIND_MEMPOOL_FREE(h,a)
# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
# define VALGRIND_MAKE_MEM_NOACCESS(a,s)
# define VALGRIND_MAKE_MEM_DEFINED(a,s)
# define VALGRIND_MAKE_MEM_UNDEFINED(a,s)
# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0)
# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0)
#endif /* ! USE_VALGRIND */
#ifdef __SANITIZE_THREAD__
# define ATTRIBUTE_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread, noinline))
#else
# define ATTRIBUTE_NO_SANITIZE_THREAD
#endif
#ifdef __SANITIZE_ADDRESS__
# include <sanitizer/asan_interface.h>
# define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address, noinline))
#else
# define ASAN_POISON_MEMORY_REGION(addr, size) \
((void)(addr), (void)(size))
# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
((void)(addr), (void)(size))
# define ATTRIBUTE_NO_SANITIZE_ADDRESS
#endif /* __SANITIZE_ADDRESS__ */

181
dll.vcxproj Normal file
View File

@ -0,0 +1,181 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{6D19209B-ECE7-4B9C-941C-0AA2B484F199}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<TargetName>mdbx</TargetName>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<TargetName>mdbx</TargetName>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<TargetName>mdbx</TargetName>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<TargetName>mdbx</TargetName>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PreprocessorDefinitions>WIN32;_DEBUG;_WINDOWS;_USRDLL;LIBMDBX_EXPORTS;%(PreprocessorDefinitions);MDBX_DEBUG=1</PreprocessorDefinitions>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
<WarningLevel>Level4</WarningLevel>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<Optimization>Disabled</Optimization>
<StringPooling>true</StringPooling>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<TargetMachine>MachineX86</TargetMachine>
<GenerateDebugInformation>true</GenerateDebugInformation>
<SubSystem>Windows</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;_USRDLL;LIBMDBX_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
<WarningLevel>Level4</WarningLevel>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<StringPooling>true</StringPooling>
<Optimization>Full</Optimization>
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
<IntrinsicFunctions>true</IntrinsicFunctions>
<FavorSizeOrSpeed>Size</FavorSizeOrSpeed>
<OmitFramePointers>true</OmitFramePointers>
<WholeProgramOptimization>true</WholeProgramOptimization>
</ClCompile>
<Link>
<TargetMachine>MachineX86</TargetMachine>
<GenerateDebugInformation>true</GenerateDebugInformation>
<SubSystem>Windows</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
</ClCompile>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PreprocessorDefinitions>WIN64;_DEBUG;_WINDOWS;_USRDLL;LIBMDBX_EXPORTS;%(PreprocessorDefinitions);MDBX_DEBUG=1</PreprocessorDefinitions>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
<StringPooling>true</StringPooling>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<PreprocessorDefinitions>WIN64;NDEBUG;_WINDOWS;_USRDLL;LIBMDBX_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
<StringPooling>true</StringPooling>
<Optimization>Full</Optimization>
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
<IntrinsicFunctions>true</IntrinsicFunctions>
<FavorSizeOrSpeed>Size</FavorSizeOrSpeed>
<OmitFramePointers>true</OmitFramePointers>
<WholeProgramOptimization>true</WholeProgramOptimization>
<WarningLevel>Level4</WarningLevel>
</ClCompile>
<Link>
<LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="src\lck-windows.c" />
<ClCompile Include="src\mdbx.c" />
<ClCompile Include="src\osal.c" />
<ClCompile Include="src\version.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="mdbx.h" />
<ClInclude Include="src\bits.h" />
<ClInclude Include="src\defs.h" />
<ClInclude Include="src\osal.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

195
intro.doc
View File

@ -1,195 +0,0 @@
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2015-2017 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/** @page starting Getting Started
LMDB is compact, fast, powerful, and robust and implements a simplified
variant of the BerkeleyDB (BDB) API. (BDB is also very powerful, and verbosely
documented in its own right.) After reading this page, the main
\ref mdb documentation should make sense. Thanks to Bert Hubert
for creating the
<a href="https://github.com/ahupowerdns/ahutils/blob/master/lmdb-semantics.md">
initial version</a> of this writeup.
Everything starts with an environment, created by #mdb_env_create().
Once created, this environment must also be opened with #mdb_env_open().
#mdb_env_open() gets passed a name which is interpreted as a directory
path. Note that this directory must exist already, it is not created
for you. Within that directory, a lock file and a storage file will be
generated. If you don't want to use a directory, you can pass the
#MDB_NOSUBDIR option, in which case the path you provided is used
directly as the data file, and another file with a "-lock" suffix
added will be used for the lock file.
Once the environment is open, a transaction can be created within it
using #mdb_txn_begin(). Transactions may be read-write or read-only,
and read-write transactions may be nested. A transaction must only
be used by one thread at a time. Transactions are always required,
even for read-only access. The transaction provides a consistent
view of the data.
Once a transaction has been created, a database can be opened within it
using #mdb_dbi_open(). If only one database will ever be used in the
environment, a NULL can be passed as the database name. For named
databases, the #MDB_CREATE flag must be used to create the database
if it doesn't already exist. Also, #mdb_env_set_maxdbs() must be
called after #mdb_env_create() and before #mdb_env_open() to set the
maximum number of named databases you want to support.
Note: a single transaction can open multiple databases. Generally
databases should only be opened once, by the first transaction in
the process. After the first transaction completes, the database
handles can freely be used by all subsequent transactions.
Within a transaction, #mdb_get() can retrieve and #mdb_put() can store single
key/value pairs if that is all you need to do (but see \ref Cursors
below if you want to do more).
A key/value pair is expressed as two #MDB_val structures. This struct
has two fields, \c mv_size and \c mv_data. The data is a \c void pointer to
an array of \c mv_size bytes.
Because LMDB is very efficient (and usually zero-copy), the data returned
in an #MDB_val structure may be memory-mapped straight from disk. In
other words <b>look but do not touch</b> (or free() for that matter).
Once a transaction is closed, the values can no longer be used, so
make a copy if you need to keep them after that.
@section Cursors Cursors
To do more powerful things, we must use a cursor.
Within the transaction, a cursor can be created with #mdb_cursor_open().
With this cursor we can store/retrieve/delete (multiple) values using
#mdb_cursor_get(), #mdb_cursor_put(), and #mdb_cursor_del().
#mdb_cursor_get() positions itself depending on the cursor operation
requested, and for some operations, on the supplied key. For example,
to list all key/value pairs in a database, use operation #MDB_FIRST for
the first call to #mdb_cursor_get(), and #MDB_NEXT on subsequent calls,
until the end is hit.
To retrieve all keys starting from a specified key value, use #MDB_SET.
For more cursor operations, see the \ref mdb docs.
When using #mdb_cursor_put(), either the function will position the
cursor for you based on the \b key, or you can use operation
#MDB_CURRENT to use the current position of the cursor. Note that
\b key must then match the current position's key.
@subsection summary Summarizing the Opening
So we have a cursor in a transaction which opened a database in an
environment which is opened from a filesystem after it was
separately created.
Or, we create an environment, open it from a filesystem, create a
transaction within it, open a database within that transaction,
and create a cursor within all of the above.
Got it?
@section thrproc Threads and Processes
LMDB uses POSIX locks on files, and these locks have issues if one
process opens a file multiple times. Because of this, do not
#mdb_env_open() a file multiple times from a single process. Instead,
share the LMDB environment that has opened the file across all threads.
Otherwise, if a single process opens the same environment multiple times,
closing it once will remove all the locks held on it, and the other
instances will be vulnerable to corruption from other processes.
Also note that a transaction is tied to one thread by default using
Thread Local Storage. If you want to pass read-only transactions across
threads, you can use the #MDB_NOTLS option on the environment.
@section txns Transactions, Rollbacks, etc.
To actually get anything done, a transaction must be committed using
#mdb_txn_commit(). Alternatively, all of a transaction's operations
can be discarded using #mdb_txn_abort(). In a read-only transaction,
any cursors will \b not automatically be freed. In a read-write
transaction, all cursors will be freed and must not be used again.
For read-only transactions, obviously there is nothing to commit to
storage. The transaction still must eventually be aborted to close
any database handle(s) opened in it, or committed to keep the
database handles around for reuse in new transactions.
In addition, as long as a transaction is open, a consistent view of
the database is kept alive, which requires storage. A read-only
transaction that no longer requires this consistent view should
be terminated (committed or aborted) when the view is no longer
needed (but see below for an optimization).
There can be multiple simultaneously active read-only transactions
but only one that can write. Once a single read-write transaction
is opened, all further attempts to begin one will block until the
first one is committed or aborted. This has no effect on read-only
transactions, however, and they may continue to be opened at any time.
@section dupkeys Duplicate Keys
#mdb_get() and #mdb_put() respectively have no and only some support
for multiple key/value pairs with identical keys. If there are multiple
values for a key, #mdb_get() will only return the first value.
When multiple values for one key are required, pass the #MDB_DUPSORT
flag to #mdb_dbi_open(). In an #MDB_DUPSORT database, by default
#mdb_put() will not replace the value for a key if the key existed
already. Instead it will add the new value to the key. In addition,
#mdb_del() will pay attention to the value field too, allowing for
specific values of a key to be deleted.
Finally, additional cursor operations become available for
traversing through and retrieving duplicate values.
@section optim Some Optimization
If you frequently begin and abort read-only transactions, as an
optimization, it is possible to only reset and renew a transaction.
#mdb_txn_reset() releases any old copies of data kept around for
a read-only transaction. To reuse this reset transaction, call
#mdb_txn_renew() on it. Any cursors in this transaction must also
be renewed using #mdb_cursor_renew().
Note that #mdb_txn_reset() is similar to #mdb_txn_abort() and will
close any databases you opened within the transaction.
To permanently free a transaction, reset or not, use #mdb_txn_abort().
@section cleanup Cleaning Up
For read-only transactions, any cursors created within it must
be closed using #mdb_cursor_close().
It is very rarely necessary to close a database handle, and in
general they should just be left open.
@section onward The Full API
The full \ref mdb documentation lists further details, like how to:
\li size a database (the default limits are intentionally small)
\li drop and clean a database
\li detect and report errors
\li optimize (bulk) loading speed
\li (temporarily) reduce robustness to gain even more speed
\li gather statistics about the database
\li define custom sort orders
*/

2
libmdbx.config Normal file
View File

@ -0,0 +1,2 @@
// Add predefined macros for your project here. For example:
// #define THE_ANSWER 42

1
libmdbx.creator Normal file
View File

@ -0,0 +1 @@
[General]

48
libmdbx.files Normal file
View File

@ -0,0 +1,48 @@
AUTHORS
LICENSE
Makefile
README.md
TODO.md
mdbx.h
src/bits.h
src/defs.h
src/lck-posix.c
src/lck-windows.c
src/mdbx.c
src/osal.c
src/osal.h
src/tools/mdbx_chk.c
src/tools/mdbx_copy.1
src/tools/mdbx_copy.c
src/tools/mdbx_dump.1
src/tools/mdbx_dump.c
src/tools/mdbx_load.1
src/tools/mdbx_load.c
src/tools/mdbx_stat.1
src/tools/mdbx_stat.c
src/version.c
test/actor.cc
test/base.h
test/chrono.cc
test/chrono.h
test/config.h
test/dead.cc
test/hill.cc
test/jitter.cc
test/keygen.cc
test/keygen.h
test/log.cc
test/log.h
test/main.cc
test/config.cc
test/cases.cc
test/osal-unix.cc
test/osal-windows.cc
test/osal.h
test/test.cc
test/test.h
test/utils.cc
test/utils.h
tutorial/README.md
tutorial/sample-bdb.txt
tutorial/sample-mdb.txt

4
libmdbx.includes Normal file
View File

@ -0,0 +1,4 @@
.
src
src/tools
test

1850
lmdb.h
View File

File diff suppressed because it is too large Load Diff

10765
mdb.c
View File

File diff suppressed because it is too large Load Diff

954
mdb_chk.c
View File

@ -1,954 +0,0 @@
/* mdbx_chk.c - memory-mapped database check tool */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2015,2016 Peter-Service R&D LLC.
*
* This file is part of libmdbx.
*
* ReOpenMDBX is free software; you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* ReOpenMDBX is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <unistd.h>
#include <signal.h>
#include <stdarg.h>
#include <malloc.h>
#include <time.h>
#include "midl.h"
#include "mdbx.h"
typedef struct flagbit {
int bit;
char *name;
} flagbit;
flagbit dbflags[] = {
{ MDB_DUPSORT, "dupsort" },
{ MDB_INTEGERKEY, "integerkey" },
{ MDB_REVERSEKEY, "reversekey" },
{ MDB_DUPFIXED, "dupfixed" },
{ MDB_REVERSEDUP, "reversedup" },
{ MDB_INTEGERDUP, "integerdup" },
{ 0, NULL }
};
static volatile sig_atomic_t gotsignal;
static void signal_handler( int sig ) {
(void) sig;
gotsignal = 1;
}
#define MAX_DBI 32768
#define EXIT_INTERRUPTED (EXIT_FAILURE+4)
#define EXIT_FAILURE_SYS (EXIT_FAILURE+3)
#define EXIT_FAILURE_MDB (EXIT_FAILURE+2)
#define EXIT_FAILURE_CHECK_MAJOR (EXIT_FAILURE+1)
#define EXIT_FAILURE_CHECK_MINOR EXIT_FAILURE
struct {
const char* dbi_names[MAX_DBI];
size_t dbi_pages[MAX_DBI];
size_t dbi_empty_pages[MAX_DBI];
size_t dbi_payload_bytes[MAX_DBI];
size_t dbi_lost_bytes[MAX_DBI];
short *pagemap;
size_t total_payload_bytes;
size_t pgcount;
} walk;
static __attribute__((constructor))
void init_walk(void)
{
walk.dbi_names[0] = "@gc";
}
size_t total_unused_bytes;
int exclusive = 2;
MDB_env *env;
MDB_txn *txn, *locktxn;
MDBX_envinfo info;
MDBX_stat stat;
size_t maxkeysize, reclaimable_pages, freedb_pages, lastpgno;
size_t userdb_count, skipped_subdb;
unsigned verbose, quiet;
const char* only_subdb;
struct problem {
struct problem* pr_next;
size_t count;
const char* caption;
};
struct problem* problems_list;
size_t total_problems;
static void __attribute__ ((format (printf, 1, 2)))
print(const char* msg, ...) {
if (! quiet) {
va_list args;
fflush(stderr);
va_start(args, msg);
vfprintf(stdout, msg, args);
va_end(args);
}
}
static void __attribute__ ((format (printf, 1, 2)))
error(const char* msg, ...) {
total_problems++;
if (! quiet) {
va_list args;
fflush(stdout);
va_start(args, msg);
vfprintf(stderr, msg, args);
va_end(args);
fflush(NULL);
}
}
static void pagemap_cleanup(void) {
int i;
for( i = 1; i < MAX_DBI; ++i ) {
if (walk.dbi_names[i]) {
free((void *) walk.dbi_names[i]);
walk.dbi_names[i] = NULL;
}
}
free(walk.pagemap);
walk.pagemap = NULL;
}
static int pagemap_lookup_dbi(const char* dbi) {
static int last;
int i;
if (last > 0 && strcmp(walk.dbi_names[last], dbi) == 0)
return last;
for(i = 1; walk.dbi_names[i] && last < MAX_DBI; ++i)
if (strcmp(walk.dbi_names[i], dbi) == 0)
return last = i;
if (i == MAX_DBI)
return -1;
walk.dbi_names[i] = strdup(dbi);
if (verbose > 1) {
print(" - found '%s' area\n", dbi);
fflush(NULL);
}
return last = i;
}
static void problem_add(const char* object, size_t entry_number, const char* msg, const char *extra, ...) {
total_problems++;
if (! quiet) {
int need_fflush = 0;
struct problem* p;
for (p = problems_list; p; p = p->pr_next)
if (p->caption == msg)
break;
if (! p) {
p = calloc(1, sizeof(*p));
p->caption = msg;
p->pr_next = problems_list;
problems_list = p;
need_fflush = 1;
}
p->count++;
if (verbose > 1) {
print(" %s #%zu: %s", object, entry_number, msg);
if (extra) {
va_list args;
printf(" (");
va_start(args, extra);
vfprintf(stdout, extra, args);
va_end(args);
printf(")");
}
printf("\n");
if (need_fflush)
fflush(NULL);
}
}
}
static struct problem* problems_push() {
struct problem* p = problems_list;
problems_list = NULL;
return p;
}
static size_t problems_pop(struct problem* list) {
size_t count = 0;
if (problems_list) {
int i;
print(" - problems: ");
for (i = 0; problems_list; ++i) {
struct problem* p = problems_list->pr_next;
count += problems_list->count;
print("%s%s (%zu)", i ? ", " : "", problems_list->caption, problems_list->count);
free(problems_list);
problems_list = p;
}
print("\n");
fflush(NULL);
}
problems_list = list;
return count;
}
static int pgvisitor(size_t pgno, unsigned pgnumber, void* ctx, const char* dbi,
const char* type, int nentries, int payload_bytes, int header_bytes, int unused_bytes)
{
(void) ctx;
if (type) {
size_t page_bytes = payload_bytes + header_bytes + unused_bytes;
size_t page_size = pgnumber * stat.base.ms_psize;
int index = pagemap_lookup_dbi(dbi);
if (index < 0)
return ENOMEM;
if (verbose > 2 && (!only_subdb || strcmp(only_subdb, dbi) == 0)) {
if (pgnumber == 1)
print(" %s-page %zu", type, pgno);
else
print(" %s-span %zu[%u]", type, pgno, pgnumber);
print(" of %s: header %i, payload %i, unused %i\n",
dbi, header_bytes, payload_bytes, unused_bytes);
}
walk.pgcount += pgnumber;
if (unused_bytes < 0 || (size_t) unused_bytes > page_size)
problem_add("page", pgno, "illegal unused-bytes", "%zu < %i < %zu",
0, unused_bytes, stat.base.ms_psize);
if (header_bytes < (int) sizeof(long) || (size_t) header_bytes >= stat.base.ms_psize - sizeof(long))
problem_add("page", pgno, "illegal header-length", "%zu < %i < %zu",
sizeof(long), header_bytes, stat.base.ms_psize - sizeof(long));
if (payload_bytes < 1) {
if (nentries > 1) {
problem_add("page", pgno, "zero size-of-entry", "payload %i bytes, %i entries",
payload_bytes, nentries);
if ((size_t) header_bytes + unused_bytes < page_size) {
/* LY: hush a misuse error */
page_bytes = page_size;
}
} else {
problem_add("page", pgno, "empty", "payload %i bytes, %i entries",
payload_bytes, nentries);
walk.dbi_empty_pages[index] += 1;
}
}
if (page_bytes != page_size) {
problem_add("page", pgno, "misused", "%zu != %zu (%ih + %ip + %iu)",
page_size, page_bytes, header_bytes, payload_bytes, unused_bytes);
if (page_size > page_bytes)
walk.dbi_lost_bytes[index] += page_size - page_bytes;
} else {
walk.dbi_payload_bytes[index] += payload_bytes + header_bytes;
walk.total_payload_bytes += payload_bytes + header_bytes;
}
if (pgnumber) {
do {
if (pgno >= lastpgno)
problem_add("page", pgno, "wrong page-no",
"%zu > %zi", pgno, lastpgno);
else if (walk.pagemap[pgno])
problem_add("page", pgno, "already used",
"in %s", walk.dbi_names[walk.pagemap[pgno]]);
else {
walk.pagemap[pgno] = index;
walk.dbi_pages[index] += 1;
}
++pgno;
} while(--pgnumber);
}
}
return gotsignal ? EINTR : MDB_SUCCESS;
}
typedef int (visitor)(size_t record_number, MDB_val *key, MDB_val* data);
static int process_db(MDB_dbi dbi, char *name, visitor *handler, int silent);
static int handle_userdb(size_t record_number, MDB_val *key, MDB_val* data) {
(void) record_number;
(void) key;
(void) data;
return MDB_SUCCESS;
}
static int handle_freedb(size_t record_number, MDB_val *key, MDB_val* data) {
char *bad = "";
size_t pg, prev;
ssize_t i, number, span = 0;
size_t *iptr = data->mv_data, txnid = *(size_t*)key->mv_data;
if (key->mv_size != sizeof(txnid))
problem_add("entry", record_number, "wrong txn-id size", "key-size %zi", key->mv_size);
else if (txnid < 1 || txnid > info.base.me_last_txnid)
problem_add("entry", record_number, "wrong txn-id", "%zu", txnid);
if (data->mv_size < sizeof(size_t) || data->mv_size % sizeof(size_t))
problem_add("entry", record_number, "wrong idl size", "%zu", data->mv_size);
else {
number = *iptr++;
if (number >= MDB_IDL_UM_MAX)
problem_add("entry", record_number, "wrong idl length", "%zi", number);
else if ((number + 1) * sizeof(size_t) != data->mv_size)
problem_add("entry", record_number, "mismatch idl length", "%zi != %zu",
number * sizeof(size_t), data->mv_size);
else {
freedb_pages += number;
if (info.me_tail_txnid > txnid)
reclaimable_pages += number;
for (i = number, prev = 1; --i >= 0; ) {
pg = iptr[i];
if (pg < 2 /* META_PAGE */ || pg > info.base.me_last_pgno)
problem_add("entry", record_number, "wrong idl entry", "2 < %zi < %zi",
pg, info.base.me_last_pgno);
else if (pg <= prev) {
bad = " [bad sequence]";
problem_add("entry", record_number, "bad sequence", "%zi <= %zi",
pg, prev);
}
prev = pg;
pg += span;
for (; i >= span && iptr[i - span] == pg; span++, pg++) ;
}
if (verbose > 2 && !only_subdb) {
print(" transaction %zu, %zd pages, maxspan %zd%s\n",
*(size_t *)key->mv_data, number, span, bad);
if (verbose > 3) {
int j = number - 1;
while (j >= 0) {
pg = iptr[j];
for (span = 1; --j >= 0 && iptr[j] == pg + span; span++) ;
if (span > 1)
print(" %9zu[%zd]\n", pg, span);
else
print(" %9zu\n", pg);
}
}
}
}
}
return MDB_SUCCESS;
}
static int handle_maindb(size_t record_number, MDB_val *key, MDB_val* data) {
char *name;
int rc;
size_t i;
name = key->mv_data;
for(i = 0; i < key->mv_size; ++i) {
if (name[i] < ' ')
return handle_userdb(record_number, key, data);
}
name = malloc(key->mv_size + 1);
memcpy(name, key->mv_data, key->mv_size);
name[key->mv_size] = '\0';
userdb_count++;
rc = process_db(-1, name, handle_userdb, 0);
free(name);
if (rc != MDB_INCOMPATIBLE)
return rc;
return handle_userdb(record_number, key, data);
}
static int process_db(MDB_dbi dbi, char *name, visitor *handler, int silent)
{
MDB_cursor *mc;
MDBX_stat ms;
MDB_val key, data;
MDB_val prev_key, prev_data;
unsigned flags;
int rc, i;
struct problem* saved_list;
size_t problems_count;
unsigned record_count = 0, dups = 0;
size_t key_bytes = 0, data_bytes = 0;
if (0 > (int) dbi) {
rc = mdbx_dbi_open(txn, name, 0, &dbi);
if (rc) {
if (!name || rc != MDB_INCOMPATIBLE) /* LY: mainDB's record is not a user's DB. */ {
error(" - mdbx_open '%s' failed, error %d %s\n",
name ? name : "main", rc, mdbx_strerror(rc));
}
return rc;
}
}
if (dbi >= 2 /* CORE_DBS */ && name && only_subdb && strcmp(only_subdb, name)) {
if (verbose) {
print("Skip processing '%s'...\n", name);
fflush(NULL);
}
skipped_subdb++;
return MDB_SUCCESS;
}
if (! silent && verbose) {
print("Processing '%s'...\n", name ? name : "main");
fflush(NULL);
}
rc = mdbx_dbi_flags(txn, dbi, &flags);
if (rc) {
error(" - mdbx_dbi_flags failed, error %d %s\n", rc, mdbx_strerror(rc));
return rc;
}
rc = mdbx_stat(txn, dbi, &ms, sizeof(ms));
if (rc) {
error(" - mdbx_stat failed, error %d %s\n", rc, mdbx_strerror(rc));
return rc;
}
if (! silent && verbose) {
print(" - dbi-id %d, flags:", dbi);
if (! flags)
print(" none");
else {
for (i=0; dbflags[i].bit; i++)
if (flags & dbflags[i].bit)
print(" %s", dbflags[i].name);
}
print(" (0x%02X)\n", flags);
if (verbose > 1) {
print(" - page size %u, entries %zu\n", ms.base.ms_psize, ms.base.ms_entries);
print(" - b-tree depth %u, pages: branch %zu, leaf %zu, overflow %zu\n",
ms.base.ms_depth, ms.base.ms_branch_pages, ms.base.ms_leaf_pages, ms.base.ms_overflow_pages);
}
}
rc = mdbx_cursor_open(txn, dbi, &mc);
if (rc) {
error(" - mdbx_cursor_open failed, error %d %s\n", rc, mdbx_strerror(rc));
return rc;
}
saved_list = problems_push();
prev_key.mv_data = NULL;
prev_data.mv_size = 0;
rc = mdbx_cursor_get(mc, &key, &data, MDB_FIRST);
while (rc == MDB_SUCCESS) {
if (gotsignal) {
print(" - interrupted by signal\n");
fflush(NULL);
rc = EINTR;
goto bailout;
}
if (key.mv_size > maxkeysize) {
problem_add("entry", record_count, "key length exceeds max-key-size",
"%zu > %zu", key.mv_size, maxkeysize);
} else if ((flags & MDB_INTEGERKEY)
&& key.mv_size != sizeof(size_t) && key.mv_size != sizeof(int)) {
problem_add("entry", record_count, "wrong key length",
"%zu != %zu", key.mv_size, sizeof(size_t));
}
if ((flags & MDB_INTEGERDUP)
&& data.mv_size != sizeof(size_t) && data.mv_size != sizeof(int)) {
problem_add("entry", record_count, "wrong data length",
"%zu != %zu", data.mv_size, sizeof(size_t));
}
if (prev_key.mv_data) {
if ((flags & MDB_DUPFIXED) && prev_data.mv_size != data.mv_size) {
problem_add("entry", record_count, "different data length",
"%zu != %zu", prev_data.mv_size, data.mv_size);
}
int cmp = mdbx_cmp(txn, dbi, &prev_key, &key);
if (cmp > 0) {
problem_add("entry", record_count, "broken ordering of entries", NULL);
} else if (cmp == 0) {
++dups;
if (! (flags & MDB_DUPSORT))
problem_add("entry", record_count, "duplicated entries", NULL);
else if (flags & MDB_INTEGERDUP) {
cmp = mdbx_dcmp(txn, dbi, &prev_data, &data);
if (cmp > 0)
problem_add("entry", record_count, "broken ordering of multi-values", NULL);
}
}
} else if (verbose) {
if (flags & MDB_INTEGERKEY)
print(" - fixed key-size %zu\n", key.mv_size );
if (flags & (MDB_INTEGERDUP | MDB_DUPFIXED))
print(" - fixed data-size %zu\n", data.mv_size );
}
if (handler) {
rc = handler(record_count, &key, &data);
if (rc)
goto bailout;
}
record_count++;
key_bytes += key.mv_size;
data_bytes += data.mv_size;
prev_key = key;
prev_data = data;
rc = mdbx_cursor_get(mc, &key, &data, MDB_NEXT);
}
if (rc != MDB_NOTFOUND)
error(" - mdbx_cursor_get failed, error %d %s\n", rc, mdbx_strerror(rc));
else
rc = 0;
if (record_count != ms.base.ms_entries)
problem_add("entry", record_count, "differentent number of entries",
"%zu != %zu", record_count, ms.base.ms_entries);
bailout:
problems_count = problems_pop(saved_list);
if (! silent && verbose) {
print(" - summary: %u records, %u dups, %zu key's bytes, %zu data's bytes, %zu problems\n",
record_count, dups, key_bytes, data_bytes, problems_count);
fflush(NULL);
}
mdbx_cursor_close(mc);
return rc || problems_count;
}
static void usage(char *prog)
{
fprintf(stderr, "usage: %s dbpath [-V] [-v] [-n] [-q] [-w] [-c] [-d] [-s subdb]\n"
" -V\t\tshow version\n"
" -v\t\tmore verbose, could be used multiple times\n"
" -n\t\tNOSUBDIR mode for open\n"
" -q\t\tbe quiet\n"
" -w\t\tlock DB for writing while checking\n"
" -d\t\tdisable page-by-page traversal of b-tree\n"
" -s subdb\tprocess a specific subdatabase only\n"
" -c\t\tforce cooperative mode (don't try exclusive)\n", prog);
exit(EXIT_INTERRUPTED);
}
const char* meta_synctype(size_t sign) {
switch(sign) {
case 0:
return "no-sync/legacy";
case 1:
return "weak";
default:
return "steady";
}
}
int meta_lt(size_t txn1, size_t sign1, size_t txn2, size_t sign2) {
return ((sign1 > 1) == (sign2 > 1)) ? txn1 < txn2 : txn2 && sign2 > 1;
}
int main(int argc, char *argv[])
{
int i, rc;
char *prog = argv[0];
char *envname;
int envflags = MDB_RDONLY;
int problems_maindb = 0, problems_freedb = 0, problems_meta = 0;
int dont_traversal = 0;
size_t n;
struct timespec timestamp_start, timestamp_finish;
double elapsed;
atexit(pagemap_cleanup);
if (clock_gettime(CLOCK_MONOTONIC, &timestamp_start)) {
rc = errno;
error("clock_gettime failed, error %d %s\n", rc, mdbx_strerror(rc));
return EXIT_FAILURE_SYS;
}
if (argc < 2) {
usage(prog);
}
while ((i = getopt(argc, argv, "Vvqnwcds:")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(EXIT_SUCCESS);
break;
case 'v':
verbose++;
break;
case 'q':
quiet = 1;
break;
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 'w':
envflags &= ~MDB_RDONLY;
break;
case 'c':
exclusive = 0;
break;
case 'd':
dont_traversal = 1;
break;
case 's':
if (only_subdb && strcmp(only_subdb, optarg))
usage(prog);
only_subdb = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
envname = argv[optind];
print("Running mdbx_chk for '%s' in %s mode...\n",
envname, (envflags & MDB_RDONLY) ? "read-only" : "write-lock");
fflush(NULL);
rc = mdbx_env_create(&env);
if (rc) {
error("mdbx_env_create failed, error %d %s\n", rc, mdbx_strerror(rc));
return rc < 0 ? EXIT_FAILURE_MDB : EXIT_FAILURE_SYS;
}
rc = mdbx_env_get_maxkeysize(env);
if (rc < 0) {
error("mdbx_env_get_maxkeysize failed, error %d %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
maxkeysize = rc;
rc = mdbx_env_set_maxdbs(env, MAX_DBI);
if (rc < 0) {
error("mdbx_env_set_maxdbs failed, error %d %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_env_open_ex(env, envname, envflags, 0664, &exclusive);
if (rc) {
error("mdbx_env_open failed, error %d %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
if (verbose)
print(" - %s mode\n", exclusive ? "monopolistic" : "cooperative");
if (! (envflags & MDB_RDONLY)) {
rc = mdbx_txn_begin(env, NULL, 0, &locktxn);
if (rc) {
error("mdbx_txn_begin(lock-write) failed, error %d %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
}
rc = mdbx_txn_begin(env, NULL, MDB_RDONLY, &txn);
if (rc) {
error("mdbx_txn_begin(read-only) failed, error %d %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_env_info(env, &info, sizeof(info));
if (rc) {
error("mdbx_env_info failed, error %d %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_env_stat(env, &stat, sizeof(stat));
if (rc) {
error("mdbx_env_stat failed, error %d %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
lastpgno = info.base.me_last_pgno + 1;
errno = 0;
if (verbose) {
double k = 1024.0;
const char sf[] = "KMGTPEZY"; /* LY: Kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta! */
for(i = 0; sf[i+1] && info.base.me_mapsize / k > 1000.0; ++i)
k *= 1024;
print(" - map size %zu (%.2f %cb)\n", info.base.me_mapsize,
info.base.me_mapsize / k, sf[i]);
if (info.base.me_mapaddr)
print(" - mapaddr %p\n", info.base.me_mapaddr);
print(" - pagesize %u, max keysize %zu (%s), max readers %u\n",
stat.base.ms_psize, maxkeysize,
(maxkeysize == 511) ? "default" :
(maxkeysize == 0) ? "devel" : "custom",
info.base.me_maxreaders);
print(" - transactions: last %zu, bottom %zu, lag reading %zi\n", info.base.me_last_txnid,
info.me_tail_txnid, info.base.me_last_txnid - info.me_tail_txnid);
print(" - meta-1: %s %zu, %s",
meta_synctype(info.me_meta1_sign), info.me_meta1_txnid,
meta_lt(info.me_meta1_txnid, info.me_meta1_sign,
info.me_meta2_txnid, info.me_meta2_sign) ? "tail" : "head");
if (info.me_meta1_txnid > info.base.me_last_txnid)
print(", rolled-back %zu (%zu >>> %zu)",
info.me_meta1_txnid - info.base.me_last_txnid,
info.me_meta1_txnid, info.base.me_last_txnid);
print("\n");
print(" - meta-2: %s %zu, %s",
meta_synctype(info.me_meta2_sign), info.me_meta2_txnid,
meta_lt(info.me_meta2_txnid, info.me_meta2_sign,
info.me_meta1_txnid, info.me_meta1_sign) ? "tail" : "head");
if (info.me_meta2_txnid > info.base.me_last_txnid)
print(", rolled-back %zu (%zu >>> %zu)",
info.me_meta2_txnid - info.base.me_last_txnid,
info.me_meta2_txnid, info.base.me_last_txnid);
print("\n");
}
if (exclusive > 1) {
if (verbose)
print(" - perform full check last-txn-id with meta-pages\n");
if (! meta_lt(info.me_meta1_txnid, info.me_meta1_sign,
info.me_meta2_txnid, info.me_meta2_sign)
&& info.me_meta1_txnid != info.base.me_last_txnid) {
print(" - meta-1 txn-id mismatch last-txn-id (%zi != %zi)\n",
info.me_meta1_txnid, info.base.me_last_txnid);
++problems_meta;
}
if (! meta_lt(info.me_meta2_txnid, info.me_meta2_sign,
info.me_meta1_txnid, info.me_meta1_sign)
&& info.me_meta2_txnid != info.base.me_last_txnid) {
print(" - meta-2 txn-id mismatch last-txn-id (%zi != %zi)\n",
info.me_meta2_txnid, info.base.me_last_txnid);
++problems_meta;
}
} else if (locktxn) {
if (verbose)
print(" - perform lite check last-txn-id with meta-pages (not a monopolistic mode)\n");
size_t last = (info.me_meta2_txnid > info.me_meta1_txnid) ? info.me_meta2_txnid : info.me_meta1_txnid;
if (last != info.base.me_last_txnid) {
print(" - last-meta mismatch last-txn-id (%zi != %zi)\n",
last, info.base.me_last_txnid);
++problems_meta;
}
} else if (verbose) {
print(" - skip check last-txn-id with meta-pages (monopolistic or write-lock mode only)\n");
}
if (!dont_traversal) {
struct problem* saved_list;
size_t traversal_problems;
size_t empty_pages, lost_bytes;
print("Traversal b-tree...\n");
fflush(NULL);
walk.pagemap = calloc(lastpgno, sizeof(*walk.pagemap));
if (! walk.pagemap) {
rc = errno ? errno : ENOMEM;
error("calloc failed, error %d %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
saved_list = problems_push();
rc = mdbx_env_pgwalk(txn, pgvisitor, NULL);
traversal_problems = problems_pop(saved_list);
if (rc) {
if (rc == EINTR && gotsignal) {
print(" - interrupted by signal\n");
fflush(NULL);
} else {
error("mdbx_env_pgwalk failed, error %d %s\n", rc, mdbx_strerror(rc));
}
goto bailout;
}
for( n = 0; n < lastpgno; ++n)
if (! walk.pagemap[n])
walk.dbi_pages[0] += 1;
empty_pages = lost_bytes = 0;
for (i = 1; i < MAX_DBI && walk.dbi_names[i]; ++i) {
empty_pages += walk.dbi_empty_pages[i];
lost_bytes += walk.dbi_lost_bytes[i];
}
if (verbose) {
size_t total_page_bytes = walk.pgcount * stat.base.ms_psize;
print(" - dbi pages: %zu total", walk.pgcount);
if (verbose > 1)
for (i = 1; i < MAX_DBI && walk.dbi_names[i]; ++i)
print(", %s %zu", walk.dbi_names[i], walk.dbi_pages[i]);
print(", %s %zu\n", walk.dbi_names[0], walk.dbi_pages[0]);
if (verbose > 1) {
print(" - space info: total %zu bytes, payload %zu (%.1f%%), unused %zu (%.1f%%)\n",
total_page_bytes, walk.total_payload_bytes,
walk.total_payload_bytes * 100.0 / total_page_bytes,
total_page_bytes - walk.total_payload_bytes,
(total_page_bytes - walk.total_payload_bytes) * 100.0 / total_page_bytes);
for (i = 1; i < MAX_DBI && walk.dbi_names[i]; ++i) {
size_t dbi_bytes = walk.dbi_pages[i] * stat.base.ms_psize;
print(" %s: subtotal %zu bytes (%.1f%%), payload %zu (%.1f%%), unused %zu (%.1f%%)",
walk.dbi_names[i],
dbi_bytes, dbi_bytes * 100.0 / total_page_bytes,
walk.dbi_payload_bytes[i], walk.dbi_payload_bytes[i] * 100.0 / dbi_bytes,
dbi_bytes - walk.dbi_payload_bytes[i],
(dbi_bytes - walk.dbi_payload_bytes[i]) * 100.0 / dbi_bytes);
if (walk.dbi_empty_pages[i])
print(", %zu empty pages", walk.dbi_empty_pages[i]);
if (walk.dbi_lost_bytes[i])
print(", %zu bytes lost", walk.dbi_lost_bytes[i]);
print("\n");
}
}
print(" - summary: average fill %.1f%%", walk.total_payload_bytes * 100.0 / total_page_bytes);
if (empty_pages)
print(", %zu empty pages", empty_pages);
if (lost_bytes)
print(", %zu bytes lost", lost_bytes);
print(", %zu problems\n", traversal_problems);
}
} else if (verbose) {
print("Skipping b-tree walk...\n");
fflush(NULL);
}
if (! verbose)
print("Iterating DBIs...\n");
problems_maindb = process_db(-1, /* MAIN_DBI */ NULL, NULL, 0);
problems_freedb = process_db(0 /* FREE_DBI */, "free", handle_freedb, 0);
if (verbose) {
size_t value = info.base.me_mapsize / stat.base.ms_psize;
double percent = value / 100.0;
print(" - pages info: %zu total", value);
print(", allocated %zu (%.1f%%)", lastpgno, lastpgno / percent);
if (verbose > 1) {
value = info.base.me_mapsize / stat.base.ms_psize - lastpgno;
print(", remained %zu (%.1f%%)", value, value / percent);
value = lastpgno - freedb_pages;
print(", used %zu (%.1f%%)", value, value / percent);
print(", gc %zu (%.1f%%)", freedb_pages, freedb_pages / percent);
value = freedb_pages - reclaimable_pages;
print(", detained %zu (%.1f%%)", value, value / percent);
print(", reclaimable %zu (%.1f%%)", reclaimable_pages, reclaimable_pages / percent);
}
value = info.base.me_mapsize / stat.base.ms_psize - lastpgno + reclaimable_pages;
print(", available %zu (%.1f%%)\n", value, value / percent);
}
if (problems_maindb == 0 && problems_freedb == 0) {
if (!dont_traversal && (exclusive || locktxn)) {
if (walk.pgcount != lastpgno - freedb_pages) {
error("used pages mismatch (%zu != %zu)\n", walk.pgcount, lastpgno - freedb_pages);
}
if (walk.dbi_pages[0] != freedb_pages) {
error("gc pages mismatch (%zu != %zu)\n", walk.dbi_pages[0], freedb_pages);
}
} else if (verbose) {
print(" - skip check used and gc pages (btree-traversal with monopolistic or write-lock mode only)\n");
}
if (! process_db(-1, NULL, handle_maindb, 1)) {
if (! userdb_count && verbose)
print(" - does not contain multiple databases\n");
}
}
bailout:
if (txn)
mdbx_txn_abort(txn);
if (locktxn)
mdbx_txn_abort(locktxn);
if (env)
mdbx_env_close(env);
fflush(NULL);
if (rc) {
if (rc < 0)
return gotsignal ? EXIT_INTERRUPTED : EXIT_FAILURE_SYS;
return EXIT_FAILURE_MDB;
}
if (clock_gettime(CLOCK_MONOTONIC, &timestamp_finish)) {
rc = errno;
error("clock_gettime failed, error %d %s\n", rc, mdbx_strerror(rc));
return EXIT_FAILURE_SYS;
}
elapsed = timestamp_finish.tv_sec - timestamp_start.tv_sec
+ (timestamp_finish.tv_nsec - timestamp_start.tv_nsec) * 1e-9;
total_problems += problems_meta;
if (total_problems || problems_maindb || problems_freedb) {
print("Total %zu error(s) is detected, elapsed %.3f seconds.\n",
total_problems, elapsed);
if (problems_meta || problems_maindb || problems_freedb)
return EXIT_FAILURE_CHECK_MAJOR;
return EXIT_FAILURE_CHECK_MINOR;
}
print("No error is detected, elapsed %.3f seconds\n", elapsed);
return EXIT_SUCCESS;
}

81
mdb_copy.c
View File

@ -1,81 +0,0 @@
/* mdb_copy.c - memory-mapped database backup tool */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2012-2017 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include "mdbx.h"
static void
sighandle(int sig)
{
(void) sig;
}
int main(int argc,char * argv[])
{
int rc;
MDB_env *env = NULL;
const char *progname = argv[0], *act;
unsigned flags = MDB_RDONLY;
unsigned cpflags = 0;
for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
if (argv[1][1] == 'n' && argv[1][2] == '\0')
flags |= MDB_NOSUBDIR;
else if (argv[1][1] == 'c' && argv[1][2] == '\0')
cpflags |= MDB_CP_COMPACT;
else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
printf("%s\n", MDB_VERSION_STRING);
exit(0);
} else
argc = 0;
}
if (argc<2 || argc>3) {
fprintf(stderr, "usage: %s [-V] [-c] [-n] srcpath [dstpath]\n", progname);
exit(EXIT_FAILURE);
}
#ifdef SIGPIPE
signal(SIGPIPE, sighandle);
#endif
#ifdef SIGHUP
signal(SIGHUP, sighandle);
#endif
signal(SIGINT, sighandle);
signal(SIGTERM, sighandle);
act = "opening environment";
rc = mdb_env_create(&env);
if (rc == MDB_SUCCESS) {
rc = mdb_env_open(env, argv[1], flags, 0640);
}
if (rc == MDB_SUCCESS) {
act = "copying";
if (argc == 2)
rc = mdb_env_copyfd2(env, STDOUT_FILENO, cpflags);
else
rc = mdb_env_copy2(env, argv[2], cpflags);
}
if (rc)
fprintf(stderr, "%s: %s failed, error %d (%s)\n",
progname, act, rc, mdb_strerror(rc));
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

314
mdb_dump.c
View File

@ -1,314 +0,0 @@
/* mdb_dump.c - memory-mapped database dump tool */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2011-2017 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <unistd.h>
#include <signal.h>
#include "mdbx.h"
#define PRINT 1
static int mode;
typedef struct flagbit {
int bit;
char *name;
} flagbit;
flagbit dbflags[] = {
{ MDB_REVERSEKEY, "reversekey" },
{ MDB_DUPSORT, "dupsort" },
{ MDB_INTEGERKEY, "integerkey" },
{ MDB_DUPFIXED, "dupfixed" },
{ MDB_INTEGERDUP, "integerdup" },
{ MDB_REVERSEDUP, "reversedup" },
{ 0, NULL }
};
static volatile sig_atomic_t gotsig;
static void dumpsig( int sig )
{
(void) sig;
gotsig = 1;
}
static const char hexc[] = "0123456789abcdef";
static void hex(unsigned char c)
{
putchar(hexc[c >> 4]);
putchar(hexc[c & 0xf]);
}
static void text(MDB_val *v)
{
unsigned char *c, *end;
putchar(' ');
c = v->mv_data;
end = c + v->mv_size;
while (c < end) {
if (isprint(*c)) {
putchar(*c);
} else {
putchar('\\');
hex(*c);
}
c++;
}
putchar('\n');
}
static void byte(MDB_val *v)
{
unsigned char *c, *end;
putchar(' ');
c = v->mv_data;
end = c + v->mv_size;
while (c < end) {
hex(*c++);
}
putchar('\n');
}
/* Dump in BDB-compatible format */
static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
{
MDB_cursor *mc;
MDB_stat ms;
MDB_val key, data;
MDB_envinfo info;
unsigned int flags;
int rc, i;
rc = mdb_dbi_flags(txn, dbi, &flags);
if (rc) return rc;
rc = mdb_stat(txn, dbi, &ms);
if (rc) return rc;
rc = mdb_env_info(mdb_txn_env(txn), &info);
if (rc) return rc;
printf("VERSION=3\n");
printf("format=%s\n", mode & PRINT ? "print" : "bytevalue");
if (name)
printf("database=%s\n", name);
printf("type=btree\n");
printf("mapsize=%zu\n", info.me_mapsize);
if (info.me_mapaddr)
printf("mapaddr=%p\n", info.me_mapaddr);
printf("maxreaders=%u\n", info.me_maxreaders);
for (i=0; dbflags[i].bit; i++)
if (flags & dbflags[i].bit)
printf("%s=1\n", dbflags[i].name);
printf("db_pagesize=%d\n", ms.ms_psize);
printf("HEADER=END\n");
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) return rc;
while ((rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT)) == MDB_SUCCESS) {
if (gotsig) {
rc = EINTR;
break;
}
if (mode & PRINT) {
text(&key);
text(&data);
} else {
byte(&key);
byte(&data);
}
}
printf("DATA=END\n");
if (rc == MDB_NOTFOUND)
rc = MDB_SUCCESS;
return rc;
}
static void usage(char *prog)
{
fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb] dbpath\n", prog);
exit(EXIT_FAILURE);
}
int main(int argc, char *argv[])
{
int i, rc;
MDB_env *env;
MDB_txn *txn;
MDB_dbi dbi;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envflags = 0, list = 0;
if (argc < 2) {
usage(prog);
}
/* -a: dump main DB and all subDBs
* -s: dump only the named subDB
* -n: use NOSUBDIR flag on env_open
* -p: use printable characters
* -f: write to file instead of stdout
* -V: print version and exit
* (default) dump only the main DB
*/
while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(0);
break;
case 'l':
list = 1;
/*FALLTHROUGH*/;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'f':
if (freopen(optarg, "w", stdout) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n",
prog, optarg, strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 'p':
mode |= PRINT;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
#ifdef SIGPIPE
signal(SIGPIPE, dumpsig);
#endif
#ifdef SIGHUP
signal(SIGHUP, dumpsig);
#endif
signal(SIGINT, dumpsig);
signal(SIGTERM, dumpsig);
envname = argv[optind];
rc = mdb_env_create(&env);
if (rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname) {
mdb_env_set_maxdbs(env, 2);
}
rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
if (alldbs) {
MDB_cursor *cursor;
MDB_val key;
int count = 0;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) {
char *str;
MDB_dbi db2;
if (memchr(key.mv_data, '\0', key.mv_size))
continue;
count++;
str = malloc(key.mv_size+1);
memcpy(str, key.mv_data, key.mv_size);
str[key.mv_size] = '\0';
rc = mdb_open(txn, str, 0, &db2);
if (rc == MDB_SUCCESS) {
if (list) {
printf("%s\n", str);
list++;
} else {
rc = dumpit(txn, db2, str);
if (rc)
break;
}
mdb_close(env, db2);
}
free(str);
if (rc) continue;
}
mdb_cursor_close(cursor);
if (!count) {
fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, envname);
rc = MDB_NOTFOUND;
} else if (rc == MDB_INCOMPATIBLE) {
/* LY: the record it not a named sub-db. */
rc = MDB_SUCCESS;
}
} else {
rc = dumpit(txn, dbi, subname);
}
if (rc && rc != MDB_NOTFOUND)
fprintf(stderr, "%s: %s: %s\n", prog, envname, mdb_strerror(rc));
mdb_close(env, dbi);
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

456
mdb_load.c
View File

@ -1,456 +0,0 @@
/* mdb_load.c - memory-mapped database load tool */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2011-2017 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <ctype.h>
#include <unistd.h>
#include "mdbx.h"
#define PRINT 1
#define NOHDR 2
static int mode;
static char *subname = NULL;
static size_t lineno;
static int version;
static int dbi_flags;
static char *prog;
static int Eof;
static MDB_envinfo info;
static MDB_val kbuf, dbuf;
#define STRLENOF(s) (sizeof(s)-1)
typedef struct flagbit {
int bit;
char *name;
int len;
} flagbit;
#define S(s) s, STRLENOF(s)
flagbit dbflags[] = {
{ MDB_REVERSEKEY, S("reversekey") },
{ MDB_DUPSORT, S("dupsort") },
{ MDB_INTEGERKEY, S("integerkey") },
{ MDB_DUPFIXED, S("dupfixed") },
{ MDB_INTEGERDUP, S("integerdup") },
{ MDB_REVERSEDUP, S("reversedup") },
{ 0, NULL, 0 }
};
static void readhdr(void)
{
char *ptr;
dbi_flags = 0;
while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) {
lineno++;
if (!strncmp(dbuf.mv_data, "db_pagesize=", STRLENOF("db_pagesize="))
|| !strncmp(dbuf.mv_data, "duplicates=", STRLENOF("duplicates="))) {
/* LY: silently ignore information fields. */
continue;
} else if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) {
version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION="));
if (version > 3) {
fprintf(stderr, "%s: line %zd: unsupported VERSION %d\n",
prog, lineno, version);
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.mv_data, "HEADER=END", STRLENOF("HEADER=END"))) {
break;
} else if (!strncmp(dbuf.mv_data, "format=", STRLENOF("format="))) {
if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print")))
mode |= PRINT;
else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) {
fprintf(stderr, "%s: line %zd: unsupported FORMAT %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.mv_data, "database=", STRLENOF("database="))) {
ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
if (ptr) *ptr = '\0';
if (subname) free(subname);
subname = strdup((char *)dbuf.mv_data+STRLENOF("database="));
} else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) {
if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree"))) {
fprintf(stderr, "%s: line %zd: unsupported type %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("type="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.mv_data, "mapaddr=", STRLENOF("mapaddr="))) {
int i;
ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
if (ptr) *ptr = '\0';
i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr);
if (i != 1) {
fprintf(stderr, "%s: line %zd: invalid mapaddr %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.mv_data, "mapsize=", STRLENOF("mapsize="))) {
int i;
ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
if (ptr) *ptr = '\0';
i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%zu", &info.me_mapsize);
if (i != 1) {
fprintf(stderr, "%s: line %zd: invalid mapsize %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.mv_data, "maxreaders=", STRLENOF("maxreaders="))) {
int i;
ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
if (ptr) *ptr = '\0';
i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders);
if (i != 1) {
fprintf(stderr, "%s: line %zd: invalid maxreaders %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders="));
exit(EXIT_FAILURE);
}
} else {
int i;
for (i=0; dbflags[i].bit; i++) {
if (!strncmp(dbuf.mv_data, dbflags[i].name, dbflags[i].len) &&
((char *)dbuf.mv_data)[dbflags[i].len] == '=') {
if (((char *)dbuf.mv_data)[dbflags[i].len+1] == '1')
dbi_flags |= dbflags[i].bit;
break;
}
}
if (!dbflags[i].bit) {
ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size);
if (!ptr) {
fprintf(stderr, "%s: line %zd: unexpected format\n",
prog, lineno);
exit(EXIT_FAILURE);
} else {
*ptr = '\0';
fprintf(stderr, "%s: line %zd: unrecognized keyword ignored: %s\n",
prog, lineno, (char *)dbuf.mv_data);
}
}
}
}
}
static void badend(void)
{
fprintf(stderr, "%s: line %zd: unexpected end of input\n",
prog, lineno);
}
static int unhex(unsigned char *c2)
{
int x, c;
x = *c2++ & 0x4f;
if (x & 0x40)
x -= 55;
c = x << 4;
x = *c2 & 0x4f;
if (x & 0x40)
x -= 55;
c |= x;
return c;
}
static int readline(MDB_val *out, MDB_val *buf)
{
unsigned char *c1, *c2, *end;
size_t len, l2;
int c;
if (!(mode & NOHDR)) {
c = fgetc(stdin);
if (c == EOF) {
Eof = 1;
return EOF;
}
if (c != ' ') {
lineno++;
if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
badend:
Eof = 1;
badend();
return EOF;
}
if (c == 'D' && !strncmp(buf->mv_data, "ATA=END", STRLENOF("ATA=END")))
return EOF;
goto badend;
}
}
if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
Eof = 1;
return EOF;
}
lineno++;
c1 = buf->mv_data;
len = strlen((char *)c1);
l2 = len;
/* Is buffer too short? */
while (c1[len-1] != '\n') {
buf->mv_data = realloc(buf->mv_data, buf->mv_size*2);
if (!buf->mv_data) {
Eof = 1;
fprintf(stderr, "%s: line %zd: out of memory, line too long\n",
prog, lineno);
return EOF;
}
c1 = buf->mv_data;
c1 += l2;
if (fgets((char *)c1, buf->mv_size+1, stdin) == NULL) {
Eof = 1;
badend();
return EOF;
}
buf->mv_size *= 2;
len = strlen((char *)c1);
l2 += len;
}
c1 = c2 = buf->mv_data;
len = l2;
c1[--len] = '\0';
end = c1 + len;
if (mode & PRINT) {
while (c2 < end) {
if (*c2 == '\\') {
if (c2[1] == '\\') {
c1++; c2 += 2;
} else {
if (c2+3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
Eof = 1;
badend();
return EOF;
}
*c1++ = unhex(++c2);
c2 += 2;
}
} else {
/* copies are redundant when no escapes were used */
*c1++ = *c2++;
}
}
} else {
/* odd length not allowed */
if (len & 1) {
Eof = 1;
badend();
return EOF;
}
while (c2 < end) {
if (!isxdigit(*c2) || !isxdigit(c2[1])) {
Eof = 1;
badend();
return EOF;
}
*c1++ = unhex(c2);
c2 += 2;
}
}
c2 = out->mv_data = buf->mv_data;
out->mv_size = c1 - c2;
return 0;
}
static void usage(void)
{
fprintf(stderr, "usage: %s [-V] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
exit(EXIT_FAILURE);
}
int main(int argc, char *argv[])
{
int i, rc;
MDB_env *env;
MDB_txn *txn;
MDB_cursor *mc;
MDB_dbi dbi;
char *envname;
int envflags = 0, putflags = 0;
prog = argv[0];
if (argc < 2) {
usage();
}
/* -f: load file instead of stdin
* -n: use NOSUBDIR flag on env_open
* -s: load into named subDB
* -N: use NOOVERWRITE on puts
* -T: read plaintext
* -V: print version and exit
*/
while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(0);
break;
case 'f':
if (freopen(optarg, "r", stdin) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n",
prog, optarg, strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 's':
subname = strdup(optarg);
break;
case 'N':
putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
break;
case 'T':
mode |= NOHDR | PRINT;
break;
default:
usage();
}
}
if (optind != argc - 1)
usage();
dbuf.mv_size = 4096;
dbuf.mv_data = malloc(dbuf.mv_size);
if (!(mode & NOHDR))
readhdr();
envname = argv[optind];
rc = mdb_env_create(&env);
if (rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
mdb_env_set_maxdbs(env, 2);
if (info.me_maxreaders)
mdb_env_set_maxreaders(env, info.me_maxreaders);
if (info.me_mapsize)
mdb_env_set_mapsize(env, info.me_mapsize);
if (info.me_mapaddr)
envflags |= MDB_FIXEDMAP;
rc = mdb_env_open(env, envname, envflags, 0664);
if (rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
kbuf.mv_data = malloc(kbuf.mv_size);
while(!Eof) {
MDB_val key, data;
int batch = 0;
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_open(txn, subname, dbi_flags|MDB_CREATE, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while(1) {
rc = readline(&key, &kbuf);
if (rc) /* rc == EOF */
break;
rc = readline(&data, &dbuf);
if (rc) {
fprintf(stderr, "%s: line %zd: failed to read key value\n", prog, lineno);
goto txn_abort;
}
rc = mdb_cursor_put(mc, &key, &data, putflags);
if (rc == MDB_KEYEXIST && putflags)
continue;
if (rc) {
fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
batch++;
if (batch == 100) {
rc = mdb_txn_commit(txn);
if (rc) {
fprintf(stderr, "%s: line %zd: txn_commit: %s\n",
prog, lineno, mdb_strerror(rc));
goto env_close;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
batch = 0;
}
}
rc = mdb_txn_commit(txn);
txn = NULL;
if (rc) {
fprintf(stderr, "%s: line %zd: txn_commit: %s\n",
prog, lineno, mdb_strerror(rc));
goto env_close;
}
mdb_dbi_close(env, dbi);
if(!(mode & NOHDR))
readhdr();
}
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

299
mdb_stat.c
View File

@ -1,299 +0,0 @@
/* mdb_stat.c - memory-mapped database status tool */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2011-2017 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "mdbx.h"
static void prstat(MDBX_stat *ms)
{
#if 0
printf(" Page size: %u\n", ms->base.ms_psize);
#endif
printf(" Tree depth: %u\n", ms->base.ms_depth);
printf(" Branch pages: %zu\n", ms->base.ms_branch_pages);
printf(" Leaf pages: %zu\n", ms->base.ms_leaf_pages);
printf(" Overflow pages: %zu\n", ms->base.ms_overflow_pages);
printf(" Entries: %zu\n", ms->base.ms_entries);
}
static void usage(char *prog)
{
fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb] dbpath\n", prog);
exit(EXIT_FAILURE);
}
int main(int argc, char *argv[])
{
int i, rc;
MDB_env *env;
MDB_txn *txn;
MDB_dbi dbi;
MDBX_stat mst;
MDBX_envinfo mei;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0;
if (argc < 2) {
usage(prog);
}
/* -a: print stat of main DB and all subDBs
* -s: print stat of only the named subDB
* -e: print env info
* -f: print freelist info
* -r: print reader info
* -n: use NOSUBDIR flag on env_open
* -V: print version and exit
* (default) print stat of only the main DB
*/
while ((i = getopt(argc, argv, "Vaefnrs:")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(0);
break;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'e':
envinfo++;
break;
case 'f':
freinfo++;
break;
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 'r':
rdrinfo++;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
envname = argv[optind];
rc = mdb_env_create(&env);
if (rc) {
fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname) {
mdb_env_set_maxdbs(env, 4);
}
rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
if (envinfo) {
(void)mdbx_env_stat(env, &mst, sizeof(mst));
(void)mdbx_env_info(env, &mei, sizeof(mei));
printf("Environment Info\n");
printf(" Map address: %p\n", mei.base.me_mapaddr);
printf(" Map size: %zu\n", mei.base.me_mapsize);
printf(" Page size: %u\n", mst.base.ms_psize);
printf(" Max pages: %zu\n", mei.base.me_mapsize / mst.base.ms_psize);
printf(" Number of pages used: %zu\n", mei.base.me_last_pgno+1);
printf(" Last transaction ID: %zu\n", mei.base.me_last_txnid);
printf(" Tail transaction ID: %zu (%zi)\n",
mei.me_tail_txnid, mei.me_tail_txnid - mei.base.me_last_txnid);
printf(" Max readers: %u\n", mei.base.me_maxreaders);
printf(" Number of readers used: %u\n", mei.base.me_numreaders);
} else {
/* LY: zap warnings from gcc */
memset(&mst, 0, sizeof(mst));
memset(&mei, 0, sizeof(mei));
}
if (rdrinfo) {
printf("Reader Table Status\n");
rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout);
if (rdrinfo > 1) {
int dead;
mdb_reader_check(env, &dead);
printf(" %d stale readers cleared.\n", dead);
rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout);
}
if (!(subname || alldbs || freinfo))
goto env_close;
}
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
if (freinfo) {
MDB_cursor *cursor;
MDB_val key, data;
size_t pages = 0, *iptr;
size_t reclaimable = 0;
printf("Freelist Status\n");
dbi = 0;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdbx_stat(txn, dbi, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
prstat(&mst);
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
iptr = data.mv_data;
pages += *iptr;
if (envinfo && mei.me_tail_txnid > *(size_t *)key.mv_data)
reclaimable += *iptr;
if (freinfo > 1) {
char *bad = "";
size_t pg, prev;
ssize_t i, j, span = 0;
j = *iptr++;
for (i = j, prev = 1; --i >= 0; ) {
pg = iptr[i];
if (pg <= prev)
bad = " [bad sequence]";
prev = pg;
pg += span;
for (; i >= span && iptr[i-span] == pg; span++, pg++) ;
}
printf(" Transaction %zu, %zd pages, maxspan %zd%s\n",
*(size_t *)key.mv_data, j, span, bad);
if (freinfo > 2) {
for (--j; j >= 0; ) {
pg = iptr[j];
for (span=1; --j >= 0 && iptr[j] == pg+span; span++) ;
if (span>1)
printf(" %9zu[%zd]\n", pg, span);
else
printf(" %9zu\n", pg);
}
}
}
}
mdb_cursor_close(cursor);
if (envinfo) {
size_t value = mei.base.me_mapsize / mst.base.ms_psize;
double percent = value / 100.0;
printf("Page Allocation Info\n");
printf(" Max pages: %9zu 100%%\n", value);
value = mei.base.me_last_pgno+1;
printf(" Number of pages used: %zu %.1f%%\n", value, value / percent);
value = mei.base.me_mapsize / mst.base.ms_psize - (mei.base.me_last_pgno+1);
printf(" Remained: %zu %.1f%%\n", value, value / percent);
value = mei.base.me_last_pgno+1 - pages;
printf(" Used now: %zu %.1f%%\n", value, value / percent);
value = pages;
printf(" Unallocated: %zu %.1f%%\n", value, value / percent);
value = pages - reclaimable;
printf(" Detained: %zu %.1f%%\n", value, value / percent);
value = reclaimable;
printf(" Reclaimable: %zu %.1f%%\n", value, value / percent);
value = mei.base.me_mapsize / mst.base.ms_psize - (mei.base.me_last_pgno+1) + reclaimable;
printf(" Available: %zu %.1f%%\n", value, value / percent);
} else
printf(" Free pages: %zu\n", pages);
}
rc = mdb_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdbx_stat(txn, dbi, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
printf("Status of %s\n", subname ? subname : "Main DB");
prstat(&mst);
if (alldbs) {
MDB_cursor *cursor;
MDB_val key;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) {
char *str;
MDB_dbi db2;
if (memchr(key.mv_data, '\0', key.mv_size))
continue;
str = malloc(key.mv_size+1);
memcpy(str, key.mv_data, key.mv_size);
str[key.mv_size] = '\0';
rc = mdb_open(txn, str, 0, &db2);
if (rc == MDB_SUCCESS)
printf("Status of %s\n", str);
free(str);
if (rc) continue;
rc = mdbx_stat(txn, db2, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
prstat(&mst);
mdb_close(env, db2);
}
mdb_cursor_close(cursor);
}
if (rc == MDB_NOTFOUND)
rc = MDB_SUCCESS;
mdb_close(env, dbi);
txn_abort:
mdb_txn_abort(txn);
env_close:
mdb_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

881
mdbx.c
View File

@ -1,881 +0,0 @@
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "mdbx.h"
int mdb_runtime_flags = MDBX_DBG_PRINT
#if MDB_DEBUG
| MDBX_DBG_ASSERT
#endif
#if MDB_DEBUG > 1
| MDBX_DBG_TRACE
#endif
#if MDB_DEBUG > 2
| MDBX_DBG_AUDIT
#endif
#if MDB_DEBUG > 3
| MDBX_DBG_EXTRA
#endif
;
static MDBX_debug_func *mdb_debug_logger;
int mdbx_setup_debug(int flags, MDBX_debug_func* logger, long edge_txn);
#include "mdb.c"
int __cold
mdbx_setup_debug(int flags, MDBX_debug_func* logger, long edge_txn) {
unsigned ret = mdb_runtime_flags;
if (flags != (int) MDBX_DBG_DNT)
mdb_runtime_flags = flags;
if (logger != (MDBX_debug_func*) MDBX_DBG_DNT)
mdb_debug_logger = logger;
if (edge_txn != (long) MDBX_DBG_DNT) {
#if MDB_DEBUG
mdb_debug_edge = edge_txn;
#endif
}
return ret;
}
static txnid_t __cold
mdbx_oomkick(MDB_env *env, txnid_t oldest)
{
int retry;
txnid_t snap;
mdb_debug("DB size maxed out");
for(retry = 0; ; ++retry) {
int reader;
if (mdb_reader_check(env, NULL))
break;
snap = mdb_find_oldest(env, &reader);
if (oldest < snap || reader < 0) {
if (retry && env->me_oom_func) {
/* LY: notify end of oom-loop */
env->me_oom_func(env, 0, 0, oldest, snap - oldest, -retry);
}
return snap;
}
MDB_reader *r;
pthread_t tid;
pid_t pid;
int rc;
if (!env->me_oom_func)
break;
r = &env->me_txns->mti_readers[ reader ];
pid = r->mr_pid;
tid = r->mr_tid;
if (r->mr_txnid != oldest || pid <= 0)
continue;
rc = env->me_oom_func(env, pid, (void*) tid, oldest,
mdb_meta_head_w(env)->mm_txnid - oldest, retry);
if (rc < 0)
break;
if (rc) {
r->mr_txnid = ~(txnid_t)0;
if (rc > 1) {
r->mr_tid = 0;
r->mr_pid = 0;
mdbx_coherent_barrier();
}
}
}
if (retry && env->me_oom_func) {
/* LY: notify end of oom-loop */
env->me_oom_func(env, 0, 0, oldest, 0, -retry);
}
return mdb_find_oldest(env, NULL);
}
int __cold
mdbx_env_set_syncbytes(MDB_env *env, size_t bytes)
{
if (unlikely(!env))
return EINVAL;
if(unlikely(env->me_signature != MDBX_ME_SIGNATURE))
return MDB_VERSION_MISMATCH;
env->me_sync_threshold = bytes;
return env->me_map ? mdb_env_sync(env, 0) : MDB_SUCCESS;
}
void __cold
mdbx_env_set_oomfunc(MDB_env *env, MDBX_oom_func *oomfunc)
{
if (likely(env && env->me_signature == MDBX_ME_SIGNATURE))
env->me_oom_func = oomfunc;
}
MDBX_oom_func* __cold
mdbx_env_get_oomfunc(MDB_env *env)
{
return likely(env && env->me_signature == MDBX_ME_SIGNATURE)
? env->me_oom_func : NULL;
}
ATTRIBUTE_NO_SANITIZE_THREAD /* LY: avoid tsan-trap by me_txn, mm_last_pg and mt_next_pgno */
int mdbx_txn_straggler(MDB_txn *txn, int *percent)
{
MDB_env *env;
MDB_meta *meta;
txnid_t recent, lag;
size_t maxpg;
if(unlikely(!txn))
return -EINVAL;
if(unlikely(txn->mt_signature != MDBX_MT_SIGNATURE))
return MDB_VERSION_MISMATCH;
env = txn->mt_env;
maxpg = env->me_maxpg;
if (unlikely((txn->mt_flags & MDB_RDONLY) == 0)) {
*percent = (int)((txn->mt_next_pgno * 100ull + maxpg / 2) / maxpg);
return -1;
}
do {
meta = mdb_meta_head_r(env);
recent = meta->mm_txnid;
if (percent) {
pgno_t last = meta->mm_last_pg + 1;
*percent = (int)((last * 100ull + maxpg / 2) / maxpg);
}
} while (unlikely(recent != meta->mm_txnid));
lag = recent - txn->mt_u.reader->mr_txnid;
return (0 > (long) lag) ? ~0u >> 1: lag;
}
typedef struct mdb_walk_ctx {
MDB_txn *mw_txn;
void *mw_user;
MDBX_pgvisitor_func *mw_visitor;
} mdb_walk_ctx_t;
/** Depth-first tree traversal. */
static int __cold
mdb_env_walk(mdb_walk_ctx_t *ctx, const char* dbi, pgno_t pg, int deep)
{
MDB_page *mp;
int rc, i, nkeys;
unsigned header_size, unused_size, payload_size, align_bytes;
const char* type;
if (pg == P_INVALID)
return MDB_SUCCESS; /* empty db */
MDB_cursor mc;
memset(&mc, 0, sizeof(mc));
mc.mc_snum = 1;
mc.mc_txn = ctx->mw_txn;
rc = mdb_page_get(&mc, pg, &mp, NULL);
if (rc)
return rc;
if (pg != mp->mp_p.p_pgno)
return MDB_CORRUPTED;
nkeys = NUMKEYS(mp);
header_size = IS_LEAF2(mp) ? PAGEHDRSZ : PAGEBASE + mp->mp_lower;
unused_size = SIZELEFT(mp);
payload_size = 0;
/* LY: Don't use mask here, e.g bitwise (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP).
* Pages should not me marked dirty/loose or otherwise. */
switch (mp->mp_flags) {
case P_BRANCH:
type = "branch";
if (nkeys < 1)
return MDB_CORRUPTED;
break;
case P_LEAF:
type = "leaf";
break;
case P_LEAF|P_SUBP:
type = "dupsort-subleaf";
break;
case P_LEAF|P_LEAF2:
type = "dupfixed-leaf";
break;
case P_LEAF|P_LEAF2|P_SUBP:
type = "dupsort-dupfixed-subleaf";
break;
case P_META:
case P_OVERFLOW:
default:
return MDB_CORRUPTED;
}
for (align_bytes = i = 0; i < nkeys;
align_bytes += ((payload_size + align_bytes) & 1), i++) {
MDB_node *node;
if (IS_LEAF2(mp)) {
/* LEAF2 pages have no mp_ptrs[] or node headers */
payload_size += mp->mp_leaf2_ksize;
continue;
}
node = NODEPTR(mp, i);
payload_size += NODESIZE + node->mn_ksize;
if (IS_BRANCH(mp)) {
rc = mdb_env_walk(ctx, dbi, NODEPGNO(node), deep);
if (rc)
return rc;
continue;
}
assert(IS_LEAF(mp));
if (node->mn_flags & F_BIGDATA) {
MDB_page *omp;
pgno_t *opg;
size_t over_header, over_payload, over_unused;
payload_size += sizeof(pgno_t);
opg = NODEDATA(node);
rc = mdb_page_get(&mc, *opg, &omp, NULL);
if (rc)
return rc;
if (*opg != omp->mp_p.p_pgno)
return MDB_CORRUPTED;
/* LY: Don't use mask here, e.g bitwise (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP).
* Pages should not me marked dirty/loose or otherwise. */
if (P_OVERFLOW != omp->mp_flags)
return MDB_CORRUPTED;
over_header = PAGEHDRSZ;
over_payload = NODEDSZ(node);
over_unused = omp->mp_pages * ctx->mw_txn->mt_env->me_psize
- over_payload - over_header;
rc = ctx->mw_visitor(*opg, omp->mp_pages, ctx->mw_user, dbi,
"overflow-data", 1, over_payload, over_header, over_unused);
if (rc)
return rc;
continue;
}
payload_size += NODEDSZ(node);
if (node->mn_flags & F_SUBDATA) {
MDB_db *db = NODEDATA(node);
char* name = NULL;
if (! (node->mn_flags & F_DUPDATA)) {
name = NODEKEY(node);
int namelen = (char*) db - name;
name = memcpy(alloca(namelen + 1), name, namelen);
name[namelen] = 0;
}
rc = mdb_env_walk(ctx, (name && name[0]) ? name : dbi,
db->md_root, deep + 1);
if (rc)
return rc;
}
}
return ctx->mw_visitor(mp->mp_p.p_pgno, 1, ctx->mw_user, dbi, type,
nkeys, payload_size, header_size, unused_size + align_bytes);
}
int __cold
mdbx_env_pgwalk(MDB_txn *txn, MDBX_pgvisitor_func* visitor, void* user)
{
mdb_walk_ctx_t ctx;
int rc;
if (unlikely(!txn))
return MDB_BAD_TXN;
if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE))
return MDB_VERSION_MISMATCH;
ctx.mw_txn = txn;
ctx.mw_user = user;
ctx.mw_visitor = visitor;
rc = visitor(0, 2, user, "lmdb", "meta", 2, sizeof(MDB_meta)*2, PAGEHDRSZ*2,
(txn->mt_env->me_psize - sizeof(MDB_meta) - PAGEHDRSZ) *2);
if (! rc)
rc = mdb_env_walk(&ctx, "free", txn->mt_dbs[FREE_DBI].md_root, 0);
if (! rc)
rc = mdb_env_walk(&ctx, "main", txn->mt_dbs[MAIN_DBI].md_root, 0);
if (! rc)
rc = visitor(P_INVALID, 0, user, NULL, NULL, 0, 0, 0, 0);
return rc;
}
int mdbx_canary_put(MDB_txn *txn, const mdbx_canary* canary)
{
if (unlikely(!txn))
return EINVAL;
if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE))
return MDB_VERSION_MISMATCH;
if (unlikely(F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)))
return EACCES;
if (likely(canary)) {
txn->mt_canary.x = canary->x;
txn->mt_canary.y = canary->y;
txn->mt_canary.z = canary->z;
}
txn->mt_canary.v = txn->mt_txnid;
return MDB_SUCCESS;
}
size_t mdbx_canary_get(MDB_txn *txn, mdbx_canary* canary)
{
if(unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE))
return 0;
if (likely(canary))
*canary = txn->mt_canary;
return txn->mt_txnid;
}
int mdbx_cursor_on_first(MDB_cursor *mc)
{
if (unlikely(mc == NULL))
return EINVAL;
if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE))
return MDB_VERSION_MISMATCH;
if (!(mc->mc_flags & C_INITIALIZED))
return MDBX_RESULT_FALSE;
unsigned i;
for(i = 0; i < mc->mc_snum; ++i) {
if (mc->mc_ki[i])
return MDBX_RESULT_FALSE;
}
return MDBX_RESULT_TRUE;
}
int mdbx_cursor_on_last(MDB_cursor *mc)
{
if (unlikely(mc == NULL))
return EINVAL;
if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE))
return MDB_VERSION_MISMATCH;
if (!(mc->mc_flags & C_INITIALIZED))
return MDBX_RESULT_FALSE;
unsigned i;
for(i = 0; i < mc->mc_snum; ++i) {
unsigned nkeys = NUMKEYS(mc->mc_pg[i]);
if (mc->mc_ki[i] < nkeys - 1)
return MDBX_RESULT_FALSE;
}
return MDBX_RESULT_TRUE;
}
int mdbx_cursor_eof(MDB_cursor *mc)
{
if (unlikely(mc == NULL))
return EINVAL;
if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE))
return MDB_VERSION_MISMATCH;
if ((mc->mc_flags & C_INITIALIZED) == 0)
return MDBX_RESULT_TRUE;
if (mc->mc_snum == 0)
return MDBX_RESULT_TRUE;
if ((mc->mc_flags & C_EOF)
&& mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top]))
return MDBX_RESULT_TRUE;
return MDBX_RESULT_FALSE;
}
static int mdbx_is_samedata(const MDB_val* a, const MDB_val* b) {
return a->iov_len == b->iov_len
&& memcmp(a->iov_base, b->iov_base, a->iov_len) == 0;
}
/* Позволяет обновить или удалить существующую запись с получением
* в old_data предыдущего значения данных. При этом если new_data равен
* нулю, то выполняется удаление, иначе обновление/вставка.
*
* Текущее значение может находиться в уже измененной (грязной) странице.
* В этом случае страница будет перезаписана при обновлении, а само старое
* значение утрачено. Поэтому исходно в old_data должен быть передан
* дополнительный буфер для копирования старого значения.
* Если переданный буфер слишком мал, то функция вернет -1, установив
* old_data->iov_len в соответствующее значение.
*
* Для не-уникальных ключей также возможен второй сценарий использования,
* когда посредством old_data из записей с одинаковым ключом для
* удаления/обновления выбирается конкретная. Для выбора этого сценария
* во flags следует одновременно указать MDB_CURRENT и MDB_NOOVERWRITE.
* Именно эта комбинация выбрана, так как она лишена смысла, и этим позволяет
* идентифицировать запрос такого сценария.
*
* Функция может быть замещена соответствующими операциями с курсорами
* после двух доработок (TODO):
* - внешняя аллокация курсоров, в том числе на стеке (без malloc).
* - получения статуса страницы по адресу (знать о P_DIRTY).
*/
int mdbx_replace(MDB_txn *txn, MDB_dbi dbi,
MDB_val *key, MDB_val *new_data, MDB_val *old_data, unsigned flags)
{
MDB_cursor mc;
MDB_xcursor mx;
if (unlikely(!key || !old_data || !txn || old_data == new_data))
return EINVAL;
if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE))
return MDB_VERSION_MISMATCH;
if (unlikely(old_data->iov_base == NULL && old_data->iov_len))
return EINVAL;
if (unlikely(new_data == NULL && !(flags & MDB_CURRENT)))
return EINVAL;
if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)))
return EINVAL;
if (unlikely(flags & ~(MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP|MDB_CURRENT)))
return EINVAL;
if (unlikely(txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)))
return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
mdb_cursor_init(&mc, txn, dbi, &mx);
mc.mc_next = txn->mt_cursors[dbi];
txn->mt_cursors[dbi] = &mc;
int rc;
MDB_val present_key = *key;
if (F_ISSET(flags, MDB_CURRENT | MDB_NOOVERWRITE)) {
/* в old_data значение для выбора конкретного дубликата */
if (unlikely(!(txn->mt_dbs[dbi].md_flags & MDB_DUPSORT))) {
rc = EINVAL;
goto bailout;
}
/* убираем лишний бит, он был признаком запрошенного режима */
flags -= MDB_NOOVERWRITE;
rc = mdbx_cursor_get(&mc, &present_key, old_data, MDB_GET_BOTH);
if (rc != MDB_SUCCESS)
goto bailout;
if (new_data) {
/* обновление конкретного дубликата */
if (mdbx_is_samedata(old_data, new_data))
/* если данные совпадают, то ничего делать не надо */
goto bailout;
#if 0 /* LY: исправлено в mdbx_cursor_put(), здесь в качестве памятки */
MDB_node *leaf = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]);
if (F_ISSET(leaf->mn_flags, F_DUPDATA)
&& mc.mc_xcursor->mx_db.md_entries > 1) {
/* Если у ключа больше одного значения, то
* сначала удаляем найденое "старое" значение.
*
* Этого можно не делать, так как MDBX уже
* обучен корректно обрабатывать такие ситуации.
*
* Однако, следует помнить, что в LMDB при
* совпадении размера данных, значение будет
* просто перезаписано с нарушением
* упорядоченности, что сломает поиск. */
rc = mdbx_cursor_del(&mc, 0);
if (rc != MDB_SUCCESS)
goto bailout;
flags -= MDB_CURRENT;
}
#endif
}
} else {
/* в old_data буфер для сохранения предыдущего значения */
if (unlikely(new_data && old_data->iov_base == new_data->iov_base))
return EINVAL;
MDB_val present_data;
rc = mdbx_cursor_get(&mc, &present_key, &present_data, MDB_SET_KEY);
if (unlikely(rc != MDB_SUCCESS)) {
old_data->iov_base = NULL;
old_data->iov_len = rc;
if (rc != MDB_NOTFOUND || (flags & MDB_CURRENT))
goto bailout;
} else if (flags & MDB_NOOVERWRITE) {
rc = MDB_KEYEXIST;
*old_data = present_data;
goto bailout;
} else {
MDB_page *page = mc.mc_pg[mc.mc_top];
if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) {
if (flags & MDB_CURRENT) {
/* для не-уникальных ключей позволяем update/delete только если ключ один */
MDB_node *leaf = NODEPTR(page, mc.mc_ki[mc.mc_top]);
if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
mdb_tassert(txn, XCURSOR_INITED(&mc) && mc.mc_xcursor->mx_db.md_entries > 1);
if (mc.mc_xcursor->mx_db.md_entries > 1) {
rc = MDBX_EMULTIVAL;
goto bailout;
}
}
/* если данные совпадают, то ничего делать не надо */
if (new_data && mdbx_is_samedata(&present_data, new_data)) {
*old_data = *new_data;
goto bailout;
}
/* В оригинальной LMDB фладок MDB_CURRENT здесь приведет
* к замене данных без учета MDB_DUPSORT сортировки,
* но здесь это в любом случае допустимо, так как мы
* проверили что для ключа есть только одно значение. */
} else if ((flags & MDB_NODUPDATA) && mdbx_is_samedata(&present_data, new_data)) {
/* если данные совпадают и установлен MDB_NODUPDATA */
rc = MDB_KEYEXIST;
goto bailout;
}
} else {
/* если данные совпадают, то ничего делать не надо */
if (new_data && mdbx_is_samedata(&present_data, new_data)) {
*old_data = *new_data;
goto bailout;
}
flags |= MDB_CURRENT;
}
if (page->mp_flags & P_DIRTY) {
if (unlikely(old_data->iov_len < present_data.iov_len)) {
old_data->iov_base = NULL;
old_data->iov_len = present_data.iov_len;
rc = MDBX_RESULT_TRUE;
goto bailout;
}
memcpy(old_data->iov_base, present_data.iov_base, present_data.iov_len);
old_data->iov_len = present_data.iov_len;
} else {
*old_data = present_data;
}
}
}
if (likely(new_data))
rc = mdbx_cursor_put(&mc, key, new_data, flags);
else
rc = mdbx_cursor_del(&mc, 0);
bailout:
txn->mt_cursors[dbi] = mc.mc_next;
return rc;
}
int
mdbx_get_ex(MDB_txn *txn, MDB_dbi dbi,
MDB_val *key, MDB_val *data, int* values_count)
{
DKBUF;
mdb_debug("===> get db %u key [%s]", dbi, DKEY(key));
if (unlikely(!key || !data || !txn))
return EINVAL;
if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE))
return MDB_VERSION_MISMATCH;
if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)))
return EINVAL;
if (unlikely(txn->mt_flags & MDB_TXN_BLOCKED))
return MDB_BAD_TXN;
MDB_cursor mc;
MDB_xcursor mx;
mdb_cursor_init(&mc, txn, dbi, &mx);
int exact = 0;
int rc = mdb_cursor_set(&mc, key, data, MDB_SET_KEY, &exact);
if (unlikely(rc != MDB_SUCCESS)) {
if (rc == MDB_NOTFOUND && values_count)
*values_count = 0;
return rc;
}
if (values_count) {
*values_count = 1;
if (mc.mc_xcursor != NULL) {
MDB_node *leaf = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]);
if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
mdb_tassert(txn, mc.mc_xcursor == &mx
&& (mx.mx_cursor.mc_flags & C_INITIALIZED));
*values_count = mx.mx_db.md_entries;
}
}
}
return MDB_SUCCESS;
}
/* Функция сообщает находится ли указанный адрес в "грязной" странице у
* заданной пишущей транзакции. В конечном счете это позволяет избавиться от
* лишнего копирования данных из НЕ-грязных страниц.
*
* "Грязные" страницы - это те, которые уже были изменены в ходе пишущей
* транзакции. Соответственно, какие-либо дальнейшие изменения могут привести
* к перезаписи таких страниц. Поэтому все функции, выполняющие изменения, в
* качестве аргументов НЕ должны получать указатели на данные в таких
* страницах. В свою очередь "НЕ грязные" страницы перед модификацией будут
* скопированы.
*
* Другими словами, данные из "грязных" страниц должны быть либо скопированы
* перед передачей в качестве аргументов для дальнейших модификаций, либо
* отвергнуты на стадии проверки корректности аргументов.
*
* Таким образом, функция позволяет как избавится от лишнего копирования,
* так и выполнить более полную проверку аргументов.
*
* ВАЖНО: Передаваемый указатель должен указывать на начало данных. Только
* так гарантируется что актуальный заголовок страницы будет физически
* расположен в той-же странице памяти, в том числе для многостраничных
* P_OVERFLOW страниц с длинными данными. */
int mdbx_is_dirty(const MDB_txn *txn, const void* ptr)
{
if (unlikely(!txn))
return EINVAL;
if(unlikely(txn->mt_signature != MDBX_MT_SIGNATURE))
return MDB_VERSION_MISMATCH;
if (unlikely(txn->mt_flags & MDB_TXN_RDONLY))
return MDB_BAD_TXN;
const MDB_env *env = txn->mt_env;
const uintptr_t mask = ~(uintptr_t) (env->me_psize - 1);
const MDB_page *page = (const MDB_page *) ((uintptr_t) ptr & mask);
/* LY: Тут не всё хорошо с абсолютной достоверностью результата,
* так как флажок P_DIRTY в LMDB может означать не совсем то,
* что было исходно задумано, детали см в логике кода mdb_page_touch().
*
* Более того, в режиме БЕЗ WRITEMAP грязные страницы выделяются через
* malloc(), т.е. находятся вне mmap-диаппазона.
*
* Тем не менее, однозначно страница "не грязная" если:
* - адрес находится внутри mmap-диаппазона и в заголовке страницы
* нет флажка P_DIRTY, то однозначно страница "не грязная".
* - адрес вне mmap-диаппазона и его нет среди списка "грязных" страниц.
*/
if (env->me_map < (char*) page) {
const size_t used_size = env->me_psize * txn->mt_next_pgno;
if (env->me_map + used_size > (char*) page) {
/* страница внутри диапазона */
if (page->mp_flags & P_DIRTY)
return MDBX_RESULT_TRUE;
return MDBX_RESULT_FALSE;
}
/* Гипотетически здесь возможна ситуация, когда указатель адресует что-то
* в пределах mmap, но за границей распределенных страниц. Это тяжелая
* ошибка, которой не возможно добиться без каких-то мега-нарушений.
* Поэтому не проверяем этот случай кроме как assert-ом, ибо бестолку. */
mdb_tassert(txn, env->me_map + env->me_mapsize > (char*) page);
}
/* Страница вне mmap-диаппазона */
if (env->me_flags & MDB_WRITEMAP)
/* Если MDB_WRITEMAP, то результат уже ясен. */
return MDBX_RESULT_FALSE;
/* Смотрим список грязных страниц у заданной транзакции. */
MDB_ID2 *list = txn->mt_u.dirty_list;
if (list) {
unsigned i, n = list[0].mid;
for (i = 1; i <= n; i++) {
const MDB_page *dirty = list[i].mptr;
if (dirty == page)
return MDBX_RESULT_TRUE;
}
}
/* При вложенных транзакциях, страница может быть в dirty-списке
* родительской транзакции, но в этом случае она будет скопирована перед
* изменением в текущей транзакции, т.е. относительно заданной транзакции
* проверяемый адрес "не грязный". */
return MDBX_RESULT_FALSE;
}
int mdbx_dbi_open_ex(MDB_txn *txn, const char *name, unsigned flags,
MDB_dbi *pdbi, MDB_cmp_func *keycmp, MDB_cmp_func *datacmp)
{
int rc = mdbx_dbi_open(txn, name, flags, pdbi);
if (likely(rc == MDB_SUCCESS)) {
MDB_dbi dbi = *pdbi;
unsigned flags = txn->mt_dbs[dbi].md_flags;
txn->mt_dbxs[dbi].md_cmp = keycmp ? keycmp : mdbx_default_keycmp(flags);
txn->mt_dbxs[dbi].md_dcmp = datacmp ? datacmp : mdbx_default_datacmp(flags);
}
return rc;
}
/* attribute support functions for Nexenta ***********************************/
static __inline int
mdbx_attr_peek(MDB_val *data, mdbx_attr_t *attrptr)
{
if (unlikely(data->mv_size < sizeof(mdbx_attr_t)))
return MDB_INCOMPATIBLE;
if (likely(attrptr != NULL))
*attrptr = *(mdbx_attr_t*) data->mv_data;
data->mv_size -= sizeof(mdbx_attr_t);
data->mv_data = likely(data->mv_size > 0)
? ((mdbx_attr_t*) data->mv_data) + 1 : NULL;
return MDB_SUCCESS;
}
static __inline int
mdbx_attr_poke(MDB_val *reserved, MDB_val *data, mdbx_attr_t attr, unsigned flags)
{
mdbx_attr_t *space = reserved->mv_data;
if (flags & MDB_RESERVE) {
if (likely(data != NULL)) {
data->mv_data = data->mv_size ? space + 1 : NULL;
}
} else {
*space = attr;
if (likely(data != NULL)) {
memcpy(space + 1, data->mv_data, data->mv_size );
}
}
return MDB_SUCCESS;
}
int
mdbx_cursor_get_attr(MDB_cursor *mc, MDB_val *key, MDB_val *data,
mdbx_attr_t *attrptr, MDB_cursor_op op)
{
int rc = mdbx_cursor_get(mc, key, data, op);
if (unlikely(rc != MDB_SUCCESS))
return rc;
return mdbx_attr_peek(data, attrptr);
}
int
mdbx_get_attr(MDB_txn *txn, MDB_dbi dbi,
MDB_val *key, MDB_val *data, uint64_t *attrptr)
{
int rc = mdbx_get(txn, dbi, key, data);
if (unlikely(rc != MDB_SUCCESS))
return rc;
return mdbx_attr_peek(data, attrptr);
}
int
mdbx_put_attr(MDB_txn *txn, MDB_dbi dbi,
MDB_val *key, MDB_val *data, mdbx_attr_t attr, unsigned flags)
{
MDB_val reserve = {
.mv_data = NULL,
.mv_size = (data ? data->mv_size : 0) + sizeof(mdbx_attr_t)
};
int rc = mdbx_put(txn, dbi, key, &reserve, flags | MDB_RESERVE);
if (unlikely(rc != MDB_SUCCESS))
return rc;
return mdbx_attr_poke(&reserve, data, attr, flags);
}
int mdbx_cursor_put_attr(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
mdbx_attr_t attr, unsigned flags)
{
MDB_val reserve = {
.mv_data = NULL,
.mv_size = (data ? data->mv_size : 0) + sizeof(mdbx_attr_t)
};
int rc = mdbx_cursor_put(cursor, key, &reserve, flags | MDB_RESERVE);
if (unlikely(rc != MDB_SUCCESS))
return rc;
return mdbx_attr_poke(&reserve, data, attr, flags);
}
int mdbx_set_attr(MDB_txn *txn, MDB_dbi dbi,
MDB_val *key, MDB_val *data, mdbx_attr_t attr)
{
MDB_cursor mc;
MDB_xcursor mx;
MDB_val old_data;
mdbx_attr_t old_attr;
int rc;
if (unlikely(!key || !txn))
return EINVAL;
if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE))
return MDB_VERSION_MISMATCH;
if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)))
return EINVAL;
if (unlikely(txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)))
return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
mdb_cursor_init(&mc, txn, dbi, &mx);
rc = mdb_cursor_set(&mc, key, &old_data, MDB_SET, NULL);
if (unlikely(rc != MDB_SUCCESS)) {
if (rc == MDB_NOTFOUND && data) {
mc.mc_next = txn->mt_cursors[dbi];
txn->mt_cursors[dbi] = &mc;
rc = mdbx_cursor_put_attr(&mc, key, data, attr, 0);
txn->mt_cursors[dbi] = mc.mc_next;
}
return rc;
}
rc = mdbx_attr_peek(&old_data, &old_attr);
if (unlikely(rc != MDB_SUCCESS))
return rc;
if (old_attr == attr && (!data ||
(data->mv_size == old_data.mv_size
&& memcpy(data->mv_data, old_data.mv_data, old_data.mv_size) == 0)))
return MDB_SUCCESS;
mc.mc_next = txn->mt_cursors[dbi];
txn->mt_cursors[dbi] = &mc;
rc = mdbx_cursor_put_attr(&mc, key, data ? data : &old_data, attr, MDB_CURRENT);
txn->mt_cursors[dbi] = mc.mc_next;
return rc;
}

1940
mdbx.h
View File

File diff suppressed because it is too large Load Diff

97
mdbx.sln Normal file
View File

@ -0,0 +1,97 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 14
VisualStudioVersion = 14.0.25420.1
MinimumVisualStudioVersion = 10.0.40219.1
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test", "test\test.vcxproj", "{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dll", "dll.vcxproj", "{6D19209B-ECE7-4B9C-941C-0AA2B484F199}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{0A147F9F-22D5-44E6-B389-218CFFB0C524}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mdbx_load", "src\tools\mdbx_load.vcxproj", "{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mdbx_dump", "src\tools\mdbx_dump.vcxproj", "{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mdbx_copy", "src\tools\mdbx_copy.vcxproj", "{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mdbx_chk", "src\tools\mdbx_chk.vcxproj", "{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mdbx_stat", "src\tools\mdbx_stat.vcxproj", "{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64
Debug|x86 = Debug|x86
Release|x64 = Release|x64
Release|x86 = Release|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Debug|x64.ActiveCfg = Debug|x64
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Debug|x64.Build.0 = Debug|x64
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Debug|x86.ActiveCfg = Debug|Win32
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Debug|x86.Build.0 = Debug|Win32
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Release|x64.ActiveCfg = Release|x64
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Release|x64.Build.0 = Release|x64
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Release|x86.ActiveCfg = Release|Win32
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Release|x86.Build.0 = Release|Win32
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Debug|x64.ActiveCfg = Debug|x64
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Debug|x64.Build.0 = Debug|x64
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Debug|x86.ActiveCfg = Debug|Win32
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Debug|x86.Build.0 = Debug|Win32
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Release|x64.ActiveCfg = Release|x64
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Release|x64.Build.0 = Release|x64
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Release|x86.ActiveCfg = Release|Win32
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Release|x86.Build.0 = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Debug|x64.ActiveCfg = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Debug|x64.Build.0 = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Debug|x86.ActiveCfg = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Debug|x86.Build.0 = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Release|x64.ActiveCfg = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Release|x64.Build.0 = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Release|x86.ActiveCfg = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Release|x86.Build.0 = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Debug|x64.ActiveCfg = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Debug|x64.Build.0 = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Debug|x86.ActiveCfg = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Debug|x86.Build.0 = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Release|x64.ActiveCfg = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Release|x64.Build.0 = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Release|x86.ActiveCfg = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Release|x86.Build.0 = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Debug|x64.ActiveCfg = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Debug|x64.Build.0 = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Debug|x86.ActiveCfg = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Debug|x86.Build.0 = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Release|x64.ActiveCfg = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Release|x64.Build.0 = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Release|x86.ActiveCfg = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Release|x86.Build.0 = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Debug|x64.ActiveCfg = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Debug|x64.Build.0 = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Debug|x86.ActiveCfg = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Debug|x86.Build.0 = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Release|x64.ActiveCfg = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Release|x64.Build.0 = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Release|x86.ActiveCfg = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Release|x86.Build.0 = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Debug|x64.ActiveCfg = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Debug|x64.Build.0 = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Debug|x86.ActiveCfg = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Debug|x86.Build.0 = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Release|x64.ActiveCfg = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Release|x64.Build.0 = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Release|x86.ActiveCfg = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Release|x86.Build.0 = Release|Win32
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB} = {0A147F9F-22D5-44E6-B389-218CFFB0C524}
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC} = {0A147F9F-22D5-44E6-B389-218CFFB0C524}
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD} = {0A147F9F-22D5-44E6-B389-218CFFB0C524}
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE} = {0A147F9F-22D5-44E6-B389-218CFFB0C524}
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF} = {0A147F9F-22D5-44E6-B389-218CFFB0C524}
EndGlobalSection
EndGlobal

361
midl.c
View File

@ -1,361 +0,0 @@
/** @file midl.c
* @brief ldap bdb back-end ID List functions */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2000-2017 The OpenLDAP Foundation.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <limits.h>
#include <string.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/types.h>
#include "midl.h"
/** @defgroup internal LMDB Internals
* @{
*/
/** @defgroup idls ID List Management
* @{
*/
static unsigned __hot
mdb_midl_search( MDB_IDL ids, MDB_ID id )
{
/*
* binary search of id in ids
* if found, returns position of id
* if not found, returns first position greater than id
*/
unsigned base = 0;
unsigned cursor = 1;
int val = 0;
unsigned n = ids[0];
while( 0 < n ) {
unsigned pivot = n >> 1;
cursor = base + pivot + 1;
val = mdbx_cmp2int( ids[cursor], id );
if( val < 0 ) {
n = pivot;
} else if ( val > 0 ) {
base = cursor;
n -= pivot + 1;
} else {
return cursor;
}
}
if( val > 0 ) {
++cursor;
}
return cursor;
}
#if 0 /* superseded by append/sort */
static int mdb_midl_insert( MDB_IDL ids, MDB_ID id )
{
unsigned x, i;
x = mdb_midl_search( ids, id );
assert( x > 0 );
if( x < 1 ) {
/* internal error */
return -2;
}
if ( x <= ids[0] && ids[x] == id ) {
/* duplicate */
assert(0);
return -1;
}
if ( ++ids[0] >= MDB_IDL_DB_MAX ) {
/* no room */
--ids[0];
return -2;
} else {
/* insert id */
for (i=ids[0]; i>x; i--)
ids[i] = ids[i-1];
ids[x] = id;
}
return 0;
}
#endif
static MDB_IDL mdb_midl_alloc(int num)
{
MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID));
if (ids) {
*ids++ = num;
*ids = 0;
}
return ids;
}
static void mdb_midl_free(MDB_IDL ids)
{
if (ids)
free(ids-1);
}
static void mdb_midl_shrink( MDB_IDL *idp )
{
MDB_IDL ids = *idp;
if (*(--ids) > MDB_IDL_UM_MAX &&
(ids = realloc(ids, (MDB_IDL_UM_MAX+2) * sizeof(MDB_ID))))
{
*ids++ = MDB_IDL_UM_MAX;
*idp = ids;
}
}
static int mdb_midl_grow( MDB_IDL *idp, int num )
{
MDB_IDL idn = *idp-1;
/* grow it */
idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID));
if (!idn)
return ENOMEM;
*idn++ += num;
*idp = idn;
return 0;
}
static int mdb_midl_need( MDB_IDL *idp, unsigned num )
{
MDB_IDL ids = *idp;
num += ids[0];
if (num > ids[-1]) {
num = (num + num/4 + (256 + 2)) & -256;
if (!(ids = realloc(ids-1, num * sizeof(MDB_ID))))
return ENOMEM;
*ids++ = num - 2;
*idp = ids;
}
return 0;
}
static int mdb_midl_append( MDB_IDL *idp, MDB_ID id )
{
MDB_IDL ids = *idp;
/* Too big? */
if (ids[0] >= ids[-1]) {
if (mdb_midl_grow(idp, MDB_IDL_UM_MAX))
return ENOMEM;
ids = *idp;
}
ids[0]++;
ids[ids[0]] = id;
return 0;
}
static int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app )
{
MDB_IDL ids = *idp;
/* Too big? */
if (ids[0] + app[0] >= ids[-1]) {
if (mdb_midl_grow(idp, app[0]))
return ENOMEM;
ids = *idp;
}
memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID));
ids[0] += app[0];
return 0;
}
static int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n )
{
MDB_ID *ids = *idp, len = ids[0];
/* Too big? */
if (len + n > ids[-1]) {
if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX))
return ENOMEM;
ids = *idp;
}
ids[0] = len + n;
ids += len;
while (n)
ids[n--] = id++;
return 0;
}
static void __hot
mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge )
{
MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k;
idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */
old_id = idl[j];
while (i) {
merge_id = merge[i--];
for (; old_id < merge_id; old_id = idl[--j])
idl[k--] = old_id;
idl[k--] = merge_id;
}
idl[0] = total;
}
/* Quicksort + Insertion sort for small arrays */
#define SMALL 8
#define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; }
static void __hot
mdb_midl_sort( MDB_IDL ids )
{
/* Max possible depth of int-indexed tree * 2 items/level */
int istack[sizeof(int)*CHAR_BIT * 2];
int i,j,k,l,ir,jstack;
MDB_ID a, itmp;
ir = (int)ids[0];
l = 1;
jstack = 0;
for(;;) {
if (ir - l < SMALL) { /* Insertion sort */
for (j=l+1;j<=ir;j++) {
a = ids[j];
for (i=j-1;i>=1;i--) {
if (ids[i] >= a) break;
ids[i+1] = ids[i];
}
ids[i+1] = a;
}
if (jstack == 0) break;
ir = istack[jstack--];
l = istack[jstack--];
} else {
k = (l + ir) >> 1; /* Choose median of left, center, right */
MIDL_SWAP(ids[k], ids[l+1]);
if (ids[l] < ids[ir]) {
MIDL_SWAP(ids[l], ids[ir]);
}
if (ids[l+1] < ids[ir]) {
MIDL_SWAP(ids[l+1], ids[ir]);
}
if (ids[l] < ids[l+1]) {
MIDL_SWAP(ids[l], ids[l+1]);
}
i = l+1;
j = ir;
a = ids[l+1];
for(;;) {
do i++; while(ids[i] > a);
do j--; while(ids[j] < a);
if (j < i) break;
MIDL_SWAP(ids[i],ids[j]);
}
ids[l+1] = ids[j];
ids[j] = a;
jstack += 2;
if (ir-i+1 >= j-l) {
istack[jstack] = ir;
istack[jstack-1] = i;
ir = j-1;
} else {
istack[jstack] = j-1;
istack[jstack-1] = l;
l = i;
}
}
}
}
static unsigned __hot
mdb_mid2l_search( MDB_ID2L ids, MDB_ID id )
{
/*
* binary search of id in ids
* if found, returns position of id
* if not found, returns first position greater than id
*/
unsigned base = 0;
unsigned cursor = 1;
int val = 0;
unsigned n = (unsigned)ids[0].mid;
while( 0 < n ) {
unsigned pivot = n >> 1;
cursor = base + pivot + 1;
val = mdbx_cmp2int( id, ids[cursor].mid );
if( val < 0 ) {
n = pivot;
} else if ( val > 0 ) {
base = cursor;
n -= pivot + 1;
} else {
return cursor;
}
}
if( val > 0 ) {
++cursor;
}
return cursor;
}
static int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id )
{
unsigned x, i;
x = mdb_mid2l_search( ids, id->mid );
if( x < 1 ) {
/* internal error */
return -2;
}
if ( x <= ids[0].mid && ids[x].mid == id->mid ) {
/* duplicate */
return -1;
}
if ( ids[0].mid >= MDB_IDL_UM_MAX ) {
/* too big */
return -2;
} else {
/* insert id */
ids[0].mid++;
for (i=(unsigned)ids[0].mid; i>x; i--)
ids[i] = ids[i-1];
ids[x] = *id;
}
return 0;
}
static int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id )
{
/* Too big? */
if (ids[0].mid >= MDB_IDL_UM_MAX) {
return -2;
}
ids[0].mid++;
ids[ids[0].mid] = *id;
return 0;
}
/** @} */
/** @} */

190
midl.h
View File

@ -1,190 +0,0 @@
/** @file midl.h
* @brief LMDB ID List header file.
*
* This file was originally part of back-bdb but has been
* modified for use in libmdb. Most of the macros defined
* in this file are unused, just left over from the original.
*
* This file is only used internally in libmdb and its definitions
* are not exposed publicly.
*/
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2000-2017 The OpenLDAP Foundation.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#ifndef _MDB_MIDL_H_
#define _MDB_MIDL_H_
#include <stddef.h>
#ifdef __cplusplus
extern "C" {
#endif
/** @defgroup internal LMDB Internals
* @{
*/
/** @defgroup idls ID List Management
* @{
*/
/** A generic unsigned ID number. These were entryIDs in back-bdb.
* Preferably it should have the same size as a pointer.
*/
typedef size_t MDB_ID;
/** An IDL is an ID List, a sorted array of IDs. The first
* element of the array is a counter for how many actual
* IDs are in the list. In the original back-bdb code, IDLs are
* sorted in ascending order. For libmdb IDLs are sorted in
* descending order.
*/
typedef MDB_ID *MDB_IDL;
/* IDL sizes - likely should be even bigger
* limiting factors: sizeof(ID), thread stack size
*/
#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */
#define MDB_IDL_DB_SIZE (1<<MDB_IDL_LOGN)
#define MDB_IDL_UM_SIZE (1<<(MDB_IDL_LOGN+1))
#define MDB_IDL_DB_MAX (MDB_IDL_DB_SIZE-1)
#define MDB_IDL_UM_MAX (MDB_IDL_UM_SIZE-1)
#define MDB_IDL_SIZEOF(ids) (((ids)[0]+1) * sizeof(MDB_ID))
#define MDB_IDL_IS_ZERO(ids) ( (ids)[0] == 0 )
#define MDB_IDL_CPY( dst, src ) (memcpy( dst, src, MDB_IDL_SIZEOF( src ) ))
#define MDB_IDL_FIRST( ids ) ( (ids)[1] )
#define MDB_IDL_LAST( ids ) ( (ids)[(ids)[0]] )
/** Current max length of an #mdb_midl_alloc()ed IDL */
#define MDB_IDL_ALLOCLEN( ids ) ( (ids)[-1] )
#ifdef MDBX_MODE_ENABLED
/** Append ID to IDL. The IDL must be big enough. */
#define mdb_midl_xappend(idl, id) do { \
MDB_ID *xidl = (idl), xlen = ++(xidl[0]); \
xidl[xlen] = (id); \
} while (0)
/** Search for an ID in an IDL.
* @param[in] ids The IDL to search.
* @param[in] id The ID to search for.
* @return The index of the first ID greater than or equal to \b id.
*/
static unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id );
/** Allocate an IDL.
* Allocates memory for an IDL of the given size.
* @return IDL on success, NULL on failure.
*/
static MDB_IDL mdb_midl_alloc(int num);
/** Free an IDL.
* @param[in] ids The IDL to free.
*/
static void mdb_midl_free(MDB_IDL ids);
/** Shrink an IDL.
* Return the IDL to the default size if it has grown larger.
* @param[in,out] idp Address of the IDL to shrink.
*/
static void mdb_midl_shrink(MDB_IDL *idp);
/** Make room for num additional elements in an IDL.
* @param[in,out] idp Address of the IDL.
* @param[in] num Number of elements to make room for.
* @return 0 on success, ENOMEM on failure.
*/
static int mdb_midl_need(MDB_IDL *idp, unsigned num);
/** Append an ID onto an IDL.
* @param[in,out] idp Address of the IDL to append to.
* @param[in] id The ID to append.
* @return 0 on success, ENOMEM if the IDL is too large.
*/
static int mdb_midl_append( MDB_IDL *idp, MDB_ID id );
/** Append an IDL onto an IDL.
* @param[in,out] idp Address of the IDL to append to.
* @param[in] app The IDL to append.
* @return 0 on success, ENOMEM if the IDL is too large.
*/
static int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app );
/** Append an ID range onto an IDL.
* @param[in,out] idp Address of the IDL to append to.
* @param[in] id The lowest ID to append.
* @param[in] n Number of IDs to append.
* @return 0 on success, ENOMEM if the IDL is too large.
*/
static int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n );
/** Merge an IDL onto an IDL. The destination IDL must be big enough.
* @param[in] idl The IDL to merge into.
* @param[in] merge The IDL to merge.
*/
static void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge );
/** Sort an IDL.
* @param[in,out] ids The IDL to sort.
*/
static void mdb_midl_sort( MDB_IDL ids );
/** An ID2 is an ID/pointer pair.
*/
typedef struct MDB_ID2 {
MDB_ID mid; /**< The ID */
void *mptr; /**< The pointer */
} MDB_ID2;
/** An ID2L is an ID2 List, a sorted array of ID2s.
* The first element's \b mid member is a count of how many actual
* elements are in the array. The \b mptr member of the first element is unused.
* The array is sorted in ascending order by \b mid.
*/
typedef MDB_ID2 *MDB_ID2L;
/** Search for an ID in an ID2L.
* @param[in] ids The ID2L to search.
* @param[in] id The ID to search for.
* @return The index of the first ID2 whose \b mid member is greater than or equal to \b id.
*/
static unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id );
/** Insert an ID2 into a ID2L.
* @param[in,out] ids The ID2L to insert into.
* @param[in] id The ID2 to insert.
* @return 0 on success, -1 if the ID was already present in the ID2L.
*/
static int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id );
/** Append an ID2 into a ID2L.
* @param[in,out] ids The ID2L to append into.
* @param[in] id The ID2 to append.
* @return 0 on success, -2 if the ID2L is too big.
*/
static int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id );
#endif /* #ifdef MDBX_MODE_ENABLED */
/** @} */
/** @} */
#ifdef __cplusplus
}
#endif
#endif /* _MDB_MIDL_H_ */

223
mtest0.c
View File

@ -1,223 +0,0 @@
/* mtest.c - memory-mapped database tester/toy */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2011-2017 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <unistd.h>
#include <errno.h>
#include <sys/stat.h>
#include "mdbx.h"
#include <pthread.h>
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
#ifndef DBPATH
# define DBPATH "./testdb"
#endif
void* thread_entry(void *ctx)
{
MDB_env *env = ctx;
MDB_txn *txn;
int rc;
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
mdb_txn_abort(txn);
return NULL;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor, *cur2;
MDB_cursor_op op;
int count;
int *values;
char sval[32] = "";
int env_oflags;
struct stat db_stat, exe_stat;
(void) argc;
(void) argv;
srand(time(NULL));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 42));
E(mdb_env_set_mapsize(env, 10485760));
E(stat("/proc/self/exe", &exe_stat)?errno:0);
E(stat(DBPATH "/.", &db_stat)?errno:0);
env_oflags = MDB_FIXEDMAP | MDB_NOSYNC;
if (major(db_stat.st_dev) != major(exe_stat.st_dev)) {
/* LY: Assume running inside a CI-environment:
* 1) don't use FIXEDMAP to avoid EBUSY in case collision,
* which could be inspired by address space randomisation feature.
* 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage.
*/
env_oflags = 0;
}
E(mdb_env_open(env, DBPATH, env_oflags, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, NULL, 0, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
/* Set <data> in each iteration, since MDB_NOOVERWRITE may modify it */
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
j++;
data.mv_size = sizeof(sval);
data.mv_data = sval;
}
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
key.mv_data = sval;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%03x ", values[i]);
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor last\n");
E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor last/prev\n");
E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
E(mdb_cursor_get(cursor, &key, &data, MDB_PREV));
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
printf("Deleting with cursor\n");
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_cursor_open(txn, dbi, &cur2));
for (i=0; i<50; i++) {
if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT)))
break;
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
E(mdb_del(txn, dbi, &key, NULL));
}
printf("Restarting cursor in txn\n");
for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op)))
break;
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cur2);
E(mdb_txn_commit(txn));
for(i = 0; i < 41; ++i) {
pthread_t thread;
pthread_create(&thread, NULL, thread_entry, env);
}
printf("Restarting cursor outside txn\n");
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op)))
break;
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

200
mtest1.c
View File

@ -1,200 +0,0 @@
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/* Based on mtest2.c - memory-mapped database tester/toy */
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <unistd.h>
#include <errno.h>
#include <sys/stat.h>
#include "mdbx.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
#ifndef DBPATH
# define DBPATH "./testdb"
#endif
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32] = "";
int env_oflags;
struct stat db_stat, exe_stat;
(void) argc;
(void) argv;
srand(time(NULL));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(stat("/proc/self/exe", &exe_stat)?errno:0);
E(stat(DBPATH "/.", &db_stat)?errno:0);
env_oflags = MDB_FIXEDMAP | MDB_NOSYNC;
if (major(db_stat.st_dev) != major(exe_stat.st_dev)) {
/* LY: Assume running inside a CI-environment:
* 1) don't use FIXEDMAP to avoid EBUSY in case collision,
* which could be inspired by address space randomisation feature.
* 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage.
*/
env_oflags = 0;
}
/* LY: especially here we always needs MDB_NOSYNC
* for testing mdbx_env_close_ex() and "redo-to-steady" on open. */
env_oflags |= MDB_NOSYNC;
E(mdb_env_open(env, DBPATH, env_oflags, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
if (mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi) == MDB_SUCCESS)
E(mdb_drop(txn, dbi, 1));
E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
printf("check-preset-a\n");
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_a = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
++present_a;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_a == count - j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_env_sync(env, 1);
int deleted = 0;
key.mv_data = sval;
for (i = count - 1; i > -1; i -= (rand()%5)) {
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%03x ", values[i]);
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
deleted++;
}
}
free(values);
printf("Deleted %d values\n", deleted);
printf("check-preset-b.cursor-next\n");
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_b = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++present_b;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_b == present_a - deleted, "mismatch");
printf("check-preset-b.cursor-prev\n");
j = 1;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++j;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_b == j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
/********************* LY: kept DB dirty ****************/
mdbx_env_close_ex(env, 1);
E(mdb_env_create(&env));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, DBPATH, env_oflags, 0664));
printf("check-preset-c.cursor-next\n");
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_dbi_open(txn, "id1", 0, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
int present_c = 0;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++present_c;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Rolled back %d deletion(s)\n", present_c - (present_a - deleted));
CHECK(present_c > present_a - deleted, "mismatch");
printf("check-preset-d.cursor-prev\n");
j = 1;
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
++j;
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
CHECK(present_c == j, "mismatch");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdbx_env_close_ex(env, 0);
return 0;
}

152
mtest2.c
View File

@ -1,152 +0,0 @@
/* mtest2.c - memory-mapped database tester/toy */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2011-2017 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/* Just like mtest.c, but using a subDB instead of the main DB */
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <unistd.h>
#include <errno.h>
#include <sys/stat.h>
#include "mdbx.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
#ifndef DBPATH
# define DBPATH "./testdb"
#endif
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32] = "";
int env_oflags;
struct stat db_stat, exe_stat;
(void) argc;
(void) argv;
srand(time(NULL));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(stat("/proc/self/exe", &exe_stat)?errno:0);
E(stat(DBPATH "/.", &db_stat)?errno:0);
env_oflags = MDB_FIXEDMAP | MDB_NOSYNC;
if (major(db_stat.st_dev) != major(exe_stat.st_dev)) {
/* LY: Assume running inside a CI-environment:
* 1) don't use FIXEDMAP to avoid EBUSY in case collision,
* which could be inspired by address space randomisation feature.
* 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage.
*/
env_oflags = 0;
}
E(mdb_env_open(env, DBPATH, env_oflags, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
if (mdb_dbi_open(txn, "id2", MDB_CREATE, &dbi) == MDB_SUCCESS)
E(mdb_drop(txn, dbi, 1));
E(mdb_dbi_open(txn, "id2", MDB_CREATE, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
key.mv_data = sval;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%03x ", values[i]);
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

161
mtest3.c
View File

@ -1,161 +0,0 @@
/* mtest3.c - memory-mapped database tester/toy */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2011-2017 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/* Tests for sorted duplicate DBs */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <errno.h>
#include <sys/stat.h>
#include "mdbx.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
#ifndef DBPATH
# define DBPATH "./testdb"
#endif
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32];
char kval[sizeof(int)];
int env_oflags;
struct stat db_stat, exe_stat;
(void) argc;
(void) argv;
srand(time(NULL));
memset(sval, 0, sizeof(sval));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(stat("/proc/self/exe", &exe_stat)?errno:0);
E(stat(DBPATH "/.", &db_stat)?errno:0);
env_oflags = MDB_FIXEDMAP | MDB_NOSYNC;
if (major(db_stat.st_dev) != major(exe_stat.st_dev)) {
/* LY: Assume running inside a CI-environment:
* 1) don't use FIXEDMAP to avoid EBUSY in case collision,
* which could be inspired by address space randomisation feature.
* 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage.
*/
env_oflags = 0;
}
E(mdb_env_open(env, DBPATH, env_oflags, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
if (mdb_dbi_open(txn, "id3", MDB_CREATE, &dbi) == MDB_SUCCESS)
E(mdb_drop(txn, dbi, 1));
E(mdb_dbi_open(txn, "id3", MDB_CREATE|MDB_DUPSORT, &dbi));
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
if (!(i & 0x0f))
sprintf(kval, "%03x", values[i]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(kval, "%03x", values[i & ~0x0f]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

196
mtest4.c
View File

@ -1,196 +0,0 @@
/* mtest4.c - memory-mapped database tester/toy */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2011-2017 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/* Tests for sorted duplicate DBs with fixed-size keys */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <errno.h>
#include <sys/stat.h>
#include "mdbx.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
#ifndef DBPATH
# define DBPATH "./testdb"
#endif
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[8];
char kval[sizeof(int)];
int env_oflags;
struct stat db_stat, exe_stat;
(void) argc;
(void) argv;
memset(sval, 0, sizeof(sval));
count = 510;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = i*5;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(stat("/proc/self/exe", &exe_stat)?errno:0);
E(stat(DBPATH "/.", &db_stat)?errno:0);
env_oflags = MDB_FIXEDMAP | MDB_NOSYNC;
if (major(db_stat.st_dev) != major(exe_stat.st_dev)) {
/* LY: Assume running inside a CI-environment:
* 1) don't use FIXEDMAP to avoid EBUSY in case collision,
* which could be inspired by address space randomisation feature.
* 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage.
*/
env_oflags = 0;
}
E(mdb_env_open(env, DBPATH, env_oflags, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
if (mdb_dbi_open(txn, "id4", MDB_CREATE, &dbi) == MDB_SUCCESS)
E(mdb_drop(txn, dbi, 1));
E(mdb_dbi_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
strcpy(kval, "001");
for (i=0;i<count;i++) {
sprintf(sval, "%07x", values[i]);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
/* there should be one full page of dups now.
*/
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
/* test all 3 branches of split code:
* 1: new key in lower half
* 2: new key at split point
* 3: new key in upper half
*/
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
sprintf(sval, "%07x", values[3]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
mdb_txn_abort(txn);
sprintf(sval, "%07x", values[255]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
mdb_txn_abort(txn);
sprintf(sval, "%07x", values[500]+1);
E(mdb_txn_begin(env, NULL, 0, &txn));
(void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA));
E(mdb_txn_commit(txn));
/* Try MDB_NEXT_MULTIPLE */
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_MULTIPLE)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
for (i= count - 1; i > -1; i-= (rand()%3)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(sval, "%07x", values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

163
mtest5.c
View File

@ -1,163 +0,0 @@
/* mtest5.c - memory-mapped database tester/toy */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2011-2017 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/* Tests for sorted duplicate DBs using cursor_put */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <errno.h>
#include <sys/stat.h>
#include "mdbx.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
#ifndef DBPATH
# define DBPATH "./testdb"
#endif
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
int count;
int *values;
char sval[32];
char kval[sizeof(int)];
int env_oflags;
struct stat db_stat, exe_stat;
(void) argc;
(void) argv;
srand(time(NULL));
memset(sval, 0, sizeof(sval));
count = (rand()%384) + 64;
values = (int *)malloc(count*sizeof(int));
for(i = 0;i<count;i++) {
values[i] = rand()%1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(stat("/proc/self/exe", &exe_stat)?errno:0);
E(stat(DBPATH "/.", &db_stat)?errno:0);
env_oflags = MDB_FIXEDMAP | MDB_NOSYNC;
if (major(db_stat.st_dev) != major(exe_stat.st_dev)) {
/* LY: Assume running inside a CI-environment:
* 1) don't use FIXEDMAP to avoid EBUSY in case collision,
* which could be inspired by address space randomisation feature.
* 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage.
*/
env_oflags = 0;
}
E(mdb_env_open(env, DBPATH, env_oflags, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
if (mdb_dbi_open(txn, "id5", MDB_CREATE, &dbi) == MDB_SUCCESS)
E(mdb_drop(txn, dbi, 1));
E(mdb_dbi_open(txn, "id5", MDB_CREATE|MDB_DUPSORT, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
if (!(i & 0x0f))
sprintf(kval, "%03x", values[i]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
if (RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NODUPDATA)))
j++;
}
if (j) printf("%d duplicates skipped\n", j);
mdb_cursor_close(cursor);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(kval, "%03x", values[i & ~0x0f]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

170
mtest6.c
View File

@ -1,170 +0,0 @@
/* mtest6.c - memory-mapped database tester/toy */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2011-2017 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/* Tests for DB splits and merges */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <errno.h>
#include <sys/stat.h>
#include "mdbx.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
#ifndef DBPATH
# define DBPATH "./testdb"
#endif
char dkbuf[1024];
int main(int argc,char * argv[])
{
int i = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data, sdata;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
long kval;
char *sval;
int env_oflags;
struct stat db_stat, exe_stat;
(void) argc;
(void) argv;
srand(time(NULL));
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(stat("/proc/self/exe", &exe_stat)?errno:0);
E(stat(DBPATH "/.", &db_stat)?errno:0);
env_oflags = MDB_FIXEDMAP | MDB_NOSYNC;
if (major(db_stat.st_dev) != major(exe_stat.st_dev)) {
/* LY: Assume running inside a CI-environment:
* 1) don't use FIXEDMAP to avoid EBUSY in case collision,
* which could be inspired by address space randomisation feature.
* 2) drop MDB_NOSYNC expecting that DBPATH is at a tmpfs or some dedicated storage.
*/
env_oflags = 0;
}
E(mdb_env_open(env, DBPATH, env_oflags, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
if (mdb_dbi_open(txn, "id6", MDB_CREATE, &dbi) == MDB_SUCCESS)
E(mdb_drop(txn, dbi, 1));
E(mdb_dbi_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
E(mdb_stat(txn, dbi, &mst));
sval = calloc(1, mst.ms_psize / 4);
key.mv_size = sizeof(long);
key.mv_data = &kval;
sdata.mv_size = mst.ms_psize / 4 - 30;
sdata.mv_data = sval;
printf("Adding 12 values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5;
sprintf(sval, "%08lx", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+4;
sprintf(sval, "%08lx", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+1;
sprintf(sval, "%08lx", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST));
do {
printf("key: %p %s, data: %p %.*s\n",
key.mv_data, mdb_dkey(&key, dkbuf),
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
} while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0);
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_commit(txn);
#if 0
int j=0;
int count = 333;
int *values = alloca(sizeof(int) * count);
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
sprintf(kval, "%03x", values[i & ~0x0f]);
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
key.mv_size = sizeof(int);
key.mv_data = kval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
#endif
mdb_env_close(env);
free(sval);
return 0;
}

124
mtest7.c
View File

@ -1,124 +0,0 @@
/* mtest7.c - memory-mapped database tester/toy */
/*
* Copyright 2015 Ilya Usvyatsky, Nexenta Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/* Tests for DB attributes */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <sys/time.h>
#include "mdbx.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
char dkbuf[1024];
#ifndef DBPATH
# define DBPATH "./testdb/data.mdb"
#endif
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
int count;
int *values;
char sval[32];
uint64_t *timestamps, timestamp;
struct timeval tv;
int env_opt = MDB_NOMEMINIT | MDB_NOSYNC | MDB_NOSUBDIR | MDB_NORDAHEAD;
srand(time(NULL));
memset(sval, 0, sizeof(sval));
count = (rand()%384) + 64;
if (argc > 1)
count = atoi(argv[1]);
values = (int *)malloc(count*sizeof(int));
timestamps = (uint64_t *)calloc(count,sizeof(uint64_t));
unlink(DBPATH);
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 104857600));
E(mdb_env_set_maxdbs(env, 8));
E(mdb_env_open(env, DBPATH, env_opt, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, "id7", MDB_CREATE|MDB_INTEGERKEY, &dbi));
key.mv_size = sizeof(int);
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
(void)gettimeofday(&tv, NULL);
timestamps[i] = tv.tv_usec + 1000000UL * tv.tv_sec;
values[i] = rand()%16383 ^ (timestamps[i] & 0xffff);
key.mv_data = values + i;
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
E(mdbx_put_attr(txn, dbi, &key, &data, timestamps[i], MDB_NODUPDATA));
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
mdb_env_close(env);
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 8));
E(mdb_env_open(env, DBPATH, env_opt, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, "id7", MDB_CREATE|MDB_INTEGERKEY, &dbi));
for (i=0;!rc&&i<count;i++) {
if (!timestamps[i])
continue;
key.mv_data = values + i;
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
E(mdbx_get_attr(txn, dbi, &key, &data, &timestamp));
if (timestamps[i] != timestamp) {
for (j = 0; j < count; ++j) {
if (j != i && values[i] == values[j] &&
timestamp == timestamps[j]) {
printf("Duplicate keys "
"%d %d %d %d %lu %lu\n",
i, j, values[i], values[j],
timestamps[i], timestamps[j]);
break;
}
}
if (j >= count) {
printf("Timestamp mismatch "
"%d %03x %d %lu != %lu\n",
i, values[i], values[i], timestamps[i],
timestamp);
break;
}
}
}
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
mdb_env_close(env);
return 0;
}

146
mtest8.c
View File

@ -1,146 +0,0 @@
/* mtest8.c - memory-mapped database tester/toy */
/*
* Copyright 2015 Ilya Usvyatsky, Nexenta Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
/* Tests for DB attributes */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <sys/time.h>
#include "mdbx.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
char dkbuf[1024];
#ifndef DBPATH
# define DBPATH "./testdb/data.mdb"
#endif
int main(int argc,char * argv[])
{
int i = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
int count;
int *values;
char sval[8000];
uint64_t *timestamps, timestamp;
struct timeval tv;
int env_opt = MDB_NOMEMINIT | MDB_NOSYNC | MDB_NOSUBDIR | MDB_NORDAHEAD;
srand(time(NULL));
memset(sval, 0, sizeof(sval));
count = 2000; //(rand()%384) + 64;
if (argc > 1)
count = atoi(argv[1]);
values = (int *)malloc(count*sizeof(int));
timestamps = (uint64_t *)calloc(count,sizeof(uint64_t));
key.mv_size = sizeof(int);
data.mv_size = sizeof(sval);
data.mv_data = sval;
values[0] = 42;
values[1] = 17;
for (i = 2; i < count; ++i)
values[i] = values[i - 1] + values[i - 2];
unlink(DBPATH);
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 104857600));
E(mdb_env_set_maxdbs(env, 8));
E(mdb_env_open(env, DBPATH, env_opt, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, "id8", MDB_CREATE|MDB_INTEGERKEY, &dbi));
for (i = 0; i < count; ++i) {
(void)gettimeofday(&tv, NULL);
timestamps[i] = tv.tv_usec + 1000000UL * tv.tv_sec;
snprintf(sval, 4000, "Value %d\n", values[i]);
snprintf(sval + 4000, 4000, "Value %d\n", values[i]);
key.mv_data = values + i;
E(mdbx_put_attr(txn, dbi, &key, &data, timestamps[i], MDB_NOOVERWRITE));
}
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
mdb_env_close(env);
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 8));
E(mdb_env_open(env, DBPATH, env_opt, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, "id8", MDB_INTEGERKEY, &dbi));
for (i = 0; i < count; ++i) {
key.mv_data = values + i;
E(mdbx_get_attr(txn, dbi, &key, &data, &timestamp));
E(timestamps[i] != timestamp);
}
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
mdb_env_close(env);
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 104857600));
E(mdb_env_set_maxdbs(env, 8));
E(mdb_env_open(env, DBPATH, env_opt, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, "id8", MDB_INTEGERKEY, &dbi));
for (i = 0; i < count; ++i) {
(void)gettimeofday(&tv, NULL);
timestamps[i] = tv.tv_usec + 1000000UL * tv.tv_sec;
key.mv_data = values + i;
E(mdbx_set_attr(txn, dbi, &key, NULL, timestamps[i]));
}
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
mdb_env_close(env);
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 8));
E(mdb_env_open(env, DBPATH, env_opt, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, "id8", MDB_INTEGERKEY, &dbi));
for (i = 0; i < count; ++i) {
key.mv_data = values + i;
E(mdbx_get_attr(txn, dbi, &key, &data, &timestamp));
E(timestamps[i] != timestamp);
}
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
mdb_env_close(env);
return 0;
}

66
sample-mdb.txt
View File

@ -1,66 +0,0 @@
/* sample-mdb.txt - MDB toy/sample
*
* Do a line-by-line comparison of this and sample-bdb.txt
*/
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2012-2017 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include "lmdb.h"
int main(int argc,char * argv[])
{
int rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_cursor *cursor;
char sval[32];
/* Note: Most error checking omitted for simplicity */
rc = mdb_env_create(&env);
rc = mdb_env_open(env, "./testdb", 0, 0664);
rc = mdb_txn_begin(env, NULL, 0, &txn);
rc = mdb_dbi_open(txn, NULL, 0, &dbi);
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
sprintf(sval, "%03x %d foo bar", 32, 3141592);
rc = mdb_put(txn, dbi, &key, &data, 0);
rc = mdb_txn_commit(txn);
if (rc) {
fprintf(stderr, "mdb_txn_commit: (%d) %s\n", rc, mdb_strerror(rc));
goto leave;
}
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
rc = mdb_cursor_open(txn, dbi, &cursor);
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
leave:
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

1197
src/bits.h Normal file
View File

File diff suppressed because it is too large Load Diff

393
src/defs.h Normal file
View File

@ -0,0 +1,393 @@
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
/* *INDENT-OFF* */
/* clang-format off */
#ifndef __GNUC_PREREQ
# if defined(__GNUC__) && defined(__GNUC_MINOR__)
# define __GNUC_PREREQ(maj, min) \
((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
# else
# define __GNUC_PREREQ(maj, min) (0)
# endif
#endif /* __GNUC_PREREQ */
#ifndef __CLANG_PREREQ
# ifdef __clang__
# define __CLANG_PREREQ(maj,min) \
((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
# else
# define __CLANG_PREREQ(maj,min) (0)
# endif
#endif /* __CLANG_PREREQ */
#ifndef __GLIBC_PREREQ
# if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
# define __GLIBC_PREREQ(maj, min) \
((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
# else
# define __GLIBC_PREREQ(maj, min) (0)
# endif
#endif /* __GLIBC_PREREQ */
#ifndef __has_attribute
# define __has_attribute(x) (0)
#endif
#ifndef __has_feature
# define __has_feature(x) (0)
#endif
#ifndef __has_extension
# define __has_extension(x) (0)
#endif
#ifndef __has_builtin
# define __has_builtin(x) (0)
#endif
#if __has_feature(thread_sanitizer)
# define __SANITIZE_THREAD__ 1
#endif
#if __has_feature(address_sanitizer)
# define __SANITIZE_ADDRESS__ 1
#endif
/*----------------------------------------------------------------------------*/
#ifndef __extern_C
# ifdef __cplusplus
# define __extern_C extern "C"
# else
# define __extern_C
# endif
#endif /* __extern_C */
#ifndef __cplusplus
# ifndef bool
# define bool _Bool
# endif
# ifndef true
# define true (1)
# endif
# ifndef false
# define false (0)
# endif
#endif
#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
# define nullptr NULL
#endif
/*----------------------------------------------------------------------------*/
#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__))
# define __thread __declspec(thread)
#endif /* __thread */
#ifndef __alwaysinline
# if defined(__GNUC__) || __has_attribute(always_inline)
# define __alwaysinline __inline __attribute__((always_inline))
# elif defined(_MSC_VER)
# define __alwaysinline __forceinline
# else
# define __alwaysinline
# endif
#endif /* __alwaysinline */
#ifndef __noinline
# if defined(__GNUC__) || __has_attribute(noinline)
# define __noinline __attribute__((noinline))
# elif defined(_MSC_VER)
# define __noinline __declspec(noinline)
# elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
# define __noinline inline
# elif !defined(__INTEL_COMPILER)
# define __noinline /* FIXME ? */
# endif
#endif /* __noinline */
#ifndef __must_check_result
# if defined(__GNUC__) || __has_attribute(warn_unused_result)
# define __must_check_result __attribute__((warn_unused_result))
# else
# define __must_check_result
# endif
#endif /* __must_check_result */
#ifndef __deprecated
# if defined(__GNUC__) || __has_attribute(deprecated)
# define __deprecated __attribute__((deprecated))
# elif defined(_MSC_VER)
# define __deprecated __declspec(deprecated)
# else
# define __deprecated
# endif
#endif /* __deprecated */
#ifndef __packed
# if defined(__GNUC__) || __has_attribute(packed)
# define __packed __attribute__((packed))
# else
# define __packed
# endif
#endif /* __packed */
#ifndef __aligned
# if defined(__GNUC__) || __has_attribute(aligned)
# define __aligned(N) __attribute__((aligned(N)))
# elif defined(_MSC_VER)
# define __aligned(N) __declspec(align(N))
# else
# define __aligned(N)
# endif
#endif /* __aligned */
#ifndef __noreturn
# if defined(__GNUC__) || __has_attribute(noreturn)
# define __noreturn __attribute__((noreturn))
# elif defined(_MSC_VER)
# define __noreturn __declspec(noreturn)
# else
# define __noreturn
# endif
#endif /* __noreturn */
#ifndef __nothrow
# if defined(__GNUC__) || __has_attribute(nothrow)
# define __nothrow __attribute__((nothrow))
# elif defined(_MSC_VER) && defined(__cplusplus)
# define __nothrow __declspec(nothrow)
# else
# define __nothrow
# endif
#endif /* __nothrow */
#ifndef __pure_function
/* Many functions have no effects except the return value and their
* return value depends only on the parameters and/or global variables.
* Such a function can be subject to common subexpression elimination
* and loop optimization just as an arithmetic operator would be.
* These functions should be declared with the attribute pure. */
# if defined(__GNUC__) || __has_attribute(pure)
# define __pure_function __attribute__((pure))
# else
# define __pure_function
# endif
#endif /* __pure_function */
#ifndef __const_function
/* Many functions do not examine any values except their arguments,
* and have no effects except the return value. Basically this is just
* slightly more strict class than the PURE attribute, since function
* is not allowed to read global memory.
*
* Note that a function that has pointer arguments and examines the
* data pointed to must not be declared const. Likewise, a function
* that calls a non-const function usually must not be const.
* It does not make sense for a const function to return void. */
# if defined(__GNUC__) || __has_attribute(const)
# define __const_function __attribute__((const))
# else
# define __const_function
# endif
#endif /* __const_function */
#ifndef __dll_hidden
# if defined(__GNUC__) || __has_attribute(visibility)
# define __hidden __attribute__((visibility("hidden")))
# else
# define __hidden
# endif
#endif /* __dll_hidden */
#ifndef __optimize
# if defined(__OPTIMIZE__)
# if defined(__clang__) && !__has_attribute(optimize)
# define __optimize(ops)
# elif defined(__GNUC__) || __has_attribute(optimize)
# define __optimize(ops) __attribute__((optimize(ops)))
# else
# define __optimize(ops)
# endif
# else
# define __optimize(ops)
# endif
#endif /* __optimize */
#ifndef __hot
# if defined(__OPTIMIZE__)
# if defined(__clang__) && !__has_attribute(hot)
/* just put frequently used functions in separate section */
# define __hot __attribute__((section("text.hot"))) __optimize("O3")
# elif defined(__GNUC__) || __has_attribute(hot)
# define __hot __attribute__((hot)) __optimize("O3")
# else
# define __hot __optimize("O3")
# endif
# else
# define __hot
# endif
#endif /* __hot */
#ifndef __cold
# if defined(__OPTIMIZE__)
# if defined(__clang__) && !__has_attribute(cold)
/* just put infrequently used functions in separate section */
# define __cold __attribute__((section("text.unlikely"))) __optimize("Os")
# elif defined(__GNUC__) || __has_attribute(cold)
# define __cold __attribute__((cold)) __optimize("Os")
# else
# define __cold __optimize("Os")
# endif
# else
# define __cold
# endif
#endif /* __cold */
#ifndef __flatten
# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(flatten))
# define __flatten __attribute__((flatten))
# else
# define __flatten
# endif
#endif /* __flatten */
#ifndef likely
# if defined(__GNUC__) || defined(__clang__)
# define likely(cond) __builtin_expect(!!(cond), 1)
# else
# define likely(x) (x)
# endif
#endif /* likely */
#ifndef unlikely
# if defined(__GNUC__) || defined(__clang__)
# define unlikely(cond) __builtin_expect(!!(cond), 0)
# else
# define unlikely(x) (x)
# endif
#endif /* unlikely */
#if !defined(__noop) && !defined(_MSC_VER)
static __inline int __do_noop(void* crutch, ...) {
(void) crutch; return 0;
}
# define __noop(...) __do_noop(0, __VA_ARGS__)
#endif /* __noop */
/* Wrapper around __func__, which is a C99 feature */
#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
# define mdbx_func_ __func__
#elif (defined(__GNUC__) && __GNUC__ >= 2) || defined(__clang__) || defined(_MSC_VER)
# define mdbx_func_ __FUNCTION__
#else
# define mdbx_func_ "<mdbx_unknown>"
#endif
/*----------------------------------------------------------------------------*/
#if defined(USE_VALGRIND)
# include <valgrind/memcheck.h>
# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE
/* LY: available since Valgrind 3.10 */
# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# endif
#else
# define VALGRIND_CREATE_MEMPOOL(h,r,z)
# define VALGRIND_DESTROY_MEMPOOL(h)
# define VALGRIND_MEMPOOL_TRIM(h,a,s)
# define VALGRIND_MEMPOOL_ALLOC(h,a,s)
# define VALGRIND_MEMPOOL_FREE(h,a)
# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
# define VALGRIND_MAKE_MEM_NOACCESS(a,s)
# define VALGRIND_MAKE_MEM_DEFINED(a,s)
# define VALGRIND_MAKE_MEM_UNDEFINED(a,s)
# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0)
# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0)
# define RUNNING_ON_VALGRIND (0)
#endif /* USE_VALGRIND */
#ifdef __SANITIZE_ADDRESS__
# include <sanitizer/asan_interface.h>
#else
# define ASAN_POISON_MEMORY_REGION(addr, size) \
((void)(addr), (void)(size))
# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
((void)(addr), (void)(size))
#endif /* __SANITIZE_ADDRESS__ */
/*----------------------------------------------------------------------------*/
#ifndef ARRAY_LENGTH
# ifdef __cplusplus
template <typename T, size_t N>
char (&__ArraySizeHelper(T (&array)[N]))[N];
# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array)))
# else
# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
# endif
#endif /* ARRAY_LENGTH */
#ifndef ARRAY_END
# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)])
#endif /* ARRAY_END */
#ifndef STRINGIFY
# define STRINGIFY_HELPER(x) #x
# define STRINGIFY(x) STRINGIFY_HELPER(x)
#endif /* STRINGIFY */
#ifndef offsetof
# define offsetof(type, member) __builtin_offsetof(type, member)
#endif /* offsetof */
#ifndef container_of
# define container_of(ptr, type, member) \
((type *)((char *)(ptr) - offsetof(type, member)))
#endif /* container_of */
#define MDBX_TETRAD(a, b, c, d) \
((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3])
#define FIXME "FIXME: " __FILE__ ", " STRINGIFY(__LINE__)
#ifndef STATIC_ASSERT_MSG
# if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \
|| __has_feature(c_static_assert)
# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg)
# elif defined(static_assert)
# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg)
# elif defined(_MSC_VER)
# include <crtdbg.h>
# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
# else
# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;}
# endif
#endif /* STATIC_ASSERT */
#ifndef STATIC_ASSERT
# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr)
#endif
/* *INDENT-ON* */
/* clang-format on */

321
src/lck-posix.c Normal file
View File

@ -0,0 +1,321 @@
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "./bits.h"
/* Some platforms define the EOWNERDEAD error code
* even though they don't support Robust Mutexes.
* Compile with -DMDBX_USE_ROBUST=0. */
#ifndef MDBX_USE_ROBUST
/* Howard Chu: Android currently lacks Robust Mutex support */
#if defined(EOWNERDEAD) && \
!defined(ANDROID) /* LY: glibc before 2.10 has a troubles with Robust \
Mutex too. */ \
&& __GLIBC_PREREQ(2, 10)
#define MDBX_USE_ROBUST 1
#else
#define MDBX_USE_ROBUST 0
#endif
#endif /* MDBX_USE_ROBUST */
/*----------------------------------------------------------------------------*/
/* rthc */
static pthread_mutex_t mdbx_rthc_mutex = PTHREAD_MUTEX_INITIALIZER;
void mdbx_rthc_lock(void) {
mdbx_ensure(NULL, pthread_mutex_lock(&mdbx_rthc_mutex) == 0);
}
void mdbx_rthc_unlock(void) {
mdbx_ensure(NULL, pthread_mutex_unlock(&mdbx_rthc_mutex) == 0);
}
void __attribute__((destructor)) mdbx_global_destructor(void) {
mdbx_rthc_cleanup();
}
/*----------------------------------------------------------------------------*/
/* lck */
#ifndef OFF_T_MAX
#define OFF_T_MAX (sizeof(off_t) > 4 ? INT64_MAX : INT32_MAX)
#endif
#define LCK_WHOLE OFF_T_MAX
static int mdbx_lck_op(mdbx_filehandle_t fd, int op, int lck, off_t offset,
off_t len) {
for (;;) {
int rc;
struct flock lock_op;
memset(&lock_op, 0, sizeof(lock_op));
lock_op.l_type = lck;
lock_op.l_whence = SEEK_SET;
lock_op.l_start = offset;
lock_op.l_len = len;
if ((rc = fcntl(fd, op, &lock_op)) == 0) {
if (op == F_GETLK && lock_op.l_type != F_UNLCK)
rc = -lock_op.l_pid;
} else if ((rc = errno) == EINTR) {
continue;
}
return rc;
}
}
static __inline int mdbx_lck_exclusive(int lfd) {
assert(lfd != INVALID_HANDLE_VALUE);
if (flock(lfd, LOCK_EX | LOCK_NB))
return errno;
return mdbx_lck_op(lfd, F_SETLK, F_WRLCK, 0, 1);
}
static __inline int mdbx_lck_shared(int lfd) {
assert(lfd != INVALID_HANDLE_VALUE);
while (flock(lfd, LOCK_SH)) {
int rc = errno;
if (rc != EINTR)
return rc;
}
return mdbx_lck_op(lfd, F_SETLKW, F_RDLCK, 0, 1);
}
int mdbx_lck_downgrade(MDBX_env *env, bool complete) {
return complete ? mdbx_lck_shared(env->me_lfd) : MDBX_SUCCESS;
}
int mdbx_lck_upgrade(MDBX_env *env) { return mdbx_lck_exclusive(env->me_lfd); }
int mdbx_rpid_set(MDBX_env *env) {
return mdbx_lck_op(env->me_lfd, F_SETLK, F_WRLCK, env->me_pid, 1);
}
int mdbx_rpid_clear(MDBX_env *env) {
return mdbx_lck_op(env->me_lfd, F_SETLKW, F_UNLCK, env->me_pid, 1);
}
/* Checks reader by pid.
*
* Returns:
* MDBX_RESULT_TRUE, if pid is live (unable to acquire lock)
* MDBX_RESULT_FALSE, if pid is dead (lock acquired)
* or otherwise the errcode. */
int mdbx_rpid_check(MDBX_env *env, mdbx_pid_t pid) {
int rc = mdbx_lck_op(env->me_lfd, F_GETLK, F_WRLCK, pid, 1);
if (rc == 0)
return MDBX_RESULT_FALSE;
if (rc < 0 && -rc == pid)
return MDBX_RESULT_TRUE;
return rc;
}
/*---------------------------------------------------------------------------*/
static int mdbx_mutex_failed(MDBX_env *env, pthread_mutex_t *mutex, int rc);
int mdbx_lck_init(MDBX_env *env) {
pthread_mutexattr_t ma;
int rc = pthread_mutexattr_init(&ma);
if (rc)
return rc;
rc = pthread_mutexattr_setpshared(&ma, PTHREAD_PROCESS_SHARED);
if (rc)
goto bailout;
#if MDBX_USE_ROBUST
#if __GLIBC_PREREQ(2, 12)
rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST);
#else
rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP);
#endif
if (rc)
goto bailout;
#endif /* MDBX_USE_ROBUST */
#if _POSIX_C_SOURCE >= 199506L
rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT);
if (rc == ENOTSUP)
rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE);
if (rc)
goto bailout;
#endif /* PTHREAD_PRIO_INHERIT */
rc = pthread_mutex_init(&env->me_lck->mti_rmutex, &ma);
if (rc)
goto bailout;
rc = pthread_mutex_init(&env->me_lck->mti_wmutex, &ma);
bailout:
pthread_mutexattr_destroy(&ma);
return rc;
}
void mdbx_lck_destroy(MDBX_env *env) {
if (env->me_lfd != INVALID_HANDLE_VALUE) {
/* try get exclusive access */
if (env->me_lck && mdbx_lck_exclusive(env->me_lfd) == 0) {
mdbx_info("%s: got exclusive, drown mutexes", mdbx_func_);
int rc = pthread_mutex_destroy(&env->me_lck->mti_rmutex);
if (rc == 0)
rc = pthread_mutex_destroy(&env->me_lck->mti_wmutex);
assert(rc == 0);
(void)rc;
/* lock would be released (by kernel) while the me_lfd will be closed */
}
}
}
static int mdbx_robust_lock(MDBX_env *env, pthread_mutex_t *mutex) {
int rc = pthread_mutex_lock(mutex);
if (unlikely(rc != 0))
rc = mdbx_mutex_failed(env, mutex, rc);
return rc;
}
static int mdbx_robust_unlock(MDBX_env *env, pthread_mutex_t *mutex) {
int rc = pthread_mutex_unlock(mutex);
if (unlikely(rc != 0))
rc = mdbx_mutex_failed(env, mutex, rc);
return rc;
}
int mdbx_rdt_lock(MDBX_env *env) {
mdbx_trace(">>");
int rc = mdbx_robust_lock(env, &env->me_lck->mti_rmutex);
mdbx_trace("<< rc %d", rc);
return rc;
}
void mdbx_rdt_unlock(MDBX_env *env) {
mdbx_trace(">>");
int rc = mdbx_robust_unlock(env, &env->me_lck->mti_rmutex);
mdbx_trace("<< rc %d", rc);
if (unlikely(MDBX_IS_ERROR(rc)))
mdbx_panic("%s() failed: errcode %d\n", mdbx_func_, rc);
}
int mdbx_txn_lock(MDBX_env *env) {
mdbx_trace(">>");
int rc = mdbx_robust_lock(env, &env->me_lck->mti_wmutex);
mdbx_trace("<< rc %d", rc);
return MDBX_IS_ERROR(rc) ? rc : MDBX_SUCCESS;
}
void mdbx_txn_unlock(MDBX_env *env) {
mdbx_trace(">>");
int rc = mdbx_robust_unlock(env, &env->me_lck->mti_wmutex);
mdbx_trace("<< rc %d", rc);
if (unlikely(MDBX_IS_ERROR(rc)))
mdbx_panic("%s() failed: errcode %d\n", mdbx_func_, rc);
}
static int internal_seize_lck(int lfd) {
assert(lfd != INVALID_HANDLE_VALUE);
/* try exclusive access */
int rc = mdbx_lck_exclusive(lfd);
if (rc == 0)
/* got exclusive */
return MDBX_RESULT_TRUE;
if (rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK) {
/* get shared access */
rc = mdbx_lck_shared(lfd);
if (rc == 0) {
/* got shared, try exclusive again */
rc = mdbx_lck_exclusive(lfd);
if (rc == 0)
/* now got exclusive */
return MDBX_RESULT_TRUE;
if (rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK)
/* unable exclusive, but stay shared */
return MDBX_RESULT_FALSE;
}
}
assert(MDBX_IS_ERROR(rc));
return rc;
}
int mdbx_lck_seize(MDBX_env *env) {
assert(env->me_fd != INVALID_HANDLE_VALUE);
if (env->me_lfd == INVALID_HANDLE_VALUE) {
/* LY: without-lck mode (e.g. on read-only filesystem) */
int rc = mdbx_lck_op(env->me_fd, F_SETLK, F_RDLCK, 0, LCK_WHOLE);
if (rc != 0) {
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_, "without-lck", rc);
return rc;
}
return MDBX_RESULT_FALSE;
}
if ((env->me_flags & MDBX_RDONLY) == 0) {
/* Check that another process don't operates in without-lck mode. */
int rc = mdbx_lck_op(env->me_fd, F_SETLK, F_WRLCK, env->me_pid, 1);
if (rc != 0) {
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_,
"lock-against-without-lck", rc);
return rc;
}
}
return internal_seize_lck(env->me_lfd);
}
#if !__GLIBC_PREREQ(2, 12) && !defined(pthread_mutex_consistent)
#define pthread_mutex_consistent(mutex) pthread_mutex_consistent_np(mutex)
#endif
static int __cold mdbx_mutex_failed(MDBX_env *env, pthread_mutex_t *mutex,
int rc) {
#if MDBX_USE_ROBUST
if (rc == EOWNERDEAD) {
/* We own the mutex. Clean up after dead previous owner. */
int rlocked = (mutex == &env->me_lck->mti_rmutex);
rc = MDBX_SUCCESS;
if (!rlocked) {
if (unlikely(env->me_txn)) {
/* env is hosed if the dead thread was ours */
env->me_flags |= MDBX_FATAL_ERROR;
env->me_txn = NULL;
rc = MDBX_PANIC;
}
}
mdbx_notice("%cmutex owner died, %s", (rlocked ? 'r' : 'w'),
(rc ? "this process' env is hosed" : "recovering"));
int check_rc = mdbx_reader_check0(env, rlocked, NULL);
check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc;
int mreco_rc = pthread_mutex_consistent(mutex);
check_rc = (mreco_rc == 0) ? check_rc : mreco_rc;
if (unlikely(mreco_rc))
mdbx_error("mutex recovery failed, %s", mdbx_strerror(mreco_rc));
rc = (rc == MDBX_SUCCESS) ? check_rc : rc;
if (MDBX_IS_ERROR(rc))
pthread_mutex_unlock(mutex);
return rc;
}
#endif /* MDBX_USE_ROBUST */
mdbx_error("mutex (un)lock failed, %s", mdbx_strerror(rc));
if (rc != EDEADLK) {
env->me_flags |= MDBX_FATAL_ERROR;
rc = MDBX_PANIC;
}
return rc;
}

449
src/lck-windows.c Normal file
View File

@ -0,0 +1,449 @@
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "./bits.h"
/* PREAMBLE FOR WINDOWS:
*
* We are not concerned for performance here.
* If you are running Windows a performance could NOT be the goal.
* Otherwise please use Linux.
*
* Regards,
* LY
*/
/*----------------------------------------------------------------------------*/
/* rthc */
static CRITICAL_SECTION rthc_critical_section;
static void NTAPI tls_callback(PVOID module, DWORD reason, PVOID reserved) {
(void)module;
(void)reserved;
switch (reason) {
case DLL_PROCESS_ATTACH:
InitializeCriticalSection(&rthc_critical_section);
break;
case DLL_PROCESS_DETACH:
DeleteCriticalSection(&rthc_critical_section);
break;
case DLL_THREAD_ATTACH:
break;
case DLL_THREAD_DETACH:
mdbx_rthc_cleanup();
break;
}
}
void mdbx_rthc_lock(void) { EnterCriticalSection(&rthc_critical_section); }
void mdbx_rthc_unlock(void) { LeaveCriticalSection(&rthc_critical_section); }
/* *INDENT-OFF* */
/* clang-format off */
#if defined(_MSC_VER)
# pragma const_seg(push)
# pragma data_seg(push)
# ifdef _WIN64
/* kick a linker to create the TLS directory if not already done */
# pragma comment(linker, "/INCLUDE:_tls_used")
/* Force some symbol references. */
# pragma comment(linker, "/INCLUDE:mdbx_tls_callback")
/* specific const-segment for WIN64 */
# pragma const_seg(".CRT$XLB")
const
# else
/* kick a linker to create the TLS directory if not already done */
# pragma comment(linker, "/INCLUDE:__tls_used")
/* Force some symbol references. */
# pragma comment(linker, "/INCLUDE:_mdbx_tls_callback")
/* specific data-segment for WIN32 */
# pragma data_seg(".CRT$XLB")
# endif
PIMAGE_TLS_CALLBACK mdbx_tls_callback = tls_callback;
# pragma data_seg(pop)
# pragma const_seg(pop)
#elif defined(__GNUC__)
# ifdef _WIN64
const
# endif
PIMAGE_TLS_CALLBACK mdbx_tls_callback __attribute__((section(".CRT$XLB"), used))
= tls_callback;
#else
# error FIXME
#endif
/* *INDENT-ON* */
/* clang-format on */
/*----------------------------------------------------------------------------*/
#define LCK_SHARED 0
#define LCK_EXCLUSIVE LOCKFILE_EXCLUSIVE_LOCK
#define LCK_WAITFOR 0
#define LCK_DONTWAIT LOCKFILE_FAIL_IMMEDIATELY
static __inline BOOL flock(mdbx_filehandle_t fd, DWORD flags, uint64_t offset,
size_t bytes) {
OVERLAPPED ov;
ov.hEvent = 0;
ov.Offset = (DWORD)offset;
ov.OffsetHigh = HIGH_DWORD(offset);
return LockFileEx(fd, flags, 0, (DWORD)bytes, HIGH_DWORD(bytes), &ov);
}
static __inline BOOL funlock(mdbx_filehandle_t fd, uint64_t offset,
size_t bytes) {
return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes,
HIGH_DWORD(bytes));
}
/*----------------------------------------------------------------------------*/
/* global `write` lock for write-txt processing,
* exclusive locking both meta-pages) */
#define LCK_MAXLEN (1u + (size_t)(MAXSSIZE_T))
#define LCK_META_OFFSET 0
#define LCK_META_LEN 0x10000u
#define LCK_BODY_OFFSET LCK_META_LEN
#define LCK_BODY_LEN (LCK_MAXLEN - LCK_BODY_OFFSET + 1u)
#define LCK_META LCK_META_OFFSET, LCK_META_LEN
#define LCK_BODY LCK_BODY_OFFSET, LCK_BODY_LEN
#define LCK_WHOLE 0, LCK_MAXLEN
int mdbx_txn_lock(MDBX_env *env) {
if (flock(env->me_fd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_BODY))
return MDBX_SUCCESS;
return GetLastError();
}
void mdbx_txn_unlock(MDBX_env *env) {
if (!funlock(env->me_fd, LCK_BODY))
mdbx_panic("%s failed: errcode %u", mdbx_func_, GetLastError());
}
/*----------------------------------------------------------------------------*/
/* global `read` lock for readers registration,
* exclusive locking `mti_numreaders` (second) cacheline */
#define LCK_LO_OFFSET 0
#define LCK_LO_LEN offsetof(MDBX_lockinfo, mti_numreaders)
#define LCK_UP_OFFSET LCK_LO_LEN
#define LCK_UP_LEN (MDBX_LOCKINFO_WHOLE_SIZE - LCK_UP_OFFSET)
#define LCK_LOWER LCK_LO_OFFSET, LCK_LO_LEN
#define LCK_UPPER LCK_UP_OFFSET, LCK_UP_LEN
int mdbx_rdt_lock(MDBX_env *env) {
if (env->me_lfd == INVALID_HANDLE_VALUE)
return MDBX_SUCCESS; /* readonly database in readonly filesystem */
/* transite from S-? (used) to S-E (locked), e.g. exclusive lock upper-part */
if (flock(env->me_lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER))
return MDBX_SUCCESS;
return GetLastError();
}
void mdbx_rdt_unlock(MDBX_env *env) {
if (env->me_lfd != INVALID_HANDLE_VALUE) {
/* transite from S-E (locked) to S-? (used), e.g. unlock upper-part */
if (!funlock(env->me_lfd, LCK_UPPER))
mdbx_panic("%s failed: errcode %u", mdbx_func_, GetLastError());
}
}
/*----------------------------------------------------------------------------*/
/* global `initial` lock for lockfile initialization,
* exclusive/shared locking first cacheline */
/* FIXME: locking schema/algo descritpion.
?-? = free
S-? = used
E-? = exclusive-read
?-S
?-E = middle
S-S
S-E = locked
E-S
E-E = exclusive-write
*/
int mdbx_lck_init(MDBX_env *env) {
(void)env;
return MDBX_SUCCESS;
}
/* Seize state as 'exclusive-write' (E-E and returns MDBX_RESULT_TRUE)
* or as 'used' (S-? and returns MDBX_RESULT_FALSE), otherwise returns an error
*/
static int internal_seize_lck(HANDLE lfd) {
int rc;
assert(lfd != INVALID_HANDLE_VALUE);
/* 1) now on ?-? (free), get ?-E (middle) */
mdbx_jitter4testing(false);
if (!flock(lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER)) {
rc = GetLastError() /* 2) something went wrong, give up */;
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_,
"?-?(free) >> ?-E(middle)", rc);
return rc;
}
/* 3) now on ?-E (middle), try E-E (exclusive-write) */
mdbx_jitter4testing(false);
if (flock(lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER))
return MDBX_RESULT_TRUE /* 4) got E-E (exclusive-write), done */;
/* 5) still on ?-E (middle) */
rc = GetLastError();
mdbx_jitter4testing(false);
if (rc != ERROR_SHARING_VIOLATION && rc != ERROR_LOCK_VIOLATION) {
/* 6) something went wrong, give up */
if (!funlock(lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", mdbx_func_,
"?-E(middle) >> ?-?(free)", GetLastError());
return rc;
}
/* 7) still on ?-E (middle), try S-E (locked) */
mdbx_jitter4testing(false);
rc = flock(lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER) ? MDBX_RESULT_FALSE
: GetLastError();
mdbx_jitter4testing(false);
if (rc != MDBX_RESULT_FALSE)
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_,
"?-E(middle) >> S-E(locked)", rc);
/* 8) now on S-E (locked) or still on ?-E (middle),
* transite to S-? (used) or ?-? (free) */
if (!funlock(lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", mdbx_func_,
"X-E(locked/middle) >> X-?(used/free)", GetLastError());
/* 9) now on S-? (used, DONE) or ?-? (free, FAILURE) */
return rc;
}
int mdbx_lck_seize(MDBX_env *env) {
int rc;
assert(env->me_fd != INVALID_HANDLE_VALUE);
if (env->me_lfd == INVALID_HANDLE_VALUE) {
/* LY: without-lck mode (e.g. on read-only filesystem) */
mdbx_jitter4testing(false);
if (!flock(env->me_fd, LCK_SHARED | LCK_DONTWAIT, LCK_WHOLE)) {
rc = GetLastError();
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_, "without-lck", rc);
return rc;
}
return MDBX_RESULT_FALSE;
}
rc = internal_seize_lck(env->me_lfd);
mdbx_jitter4testing(false);
if (rc == MDBX_RESULT_TRUE && (env->me_flags & MDBX_RDONLY) == 0) {
/* Check that another process don't operates in without-lck mode.
* Doing such check by exclusive locking the body-part of db. Should be
* noted:
* - we need an exclusive lock for do so;
* - we can't lock meta-pages, otherwise other process could get an error
* while opening db in valid (non-conflict) mode. */
if (!flock(env->me_fd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_BODY)) {
rc = GetLastError();
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_,
"lock-against-without-lck", rc);
mdbx_jitter4testing(false);
mdbx_lck_destroy(env);
} else {
mdbx_jitter4testing(false);
if (!funlock(env->me_fd, LCK_BODY))
mdbx_panic("%s(%s) failed: errcode %u", mdbx_func_,
"unlock-against-without-lck", GetLastError());
}
}
return rc;
}
int mdbx_lck_downgrade(MDBX_env *env, bool complete) {
/* Transite from exclusive state (E-?) to used (S-?) */
assert(env->me_fd != INVALID_HANDLE_VALUE);
assert(env->me_lfd != INVALID_HANDLE_VALUE);
/* 1) must be at E-E (exclusive-write) */
if (!complete) {
/* transite from E-E to E_? (exclusive-read) */
if (!funlock(env->me_lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", mdbx_func_,
"E-E(exclusive-write) >> E-?(exclusive-read)", GetLastError());
return MDBX_SUCCESS /* 2) now at E-? (exclusive-read), done */;
}
/* 3) now at E-E (exclusive-write), transite to ?_E (middle) */
if (!funlock(env->me_lfd, LCK_LOWER))
mdbx_panic("%s(%s) failed: errcode %u", mdbx_func_,
"E-E(exclusive-write) >> ?-E(middle)", GetLastError());
/* 4) now at ?-E (middle), transite to S-E (locked) */
if (!flock(env->me_lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER)) {
int rc = GetLastError() /* 5) something went wrong, give up */;
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_,
"?-E(middle) >> S-E(locked)", rc);
return rc;
}
/* 6) got S-E (locked), continue transition to S-? (used) */
if (!funlock(env->me_lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", mdbx_func_,
"S-E(locked) >> S-?(used)", GetLastError());
return MDBX_SUCCESS /* 7) now at S-? (used), done */;
}
int mdbx_lck_upgrade(MDBX_env *env) {
/* Transite from locked state (S-E) to exclusive-write (E-E) */
assert(env->me_fd != INVALID_HANDLE_VALUE);
assert(env->me_lfd != INVALID_HANDLE_VALUE);
/* 1) must be at S-E (locked), transite to ?_E (middle) */
if (!funlock(env->me_lfd, LCK_LOWER))
mdbx_panic("%s(%s) failed: errcode %u", mdbx_func_,
"S-E(locked) >> ?-E(middle)", GetLastError());
/* 3) now on ?-E (middle), try E-E (exclusive-write) */
mdbx_jitter4testing(false);
if (flock(env->me_lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER))
return MDBX_RESULT_TRUE; /* 4) got E-E (exclusive-write), done */
/* 5) still on ?-E (middle) */
int rc = GetLastError();
mdbx_jitter4testing(false);
if (rc != ERROR_SHARING_VIOLATION && rc != ERROR_LOCK_VIOLATION) {
/* 6) something went wrong, report but continue */
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_,
"?-E(middle) >> E-E(exclusive-write)", rc);
}
/* 7) still on ?-E (middle), try restore S-E (locked) */
mdbx_jitter4testing(false);
rc = flock(env->me_lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER)
? MDBX_RESULT_FALSE
: GetLastError();
mdbx_jitter4testing(false);
if (rc != MDBX_RESULT_FALSE) {
mdbx_fatal("%s(%s) failed: errcode %u", mdbx_func_,
"?-E(middle) >> S-E(locked)", rc);
return rc;
}
/* 8) now on S-E (locked) */
return MDBX_RESULT_FALSE;
}
void mdbx_lck_destroy(MDBX_env *env) {
int rc;
if (env->me_lfd != INVALID_HANDLE_VALUE) {
/* double `unlock` for robustly remove overlapped shared/exclusive locks */
while (funlock(env->me_lfd, LCK_LOWER))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_lfd, LCK_UPPER))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
}
if (env->me_fd != INVALID_HANDLE_VALUE) {
/* explicitly unlock to avoid latency for other processes (windows kernel
* releases such locks via deferred queues) */
while (funlock(env->me_fd, LCK_BODY))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_fd, LCK_META))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_fd, LCK_WHOLE))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
}
}
/*----------------------------------------------------------------------------*/
/* reader checking (by pid) */
int mdbx_rpid_set(MDBX_env *env) {
(void)env;
return MDBX_SUCCESS;
}
int mdbx_rpid_clear(MDBX_env *env) {
(void)env;
return MDBX_SUCCESS;
}
/* Checks reader by pid.
*
* Returns:
* MDBX_RESULT_TRUE, if pid is live (unable to acquire lock)
* MDBX_RESULT_FALSE, if pid is dead (lock acquired)
* or otherwise the errcode. */
int mdbx_rpid_check(MDBX_env *env, mdbx_pid_t pid) {
(void)env;
HANDLE hProcess = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid);
int rc;
if (hProcess) {
rc = WaitForSingleObject(hProcess, 0);
CloseHandle(hProcess);
} else {
rc = GetLastError();
}
switch (rc) {
case ERROR_INVALID_PARAMETER:
/* pid seem invalid */
return MDBX_RESULT_FALSE;
case WAIT_OBJECT_0:
/* process just exited */
return MDBX_RESULT_FALSE;
case WAIT_TIMEOUT:
/* pid running */
return MDBX_RESULT_TRUE;
default:
/* failure */
return rc;
}
}

11285
src/mdbx.c Normal file
View File

File diff suppressed because it is too large Load Diff

1008
src/osal.c Normal file
View File

File diff suppressed because it is too large Load Diff

613
src/osal.h Normal file
View File

@ -0,0 +1,613 @@
/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
/*----------------------------------------------------------------------------*/
/* Microsoft compiler generates a lot of warning for self includes... */
#ifdef _MSC_VER
#pragma warning(push, 1)
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
* semantics are not enabled. Specify /EHsc */
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
* mode specified; termination on exception is \
* not guaranteed. Specify /EHsc */
#if !defined(_CRT_SECURE_NO_WARNINGS)
#define _CRT_SECURE_NO_WARNINGS
#endif
#endif /* _MSC_VER (warnings) */
/*----------------------------------------------------------------------------*/
/* C99 includes */
#include <inttypes.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <assert.h>
#include <fcntl.h>
#include <limits.h>
#include <malloc.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
#ifndef _POSIX_C_SOURCE
#ifdef _POSIX_SOURCE
#define _POSIX_C_SOURCE 1
#else
#define _POSIX_C_SOURCE 0
#endif
#endif
#ifndef _XOPEN_SOURCE
#define _XOPEN_SOURCE 0
#endif
/*----------------------------------------------------------------------------*/
/* Systems includes */
#if defined(_WIN32) || defined(_WIN64)
#include <windows.h>
#include <winnt.h>
#define HAVE_SYS_STAT_H
#define HAVE_SYS_TYPES_H
typedef HANDLE mdbx_thread_t;
typedef unsigned mdbx_thread_key_t;
#define MDBX_OSAL_SECTION HANDLE
#define MAP_FAILED NULL
#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0))
#define THREAD_CALL WINAPI
#define THREAD_RESULT DWORD
typedef struct {
HANDLE mutex;
HANDLE event;
} mdbx_condmutex_t;
typedef CRITICAL_SECTION mdbx_fastmutex_t;
#else
#include <pthread.h>
#include <sys/file.h>
#include <sys/mman.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/uio.h>
#include <unistd.h>
typedef pthread_t mdbx_thread_t;
typedef pthread_key_t mdbx_thread_key_t;
#define INVALID_HANDLE_VALUE (-1)
#define THREAD_CALL
#define THREAD_RESULT void *
typedef struct {
pthread_mutex_t mutex;
pthread_cond_t cond;
} mdbx_condmutex_t;
typedef pthread_mutex_t mdbx_fastmutex_t;
#endif /* Platform */
#ifndef SSIZE_MAX
#define SSIZE_MAX INTPTR_MAX
#endif
#ifdef HAVE_SYS_STAT_H
#include <sys/stat.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_SYS_FILE_H
#include <sys/file.h>
#endif
#if !defined(UNALIGNED_OK)
#if defined(__i386) || defined(__x86_64__) || defined(_M_IX86) || \
defined(_M_X64) || defined(i386) || defined(_X86_) || defined(__i386__) || \
defined(_X86_64_)
#define UNALIGNED_OK 1
#else
#define UNALIGNED_OK 0
#endif
#endif /* UNALIGNED_OK */
#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
#error \
"Sanity checking failed: Two's complement, reasonably sized integer types"
#endif
/*----------------------------------------------------------------------------*/
/* Compiler's includes for builtins/intrinsics */
#ifdef _MSC_VER
#if _MSC_FULL_VER < 190024215
#if _MSC_FULL_VER < 180040629 && defined(_M_IX86)
#error Please use Visual Studio 2015 (MSC 19.0) or newer for 32-bit target.
#else
#pragma message( \
"It is recommended to use Visual Studio 2015 (MSC 19.0) or newer.")
#endif
#endif
#include <intrin.h>
#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
#if defined(__i386__) || defined(__x86_64__)
#include <cpuid.h>
#include <x86intrin.h>
#endif
#elif defined(__INTEL_COMPILER)
#include <intrin.h>
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
#include <mbarrier.h>
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
#include <machine/sys/inline.h>
#elif defined(__IBMC__) && defined(__powerpc)
#include <atomic.h>
#elif defined(_AIX)
#include <builtins.h>
#include <sys/atomic_op.h>
#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
#include <c_asm.h>
#include <machine/builtins.h>
#elif defined(__MWERKS__)
/* CodeWarrior - troubles ? */
#pragma gcc_extensions
#elif defined(__SNC__)
/* Sony PS3 - troubles ? */
#else
#error Unknown C compiler, please use GNU C 5.x or newer
#endif /* Compiler */
/*----------------------------------------------------------------------------*/
/* Byteorder */
#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
!defined(__ORDER_BIG_ENDIAN__)
#if defined(HAVE_ENDIAN_H)
#include <endian.h>
#elif defined(HAVE_SYS_PARAM_H)
#include <sys/param.h> /* for endianness */
#elif defined(HAVE_NETINET_IN_H) && defined(HAVE_RESOLV_H)
#include <netinet/in.h>
#include <resolv.h> /* defines BYTE_ORDER on HPUX and Solaris */
#endif
#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
#define __BYTE_ORDER__ __BYTE_ORDER
#else
#define __ORDER_LITTLE_ENDIAN__ 1234
#define __ORDER_BIG_ENDIAN__ 4321
#if defined(__LITTLE_ENDIAN__) || defined(_LITTLE_ENDIAN) || \
defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
defined(__i386) || defined(__x86_64__) || defined(_M_IX86) || \
defined(_M_X64) || defined(i386) || defined(_X86_) || defined(__i386__) || \
defined(_X86_64_) || defined(_M_ARM) || defined(_M_ARM64) || \
defined(__e2k__)
#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
#elif defined(__BIG_ENDIAN__) || defined(_BIG_ENDIAN) || defined(__ARMEB__) || \
defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || \
defined(_MIPSEB) || defined(__MIPSEB) || defined(_M_IA64)
#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
#else
#error __BYTE_ORDER__ should be defined.
#endif
#endif
#endif
#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \
__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__
#error Unsupported byte order.
#endif
/*----------------------------------------------------------------------------*/
/* Memory/Compiler barriers, cache coherence */
static __inline void mdbx_compiler_barrier(void) {
#if defined(__clang__) || defined(__GNUC__)
__asm__ __volatile__("" ::: "memory");
#elif defined(_MSC_VER)
_ReadWriteBarrier();
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
__memory_barrier();
if (type > MDBX_BARRIER_COMPILER)
#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
__mf();
#elif defined(__i386__) || defined(__x86_64__)
_mm_mfence();
#else
#error "Unknown target for Intel Compiler, please report to us."
#endif
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
__compiler_barrier();
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
_Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
defined(__ppc64__) || defined(__powerpc64__)
__fence();
#else
#error "Could not guess the kind of compiler, please report to us."
#endif
}
static __inline void mdbx_memory_barrier(void) {
#if __has_extension(c_atomic) || __has_extension(cxx_atomic)
__c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif defined(__ATOMIC_SEQ_CST)
__atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif defined(__clang__) || defined(__GNUC__)
__sync_synchronize();
#elif defined(_MSC_VER)
MemoryBarrier();
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
__mf();
#elif defined(__i386__) || defined(__x86_64__)
_mm_mfence();
#else
#error "Unknown target for Intel Compiler, please report to us."
#endif
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
__machine_rw_barrier();
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
_Asm_mf();
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
defined(__ppc64__) || defined(__powerpc64__)
__lwsync();
#else
#error "Could not guess the kind of compiler, please report to us."
#endif
}
/*----------------------------------------------------------------------------*/
/* Cache coherence and invalidation */
#if defined(__i386__) || defined(__x86_64__) || defined(_M_AMD64) || \
defined(_M_IX86) || defined(__i386) || defined(__amd64) || \
defined(i386) || defined(__x86_64) || defined(_AMD64_) || defined(_M_X64)
#define MDBX_CACHE_IS_COHERENT 1
#elif defined(__hppa) || defined(__hppa__)
#define MDBX_CACHE_IS_COHERENT 1
#endif
#ifndef MDBX_CACHE_IS_COHERENT
#define MDBX_CACHE_IS_COHERENT 0
#endif
#ifndef MDBX_CACHELINE_SIZE
#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE)
#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE
#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
#define MDBX_CACHELINE_SIZE 128
#else
#define MDBX_CACHELINE_SIZE 64
#endif
#endif /* MDBX_CACHELINE_SIZE */
#ifndef __cache_aligned
#define __cache_aligned __aligned(MDBX_CACHELINE_SIZE)
#endif
#if MDBX_CACHE_IS_COHERENT
#define mdbx_coherent_barrier() mdbx_compiler_barrier()
#else
#define mdbx_coherent_barrier() mdbx_memory_barrier()
#endif
#if defined(__mips) && defined(__linux)
/* Only MIPS has explicit cache control */
#include <asm/cachectl.h>
#endif
static __inline void mdbx_invalidate_cache(void *addr, size_t nbytes) {
mdbx_coherent_barrier();
#if defined(__mips) && defined(__linux)
/* MIPS has cache coherency issues.
* Note: for any nbytes >= on-chip cache size, entire is flushed. */
cacheflush(addr, nbytes, DCACHE);
#elif defined(_M_MRX000) || defined(_MIPS_)
#error "Sorry, cacheflush() for MIPS not implemented"
#else
/* LY: assume no relevant mmap/dcache issues. */
(void)addr;
(void)nbytes;
#endif
}
/*----------------------------------------------------------------------------*/
/* libc compatibility stuff */
#ifndef mdbx_assert_fail
void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func,
int line);
#endif /* mdbx_assert_fail */
#if __GLIBC_PREREQ(2, 1)
#define mdbx_asprintf asprintf
#else
int mdbx_asprintf(char **strp, const char *fmt, ...);
#endif
#ifdef _MSC_VER
#ifndef snprintf
#define snprintf(buffer, buffer_size, format, ...) \
_snprintf_s(buffer, buffer_size, _TRUNCATE, format, __VA_ARGS__)
#endif /* snprintf */
#ifndef vsnprintf
#define vsnprintf(buffer, buffer_size, format, args) \
_vsnprintf_s(buffer, buffer_size, _TRUNCATE, format, args)
#endif /* vsnprintf */
#endif /* _MSC_VER */
/*----------------------------------------------------------------------------*/
/* OS abstraction layer stuff */
/* max bytes to write in one call */
#define MAX_WRITE UINT32_C(0x3fff0000)
/* Get the size of a memory page for the system.
* This is the basic size that the platform's memory manager uses, and is
* fundamental to the use of memory-mapped files. */
static __inline size_t mdbx_syspagesize(void) {
#if defined(_WIN32) || defined(_WIN64)
SYSTEM_INFO si;
GetSystemInfo(&si);
return si.dwPageSize;
#else
return sysconf(_SC_PAGE_SIZE);
#endif
}
static __inline char *mdbx_strdup(const char *str) {
#ifdef _MSC_VER
return _strdup(str);
#else
return strdup(str);
#endif
}
static __inline int mdbx_get_errno(void) {
#if defined(_WIN32) || defined(_WIN64)
DWORD rc = GetLastError();
#else
int rc = errno;
#endif
return rc;
}
int mdbx_memalign_alloc(size_t alignment, size_t bytes, void **result);
void mdbx_memalign_free(void *ptr);
int mdbx_condmutex_init(mdbx_condmutex_t *condmutex);
int mdbx_condmutex_lock(mdbx_condmutex_t *condmutex);
int mdbx_condmutex_unlock(mdbx_condmutex_t *condmutex);
int mdbx_condmutex_signal(mdbx_condmutex_t *condmutex);
int mdbx_condmutex_wait(mdbx_condmutex_t *condmutex);
int mdbx_condmutex_destroy(mdbx_condmutex_t *condmutex);
int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex);
int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex);
int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex);
int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex);
int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, int iovcnt,
uint64_t offset, size_t expected_written);
int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count, uint64_t offset);
int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf, size_t count,
uint64_t offset);
int mdbx_write(mdbx_filehandle_t fd, const void *buf, size_t count);
int mdbx_thread_create(mdbx_thread_t *thread,
THREAD_RESULT(THREAD_CALL *start_routine)(void *),
void *arg);
int mdbx_thread_join(mdbx_thread_t thread);
int mdbx_thread_key_create(mdbx_thread_key_t *key);
void mdbx_thread_key_delete(mdbx_thread_key_t key);
void *mdbx_thread_rthc_get(mdbx_thread_key_t key);
void mdbx_thread_rthc_set(mdbx_thread_key_t key, const void *value);
int mdbx_filesync(mdbx_filehandle_t fd, bool fullsync);
int mdbx_filesize_sync(mdbx_filehandle_t fd);
int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length);
int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length);
int mdbx_openfile(const char *pathname, int flags, mode_t mode,
mdbx_filehandle_t *fd);
int mdbx_closefile(mdbx_filehandle_t fd);
typedef struct mdbx_mmap_param {
union {
void *address;
uint8_t *dxb;
struct MDBX_lockinfo *lck;
};
mdbx_filehandle_t fd;
size_t length; /* mapping length, but NOT a size of file or DB */
#if defined(_WIN32) || defined(_WIN64)
size_t current; /* mapped region size, e.g. file and DB */
#endif
#ifdef MDBX_OSAL_SECTION
MDBX_OSAL_SECTION section;
#endif
} mdbx_mmap_t;
int mdbx_mmap(int flags, mdbx_mmap_t *map, size_t must, size_t limit);
int mdbx_munmap(mdbx_mmap_t *map);
int mdbx_mlock(mdbx_mmap_t *map, size_t length);
int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t current, size_t wanna);
int mdbx_msync(mdbx_mmap_t *map, size_t offset, size_t length, int async);
static __inline mdbx_pid_t mdbx_getpid(void) {
#if defined(_WIN32) || defined(_WIN64)
return GetCurrentProcessId();
#else
return getpid();
#endif
}
static __inline mdbx_tid_t mdbx_thread_self(void) {
#if defined(_WIN32) || defined(_WIN64)
return GetCurrentThreadId();
#else
return pthread_self();
#endif
}
void mdbx_osal_jitter(bool tiny);
/*----------------------------------------------------------------------------*/
/* lck stuff */
#if defined(_WIN32) || defined(_WIN64)
#undef MDBX_OSAL_LOCK
#define MDBX_OSAL_LOCK_SIGN UINT32_C(0xF10C)
#else
#define MDBX_OSAL_LOCK pthread_mutex_t
#define MDBX_OSAL_LOCK_SIGN UINT32_C(0x8017)
#endif
int mdbx_lck_init(MDBX_env *env);
int mdbx_lck_seize(MDBX_env *env);
int mdbx_lck_downgrade(MDBX_env *env, bool complete);
int mdbx_lck_upgrade(MDBX_env *env);
void mdbx_lck_destroy(MDBX_env *env);
int mdbx_rdt_lock(MDBX_env *env);
void mdbx_rdt_unlock(MDBX_env *env);
int mdbx_txn_lock(MDBX_env *env);
void mdbx_txn_unlock(MDBX_env *env);
int mdbx_rpid_set(MDBX_env *env);
int mdbx_rpid_clear(MDBX_env *env);
/* Checks reader by pid.
*
* Returns:
* MDBX_RESULT_TRUE, if pid is live (unable to acquire lock)
* MDBX_RESULT_FALSE, if pid is dead (lock acquired)
* or otherwise the errcode. */
int mdbx_rpid_check(MDBX_env *env, mdbx_pid_t pid);
/*----------------------------------------------------------------------------*/
/* Atomics */
#if (__STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_ATOMICS__) && \
(__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \
!(defined(__GNUC__) || defined(__clang__)))
#include <stdatomic.h>
#elif defined(__GNUC__) || defined(__clang__)
/* LY: nothing required */
#elif defined(_MSC_VER)
#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */
#pragma warning(disable : 4133) /* 'function': incompatible types - from \
'size_t' to 'LONGLONG' */
#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
'std::size_t', possible loss of data */
#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
'long', possible loss of data */
#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange)
#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64)
#elif defined(__APPLE__)
#include <libkern/OSAtomic.h>
#else
#error FIXME atomic-ops
#endif
static __inline uint32_t mdbx_atomic_add32(volatile uint32_t *p, uint32_t v) {
#if defined(ATOMIC_VAR_INIT)
assert(atomic_is_lock_free(p));
return atomic_fetch_add((_Atomic uint32_t *)p, v);
#elif defined(__GNUC__) || defined(__clang__)
return __sync_fetch_and_add(p, v);
#else
#ifdef _MSC_VER
return _InterlockedExchangeAdd(p, v);
#endif
#ifdef __APPLE__
return OSAtomicAdd32(v, (volatile int32_t *)p);
#endif
#endif
}
static __inline uint64_t mdbx_atomic_add64(volatile uint64_t *p, uint64_t v) {
#ifdef ATOMIC_VAR_INIT
assert(atomic_is_lock_free(p));
return atomic_fetch_add((_Atomic uint64_t *)p, v);
#elif defined(__GNUC__) || defined(__clang__)
return __sync_fetch_and_add(p, v);
#else
#ifdef _MSC_VER
return _InterlockedExchangeAdd64(p, v);
#endif
#ifdef __APPLE__
return OSAtomicAdd64(v, (volatile int64_t *)p);
#endif
#endif
}
#define mdbx_atomic_sub32(p, v) mdbx_atomic_add32(p, -(v))
#define mdbx_atomic_sub64(p, v) mdbx_atomic_add64(p, -(v))
static __inline bool mdbx_atomic_compare_and_swap32(volatile uint32_t *p,
uint32_t c, uint32_t v) {
#ifdef ATOMIC_VAR_INIT
assert(atomic_is_lock_free(p));
return atomic_compare_exchange_strong((_Atomic uint32_t *)p, &c, v);
#elif defined(__GNUC__) || defined(__clang__)
return __sync_bool_compare_and_swap(p, c, v);
#else
#ifdef _MSC_VER
return c == _InterlockedCompareExchange(p, v, c);
#endif
#ifdef __APPLE__
return c == OSAtomicCompareAndSwap32Barrier(c, v, (volatile int32_t *)p);
#endif
#endif
}
static __inline bool mdbx_atomic_compare_and_swap64(volatile uint64_t *p,
uint64_t c, uint64_t v) {
#ifdef ATOMIC_VAR_INIT
assert(atomic_is_lock_free(p));
return atomic_compare_exchange_strong((_Atomic uint64_t *)p, &c, v);
#elif defined(__GNUC__) || defined(__clang__)
return __sync_bool_compare_and_swap(p, c, v);
#else
#ifdef _MSC_VER
return c == _InterlockedCompareExchange64(p, v, c);
#endif
#ifdef __APPLE__
return c == OSAtomicCompareAndSwap64Barrier(c, v, (volatile uint64_t *)p);
#endif
#endif
}
/*----------------------------------------------------------------------------*/
#ifdef _MSC_VER
#pragma warning(pop)
#endif

1138
src/tools/mdbx_chk.c Normal file
View File

File diff suppressed because it is too large Load Diff

166
src/tools/mdbx_chk.vcxproj Normal file
View File

@ -0,0 +1,166 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>mdbx_chk</RootNamespace>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ProjectReference Include="..\..\dll.vcxproj">
<Project>{6d19209b-ece7-4b9c-941c-0aa2b484f199}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<ClCompile Include="mdbx_chk.c" />
<ClCompile Include="wingetopt.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\mdbx.h" />
<ClInclude Include="wingetopt.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

14
mdb_copy.1 → src/tools/mdbx_copy.1
View File

@ -1,12 +1,12 @@
.\" Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2012-2017 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14"
.TH MDBX_COPY 1 "2014/06/20" "LMDB 0.9.14"
.SH NAME
mdb_copy \- LMDB environment copy tool
mdbx_copy \- MDBX environment copy tool
.SH SYNOPSIS
.B mdb_copy
.B mdbx_copy
[\c
.BR \-V ]
[\c
@ -18,8 +18,8 @@ mdb_copy \- LMDB environment copy tool
.BR dstpath ]
.SH DESCRIPTION
The
.B mdb_copy
utility copies an LMDB environment. The environment can
.B mdbx_copy
utility copies an MDBX environment. The environment can
be copied regardless of whether it is currently in use.
No lockfile is created, since it gets recreated at need.
@ -52,6 +52,6 @@ This utility can trigger significant file size growth if run
in parallel with write transactions, because pages which they
free during copying cannot be reused until the copy is done.
.SH "SEE ALSO"
.BR mdb_stat (1)
.BR mdbx_stat (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

114
src/tools/mdbx_copy.c Normal file
View File

@ -0,0 +1,114 @@
/* mdbx_copy.c - memory-mapped database backup tool */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#if _MSC_VER == 1900
/* LY: MSVC 2015 has buggy/inconsistent PRIuPTR/PRIxPTR macros and format-arg
checker for size_t typedef. */
#pragma warning(disable : 4777) /* format string '%10u' requires an argument \
of type 'unsigned int', but variadic \
argument 1 has type 'std::size_t' */
#endif
#endif /* _MSC_VER (warnings) */
#include "../bits.h"
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
int main(int argc, char *argv[]) {
int rc;
MDBX_env *env = NULL;
const char *progname = argv[0], *act;
unsigned flags = MDBX_RDONLY;
unsigned cpflags = 0;
for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
if (argv[1][1] == 'n' && argv[1][2] == '\0')
flags |= MDBX_NOSUBDIR;
else if (argv[1][1] == 'c' && argv[1][2] == '\0')
cpflags |= MDBX_CP_COMPACT;
else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
printf("%s (%s, build %s)\n", mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_build.datetime);
exit(EXIT_SUCCESS);
} else
argc = 0;
}
if (argc < 2 || argc > 3) {
fprintf(stderr, "usage: %s [-V] [-c] [-n] srcpath [dstpath]\n", progname);
exit(EXIT_FAILURE);
}
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
act = "opening environment";
rc = mdbx_env_create(&env);
if (rc == MDBX_SUCCESS) {
rc = mdbx_env_open(env, argv[1], flags, 0640);
}
if (rc == MDBX_SUCCESS) {
act = "copying";
if (argc == 2) {
mdbx_filehandle_t fd;
#if defined(_WIN32) || defined(_WIN64)
fd = GetStdHandle(STD_OUTPUT_HANDLE);
#else
fd = fileno(stdout);
#endif
rc = mdbx_env_copy2fd(env, fd, cpflags);
} else
rc = mdbx_env_copy(env, argv[2], cpflags);
}
if (rc)
fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc,
mdbx_strerror(rc));
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

166
src/tools/mdbx_copy.vcxproj Normal file
View File

@ -0,0 +1,166 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>mdbx_copy</RootNamespace>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ProjectReference Include="..\..\dll.vcxproj">
<Project>{6d19209b-ece7-4b9c-941c-0aa2b484f199}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<ClCompile Include="mdbx_copy.c" />
<ClCompile Include="wingetopt.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\mdbx.h" />
<ClInclude Include="wingetopt.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

18
mdb_dump.1 → src/tools/mdbx_dump.1
View File

@ -1,12 +1,12 @@
.\" Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2014-2017 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14"
.TH MDBX_DUMP 1 "2014/06/20" "LMDB 0.9.14"
.SH NAME
mdb_dump \- LMDB environment export tool
mdbx_dump \- MDBX environment export tool
.SH SYNOPSIS
.B mdb_dump
.B mdbx_dump
[\c
.BR \-V ]
[\c
@ -23,11 +23,11 @@ mdb_dump \- LMDB environment export tool
.BR \ envpath
.SH DESCRIPTION
The
.B mdb_dump
.B mdbx_dump
utility reads a database and writes its contents to the
standard output using a portable flat-text format
understood by the
.BR mdb_load (1)
.BR mdbx_load (1)
utility.
.SH OPTIONS
.TP
@ -42,7 +42,7 @@ List the databases stored in the environment. Just the
names will be listed, no data will be output.
.TP
.BR \-n
Dump an LMDB database which does not use subdirectories.
Dump an MDBX database which does not use subdirectories.
.TP
.BR \-p
If characters in either the key or data items are printing characters (as
@ -69,9 +69,9 @@ will result in new databases that use the default comparison functions.
damaged beyond repair permitting neither record storage nor retrieval.\fP
The only available workaround is to modify the source for the
.BR mdb_load (1)
.BR mdbx_load (1)
utility to load the database using the correct comparison functions.
.SH "SEE ALSO"
.BR mdb_load (1)
.BR mdbx_load (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

342
src/tools/mdbx_dump.c Normal file
View File

@ -0,0 +1,342 @@
/* mdbx_dump.c - memory-mapped database dump tool */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#if _MSC_VER == 1900
/* LY: MSVC 2015 has buggy/inconsistent PRIuPTR/PRIxPTR macros and format-arg
checker for size_t typedef. */
#pragma warning(disable : 4777) /* format string '%10u' requires an argument \
of type 'unsigned int', but variadic \
argument 1 has type 'std::size_t' */
#endif
#endif /* _MSC_VER (warnings) */
#include "../bits.h"
#include <ctype.h>
#define PRINT 1
static int mode;
typedef struct flagbit {
int bit;
char *name;
} flagbit;
flagbit dbflags[] = {{MDBX_REVERSEKEY, "reversekey"},
{MDBX_DUPSORT, "dupsort"},
{MDBX_INTEGERKEY, "integerkey"},
{MDBX_DUPFIXED, "dupfixed"},
{MDBX_INTEGERDUP, "integerdup"},
{MDBX_REVERSEDUP, "reversedup"},
{0, NULL}};
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
static const char hexc[] = "0123456789abcdef";
static void dumpbyte(unsigned char c) {
putchar(hexc[c >> 4]);
putchar(hexc[c & 0xf]);
}
static void text(MDBX_val *v) {
unsigned char *c, *end;
putchar(' ');
c = v->iov_base;
end = c + v->iov_len;
while (c < end) {
if (isprint(*c) && *c != '\\') {
putchar(*c);
} else {
putchar('\\');
dumpbyte(*c);
}
c++;
}
putchar('\n');
}
static void dumpval(MDBX_val *v) {
unsigned char *c, *end;
putchar(' ');
c = v->iov_base;
end = c + v->iov_len;
while (c < end) {
dumpbyte(*c++);
}
putchar('\n');
}
/* Dump in BDB-compatible format */
static int dumpit(MDBX_txn *txn, MDBX_dbi dbi, char *name) {
MDBX_cursor *mc;
MDBX_stat ms;
MDBX_val key, data;
MDBX_envinfo info;
unsigned int flags;
int rc, i;
rc = mdbx_dbi_flags(txn, dbi, &flags);
if (rc)
return rc;
rc = mdbx_dbi_stat(txn, dbi, &ms, sizeof(ms));
if (rc)
return rc;
rc = mdbx_env_info(mdbx_txn_env(txn), &info, sizeof(info));
if (rc)
return rc;
printf("VERSION=3\n");
printf("format=%s\n", mode & PRINT ? "print" : "bytevalue");
if (name)
printf("database=%s\n", name);
printf("type=btree\n");
printf("mapsize=%" PRIu64 "\n", info.mi_mapsize);
printf("maxreaders=%u\n", info.mi_maxreaders);
for (i = 0; dbflags[i].bit; i++)
if (flags & dbflags[i].bit)
printf("%s=1\n", dbflags[i].name);
printf("db_pagesize=%d\n", ms.ms_psize);
printf("HEADER=END\n");
rc = mdbx_cursor_open(txn, dbi, &mc);
if (rc)
return rc;
while ((rc = mdbx_cursor_get(mc, &key, &data, MDBX_NEXT)) == MDBX_SUCCESS) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
if (mode & PRINT) {
text(&key);
text(&data);
} else {
dumpval(&key);
dumpval(&data);
}
}
printf("DATA=END\n");
if (rc == MDBX_NOTFOUND)
rc = MDBX_SUCCESS;
return rc;
}
static void usage(char *prog) {
fprintf(stderr,
"usage: %s [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb] dbpath\n",
prog);
exit(EXIT_FAILURE);
}
int main(int argc, char *argv[]) {
int i, rc;
MDBX_env *env;
MDBX_txn *txn;
MDBX_dbi dbi;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envflags = 0, list = 0;
if (argc < 2) {
usage(prog);
}
/* -a: dump main DB and all subDBs
* -s: dump only the named subDB
* -n: use NOSUBDIR flag on env_open
* -p: use printable characters
* -f: write to file instead of stdout
* -V: print version and exit
* (default) dump only the main DB
*/
while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) {
switch (i) {
case 'V':
printf("%s (%s, build %s)\n", mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_build.datetime);
exit(EXIT_SUCCESS);
break;
case 'l':
list = 1;
/*FALLTHROUGH*/;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'f':
if (freopen(optarg, "w", stdout) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDBX_NOSUBDIR;
break;
case 'p':
mode |= PRINT;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
envname = argv[optind];
rc = mdbx_env_create(&env);
if (rc) {
fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc,
mdbx_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname) {
mdbx_env_set_maxdbs(env, 2);
}
rc = mdbx_env_open(env, envname, envflags | MDBX_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_dbi_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc));
goto txn_abort;
}
if (alldbs) {
MDBX_cursor *cursor;
MDBX_val key;
int count = 0;
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
while ((rc = mdbx_cursor_get(cursor, &key, NULL, MDBX_NEXT_NODUP)) == 0) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
char *str;
MDBX_dbi db2;
if (memchr(key.iov_base, '\0', key.iov_len))
continue;
count++;
str = malloc(key.iov_len + 1);
memcpy(str, key.iov_base, key.iov_len);
str[key.iov_len] = '\0';
rc = mdbx_dbi_open(txn, str, 0, &db2);
if (rc == MDBX_SUCCESS) {
if (list) {
printf("%s\n", str);
list++;
} else {
rc = dumpit(txn, db2, str);
if (rc)
break;
}
mdbx_dbi_close(env, db2);
}
free(str);
if (rc)
continue;
}
mdbx_cursor_close(cursor);
if (!count) {
fprintf(stderr, "%s: %s does not contain multiple databases\n", prog,
envname);
rc = MDBX_NOTFOUND;
} else if (rc == MDBX_INCOMPATIBLE) {
/* LY: the record it not a named sub-db. */
rc = MDBX_SUCCESS;
}
} else {
rc = dumpit(txn, dbi, subname);
}
if (rc && rc != MDBX_NOTFOUND)
fprintf(stderr, "%s: %s: %s\n", prog, envname, mdbx_strerror(rc));
mdbx_dbi_close(env, dbi);
txn_abort:
mdbx_txn_abort(txn);
env_close:
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

166
src/tools/mdbx_dump.vcxproj Normal file
View File

@ -0,0 +1,166 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>mdbx_dump</RootNamespace>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ProjectReference Include="..\..\dll.vcxproj">
<Project>{6d19209b-ece7-4b9c-941c-0aa2b484f199}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<ClCompile Include="mdbx_dump.c" />
<ClCompile Include="wingetopt.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\mdbx.h" />
<ClInclude Include="wingetopt.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

22
mdb_load.1 → src/tools/mdbx_load.1
View File

@ -1,12 +1,12 @@
.\" Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2014-2017 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14"
.TH MDBX_LOAD 1 "2014/06/20" "LMDB 0.9.14"
.SH NAME
mdb_load \- LMDB environment import tool
mdbx_load \- MDBX environment import tool
.SH SYNOPSIS
.B mdb_load
.B mdbx_load
[\c
.BR \-V ]
[\c
@ -22,15 +22,15 @@ mdb_load \- LMDB environment import tool
.BR \ envpath
.SH DESCRIPTION
The
.B mdb_load
.B mdbx_load
utility reads from the standard input and loads it into the
LMDB environment
MDBX environment
.BR envpath .
The input to
.B mdb_load
.B mdbx_load
must be in the output format specified by the
.BR mdb_dump (1)
.BR mdbx_dump (1)
utility or as specified by the
.B -T
option below.
@ -43,7 +43,7 @@ Write the library version number to the standard output, and exit.
Read from the specified file instead of from the standard input.
.TP
.BR \-n
Load an LMDB database which does not use subdirectories.
Load an MDBX database which does not use subdirectories.
.TP
.BR \-s \ subdb
Load a specific subdatabase. If no database is specified, data is loaded into the main database.
@ -66,7 +66,7 @@ character; for example, \\0a is a newline character in the ASCII character set.
For this reason, any backslash or newline characters that naturally occur in the text
input must be escaped to avoid misinterpretation by
.BR mdb_load .
.BR mdbx_load .
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
@ -74,6 +74,6 @@ Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
.SH "SEE ALSO"
.BR mdb_dump (1)
.BR mdbx_dump (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

520
src/tools/mdbx_load.c Normal file
View File

@ -0,0 +1,520 @@
/* mdbx_load.c - memory-mapped database load tool */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#if _MSC_VER == 1900
/* LY: MSVC 2015 has buggy/inconsistent PRIuPTR/PRIxPTR macros and format-arg
checker for size_t typedef. */
#pragma warning(disable : 4777) /* format string '%10u' requires an argument \
of type 'unsigned int', but variadic \
argument 1 has type 'std::size_t' */
#endif
#endif /* _MSC_VER (warnings) */
#include "../bits.h"
#include <ctype.h>
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
#define PRINT 1
#define NOHDR 2
static int mode;
static char *subname = NULL;
static size_t lineno;
static int version;
static int dbi_flags;
static char *prog;
static int Eof;
static MDBX_envinfo envinfo;
static MDBX_val kbuf, dbuf;
#define STRLENOF(s) (sizeof(s) - 1)
typedef struct flagbit {
int bit;
char *name;
int len;
} flagbit;
#define S(s) s, STRLENOF(s)
flagbit dbflags[] = {{MDBX_REVERSEKEY, S("reversekey")},
{MDBX_DUPSORT, S("dupsort")},
{MDBX_INTEGERKEY, S("integerkey")},
{MDBX_DUPFIXED, S("dupfixed")},
{MDBX_INTEGERDUP, S("integerdup")},
{MDBX_REVERSEDUP, S("reversedup")},
{0, NULL, 0}};
static void readhdr(void) {
char *ptr;
dbi_flags = 0;
while (fgets(dbuf.iov_base, (int)dbuf.iov_len, stdin) != NULL) {
lineno++;
if (!strncmp(dbuf.iov_base, "db_pagesize=", STRLENOF("db_pagesize=")) ||
!strncmp(dbuf.iov_base, "duplicates=", STRLENOF("duplicates="))) {
/* LY: silently ignore information fields. */
continue;
} else if (!strncmp(dbuf.iov_base, "VERSION=", STRLENOF("VERSION="))) {
version = atoi((char *)dbuf.iov_base + STRLENOF("VERSION="));
if (version > 3) {
fprintf(stderr, "%s: line %" PRIiPTR ": unsupported VERSION %d\n", prog,
lineno, version);
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "HEADER=END", STRLENOF("HEADER=END"))) {
break;
} else if (!strncmp(dbuf.iov_base, "format=", STRLENOF("format="))) {
if (!strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "print",
STRLENOF("print")))
mode |= PRINT;
else if (strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "bytevalue",
STRLENOF("bytevalue"))) {
fprintf(stderr, "%s: line %" PRIiPTR ": unsupported FORMAT %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("FORMAT="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "database=", STRLENOF("database="))) {
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
if (subname)
free(subname);
subname = strdup((char *)dbuf.iov_base + STRLENOF("database="));
} else if (!strncmp(dbuf.iov_base, "type=", STRLENOF("type="))) {
if (strncmp((char *)dbuf.iov_base + STRLENOF("type="), "btree",
STRLENOF("btree"))) {
fprintf(stderr, "%s: line %" PRIiPTR ": unsupported type %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("type="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "mapaddr=", STRLENOF("mapaddr="))) {
int i;
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
void *unused;
i = sscanf((char *)dbuf.iov_base + STRLENOF("mapaddr="), "%p", &unused);
if (i != 1) {
fprintf(stderr, "%s: line %" PRIiPTR ": invalid mapaddr %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("mapaddr="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "mapsize=", STRLENOF("mapsize="))) {
int i;
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
i = sscanf((char *)dbuf.iov_base + STRLENOF("mapsize="), "%" PRIu64 "",
&envinfo.mi_mapsize);
if (i != 1) {
fprintf(stderr, "%s: line %" PRIiPTR ": invalid mapsize %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("mapsize="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "maxreaders=",
STRLENOF("maxreaders="))) {
int i;
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
i = sscanf((char *)dbuf.iov_base + STRLENOF("maxreaders="), "%u",
&envinfo.mi_maxreaders);
if (i != 1) {
fprintf(stderr, "%s: line %" PRIiPTR ": invalid maxreaders %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("maxreaders="));
exit(EXIT_FAILURE);
}
} else {
int i;
for (i = 0; dbflags[i].bit; i++) {
if (!strncmp(dbuf.iov_base, dbflags[i].name, dbflags[i].len) &&
((char *)dbuf.iov_base)[dbflags[i].len] == '=') {
if (((char *)dbuf.iov_base)[dbflags[i].len + 1] == '1')
dbi_flags |= dbflags[i].bit;
break;
}
}
if (!dbflags[i].bit) {
ptr = memchr(dbuf.iov_base, '=', dbuf.iov_len);
if (!ptr) {
fprintf(stderr, "%s: line %" PRIiPTR ": unexpected format\n", prog,
lineno);
exit(EXIT_FAILURE);
} else {
*ptr = '\0';
fprintf(stderr,
"%s: line %" PRIiPTR ": unrecognized keyword ignored: %s\n",
prog, lineno, (char *)dbuf.iov_base);
}
}
}
}
}
static void badend(void) {
fprintf(stderr, "%s: line %" PRIiPTR ": unexpected end of input\n", prog,
lineno);
}
static int unhex(unsigned char *c2) {
int x, c;
x = *c2++ & 0x4f;
if (x & 0x40)
x -= 55;
c = x << 4;
x = *c2 & 0x4f;
if (x & 0x40)
x -= 55;
c |= x;
return c;
}
static int readline(MDBX_val *out, MDBX_val *buf) {
unsigned char *c1, *c2, *end;
size_t len, l2;
int c;
if (!(mode & NOHDR)) {
c = fgetc(stdin);
if (c == EOF) {
Eof = 1;
return EOF;
}
if (c != ' ') {
lineno++;
if (fgets(buf->iov_base, (int)buf->iov_len, stdin) == NULL) {
badend:
Eof = 1;
badend();
return EOF;
}
if (c == 'D' && !strncmp(buf->iov_base, "ATA=END", STRLENOF("ATA=END")))
return EOF;
goto badend;
}
}
if (fgets(buf->iov_base, (int)buf->iov_len, stdin) == NULL) {
Eof = 1;
return EOF;
}
lineno++;
c1 = buf->iov_base;
len = strlen((char *)c1);
l2 = len;
/* Is buffer too short? */
while (c1[len - 1] != '\n') {
buf->iov_base = realloc(buf->iov_base, buf->iov_len * 2);
if (!buf->iov_base) {
Eof = 1;
fprintf(stderr, "%s: line %" PRIiPTR ": out of memory, line too long\n",
prog, lineno);
return EOF;
}
c1 = buf->iov_base;
c1 += l2;
if (fgets((char *)c1, (int)buf->iov_len + 1, stdin) == NULL) {
Eof = 1;
badend();
return EOF;
}
buf->iov_len *= 2;
len = strlen((char *)c1);
l2 += len;
}
c1 = c2 = buf->iov_base;
len = l2;
c1[--len] = '\0';
end = c1 + len;
if (mode & PRINT) {
while (c2 < end) {
if (*c2 == '\\') {
if (c2[1] == '\\') {
c1++;
c2 += 2;
} else {
if (c2 + 3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
Eof = 1;
badend();
return EOF;
}
*c1++ = (char)unhex(++c2);
c2 += 2;
}
} else {
/* copies are redundant when no escapes were used */
*c1++ = *c2++;
}
}
} else {
/* odd length not allowed */
if (len & 1) {
Eof = 1;
badend();
return EOF;
}
while (c2 < end) {
if (!isxdigit(*c2) || !isxdigit(c2[1])) {
Eof = 1;
badend();
return EOF;
}
*c1++ = (char)unhex(c2);
c2 += 2;
}
}
c2 = out->iov_base = buf->iov_base;
out->iov_len = c1 - c2;
return 0;
}
static void usage(void) {
fprintf(stderr, "usage: %s [-V] [-f input] [-n] [-s name] [-N] [-T] dbpath\n",
prog);
exit(EXIT_FAILURE);
}
int main(int argc, char *argv[]) {
int i, rc;
MDBX_env *env = NULL;
MDBX_txn *txn = NULL;
MDBX_cursor *mc = NULL;
MDBX_dbi dbi;
char *envname = NULL;
int envflags = 0, putflags = 0;
prog = argv[0];
if (argc < 2) {
usage();
}
/* -f: load file instead of stdin
* -n: use NOSUBDIR flag on env_open
* -s: load into named subDB
* -N: use NOOVERWRITE on puts
* -T: read plaintext
* -V: print version and exit
*/
while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
switch (i) {
case 'V':
printf("%s (%s, build %s)\n", mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_build.datetime);
exit(EXIT_SUCCESS);
break;
case 'f':
if (freopen(optarg, "r", stdin) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDBX_NOSUBDIR;
break;
case 's':
subname = strdup(optarg);
break;
case 'N':
putflags = MDBX_NOOVERWRITE | MDBX_NODUPDATA;
break;
case 'T':
mode |= NOHDR | PRINT;
break;
default:
usage();
}
}
if (optind != argc - 1)
usage();
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
dbuf.iov_len = 4096;
dbuf.iov_base = malloc(dbuf.iov_len);
if (!(mode & NOHDR))
readhdr();
envname = argv[optind];
rc = mdbx_env_create(&env);
if (rc) {
fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc,
mdbx_strerror(rc));
return EXIT_FAILURE;
}
mdbx_env_set_maxdbs(env, 2);
if (envinfo.mi_maxreaders)
mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders);
if (envinfo.mi_mapsize) {
if (envinfo.mi_mapsize > SIZE_MAX) {
fprintf(stderr, "mdbx_env_set_mapsize failed, error %d %s\n", rc,
mdbx_strerror(MDBX_TOO_LARGE));
return EXIT_FAILURE;
}
mdbx_env_set_mapsize(env, (size_t)envinfo.mi_mapsize);
}
#ifdef MDBX_FIXEDMAP
if (info.mi_mapaddr)
envflags |= MDBX_FIXEDMAP;
#endif
rc = mdbx_env_open(env, envname, envflags, 0664);
if (rc) {
fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
kbuf.iov_len = mdbx_env_get_maxkeysize(env) * 2 + 2;
kbuf.iov_base = malloc(kbuf.iov_len);
while (!Eof) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
MDBX_val key, data;
int batch = 0;
rc = mdbx_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_dbi_open(txn, subname, dbi_flags | MDBX_CREATE, &dbi);
if (rc) {
fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc));
goto txn_abort;
}
rc = mdbx_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
while (1) {
rc = readline(&key, &kbuf);
if (rc) /* rc == EOF */
break;
rc = readline(&data, &dbuf);
if (rc) {
fprintf(stderr, "%s: line %" PRIiPTR ": failed to read key value\n",
prog, lineno);
goto txn_abort;
}
rc = mdbx_cursor_put(mc, &key, &data, putflags);
if (rc == MDBX_KEYEXIST && putflags)
continue;
if (rc) {
fprintf(stderr, "mdbx_cursor_put failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
batch++;
if (batch == 100) {
rc = mdbx_txn_commit(txn);
if (rc) {
fprintf(stderr, "%s: line %" PRIiPTR ": txn_commit: %s\n", prog,
lineno, mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
batch = 0;
}
}
rc = mdbx_txn_commit(txn);
txn = NULL;
if (rc) {
fprintf(stderr, "%s: line %" PRIiPTR ": txn_commit: %s\n", prog, lineno,
mdbx_strerror(rc));
goto env_close;
}
mdbx_dbi_close(env, dbi);
if (!(mode & NOHDR))
readhdr();
}
txn_abort:
mdbx_txn_abort(txn);
env_close:
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

166
src/tools/mdbx_load.vcxproj Normal file
View File

@ -0,0 +1,166 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>mdbx_load</RootNamespace>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ProjectReference Include="..\..\dll.vcxproj">
<Project>{6d19209b-ece7-4b9c-941c-0aa2b484f199}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<ClCompile Include="mdbx_load.c" />
<ClCompile Include="wingetopt.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\mdbx.h" />
<ClInclude Include="wingetopt.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

16
mdb_stat.1 → src/tools/mdbx_stat.1
View File

@ -1,12 +1,12 @@
.\" Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2012-2017 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14"
.TH MDBX_STAT 1 "2014/06/20" "LMDB 0.9.14"
.SH NAME
mdb_stat \- LMDB environment status tool
mdbx_stat \- MDBX environment status tool
.SH SYNOPSIS
.B mdb_stat
.B mdbx_stat
[\c
.BR \-V ]
[\c
@ -23,8 +23,8 @@ mdb_stat \- LMDB environment status tool
.BR \ envpath
.SH DESCRIPTION
The
.B mdb_stat
utility displays the status of an LMDB environment.
.B mdbx_stat
utility displays the status of an MDBX environment.
.SH OPTIONS
.TP
.BR \-V
@ -39,7 +39,7 @@ If \fB\-ff\fP is given, summarize each freelist entry.
If \fB\-fff\fP is given, display the full list of page IDs in the freelist.
.TP
.BR \-n
Display the status of an LMDB database which does not use subdirectories.
Display the status of an MDBX database which does not use subdirectories.
.TP
.BR \-r
Display information about the environment reader table.
@ -61,6 +61,6 @@ Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
.SH "SEE ALSO"
.BR mdb_copy (1)
.BR mdbx_copy (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

369
src/tools/mdbx_stat.c Normal file
View File

@ -0,0 +1,369 @@
/* mdbx_stat.c - memory-mapped database status tool */
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#if _MSC_VER == 1900
/* LY: MSVC 2015 has buggy/inconsistent PRIuPTR/PRIxPTR macros and format-arg
checker for size_t typedef. */
#pragma warning(disable : 4777) /* format string '%10u' requires an argument \
of type 'unsigned int', but variadic \
argument 1 has type 'std::size_t' */
#endif
#endif /* _MSC_VER (warnings) */
#include "../bits.h"
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
static void prstat(MDBX_stat *ms) {
printf(" Pagesize: %u\n", ms->ms_psize);
printf(" Tree depth: %u\n", ms->ms_depth);
printf(" Branch pages: %" PRIu64 "\n", ms->ms_branch_pages);
printf(" Leaf pages: %" PRIu64 "\n", ms->ms_leaf_pages);
printf(" Overflow pages: %" PRIu64 "\n", ms->ms_overflow_pages);
printf(" Entries: %" PRIu64 "\n", ms->ms_entries);
}
static void usage(char *prog) {
fprintf(stderr,
"usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb] dbpath\n",
prog);
exit(EXIT_FAILURE);
}
int main(int argc, char *argv[]) {
int o, rc;
MDBX_env *env;
MDBX_txn *txn;
MDBX_dbi dbi;
MDBX_stat mst;
MDBX_envinfo mei;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0;
if (argc < 2) {
usage(prog);
}
/* -a: print stat of main DB and all subDBs
* -s: print stat of only the named subDB
* -e: print env info
* -f: print freelist info
* -r: print reader info
* -n: use NOSUBDIR flag on env_open
* -V: print version and exit
* (default) print stat of only the main DB
*/
while ((o = getopt(argc, argv, "Vaefnrs:")) != EOF) {
switch (o) {
case 'V':
printf("%s (%s, build %s)\n", mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_build.datetime);
exit(EXIT_SUCCESS);
break;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'e':
envinfo++;
break;
case 'f':
freinfo++;
break;
case 'n':
envflags |= MDBX_NOSUBDIR;
break;
case 'r':
rdrinfo++;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
envname = argv[optind];
rc = mdbx_env_create(&env);
if (rc) {
fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc,
mdbx_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname) {
mdbx_env_set_maxdbs(env, 4);
}
rc = mdbx_env_open(env, envname, envflags | MDBX_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
if (envinfo) {
(void)mdbx_env_stat(env, &mst, sizeof(mst));
(void)mdbx_env_info(env, &mei, sizeof(mei));
printf("Environment Info\n");
printf(" Pagesize: %u\n", mst.ms_psize);
if (mei.mi_geo.lower != mei.mi_geo.upper) {
printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64
"/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64
"/-%" PRIu64 ")\n",
mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow,
mei.mi_geo.shrink, mei.mi_geo.lower / mst.ms_psize,
mei.mi_geo.upper / mst.ms_psize, mei.mi_geo.grow / mst.ms_psize,
mei.mi_geo.shrink / mst.ms_psize);
printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize);
} else {
printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize);
}
printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n",
mei.mi_mapsize, mei.mi_mapsize / mst.ms_psize);
printf(" Number of pages used: %" PRIu64 "\n", mei.mi_last_pgno + 1);
printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid);
printf(" Tail transaction ID: %" PRIu64 " (%" PRIi64 ")\n",
mei.mi_latter_reader_txnid,
mei.mi_latter_reader_txnid - mei.mi_recent_txnid);
printf(" Max readers: %u\n", mei.mi_maxreaders);
printf(" Number of readers used: %u\n", mei.mi_numreaders);
} else {
/* LY: zap warnings from gcc */
memset(&mst, 0, sizeof(mst));
memset(&mei, 0, sizeof(mei));
}
if (rdrinfo) {
printf("Reader Table Status\n");
rc = mdbx_reader_list(env, (MDBX_msg_func *)fputs, stdout);
if (rdrinfo > 1) {
int dead;
mdbx_reader_check(env, &dead);
printf(" %d stale readers cleared.\n", dead);
rc = mdbx_reader_list(env, (MDBX_msg_func *)fputs, stdout);
}
if (!(subname || alldbs || freinfo))
goto env_close;
}
rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
if (freinfo) {
MDBX_cursor *cursor;
MDBX_val key, data;
pgno_t pages = 0, *iptr;
pgno_t reclaimable = 0;
printf("Freelist Status\n");
dbi = 0;
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
prstat(&mst);
while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == 0) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
iptr = data.iov_base;
pages += *iptr;
if (envinfo && mei.mi_latter_reader_txnid > *(size_t *)key.iov_base)
reclaimable += *iptr;
if (freinfo > 1) {
char *bad = "";
pgno_t pg, prev;
ssize_t i, j, span = 0;
j = *iptr++;
for (i = j, prev = 1; --i >= 0;) {
pg = iptr[i];
if (pg <= prev)
bad = " [bad sequence]";
prev = pg;
pg += (unsigned)span;
for (; i >= span && iptr[i - span] == pg; span++, pg++)
;
}
printf(" Transaction %" PRIaTXN ", %" PRIiPTR
" pages, maxspan %" PRIiPTR "%s\n",
*(txnid_t *)key.iov_base, j, span, bad);
if (freinfo > 2) {
for (--j; j >= 0;) {
pg = iptr[j];
for (span = 1; --j >= 0 && iptr[j] == pg + span; span++)
;
if (span > 1)
printf(" %9" PRIaPGNO "[%" PRIiPTR "]\n", pg, span);
else
printf(" %9" PRIaPGNO "\n", pg);
}
}
}
}
mdbx_cursor_close(cursor);
if (envinfo) {
uint64_t value = mei.mi_mapsize / mst.ms_psize;
double percent = value / 100.0;
printf("Page Allocation Info\n");
printf(" Max pages: %" PRIu64 " 100%%\n", value);
value = mei.mi_last_pgno + 1;
printf(" Pages used: %" PRIu64 " %.1f%%\n", value, value / percent);
value = mei.mi_mapsize / mst.ms_psize - (mei.mi_last_pgno + 1);
printf(" Remained: %" PRIu64 " %.1f%%\n", value, value / percent);
value = mei.mi_last_pgno + 1 - pages;
printf(" Used now: %" PRIu64 " %.1f%%\n", value, value / percent);
value = pages;
printf(" Unallocated: %" PRIu64 " %.1f%%\n", value, value / percent);
value = pages - reclaimable;
printf(" Detained: %" PRIu64 " %.1f%%\n", value, value / percent);
value = reclaimable;
printf(" Reclaimable: %" PRIu64 " %.1f%%\n", value, value / percent);
value =
mei.mi_mapsize / mst.ms_psize - (mei.mi_last_pgno + 1) + reclaimable;
printf(" Available: %" PRIu64 " %.1f%%\n", value, value / percent);
} else
printf(" Free pages: %" PRIaPGNO "\n", pages);
}
rc = mdbx_dbi_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc));
goto txn_abort;
}
rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
printf("Status of %s\n", subname ? subname : "Main DB");
prstat(&mst);
if (alldbs) {
MDBX_cursor *cursor;
MDBX_val key;
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
while ((rc = mdbx_cursor_get(cursor, &key, NULL, MDBX_NEXT_NODUP)) == 0) {
char *str;
MDBX_dbi db2;
if (memchr(key.iov_base, '\0', key.iov_len))
continue;
str = malloc(key.iov_len + 1);
memcpy(str, key.iov_base, key.iov_len);
str[key.iov_len] = '\0';
rc = mdbx_dbi_open(txn, str, 0, &db2);
if (rc == MDBX_SUCCESS)
printf("Status of %s\n", str);
free(str);
if (rc)
continue;
rc = mdbx_dbi_stat(txn, db2, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
prstat(&mst);
mdbx_dbi_close(env, db2);
}
mdbx_cursor_close(cursor);
}
if (rc == MDBX_NOTFOUND)
rc = MDBX_SUCCESS;
mdbx_dbi_close(env, dbi);
txn_abort:
mdbx_txn_abort(txn);
env_close:
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

166
src/tools/mdbx_stat.vcxproj Normal file
View File

@ -0,0 +1,166 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>mdbx_stat</RootNamespace>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ProjectReference Include="..\..\dll.vcxproj">
<Project>{6d19209b-ece7-4b9c-941c-0aa2b484f199}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<ClCompile Include="mdbx_stat.c" />
<ClCompile Include="wingetopt.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\mdbx.h" />
<ClInclude Include="wingetopt.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

73
src/tools/wingetopt.c Normal file
View File

@ -0,0 +1,73 @@
/*
* POSIX getopt for Windows
*
* AT&T Public License
*
* Code given out at the 1985 UNIFORUM conference in Dallas.
*/
#include "wingetopt.h"
#include <stdio.h>
#include <string.h>
#ifndef NULL
#define NULL 0
#endif
#ifndef EOF
#define EOF (-1)
#endif
#define ERR(s, c) \
if (opterr) { \
fputs(argv[0], stderr); \
fputs(s, stderr); \
fputc(c, stderr); \
}
int opterr = 1;
int optind = 1;
int optopt;
char *optarg;
int getopt(int argc, char *const argv[], const char *opts) {
static int sp = 1;
int c;
char *cp;
if (sp == 1) {
if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0')
return EOF;
else if (strcmp(argv[optind], "--") == 0) {
optind++;
return EOF;
}
}
optopt = c = argv[optind][sp];
if (c == ':' || (cp = strchr(opts, c)) == NULL) {
ERR(": illegal option -- ", c);
if (argv[optind][++sp] == '\0') {
optind++;
sp = 1;
}
return '?';
}
if (*++cp == ':') {
if (argv[optind][sp + 1] != '\0')
optarg = &argv[optind++][sp + 1];
else if (++optind >= argc) {
ERR(": option requires an argument -- ", c);
sp = 1;
return '?';
} else
optarg = argv[optind++];
sp = 1;
} else {
if (argv[optind][++sp] == '\0') {
sp = 1;
optind++;
}
optarg = NULL;
}
return c;
}

26
src/tools/wingetopt.h Normal file
View File

@ -0,0 +1,26 @@
/*
* POSIX getopt for Windows
*
* AT&T Public License
*
* Code given out at the 1985 UNIFORUM conference in Dallas.
*/
#ifndef _WINGETOPT_H_
#define _WINGETOPT_H_
#ifdef __cplusplus
extern "C" {
#endif
extern int opterr;
extern int optind;
extern int optopt;
extern char *optarg;
int getopt(int argc, char *const argv[], const char *optstring);
#ifdef __cplusplus
}
#endif
#endif /* _GETOPT_H_ */

34
src/version.c Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "./bits.h"
#if MDBX_VERSION_MAJOR != 0 || MDBX_VERSION_MINOR != 0
#error "API version mismatch!"
#endif
#define MDBX_VERSION_RELEASE 0
#define MDBX_VERSION_REVISION 0
const struct mdbx_version_info mdbx_version = {
MDBX_VERSION_MAJOR,
MDBX_VERSION_MINOR,
MDBX_VERSION_RELEASE,
MDBX_VERSION_REVISION,
{"@MDBX_GIT_TIMESTAMP@", "@MDBX_GIT_TREE@", "@MDBX_GIT_COMMIT@",
"@MDBX_GIT_DESCRIBE@"}};
const struct mdbx_build_info mdbx_build = {
"@MDBX_BUILD_TIMESTAMP@", "@MDBX_BUILD_TAGRET@", "@MDBX_BUILD_OPTIONS@",
"@MDBX_BUILD_COMPILER@", "@MDBX_BUILD_FLAGS@"};

105
test/base.h Normal file
View File

@ -0,0 +1,105 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#ifndef NOMINMAX
#define NOMINMAX
#endif
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
#ifdef _MSC_VER
#define _CRT_SECURE_NO_WARNINGS
#pragma warning(push, 1)
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
semantics are not enabled. Specify /EHsc */
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
mode specified; termination on exception \
is not guaranteed. Specify /EHsc */
#endif /* _MSC_VER (warnings) */
/* If you wish to build your application for a previous Windows platform,
* include WinSDKVer.h and set the _WIN32_WINNT macro to the platform you
* wish to support before including SDKDDKVer.h.
*
* TODO: #define _WIN32_WINNT WIN32_MUSTDIE */
#include <SDKDDKVer.h>
#endif /* WINDOWS */
#include <errno.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
#include <io.h>
#else
#include <fcntl.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#endif
#ifdef _BSD_SOURCE
#include <endian.h>
#endif
#include <algorithm>
#include <cassert>
#include <cinttypes> // for PRId64, PRIu64
#include <cstdarg>
#include <cstddef>
#include <cstdint>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <type_traits>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#ifdef _MSC_VER
#include <intrin.h>
#endif
#if defined(__i386__) || defined(__x86_64__)
#include <x86intrin.h>
#endif
#include "../mdbx.h"
#include "../src/defs.h"
#ifdef _MSC_VER
#pragma warning(pop)
#pragma warning(disable : 4201) /* nonstandard extension used : \
nameless struct / union */
#pragma warning(disable : 4127) /* conditional expression is constant */
#if _MSC_VER < 1900
#pragma warning(disable : 4510) /* default constructor could \
not be generated */
#pragma warning(disable : 4512) /* assignment operator could \
not be generated */
#pragma warning(disable : 4610) /* user-defined constructor required */
#define snprintf _snprintf
#pragma warning(disable : 4996) /* 'vsnprintf': This function or variable \
may be unsafe */
#endif
#endif /* _MSC_VER */

96
test/cases.cc Normal file
View File

@ -0,0 +1,96 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
void configure_actor(unsigned &last_space_id, const actor_testcase testcase,
const char *space_id_cstr, const actor_params &params) {
unsigned wait4id = 0;
if (params.waitfor_nops) {
for (auto i = global::actors.rbegin(); i != global::actors.rend(); ++i) {
if (i->is_waitable(params.waitfor_nops)) {
if (i->signal_nops && i->signal_nops != params.waitfor_nops)
failure("Previous waitable actor (id=%u) already linked on %u-ops\n",
i->actor_id, i->signal_nops);
wait4id = i->actor_id;
i->signal_nops = params.waitfor_nops;
break;
}
}
if (!wait4id)
failure("No previous waitable actor for %u-ops\n", params.waitfor_nops);
}
unsigned space_id = 0;
if (!space_id_cstr || strcmp(space_id_cstr, "auto") == 0)
space_id = last_space_id + 1;
else {
char *end = nullptr;
errno = 0;
space_id = strtoul(space_id_cstr, &end, 0);
if (errno)
failure_perror("Expects an integer value for space-id\n", errno);
if (end && *end)
failure("The '%s' is unexpected for space-id\n", end);
}
if (space_id > ACTOR_ID_MAX)
failure("Invalid space-id %u\n", space_id);
last_space_id = space_id;
log_trace("configure_actor: space %u for %s", space_id,
testcase2str(testcase));
global::actors.emplace_back(
actor_config(testcase, params, space_id, wait4id));
global::databases.insert(params.pathname_db);
}
void testcase_setup(const char *casename, actor_params &params,
unsigned &last_space_id) {
if (strcmp(casename, "basic") == 0) {
log_notice(">>> testcase_setup(%s)", casename);
configure_actor(last_space_id, ac_jitter, nullptr, params);
configure_actor(last_space_id, ac_hill, nullptr, params);
configure_actor(last_space_id, ac_jitter, nullptr, params);
configure_actor(last_space_id, ac_hill, nullptr, params);
configure_actor(last_space_id, ac_jitter, nullptr, params);
configure_actor(last_space_id, ac_hill, nullptr, params);
log_notice("<<< testcase_setup(%s): done", casename);
} else {
failure("unknown testcase `%s`", casename);
}
}
void keycase_setup(const char *casename, actor_params &params) {
if (strcmp(casename, "random") == 0 || strcmp(casename, "prng") == 0) {
log_notice(">>> keycase_setup(%s)", casename);
params.keygen.keycase = kc_random;
// TODO
log_notice("<<< keycase_setup(%s): done", casename);
} else if (strcmp(casename, "dashes") == 0 ||
strcmp(casename, "aside") == 0) {
log_notice(">>> keycase_setup(%s)", casename);
params.keygen.keycase = kc_dashes;
// TODO
log_notice("<<< keycase_setup(%s): done", casename);
} else if (strcmp(casename, "custom") == 0) {
log_notice("=== keycase_setup(%s): skip", casename);
params.keygen.keycase = kc_custom;
} else {
failure("unknown keycase `%s`", casename);
}
}
/* TODO */

129
test/chrono.cc Normal file
View File

@ -0,0 +1,129 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
namespace chrono {
#define NSEC_PER_SEC 1000000000u
uint32_t ns2fractional(uint32_t ns) {
assert(ns < NSEC_PER_SEC);
/* LY: здесь и далее используется "длинное деление", которое
* для ясности кода оставлено как есть (без ручной оптимизации). Так как
* GCC, Clang и даже MSVC сами давно умеют конвертировать деление на
* константу в быструю reciprocal-форму. */
return ((uint64_t)ns << 32) / NSEC_PER_SEC;
}
uint32_t fractional2ns(uint32_t fractional) {
return (fractional * (uint64_t)NSEC_PER_SEC) >> 32;
}
#define USEC_PER_SEC 1000000u
uint32_t us2fractional(uint32_t us) {
assert(us < USEC_PER_SEC);
return ((uint64_t)us << 32) / USEC_PER_SEC;
}
uint32_t fractional2us(uint32_t fractional) {
return (fractional * (uint64_t)USEC_PER_SEC) >> 32;
}
#define MSEC_PER_SEC 1000u
uint32_t ms2fractional(uint32_t ms) {
assert(ms < MSEC_PER_SEC);
return ((uint64_t)ms << 32) / MSEC_PER_SEC;
}
uint32_t fractional2ms(uint32_t fractional) {
return (fractional * (uint64_t)MSEC_PER_SEC) >> 32;
}
time from_ns(uint64_t ns) {
time result;
result.fixedpoint = ((ns / NSEC_PER_SEC) << 32) |
ns2fractional((uint32_t)(ns % NSEC_PER_SEC));
return result;
}
time from_us(uint64_t us) {
time result;
result.fixedpoint = ((us / USEC_PER_SEC) << 32) |
us2fractional((uint32_t)(us % USEC_PER_SEC));
return result;
}
time from_ms(uint64_t ms) {
time result;
result.fixedpoint = ((ms / MSEC_PER_SEC) << 32) |
ms2fractional((uint32_t)(ms % MSEC_PER_SEC));
return result;
}
time now_realtime() {
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
static void(WINAPI * query_time)(LPFILETIME);
if (!query_time) {
query_time = (void(WINAPI *)(LPFILETIME))GetProcAddress(
GetModuleHandle(TEXT("kernel32.dll")),
"GetSystemTimePreciseAsFileTime");
if (!query_time)
query_time = GetSystemTimeAsFileTime;
}
FILETIME filetime;
query_time(&filetime);
uint64_t ns100 =
(uint64_t)filetime.dwHighDateTime << 32 | filetime.dwLowDateTime;
return from_ns((ns100 - UINT64_C(116444736000000000)) * 100u);
#else
struct timespec ts;
if (unlikely(clock_gettime(CLOCK_REALTIME, &ts)))
failure_perror("clock_gettime(CLOCK_REALTIME", errno);
return from_timespec(ts);
#endif
}
time now_motonic() {
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
static uint64_t reciprocal;
static LARGE_INTEGER Frequency;
if (reciprocal == 0) {
if (!QueryPerformanceFrequency(&Frequency))
failure_perror("QueryPerformanceFrequency()", GetLastError());
reciprocal = (((UINT64_C(1) << 48) + Frequency.QuadPart / 2 + 1) /
Frequency.QuadPart);
assert(reciprocal);
}
LARGE_INTEGER Counter;
if (!QueryPerformanceCounter(&Counter))
failure_perror("QueryPerformanceCounter()", GetLastError());
time result;
result.fixedpoint = (Counter.QuadPart / Frequency.QuadPart) << 32;
uint64_t mod = Counter.QuadPart % Frequency.QuadPart;
result.fixedpoint += (mod * reciprocal) >> 16;
return result;
#else
struct timespec ts;
if (unlikely(clock_gettime(CLOCK_MONOTONIC, &ts)))
failure_perror("clock_gettime(CLOCK_MONOTONIC)", errno);
return from_timespec(ts);
#endif
}
} /* namespace chrono */

96
test/chrono.h Normal file
View File

@ -0,0 +1,96 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "log.h"
#include "utils.h"
namespace chrono {
typedef union time {
uint64_t fixedpoint;
struct __packed {
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
uint32_t fractional;
union {
uint32_t utc;
uint32_t integer;
};
#else
union {
uint32_t utc;
uint32_t integer;
};
uint32_t fractional;
#endif
};
void reset() { fixedpoint = 0; }
uint32_t seconds() const { return utc; }
} time;
uint32_t ns2fractional(uint32_t);
uint32_t fractional2ns(uint32_t);
uint32_t us2fractional(uint32_t);
uint32_t fractional2us(uint32_t);
uint32_t ms2fractional(uint32_t);
uint32_t fractional2ms(uint32_t);
time from_ns(uint64_t us);
time from_us(uint64_t ns);
time from_ms(uint64_t ms);
inline time from_seconds(uint64_t seconds) {
assert(seconds < UINT32_MAX);
time result;
result.fixedpoint = seconds << 32;
return result;
}
inline time from_utc(time_t utc) {
assert(utc >= 0);
return from_seconds(utc);
}
inline time infinite() {
time result;
result.fixedpoint = UINT64_MAX;
return result;
}
#if defined(HAVE_TIMESPEC_TV_NSEC) || defined(__timespec_defined) || \
defined(CLOCK_REALTIME)
inline time from_timespec(const struct timespec &ts) {
time result;
result.fixedpoint =
((uint64_t)ts.tv_sec << 32) | ns2fractional((uint32_t)ts.tv_nsec);
return result;
}
#endif /* HAVE_TIMESPEC_TV_NSEC */
#if defined(HAVE_TIMEVAL_TV_USEC) || defined(_STRUCT_TIMEVAL)
inline time from_timeval(const struct timeval &tv) {
time result;
result.fixedpoint =
((uint64_t)tv.tv_sec << 32) | us2fractional((uint32_t)tv.tv_usec);
return result;
}
#endif /* HAVE_TIMEVAL_TV_USEC */
time now_realtime();
time now_motonic();
} /* namespace chrono */

468
test/config.cc Normal file
View File

@ -0,0 +1,468 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
#if defined(_MSC_VER) && !defined(strcasecmp)
#define strcasecmp(str, len) _stricmp(str, len)
#endif /* _MSC_VER && strcasecmp() */
namespace config {
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
const char **value, const char *default_value) {
assert(narg < argc);
const char *current = argv[narg];
const size_t optlen = strlen(option);
if (strncmp(current, "--", 2) || strncmp(current + 2, option, optlen))
return false;
if (!value) {
if (current[optlen + 2] == '=')
failure("Option '--%s' doen't accept any value\n", option);
return true;
}
*value = nullptr;
if (current[optlen + 2] == '=') {
*value = &current[optlen + 3];
return true;
}
if (narg + 1 < argc && strncmp("--", argv[narg + 1], 2) != 0) {
*value = argv[narg + 1];
++narg;
return true;
}
if (default_value) {
*value = default_value;
return true;
}
failure("No value given for '--%s' option\n", option);
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty) {
const char *value_cstr;
if (!parse_option(argc, argv, narg, option, &value_cstr,
allow_empty ? "" : nullptr))
return false;
if (!allow_empty && strlen(value_cstr) == 0)
failure("Value for option '--%s' could't be empty\n", option);
value = value_cstr;
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &mask, const option_verb *verbs) {
const char *list;
if (!parse_option(argc, argv, narg, option, &list))
return false;
mask = 0;
while (*list) {
if (*list == ',' || *list == ' ' || *list == '\t') {
++list;
continue;
}
const char *const comma = strchr(list, ',');
const size_t len = (comma) ? comma - list : strlen(list);
const option_verb *scan = verbs;
while (true) {
if (!scan->verb)
failure("Unknown verb '%.*s', for option '==%s'\n", (int)len, list,
option);
if (strlen(scan->verb) == len && strncmp(list, scan->verb, len) == 0) {
mask |= scan->mask;
list += len;
break;
}
++scan;
}
}
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint64_t &value, const scale_mode scale,
const uint64_t minval, const uint64_t maxval) {
const char *value_cstr;
if (!parse_option(argc, argv, narg, option, &value_cstr))
return false;
char *suffix = nullptr;
errno = 0;
unsigned long raw = strtoul(value_cstr, &suffix, 0);
if (errno)
failure("Option '--%s' expects a numeric value (%s)\n", option,
test_strerror(errno));
uint64_t multipler = 1;
if (suffix && *suffix) {
if (scale == no_scale)
failure("Option '--%s' doen't accepts suffixes, so '%s' is unexpected\n",
option, suffix);
if (strcmp(suffix, "K") == 0 || strcasecmp(suffix, "Kilo") == 0)
multipler = (scale == decimal) ? UINT64_C(1000) : UINT64_C(1024);
else if (strcmp(suffix, "M") == 0 || strcasecmp(suffix, "Mega") == 0)
multipler =
(scale == decimal) ? UINT64_C(1000) * 1000 : UINT64_C(1024) * 1024;
else if (strcmp(suffix, "G") == 0 || strcasecmp(suffix, "Giga") == 0)
multipler = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000
: UINT64_C(1024) * 1024 * 1024;
else if (strcmp(suffix, "T") == 0 || strcasecmp(suffix, "Tera") == 0)
multipler = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 * 1000
: UINT64_C(1024) * 1024 * 1024 * 1024;
else if (scale == duration &&
(strcmp(suffix, "s") == 0 || strcasecmp(suffix, "Seconds") == 0))
multipler = 1;
else if (scale == duration &&
(strcmp(suffix, "m") == 0 || strcasecmp(suffix, "Minutes") == 0))
multipler = 60;
else if (scale == duration &&
(strcmp(suffix, "h") == 0 || strcasecmp(suffix, "Hours") == 0))
multipler = 3600;
else if (scale == duration &&
(strcmp(suffix, "d") == 0 || strcasecmp(suffix, "Days") == 0))
multipler = 3600 * 24;
else
failure(
"Option '--%s' expects a numeric value with Kilo/Mega/Giga/Tera %s"
"suffixes, but '%s' is unexpected\n",
option, (scale == duration) ? "or Seconds/Minutes/Hours/Days " : "",
suffix);
}
if (raw >= UINT64_MAX / multipler)
failure("The value for option '--%s' is too huge\n", option);
value = raw * multipler;
if (maxval && value > maxval)
failure("The maximal value for option '--%s' is %" PRIu64 "\n", option,
maxval);
if (value < minval)
failure("The minimal value for option '--%s' is %" PRIu64 "\n", option,
minval);
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &value, const scale_mode scale,
const unsigned minval, const unsigned maxval) {
uint64_t huge;
if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval))
return false;
value = (unsigned)huge;
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint8_t &value, const uint8_t minval, const uint8_t maxval) {
uint64_t huge;
if (!parse_option(argc, argv, narg, option, huge, no_scale, minval, maxval))
return false;
value = (uint8_t)huge;
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
bool &value) {
const char *value_cstr = NULL;
if (!parse_option(argc, argv, narg, option, &value_cstr, "yes")) {
const char *current = argv[narg];
if (strncmp(current, "--no-", 5) == 0 && strcmp(current + 5, option) == 0) {
value = false;
return true;
}
if (strncmp(current, "--dont-", 7) == 0 &&
strcmp(current + 7, option) == 0) {
value = false;
return true;
}
return false;
}
if (!value_cstr) {
value = true;
return true;
}
if (strcasecmp(value_cstr, "yes") == 0 || strcasecmp(value_cstr, "1") == 0) {
value = true;
return true;
}
if (strcasecmp(value_cstr, "no") == 0 || strcasecmp(value_cstr, "0") == 0) {
value = false;
return true;
}
failure(
"Option '--%s' expects a 'boolean' value Yes/No, so '%s' is unexpected\n",
option, value_cstr);
}
//-----------------------------------------------------------------------------
const struct option_verb mode_bits[] = {
{"rdonly", MDBX_RDONLY}, {"mapasync", MDBX_MAPASYNC},
{"utterly", MDBX_UTTERLY_NOSYNC}, {"nosubdir", MDBX_NOSUBDIR},
{"nosync", MDBX_NOSYNC}, {"nometasync", MDBX_NOMETASYNC},
{"writemap", MDBX_WRITEMAP}, {"notls", MDBX_NOTLS},
{"nordahead", MDBX_NORDAHEAD}, {"nomeminit", MDBX_NOMEMINIT},
{"coalesce", MDBX_COALESCE}, {"lifo", MDBX_LIFORECLAIM},
{"perturb", MDBX_PAGEPERTURB}, {nullptr, 0}};
const struct option_verb table_bits[] = {
{"key.reverse", MDBX_REVERSEKEY},
{"key.integer", MDBX_INTEGERKEY},
{"data.integer", MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT},
{"data.fixed", MDBX_DUPFIXED | MDBX_DUPSORT},
{"data.reverse", MDBX_REVERSEDUP | MDBX_DUPSORT},
{"data.dups", MDBX_DUPSORT},
{nullptr, 0}};
static void dump_verbs(const char *caption, size_t bits,
const struct option_verb *verbs) {
log_info("%s: 0x%" PRIx64 " = ", caption, (uint64_t)bits);
const char *comma = "";
while (verbs->mask && bits) {
if ((bits & verbs->mask) == verbs->mask) {
logging::feed("%s%s", comma, verbs->verb);
bits -= verbs->mask;
comma = ", ";
}
++verbs;
}
logging::feed("\n");
}
static void dump_duration(const char *caption, unsigned duration) {
log_info("%s: ", caption);
if (duration) {
if (duration > 24 * 3600)
logging::feed("%u_", duration / (24 * 3600));
if (duration > 3600)
logging::feed("%02u:", (duration % (24 * 3600)) / 3600);
logging::feed("%02u:%02u", (duration % 3600) / 60, duration % 60);
} else {
logging::feed("INFINITE");
}
logging::feed("\n");
}
void dump(const char *title) {
logging::local_suffix indent(title);
for (auto i = global::actors.begin(); i != global::actors.end(); ++i) {
const std::string tableid =
i->space_id ? "MAINDB" : ("SUB#" + std::to_string(i->space_id));
log_info("#%u, testcase %s, space_id/table %u\n", i->actor_id,
testcase2str(i->testcase), i->space_id);
indent.push();
if (i->params.loglevel) {
log_info("log: level %u, %s\n", i->params.loglevel,
i->params.pathname_log.empty() ? "console"
: i->params.pathname_log.c_str());
}
log_info("database: %s, size %" PRIu64 "\n", i->params.pathname_db.c_str(),
i->params.size);
dump_verbs("mode", i->params.mode_flags, mode_bits);
dump_verbs("table", i->params.table_flags, table_bits);
if (i->params.test_nops)
log_info("iterations/records %u\n", i->params.test_nops);
else
dump_duration("duration", i->params.test_duration);
if (i->params.nrepeat)
log_info("repeat %u\n", i->params.nrepeat);
else
log_info("repeat ETERNALLY\n");
log_info("threads %u\n", i->params.nthreads);
log_info("keygen.case: %s\n", keygencase2str(i->params.keygen.keycase));
log_info("keygen.seed: %u\n", i->params.keygen.seed);
log_info("key: minlen %u, maxlen %u\n", i->params.keylen_min,
i->params.keylen_max);
log_info("data: minlen %u, maxlen %u\n", i->params.datalen_min,
i->params.datalen_max);
log_info("batch: read %u, write %u\n", i->params.batch_read,
i->params.batch_write);
if (i->params.waitfor_nops)
log_info("wait: actor %u for %u ops\n", i->wait4id,
i->params.waitfor_nops);
else if (i->params.delaystart)
dump_duration("delay", i->params.delaystart);
else
log_info("no-delay\n");
log_info("limits: readers %u, tables %u\n", i->params.max_readers,
i->params.max_tables);
log_info("drop table: %s\n", i->params.drop_table ? "Yes" : "No");
indent.pop();
}
dump_duration("timeout", global::config::timeout_duration_seconds);
log_info("cleanup: before %s, after %s\n",
global::config::cleanup_before ? "Yes" : "No",
global::config::cleanup_after ? "Yes" : "No");
log_info("failfast: %s\n", global::config::failfast ? "Yes" : "No");
log_info("progress indicator: %s\n",
global::config::progress_indicator ? "Yes" : "No");
}
} /* namespace config */
//-----------------------------------------------------------------------------
using namespace config;
actor_config::actor_config(actor_testcase testcase, const actor_params &params,
unsigned space_id, unsigned wait4id)
: params(params) {
this->space_id = space_id;
this->actor_id = 1 + (unsigned)global::actors.size();
this->testcase = testcase;
this->wait4id = wait4id;
signal_nops = 0;
}
const std::string actor_config::serialize(const char *prefix) const {
simple_checksum checksum;
std::string result;
if (prefix)
result.append(prefix);
checksum.push(params.pathname_db);
result.append(params.pathname_db);
result.append("|");
checksum.push(params.pathname_log);
result.append(params.pathname_log);
result.append("|");
static_assert(std::is_pod<actor_params_pod>::value,
"actor_params_pod should by POD");
result.append(data2hex(static_cast<const actor_params_pod *>(&params),
sizeof(actor_params_pod), checksum));
result.append("|");
static_assert(std::is_pod<actor_config_pod>::value,
"actor_config_pod should by POD");
result.append(data2hex(static_cast<const actor_config_pod *>(this),
sizeof(actor_config_pod), checksum));
result.append("|");
result.append(osal_serialize(checksum));
result.append("|");
result.append(std::to_string(checksum.value));
return result;
}
bool actor_config::deserialize(const char *str, actor_config &config) {
simple_checksum checksum;
TRACE(">> actor_config::deserialize: %s\n", str);
const char *slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-1\n");
return false;
}
config.params.pathname_db.assign(str, slash - str);
checksum.push(config.params.pathname_db);
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-2\n");
return false;
}
config.params.pathname_log.assign(str, slash - str);
checksum.push(config.params.pathname_log);
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-3\n");
return false;
}
static_assert(std::is_pod<actor_params_pod>::value,
"actor_params_pod should by POD");
if (!hex2data(str, slash, static_cast<actor_params_pod *>(&config.params),
sizeof(actor_params_pod), checksum)) {
TRACE("<< actor_config::deserialize: actor_params_pod(%.*s)\n",
(int)(slash - str), str);
return false;
}
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-4\n");
return false;
}
static_assert(std::is_pod<actor_config_pod>::value,
"actor_config_pod should by POD");
if (!hex2data(str, slash, static_cast<actor_config_pod *>(&config),
sizeof(actor_config_pod), checksum)) {
TRACE("<< actor_config::deserialize: actor_config_pod(%.*s)\n",
(int)(slash - str), str);
return false;
}
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-5\n");
return false;
}
if (!config.osal_deserialize(str, slash, checksum)) {
TRACE("<< actor_config::deserialize: osal\n");
return false;
}
str = slash + 1;
uint64_t verify = std::stoull(std::string(str));
if (checksum.value != verify) {
TRACE("<< actor_config::deserialize: checksum mismatch\n");
return false;
}
TRACE("<< actor_config::deserialize: OK\n");
return true;
}

271
test/config.h Normal file
View File

@ -0,0 +1,271 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "log.h"
#include "utils.h"
#define ACTOR_ID_MAX INT16_MAX
enum actor_testcase { ac_none, ac_hill, ac_deadread, ac_deadwrite, ac_jitter };
enum actor_status {
as_unknown,
as_debuging,
as_running,
as_successful,
as_killed,
as_failed
};
const char *testcase2str(const actor_testcase);
const char *status2str(actor_status status);
enum keygen_case {
kc_random, /* [ 6.. 2.. 7.. 4.. 0.. 1.. 5.. 3.. ] */
kc_dashes, /* [ 0123.. 4567.. ] */
kc_custom,
/* TODO: more cases */
};
const char *keygencase2str(const keygen_case);
//-----------------------------------------------------------------------------
namespace config {
enum scale_mode { no_scale, decimal, binary, duration };
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
const char **value, const char *default_value = nullptr);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty = false);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
bool &value);
struct option_verb {
const char *const verb;
unsigned mask;
};
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &mask, const option_verb *verbs);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint64_t &value, const scale_mode scale,
const uint64_t minval = 0, const uint64_t maxval = INT64_MAX);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &value, const scale_mode scale,
const unsigned minval = 0, const unsigned maxval = INT32_MAX);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint8_t &value, const uint8_t minval = 0,
const uint8_t maxval = 255);
//-----------------------------------------------------------------------------
#pragma pack(push, 1)
struct keygen_params_pod {
keygen_case keycase;
/* Параметры генератора пар key-value.
*
* Ключи и значения генерируются по задаваемым параметрам на основе "плоской"
* исходной координаты. При этом, в общем случае, в процессе тестов исходная
* координата последовательно итерируется в заданном диапазоне, а необходимые
* паттерны/последовательности/узоры получаются за счет преобразования
* исходной координаты, согласно описанным ниже параметрам.
*
* Стоит отметить, что порядок описания параметров для удобства совпадает с
* порядком их использования, т.е. с порядком соответствующих преобразований.
*
* Второе важное замечание касается ограничений одновременной координированной
* генерации паттеров как для ключей, так и для значений. Суть в том, что
* такая возможность не нужна по следующим причинам:
* - libmdbx поддерживает два существенно различающихся вида таблиц,
* "уникальные" (без дубликатов и без multi-value), и так называемые
* "с дубликатами" (c multi-value).
* - Для таблиц "без дубликатов" только размер связанных к ключами значений
* (данных) оказывает влияния на работу движка, непосредственно содержимое
* данных не анализируется движком и не оказывает влияния на его работу.
* - Для таблиц "с дубликатами", при наличии более одного значения для
* некоторого ключа, формируется дочернее btree-поддерево. Это дерево
* формируется в отдельном "кусте" страниц и обслуживается независимо
* от окружения родительского ключа.
* - Таким образом, паттерн генерации значений имеет смысл только для
* таблиц "с дубликатами" и только в контексте одного значения ключа.
* Иначе говоря, нет смысла в со-координации генерации паттернов для
* ключей и значений. Более того, генерацию значений всегда необходимо
* рассматривать в контексте связки с одним значением ключа.
*
* width:
* Большинство тестов предполагают создание или итерирование некоторого
* количества записей. При этом требуется итерирование или генерация
* значений и ключей из некоторого ограниченного пространства вариантов.
*
* Параметр width задает такую ширину пространства вариантов в битах.
* Таким образом мощность пространства вариантов (пока) всегда равна
* степени двойки. Это ограничение можно снять, но ценой увеличения
* вычислительной сложности, включая потерю простоты и прозрачности.
*
* С другой стороны, не-битовый width может быть полезен:
* - Позволит генерировать ключи/значения в точно задаваемом диапазоне.
* Например, перебрать в псевдо-случайном порядке 10001 значение.
* - Позволит поровну разделять заданное пространство (диапазон)
* ключей/значений между количеством потоков некратным степени двойки.
*
* mesh и seed:
* Позволяют получить псевдо-случайные последовательности ключей/значений.
* Параметр mesh задает сколько младших бит исходной плоской координаты
* будет "перемешано" (инъективно отображено), а параметр seed позволяет
* выбрать конкретный вариант "перемешивания".
*
* Перемешивание выполняется при ненулевом значении mesh. Перемешивание
* реализуется посредством применения двух инъективных функций для
* заданного количества бит:
* - применяется первая инъективная функция;
* - к результату добавляется salt полученный из seed;
* - применяется вторая инъективная функция;
*
* Следует отметить, что mesh умышленно позволяет перемешать только младшую
* часть, что при ненулевом значении split (см далее) не позволяет получать
* псевдо-случайные значений ключей без псевдо-случайности в значениях.
*
* Такое ограничение соответствуют внутренней алгоритмике libmdbx. Проще
* говоря мы можем проверить движок псевдо-случайной последовательностью
* ключей на таблицах без дубликатов (без multi-value), а затем проверить
* корректность работу псевдо-случайной последовательностью значений на
* таблицах с дубликатами (с multi-value), опционально добавляя
* псевдо-случайности к последовательности ключей. Однако, нет смысла
* генерировать псевдо-случайные ключи, одновременно с формированием
* какого-либо паттерна в значениях, так как содержимое в данных либо
* не будет иметь значения (для таблиц без дубликатов), либо будет
* обрабатываться в отдельных btree-поддеревьях.
*
* rotate и offset:
* Для проверки слияния и разделения страниц внутри движка требуются
* генерация ключей/значений в виде не-смежных последовательностей, как-бы
* в виде "пунктира", который постепенно заполняет весь заданных диапазон.
*
* Параметры позволяют генерировать такой "пунктир". Соответственно rotate
* задает циклический сдвиг вправо, а offset задает смещение, точнее говоря
* сложение по модулю внутри диапазона заданного посредством width.
*
* Например, при rotate равном 1 (циклический сдвиг вправо на 1 бит),
* четные и нечетные исходные значения сложатся в две линейные
* последовательности, которые постепенно закроют старшую и младшую
* половины диапазона.
*
* split:
* Для таблиц без дубликатов (без multi-value ключей) фактически требуется
* генерация только ключей, а данные могут быть постоянным. Но для таблиц с
* дубликатами (с multi-value ключами) также требуется генерация значений.
*
* Ненулевое значение параметра split фактически включает генерацию значений,
* при этом значение split определяет сколько бит исходного абстрактного
* номера будет отрезано для генерации значения.
*/
uint8_t width;
uint8_t mesh;
uint8_t rotate;
uint8_t split;
uint32_t seed;
uint64_t offset;
};
struct actor_params_pod {
unsigned loglevel;
unsigned mode_flags;
unsigned table_flags;
uint64_t size;
unsigned test_duration;
unsigned test_nops;
unsigned nrepeat;
unsigned nthreads;
unsigned keylen_min, keylen_max;
unsigned datalen_min, datalen_max;
unsigned batch_read;
unsigned batch_write;
unsigned delaystart;
unsigned waitfor_nops;
unsigned max_readers;
unsigned max_tables;
keygen_params_pod keygen;
bool drop_table;
};
struct actor_config_pod {
unsigned actor_id, space_id;
actor_testcase testcase;
unsigned wait4id;
unsigned signal_nops;
};
#pragma pack(pop)
extern const struct option_verb mode_bits[];
extern const struct option_verb table_bits[];
void dump(const char *title = "config-dump: ");
} /* namespace config */
struct actor_params : public config::actor_params_pod {
std::string pathname_log;
std::string pathname_db;
void set_defaults(void);
};
struct actor_config : public config::actor_config_pod {
actor_params params;
bool wanna_event4signalling() const { return true /* TODO ? */; }
actor_config(actor_testcase testcase, const actor_params &params,
unsigned space_id, unsigned wait4id);
actor_config(const char *str) {
if (!deserialize(str, *this))
failure("Invalid internal parameter '%s'\n", str);
}
const std::string osal_serialize(simple_checksum &) const;
bool osal_deserialize(const char *str, const char *end, simple_checksum &);
const std::string serialize(const char *prefix) const;
static bool deserialize(const char *str, actor_config &config);
bool is_waitable(size_t nops) const {
switch (testcase) {
case ac_hill:
if (!params.test_nops || params.test_nops >= nops)
return true;
default:
return false;
}
}
};

63
test/dead.cc Normal file
View File

@ -0,0 +1,63 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_deadread::setup() {
log_trace(">> setup");
if (!inherited::setup())
return false;
log_trace("<< setup");
return true;
}
bool testcase_deadread::run() {
db_open();
txn_begin(true);
return true;
}
bool testcase_deadread::teardown() {
log_trace(">> teardown");
cursor_guard.release();
txn_guard.release();
db_guard.release();
return inherited::teardown();
}
//-----------------------------------------------------------------------------
bool testcase_deadwrite::setup() {
log_trace(">> setup");
if (!inherited::setup())
return false;
log_trace("<< setup");
return true;
}
bool testcase_deadwrite::run() {
db_open();
txn_begin(false);
return true;
}
bool testcase_deadwrite::teardown() {
log_trace(">> teardown");
cursor_guard.release();
txn_guard.release();
db_guard.release();
return inherited::teardown();
}

227
test/hill.cc Normal file
View File

@ -0,0 +1,227 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_hill::setup() {
log_trace(">> setup");
if (!inherited::setup())
return false;
/* TODO */
log_trace("<< setup");
return true;
}
bool testcase_hill::run() {
db_open();
txn_begin(false);
MDBX_dbi dbi = db_table_open(true);
txn_end(false);
/* LY: тест "холмиком":
* - сначала наполняем таблицу циклическими CRUD-манипуляциями,
* которые в каждом цикле делают несколько операций, включая удаление,
* но в результате добавляют записи.
* - затем очищаем таблицу также CRUD-манипуляциями, но уже с другой
* пропорцией удалений.
*
* При этом очень многое зависит от порядка перебора ключей:
* - (псевдо)случайное распределение требуется лишь для полноты картины,
* но в целом не покрывает важных кейсов.
* - кроме (псевдо)случайного перебора требуется последовательное
* итерирование ключей интервалами различной ширины, с тем чтобы
* проверить различные варианты как разделения, так и слияния страниц
* внутри движка.
* - при не-уникальных ключах (MDBX_DUPSORT с подвариантами), для каждого
* повтора внутри движка формируется вложенное btree-дерево,
* соответственно требуется соблюдение аналогичных принципов
* итерирования для значений.
*/
/* TODO: работа в несколько потоков */
keyvalue_maker.setup(config.params, 0 /* thread_number */);
keygen::buffer a_key = keygen::alloc(config.params.keylen_max);
keygen::buffer a_data_0 = keygen::alloc(config.params.datalen_max);
keygen::buffer a_data_1 = keygen::alloc(config.params.datalen_max);
keygen::buffer b_key = keygen::alloc(config.params.keylen_max);
keygen::buffer b_data = keygen::alloc(config.params.datalen_max);
const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT)
? MDBX_NODUPDATA
: MDBX_NODUPDATA | MDBX_NOOVERWRITE;
const unsigned update_flags =
MDBX_CURRENT | MDBX_NODUPDATA | MDBX_NOOVERWRITE;
uint64_t serial_count = 0;
unsigned txn_nops = 0;
if (!txn_guard)
txn_begin(false);
while (should_continue()) {
const keygen::serial_t a_serial = serial_count;
if (unlikely(!keyvalue_maker.increment(serial_count, 1)))
failure("uphill: unexpected key-space overflow");
const keygen::serial_t b_serial = serial_count;
assert(b_serial > a_serial);
// создаем первую запись из пары
const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31);
log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift,
a_serial);
generate_pair(a_serial, a_key, a_data_1, age_shift);
int rc = mdbx_put(txn_guard.get(), dbi, &a_key->value, &a_data_1->value,
insert_flags);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_put(insert-a.1)", rc);
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
txn_nops = 0;
}
// создаем вторую запись из пары
log_trace("uphill: insert-b %" PRIu64, b_serial);
generate_pair(b_serial, b_key, b_data, 0);
rc = mdbx_put(txn_guard.get(), dbi, &b_key->value, &b_data->value,
insert_flags);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_put(insert-b)", rc);
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
txn_nops = 0;
}
// обновляем данные в первой записи
log_trace("uphill: update-a (age %" PRIu64 "->0) %" PRIu64, age_shift,
a_serial);
generate_pair(a_serial, a_key, a_data_0, 0);
rc = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_0->value,
&a_data_1->value, update_flags);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_put(update-a: 1->0)", rc);
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
txn_nops = 0;
}
// удаляем вторую запись
log_trace("uphill: delete-b %" PRIu64, b_serial);
rc = mdbx_del(txn_guard.get(), dbi, &b_key->value, &b_data->value);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_del(b)", rc);
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
txn_nops = 0;
}
report(1);
if (!keyvalue_maker.increment(serial_count, 1)) {
// дошли до границы пространства ключей
serial_count = a_serial;
goto overflow;
}
}
while (serial_count > 0) {
if (unlikely(!keyvalue_maker.increment(serial_count, -2)))
failure("downhill: unexpected key-space underflow");
overflow:
const keygen::serial_t a_serial = serial_count;
const keygen::serial_t b_serial = a_serial + 1;
assert(b_serial > a_serial);
// обновляем первую запись из пары
const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31);
log_trace("downhill: update-a (age 0->%" PRIu64 ") %" PRIu64, age_shift,
a_serial);
generate_pair(a_serial, a_key, a_data_0, 0);
generate_pair(a_serial, a_key, a_data_1, age_shift);
if (a_serial == 808)
log_trace("!!!");
int rc = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_1->value,
&a_data_0->value, update_flags);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_put(update-a: 0->1)", rc);
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
txn_nops = 0;
}
// создаем вторую запись из пары
log_trace("downhill: insert-b %" PRIu64, b_serial);
generate_pair(b_serial, b_key, b_data, 0);
rc = mdbx_put(txn_guard.get(), dbi, &b_key->value, &b_data->value,
insert_flags);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_put(insert-b)", rc);
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
txn_nops = 0;
}
// удаляем первую запись
log_trace("downhill: delete-a (age %" PRIu64 ") %" PRIu64, age_shift,
a_serial);
rc = mdbx_del(txn_guard.get(), dbi, &a_key->value, &a_data_1->value);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_del(a)", rc);
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
txn_nops = 0;
}
// удаляем вторую запись
log_trace("downhill: delete-b %" PRIu64, b_serial);
rc = mdbx_del(txn_guard.get(), dbi, &b_key->value, &b_data->value);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_del(b)", rc);
if (++txn_nops >= config.params.batch_write) {
txn_restart(false, false);
txn_nops = 0;
}
report(1);
}
if (txn_guard)
txn_end(false);
if (dbi) {
if (config.params.drop_table && !mode_readonly()) {
txn_begin(false);
db_table_drop(dbi);
txn_end(false);
} else
db_table_close(dbi);
}
return true;
}
bool testcase_hill::teardown() {
log_trace(">> teardown");
return inherited::teardown();
}

69
test/jitter.cc Normal file
View File

@ -0,0 +1,69 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_jitter::setup() {
log_trace(">> setup");
if (!inherited::setup())
return false;
log_trace("<< setup");
return true;
}
bool testcase_jitter::run() {
while (should_continue()) {
jitter_delay();
db_open();
if (flipcoin()) {
jitter_delay();
txn_begin(true);
fetch_canary();
jitter_delay();
txn_end(flipcoin());
}
jitter_delay();
txn_begin(mode_readonly());
jitter_delay();
if (!mode_readonly()) {
fetch_canary();
update_canary(1);
/* TODO:
* - db_setsize()
* ...
*/
}
txn_end(flipcoin());
if (flipcoin()) {
jitter_delay();
txn_begin(true);
jitter_delay();
txn_end(flipcoin());
}
jitter_delay();
db_close();
report(1);
}
return true;
}
bool testcase_jitter::teardown() {
log_trace(">> teardown");
return inherited::teardown();
}

238
test/keygen.cc Normal file
View File

@ -0,0 +1,238 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
namespace keygen {
static inline __pure_function serial_t mask(unsigned bits) {
assert(bits > 0 && bits <= serial_maxwith);
return serial_allones >> (serial_maxwith - bits);
}
/* LY: https://en.wikipedia.org/wiki/Injective_function */
serial_t injective(const serial_t serial,
const unsigned bits /* at least serial_minwith (8) */,
const serial_t salt) {
assert(bits > serial_minwith && bits <= serial_maxwith);
/* LY: All these "magic" prime numbers were found
* and verified with a bit of brute force. */
static const uint64_t m[64 - serial_minwith] = {
/* 8 - 24 */
113, 157, 397, 653, 1753, 5641, 9697, 23873, 25693, 80833, 105953, 316937,
309277, 834497, 1499933, 4373441, 10184137,
/* 25 - 64 */
10184137, 17279209, 33990377, 67295161, 284404553, 1075238767, 6346721573,
6924051577, 19204053433, 45840188887, 53625693977, 73447827913,
141638870249, 745683604649, 1283334050489, 1100828289853, 2201656586197,
5871903036137, 11238507001417, 45264020802263, 105008404482889,
81921776907059, 199987980256399, 307207457507641, 946769023178273,
2420886491930041, 3601632139991929, 11984491914483833, 21805846439714153,
23171543400565993, 53353226456762893, 155627817337932409,
227827205384840249, 816509268558278821, 576933057762605689,
2623957345935638441, 5048241705479929949, 4634245581946485653};
static const uint8_t s[64 - serial_minwith] = {
/* 8 - 24 */
2, 3, 4, 4, 2, 4, 3, 3, 7, 3, 3, 4, 8, 3, 10, 3, 11,
/* 25 - 64 */
11, 9, 9, 9, 11, 10, 5, 14, 11, 16, 14, 12, 13, 16, 19, 10, 10, 21, 7, 20,
10, 14, 22, 19, 3, 21, 18, 19, 26, 24, 2, 21, 25, 29, 24, 10, 11, 14};
serial_t result = serial * m[bits - 8];
if (salt) {
const unsigned left = bits / 2;
const unsigned right = bits - left;
result = (result << left) | ((result & mask(bits)) >> right);
result = (result ^ salt) * m[bits - 8];
}
result ^= result << s[bits - 8];
result &= mask(bits);
log_trace("keygen-injective: serial %" PRIu64 " into %" PRIu64, serial,
result);
return result;
}
void __hot maker::pair(serial_t serial, const buffer &key, buffer &value,
serial_t value_age) {
assert(mapping.width >= serial_minwith && mapping.width <= serial_maxwith);
assert(mapping.split <= mapping.width);
assert(mapping.mesh <= mapping.width);
assert(mapping.rotate <= mapping.width);
assert(mapping.offset <= mask(mapping.width));
assert(!(key_essentials.flags & (MDBX_INTEGERDUP | MDBX_REVERSEDUP)));
assert(!(value_essentials.flags & (MDBX_INTEGERKEY | MDBX_REVERSEKEY)));
log_trace("keygen-pair: serial %" PRIu64 ", data-age %" PRIu64, serial,
value_age);
if (mapping.mesh >= serial_minwith) {
serial =
(serial & ~mask(mapping.mesh)) | injective(serial, mapping.mesh, salt);
log_trace("keygen-pair: mesh %" PRIu64, serial);
}
if (mapping.rotate) {
const unsigned right = mapping.rotate;
const unsigned left = mapping.width - right;
serial = (serial << left) | ((serial & mask(mapping.width)) >> right);
log_trace("keygen-pair: rotate %" PRIu64 ", 0x%" PRIx64, serial, serial);
}
serial = (serial + mapping.offset) & mask(mapping.width);
log_trace("keygen-pair: offset %" PRIu64, serial);
serial += base;
serial_t key_serial = serial;
serial_t value_serial = value_age;
if (mapping.split) {
key_serial = serial >> mapping.split;
value_serial =
(serial & mask(mapping.split)) | (value_age << mapping.split);
}
log_trace("keygen-pair: key %" PRIu64 ", value %" PRIu64, key_serial,
value_serial);
mk(key_serial, key_essentials, *key);
mk(value_serial, value_essentials, *value);
if (log_enabled(logging::trace)) {
char dump_key[128], dump_value[128];
log_trace("keygen-pair: key %s, value %s",
mdbx_dkey(&key->value, dump_key, sizeof(dump_key)),
mdbx_dkey(&value->value, dump_value, sizeof(dump_value)));
}
}
void maker::setup(const config::actor_params_pod &actor,
unsigned thread_number) {
key_essentials.flags =
actor.table_flags & (MDBX_INTEGERKEY | MDBX_REVERSEKEY);
assert(actor.keylen_min < UINT8_MAX);
key_essentials.minlen = (uint8_t)actor.keylen_min;
assert(actor.keylen_max < UINT16_MAX);
key_essentials.maxlen = (uint16_t)actor.keylen_max;
value_essentials.flags =
actor.table_flags & (MDBX_INTEGERDUP | MDBX_REVERSEDUP);
assert(actor.datalen_min < UINT8_MAX);
value_essentials.minlen = (uint8_t)actor.datalen_min;
assert(actor.datalen_max < UINT16_MAX);
value_essentials.maxlen = (uint16_t)actor.datalen_max;
assert(thread_number < 2);
(void)thread_number;
mapping = actor.keygen;
salt = actor.keygen.seed * UINT64_C(14653293970879851569);
// FIXME: TODO
base = 0;
}
bool maker::increment(serial_t &serial, int delta) {
if (serial > mask(mapping.width)) {
log_extra("keygen-increment: %" PRIu64 " > %" PRIu64 ", overflow", serial,
mask(mapping.width));
return false;
}
serial_t target = serial + (int64_t)delta;
if (target > mask(mapping.width)) {
log_extra("keygen-increment: %" PRIu64 "%-d => %" PRIu64 ", overflow",
serial, delta, target);
return false;
}
log_extra("keygen-increment: %" PRIu64 "%-d => %" PRIu64 ", continue", serial,
delta, target);
serial = target;
return true;
}
//-----------------------------------------------------------------------------
size_t length(serial_t serial) {
size_t n = 0;
if (serial > UINT32_MAX) {
n = 4;
serial >>= 32;
}
if (serial > UINT16_MAX) {
n += 2;
serial >>= 16;
}
if (serial > UINT8_MAX) {
n += 1;
serial >>= 8;
}
return (serial > 0) ? n + 1 : n;
}
buffer alloc(size_t limit) {
result *ptr = (result *)malloc(sizeof(result) + limit);
if (unlikely(ptr == nullptr))
failure_perror("malloc(keyvalue_buffer)", errno);
ptr->value.iov_base = ptr->bytes;
ptr->value.iov_len = 0;
ptr->limit = limit;
return buffer(ptr);
}
void __hot maker::mk(const serial_t serial, const essentials &params,
result &out) {
assert(out.limit >= params.maxlen);
assert(params.maxlen >= params.minlen);
assert(params.maxlen >= length(serial));
out.value.iov_base = out.bytes;
out.value.iov_len = params.minlen;
if (params.flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) {
assert(params.maxlen == params.minlen);
assert(params.minlen == 4 || params.minlen == 8);
if (is_byteorder_le() || params.minlen == 8)
out.u64 = serial;
else
out.u32 = (uint32_t)serial;
} else if (params.flags & (MDBX_REVERSEKEY | MDBX_REVERSEDUP)) {
if (out.value.iov_len > 8) {
memset(out.bytes, '\0', out.value.iov_len - 8);
unaligned::store(out.bytes + out.value.iov_len - 8, htobe64(serial));
} else {
out.u64 = htobe64(serial);
if (out.value.iov_len < 8) {
out.value.iov_len = std::max(length(serial), out.value.iov_len);
out.value.iov_base = out.bytes + 8 - out.value.iov_len;
}
}
} else {
out.u64 = htole64(serial);
if (out.value.iov_len > 8)
memset(out.bytes + 8, '\0', out.value.iov_len - 8);
else
out.value.iov_len = std::max(length(serial), out.value.iov_len);
}
assert(out.value.iov_len >= params.minlen);
assert(out.value.iov_len <= params.maxlen);
assert(out.value.iov_len >= length(serial));
assert(out.value.iov_base >= out.bytes);
assert((uint8_t *)out.value.iov_base + out.value.iov_len <=
out.bytes + out.limit);
}
} /* namespace keygen */

125
test/keygen.h Normal file
View File

@ -0,0 +1,125 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "config.h"
#include "log.h"
#include "utils.h"
namespace keygen {
/* Под "генерацией ключей" здесь понимается генерация обоих значений для
* пар key-value, т.е. не только ключей, но и ассоциированных с ними данных.
*/
/* Генерацию ключей нельзя отнести к простым задачам, так как требования
* примерно следующие:
* - генерация разного количества уникальных ключей различной длины
* в задаваемом диапазоне;
* - возможность выбора как псевдо-случайного порядка ключей,
* так и по некоторым специфическим законам (ограниченными упорядоченными
* последовательностями, в шахматном порядке по граница диапазона и т.д.);
* - возможность генерации дубликатов с задаваемым законом распределения;
* - возможность генерации непересекающимися кластерами для параллельного
* использования в нескольких потоках;
* - использовать минимум ресурсов, как CPU, так и RAM, в том числе
* включая cache pollution и ram bandwidth.
*
* При этом заведомо известно, что для MDBX не имеет значения:
* - используемый алфавит (значения байтов);
* - частотное распределение по алфавиту;
* - абсолютное значение ключей или разность между отдельными значениями;
*
* Соответственно, в общих чертах, схема генерации следующая:
* - вводится плоская одномерная "координата" uint64_t;
* - генерация специфических паттернов (последовательностей)
* реализуется посредством соответствующих преобразований "координат", при
* этом все подобные преобразования выполняются только над "координатой";
* - итоговая "координата" преобразуется в 8-байтное суррогатное значение
* ключа;
* - для получения ключей длиной МЕНЕЕ 8 байт суррогат может усекаться
* до ненулевых байт, в том числе до нулевой длины;
* - для получения ключей длиной БОЛЕЕ 8 байт суррогат дополняется
* нулями или псевдослучайной последовательностью;
*
* Механизм генерации паттернов:
* - реализованный механизм является компромиссом между скоростью/простотой
* и гибкостью, необходимой для получения последовательностей, которых
* будет достаточно для проверки сценариев разделения и слияния страниц
* с данными внутри mdbx;
* - псевдо-случайные паттерны реализуются посредством набора инъективных
* отображающих функций;
* - не-псевдо-случайные паттерны реализуются посредством параметризируемого
* трех-этапного преобразования:
* 1) смещение (сложение) по модулю;
* 2) циклический сдвиг;
* 3) добавление абсолютного смещения (базы);
*/
typedef uint64_t serial_t;
enum : serial_t {
serial_minwith = 8,
serial_maxwith = sizeof(serial_t) * 8,
serial_allones = ~(serial_t)0
};
struct result {
MDBX_val value;
size_t limit;
union {
uint8_t bytes[sizeof(uint64_t)];
uint32_t u32;
uint64_t u64;
};
};
//-----------------------------------------------------------------------------
struct buffer_deleter : public std::unary_function<void, result *> {
void operator()(result *buffer) const { free(buffer); }
};
typedef std::unique_ptr<result, buffer_deleter> buffer;
buffer alloc(size_t limit);
class maker {
config::keygen_params_pod mapping;
serial_t base;
serial_t salt;
struct essentials {
uint8_t minlen;
uint8_t flags;
uint16_t maxlen;
} key_essentials, value_essentials;
static void mk(const serial_t serial, const essentials &params, result &out);
public:
maker() { memset(this, 0, sizeof(*this)); }
void pair(serial_t serial, const buffer &key, buffer &value,
serial_t value_age);
void setup(const config::actor_params_pod &actor, unsigned thread_number);
bool increment(serial_t &serial, int delta);
};
size_t length(serial_t serial);
} /* namespace keygen */

287
test/log.cc Normal file
View File

@ -0,0 +1,287 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
static void fflushall() { fflush(nullptr); }
void failure(const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
fflushall();
logging::output(logging::failure, fmt, ap);
va_end(ap);
fflushall();
exit(EXIT_FAILURE);
}
const char *test_strerror(int errnum) {
static __thread char buf[1024];
return mdbx_strerror_r(errnum, buf, sizeof(buf));
}
void __noreturn failure_perror(const char *what, int errnum) {
failure("%s failed: %s (%d)\n", what, test_strerror(errnum), errnum);
}
//-----------------------------------------------------------------------------
namespace logging {
static std::string prefix;
static std::string suffix;
static loglevel level;
static FILE *last;
void setup(loglevel _level, const std::string &_prefix) {
level = (_level > error) ? failure : _level;
prefix = _prefix;
}
void setup(const std::string &_prefix) { prefix = _prefix; }
const char *level2str(const loglevel alevel) {
switch (alevel) {
default:
return "invalid/unknown";
case extra:
return "extra";
case trace:
return "trace";
case verbose:
return "verbose";
case info:
return "info";
case notice:
return "notice";
case warning:
return "warning";
case error:
return "error";
case failure:
return "failure";
}
}
bool output(const loglevel priority, const char *format, ...) {
if (priority < level)
return false;
va_list ap;
va_start(ap, format);
output(priority, format, ap);
va_end(ap);
return true;
}
bool output(const logging::loglevel priority, const char *format, va_list ap) {
if (last) {
putc('\n', last);
fflush(last);
last = nullptr;
}
if (priority < level)
return false;
chrono::time now = chrono::now_realtime();
struct tm tm;
#ifdef _MSC_VER
int rc = _localtime32_s(&tm, (const __time32_t *)&now.utc);
#else
time_t time = now.utc;
int rc = localtime_r(&time, &tm) ? MDBX_SUCCESS : errno;
#endif
if (rc != MDBX_SUCCESS)
failure_perror("localtime_r()", rc);
last = stdout;
fprintf(last,
"[ %02d%02d%02d-%02d:%02d:%02d.%06d_%05u %-10s %.4s ] %s" /* TODO */,
tm.tm_year - 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min,
tm.tm_sec, chrono::fractional2us(now.fractional), osal_getpid(),
prefix.c_str(), level2str(priority), suffix.c_str());
va_list ones;
memset(&ones, 0, sizeof(ones)) /* zap MSVC and other stupid compilers */;
if (priority >= error)
va_copy(ones, ap);
vfprintf(last, format, ap);
size_t len = strlen(format);
char end = len ? format[len - 1] : '\0';
switch (end) {
default:
putc('\n', last);
// fall through
case '\n':
fflush(last);
last = nullptr;
// fall through
case ' ':
case '_':
case ':':
case '|':
case ',':
case '\t':
case '\b':
case '\r':
case '\0':
break;
}
if (priority >= error) {
if (last != stderr) {
fprintf(stderr, "[ %05u %-10s %.4s ] %s", osal_getpid(), prefix.c_str(),
level2str(priority), suffix.c_str());
vfprintf(stderr, format, ones);
if (end != '\n')
putc('\n', stderr);
fflush(stderr);
}
va_end(ones);
}
return true;
}
bool feed(const char *format, va_list ap) {
if (!last)
return false;
vfprintf(last, format, ap);
size_t len = strlen(format);
if (len && format[len - 1] == '\n') {
fflush(last);
last = nullptr;
}
return true;
}
bool feed(const char *format, ...) {
if (!last)
return false;
va_list ap;
va_start(ap, format);
feed(format, ap);
va_end(ap);
return true;
}
local_suffix::local_suffix(const char *c_str)
: trim_pos(suffix.size()), indent(0) {
suffix.append(c_str);
}
local_suffix::local_suffix(const std::string &str)
: trim_pos(suffix.size()), indent(0) {
suffix.append(str);
}
void local_suffix::push() {
indent += 1;
suffix.push_back('\t');
}
void local_suffix::pop() {
assert(indent > 0);
if (indent > 0) {
indent -= 1;
suffix.pop_back();
}
}
local_suffix::~local_suffix() { suffix.erase(trim_pos); }
} /* namespace log */
void log_extra(const char *msg, ...) {
if (logging::extra >= logging::level) {
va_list ap;
va_start(ap, msg);
logging::output(logging::extra, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_trace(const char *msg, ...) {
if (logging::trace >= logging::level) {
va_list ap;
va_start(ap, msg);
logging::output(logging::trace, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_verbose(const char *msg, ...) {
if (logging::verbose >= logging::level) {
va_list ap;
va_start(ap, msg);
logging::output(logging::verbose, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_info(const char *msg, ...) {
if (logging::info >= logging::level) {
va_list ap;
va_start(ap, msg);
logging::output(logging::info, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_notice(const char *msg, ...) {
if (logging::notice >= logging::level) {
va_list ap;
va_start(ap, msg);
logging::output(logging::notice, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_warning(const char *msg, ...) {
if (logging::warning >= logging::level) {
va_list ap;
va_start(ap, msg);
logging::output(logging::warning, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_error(const char *msg, ...) {
if (logging::error >= logging::level) {
va_list ap;
va_start(ap, msg);
logging::output(logging::error, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_trouble(const char *where, const char *what, int errnum) {
log_error("%s: %s %s", where, what, test_strerror(errnum));
}
bool log_enabled(const logging::loglevel priority) {
return (priority >= logging::level);
}

90
test/log.h Normal file
View File

@ -0,0 +1,90 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
void __noreturn usage(void);
#ifdef __GNUC__
#define __printf_args(format_index, first_arg) \
__attribute__((format(printf, format_index, first_arg)))
#else
#define __printf_args(format_index, first_arg)
#endif
void __noreturn __printf_args(1, 2) failure(const char *fmt, ...);
void __noreturn failure_perror(const char *what, int errnum);
const char *test_strerror(int errnum);
namespace logging {
enum loglevel {
extra,
trace,
verbose,
info,
notice,
warning,
error,
failure,
};
const char *level2str(const loglevel level);
void setup(loglevel level, const std::string &prefix);
void setup(const std::string &prefix);
bool output(const loglevel priority, const char *format, va_list ap);
bool __printf_args(2, 3)
output(const loglevel priority, const char *format, ...);
bool feed(const char *format, va_list ap);
bool __printf_args(1, 2) feed(const char *format, ...);
class local_suffix {
protected:
size_t trim_pos;
int indent;
public:
local_suffix(const local_suffix &) = delete;
local_suffix(const local_suffix &&) = delete;
const local_suffix &operator=(const local_suffix &) = delete;
local_suffix(const char *c_str);
local_suffix(const std::string &str);
void push();
void pop();
~local_suffix();
};
} /* namespace log */
void __printf_args(1, 2) log_extra(const char *msg, ...);
void __printf_args(1, 2) log_trace(const char *msg, ...);
void __printf_args(1, 2) log_verbose(const char *msg, ...);
void __printf_args(1, 2) log_info(const char *msg, ...);
void __printf_args(1, 2) log_notice(const char *msg, ...);
void __printf_args(1, 2) log_warning(const char *msg, ...);
void __printf_args(1, 2) log_error(const char *msg, ...);
void log_trouble(const char *where, const char *what, int errnum);
bool log_enabled(const logging::loglevel priority);
#ifdef _DEBUG
#define TRACE(...) log_trace(__VA_ARGS__)
#else
#define TRACE(...) __noop(__VA_ARGS__)
#endif

397
test/main.cc Normal file
View File

@ -0,0 +1,397 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
void __noreturn usage(void) {
printf("usage:\n"
"\tFIXME\n");
exit(EXIT_FAILURE);
}
//-----------------------------------------------------------------------------
void actor_params::set_defaults(void) {
pathname_log = "";
loglevel =
#ifdef NDEBUG
logging::notice;
#else
logging::trace;
#endif
pathname_db =
#ifdef __linux__
"/dev/shm/test_tmpdb.mdbx";
#else
"test_tmpdb.mdbx";
#endif
mode_flags = MDBX_NOSUBDIR | MDBX_WRITEMAP | MDBX_MAPASYNC | MDBX_NORDAHEAD |
MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_LIFORECLAIM;
table_flags = MDBX_DUPSORT;
size = 1024 * 1024 * 4;
keygen.seed = 1;
keygen.keycase = kc_random;
keygen.width = 32;
keygen.mesh = 32;
keygen.split = keygen.width / 2;
keygen.rotate = 0;
keygen.offset = 0;
test_duration = 0;
test_nops = 1000;
nrepeat = 1;
nthreads = 1;
keylen_min = 0;
keylen_max = 42;
datalen_min = 0;
datalen_max = 256;
batch_read = 4;
batch_write = 4;
delaystart = 0;
waitfor_nops = 0;
drop_table = false;
max_readers = 42;
max_tables = 42;
global::config::timeout_duration_seconds = 0 /* infinite */;
global::config::dump_config = true;
global::config::cleanup_before = true;
global::config::cleanup_after = true;
global::config::failfast = true;
global::config::progress_indicator = osal_istty(STDERR_FILENO);
}
namespace global {
std::vector<actor_config> actors;
std::unordered_map<unsigned, actor_config *> events;
std::unordered_map<mdbx_pid_t, actor_config *> pid2actor;
std::set<std::string> databases;
unsigned nactors;
chrono::time start_motonic;
chrono::time deadline_motonic;
bool singlemode;
namespace config {
unsigned timeout_duration_seconds;
bool dump_config;
bool cleanup_before;
bool cleanup_after;
bool failfast;
bool progress_indicator;
} /* namespace config */
} /* namespace global */
//-----------------------------------------------------------------------------
const char global::thunk_param_prefix[] = "--execute=";
std::string thunk_param(const actor_config &config) {
return config.serialize(global::thunk_param_prefix);
}
void cleanup() {
log_trace(">> cleanup");
/* TODO: remove each database */
log_trace("<< cleanup");
}
int main(int argc, char *const argv[]) {
#ifdef _DEBUG
log_trace("#argc = %d", argc);
for (int i = 0; i < argc; ++i)
log_trace("#argv[%d] = %s", i, argv[i]);
#endif /* _DEBUG */
if (argc < 2)
failure("No parameters given\n");
if (argc == 2 &&
strncmp(argv[1], global::thunk_param_prefix,
strlen(global::thunk_param_prefix)) == 0)
return test_execute(
actor_config(argv[1] + strlen(global::thunk_param_prefix)))
? EXIT_SUCCESS
: EXIT_FAILURE;
actor_params params;
params.set_defaults();
global::config::dump_config = true;
logging::setup((logging::loglevel)params.loglevel, "main");
unsigned last_space_id = 0;
for (int narg = 1; narg < argc; ++narg) {
const char *value = nullptr;
if (config::parse_option(argc, argv, narg, "case", &value)) {
testcase_setup(value, params, last_space_id);
continue;
}
if (config::parse_option(argc, argv, narg, "pathname", params.pathname_db))
continue;
if (config::parse_option(argc, argv, narg, "mode", params.mode_flags,
config::mode_bits))
continue;
if (config::parse_option(argc, argv, narg, "table", params.table_flags,
config::table_bits))
continue;
if (config::parse_option(argc, argv, narg, "size", params.size,
config::binary, 4096 * 4))
continue;
if (config::parse_option(argc, argv, narg, "keygen.width",
params.keygen.width, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "keygen.mesh",
params.keygen.mesh, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "keygen.seed",
params.keygen.seed, config::no_scale))
continue;
if (config::parse_option(argc, argv, narg, "keygen.split",
params.keygen.split, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "keygen.rotate",
params.keygen.rotate, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "keygen.offset",
params.keygen.offset, config::binary))
continue;
if (config::parse_option(argc, argv, narg, "keygen.case", &value)) {
keycase_setup(value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "repeat", params.nrepeat,
config::no_scale))
continue;
if (config::parse_option(argc, argv, narg, "threads", params.nthreads,
config::no_scale, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "timeout",
global::config::timeout_duration_seconds,
config::duration, 1))
continue;
if (config::parse_option(argc, argv, narg, "keylen.min", params.keylen_min,
config::no_scale, 0, params.keylen_max))
continue;
if (config::parse_option(argc, argv, narg, "keylen.max", params.keylen_max,
config::no_scale, params.keylen_min,
mdbx_get_maxkeysize(0)))
continue;
if (config::parse_option(argc, argv, narg, "datalen.min",
params.datalen_min, config::no_scale, 0,
params.datalen_max))
continue;
if (config::parse_option(argc, argv, narg, "datalen.max",
params.datalen_max, config::no_scale,
params.datalen_min, MDBX_MAXDATASIZE))
continue;
if (config::parse_option(argc, argv, narg, "batch.read", params.batch_read,
config::no_scale, 1))
continue;
if (config::parse_option(argc, argv, narg, "batch.write",
params.batch_write, config::no_scale, 1))
continue;
if (config::parse_option(argc, argv, narg, "delay", params.delaystart,
config::duration))
continue;
if (config::parse_option(argc, argv, narg, "wait4ops", params.waitfor_nops,
config::decimal))
continue;
if (config::parse_option(argc, argv, narg, "drop", params.drop_table))
continue;
if (config::parse_option(argc, argv, narg, "dump-config",
global::config::dump_config))
continue;
if (config::parse_option(argc, argv, narg, "cleanup-before",
global::config::cleanup_before))
continue;
if (config::parse_option(argc, argv, narg, "cleanup-after",
global::config::cleanup_after))
continue;
if (config::parse_option(argc, argv, narg, "max-readers",
params.max_readers, config::no_scale, 1, 255))
continue;
if (config::parse_option(argc, argv, narg, "max-tables", params.max_tables,
config::no_scale, 1, INT16_MAX))
continue;
if (config::parse_option(argc, argv, narg, "no-delay", nullptr)) {
params.delaystart = 0;
continue;
}
if (config::parse_option(argc, argv, narg, "no-wait", nullptr)) {
params.waitfor_nops = 0;
continue;
}
if (config::parse_option(argc, argv, narg, "duration", params.test_duration,
config::duration, 1)) {
params.test_nops = 0;
continue;
}
if (config::parse_option(argc, argv, narg, "nops", params.test_nops,
config::decimal, 1)) {
params.test_duration = 0;
continue;
}
if (config::parse_option(argc, argv, narg, "hill", &value, "auto")) {
configure_actor(last_space_id, ac_hill, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "jitter", nullptr)) {
configure_actor(last_space_id, ac_jitter, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "dead.reader", nullptr)) {
configure_actor(last_space_id, ac_deadread, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "dead.writer", nullptr)) {
configure_actor(last_space_id, ac_deadwrite, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "failfast",
global::config::failfast))
continue;
if (config::parse_option(argc, argv, narg, "progress",
global::config::progress_indicator))
continue;
if (*argv[narg] != '-')
testcase_setup(argv[narg], params, last_space_id);
else
failure("Unknown option '%s'\n", argv[narg]);
}
if (global::config::dump_config)
config::dump();
//--------------------------------------------------------------------------
if (global::actors.empty()) {
log_notice("no testcase(s) configured, exiting");
return EXIT_SUCCESS;
}
bool failed = false;
global::start_motonic = chrono::now_motonic();
global::deadline_motonic.fixedpoint =
(global::config::timeout_duration_seconds == 0)
? chrono::infinite().fixedpoint
: global::start_motonic.fixedpoint +
chrono::from_seconds(global::config::timeout_duration_seconds)
.fixedpoint;
if (global::config::cleanup_before)
cleanup();
if (global::actors.size() == 1) {
logging::setup("main");
global::singlemode = true;
if (!test_execute(global::actors.front()))
failed = true;
} else {
logging::setup("overlord");
log_trace("=== preparing...");
log_trace(">> osal_setup");
osal_setup(global::actors);
log_trace("<< osal_setup");
for (auto &a : global::actors) {
mdbx_pid_t pid;
log_trace(">> actor_start");
int rc = osal_actor_start(a, pid);
log_trace("<< actor_start");
if (rc) {
log_trace(">> killall_actors: (%s)", "start failed");
osal_killall_actors();
log_trace("<< killall_actors");
failure("Failed to start actor #%u (%s)\n", a.actor_id,
test_strerror(rc));
}
global::pid2actor[pid] = &a;
}
log_trace("=== ready to start...");
atexit(osal_killall_actors);
log_trace(">> wait4barrier");
osal_wait4barrier();
log_trace("<< wait4barrier");
size_t left = global::actors.size();
log_trace("=== polling...");
while (left > 0) {
unsigned timeout_seconds_left = INT_MAX;
chrono::time now_motonic = chrono::now_motonic();
if (now_motonic.fixedpoint >= global::deadline_motonic.fixedpoint)
timeout_seconds_left = 0;
else {
chrono::time left_motonic;
left_motonic.fixedpoint =
global::deadline_motonic.fixedpoint - now_motonic.fixedpoint;
timeout_seconds_left = left_motonic.seconds();
}
mdbx_pid_t pid;
int rc = osal_actor_poll(pid, timeout_seconds_left);
if (rc)
failure("Poll error: %s (%d)\n", test_strerror(rc), rc);
if (pid) {
actor_status status = osal_actor_info(pid);
actor_config *actor = global::pid2actor.at(pid);
if (!actor)
continue;
log_info("actor #%u, id %d, pid %u: %s\n", actor->actor_id,
actor->space_id, pid, status2str(status));
if (status > as_running) {
left -= 1;
if (status != as_successful) {
if (global::config::failfast && !failed) {
log_trace(">> killall_actors: (%s)", "failfast");
osal_killall_actors();
log_trace("<< killall_actors");
}
failed = true;
}
}
} else {
if (timeout_seconds_left == 0)
failure("Timeout\n");
}
}
log_trace("=== done...");
}
log_notice("RESULT: %s\n", failed ? "Failed" : "Successful");
if (global::config::cleanup_before) {
if (failed)
log_info("skip cleanup");
else
cleanup();
}
return failed ? EXIT_FAILURE : EXIT_SUCCESS;
}

274
test/osal-unix.cc Normal file
View File

@ -0,0 +1,274 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
#include <pthread.h>
#include <signal.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
struct shared_t {
pthread_barrier_t barrier;
pthread_mutex_t mutex;
size_t conds_size;
pthread_cond_t conds[1];
};
static shared_t *shared;
void osal_wait4barrier(void) {
assert(shared != nullptr && shared != MAP_FAILED);
int rc = pthread_barrier_wait(&shared->barrier);
if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) {
failure_perror("pthread_barrier_wait(shared)", rc);
}
}
void osal_setup(const std::vector<actor_config> &actors) {
assert(shared == nullptr);
pthread_mutexattr_t mutexattr;
int rc = pthread_mutexattr_init(&mutexattr);
if (rc)
failure_perror("pthread_mutexattr_init()", rc);
rc = pthread_mutexattr_setpshared(&mutexattr, PTHREAD_PROCESS_SHARED);
if (rc)
failure_perror("pthread_mutexattr_setpshared()", rc);
pthread_barrierattr_t barrierattr;
rc = pthread_barrierattr_init(&barrierattr);
if (rc)
failure_perror("pthread_barrierattr_init()", rc);
rc = pthread_barrierattr_setpshared(&barrierattr, PTHREAD_PROCESS_SHARED);
if (rc)
failure_perror("pthread_barrierattr_setpshared()", rc);
pthread_condattr_t condattr;
rc = pthread_condattr_init(&condattr);
if (rc)
failure_perror("pthread_condattr_init()", rc);
rc = pthread_condattr_setpshared(&condattr, PTHREAD_PROCESS_SHARED);
if (rc)
failure_perror("pthread_condattr_setpshared()", rc);
shared = (shared_t *)mmap(
nullptr, sizeof(shared_t) + actors.size() * sizeof(pthread_cond_t),
PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (MAP_FAILED == (void *)shared)
failure_perror("mmap(shared_conds)", errno);
rc = pthread_mutex_init(&shared->mutex, &mutexattr);
if (rc)
failure_perror("pthread_mutex_init(shared)", rc);
rc = pthread_barrier_init(&shared->barrier, &barrierattr, actors.size() + 1);
if (rc)
failure_perror("pthread_barrier_init(shared)", rc);
const size_t n = actors.size() + 1;
for (size_t i = 0; i < n; ++i) {
pthread_cond_t *event = &shared->conds[i];
rc = pthread_cond_init(event, &condattr);
if (rc)
failure_perror("pthread_cond_init(shared)", rc);
log_trace("osal_setup: event(shared pthread_cond) %" PRIuPTR " -> %p", i,
event);
}
shared->conds_size = actors.size() + 1;
pthread_barrierattr_destroy(&barrierattr);
pthread_condattr_destroy(&condattr);
pthread_mutexattr_destroy(&mutexattr);
}
void osal_broadcast(unsigned id) {
assert(shared != nullptr && shared != MAP_FAILED);
log_trace("osal_broadcast: event %u", id);
if (id >= shared->conds_size)
failure("osal_broadcast: id > limit");
int rc = pthread_cond_broadcast(shared->conds + id);
if (rc)
failure_perror("sem_post(shared)", rc);
}
int osal_waitfor(unsigned id) {
assert(shared != nullptr && shared != MAP_FAILED);
log_trace("osal_waitfor: event %u", id);
if (id >= shared->conds_size)
failure("osal_waitfor: id > limit");
int rc = pthread_mutex_lock(&shared->mutex);
if (rc != 0)
failure_perror("pthread_mutex_lock(shared)", rc);
rc = pthread_cond_wait(shared->conds + id, &shared->mutex);
if (rc && rc != EINTR)
failure_perror("pthread_cond_wait(shared)", rc);
rc = pthread_mutex_unlock(&shared->mutex);
if (rc != 0)
failure_perror("pthread_mutex_unlock(shared)", rc);
return (rc == 0) ? true : false;
}
//-----------------------------------------------------------------------------
const std::string
actor_config::osal_serialize(simple_checksum &checksum) const {
(void)checksum;
/* not used in workload, but just for testing */
return "unix.fork";
}
bool actor_config::osal_deserialize(const char *str, const char *end,
simple_checksum &checksum) {
(void)checksum;
/* not used in workload, but just for testing */
return strncmp(str, "unix.fork", 9) == 0 && str + 9 == end;
}
//-----------------------------------------------------------------------------
static std::unordered_map<pid_t, actor_status> childs;
static void handler_SIGCHLD(int unused) { (void)unused; }
mdbx_pid_t osal_getpid(void) { return getpid(); }
int osal_delay(unsigned seconds) { return sleep(seconds) ? errno : 0; }
int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) {
if (childs.empty())
signal(SIGCHLD, handler_SIGCHLD);
pid = fork();
if (pid == 0) {
const bool result = test_execute(config);
exit(result ? EXIT_SUCCESS : EXIT_FAILURE);
}
if (pid < 0)
return errno;
log_trace("osal_actor_start: fork pid %i for %u", pid, config.actor_id);
childs[pid] = as_running;
return 0;
}
actor_status osal_actor_info(const mdbx_pid_t pid) { return childs.at(pid); }
void osal_killall_actors(void) {
for (auto &pair : childs) {
kill(pair.first, SIGKILL);
pair.second = as_killed;
}
}
int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) {
retry:
int status, options = WNOHANG;
#ifdef WUNTRACED
options |= WUNTRACED;
#endif
#ifdef WCONTINUED
options |= WCONTINUED;
#endif
pid = waitpid(0, &status, options);
if (pid > 0) {
if (WIFEXITED(status))
childs[pid] =
(WEXITSTATUS(status) == EXIT_SUCCESS) ? as_successful : as_failed;
else if (WIFSIGNALED(status) || WCOREDUMP(status))
childs[pid] = as_killed;
else if (WIFSTOPPED(status))
childs[pid] = as_debuging;
else if (WIFCONTINUED(status))
childs[pid] = as_running;
else {
assert(false);
}
return 0;
}
if (pid == 0) {
if (timeout && sleep(timeout))
goto retry;
return 0;
}
switch (errno) {
case EINTR:
pid = 0;
return 0;
case ECHILD:
default:
pid = 0;
return errno;
}
}
void osal_yield(void) {
if (sched_yield())
failure_perror("sched_yield()", errno);
}
void osal_udelay(unsigned us) {
chrono::time until, now = chrono::now_motonic();
until.fixedpoint = now.fixedpoint + chrono::from_us(us).fixedpoint;
struct timespec ts;
static unsigned threshold_us;
if (threshold_us == 0) {
if (clock_getres(CLOCK_PROCESS_CPUTIME_ID, &ts)) {
int rc = errno;
failure_perror("clock_getres(CLOCK_PROCESS_CPUTIME_ID)", rc);
}
chrono::time threshold = chrono::from_timespec(ts);
assert(threshold.seconds() == 0);
threshold_us = chrono::fractional2us(threshold.fractional);
if (threshold_us < 1000)
threshold_us = 1000;
}
ts.tv_sec = ts.tv_nsec = 0;
if (us > threshold_us) {
ts.tv_sec = us / 1000000u;
ts.tv_nsec = (us % 1000000u) * 1000u;
}
do {
if (us > threshold_us) {
if (nanosleep(&ts, &ts)) {
int rc = errno;
/* if (rc == EINTR) { ... } ? */
failure_perror("usleep()", rc);
}
us = ts.tv_sec * 1000000u + ts.tv_nsec / 1000u;
}
cpu_relax();
now = chrono::now_motonic();
} while (until.fixedpoint > now.fixedpoint);
}
bool osal_istty(int fd) { return isatty(fd) == 1; }

307
test/osal-windows.cc Normal file
View File

@ -0,0 +1,307 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
static std::unordered_map<unsigned, HANDLE> events;
static HANDLE hBarrierSemaphore, hBarrierEvent;
static int waitstatus2errcode(DWORD result) {
switch (result) {
case WAIT_OBJECT_0:
return MDBX_SUCCESS;
case WAIT_FAILED:
return GetLastError();
case WAIT_ABANDONED:
return ERROR_ABANDONED_WAIT_0;
case WAIT_IO_COMPLETION:
return ERROR_USER_APC;
case WAIT_TIMEOUT:
return ERROR_TIMEOUT;
default:
return ERROR_UNHANDLED_ERROR;
}
}
void osal_wait4barrier(void) {
DWORD rc = WaitForSingleObject(hBarrierSemaphore, 0);
switch (rc) {
default:
failure_perror("WaitForSingleObject(BarrierSemaphore)",
waitstatus2errcode(rc));
case WAIT_OBJECT_0:
rc = WaitForSingleObject(hBarrierEvent, INFINITE);
if (rc != WAIT_OBJECT_0)
failure_perror("WaitForSingleObject(BarrierEvent)",
waitstatus2errcode(rc));
break;
case WAIT_TIMEOUT:
if (!SetEvent(hBarrierEvent))
failure_perror("SetEvent(BarrierEvent)", GetLastError());
break;
}
}
static HANDLE make_inharitable(HANDLE hHandle) {
assert(hHandle != NULL && hHandle != INVALID_HANDLE_VALUE);
if (!DuplicateHandle(GetCurrentProcess(), hHandle, GetCurrentProcess(),
&hHandle, 0, TRUE,
DUPLICATE_CLOSE_SOURCE | DUPLICATE_SAME_ACCESS))
failure_perror("DuplicateHandle()", GetLastError());
return hHandle;
}
void osal_setup(const std::vector<actor_config> &actors) {
assert(events.empty());
const size_t n = actors.size() + 1;
events.reserve(n);
for (unsigned i = 0; i < n; ++i) {
HANDLE hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
if (!hEvent)
failure_perror("CreateEvent()", GetLastError());
hEvent = make_inharitable(hEvent);
log_trace("osal_setup: event %" PRIuPTR " -> %p", i, hEvent);
events[i] = hEvent;
}
hBarrierSemaphore = CreateSemaphore(NULL, 0, (LONG)actors.size(), NULL);
if (!hBarrierSemaphore)
failure_perror("CreateSemaphore(BarrierSemaphore)", GetLastError());
hBarrierSemaphore = make_inharitable(hBarrierSemaphore);
hBarrierEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
if (!hBarrierEvent)
failure_perror("CreateEvent(BarrierEvent)", GetLastError());
hBarrierEvent = make_inharitable(hBarrierEvent);
}
void osal_broadcast(unsigned id) {
log_trace("osal_broadcast: event %u", id);
if (!SetEvent(events.at(id)))
failure_perror("SetEvent()", GetLastError());
}
int osal_waitfor(unsigned id) {
log_trace("osal_waitfor: event %u", id);
DWORD rc = WaitForSingleObject(events.at(id), INFINITE);
return waitstatus2errcode(rc);
}
mdbx_pid_t osal_getpid(void) { return GetCurrentProcessId(); }
int osal_delay(unsigned seconds) {
Sleep(seconds * 1000u);
return 0;
}
//-----------------------------------------------------------------------------
const std::string
actor_config::osal_serialize(simple_checksum &checksum) const {
checksum.push(hBarrierSemaphore);
checksum.push(hBarrierEvent);
HANDLE hWait = INVALID_HANDLE_VALUE;
if (wait4id) {
hWait = events.at(wait4id);
checksum.push(hWait);
}
HANDLE hSignal = INVALID_HANDLE_VALUE;
if (wanna_event4signalling()) {
hSignal = events.at(actor_id);
checksum.push(hSignal);
}
return format("%p.%p.%p.%p", hBarrierSemaphore, hBarrierEvent, hWait,
hSignal);
}
bool actor_config::osal_deserialize(const char *str, const char *end,
simple_checksum &checksum) {
std::string copy(str, end - str);
TRACE(">> osal_deserialize(%s)\n", copy.c_str());
assert(hBarrierSemaphore == 0);
assert(hBarrierEvent == 0);
assert(events.empty());
HANDLE hWait, hSignal;
if (sscanf_s(copy.c_str(), "%p.%p.%p.%p", &hBarrierSemaphore, &hBarrierEvent,
&hWait, &hSignal) != 4) {
TRACE("<< osal_deserialize: failed\n");
return false;
}
checksum.push(hBarrierSemaphore);
checksum.push(hBarrierEvent);
if (wait4id) {
checksum.push(hWait);
events[wait4id] = hWait;
}
if (wanna_event4signalling()) {
checksum.push(hSignal);
events[actor_id] = hSignal;
}
TRACE("<< osal_deserialize: OK\n");
return true;
}
//-----------------------------------------------------------------------------
typedef std::pair<HANDLE, actor_status> child;
static std::unordered_map<mdbx_pid_t, child> childs;
int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) {
if (childs.size() == MAXIMUM_WAIT_OBJECTS)
failure("Could't manage more that %u actors on Windows\n",
MAXIMUM_WAIT_OBJECTS);
_flushall();
STARTUPINFOA StartupInfo;
GetStartupInfoA(&StartupInfo);
char exename[_MAX_PATH];
DWORD exename_size = sizeof(exename);
if (!QueryFullProcessImageNameA(GetCurrentProcess(), 0, exename,
&exename_size))
failure_perror("QueryFullProcessImageName()", GetLastError());
std::string cmdline = "test_mdbx.child " + thunk_param(config);
PROCESS_INFORMATION ProcessInformation;
if (!CreateProcessA(exename, const_cast<char *>(cmdline.c_str()),
NULL, // Retuned process handle is not inheritable.
NULL, // Retuned thread handle is not inheritable.
TRUE, // Child inherits all inheritable handles.
NORMAL_PRIORITY_CLASS | INHERIT_PARENT_AFFINITY,
NULL, // Inherit the parent's environment.
NULL, // Inherit the parent's current directory.
&StartupInfo, &ProcessInformation))
return GetLastError();
CloseHandle(ProcessInformation.hThread);
pid = ProcessInformation.dwProcessId;
childs[pid] = std::make_pair(ProcessInformation.hProcess, as_running);
return 0;
}
actor_status osal_actor_info(const mdbx_pid_t pid) {
actor_status status = childs.at(pid).second;
if (status > as_running)
return status;
DWORD ExitCode;
if (!GetExitCodeProcess(childs.at(pid).first, &ExitCode))
failure_perror("GetExitCodeProcess()", GetLastError());
switch (ExitCode) {
case STILL_ACTIVE:
return as_running;
case EXIT_SUCCESS:
status = as_successful;
break;
// case EXCEPTION_BREAKPOINT:
case EXCEPTION_SINGLE_STEP:
status = as_debuging;
break;
case STATUS_CONTROL_C_EXIT:
case EXCEPTION_NONCONTINUABLE_EXCEPTION:
status = as_killed;
break;
default:
status = as_failed;
break;
}
childs.at(pid).second = status;
return status;
}
void osal_killall_actors(void) {
for (auto &pair : childs)
TerminateProcess(pair.second.first, STATUS_CONTROL_C_EXIT);
}
int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) {
std::vector<HANDLE> handles;
handles.reserve(childs.size());
for (const auto &pair : childs)
if (pair.second.second <= as_running)
handles.push_back(pair.second.first);
DWORD rc =
MsgWaitForMultipleObjectsEx((DWORD)handles.size(), &handles[0],
(timeout > 60) ? 60 * 1000 : timeout * 1000,
QS_ALLINPUT | QS_ALLPOSTMESSAGE, 0);
if (rc >= WAIT_OBJECT_0 && rc < WAIT_OBJECT_0 + handles.size()) {
pid = 0;
for (const auto &pair : childs)
if (pair.second.first == handles[rc - WAIT_OBJECT_0]) {
pid = pair.first;
break;
}
return 0;
}
if (rc == WAIT_TIMEOUT) {
pid = 0;
return 0;
}
return waitstatus2errcode(rc);
}
void osal_yield(void) { SwitchToThread(); }
void osal_udelay(unsigned us) {
chrono::time until, now = chrono::now_motonic();
until.fixedpoint = now.fixedpoint + chrono::from_us(us).fixedpoint;
static unsigned threshold_us;
if (threshold_us == 0) {
#if 1
unsigned timeslice_ms = 1;
while (timeBeginPeriod(timeslice_ms) == TIMERR_NOCANDO)
++timeslice_ms;
threshold_us = timeslice_ms * 1500u;
#else
ULONGLONG InterruptTimePrecise_100ns;
QueryInterruptTimePrecise(&InterruptTimePrecise_100ns);
threshold_us = InterruptTimePrecise_100ns / 5;
#endif
assert(threshold_us > 0);
}
do {
if (us > threshold_us && us > 1000) {
DWORD rc = SleepEx(us / 1000, TRUE);
if (rc)
failure_perror("SleepEx()", waitstatus2errcode(rc));
us = 0;
}
YieldProcessor();
now = chrono::now_motonic();
} while (now.fixedpoint < until.fixedpoint);
}
bool osal_istty(int fd) { return _isatty(fd) != 0; }

45
test/osal.h Normal file
View File

@ -0,0 +1,45 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
void osal_setup(const std::vector<actor_config> &actors);
void osal_broadcast(unsigned id);
int osal_waitfor(unsigned id);
int osal_actor_start(const actor_config &config, mdbx_pid_t &pid);
actor_status osal_actor_info(const mdbx_pid_t pid);
void osal_killall_actors(void);
int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout);
void osal_wait4barrier(void);
mdbx_pid_t osal_getpid(void);
int osal_delay(unsigned seconds);
void osal_udelay(unsigned us);
void osal_yield(void);
bool osal_istty(int fd);
#ifdef _MSC_VER
#ifndef STDIN_FILENO
#define STDIN_FILENO _fileno(stdin)
#endif
#ifndef STDOUT_FILENO
#define STDOUT_FILENO _fileno(stdout)
#endif
#ifndef STDERR_FILENO
#define STDERR_FILENO _fileno(stderr)
#endif
#endif /* _MSC_VER */

465
test/test.cc Normal file
View File

@ -0,0 +1,465 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
const char *testcase2str(const actor_testcase testcase) {
switch (testcase) {
default:
assert(false);
return "?!";
case ac_none:
return "none";
case ac_hill:
return "hill";
case ac_deadread:
return "deadread";
case ac_deadwrite:
return "deadwrite";
case ac_jitter:
return "jitter";
}
}
const char *status2str(actor_status status) {
switch (status) {
default:
assert(false);
return "?!";
case as_debuging:
return "debuging";
case as_running:
return "running";
case as_successful:
return "successful";
case as_killed:
return "killed";
case as_failed:
return "failed";
}
}
const char *keygencase2str(const keygen_case keycase) {
switch (keycase) {
default:
assert(false);
return "?!";
case kc_random:
return "random";
case kc_dashes:
return "dashes";
case kc_custom:
return "custom";
}
}
//-----------------------------------------------------------------------------
static void mdbx_debug_logger(int type, const char *function, int line,
const char *msg, va_list args) {
logging::loglevel level = logging::info;
if (type & MDBX_DBG_EXTRA)
level = logging::extra;
if (type & MDBX_DBG_TRACE)
level = logging::trace;
if (type & MDBX_DBG_PRINT)
level = logging::verbose;
if (!function)
function = "unknown";
if (type & MDBX_DBG_ASSERT) {
log_error("mdbx: assertion failure: %s, %d", function, line);
level = logging::failure;
}
if (logging::output(level, strncmp(function, "mdbx_", 5) == 0 ? "%s: "
: "mdbx: %s: ",
function))
logging::feed(msg, args);
if (type & MDBX_DBG_ASSERT)
abort();
}
int testcase::oom_callback(MDBX_env *env, int pid, mdbx_tid_t tid, uint64_t txn,
unsigned gap, int retry) {
testcase *self = (testcase *)mdbx_env_get_userctx(env);
if (retry == 0)
log_notice("oom_callback: waitfor pid %u, thread %" PRIuPTR
", txn #%" PRIu64 ", gap %d",
pid, (size_t)tid, txn, gap);
if (self->should_continue(true)) {
osal_yield();
if (retry > 0)
osal_udelay(retry * 100);
return 0 /* always retry */;
}
return -1;
}
void testcase::db_prepare() {
log_trace(">> db_prepare");
assert(!db_guard);
int mdbx_dbg_opts = MDBX_DBG_ASSERT | MDBX_DBG_JITTER | MDBX_DBG_DUMP;
if (config.params.loglevel <= logging::trace)
mdbx_dbg_opts |= MDBX_DBG_TRACE;
if (config.params.loglevel <= logging::verbose)
mdbx_dbg_opts |= MDBX_DBG_PRINT;
int rc = mdbx_setup_debug(mdbx_dbg_opts, mdbx_debug_logger);
log_info("set mdbx debug-opts: 0x%02x", rc);
MDBX_env *env = nullptr;
rc = mdbx_env_create(&env);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_create()", rc);
assert(env != nullptr);
db_guard.reset(env);
rc = mdbx_env_set_userctx(env, this);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_userctx()", rc);
rc = mdbx_env_set_maxreaders(env, config.params.max_readers);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_maxreaders()", rc);
rc = mdbx_env_set_maxdbs(env, config.params.max_tables);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_maxdbs()", rc);
rc = mdbx_env_set_oomfunc(env, testcase::oom_callback);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_oomfunc()", rc);
rc = mdbx_env_set_mapsize(env, (size_t)config.params.size);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_mapsize()", rc);
log_trace("<< db_prepare");
}
void testcase::db_open() {
log_trace(">> db_open");
if (!db_guard)
db_prepare();
int rc = mdbx_env_open(db_guard.get(), config.params.pathname_db.c_str(),
(unsigned)config.params.mode_flags, 0640);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_open()", rc);
log_trace("<< db_open");
}
void testcase::db_close() {
log_trace(">> db_close");
cursor_guard.reset();
txn_guard.reset();
db_guard.reset();
log_trace("<< db_close");
}
void testcase::txn_begin(bool readonly) {
log_trace(">> txn_begin(%s)", readonly ? "read-only" : "read-write");
assert(!txn_guard);
MDBX_txn *txn = nullptr;
int rc =
mdbx_txn_begin(db_guard.get(), nullptr, readonly ? MDBX_RDONLY : 0, &txn);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_txn_begin()", rc);
txn_guard.reset(txn);
log_trace("<< txn_begin(%s)", readonly ? "read-only" : "read-write");
}
void testcase::txn_end(bool abort) {
log_trace(">> txn_end(%s)", abort ? "abort" : "commit");
assert(txn_guard);
MDBX_txn *txn = txn_guard.release();
if (abort) {
int rc = mdbx_txn_abort(txn);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_txn_abort()", rc);
} else {
int rc = mdbx_txn_commit(txn);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_txn_commit()", rc);
}
log_trace("<< txn_end(%s)", abort ? "abort" : "commit");
}
void testcase::txn_restart(bool abort, bool readonly) {
if (txn_guard)
txn_end(abort);
txn_begin(readonly);
}
bool testcase::wait4start() {
if (config.wait4id) {
log_trace(">> wait4start(%u)", config.wait4id);
assert(!global::singlemode);
int rc = osal_waitfor(config.wait4id);
if (rc) {
log_trace("<< wait4start(%u), failed %s", config.wait4id,
test_strerror(rc));
return false;
}
} else {
log_trace("== skip wait4start: not needed");
}
if (config.params.delaystart) {
int rc = osal_delay(config.params.delaystart);
if (rc) {
log_trace("<< delay(%u), failed %s", config.params.delaystart,
test_strerror(rc));
return false;
}
} else {
log_trace("== skip delay: not needed");
}
return true;
}
void testcase::kick_progress(bool active) const {
chrono::time now = chrono::now_motonic();
if (active) {
static int last_point = -1;
int point = (now.fixedpoint >> 29) & 3;
if (point != last_point) {
last.progress_timestamp = now;
fprintf(stderr, "%c\b", "-\\|/"[last_point = point]);
fflush(stderr);
}
} else if (now.fixedpoint - last.progress_timestamp.fixedpoint >
chrono::from_seconds(2).fixedpoint) {
last.progress_timestamp = now;
fprintf(stderr, "%c\b", "@*"[now.utc & 1]);
fflush(stderr);
}
}
void testcase::report(size_t nops_done) {
assert(nops_done > 0);
if (!nops_done)
return;
nops_completed += nops_done;
log_verbose("== complete +%" PRIuPTR " iteration, total %" PRIuPTR " done",
nops_done, nops_completed);
if (global::config::progress_indicator)
kick_progress(true);
if (config.signal_nops && !signalled &&
config.signal_nops <= nops_completed) {
log_trace(">> signal(n-ops %" PRIuPTR ")", nops_completed);
if (!global::singlemode)
osal_broadcast(config.actor_id);
signalled = true;
log_trace("<< signal(n-ops %" PRIuPTR ")", nops_completed);
}
}
void testcase::signal() {
if (!signalled) {
log_trace(">> signal(forced)");
if (!global::singlemode)
osal_broadcast(config.actor_id);
signalled = true;
log_trace("<< signal(forced)");
}
}
bool testcase::setup() {
db_prepare();
if (!wait4start())
return false;
start_timestamp = chrono::now_motonic();
return true;
}
bool testcase::teardown() {
log_trace(">> testcase::teardown");
signal();
db_close();
log_trace("<< testcase::teardown");
return true;
}
bool testcase::should_continue(bool check_timeout_only) const {
bool result = true;
if (config.params.test_duration) {
chrono::time since;
since.fixedpoint =
chrono::now_motonic().fixedpoint - start_timestamp.fixedpoint;
if (since.seconds() >= config.params.test_duration)
result = false;
}
if (!check_timeout_only && config.params.test_nops &&
nops_completed >= config.params.test_nops)
result = false;
if (result && global::config::progress_indicator)
kick_progress(false);
return result;
}
void testcase::fetch_canary() {
mdbx_canary canary_now;
log_trace(">> fetch_canary");
int rc = mdbx_canary_get(txn_guard.get(), &canary_now);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_canary_get()", rc);
if (canary_now.v < last.canary.v)
failure("fetch_canary: %" PRIu64 "(canary-now.v) < %" PRIu64
"(canary-last.v)",
canary_now.v, last.canary.v);
if (canary_now.y < last.canary.y)
failure("fetch_canary: %" PRIu64 "(canary-now.y) < %" PRIu64
"(canary-last.y)",
canary_now.y, last.canary.y);
last.canary = canary_now;
log_trace("<< fetch_canary: db-sequence %" PRIu64
", db-sequence.txnid %" PRIu64,
last.canary.y, last.canary.v);
}
void testcase::update_canary(uint64_t increment) {
mdbx_canary canary_now = last.canary;
log_trace(">> update_canary: sequence %" PRIu64 " += %" PRIu64, canary_now.y,
increment);
canary_now.y += increment;
int rc = mdbx_canary_put(txn_guard.get(), &canary_now);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_canary_put()", rc);
log_trace("<< update_canary: sequence = %" PRIu64, canary_now.y);
}
MDBX_dbi testcase::db_table_open(bool create) {
log_trace(">> testcase::db_table_create");
char tablename_buf[16];
const char *tablename = nullptr;
if (config.space_id) {
int rc = snprintf(tablename_buf, sizeof(tablename_buf), "TBL%04u",
config.space_id);
if (rc < 4 || rc >= (int)sizeof(tablename_buf) - 1)
failure("snprintf(tablename): %d", rc);
tablename = tablename_buf;
}
log_verbose("use %s table", tablename ? tablename : "MAINDB");
MDBX_dbi handle = 0;
int rc = mdbx_dbi_open(txn_guard.get(), tablename,
(create ? MDBX_CREATE : 0) | config.params.table_flags,
&handle);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_dbi_open()", rc);
log_trace("<< testcase::db_table_create, handle %u", handle);
return handle;
}
void testcase::db_table_drop(MDBX_dbi handle) {
log_trace(">> testcase::db_table_drop, handle %u", handle);
if (config.params.drop_table) {
int rc = mdbx_drop(txn_guard.get(), handle, true);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_drop()", rc);
log_trace("<< testcase::db_table_drop");
} else {
log_trace("<< testcase::db_table_drop: not needed");
}
}
void testcase::db_table_close(MDBX_dbi handle) {
log_trace(">> testcase::db_table_close, handle %u", handle);
assert(!txn_guard);
int rc = mdbx_dbi_close(db_guard.get(), handle);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_dbi_close()", rc);
log_trace("<< testcase::db_table_close");
}
//-----------------------------------------------------------------------------
bool test_execute(const actor_config &config) {
const mdbx_pid_t pid = osal_getpid();
if (global::singlemode) {
logging::setup(format("single_%s", testcase2str(config.testcase)));
} else {
logging::setup((logging::loglevel)config.params.loglevel,
format("child_%u.%u", config.actor_id, config.space_id));
log_trace(">> wait4barrier");
osal_wait4barrier();
log_trace("<< wait4barrier");
}
try {
std::unique_ptr<testcase> test;
switch (config.testcase) {
case ac_hill:
test.reset(new testcase_hill(config, pid));
break;
case ac_deadread:
test.reset(new testcase_deadread(config, pid));
break;
case ac_deadwrite:
test.reset(new testcase_deadwrite(config, pid));
break;
case ac_jitter:
test.reset(new testcase_jitter(config, pid));
break;
default:
test.reset(new testcase(config, pid));
break;
}
if (!test->setup())
log_notice("test setup failed");
else if (!test->run())
log_notice("test failed");
else if (!test->teardown())
log_notice("test teardown failed");
else {
log_info("test successed");
return true;
}
} catch (const std::exception &pipets) {
failure("***** Exception: %s *****", pipets.what());
}
return false;
}

192
test/test.h Normal file
View File

@ -0,0 +1,192 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "chrono.h"
#include "config.h"
#include "keygen.h"
#include "log.h"
#include "osal.h"
#include "utils.h"
bool test_execute(const actor_config &config);
std::string thunk_param(const actor_config &config);
void testcase_setup(const char *casename, actor_params &params,
unsigned &last_space_id);
void configure_actor(unsigned &last_space_id, const actor_testcase testcase,
const char *space_id_cstr, const actor_params &params);
void keycase_setup(const char *casename, actor_params &params);
namespace global {
extern const char thunk_param_prefix[];
extern std::vector<actor_config> actors;
extern std::unordered_map<unsigned, actor_config *> events;
extern std::unordered_map<mdbx_pid_t, actor_config *> pid2actor;
extern std::set<std::string> databases;
extern unsigned nactors;
extern chrono::time start_motonic;
extern chrono::time deadline_motonic;
extern bool singlemode;
namespace config {
extern unsigned timeout_duration_seconds;
extern bool dump_config;
extern bool cleanup_before;
extern bool cleanup_after;
extern bool failfast;
extern bool progress_indicator;
} /* namespace config */
} /* namespace global */
//-----------------------------------------------------------------------------
struct db_deleter : public std::unary_function<void, MDBX_env *> {
void operator()(MDBX_env *env) const { mdbx_env_close(env); }
};
struct txn_deleter : public std::unary_function<void, MDBX_txn *> {
void operator()(MDBX_txn *txn) const {
int rc = mdbx_txn_abort(txn);
if (rc)
log_trouble(mdbx_func_, "mdbx_txn_abort()", rc);
}
};
struct cursor_deleter : public std::unary_function<void, MDBX_cursor *> {
void operator()(MDBX_cursor *cursor) const { mdbx_cursor_close(cursor); }
};
typedef std::unique_ptr<MDBX_env, db_deleter> scoped_db_guard;
typedef std::unique_ptr<MDBX_txn, txn_deleter> scoped_txn_guard;
typedef std::unique_ptr<MDBX_cursor, cursor_deleter> scoped_cursor_guard;
//-----------------------------------------------------------------------------
class testcase {
protected:
const actor_config &config;
const mdbx_pid_t pid;
scoped_db_guard db_guard;
scoped_txn_guard txn_guard;
scoped_cursor_guard cursor_guard;
bool signalled;
size_t nops_completed;
chrono::time start_timestamp;
keygen::buffer key;
keygen::buffer data;
keygen::maker keyvalue_maker;
struct {
mdbx_canary canary;
mutable chrono::time progress_timestamp;
} last;
static int oom_callback(MDBX_env *env, int pid, mdbx_tid_t tid, uint64_t txn,
unsigned gap, int retry);
void db_prepare();
void db_open();
void db_close();
void txn_begin(bool readonly);
void txn_end(bool abort);
void txn_restart(bool abort, bool readonly);
void fetch_canary();
void update_canary(uint64_t increment);
void kick_progress(bool active) const;
MDBX_dbi db_table_open(bool create);
void db_table_drop(MDBX_dbi handle);
void db_table_close(MDBX_dbi handle);
bool wait4start();
void report(size_t nops_done);
void signal();
bool should_continue(bool check_timeout_only = false) const;
void generate_pair(const keygen::serial_t serial, keygen::buffer &out_key,
keygen::buffer &out_value, keygen::serial_t data_age = 0) {
keyvalue_maker.pair(serial, out_key, out_value, data_age);
}
void generate_pair(const keygen::serial_t serial,
keygen::serial_t data_age = 0) {
generate_pair(serial, key, data, data_age);
}
bool mode_readonly() const {
return (config.params.mode_flags & MDBX_RDONLY) ? true : false;
}
public:
testcase(const actor_config &config, const mdbx_pid_t pid)
: config(config), pid(pid), signalled(false), nops_completed(0) {
start_timestamp.reset();
memset(&last, 0, sizeof(last));
}
virtual bool setup();
virtual bool run() { return true; }
virtual bool teardown();
virtual ~testcase() {}
};
class testcase_hill : public testcase {
typedef testcase inherited;
public:
testcase_hill(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool setup();
bool run();
bool teardown();
};
class testcase_deadread : public testcase {
typedef testcase inherited;
public:
testcase_deadread(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool setup();
bool run();
bool teardown();
};
class testcase_deadwrite : public testcase {
typedef testcase inherited;
public:
testcase_deadwrite(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool setup();
bool run();
bool teardown();
};
class testcase_jitter : public testcase {
typedef testcase inherited;
public:
testcase_jitter(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool setup();
bool run();
bool teardown();
};

201
test/test.vcxproj Normal file
View File

@ -0,0 +1,201 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\dll.vcxproj">
<Project>{6d19209b-ece7-4b9c-941c-0aa2b484f199}</Project>
</ProjectReference>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>mdbxtest</RootNamespace>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>Use</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;MDBX_DEBUG=1;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<PrecompiledHeaderFile>test.h</PrecompiledHeaderFile>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>winmm.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>Use</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;MDBX_DEBUG=1;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<PrecompiledHeaderFile>test.h</PrecompiledHeaderFile>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>winmm.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>Use</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<PrecompiledHeaderFile>test.h</PrecompiledHeaderFile>
<OmitFramePointers>true</OmitFramePointers>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>winmm.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>Use</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<PrecompiledHeaderFile>test.h</PrecompiledHeaderFile>
<OmitFramePointers>true</OmitFramePointers>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>winmm.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="base.h" />
<ClInclude Include="chrono.h" />
<ClInclude Include="config.h" />
<ClInclude Include="keygen.h" />
<ClInclude Include="log.h" />
<ClInclude Include="osal.h" />
<ClInclude Include="test.h" />
<ClInclude Include="utils.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="cases.cc" />
<ClCompile Include="chrono.cc" />
<ClCompile Include="config.cc" />
<ClCompile Include="dead.cc" />
<ClCompile Include="hill.cc" />
<ClCompile Include="jitter.cc" />
<ClCompile Include="keygen.cc" />
<ClCompile Include="log.cc" />
<ClCompile Include="main.cc" />
<ClCompile Include="osal-windows.cc">
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Create</PrecompiledHeader>
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Create</PrecompiledHeader>
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Create</PrecompiledHeader>
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Create</PrecompiledHeader>
</ClCompile>
<ClCompile Include="test.cc" />
<ClCompile Include="utils.cc" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

316
test/utils.cc Normal file
View File

@ -0,0 +1,316 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
#include <float.h>
#ifdef HAVE_IEEE754_H
#include <ieee754.h>
#endif
std::string format(const char *fmt, ...) {
va_list ap, ones;
va_start(ap, fmt);
va_copy(ones, ap);
#ifdef _MSC_VER
int needed = _vscprintf(fmt, ap);
#else
int needed = vsnprintf(nullptr, 0, fmt, ap);
#endif
assert(needed >= 0);
va_end(ap);
std::string result;
result.reserve((size_t)needed + 1);
result.resize((size_t)needed, '\0');
int actual = vsnprintf((char *)result.data(), result.capacity(), fmt, ones);
assert(actual == needed);
(void)actual;
va_end(ones);
return result;
}
std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum) {
std::string result;
if (bytes > 0) {
const uint8_t *data = (const uint8_t *)ptr;
checksum.push(data, bytes);
result.reserve(bytes * 2);
const uint8_t *const end = data + bytes;
do {
char h = *data >> 4;
char l = *data & 15;
result.push_back((l < 10) ? l + '0' : l - 10 + 'a');
result.push_back((h < 10) ? h + '0' : h - 10 + 'a');
} while (++data < end);
}
assert(result.size() == bytes * 2);
return result;
}
bool hex2data(const char *hex_begin, const char *hex_end, void *ptr,
size_t bytes, simple_checksum &checksum) {
if (bytes * 2 != (size_t)(hex_end - hex_begin))
return false;
uint8_t *data = (uint8_t *)ptr;
for (const char *hex = hex_begin; hex != hex_end; hex += 2, ++data) {
unsigned l = hex[0], h = hex[1];
if (l >= '0' && l <= '9')
l = l - '0';
else if (l >= 'A' && l <= 'F')
l = l - 'A' + 10;
else if (l >= 'a' && l <= 'f')
l = l - 'a' + 10;
else
return false;
if (h >= '0' && h <= '9')
h = h - '0';
else if (h >= 'A' && h <= 'F')
h = h - 'A' + 10;
else if (h >= 'a' && h <= 'f')
h = h - 'a' + 10;
else
return false;
uint32_t c = l + (h << 4);
checksum.push(c);
*data = (uint8_t)c;
}
return true;
}
//-----------------------------------------------------------------------------
#ifdef __mips__
static uint64_t *mips_tsc_addr;
__cold static void mips_rdtsc_init() {
int mem_fd = open("/dev/mem", O_RDONLY | O_SYNC, 0);
HIPPEUS_ENSURE(mem_fd >= 0);
mips_tsc_addr = mmap(nullptr, pagesize, PROT_READ, MAP_SHARED, mem_fd,
0x10030000 /* MIPS_ZBUS_TIMER */);
close(mem_fd);
}
#endif /* __mips__ */
uint64_t entropy_ticks(void) {
#if defined(__GNUC__) || defined(__clang__)
#if defined(__ia64__)
uint64_t ticks;
__asm __volatile("mov %0=ar.itc" : "=r"(ticks));
return ticks;
#elif defined(__hppa__)
uint64_t ticks;
__asm __volatile("mfctl 16, %0" : "=r"(ticks));
return ticks;
#elif defined(__s390__)
uint64_t ticks;
__asm __volatile("stck 0(%0)" : : "a"(&(ticks)) : "memory", "cc");
return ticks;
#elif defined(__alpha__)
uint64_t ticks;
__asm __volatile("rpcc %0" : "=r"(ticks));
return ticks;
#elif defined(__sparc_v9__)
uint64_t ticks;
__asm __volatile("rd %%tick, %0" : "=r"(ticks));
return ticks;
#elif defined(__powerpc64__) || defined(__ppc64__)
uint64_t ticks;
__asm __volatile("mfspr %0, 268" : "=r"(ticks));
return ticks;
#elif defined(__ppc__) || defined(__powerpc__)
unsigned tbl, tbu;
/* LY: Here not a problem if a high-part (tbu)
* would been updated during reading. */
__asm __volatile("mftb %0" : "=r"(tbl));
__asm __volatile("mftbu %0" : "=r"(tbu));
return (((uin64_t)tbu0) << 32) | tbl;
#elif defined(__mips__)
if (mips_tsc_addr != MAP_FAILED) {
if (unlikely(!mips_tsc_addr)) {
static pthread_once_t is_initialized = PTHREAD_ONCE_INIT;
int rc = pthread_once(&is_initialized, mips_rdtsc_init);
if (unlikely(rc))
failure_perror("pthread_once()", rc);
}
if (mips_tsc_addr != MAP_FAILED)
return *mips_tsc_addr;
}
#elif defined(__x86_64__) || defined(__i386__)
#if __GNUC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_rdtsc)
return __builtin_ia32_rdtsc();
#else
unsigned lo, hi;
/* LY: Using the "a" and "d" constraints is important for correct code. */
__asm __volatile("rdtsc" : "=a"(lo), "=d"(hi));
return (((uint64_t)hi) << 32) + lo;
#endif
#endif /* arch selector */
#elif defined(_M_IX86) || defined(_M_X64)
return __rdtsc();
#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
LARGE_INTEGER PerformanceCount;
if (QueryPerformanceCounter(&PerformanceCount))
return PerformanceCount.QuadPart;
return GetTickCount64();
#else
struct timespec ts;
#if defined(CLOCK_MONOTONIC_COARSE)
clockid_t clock = CLOCK_MONOTONIC_COARSE;
#elif defined(CLOCK_MONOTONIC_RAW)
clockid_t clock = CLOCK_MONOTONIC_RAW;
#else
clockid_t clock = CLOCK_MONOTONIC;
#endif
int rc = clock_gettime(clock, &ts);
if (unlikely(rc))
failure_perror("clock_gettime()", rc);
return (((uint64_t)ts.tv_sec) << 32) + ts.tv_nsec;
#endif
}
//-----------------------------------------------------------------------------
static __inline uint64_t bleach64(uint64_t dirty) {
return mul_64x64_high(bswap64(dirty), UINT64_C(17048867929148541611));
}
static __inline uint32_t bleach32(uint32_t dirty) {
return (uint32_t)((bswap32(dirty) * UINT64_C(2175734609)) >> 32);
}
uint64_t prng64_careless(uint64_t &state) {
state = state * UINT64_C(6364136223846793005) + 1;
return state;
}
uint64_t prng64_white(uint64_t &state) {
state = state * UINT64_C(6364136223846793005) + UINT64_C(1442695040888963407);
return bleach64(state);
}
uint32_t prng32(uint64_t &state) {
return (uint32_t)(prng64_careless(state) >> 32);
}
void prng_fill(uint64_t &state, void *ptr, size_t bytes) {
while (bytes >= 4) {
*((uint32_t *)ptr) = prng32(state);
ptr = (uint32_t *)ptr + 1;
bytes -= 4;
}
switch (bytes & 3) {
case 3: {
uint32_t u32 = prng32(state);
memcpy(ptr, &u32, 3);
} break;
case 2:
*((uint16_t *)ptr) = (uint16_t)prng32(state);
break;
case 1:
*((uint8_t *)ptr) = (uint8_t)prng32(state);
break;
case 0:
break;
}
}
static __thread uint64_t prng_state;
void prng_seed(uint64_t seed) { prng_state = bleach64(seed); }
uint32_t prng32(void) { return prng32(prng_state); }
uint64_t prng64(void) { return prng64_white(prng_state); }
void prng_fill(void *ptr, size_t bytes) { prng_fill(prng_state, ptr, bytes); }
uint64_t entropy_white() { return bleach64(entropy_ticks()); }
double double_from_lower(uint64_t salt) {
#ifdef IEEE754_DOUBLE_BIAS
ieee754_double r;
r.ieee.negative = 0;
r.ieee.exponent = IEEE754_DOUBLE_BIAS;
r.ieee.mantissa0 = (unsigned)(salt >> 32);
r.ieee.mantissa1 = (unsigned)salt;
return r.d;
#else
const uint64_t top = (UINT64_C(1) << DBL_MANT_DIG) - 1;
const double scale = 1.0 / (double)top;
return (salt & top) * scale;
#endif
}
double double_from_upper(uint64_t salt) {
#ifdef IEEE754_DOUBLE_BIAS
ieee754_double r;
r.ieee.negative = 0;
r.ieee.exponent = IEEE754_DOUBLE_BIAS;
salt >>= 64 - DBL_MANT_DIG;
r.ieee.mantissa0 = (unsigned)(salt >> 32);
r.ieee.mantissa1 = (unsigned)salt;
return r.d;
#else
const uint64_t top = (UINT64_C(1) << DBL_MANT_DIG) - 1;
const double scale = 1.0 / (double)top;
return (salt >> (64 - DBL_MANT_DIG)) * scale;
#endif
}
bool flipcoin() { return bleach32((uint32_t)entropy_ticks()) & 1; }
bool jitter(unsigned probability_percent) {
const uint32_t top = UINT32_MAX - UINT32_MAX % 100;
uint32_t dice, edge = (top) / 100 * probability_percent;
do
dice = bleach32((uint32_t)entropy_ticks());
while (dice >= top);
return dice < edge;
}
void jitter_delay(bool extra) {
unsigned dice = entropy_white() & 3;
if (dice == 0) {
log_trace("== jitter.no-delay");
} else {
log_trace(">> jitter.delay: dice %u", dice);
do {
cpu_relax();
memory_barrier();
cpu_relax();
if (dice > 1) {
osal_yield();
cpu_relax();
if (dice > 2) {
unsigned us = entropy_white() &
(extra ? 0xfffff /* 1.05 s */ : 0x3ff /* 1 ms */);
log_trace("== jitter.delay: %0.6f", us / 1000000.0);
osal_udelay(us);
}
}
} while (flipcoin());
log_trace("<< jitter.delay: dice %u", dice);
}
}

373
test/utils.h Normal file
View File

@ -0,0 +1,373 @@
/*
* Copyright 2017 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
!defined(__ORDER_BIG_ENDIAN__)
#ifndef _MSC_VER
#include <sys/param.h> /* for endianness */
#endif
#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
#define __BYTE_ORDER__ __BYTE_ORDER
#else
#define __ORDER_LITTLE_ENDIAN__ 1234
#define __ORDER_BIG_ENDIAN__ 4321
#if defined(__LITTLE_ENDIAN__) || defined(_LITTLE_ENDIAN) || \
defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
defined(__i386) || defined(__x86_64__) || defined(_M_IX86) || \
defined(_M_X64) || defined(i386) || defined(_X86_) || defined(__i386__) || \
defined(_X86_64_) || defined(_M_ARM) || defined(_M_ARM64) || \
defined(__e2k__)
#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
#elif defined(__BIG_ENDIAN__) || defined(_BIG_ENDIAN) || defined(__ARMEB__) || \
defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || \
defined(_MIPSEB) || defined(__MIPSEB) || defined(_M_IA64)
#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
#else
#error __BYTE_ORDER__ should be defined.
#endif
#endif
#endif
#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \
__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__
#error Unsupported byte order.
#endif
#if __GNUC_PREREQ(4, 4) || defined(__clang__)
#if __GNUC_PREREQ(4, 5) || defined(__clang__)
#define unreachable() __builtin_unreachable()
#endif
#define bswap64(v) __builtin_bswap64(v)
#define bswap32(v) __builtin_bswap32(v)
#if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16)
#define bswap16(v) __builtin_bswap16(v)
#endif
#elif defined(_MSC_VER)
#if _MSC_FULL_VER < 190024215
#pragma message( \
"It is recommended to use Visual Studio 2015 (MSC 19.0) or newer.")
#endif
#define unreachable() __assume(0)
#define bswap64(v) _byteswap_uint64(v)
#define bswap32(v) _byteswap_ulong(v)
#define bswap16(v) _byteswap_ushort(v)
#define rot64(v, s) _rotr64(v, s)
#define rot32(v, s) _rotr(v, s)
#if defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
#pragma intrinsic(_umul128)
#define mul_64x64_128(a, b, ph) _umul128(a, b, ph)
#pragma intrinsic(__umulh)
#define mul_64x64_high(a, b) __umulh(a, b)
#endif
#if defined(_M_IX86)
#pragma intrinsic(__emulu)
#define mul_32x32_64(a, b) __emulu(a, b)
#elif defined(_M_ARM)
#define mul_32x32_64(a, b) _arm_umull(a, b)
#endif
#endif /* compiler */
#ifndef unreachable
#define unreachable() \
do { \
} while (1)
#endif
#ifndef bswap64
#ifdef __bswap_64
#define bswap64(v) __bswap_64(v)
#else
static __inline uint64_t bswap64(uint64_t v) {
return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) |
((v << 24) & UINT64_C(0x0000ff0000000000)) |
((v << 8) & UINT64_C(0x000000ff00000000)) |
((v >> 8) & UINT64_C(0x00000000ff0000000)) |
((v >> 24) & UINT64_C(0x0000000000ff0000)) |
((v >> 40) & UINT64_C(0x000000000000ff00));
}
#endif
#endif /* bswap64 */
#ifndef bswap32
#ifdef __bswap_32
#define bswap32(v) __bswap_32(v)
#else
static __inline uint32_t bswap32(uint32_t v) {
return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) |
((v >> 8) & UINT32_C(0x0000ff00));
}
#endif
#endif /* bswap32 */
#ifndef bswap16
#ifdef __bswap_16
#define bswap16(v) __bswap_16(v)
#else
static __inline uint16_t bswap16(uint16_t v) { return v << 8 | v >> 8; }
#endif
#endif /* bswap16 */
#define is_byteorder_le() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#define is_byteorder_be() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
#ifndef htole16
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define htobe16(v) bswap16(v)
#define htole16(v) (v)
#define be16toh(v) bswap16(v)
#define le16toh(v) (v)
#else
#define htobe16(v) (v)
#define htole16(v) bswap16(v)
#define be16toh(v) (v)
#define le16toh(v) bswap16(v)
#endif
#endif /* htole16 */
#ifndef htole32
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define htobe32(v) bswap32(v)
#define htole32(v) (v)
#define be32toh(v) bswap32(v)
#define le32toh(v) (v)
#else
#define htobe32(v) (v)
#define htole32(v) bswap32(v)
#define be32toh(v) (v)
#define le32toh(v) bswap32(v)
#endif
#endif /* htole32 */
#ifndef htole64
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define htobe64(v) bswap64(v)
#define htole64(v) (v)
#define be64toh(v) bswap64(v)
#define le64toh(v) (v)
#else
#define htobe64(v) (v)
#define htole64(v) bswap_64(v)
#define be64toh(v) (v)
#define le64toh(v) bswap_64(v)
#endif
#endif /* htole64 */
namespace unaligned {
template <typename T> static __inline T load(const void *ptr) {
#if defined(_MSC_VER) && \
(defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64))
return *(const T __unaligned *)ptr;
#elif UNALIGNED_OK
return *(const T *)ptr;
#else
T local;
#if defined(__GNUC__) || defined(__clang__)
__builtin_memcpy(&local, (const T *)ptr, sizeof(T));
#else
memcpy(&local, (const T *)ptr, sizeof(T));
#endif /* __GNUC__ || __clang__ */
return local;
#endif /* UNALIGNED_OK */
}
template <typename T> static __inline void store(void *ptr, const T &value) {
#if defined(_MSC_VER) && \
(defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64))
*((T __unaligned *)ptr) = value;
#elif UNALIGNED_OK
*(volatile T *)ptr = value;
#else
#if defined(__GNUC__) || defined(__clang__)
__builtin_memcpy(ptr, &value, sizeof(T));
#else
memcpy(ptr, &value, sizeof(T));
#endif /* __GNUC__ || __clang__ */
#endif /* UNALIGNED_OK */
}
} /* namespace unaligned */
//-----------------------------------------------------------------------------
#ifndef rot64
static __inline uint64_t rot64(uint64_t v, unsigned s) {
return (v >> s) | (v << (64 - s));
}
#endif /* rot64 */
#ifndef mul_32x32_64
static __inline uint64_t mul_32x32_64(uint32_t a, uint32_t b) {
return a * (uint64_t)b;
}
#endif /* mul_32x32_64 */
#ifndef mul_64x64_128
static __inline unsigned add_with_carry(uint64_t *sum, uint64_t addend) {
*sum += addend;
return *sum < addend;
}
static __inline uint64_t mul_64x64_128(uint64_t a, uint64_t b, uint64_t *h) {
#if defined(__SIZEOF_INT128__) || \
(defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
__uint128_t r = (__uint128_t)a * (__uint128_t)b;
/* modern GCC could nicely optimize this */
*h = r >> 64;
return r;
#elif defined(mul_64x64_high)
*h = mul_64x64_high(a, b);
return a * b;
#else
/* performs 64x64 to 128 bit multiplication */
uint64_t ll = mul_32x32_64((uint32_t)a, (uint32_t)b);
uint64_t lh = mul_32x32_64(a >> 32, (uint32_t)b);
uint64_t hl = mul_32x32_64((uint32_t)a, b >> 32);
*h = mul_32x32_64(a >> 32, b >> 32) + (lh >> 32) + (hl >> 32) +
add_with_carry(&ll, lh << 32) + add_with_carry(&ll, hl << 32);
return ll;
#endif
}
#endif /* mul_64x64_128() */
#ifndef mul_64x64_high
static __inline uint64_t mul_64x64_high(uint64_t a, uint64_t b) {
uint64_t h;
mul_64x64_128(a, b, &h);
return h;
}
#endif /* mul_64x64_high */
static __inline bool is_power2(size_t x) { return (x & (x - 1)) == 0; }
static __inline size_t roundup2(size_t value, size_t granularity) {
assert(is_power2(granularity));
return (value + granularity - 1) & ~(granularity - 1);
}
//-----------------------------------------------------------------------------
static __inline void memory_barrier(void) {
#if __has_extension(c_atomic) || __has_extension(cxx_atomic)
__c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif defined(__ATOMIC_SEQ_CST)
__atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif defined(__clang__) || defined(__GNUC__)
__sync_synchronize();
#elif defined(_MSC_VER)
MemoryBarrier();
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
__mf();
#elif defined(__i386__) || defined(__x86_64__)
_mm_mfence();
#else
#error "Unknown target for Intel Compiler, please report to us."
#endif
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
__machine_rw_barrier();
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
_Asm_mf();
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
defined(__ppc64__) || defined(__powerpc64__)
__lwsync();
#else
#error "Could not guess the kind of compiler, please report to us."
#endif
}
static __inline void cpu_relax() {
#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || \
defined(_M_X64)
_mm_pause();
#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) || \
defined(YieldProcessor)
YieldProcessor();
#else
/* nope */
#endif
}
//-----------------------------------------------------------------------------
struct simple_checksum {
uint64_t value;
simple_checksum() : value(0) {}
void push(uint32_t data) {
value += data * UINT64_C(9386433910765580089) + 1;
value ^= value >> 41;
}
void push(uint64_t data) {
push((uint32_t)data);
push((uint32_t)(data >> 32));
}
void push(bool data) { push(data ? UINT32_C(0x780E) : UINT32_C(0xFA18E)); }
void push(const void *ptr, size_t bytes) {
const uint8_t *data = (const uint8_t *)ptr;
for (size_t i = 0; i < bytes; ++i)
push((uint32_t)data[i]);
}
void push(const double &data) { push(&data, sizeof(double)); }
void push(const char *cstr) { push(cstr, strlen(cstr)); }
void push(const std::string &str) { push(str.data(), str.size()); }
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
void push(const HANDLE &handle) { push(&handle, sizeof(handle)); }
#endif /* _WINDOWS */
};
std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum);
bool hex2data(const char *hex_begin, const char *hex_end, void *ptr,
size_t bytes, simple_checksum &checksum);
std::string format(const char *fmt, ...);
uint64_t entropy_ticks(void);
uint64_t entropy_white(void);
uint64_t prng64_careless(uint64_t &state);
uint64_t prng64_white(uint64_t &state);
uint32_t prng32(uint64_t &state);
void prng_fill(uint64_t &state, void *ptr, size_t bytes);
void prng_seed(uint64_t seed);
uint32_t prng32(void);
uint64_t prng64(void);
void prng_fill(void *ptr, size_t bytes);
bool flipcoin();
bool jitter(unsigned probability_percent);
void jitter_delay(bool extra = false);

1
tutorial/README.md Normal file
View File

@ -0,0 +1 @@
This directory is just a placeholder for now. Tutorial and examples will be added later.

2
sample-bdb.txt → tutorial/sample-bdb.txt
View File

@ -5,7 +5,7 @@
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2012-2017 Howard Chu, Symas Corp.
* Copyright 2012-2015 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*

66
tutorial/sample-mdb.txt Normal file
View File

@ -0,0 +1,66 @@
/* sample-mdb.txt - MDB toy/sample
*
* Do a line-by-line comparison of this and sample-bdb.txt
*/
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2012-2015 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include "mdbx.h"
int main(int argc,char * argv[])
{
int rc;
MDBX_env *env;
MDBX_dbi dbi;
MDBX_val key, data;
MDBX_txn *txn;
MDBX_cursor *cursor;
char sval[32];
/* Note: Most error checking omitted for simplicity */
rc = mdbx_env_create(&env);
rc = mdbx_env_open(env, "./testdb", 0, 0664);
rc = mdbx_txn_begin(env, NULL, 0, &txn);
rc = mdbx_dbi_open(txn, NULL, 0, &dbi);
key.iov_len = sizeof(int);
key.iov_base = sval;
data.iov_len = sizeof(sval);
data.iov_base = sval;
sprintf(sval, "%03x %d foo bar", 32, 3141592);
rc = mdbx_put(txn, dbi, &key, &data, 0);
rc = mdbx_txn_commit(txn);
if (rc) {
fprintf(stderr, "mdbx_txn_commit: (%d) %s\n", rc, mdbx_strerror(rc));
goto leave;
}
rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn);
rc = mdbx_cursor_open(txn, dbi, &cursor);
while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.iov_base, (int) key.iov_len, (char *) key.iov_base,
data.iov_base, (int) data.iov_len, (char *) data.iov_base);
}
mdbx_cursor_close(cursor);
mdbx_txn_abort(txn);
leave:
mdbx_dbi_close(env, dbi);
mdbx_env_close(env);
return 0;
}

259
wbench.c
View File

@ -1,259 +0,0 @@
/*
* Copyright 2015-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <errno.h>
#include "mdbx.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
#ifndef DBPATH
# define DBPATH "./testdb"
#endif
struct t0 {
struct rusage ru;
struct timespec ts;
};
void t0(struct t0 *t0)
{
int rc;
E(getrusage(RUSAGE_SELF, &t0->ru));
E(clock_gettime(CLOCK_MONOTONIC_RAW, &t0->ts));
}
struct info {
double wall_s, cpu_sys_s, cpu_user_s;
long iops_r, iops_w, iops_pf;
};
double delta_s(const struct timeval *begin, const struct timeval *end)
{
return end->tv_sec - begin->tv_sec
+ (end->tv_usec - begin->tv_usec) / 1000000.0;
}
double delta2_s(const struct timespec *begin, const struct timespec *end)
{
return end->tv_sec - begin->tv_sec
+ (end->tv_nsec - begin->tv_nsec) / 1000000000.0;
}
void measure(const struct t0 *t0, struct info *i)
{
struct t0 t1;
int rc;
E(clock_gettime(CLOCK_MONOTONIC_RAW, &t1.ts));
E(getrusage(RUSAGE_SELF, &t1.ru));
i->wall_s = delta2_s(&t0->ts, &t1.ts);
i->cpu_user_s = delta_s(&t0->ru.ru_utime, &t1.ru.ru_utime);
i->cpu_sys_s = delta_s(&t0->ru.ru_stime, &t1.ru.ru_stime);
i->iops_r = t1.ru.ru_inblock - t0->ru.ru_inblock;
i->iops_w = t1.ru.ru_oublock - t0->ru.ru_oublock;
i->iops_pf = t1.ru.ru_majflt - t0->ru.ru_majflt
+ t1.ru.ru_minflt - t0->ru.ru_minflt;
}
void print(struct info *i)
{
printf("wall-clock %.3f, iops: %lu reads, %lu writes, %lu page-faults, "
"cpu: %.3f user, %.3f sys\n",
i->wall_s, i->iops_r, i->iops_w, i->iops_pf, i->cpu_user_s, i->cpu_sys_s);
}
static void wbench(int flags, int mb, int count, int salt)
{
MDB_env *env;
MDB_dbi dbi;
MDB_txn *txn;
MDB_val key, data;
unsigned key_value = salt;
char data_value[777];
int i, rc;
struct t0 start;
struct info ra, rd, rs, rt;
mkdir(DBPATH, 0755);
unlink(DBPATH "/data.mdb");
unlink(DBPATH "/lock.mdb");
printf("\nProbing %d Mb, %d items, flags:", mb, count);
if (flags & MDB_NOSYNC)
printf(" NOSYNC");
if (flags & MDB_NOMETASYNC)
printf(" NOMETASYNC");
if (flags & MDB_WRITEMAP)
printf(" WRITEMAP");
if (flags & MDB_MAPASYNC)
printf(" MAPASYNC");
#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM)
if (flags & MDBX_COALESCE)
printf(" COALESCE");
if (flags & MDBX_LIFORECLAIM)
printf(" LIFO");
#endif
printf(" 0x%X\n", flags);
E(mdb_env_create(&env));
E(mdb_env_set_mapsize(env, (1ull << 20) * mb));
E(mdb_env_open(env, DBPATH, flags, 0664));
key.mv_size = sizeof(key_value);
key.mv_data = &key_value;
data.mv_size = sizeof(data_value);
data.mv_data = &data_value;
printf("\tAdding %d values...", count);
fflush(stdout);
key_value = salt;
t0(&start);
for(i = 0; i < count; ++i) {
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, NULL, 0, &dbi));
snprintf(data_value, sizeof(data_value), "value=%u", key_value);
E(mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE));
E(mdb_txn_commit(txn));
key_value = key_value * 1664525 + 1013904223;
}
measure(&start, &ra);
print(&ra);
printf("\tDeleting %d values...", count);
fflush(stdout);
key_value = salt;
t0(&start);
for(i = 0; i < count; ++i) {
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, NULL, 0, &dbi));
E(mdb_del(txn, dbi, &key, NULL));
E(mdb_txn_commit(txn));
key_value = key_value * 1664525 + 1013904223;
}
measure(&start, &rd);
print(&rd);
printf("\tCheckpoint...");
fflush(stdout);
t0(&start);
mdb_env_sync(env, 1);
measure(&start, &rs);
print(&rs);
mdb_env_close(env);
rt.wall_s = ra.wall_s + rd.wall_s + rs.wall_s;
rt.cpu_sys_s = ra.cpu_sys_s + rd.cpu_sys_s + rs.cpu_sys_s;
rt.cpu_user_s = ra.cpu_user_s + rd.cpu_user_s + rs.cpu_user_s;
rt.iops_r = ra.iops_r + rd.iops_r + rs.iops_r;
rt.iops_w = ra.iops_w + rd.iops_w + rs.iops_w;
rt.iops_pf = ra.iops_pf + rd.iops_pf + rs.iops_pf;
printf("Total ");
print(&rt);
fprintf(stderr, "flags: ");
if (flags & MDB_NOSYNC)
fprintf(stderr, " NOSYNC");
if (flags & MDB_NOMETASYNC)
fprintf(stderr, " NOMETASYNC");
if (flags & MDB_WRITEMAP)
fprintf(stderr, " WRITEMAP");
if (flags & MDB_MAPASYNC)
fprintf(stderr, " MAPASYNC");
#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM)
if (flags & MDBX_COALESCE)
fprintf(stderr, " COALESCE");
if (flags & MDBX_LIFORECLAIM)
fprintf(stderr, " LIFO");
#endif
fprintf(stderr, "\t%.3f\t%.3f\t%.3f\t%.3f\n", rt.iops_w / 1000.0, rt.cpu_user_s, rt.cpu_sys_s, rt.wall_s);
}
int main(int argc,char * argv[])
{
(void) argc;
(void) argv;
#define SALT 1
#define COUNT 10000
#define SIZE 12
printf("\nDefault 'sync' mode...");
wbench(0, SIZE, COUNT, SALT);
#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM)
// wbench(MDBX_COALESCE, SIZE, COUNT, SALT);
wbench(MDBX_COALESCE | MDBX_LIFORECLAIM, SIZE, COUNT, SALT);
// wbench(MDBX_LIFORECLAIM, SIZE, COUNT, SALT);
#endif
printf("\nno-meta-sync hack...");
wbench(MDB_NOMETASYNC, SIZE, COUNT, SALT);
#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM)
// wbench(MDB_NOMETASYNC | MDBX_COALESCE, SIZE, COUNT, SALT);
wbench(MDB_NOMETASYNC | MDBX_COALESCE | MDBX_LIFORECLAIM, SIZE, COUNT, SALT);
// wbench(MDB_NOMETASYNC | MDBX_LIFORECLAIM, SIZE, COUNT, SALT);
#endif
printf("\nno-sync...");
wbench(MDB_NOSYNC, SIZE, COUNT, SALT);
#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM)
// wbench(MDB_NOSYNC | MDBX_COALESCE, SIZE, COUNT, SALT);
// wbench(MDB_NOSYNC | MDBX_COALESCE | MDBX_LIFORECLAIM, SIZE, COUNT, SALT);
// wbench(MDB_NOSYNC | MDBX_LIFORECLAIM, SIZE, COUNT, SALT);
#endif
printf("\nr/w-map...");
wbench(MDB_WRITEMAP, SIZE, COUNT, SALT);
#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM)
// wbench(MDB_WRITEMAP | MDBX_COALESCE, SIZE, COUNT, SALT);
wbench(MDB_WRITEMAP | MDBX_COALESCE | MDBX_LIFORECLAIM, SIZE, COUNT, SALT);
// wbench(MDB_WRITEMAP | MDBX_LIFORECLAIM, SIZE, COUNT, SALT);
#endif
printf("\nasync...");
wbench(MDB_WRITEMAP | MDB_MAPASYNC, SIZE, COUNT, SALT);
#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM)
// wbench(MDB_WRITEMAP | MDB_MAPASYNC | MDBX_COALESCE, SIZE, COUNT, SALT);
wbench(MDB_WRITEMAP | MDB_MAPASYNC | MDBX_COALESCE | MDBX_LIFORECLAIM, SIZE, COUNT, SALT);
// wbench(MDB_WRITEMAP | MDB_MAPASYNC | MDBX_LIFORECLAIM, SIZE, COUNT, SALT);
#endif
printf("\nr/w-map + no-sync...");
wbench(MDB_NOSYNC | MDB_WRITEMAP, SIZE, COUNT, SALT);
#if defined(MDBX_COALESCE) && defined(MDBX_LIFORECLAIM)
// wbench(MDB_NOSYNC | MDB_WRITEMAP | MDBX_COALESCE, SIZE, COUNT, SALT);
wbench(MDB_NOSYNC | MDB_WRITEMAP | MDBX_COALESCE | MDBX_LIFORECLAIM, SIZE, COUNT, SALT);
// wbench(MDB_NOSYNC | MDB_WRITEMAP | MDBX_LIFORECLAIM, SIZE, COUNT, SALT);
#endif
return 0;
}

260
yota_test1.c
View File

@ -1,260 +0,0 @@
/*
* Copyright 2016-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2015 Vladimir Romanov <https://www.linkedin.com/in/vladimirromanov>, Yota Lab.
*
* This file is part of libmdbx.
*
* ReOpenMDBX is free software; you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* ReOpenMDBX is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <sys/time.h>
#include <sys/stat.h>
#include <limits.h>
#include <stddef.h>
#include <string.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <assert.h>
#include "mdbx.h"
#define IP_PRINTF_ARG_HOST(addr) (int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), (int)((addr) & 0xff)
char opt_db_path[PATH_MAX] = "/dev/shm/lmdb_bench1";
static MDB_env *env;
#define REC_COUNT 1000000
int64_t ids[REC_COUNT + REC_COUNT / 10];
int32_t ids_count = 0;
int64_t lmdb_add = 0;
int64_t lmdb_del = 0;
int64_t obj_id = 0;
static void add_id_to_pool(int64_t id) {
ids[ids_count] = id;
ids_count++;
}
static inline int64_t getTimeMicroseconds(void) {
struct timeval val;
gettimeofday(&val, NULL);
return val.tv_sec * ((int64_t) 1000000) + val.tv_usec;
}
static int64_t get_id_from_pool() {
if (ids_count == 0) {
return -1;
}
int32_t index = rand() % ids_count;
int64_t id = ids[index];
ids[index] = ids[ids_count - 1];
ids_count--;
return id;
}
#define LMDB_CHECK(x) \
do {\
const int rc = (x);\
if ( rc != MDB_SUCCESS ) {\
printf("Error [%d] %s in %s at %s:%d\n", rc, mdb_strerror(rc), #x, __FILE__, __LINE__); \
exit(EXIT_FAILURE); \
}\
} while(0)
static void db_connect() {
LMDB_CHECK(mdb_env_create(&env));
LMDB_CHECK(mdb_env_set_mapsize(env, 3L * 1024L * 1024L * 1024L));
LMDB_CHECK(mdb_env_set_maxdbs(env, 30));
#if defined(MDBX_LIFORECLAIM)
LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP | MDBX_LIFORECLAIM, 0664));
#else
LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP, 0664));
#endif
printf("Connection open\n");
}
typedef struct {
char session_id1[100];
char session_id2[100];
char ip[20];
uint8_t fill[100];
} session_data_t;
typedef struct {
int64_t obj_id;
int8_t event_type;
} __attribute__((__packed__)) event_data_t;
static void create_record(int64_t record_id) {
MDB_dbi dbi_session;
MDB_dbi dbi_session_id;
MDB_dbi dbi_event;
MDB_dbi dbi_ip;
event_data_t event;
MDB_txn *txn;
session_data_t data;
// transaction init
snprintf(data.session_id1, sizeof (data.session_id1), "mskugw%02ld_%02ld.gx.yota.ru;3800464060;4152;%ld", record_id % 3 + 1, record_id % 9 + 1, record_id);
snprintf(data.session_id2, sizeof (data.session_id2), "gx_service;%ld;%ld;node@spb-jsm1", record_id, record_id % 1000000000 + 99999);
snprintf(data.ip, sizeof (data.ip), "%d.%d.%d.%d", IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF));
event.obj_id = record_id;
event.event_type = 1;
MDB_val _session_id1_rec = {data.session_id1, strlen(data.session_id1)};
MDB_val _session_id2_rec = {data.session_id2, strlen(data.session_id2)};
MDB_val _ip_rec = {data.ip, strlen(data.ip)};
MDB_val _obj_id_rec = {&record_id, sizeof (record_id)};
MDB_val _data_rec = {&data, offsetof(session_data_t, fill) + (rand() % sizeof (data.fill))};
MDB_val _event_rec = {&event, sizeof (event)};
LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn));
LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session));
LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id));
LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event));
LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip));
LMDB_CHECK(mdb_put(txn, dbi_session, &_obj_id_rec, &_data_rec, MDB_NOOVERWRITE | MDB_NODUPDATA));
LMDB_CHECK(mdb_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec, MDB_NOOVERWRITE | MDB_NODUPDATA));
LMDB_CHECK(mdb_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec, MDB_NOOVERWRITE | MDB_NODUPDATA));
LMDB_CHECK(mdb_put(txn, dbi_ip, &_ip_rec, &_obj_id_rec, 0));
LMDB_CHECK(mdb_put(txn, dbi_event, &_event_rec, &_obj_id_rec, 0));
// transaction commit
LMDB_CHECK(mdb_txn_commit(txn));
lmdb_add++;
}
static void delete_record(int64_t record_id) {
MDB_dbi dbi_session;
MDB_dbi dbi_session_id;
MDB_dbi dbi_event;
MDB_dbi dbi_ip;
event_data_t event;
MDB_txn *txn;
// transaction init
LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn));
// open database in read-write mode
LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session));
LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id));
LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event));
LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip));
// put data
MDB_val _obj_id_rec = {&record_id, sizeof(record_id)};
MDB_val v_rec;
// get data
LMDB_CHECK(mdb_get(txn, dbi_session, &_obj_id_rec, &v_rec));
session_data_t* data = (session_data_t*) v_rec.mv_data;
MDB_val _session_id1_rec = {data->session_id1, strlen(data->session_id1)};
MDB_val _session_id2_rec = {data->session_id2, strlen(data->session_id2)};
MDB_val _ip_rec = {data->ip, strlen(data->ip)};
LMDB_CHECK(mdb_del(txn, dbi_session_id, &_session_id1_rec, NULL));
LMDB_CHECK(mdb_del(txn, dbi_session_id, &_session_id2_rec, NULL));
LMDB_CHECK(mdb_del(txn, dbi_ip, &_ip_rec, NULL));
event.obj_id = record_id;
event.event_type = 1;
MDB_val _event_rec = {&event, sizeof(event)};
LMDB_CHECK(mdb_del(txn, dbi_event, &_event_rec, NULL));
LMDB_CHECK(mdb_del(txn, dbi_session, &_obj_id_rec, NULL));
// transaction commit
LMDB_CHECK(mdb_txn_commit(txn));
lmdb_del++;
}
static void db_disconnect() {
mdb_env_close(env);
printf("Connection closed\n");
}
static void get_db_stat(const char* db, int64_t* ms_branch_pages, int64_t* ms_leaf_pages) {
MDB_txn *txn;
MDB_stat stat;
MDB_dbi dbi;
LMDB_CHECK(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
LMDB_CHECK(mdb_dbi_open(txn, db, MDB_CREATE, &dbi));
LMDB_CHECK(mdb_stat(txn, dbi, &stat));
mdb_txn_abort(txn);
printf("%15s | %15ld | %5u | %10ld | %10ld | %11ld |\n",
db,
stat.ms_branch_pages,
stat.ms_depth,
stat.ms_entries,
stat.ms_leaf_pages,
stat.ms_overflow_pages);
(*ms_branch_pages) += stat.ms_branch_pages;
(*ms_leaf_pages) += stat.ms_leaf_pages;
}
static void periodic_stat(void) {
int64_t ms_branch_pages = 0;
int64_t ms_leaf_pages = 0;
printf(" Name | ms_branch_pages | depth | entries | leaf_pages | overf_pages |\n");
get_db_stat("session", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("event", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages);
printf("%15s | %15ld | %5s | %10s | %10ld | %11s |\n", "", ms_branch_pages, "", "", ms_leaf_pages, "");
static int64_t prev_add;
static int64_t prev_del;
static int64_t t = -1;
if (t > 0) {
int64_t delta = getTimeMicroseconds() - t;
printf("CPS: add %ld, delete %ld, items processed - %ld\n", (lmdb_add - prev_add)*1000000 / delta, (lmdb_del - prev_del)*1000000 / delta, obj_id);
}
t = getTimeMicroseconds();
prev_add = lmdb_add;
prev_del = lmdb_del;
}
static void periodic_add_rec() {
int i;
for (i = 0; i < 10000; i++) {
if (ids_count <= REC_COUNT) {
int64_t id = obj_id++;
create_record(id);
add_id_to_pool(id);
}
if (ids_count > REC_COUNT) {
int64_t id = get_id_from_pool();
delete_record(id);
}
}
periodic_stat();
}
int main(int argc, char** argv) {
(void) argc;
(void) argv;
char filename[PATH_MAX];
mkdir(opt_db_path, 0775);
strcpy(filename, opt_db_path);
strcat(filename, "/data.mdb");
remove(filename);
strcpy(filename, opt_db_path);
strcat(filename, "/lock.mdb");
remove(filename);
db_connect();
while (1) {
periodic_add_rec();
}
db_disconnect();
return 0;
}

310
yota_test2.c
View File

@ -1,310 +0,0 @@
/*
* Copyright 2016-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2015 Vladimir Romanov <https://www.linkedin.com/in/vladimirromanov>, Yota Lab.
*
* This file is part of libmdbx.
*
* ReOpenMDBX is free software; you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* ReOpenMDBX is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <sys/time.h>
#include <sys/stat.h>
#include <limits.h>
#include <stddef.h>
#include <string.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <assert.h>
#include "mdbx.h"
#define IP_PRINTF_ARG_HOST(addr) (int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), (int)((addr) & 0xff)
char opt_db_path[PATH_MAX] = "/dev/shm/lmdb_bench2";
static MDB_env *env;
#define REC_COUNT 1024000
int64_t ids[REC_COUNT * 10];
int32_t ids_count = 0;
int64_t lmdb_add = 0;
int64_t lmdb_del = 0;
int64_t obj_id = 0;
int64_t lmdb_data_size = 0;
int64_t lmdb_key_size = 0;
static void add_id_to_pool(int64_t id) {
ids[ids_count] = id;
ids_count++;
}
static inline int64_t getTimeMicroseconds(void) {
struct timeval val;
gettimeofday(&val, NULL);
return val.tv_sec * ((int64_t) 1000000) + val.tv_usec;
}
static int64_t get_id_from_pool() {
if (ids_count == 0) {
return -1;
}
int32_t index = rand() % ids_count;
int64_t id = ids[index];
ids[index] = ids[ids_count - 1];
ids_count--;
return id;
}
#define LMDB_CHECK(x) \
do {\
const int rc = (x);\
if ( rc != MDB_SUCCESS ) {\
printf("Error [%d] %s in %s at %s:%d\n", rc, mdb_strerror(rc), #x, __FILE__, __LINE__); \
exit(EXIT_FAILURE); \
}\
} while(0)
static void db_connect() {
MDB_dbi dbi_session;
MDB_dbi dbi_session_id;
MDB_dbi dbi_event;
MDB_dbi dbi_ip;
LMDB_CHECK(mdb_env_create(&env));
LMDB_CHECK(mdb_env_set_mapsize(env, 300000L * 4096L));
LMDB_CHECK(mdb_env_set_maxdbs(env, 30));
#if defined(MDBX_LIFORECLAIM)
LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP | MDBX_LIFORECLAIM, 0664));
#else
LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP, 0664));
#endif
MDB_txn *txn;
// transaction init
LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn));
// open database in read-write mode
LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session));
LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id));
LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event));
LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip));
// transaction commit
LMDB_CHECK(mdb_txn_commit(txn));
printf("Connection open\n");
}
typedef struct {
char session_id1[100];
char session_id2[100];
char ip[20];
uint8_t fill[100];
} session_data_t;
typedef struct {
int64_t obj_id;
int8_t event_type;
} __attribute__((__packed__)) event_data_t;
static void create_record(int64_t record_id) {
MDB_dbi dbi_session;
MDB_dbi dbi_session_id;
MDB_dbi dbi_event;
MDB_dbi dbi_ip;
event_data_t event;
MDB_txn *txn;
session_data_t data;
// transaction init
snprintf(data.session_id1, sizeof (data.session_id1), "mskugw%02ld_%02ld.gx.yota.ru;3800464060;4152;%ld", record_id % 3 + 1, record_id % 9 + 1, record_id);
snprintf(data.session_id2, sizeof (data.session_id2), "gx_service;%ld;%ld;node@spb-jsm1", record_id, record_id % 1000000000 + 99999);
snprintf(data.ip, sizeof (data.ip), "%d.%d.%d.%d", IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF));
event.obj_id = record_id;
event.event_type = 1;
MDB_val _session_id1_rec = {data.session_id1, strlen(data.session_id1)};
MDB_val _session_id2_rec = {data.session_id2, strlen(data.session_id2)};
MDB_val _ip_rec = {data.ip, strlen(data.ip)};
MDB_val _obj_id_rec = {&record_id, sizeof(record_id)};
MDB_val _data_rec = {&data, offsetof(session_data_t, fill) + (rand() % sizeof (data.fill))};
MDB_val _event_rec = {&event, sizeof(event)};
LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn));
LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session));
LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id));
LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event));
LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip));
LMDB_CHECK(mdb_put(txn, dbi_session, &_obj_id_rec, &_data_rec, MDB_NOOVERWRITE | MDB_NODUPDATA));
LMDB_CHECK(mdb_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec, MDB_NOOVERWRITE | MDB_NODUPDATA));
LMDB_CHECK(mdb_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec, MDB_NOOVERWRITE | MDB_NODUPDATA));
LMDB_CHECK(mdb_put(txn, dbi_ip, &_ip_rec, &_obj_id_rec, 0));
LMDB_CHECK(mdb_put(txn, dbi_event, &_event_rec, &_obj_id_rec, 0));
lmdb_data_size += (_data_rec.mv_size + _obj_id_rec.mv_size * 4);
lmdb_key_size += (_obj_id_rec.mv_size + _session_id1_rec.mv_size + _session_id2_rec.mv_size + _ip_rec.mv_size + _event_rec.mv_size);
// transaction commit
LMDB_CHECK(mdb_txn_commit(txn));
lmdb_add++;
}
static void delete_record(int64_t record_id) {
MDB_dbi dbi_session;
MDB_dbi dbi_session_id;
MDB_dbi dbi_event;
MDB_dbi dbi_ip;
event_data_t event;
MDB_txn *txn;
// transaction init
LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn));
// open database in read-write mode
LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session));
LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id));
LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event));
LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip));
// put data
MDB_val _obj_id_rec = {&record_id, sizeof(record_id)};
MDB_val _data_rec;
// get data
LMDB_CHECK(mdb_get(txn, dbi_session, &_obj_id_rec, &_data_rec));
session_data_t* data = (session_data_t*) _data_rec.mv_data;
MDB_val _session_id1_rec = {data->session_id1, strlen(data->session_id1)};
MDB_val _session_id2_rec = {data->session_id2, strlen(data->session_id2)};
MDB_val _ip_rec = {data->ip, strlen(data->ip)};
LMDB_CHECK(mdb_del(txn, dbi_session_id, &_session_id1_rec, NULL));
LMDB_CHECK(mdb_del(txn, dbi_session_id, &_session_id2_rec, NULL));
LMDB_CHECK(mdb_del(txn, dbi_ip, &_ip_rec, NULL));
event.obj_id = record_id;
event.event_type = 1;
MDB_val _event_rec = {&event, sizeof(event)};
LMDB_CHECK(mdb_del(txn, dbi_event, &_event_rec, NULL));
LMDB_CHECK(mdb_del(txn, dbi_session, &_obj_id_rec, NULL));
lmdb_data_size -= (_data_rec.mv_size + _obj_id_rec.mv_size * 4);
lmdb_key_size -= (_obj_id_rec.mv_size + _session_id1_rec.mv_size + _session_id2_rec.mv_size + _ip_rec.mv_size + _event_rec.mv_size);
// transaction commit
LMDB_CHECK(mdb_txn_commit(txn));
lmdb_del++;
}
static void db_disconnect() {
mdb_env_close(env);
printf("Connection closed\n");
}
static void get_db_stat(const char* db, int64_t* ms_branch_pages, int64_t* ms_leaf_pages) {
MDB_txn *txn;
MDB_stat stat;
MDB_dbi dbi;
LMDB_CHECK(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
LMDB_CHECK(mdb_dbi_open(txn, db, MDB_CREATE, &dbi));
LMDB_CHECK(mdb_stat(txn, dbi, &stat));
mdb_txn_abort(txn);
printf("%15s | %15ld | %5u | %10ld | %10ld | %11ld |\n",
db,
stat.ms_branch_pages,
stat.ms_depth,
stat.ms_entries,
stat.ms_leaf_pages,
stat.ms_overflow_pages);
(*ms_branch_pages) += stat.ms_branch_pages;
(*ms_leaf_pages) += stat.ms_leaf_pages;
}
static void periodic_stat(void) {
int64_t ms_branch_pages = 0;
int64_t ms_leaf_pages = 0;
printf(" Name | ms_branch_pages | depth | entries | leaf_pages | overf_pages |\n");
get_db_stat("session", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("event", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages);
printf("%15s | %15ld | %5s | %10s | %10ld | %11s |\n", "", ms_branch_pages, "", "", ms_leaf_pages, "");
static int64_t prev_add;
static int64_t prev_del;
static int64_t t = -1;
if (t > 0) {
int64_t delta = getTimeMicroseconds() - t;
printf("CPS: add %ld, delete %ld, items processed - %ldK data=%ldK key=%ldK\n", (lmdb_add - prev_add)*1000000 / delta, (lmdb_del - prev_del)*1000000 / delta, obj_id / 1024, lmdb_data_size / 1024, lmdb_key_size / 1024);
printf("usage data=%ld%%\n", ((lmdb_data_size + lmdb_key_size) * 100) / ((ms_leaf_pages + ms_branch_pages)*4096));
}
t = getTimeMicroseconds();
prev_add = lmdb_add;
prev_del = lmdb_del;
}
//static void periodic_add_rec() {
// for (int i = 0; i < 10240; i++) {
// if (ids_count <= REC_COUNT) {
// int64_t id = obj_id++;
// create_record(id);
// add_id_to_pool(id);
// }
// if (ids_count > REC_COUNT) {
// int64_t id = get_id_from_pool();
// delete_record(id);
// }
// }
// periodic_stat();
//}
int main(int argc, char** argv) {
(void) argc;
(void) argv;
char filename[PATH_MAX];
int i;
int64_t t;
mkdir(opt_db_path, 0775);
strcpy(filename, opt_db_path);
strcat(filename, "/data.mdb");
remove(filename);
strcpy(filename, opt_db_path);
strcat(filename, "/lock.mdb");
remove(filename);
db_connect();
periodic_stat();
for (i = 0; i < 1024000; i++) {
int64_t id = obj_id++;
create_record(id);
add_id_to_pool(id);
}
periodic_stat();
t = getTimeMicroseconds();
while (1) {
int i;
int64_t now;
for (i = 0; i < 100; i++) {
int64_t id = obj_id++;
create_record(id);
add_id_to_pool(id);
id = get_id_from_pool();
delete_record(id);
}
//int64_t id = obj_id++;
//create_record(id);
//add_id_to_pool(id);
now = getTimeMicroseconds();
if ((now - t) > 100000) {
periodic_stat();
t = now;
}
}
db_disconnect();
return 0;
}