Compare commits

..

17 Commits

Author SHA1 Message Date
Leonid Yuriev
c778d3cfd4 mdbx: bump version to 0.9.x (not a release, but API changes).
Change-Id: I756f1224739df53d8503cf308c2c908f6ecd3fd1
2020-07-31 00:45:34 +03:00
Leonid Yuriev
67d27c81d7 mdbx-docs: fix typos & add refs for Doxygen. 2020-07-30 21:16:08 +03:00
Leonid Yuriev
dcd61289d8 mdbx-doc: more for Doxygen (refs, build options).
Change-Id: I8b15bce3d6638a24bdafa3171ae10f01311862d7
2020-07-29 02:18:44 +03:00
Leonid Yuriev
aa07d7a3a2 mdbx-docs: more for Doxygen (seems done).
Change-Id: I089748d738705b77e13887391a7e65a1ad327fea
2020-07-25 04:34:15 +03:00
Leonid Yuriev
a902538e34 mdbx: more Doxygen tags (almost done).
Change-Id: I696e717e37a905f68af059c51f9df327c257332e
2020-07-24 22:49:21 +03:00
Leonid Yuriev
2cd7fcb16d mdbx: fix minor Coverity issues.
Change-Id: I21c44de18e4e620ecec38ec837446150736faf79
2020-07-24 17:43:35 +03:00
Leonid Yuriev
5f4f828bae mdbx-doc: provide non-API docs via doxygen (squashed).
Change-Id: Ie33858517f964f794ec182a1e8bb630730a0f172
2020-07-23 18:44:32 +03:00
Leonid Yuriev
bb3d4ab9ba mdbx-ci: build github-pages by doxygen.
Change-Id: I9d84ca299cf0b8579157f84dae42fcbb09a6f8bc
2020-07-23 11:58:04 +03:00
Leonid Yuriev
e3efef40c4 mdbx: initial support for doxygen.
Change-Id: I5258b9f5dac981f03658ed35a799a52250a3e136
2020-07-23 11:58:04 +03:00
Leonid Yuriev
19e4fc1414 mdbx: minor fix .gitignore
Change-Id: I3b54830fed6ba161bfb631e6989f616df106450d
2020-07-23 11:58:04 +03:00
Leonid Yuriev
3d31884c3b mdbx-chk: always skip key ordering checking to avoid MDBX_CORRUPTED when using custom comparators.
Change-Id: Ieb6ea4a1c36b0e661afc13da365505ffbb5d0b5e
2020-07-23 11:47:05 +03:00
Leonid Yuriev
b6085afb5a mdbx-test: minor fix for MSVC-2015.
Change-Id: I631009ffe0b04428026492d1cab691b58fdd160a
2020-07-08 03:13:07 +03:00
Leonid Yuriev
9720ed39f5 mdbx: using enum instead of #define for flags/modes.
Resolve https://github.com/erthink/libmdbx/issues/108

Change-Id: I45897300375d2b5b9361aaba81dadcf9801fe3cf
2020-07-08 02:26:46 +03:00
Leonid Yuriev
2e0d2e65af mdbx: fix minor typos. 2020-07-07 23:16:43 +03:00
Leonid Yuriev
e989cb05ed mdbx: deprecate mdbx_dbi_open_ex() and custom comparators.
Change-Id: I1d20f77b9ba5d8d5ec891df17019377afb466b06
2020-07-07 19:33:59 +03:00
Leonid Yuriev
fab6ddee14 mdbx: add MDBX_DEPRECATED macro.
Change-Id: I87e14b37a6d152fa1f69f4a74e16244870dedb0f
2020-07-07 19:33:17 +03:00
Leonid Yuriev
e34e58f529 mdbx: reformat ChangeLog (cosmetics).
Change-Id: Ida9e18a512604c3fff8d3d88558020f694e740b0
2020-07-07 00:38:27 +03:00
36 changed files with 6275 additions and 2862 deletions

View File

@@ -1,6 +1,9 @@
version: 2
jobs:
build:
branches:
ignore:
- gh-pages
docker:
- image: circleci/buildpack-deps:20.04
environment:

View File

@@ -0,0 +1,25 @@
name: doxygen-github-pages
on:
push:
branches: master
jobs:
build:
runs-on: ubuntu-20.04
steps:
- name: Checkout code
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Install doxygen
run: sudo apt install doxygen graphviz fonts-freefont-ttf
- name: Build html docs
run: make doxygen && cp -R .circleci docs/html/
- name: Deploy gh-pages
uses: JamesIves/github-pages-deploy-action@3.5.7
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BRANCH: gh-pages
FOLDER: docs/html
SINGLE_COMMIT: true

View File

@@ -11,10 +11,12 @@ name: Upload Release Asset
jobs:
build:
name: Upload Release Asset
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- name: Checkout code
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Build assets
run: |
make release-assets

6
.gitignore vendored
View File

@@ -29,7 +29,9 @@ test/tmp.db-lck
tmp.db
tmp.db-lck
valgrind.*
src/elements/version.c
src/elements/config.h
src/version.c
src/config.h
dist/
*.tar*
docs/Doxyfile
docs/html/

58
AUTHORS
View File

@@ -1,32 +1,32 @@
Contributors
============
Alexey Naumov <alexey.naumov@gmail.com>
Chris Mikkelson <cmikk@qwest.net>
Claude Brisson <claude.brisson@gmail.com>
David Barbour <dmbarbour@gmail.com>
David Wilson <dw@botanicus.net>
dreamsxin <dreamsxin@126.com>
Hallvard Furuseth <hallvard@openldap.org>, <h.b.furuseth@usit.uio.no>
Heiko Becker <heirecka@exherbo.org>
Howard Chu <hyc@openldap.org>, <hyc@symas.com>
Ignacio Casal Quinteiro <ignacio.casal@nice-software.com>
James Rouzier <rouzier@gmail.com>
Jean-Christophe DUBOIS <jcd@tribudubois.net>
John Hewson <john@jahewson.com>
Klaus Malorny <klaus.malorny@knipp.de>
Kurt Zeilenga <kurt.zeilenga@isode.com>
Leonid Yuriev <leo@yuriev.ru>, <lyuryev@ptsecurity.com>
Lorenz Bauer <lmb@cloudflare.com>
Luke Yeager <lyeager@nvidia.com>
Martin Hedenfalk <martin@bzero.se>
Ondrej Kuznik <ondrej.kuznik@acision.com>
Orivej Desh <orivej@gmx.fr>
Oskari Timperi <oskari.timperi@iki.fi>
Pavel Medvedev <pmedvedev@gmail.com>
Philipp Storz <philipp.storz@bareos.com>
Quanah Gibson-Mount <quanah@openldap.org>
Salvador Ortiz <sog@msg.com.mx>
Sebastien Launay <sebastien@slaunay.fr>
Vladimir Romanov <vromanov@gmail.com>
Zano Foundation <crypto.sowle@gmail.com>
- Alexey Naumov <alexey.naumov@gmail.com>
- Chris Mikkelson <cmikk@qwest.net>
- Claude Brisson <claude.brisson@gmail.com>
- David Barbour <dmbarbour@gmail.com>
- David Wilson <dw@botanicus.net>
- dreamsxin <dreamsxin@126.com>
- Hallvard Furuseth <hallvard@openldap.org>, <h.b.furuseth@usit.uio.no>
- Heiko Becker <heirecka@exherbo.org>
- Howard Chu <hyc@openldap.org>, <hyc@symas.com>
- Ignacio Casal Quinteiro <ignacio.casal@nice-software.com>
- James Rouzier <rouzier@gmail.com>
- Jean-Christophe DUBOIS <jcd@tribudubois.net>
- John Hewson <john@jahewson.com>
- Klaus Malorny <klaus.malorny@knipp.de>
- Kurt Zeilenga <kurt.zeilenga@isode.com>
- Leonid Yuriev <leo@yuriev.ru>, <lyuryev@ptsecurity.com>
- Lorenz Bauer <lmb@cloudflare.com>
- Luke Yeager <lyeager@nvidia.com>
- Martin Hedenfalk <martin@bzero.se>
- Ondrej Kuznik <ondrej.kuznik@acision.com>
- Orivej Desh <orivej@gmx.fr>
- Oskari Timperi <oskari.timperi@iki.fi>
- Pavel Medvedev <pmedvedev@gmail.com>
- Philipp Storz <philipp.storz@bareos.com>
- Quanah Gibson-Mount <quanah@openldap.org>
- Salvador Ortiz <sog@msg.com.mx>
- Sebastien Launay <sebastien@slaunay.fr>
- Vladimir Romanov <vromanov@gmail.com>
- Zano Foundation <crypto.sowle@gmail.com>

View File

@@ -1,128 +1,133 @@
v0.9.x (in the development):
- TODO: API for explicit threads (de)registration.
  - TODO: Native bindings for C++.
  - TODO: Packages for AltLinux, Fedora/RHEL, Debian/Ubuntu.
ChangeLog
---------
v0.8.2 2020-07-06:
  - Added support multi-opening the same DB in a process with SysV locking (BSD).
  - Fixed warnings & minors for LCC compiler (E2K).
  - Enabled to simultaneously open the same database from processes with and without the `MDBX_WRITEMAP` option.
  - Added key-to-value, `mdbx_get_keycmp()` and `mdbx_get_datacmp()` functions (helpful to avoid using custom comparators).
- Added `ENABLE_UBSAN` CMake option to enabling the UndefinedBehaviorSanitizer from GCC/CLANG.
  - Workaround for CLANG bug https://bugs.llvm.org/show_bug.cgi?id=43275.
  - Returning MDBX_CORRUPTED in case all meta-pages are weak and no other error.
  - Refined mode bits while auto-creating LCK-file.
- Avoids unnecessary database file re-mapping in case geometry changed by another process(es).
From the user's point of view, the MDBX_UNABLE_EXTEND_MAPSIZE error will now be returned less frequently and only when using the DB in the current process really requires it to be reopened.
- Remapping on-the-fly and of the database file was implemented.
Now remapping with a change of address is performed automatically if there are no dependent readers in the current process.
## v0.9.x (in the development):
- Since v0.9 usage of custom comparators and the `mdbx_dbi_open_ex()` are deprecated.
- Support for Doxygen & [online API reference](https://erthink.github.io/libmdbx/).
- TODO: API for explicit threads (de)registration.
- TODO: Native bindings for C++.
- TODO: Packages for AltLinux, Fedora/RHEL, Debian/Ubuntu.
v0.8.1 2020-06-12:
  - Minor change versioning. The last number in the version now means the number of commits since last release/tag.
  - Provide ChangeLog file.
  - Fix for using libmdbx as a C-only sub-project with CMake.
  - Fix `mdbx_env_set_geometry()` for case it is called from an opened environment outside of a write transaction.
  - Add support for huge transactions and `MDBX_HUGE_TRANSACTIONS` build-option (default `OFF`).
  - Refine LTO (link time optimization) for clang.
  - Force enabling exceptions handling for MSVC (`/EHsc` option).
## v0.8.2 2020-07-06:
- Added support multi-opening the same DB in a process with SysV locking (BSD).
- Fixed warnings & minors for LCC compiler (E2K).
- Enabled to simultaneously open the same database from processes with and without the `MDBX_WRITEMAP` option.
- Added key-to-value, `mdbx_get_keycmp()` and `mdbx_get_datacmp()` functions (helpful to avoid using custom comparators).
- Added `ENABLE_UBSAN` CMake option to enabling the UndefinedBehaviorSanitizer from GCC/CLANG.
- Workaround for [CLANG bug](https://bugs.llvm.org/show_bug.cgi?id=43275).
- Returning `MDBX_CORRUPTED` in case all meta-pages are weak and no other error.
- Refined mode bits while auto-creating LCK-file.
- Avoids unnecessary database file re-mapping in case geometry changed by another process(es).
From the user's point of view, the `MDBX_UNABLE_EXTEND_MAPSIZE` error will now be returned less frequently and only when using the DB in the current process really requires it to be reopened.
- Remapping on-the-fly and of the database file was implemented.
Now remapping with a change of address is performed automatically if there are no dependent readers in the current process.
v0.8.0 2020-06-05:
  - Support for Android/Bionic.
  - Support for iOS.
  - Auto-handling `MDBX_NOSUBDIR` while opening for any existing database.
  - Engage github-actions to make release-assets.
  - Clarify API description.
  - Extended keygen-cases in stochastic test.
  - Fix fetching of first/lower key from LEAF2-page during page merge.
  - Fix missing comma in array of error messages.
  - Fix div-by-zero while copy-with-compaction for non-resizable environments.
  - Fixes & enhancements for custom-comparators.
  - Fix `MDBX_AVOID_CRT` option and missing `ntdll.def`.
  - Fix `mdbx_env_close()` to work correctly called concurrently from several threads.
  - Fix null-deref in an ASAN-enabled builds while opening the environment with error and/or read-only.
  - Fix AddressSanitizer errors after closing the environment.
  - Fix/workaround to avoid GCC 10.x pedantic warnings.
  - Fix using `ENODATA` for FreeBSD.
  - Avoid invalidation of DBI-handle(s) when it just closes.
  - Avoid using `pwritev()` for single-writes (up to 10% speedup for some kernels & scenarios).
  - Avoiding `MDBX_UTTERLY_NOSYNC` as result of flags merge.
  - Add `mdbx_dbi_dupsort_depthmask()` function.
  - Add `MDBX_CP_FORCE_RESIZEABLE` option.
  - Add deprecated `MDBX_MAP_RESIZED` for compatibility.
  - Add `MDBX_BUILD_TOOLS` option (default `ON`).
  - Refine `mdbx_dbi_open_ex()` to safe concurrently opening the same handle from different threads.
  - Truncate clk-file during environment closing. So a zero-length lck-file indicates that the environment was closed properly.
  - Refine `mdbx_update_gc()` for huge transactions with small sizes of database page.
  - Extends dump/load to support all MDBX attributes.
  - Avoid upsertion the same key-value data, fix related assertions.
  - Rework min/max length checking for keys & values.
  - Checking the order of keys on all pages during checking.
  - Support `CFLAGS_EXTRA` make-option for convenience.
  - Preserve the last txnid while copying with compactification.
  - Auto-reset running transaction in mdbx_txn_renew().
  - Automatically abort errored transaction in mdbx_txn_commit().
  - Auto-choose page size for large databases.
  - Rearrange source files, rework build, options-support by CMake.
  - Crutch for WSL1 (Windows subsystem for Linux).
  - Refine install/uninstall targets.
  - Support for Valgrind 3.14 and later.
  - Add check-analyzer check-ubsan check-asan check-leak targets to Makefile.
  - Minor fix/workaround to avoid UBSAN traps for `memcpy(ptr, NULL, 0)`.
  - Avoid some GCC-analyzer false-positive warnings.
## v0.8.1 2020-06-12:
- Minor change versioning. The last number in the version now means the number of commits since last release/tag.
- Provide ChangeLog file.
- Fix for using libmdbx as a C-only sub-project with CMake.
- Fix `mdbx_env_set_geometry()` for case it is called from an opened environment outside of a write transaction.
- Add support for huge transactions and `MDBX_HUGE_TRANSACTIONS` build-option (default `OFF`).
- Refine LTO (link time optimization) for clang.
- Force enabling exceptions handling for MSVC (`/EHsc` option).
v0.7.0 2020-03-18:
  - Workarounds for Wine (Windows compatibility layer for Linux).
  - `MDBX_MAP_RESIZED` renamed to `MDBX_UNABLE_EXTEND_MAPSIZE`.
  - Clarify API description, fix typos.
  - Speedup runtime checks in debug/checked builds.
  - Added checking for read/write transactions overlapping for the same thread, added `MDBX_TXN_OVERLAPPING` error and `MDBX_DBG_LEGACY_OVERLAP` option.
  - Added `mdbx_key_from_jsonInteger()`, `mdbx_key_from_double()`, `mdbx_key_from_float()`, `mdbx_key_from_int64()` and `mdbx_key_from_int32()` functions. See `mdbx.h` for description.
  - Fix compatibility (use zero for invalid DBI).
  - Refine/clarify error messages.
  - Avoids extra error messages "bad txn" from mdbx_chk when DB is corrupted.
## v0.8.0 2020-06-05:
- Support for Android/Bionic.
- Support for iOS.
- Auto-handling `MDBX_NOSUBDIR` while opening for any existing database.
- Engage github-actions to make release-assets.
- Clarify API description.
- Extended keygen-cases in stochastic test.
- Fix fetching of first/lower key from LEAF2-page during page merge.
- Fix missing comma in array of error messages.
- Fix div-by-zero while copy-with-compaction for non-resizable environments.
- Fixes & enhancements for custom-comparators.
- Fix `MDBX_AVOID_CRT` option and missing `ntdll.def`.
- Fix `mdbx_env_close()` to work correctly called concurrently from several threads.
- Fix null-deref in an ASAN-enabled builds while opening the environment with error and/or read-only.
- Fix AddressSanitizer errors after closing the environment.
- Fix/workaround to avoid GCC 10.x pedantic warnings.
- Fix using `ENODATA` for FreeBSD.
- Avoid invalidation of DBI-handle(s) when it just closes.
- Avoid using `pwritev()` for single-writes (up to 10% speedup for some kernels & scenarios).
- Avoiding `MDBX_UTTERLY_NOSYNC` as result of flags merge.
- Add `mdbx_dbi_dupsort_depthmask()` function.
- Add `MDBX_CP_FORCE_RESIZEABLE` option.
- Add deprecated `MDBX_MAP_RESIZED` for compatibility.
- Add `MDBX_BUILD_TOOLS` option (default `ON`).
- Refine `mdbx_dbi_open_ex()` to safe concurrently opening the same handle from different threads.
- Truncate clk-file during environment closing. So a zero-length lck-file indicates that the environment was closed properly.
- Refine `mdbx_update_gc()` for huge transactions with small sizes of database page.
- Extends dump/load to support all MDBX attributes.
- Avoid upsertion the same key-value data, fix related assertions.
- Rework min/max length checking for keys & values.
- Checking the order of keys on all pages during checking.
- Support `CFLAGS_EXTRA` make-option for convenience.
- Preserve the last txnid while copying with compactification.
- Auto-reset running transaction in mdbx_txn_renew().
- Automatically abort errored transaction in mdbx_txn_commit().
- Auto-choose page size for large databases.
- Rearrange source files, rework build, options-support by CMake.
- Crutch for WSL1 (Windows subsystem for Linux).
- Refine install/uninstall targets.
- Support for Valgrind 3.14 and later.
- Add check-analyzer check-ubsan check-asan check-leak targets to Makefile.
- Minor fix/workaround to avoid UBSAN traps for `memcpy(ptr, NULL, 0)`.
- Avoid some GCC-analyzer false-positive warnings.
v0.6.0 2020-01-21:
  - Fix `mdbx_load` utility for custom comparators.
  - Fix checks related to `MDBX_APPEND` flag inside `mdbx_cursor_put()`.
  - Refine/fix dbi_bind() internals.
  - Refine/fix handling STATUS_CONFLICTING_ADDRESSES.
  - Rework `MDBX_DBG_DUMP` option to avoid disk I/O performance degradation.
  - Add built-in help to test tool.
  - Fix `mdbx_env_set_geometry()` for large page size.
  - Fix env_set_geometry() for large pagesize.
  - Clarify API description & comments, fix typos.
## v0.7.0 2020-03-18:
- Workarounds for Wine (Windows compatibility layer for Linux).
- `MDBX_MAP_RESIZED` renamed to `MDBX_UNABLE_EXTEND_MAPSIZE`.
- Clarify API description, fix typos.
- Speedup runtime checks in debug/checked builds.
- Added checking for read/write transactions overlapping for the same thread, added `MDBX_TXN_OVERLAPPING` error and `MDBX_DBG_LEGACY_OVERLAP` option.
- Added `mdbx_key_from_jsonInteger()`, `mdbx_key_from_double()`, `mdbx_key_from_float()`, `mdbx_key_from_int64()` and `mdbx_key_from_int32()` functions. See `mdbx.h` for description.
- Fix compatibility (use zero for invalid DBI).
- Refine/clarify error messages.
- Avoids extra error messages "bad txn" from mdbx_chk when DB is corrupted.
v0.5.0 2019-12-31:
  - Fix returning MDBX_RESULT_TRUE from page_alloc().
  - Fix false-positive ASAN issue.
  - Fix assertion for `MDBX_NOTLS` option.
  - Rework MADV_DONTNEED threshold.
  - Fix `mdbx_chk` utility for don't checking some numbers if walking on the B-tree was disabled.
  - Use page's mp_txnid for basic integrity checking.
  - Add MDBX_FORCE_ASSERTIONS built-time option.
  - Rework MDBX_DBG_DUMP to avoid performance degradation.
  - Rename MDBX_NOSYNC to MDBX_SAFE_NOSYNC for clarity.
  - Interpret `ERROR_ACCESS_DENIED` from `OpenProcess()` as 'process exists'.
  - Avoid using FILE_FLAG_NO_BUFFERING for compatibility with small database pages.
  - Added install section for CMake.
## v0.6.0 2020-01-21:
- Fix `mdbx_load` utility for custom comparators.
- Fix checks related to `MDBX_APPEND` flag inside `mdbx_cursor_put()`.
- Refine/fix dbi_bind() internals.
- Refine/fix handling `STATUS_CONFLICTING_ADDRESSES`.
- Rework `MDBX_DBG_DUMP` option to avoid disk I/O performance degradation.
- Add built-in help to test tool.
- Fix `mdbx_env_set_geometry()` for large page size.
- Fix env_set_geometry() for large pagesize.
- Clarify API description & comments, fix typos.
v0.4.0 2019-12-02:
  - Support for Mac OSX, FreeBSD, NetBSD, OpenBSD, DragonFly BSD, OpenSolaris, OpenIndiana (AIX and HP-UX pending).
  - Use bootid for decisions of rollback.
  - Counting retired pages and extended transaction info.
  - Add MDBX_ACCEDE flag for database opening.
  - Using OFD-locks and tracking for in-process multi-opening.
  - Hot backup into pipe.
  - Support for cmake & amalgamated sources.
  - Fastest internal sort implementation.
  - New internal dirty-list implementation with lazy sorting.
  - Support for lazy-sync-to-disk with polling.
  - Extended key length.
  - Last update transaction number for each sub-database.
  - Automatic read ahead enabling/disabling.
  - More auto-compactification.
  - Using -fsanitize=undefined and -Wpedantic options.
  - Rework page merging.
  - Nested transactions.
  - API description.
  - Checking for non-local filesystems to avoid DB corruption.
## v0.5.0 2019-12-31:
- Fix returning MDBX_RESULT_TRUE from page_alloc().
- Fix false-positive ASAN issue.
- Fix assertion for `MDBX_NOTLS` option.
- Rework `MADV_DONTNEED` threshold.
- Fix `mdbx_chk` utility for don't checking some numbers if walking on the B-tree was disabled.
- Use page's mp_txnid for basic integrity checking.
- Add `MDBX_FORCE_ASSERTIONS` built-time option.
- Rework `MDBX_DBG_DUMP` to avoid performance degradation.
- Rename `MDBX_NOSYNC` to `MDBX_SAFE_NOSYNC` for clarity.
- Interpret `ERROR_ACCESS_DENIED` from `OpenProcess()` as 'process exists'.
- Avoid using `FILE_FLAG_NO_BUFFERING` for compatibility with small database pages.
- Added install section for CMake.
## v0.4.0 2019-12-02:
- Support for Mac OSX, FreeBSD, NetBSD, OpenBSD, DragonFly BSD, OpenSolaris, OpenIndiana (AIX and HP-UX pending).
- Use bootid for decisions of rollback.
- Counting retired pages and extended transaction info.
- Add `MDBX_ACCEDE` flag for database opening.
- Using OFD-locks and tracking for in-process multi-opening.
- Hot backup into pipe.
- Support for cmake & amalgamated sources.
- Fastest internal sort implementation.
- New internal dirty-list implementation with lazy sorting.
- Support for lazy-sync-to-disk with polling.
- Extended key length.
- Last update transaction number for each sub-database.
- Automatic read ahead enabling/disabling.
- More auto-compactification.
- Using -fsanitize=undefined and -Wpedantic options.
- Rework page merging.
- Nested transactions.
- API description.
- Checking for non-local filesystems to avoid DB corruption.

View File

@@ -48,7 +48,7 @@ LIBRARIES := libmdbx.a libmdbx.$(SO_SUFFIX)
TOOLS := mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_chk
MANPAGES := mdbx_stat.1 mdbx_copy.1 mdbx_dump.1 mdbx_load.1 mdbx_chk.1
.PHONY: mdbx all install clean test dist check
.PHONY: mdbx all install clean
all: $(LIBRARIES) $(TOOLS)
@@ -99,6 +99,8 @@ else
################################################################################
# Plain (non-amalgamated) sources with test
.PHONY: test dist check doxygen reformat
define uname2osal
case "$(UNAME)" in
CYGWIN*|MINGW*|MSYS*|Windows*) echo windows;;
@@ -240,6 +242,37 @@ mdbx-dylib.o: src/config.h src/version.c src/alloy.c $(ALLOY_DEPS) $(lastword $(
mdbx-static.o: src/config.h src/version.c src/alloy.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST))
$(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -ULIBMDBX_EXPORTS -c src/alloy.c -o $@
docs/Doxyfile: docs/Doxyfile.in src/version.c
sed \
-e "s|@MDBX_GIT_TIMESTAMP@|$(MDBX_GIT_TIMESTAMP)|" \
-e "s|@MDBX_GIT_TREE@|$(shell git show --no-patch --format=%T HEAD || echo 'Please install latest get version')|" \
-e "s|@MDBX_GIT_COMMIT@|$(shell git show --no-patch --format=%H HEAD || echo 'Please install latest get version')|" \
-e "s|@MDBX_GIT_DESCRIBE@|$(MDBX_GIT_DESCRIBE)|" \
-e "s|\$${MDBX_VERSION_MAJOR}|$(shell echo '$(MDBX_GIT_VERSION)' | cut -d . -f 1)|" \
-e "s|\$${MDBX_VERSION_MINOR}|$(shell echo '$(MDBX_GIT_VERSION)' | cut -d . -f 2)|" \
-e "s|\$${MDBX_VERSION_RELEASE}|$(shell echo '$(MDBX_GIT_VERSION)' | cut -d . -f 3)|" \
-e "s|\$${MDBX_VERSION_REVISION}|$(MDBX_GIT_REVISION)|" \
docs/Doxyfile.in > $@
define md-extract-section
docs/__$(1).md: $(2)
sed -n '/<!-- section-begin $(1) -->/,/<!-- section-end -->/p' $$< > $$@ && test -s $$@
endef
$(foreach section,overview mithril characteristics improvements history usage performance bindings,$(eval $(call md-extract-section,$(section),README.md)))
docs/overall.md: docs/__overview.md docs/_toc.md docs/__mithril.md docs/__history.md AUTHORS LICENSE
echo -e "\\mainpage Overall\n\\section brief Brief" | cat - $(filter %.md, $^) > $@ && echo -e "\n\n\nLicense\n=======\n" | cat AUTHORS - LICENSE >> $@
docs/intro.md: docs/_preface.md docs/__characteristics.md docs/__improvements.md docs/_restrictions.md docs/__performance.md
cat $^ | sed 's/^Performance comparison$$/Performance comparison {#performance}/' > $@
docs/usage.md: docs/__usage.md docs/_starting.md docs/__bindings.md
echo -e "\\page usage Usage\n\\section getting Getting the libmdbx" | cat - $^ | sed 's/^Bindings$$/Bindings {#bindings}/' > $@
doxygen: docs/Doxyfile docs/overall.md docs/intro.md docs/usage.md mdbx.h src/options.h ChangeLog.md AUTHORS LICENSE
rm -rf docs/html && cp mdbx.h src/options.h ChangeLog.md docs/ && (cd docs && doxygen Doxyfile) && cp AUTHORS LICENSE docs/html/
.PHONY: dist release-assets
dist: libmdbx-sources-$(MDBX_VERSION_SUFFIX).tar.gz $(lastword $(MAKEFILE_LIST))

View File

@@ -1,4 +1,4 @@
all bench bench-quartet build-test check clean clean-bench cross-gcc cross-qemu dist gcc-analyzer install mdbx memcheck reformat release-assets strip test test-asan test-fault test-leak test-singleprocess test-ubsan test-valgrind tools:
all bench bench-quartet build-test check clean clean-bench cross-gcc cross-qemu dist doxygen gcc-analyzer install mdbx memcheck reformat release-assets strip test test-asan test-fault test-leak test-singleprocess test-ubsan test-valgrind tools:
@CC=$(CC) \
CXX=`if test -n "$(CXX)" && which "$(CXX)" > /dev/null; then echo "$(CXX)"; elif test -n "$(CCC)" && which "$(CCC)" > /dev/null; then echo "$(CCC)"; else echo "c++"; fi` \
`which gmake || which gnumake || echo 'echo "GNU Make is required"; exit 2;'` \

View File

@@ -1,13 +1,14 @@
<!-- Required extensions: pymdownx.betterem, pymdownx.tilde, pymdownx.emoji, pymdownx.tasklist, pymdownx.superfences -->
libmdbx
=======
========
<!-- section-begin overview -->
_libmdbx_ is an extremely fast, compact, powerful, embedded,
transactional [key-value store](https://en.wikipedia.org/wiki/Key-value_database)
database, with [permissive license](LICENSE).
transactional [key-value database](https://en.wikipedia.org/wiki/Key-value_database),
with [permissive license](./LICENSE).
_MDBX_ has a specific set of properties and capabilities,
focused on creating unique lightweight solutions with extraordinary performance.
focused on creating unique lightweight solutions.
1. Allows **a swarm of multi-threaded processes to
[ACID]((https://en.wikipedia.org/wiki/ACID))ly read and update** several
@@ -43,13 +44,15 @@ neglected in favour of write performance.
7. Supports Linux, Windows, MacOS, Android, iOS, FreeBSD, DragonFly, Solaris,
OpenSolaris, OpenIndiana, NetBSD, OpenBSD and other systems compliant with
**POSIX.1-2008**.
<!-- section-end -->
Historically, _MDBX_ is a deeply revised and extended descendant of the amazing
[Lightning Memory-Mapped Database](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database).
_MDBX_ inherits all benefits from _LMDB_, but resolves some issues and adds [a set of improvements](#improvements-beyond-lmdb).
<!-- section-begin mithril -->
The next version is under active non-public development from scratch and will be
released as **_MithrilDB_** and `libmithrildb` for libraries & packages.
released as _**MithrilDB**_ and `libmithrildb` for libraries & packages.
Admittedly mythical [Mithril](https://en.wikipedia.org/wiki/Mithril) is
resembling silver but being stronger and lighter than steel. Therefore
_MithrilDB_ is a rightly relevant name.
@@ -58,6 +61,7 @@ _MithrilDB_ is a rightly relevant name.
> License](https://www.apache.org/licenses/LICENSE-2.0). The goal of this
> revolution is to provide a clearer and robust API, add more features and
> new valuable properties of the database.
<!-- section-end -->
[![https://t.me/libmdbx](https://raw.githubusercontent.com/wiki/erthink/libmdbx/img/telegram.png)](https://t.me/libmdbx)
[![Build Status](https://travis-ci.org/erthink/libmdbx.svg?branch=master)](https://travis-ci.org/erthink/libmdbx)
@@ -71,10 +75,10 @@ _MithrilDB_ is a rightly relevant name.
-----
## Table of Contents
- [Overview](#overview)
- [Characteristics](#characteristics)
- [Features](#features)
- [Limitations](#limitations)
- [Caveats & Gotchas](#caveats--gotchas)
- [Gotchas](#gotchas)
- [Comparison with other databases](#comparison-with-other-databases)
- [Improvements beyond LMDB](#improvements-beyond-lmdb)
- [History & Acknowledgments](#history)
@@ -90,7 +94,9 @@ _MithrilDB_ is a rightly relevant name.
- [Async-write mode](#async-write-mode)
- [Cost comparison](#cost-comparison)
# Overview
# Characteristics
<!-- section-begin characteristics -->
## Features
@@ -146,7 +152,7 @@ transaction journal. No crash recovery needed. No maintenance is required.
- **Database size**: up to `2147483648` pages (8 [TiB](https://en.wikipedia.org/wiki/Tebibyte) for default 4K pagesize, 128 [TiB](https://en.wikipedia.org/wiki/Tebibyte) for 64K pagesize).
- **Maximum sub-databases**: `32765`.
## Caveats & Gotchas
## Gotchas
1. There cannot be more than one writer at a time, i.e. no more than one write transaction at a time.
@@ -165,11 +171,14 @@ so you should reconsider using brute force techniques and double check your code
On the one hand, in the case of MDBX, a simple linear search may be more profitable than complex indexes.
On the other hand, if you make something suboptimally, you can notice detrimentally only on sufficiently large data.
### Comparison with other databases
## Comparison with other databases
For now please refer to [chapter of "BoltDB comparison with other
databases"](https://github.com/coreos/bbolt#comparison-with-other-databases)
which is also (mostly) applicable to _libmdbx_.
<!-- section-end -->
<!-- section-begin improvements -->
Improvements beyond LMDB
========================
@@ -180,7 +189,7 @@ out-of-the-box, not silently and catastrophically break down. The list
below is pruned down to the improvements most notable and obvious from
the user's point of view.
### Added Features:
## Added Features
1. Keys could be more than 2 times longer than _LMDB_.
> For DB with default page size _libmdbx_ support keys up to 1300 bytes
@@ -230,7 +239,7 @@ and/or optimize query execution plans.
12. Support for opening databases in the exclusive mode, including on a network share.
### Added Abilities:
## Added Abilities
1. Zero-length for keys and values.
@@ -248,7 +257,7 @@ pair, to the first, to the last, or not set to anything.
> _libmdbx_ allows one _at once_ with getting previous value
> and addressing the particular item from multi-value with the same key.
### Other fixes and specifics:
## Other fixes and specifics
1. Fixed more than 10 significant errors, in particular: page leaks, wrong sub-database statistics, segfault in several conditions, nonoptimal page merge strategy, updating an existing record with a change in data size (including for multimap), etc.
@@ -282,7 +291,13 @@ against incompetent user actions (aka
_libmdbx_ may be a little lag in performance tests from LMDB where the
named mutexes are used.
### History
<!-- section-end -->
<!-- section-begin history -->
# History
Historically, _MDBX_ is a deeply revised and extended descendant of the
[Lightning Memory-Mapped Database](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database).
At first the development was carried out within the
[ReOpenLDAP](https://github.com/erthink/ReOpenLDAP) project. About a
year later _libmdbx_ was separated into a standalone project, which was
@@ -292,18 +307,26 @@ conference](http://www.highload.ru/2015/abstracts/1831.html).
Since 2017 _libmdbx_ is used in [Fast Positive Tables](https://github.com/erthink/libfpta),
and development is funded by [Positive Technologies](https://www.ptsecurity.com).
### Acknowledgments
## Acknowledgments
Howard Chu <hyc@openldap.org> is the author of LMDB, from which
originated the MDBX in 2015.
Martin Hedenfalk <martin@bzero.se> is the author of `btree.c` code, which
was used to begin development of LMDB.
<!-- section-end -->
--------------------------------------------------------------------------------
Usage
=====
<!-- section-begin usage -->
Currently, libmdbx is only available in a
[source code](https://en.wikipedia.org/wiki/Source_code) form.
Packages support for common Linux distributions is planned in the future,
since release the version 1.0.
## Source code embedding
_libmdbx_ provides two official ways for integration in source code form:
@@ -316,7 +339,7 @@ _libmdbx_ provides two official ways for integration in source code form:
> This allows you to build as _libmdbx_ and testing tool.
> On the other hand, this way requires you to pull git tags, and use C++11 compiler for test tool.
**_Please, avoid using any other techniques._** Otherwise, at least
_**Please, avoid using any other techniques.**_ Otherwise, at least
don't ask for support and don't name such chimeras `libmdbx`.
The amalgamated source code could be created from the original clone of git
@@ -434,21 +457,30 @@ To build _libmdbx_ for iOS, we recommend using CMake with the
"[toolchain file](https://cmake.org/cmake/help/latest/variable/CMAKE_TOOLCHAIN_FILE.html)"
from the [ios-cmake](https://github.com/leetal/ios-cmake) project.
<!-- section-end -->
## API description
For more information and API description see the [mdbx.h](mdbx.h) header.
Please do not hesitate to point out errors in the documentation,
including creating [PR](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/proposing-changes-to-your-work-with-pull-requests) with corrections and improvements.
## Bindings
Please refer to the online [_libmdbx_ API reference](https://erthink.github.io/libmdbx/)
and/or see the [mdbx.h](mdbx.h) header.
| Runtime | GitHub | Author |
| -------- | ------ | ------ |
| Rust | [mdbx-rs](https://github.com/Kerollmops/mdbx-rs) | [@Kerollmops](https://github.com/Kerollmops) |
| Java | [mdbxjni](https://github.com/castortech/mdbxjni) | [Castor Technologies](https://castortech.com/) |
| .NET | [mdbx.NET](https://github.com/wangjia184/mdbx.NET) | [Jerry Wang](https://github.com/wangjia184) |
<!-- section-begin bindings -->
Bindings
========
| Runtime | GitHub | Author |
| ------- | ------ | ------ |
| Rust | [mdbx-rs](https://github.com/Kerollmops/mdbx-rs) | [Clément Renault](https://github.com/Kerollmops) |
| Java | [mdbxjni](https://github.com/castortech/mdbxjni) | [Castor Technologies](https://castortech.com/) |
| .NET | [mdbx.NET](https://github.com/wangjia184/mdbx.NET) | [Jerry Wang](https://github.com/wangjia184) |
<!-- section-end -->
--------------------------------------------------------------------------------
<!-- section-begin performance -->
Performance comparison
======================
@@ -585,6 +617,8 @@ syscall and by scanning the data directory.
![Comparison #6: Cost comparison](https://raw.githubusercontent.com/wiki/erthink/libmdbx/img/perf-slide-6.png)
<!-- section-end -->
--------------------------------------------------------------------------------
#### This is a mirror of the origin repository that was moved to [abf.io](https://abf.io/erthink/) because of discriminatory restrictions for Russian Crimea.

View File

@@ -1,4 +1,4 @@
version: 0.8.2.{build}
version: 0.9.0.{build}
environment:
matrix:

2533
docs/Doxyfile.in Normal file

File diff suppressed because it is too large Load Diff

47
docs/_preface.md Normal file
View File

@@ -0,0 +1,47 @@
\page intro Introduction
\section characteristics Characteristics
Preface {#preface}
------------------
> For the most part, this section is a copy of the corresponding text
> from LMDB description, but with some edits reflecting the improvements
> and enhancements were made in MDBX.
MDBX is a Btree-based database management library modeled loosely on the
BerkeleyDB API, but much simplified. The entire database (aka "environment")
is exposed in a memory map, and all data fetches return data directly from
the mapped memory, so no malloc's or memcpy's occur during data fetches.
As such, the library is extremely simple because it requires no page caching
layer of its own, and it is extremely high performance and memory-efficient.
It is also fully transactional with full ACID semantics, and when the memory
map is read-only, the database integrity cannot be corrupted by stray pointer
writes from application code.
The library is fully thread-aware and supports concurrent read/write access
from multiple processes and threads. Data pages use a copy-on-write strategy
so no active data pages are ever overwritten, which also provides resistance
to corruption and eliminates the need of any special recovery procedures
after a system crash. Writes are fully serialized; only one write transaction
may be active at a time, which guarantees that writers can never deadlock.
The database structure is multi-versioned so readers run with no locks;
writers cannot block readers, and readers don't block writers.
Unlike other well-known database mechanisms which use either write-ahead
transaction logs or append-only data writes, MDBX requires no maintenance
during operation. Both write-ahead loggers and append-only databases require
periodic checkpointing and/or compaction of their log or database files
otherwise they grow without bound. MDBX tracks retired/freed pages within the
database and re-uses them for new write operations, so the database size does
not grow without bound in normal use. It is worth noting that the "next"
version libmdbx (MithrilDB) will solve this problem.
The memory map can be used as a read-only or read-write map. It is read-only
by default as this provides total immunity to corruption. Using read-write
mode offers much higher write performance, but adds the possibility for stray
application writes thru pointers to silently corrupt the database.
Of course if your application code is known to be bug-free (...) then this is
not an issue.
If this is your first time using a transactional embedded key-value store,
you may find the \ref starting section below to be helpful.

174
docs/_restrictions.md Normal file
View File

@@ -0,0 +1,174 @@
Restrictions & Caveats {#restrictions}
======================
In addition to those listed for some functions.
## Troubleshooting the LCK-file
1. A broken LCK-file can cause sync issues, including appearance of
wrong/inconsistent data for readers. When database opened in the
cooperative read-write mode the LCK-file requires to be mapped to
memory in read-write access. In this case it is always possible for
stray/malfunctioned application could writes thru pointers to
silently corrupt the LCK-file.
Unfortunately, there is no any portable way to prevent such
corruption, since the LCK-file is updated concurrently by
multiple processes in a lock-free manner and any locking is
unwise due to a large overhead.
The "next" version of libmdbx (MithrilDB) will solve this issue.
\note Workaround: Just make all programs using the database close it;
the LCK-file is always reset on first open.
2. Stale reader transactions left behind by an aborted program cause
further writes to grow the database quickly, and stale locks can
block further operation.
MDBX checks for stale readers while opening environment and before
growth the database. But in some cases, this may not be enough.
\note Workaround: Check for stale readers periodically, using the
`mdbx_reader_check()` function or the mdbx_stat tool.
3. Stale writers will be cleared automatically by MDBX on supprted
platforms. But this is platform-specific, especially of
implementation of shared POSIX-mutexes and support for robust
mutexes. For instance there are no known issues on Linux, OSX,
Windows and FreeBSD.
\note Workaround: Otherwise just make all programs using the database
close it; the LCK-file is always reset on first open
of the environment.
## Remote filesystems
Do not use MDBX databases on remote filesystems, even between processes
on the same host. This breaks file locks on some platforms, possibly
memory map sync, and certainly sync between programs on different hosts.
On the other hand, MDBX support the exclusive database operation over
a network, and cooperative read-only access to the database placed on
a read-only network shares.
## Child processes
Do not use opened `MDBX_env` instance(s) in a child processes after `fork()`.
It would be insane to call fork() and any MDBX-functions simultaneously
from multiple threads. The best way is to prevent the presence of open
MDBX-instances during `fork()`.
The `MDBX_TXN_CHECKPID` build-time option, which is ON by default on
non-Windows platforms (i.e. where `fork()` is available), enables PID
checking at a few critical points. But this does not give any guarantees,
but only allows you to detect such errors a little sooner. Depending on
the platform, you should expect an application crash and/or database
corruption in such cases.
On the other hand, MDBX allow calling `mdbx_close_env()` in such cases to
release resources, but no more and in general this is a wrong way.
## Read-only mode
There is no pure read-only mode in a normal explicitly way, since
readers need write access to LCK-file to be ones visible for writer.
So MDBX always tries to open/create LCK-file for read-write, but switches
to without-LCK mode on appropriate errors (`EROFS`, `EACCESS`, `EPERM`)
if the read-only mode was requested by the `MDBX_RDONLY` flag which is
described below.
The "next" version of libmdbx (MithrilDB) will solve this issue for the "many
readers without writer" case.
## One thread - One transaction
A thread can only use one transaction at a time, plus any nested
read-write transactions in the non-writemap mode. Each transaction
belongs to one thread. The `MDBX_NOTLS` flag changes this for read-only
transactions. See below.
Do not start more than one transaction for a one thread. If you think
about this, it's really strange to do something with two data snapshots
at once, which may be different. MDBX checks and preventing this by
returning corresponding error code (`MDBX_TXN_OVERLAPPING`, `MDBX_BAD_RSLOT`,
`MDBX_BUSY`) unless you using `MDBX_NOTLS` option on the environment.
Nonetheless, with the `MDBX_NOTLS` option, you must know exactly what you
are doing, otherwise you will get deadlocks or reading an alien data.
## Do not open twice
Do not have open an MDBX database twice in the same process at the same
time. By default MDBX prevent this in most cases by tracking databases
opening and return `MDBX_BUSY` if anyone LCK-file is already open.
The reason for this is that when the "Open file description" locks (aka
OFD-locks) are not available, MDBX uses POSIX locks on files, and these
locks have issues if one process opens a file multiple times. If a single
process opens the same environment multiple times, closing it once will
remove all the locks held on it, and the other instances will be
vulnerable to corruption from other processes.
For compatibility with LMDB which allows multi-opening, MDBX can be
configured at runtime by `mdbx_setup_debug(MDBX_DBG_LEGACY_MULTIOPEN, ...)`
prior to calling other MDBX funcitons. In this way MDBX will track
databases opening, detect multi-opening cases and then recover POSIX file
locks as necessary. However, lock recovery can cause unexpected pauses,
such as when another process opened the database in exclusive mode before
the lock was restored - we have to wait until such a process releases the
database, and so on.
## Long-lived read transactions
Avoid long-lived read transactions, especially in the scenarios with a
high rate of write transactions. Long-lived read transactions prevents
recycling pages retired/freed by newer write transactions, thus the
database can grow quickly.
Understanding the problem of long-lived read transactions requires some
explanation, but can be difficult for quick perception. So is is
reasonable to simplify this as follows:
1. Garbage collection problem exists in all databases one way or
another, e.g. VACUUM in PostgreSQL. But in MDBX it's even more
discernible because of high transaction rate and intentional
internals simplification in favor of performance.
2. MDBX employs Multiversion concurrency control on the Copy-on-Write
basis, that allows multiple readers runs in parallel with a write
transaction without blocking. An each write transaction needs free
pages to put the changed data, that pages will be placed in the new
b-tree snapshot at commit. MDBX efficiently recycling pages from
previous created unused snapshots, BUT this is impossible if anyone
a read transaction use such snapshot.
3. Thus massive altering of data during a parallel long read operation
will increase the process's work set and may exhaust entire free
database space.
A good example of long readers is a hot backup to the slow destination
or debugging of a client application while retaining an active read
transaction. LMDB this results in `MDBX_MAP_FULL` error and subsequent write
performance degradation.
MDBX mostly solve "long-lived" readers issue by the lack-of-space callback
which allow to aborts long readers, and by the `MDBX_LIFORECLAIM` mode which
addresses subsequent performance degradation.
The "next" version of libmdbx (MithrilDB) will completely solve this.
- Avoid suspending a process with active transactions. These would then be
"long-lived" as above.
The "next" version of libmdbx (MithrilDB) will solve this issue.
- Avoid aborting a process with an active read-only transaction in scenaries
with high rate of write transactions. The transaction becomes "long-lived"
as above until a check for stale readers is performed or the LCK-file is
reset, since the process may not remove it from the lockfile. This does
not apply to write transactions if the system clears stale writers, see
above.
## Space reservation
An MDBX database configuration will often reserve considerable unused
memory address space and maybe file size for future growth. This does
not use actual memory or disk space, but users may need to understand
the difference so they won't be scared off.
\todo To write about the Read/Write Amplification Factors

241
docs/_starting.md Normal file
View File

@@ -0,0 +1,241 @@
Getting started {#starting}
===============
> This section is based on Bert Hubert's intro "LMDB Semantics", with
> edits reflecting the improvements and enhancements were made in MDBX.
> See Bert Hubert's [original](https://github.com/ahupowerdns/ahutils/blob/master/lmdb-semantics.md).
Everything starts with an environment, created by \ref mdbx_env_create().
Once created, this environment must also be opened with \ref mdbx_env_open(),
and after use be closed by \ref mdbx_env_close(). At that a non-zero value
of the last argument "mode" supposes MDBX will create database and directory
if ones does not exist. In this case the non-zero "mode" argument specifies
the file mode bits be applied when a new files are created by `open()` function.
Within that directory, a lock file (aka LCK-file) and a storage file (aka
DXB-file) will be generated. If you don't want to use a directory, you can
pass the \ref MDBX_NOSUBDIR option, in which case the path you provided is used
directly as the DXB-file, and another file with a "-lck" suffix added
will be used for the LCK-file.
Once the environment is open, a transaction can be created within it using
\ref mdbx_txn_begin(). Transactions may be read-write or read-only, and read-write
transactions may be nested. A transaction must only be used by one thread at
a time. Transactions are always required, even for read-only access. The
transaction provides a consistent view of the data.
Once a transaction has been created, a database (i.e. key-value space inside
the environment) can be opened within it using \ref mdbx_dbi_open(). If only one
database will ever be used in the environment, a `NULL` can be passed as the
database name. For named databases, the \ref MDBX_CREATE flag must be used to
create the database if it doesn't already exist. Also, \ref mdbx_env_set_maxdbs()
must be called after \ref mdbx_env_create() and before \ref mdbx_env_open() to set
the maximum number of named databases you want to support.
\note A single transaction can open multiple databases. Generally databases
should only be opened once, by the first transaction in the process.
Within a transaction, \ref mdbx_get() and \ref mdbx_put() can store single key-value
pairs if that is all you need to do (but see \ref Cursors below if you want to do
more).
A key-value pair is expressed as two \ref MDBX_val structures. This struct that is
exactly similar to POSIX's `struct iovec` and has two fields, `iov_len` and
`iov_base`. The data is a `void` pointer to an array of `iov_len` bytes.
\note The notable difference between MDBX and LMDB is that MDBX support zero
length keys.
Because MDBX is very efficient (and usually zero-copy), the data returned in
an \ref MDBX_val structure may be memory-mapped straight from disk. In other words
look but do not touch (or `free()` for that matter). Once a transaction is
closed, the values can no longer be used, so make a copy if you need to keep
them after that.
## Cursors {#Cursors}
To do more powerful things, we must use a cursor.
Within the transaction, a cursor can be created with \ref mdbx_cursor_open().
With this cursor we can store/retrieve/delete (multiple) values using
\ref mdbx_cursor_get(), \ref mdbx_cursor_put() and \ref mdbx_cursor_del().
The \ref mdbx_cursor_get() positions itself depending on the cursor operation
requested, and for some operations, on the supplied key. For example, to list
all key-value pairs in a database, use operation \ref MDBX_FIRST for the first
call to \ref mdbx_cursor_get(), and \ref MDBX_NEXT on subsequent calls, until
the end is hit.
To retrieve all keys starting from a specified key value, use \ref MDBX_SET. For
more cursor operations, see the \ref c_api reference.
When using \ref mdbx_cursor_put()\ref , either the function will position the cursor
for you based on the key, or you can use operation \ref MDBX_CURRENT to use the
current position of the cursor. \note Note that key must then match the current
position's key.
## Summarizing the opening
So we have a cursor in a transaction which opened a database in an
environment which is opened from a filesystem after it was separately
created.
Or, we create an environment, open it from a filesystem, create a transaction
within it, open a database within that transaction, and create a cursor
within all of the above.
Got it?
## Threads and processes
Do not have open an database twice in the same process at the same time, MDBX
will track and prevent this. Instead, share the MDBX environment that has
opened the file across all threads. The reason for this is:
- When the "Open file description" locks (aka OFD-locks) are not available,
MDBX uses POSIX locks on files, and these locks have issues if one process
opens a file multiple times.
- If a single process opens the same environment multiple times, closing it
once will remove all the locks held on it, and the other instances will be
vulnerable to corruption from other processes.
+ For compatibility with LMDB which allows multi-opening, MDBX can be
configured at runtime by \ref mdbx_setup_debug() with \ref MDBX_DBG_LEGACY_MULTIOPEN` option
prior to calling other MDBX funcitons. In this way MDBX will track
databases opening, detect multi-opening cases and then recover POSIX file
locks as necessary. However, lock recovery can cause unexpected pauses,
such as when another process opened the database in exclusive mode before
the lock was restored - we have to wait until such a process releases the
database, and so on.
Do not use opened MDBX environment(s) after `fork()` in a child process(es),
MDBX will check and prevent this at critical points. Instead, ensure there is
no open MDBX-instance(s) during fork(), or atleast close it immediately after
`fork()` in the child process and reopen if required - for instance by using
`pthread_atfork()`. The reason for this is:
- For competitive consistent reading, MDBX assigns a slot in the shared
table for each process that interacts with the database. This slot is
populated with process attributes, including the PID.
- After `fork()`, in order to remain connected to a database, the child
process must have its own such "slot", which can't be assigned in any
simple and robust way another than the regular.
- A write transaction from a parent process cannot continue in a child
process for obvious reasons.
- Moreover, in a multithreaded process at the fork() moment any number of
threads could run in critical and/or intermediate sections of MDBX code
with interaction and/or racing conditions with threads from other
process(es). For instance: shrinking a database or copying it to a pipe,
opening or closing environment, begining or finishing a transaction,
and so on.
= Therefore, any solution other than simply close database (and reopen if
necessary) in a child process would be both extreme complicated and so
fragile.
Do not start more than one transaction for a one thread. If you think about
this, it's really strange to do something with two data snapshots at once,
which may be different. MDBX checks and preventing this by returning
corresponding error code (\ref MDBX_TXN_OVERLAPPING, \ref MDBX_BAD_RSLOT, \ref MDBX_BUSY)
unless you using \ref MDBX_NOTLS option on the environment. Nonetheless, with the
\ref MDBX_NOTLS option, you must know exactly what you are doing, otherwise you
will get deadlocks or reading an alien data.
Also note that a transaction is tied to one thread by default using Thread
Local Storage. If you want to pass read-only transactions across threads,
you can use the \ref MDBX_NOTLS option on the environment. Nevertheless, a write
transaction entirely should only be used in one thread from start to finish.
MDBX checks this in a reasonable manner and return the \ref MDBX_THREAD_MISMATCH
error in rules violation.
## Transactions, rollbacks etc
To actually get anything done, a transaction must be committed using
\ref mdbx_txn_commit(). Alternatively, all of a transaction's operations
can be discarded using \ref mdbx_txn_abort().
\attention An important difference between MDBX and LMDB is that MDBX required
that any opened cursors can be reused and must be freed explicitly, regardless
ones was opened in a read-only or write transaction. The REASON for this is
eliminates ambiguity which helps to avoid errors such as: use-after-free,
double-free, i.e. memory corruption and segfaults.
For read-only transactions, obviously there is nothing to commit to storage.
\attention An another notable difference between MDBX and LMDB is that MDBX make
handles opened for existing databases immediately available for other
transactions, regardless this transaction will be aborted or reset. The
REASON for this is to avoiding the requirement for multiple opening a same
handles in concurrent read transactions, and tracking of such open but hidden
handles until the completion of read transactions which opened them.
In addition, as long as a transaction is open, a consistent view of the
database is kept alive, which requires storage. A read-only transaction that
no longer requires this consistent view should be terminated (committed or
aborted) when the view is no longer needed (but see below for an
optimization).
There can be multiple simultaneously active read-only transactions but only
one that can write. Once a single read-write transaction is opened, all
further attempts to begin one will block until the first one is committed or
aborted. This has no effect on read-only transactions, however, and they may
continue to be opened at any time.
## Duplicate keys aka Multi-values
\ref mdbx_get() and \ref mdbx_put() respectively have no and only some support or
multiple key-value pairs with identical keys. If there are multiple values
for a key, \ref mdbx_get() will only return the first value.
When multiple values for one key are required, pass the \ref MDBX_DUPSORT flag to
\ref mdbx_dbi_open(). In an \ref MDBX_DUPSORT database, by default \ref mdbx_put() will
not replace the value for a key if the key existed already. Instead it will add
the new value to the key. In addition, \ref mdbx_del() will pay attention to the
value field too, allowing for specific values of a key to be deleted.
Finally, additional cursor operations become available for traversing through
and retrieving duplicate values.
## Some optimization
If you frequently begin and abort read-only transactions, as an optimization,
it is possible to only reset and renew a transaction.
\ref mdbx_txn_reset() releases any old copies of data kept around for a read-only
transaction. To reuse this reset transaction, call \ref mdbx_txn_renew() on it.
Any cursors in this transaction can also be renewed using \ref mdbx_cursor_renew()
or freed by \ref mdbx_cursor_close().
To permanently free a transaction, reset or not, use \ref mdbx_txn_abort().
## Cleaning up
Any created cursors must be closed using \ref mdbx_cursor_close(). It is advisable
to repeat:
\note An important difference between MDBX and LMDB is that MDBX required that
any opened cursors can be reused and must be freed explicitly, regardless
ones was opened in a read-only or write transaction. The REASON for this is
eliminates ambiguity which helps to avoid errors such as: use-after-free,
double-free, i.e. memory corruption and segfaults.
It is very rarely necessary to close a database handle, and in general they
should just be left open. When you close a handle, it immediately becomes
unavailable for all transactions in the environment. Therefore, you should
avoid closing the handle while at least one transaction is using it.
## Now read up on the full API!
The full \ref c_api documentation lists further details below, like how to:
- Configure database size and automatic size management: \ref mdbx_env_set_geometry().
- Drop and clean a database: \ref mdbx_drop().
- Detect and report errors: \ref c_err.
- Optimize (bulk) loading speed: \ref MDBX_MULTIPLE, \ref MDBX_APPEND.
- Reduce (temporarily) robustness to gain even more speed: \ref sync_modes.
- Gather statistics about the database: \ref c_statinfo.
- Sstimate size of range query result: \ref c_rqest.
- Double perfomance by LIFO reclaiming on storages with write-back: \ref MDBX_LIFORECLAIM.
- Use sequences and canary markers: \ref mdbx_dbi_sequence(), \ref MDBX_canary.
- Use lack-of-space callback (aka OOM-KICK): \ref mdbx_env_set_oomfunc().
- Use exclusive mode: \ref MDBX_EXCLUSIVE.
- Define custom sort orders (but this is recommended to be avoided).

46
docs/_toc.md Normal file
View File

@@ -0,0 +1,46 @@
_The Future will (be) [Positive](https://www.ptsecurity.com). Всё будет хорошо._
\section toc Table of Contents
This manual is divided into parts,
each of which is divided into several sections.
1. The \ref intro
- \ref characteristics
- Preface
- Features
- Limitations
- Gotchas
- Comparison with other databases
- \ref restrictions
- \ref performance
- Integral performance
- Read Scalability
- Sync-write mode
- Lazy-write mode
- Async-write mode
- Cost comparison
2. \ref usage
- \ref getting
- Embedding
- Building
- \ref starting
- Opening
- Cursors
- Threads and processes
- Transactions
- Duplicate keys aka Multi-values
- Cleaning up
- \ref bindings
3. The `C` API manual:
- The \ref c_api reference
- The \ref mdbx.h header file reference
Please do not hesitate to point out errors in the documentation,
including creating [PR](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/proposing-changes-to-your-work-with-pull-requests) with corrections and improvements.
---
\section mithril Mithril DB

5277
mdbx.h

File diff suppressed because it is too large Load Diff

View File

@@ -9254,7 +9254,7 @@ int __cold mdbx_env_set_mapsize(MDBX_env *env, size_t size) {
}
int __cold mdbx_env_set_maxdbs(MDBX_env *env, MDBX_dbi dbs) {
if (unlikely(dbs > MAX_DBI))
if (unlikely(dbs > MDBX_MAX_DBI))
return MDBX_EINVAL;
if (unlikely(!env))
@@ -10028,26 +10028,6 @@ __cold int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy) {
: MDBX_RESULT_TRUE;
}
/* Only a subset of the mdbx_env flags can be changed
* at runtime. Changing other flags requires closing the
* environment and re-opening it with the new flags. */
#define ENV_CHANGEABLE_FLAGS \
(MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_MAPASYNC | MDBX_NOMEMINIT | \
MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE)
#define ENV_CHANGELESS_FLAGS \
(MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \
MDBX_LIFORECLAIM | MDBX_EXCLUSIVE)
#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS)
#if ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS
#error "Opps, some flags overlapped or wrong"
#endif
#if (MDBX_ACCEDE | MDBX_CREATE) != ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & \
(ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS))
#error "Opps, some flags overlapped or wrong"
#endif
/* Merge flags and avoid false MDBX_UTTERLY_NOSYNC */
static uint32_t merge_flags(const uint32_t a, const uint32_t b) {
uint32_t r = a | b;
@@ -15728,7 +15708,7 @@ static int __cold mdbx_env_compact(MDBX_env *env, MDBX_txn *read_txn,
MDBX_meta *const meta = mdbx_init_metas(env, buffer);
mdbx_meta_set_txnid(env, meta, read_txn->mt_txnid);
if (flags & MDBX_CP_FORCE_RESIZEABLE)
if (flags & MDBX_CP_FORCE_DYNAMIC_SIZE)
make_sizeable(meta);
/* copy canary sequenses if present */
@@ -15884,7 +15864,7 @@ static int __cold mdbx_env_copy_asis(MDBX_env *env, MDBX_txn *read_txn,
(MDBX_meta *)(buffer + ((uint8_t *)mdbx_meta_head(env) - env->me_map));
mdbx_txn_unlock(env);
if (flags & MDBX_CP_FORCE_RESIZEABLE)
if (flags & MDBX_CP_FORCE_DYNAMIC_SIZE)
make_sizeable(headcopy);
/* Update signature to steady */
headcopy->mm_datasync_sign = mdbx_meta_sign(headcopy);
@@ -16452,9 +16432,9 @@ static int mdbx_dbi_bind(MDBX_txn *txn, const MDBX_dbi dbi, unsigned user_flags,
return MDBX_SUCCESS;
}
int mdbx_dbi_open_ex(MDBX_txn *txn, const char *table_name, unsigned user_flags,
MDBX_dbi *dbi, MDBX_cmp_func *keycmp,
MDBX_cmp_func *datacmp) {
static int dbi_open(MDBX_txn *txn, const char *table_name, unsigned user_flags,
MDBX_dbi *dbi, MDBX_cmp_func *keycmp,
MDBX_cmp_func *datacmp) {
int rc = MDBX_EINVAL;
if (unlikely(!dbi))
return rc;
@@ -16673,7 +16653,13 @@ int mdbx_dbi_open_ex(MDBX_txn *txn, const char *table_name, unsigned user_flags,
int mdbx_dbi_open(MDBX_txn *txn, const char *table_name, unsigned table_flags,
MDBX_dbi *dbi) {
return mdbx_dbi_open_ex(txn, table_name, table_flags, dbi, nullptr, nullptr);
return dbi_open(txn, table_name, table_flags, dbi, nullptr, nullptr);
}
int mdbx_dbi_open_ex(MDBX_txn *txn, const char *table_name,
unsigned table_flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp,
MDBX_cmp_func *datacmp) {
return dbi_open(txn, table_name, table_flags, dbi, keycmp, datacmp);
}
int __cold mdbx_dbi_stat(MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *dest,
@@ -17680,7 +17666,7 @@ int __cold mdbx_env_pgwalk(MDBX_txn *txn, MDBX_pgvisitor_func *visitor,
return rc;
}
int mdbx_canary_put(MDBX_txn *txn, const mdbx_canary *canary) {
int mdbx_canary_put(MDBX_txn *txn, const MDBX_canary *canary) {
int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
@@ -17699,7 +17685,7 @@ int mdbx_canary_put(MDBX_txn *txn, const mdbx_canary *canary) {
return MDBX_SUCCESS;
}
int mdbx_canary_get(const MDBX_txn *txn, mdbx_canary *canary) {
int mdbx_canary_get(const MDBX_txn *txn, MDBX_canary *canary) {
int rc = check_txn(txn, MDBX_TXN_BLOCKED);
if (unlikely(rc != MDBX_SUCCESS))
return rc;
@@ -18834,7 +18820,7 @@ __dll_export
__has_attribute(__externally_visible__)
__attribute__((__externally_visible__))
#endif
const mdbx_build_info mdbx_build = {
const struct MDBX_build_info mdbx_build = {
#ifdef MDBX_BUILD_TIMESTAMP
MDBX_BUILD_TIMESTAMP
#else

View File

@@ -90,6 +90,11 @@
#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized array in struct/union */
#endif /* _MSC_VER (warnings) */
#if defined(MDBX_TOOLS)
#undef MDBX_DEPRECATED
#define MDBX_DEPRECATED
#endif /* MDBX_TOOLS */
#include "../mdbx.h"
#include "defs.h"
@@ -190,10 +195,6 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor;
#define MAIN_DBI 1
/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */
#define CORE_DBS 2
#define MAX_DBI (INT16_MAX - CORE_DBS)
#if MAX_DBI != MDBX_MAX_DBI
#error "Opps, MAX_DBI != MDBX_MAX_DBI"
#endif
/* Number of meta pages - also hardcoded elsewhere */
#define NUM_METAS 3
@@ -303,7 +304,7 @@ typedef struct MDBX_meta {
#define mm_psize mm_dbs[FREE_DBI].md_xsize
/* Any persistent environment flags, see mdbx_env */
#define mm_flags mm_dbs[FREE_DBI].md_flags
mdbx_canary mm_canary;
MDBX_canary mm_canary;
#define MDBX_DATASIGN_NONE 0u
#define MDBX_DATASIGN_WEAK 1u
@@ -735,7 +736,7 @@ struct MDBX_txn {
#if (TXN_FLAGS & MDBX_TXN_BEGIN_FLAGS) || \
((MDBX_TXN_BEGIN_FLAGS | TXN_FLAGS) & MDBX_SHRINK_ALLOWED)
#error "Opps, some flags overlapped or wrong"
#error "Oops, some flags overlapped or wrong"
#endif
unsigned mt_flags;
@@ -769,7 +770,7 @@ struct MDBX_txn {
* don't decrement it when individual DB handles are closed. */
MDBX_dbi mt_numdbs;
size_t mt_owner; /* thread ID that owns this transaction */
mdbx_canary mt_canary;
MDBX_canary mt_canary;
union {
struct {
@@ -1310,10 +1311,10 @@ typedef struct MDBX_node {
#define DB_INTERNAL_FLAGS DB_VALID
#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS
#error "Opps, some flags overlapped or wrong"
#error "Oops, some flags overlapped or wrong"
#endif
#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS
#error "Opps, some flags overlapped or wrong"
#error "Oops, some flags overlapped or wrong"
#endif
/* max number of pages to commit in one writev() call */
@@ -1374,3 +1375,25 @@ static __pure_function __always_inline __maybe_unused size_t
ceil_powerof2(size_t value, size_t granularity) {
return floor_powerof2(value + granularity - 1, granularity);
}
/* Only a subset of the mdbx_env flags can be changed
* at runtime. Changing other flags requires closing the
* environment and re-opening it with the new flags. */
#define ENV_CHANGEABLE_FLAGS \
(MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_MAPASYNC | MDBX_NOMEMINIT | \
MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE)
#define ENV_CHANGELESS_FLAGS \
(MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \
MDBX_LIFORECLAIM | MDBX_EXCLUSIVE)
#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS)
static __maybe_unused void static_checks(void) {
STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI,
"Oops, MDBX_MAX_DBI or CORE_DBS?");
STATIC_ASSERT_MSG((MDBX_ACCEDE | MDBX_CREATE) ==
((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) &
(ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)),
"Oops, some flags overlapped or wrong");
STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0,
"Oops, some flags overlapped or wrong");
}

View File

@@ -1,6 +1,6 @@
.\" Copyright 2015-2020 Leonid Yuriev <leo@yuriev.ru>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_CHK 1 "2020-06-05" "MDBX 0.8.x"
.TH MDBX_CHK 1 "2020-07-31" "MDBX 0.9.x"
.SH NAME
mdbx_chk \- MDBX checking tool
.SH SYNOPSIS

View File

@@ -2,7 +2,7 @@
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_COPY 1 "2020-06-05" "MDBX 0.8.x"
.TH MDBX_COPY 1 "2020-07-31" "MDBX 0.9.x"
.SH NAME
mdbx_copy \- MDBX environment copy tool
.SH SYNOPSIS

View File

@@ -2,7 +2,7 @@
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_DUMP 1 "2020-06-05" "MDBX 0.8.x"
.TH MDBX_DUMP 1 "2020-07-31" "MDBX 0.9.x"
.SH NAME
mdbx_dump \- MDBX environment export tool
.SH SYNOPSIS

View File

@@ -2,7 +2,7 @@
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_LOAD 1 "2020-06-05" "MDBX 0.8.x"
.TH MDBX_LOAD 1 "2020-07-31" "MDBX 0.9.x"
.SH NAME
mdbx_load \- MDBX environment import tool
.SH SYNOPSIS

View File

@@ -2,7 +2,7 @@
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_STAT 1 "2020-06-05" "MDBX 0.8.x"
.TH MDBX_STAT 1 "2020-07-31" "MDBX 0.9.x"
.SH NAME
mdbx_stat \- MDBX environment status tool
.SH SYNOPSIS

View File

@@ -77,7 +77,8 @@ struct {
short *pagemap;
uint64_t total_payload_bytes;
uint64_t pgcount;
walk_dbi_t dbi[MAX_DBI + CORE_DBS + /* account pseudo-entry for meta */ 1];
walk_dbi_t
dbi[MDBX_MAX_DBI + CORE_DBS + /* account pseudo-entry for meta */ 1];
} walk;
#define dbi_free walk.dbi[FREE_DBI]
@@ -1032,7 +1033,7 @@ int main(int argc, char *argv[]) {
return rc < 0 ? EXIT_FAILURE_MDB : EXIT_FAILURE_SYS;
}
rc = mdbx_env_set_maxdbs(env, MAX_DBI);
rc = mdbx_env_set_maxdbs(env, MDBX_MAX_DBI);
if (rc) {
error("mdbx_env_set_maxdbs failed, error %d %s\n", rc, mdbx_strerror(rc));
goto bailout;
@@ -1274,7 +1275,9 @@ int main(int argc, char *argv[]) {
}
saved_list = problems_push();
rc = mdbx_env_pgwalk(txn, pgvisitor, nullptr, ignore_wrong_order);
rc = mdbx_env_pgwalk(txn, pgvisitor, nullptr,
true /* always skip key ordering checking to avoid
MDBX_CORRUPTED when using custom comparators */);
traversal_problems = problems_pop(saved_list);
if (rc) {

View File

@@ -138,7 +138,7 @@ static int dump_sdb(MDBX_txn *txn, MDBX_dbi dbi, char *name) {
printf("mapsize=%" PRIu64 "\n", info.mi_geo.upper);
printf("maxreaders=%u\n", info.mi_maxreaders);
mdbx_canary canary;
MDBX_canary canary;
rc = mdbx_canary_get(txn, &canary);
if (unlikely(rc != MDBX_SUCCESS)) {
error("mdbx_canary_get", rc);

View File

@@ -109,7 +109,7 @@ static char *subname = nullptr;
static int dbi_flags;
static txnid_t txnid;
static uint64_t sequence;
static mdbx_canary canary;
static MDBX_canary canary;
static MDBX_envinfo envinfo;
#define PRINT 1
@@ -201,6 +201,7 @@ static int readhdr(void) {
str = valstr(dbuf.iov_base, "database");
if (str) {
if (*str) {
free(subname);
subname = mdbx_strdup(str);
if (!subname) {
perror("strdup()");
@@ -216,6 +217,7 @@ static int readhdr(void) {
fprintf(stderr,
"%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", prog,
lineno, str, "type");
free(subname);
exit(EXIT_FAILURE);
}
continue;
@@ -700,18 +702,13 @@ int main(int argc, char *argv[]) {
int batch = 0;
prevk.iov_len = 0;
while (rc == MDBX_SUCCESS) {
MDBX_val key;
MDBX_val key, data;
rc = readline(&key, &kbuf);
if (rc != MDBX_SUCCESS) /* rc == EOF */
if (rc == EOF)
break;
if (user_break) {
rc = MDBX_EINTR;
break;
}
MDBX_val data;
rc = readline(&data, &dbuf);
if (rc == MDBX_SUCCESS)
rc = readline(&data, &dbuf);
if (rc) {
fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n",
prog, lineno);

View File

@@ -13,21 +13,51 @@
*
*/
/* Support for huge write-transactions */
/** \defgroup build_option Build options
* The libmdbx build options.
@{ */
#ifdef DOXYGEN
/* !!! Actually this is a fake definitions !!!
* !!! for documentation generation by Doxygen !!! */
/** Controls enabling of debugging features.
*
* - `MDBX_DEBUG = 0` (by default) Disables any debugging features at all,
* including logging and assertion controls.
* Logging level and corresponding debug flags changing
* by \ref mdbx_setup_debug() will not have effect.
* - `MDBX_DEBUG > 0` Enables code for the debugging features (logging,
* assertions checking and internal audit).
* Simultaneously sets the default logging level
* to the `MDBX_DEBUG` value.
* Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`.
*
* \ingroup build_option */
#define MDBX_DEBUG 0...7
/** Disables using of GNU libc extensions. */
#define MDBX_DISABLE_GNU_SOURCE 0 or 1
#endif /* DOXYGEN */
/** Enables support for huge write-transactions */
#ifndef MDBX_HUGE_TRANSACTIONS
#define MDBX_HUGE_TRANSACTIONS 0
#endif /* MDBX_HUGE_TRANSACTIONS */
/* using fcntl(F_FULLFSYNC) with 5-10 times slowdown */
/** Using fcntl(F_FULLFSYNC) with 5-10 times slowdown */
#define MDBX_OSX_WANNA_DURABILITY 0
/* using fsync() with chance of data lost on power failure */
/** Using fsync() with chance of data lost on power failure */
#define MDBX_OSX_WANNA_SPEED 1
#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY
/** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED
* for OSX & iOS */
#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY
#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */
/* Controls checking PID against reuse DB environment after the fork() */
/** Controls checking PID against reuse DB environment after the fork() */
#ifndef MDBX_TXN_CHECKPID
#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64)
/* PID check could be ommited:
@@ -43,7 +73,7 @@
#define MDBX_TXN_CHECKPID_CONFIG STRINGIFY(MDBX_TXN_CHECKPID)
#endif /* MDBX_TXN_CHECKPID */
/* Controls checking transaction owner thread against misuse transactions from
/** Controls checking transaction owner thread against misuse transactions from
* other threads. */
#ifndef MDBX_TXN_CHECKOWNER
#define MDBX_TXN_CHECKOWNER 1
@@ -52,7 +82,7 @@
#define MDBX_TXN_CHECKOWNER_CONFIG STRINGIFY(MDBX_TXN_CHECKOWNER)
#endif /* MDBX_TXN_CHECKOWNER */
/* Does a system have battery-backed Real-Time Clock or just a fake. */
/** Does a system have battery-backed Real-Time Clock or just a fake. */
#ifndef MDBX_TRUST_RTC
#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \
defined(__OpenBSD__)
@@ -67,13 +97,25 @@
//------------------------------------------------------------------------------
#define MDBX_LOCKING_WIN32FILES -1 /* Win32 File Locking API */
#define MDBX_LOCKING_SYSV 5 /* SystemV IPC semaphores */
#define MDBX_LOCKING_POSIX1988 1988 /* POSIX-1 Shared anonymous semaphores */
#define MDBX_LOCKING_POSIX2001 2001 /* POSIX-2001 Shared Mutexes */
#define MDBX_LOCKING_POSIX2008 2008 /* POSIX-2008 Robust Mutexes */
#define MDBX_LOCKING_BENAPHORE 1995 /* BeOS Benaphores, aka Futexes */
/** Win32 File Locking API for \ref MDBX_LOCKING */
#define MDBX_LOCKING_WIN32FILES -1
/** SystemV IPC semaphores for \ref MDBX_LOCKING */
#define MDBX_LOCKING_SYSV 5
/** POSIX-1 Shared anonymous semaphores for \ref MDBX_LOCKING */
#define MDBX_LOCKING_POSIX1988 1988
/** POSIX-2001 Shared Mutexes for \ref MDBX_LOCKING */
#define MDBX_LOCKING_POSIX2001 2001
/** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */
#define MDBX_LOCKING_POSIX2008 2008
/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */
#define MDBX_LOCKING_BENAPHORE 1995
/** Advanced: Choices the locking implementation (autodetection by default). */
#if defined(_WIN32) || defined(_WIN64)
#define MDBX_LOCKING MDBX_LOCKING_WIN32FILES
#else
@@ -106,6 +148,7 @@
#endif /* MDBX_LOCKING */
#endif /* !Windows */
/** Advanced: Using POSIX OFD-locks (autodetection by default). */
#ifndef MDBX_USE_OFDLOCKS
#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \
!defined(MDBX_SAFE4QEMU) && \
@@ -123,7 +166,7 @@
#ifndef MDBX_CPU_WRITEBACK_INCOHERENT
#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
defined(__hppa__)
defined(__hppa__) || defined(DOXYGEN)
#define MDBX_CPU_WRITEBACK_INCOHERENT 0
#else
#define MDBX_CPU_WRITEBACK_INCOHERENT 1
@@ -151,7 +194,7 @@
#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */
#ifndef MDBX_64BIT_ATOMIC
#if MDBX_WORDBITS >= 64
#if MDBX_WORDBITS >= 64 || defined(DOXYGEN)
#define MDBX_64BIT_ATOMIC 1
#else
#define MDBX_64BIT_ATOMIC 0
@@ -180,7 +223,7 @@
#else
#define MDBX_64BIT_CAS 0
#endif
#elif defined(_MSC_VER) || defined(__APPLE__)
#elif defined(_MSC_VER) || defined(__APPLE__) || defined(DOXYGEN)
#define MDBX_64BIT_CAS 1
#else
#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC
@@ -213,6 +256,7 @@
#endif
#endif /* MDBX_CACHELINE_SIZE */
/** @} end of build options */
/*******************************************************************************
*******************************************************************************
******************************************************************************/

View File

@@ -1931,7 +1931,9 @@ __cold MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void) {
(fstatfs(fd, &fs) == 0 && fs.f_type == /* procfs */ 0x9FA0)
? read(fd, buf, sizeof(buf))
: -1;
close(fd);
const int err = close(fd);
assert(err == 0);
(void)err;
if (len > 0 && bootid_parse_uuid(&bin, buf, len))
return bin;
}

View File

@@ -22,7 +22,7 @@ __dll_export
__has_attribute(__externally_visible__)
__attribute__((__externally_visible__))
#endif
const mdbx_version_info mdbx_version = {
const struct MDBX_version_info mdbx_version = {
${MDBX_VERSION_MAJOR},
${MDBX_VERSION_MINOR},
${MDBX_VERSION_RELEASE},

View File

@@ -80,8 +80,10 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &mask, const option_verb *verbs) {
template <>
bool parse_option<unsigned>(int argc, char *const argv[], int &narg,
const char *option, unsigned &mask,
const option_verb *verbs) {
const char *list;
if (!parse_option(argc, argv, narg, option, &list))
return false;
@@ -213,7 +215,7 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval,
default_value))
return false;
value = (unsigned)huge;
value = unsigned(huge);
return true;
}
@@ -225,18 +227,18 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
if (!parse_option(argc, argv, narg, option, huge, no_scale, minval, maxval,
default_value))
return false;
value = (uint8_t)huge;
value = uint8_t(huge);
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int64_t &value, const int64_t minval, const int64_t maxval,
const int64_t default_value) {
uint64_t proxy = (uint64_t)value;
uint64_t proxy = uint64_t(value);
if (parse_option(argc, argv, narg, option, proxy, config::binary,
(uint64_t)minval, (uint64_t)maxval,
(uint64_t)default_value)) {
value = (int64_t)proxy;
uint64_t(minval), uint64_t(maxval),
uint64_t(default_value))) {
value = int64_t(proxy);
return true;
}
return false;
@@ -245,11 +247,11 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int32_t &value, const int32_t minval, const int32_t maxval,
const int32_t default_value) {
uint64_t proxy = (uint64_t)value;
uint64_t proxy = uint64_t(value);
if (parse_option(argc, argv, narg, option, proxy, config::binary,
(uint64_t)minval, (uint64_t)maxval,
(uint64_t)default_value)) {
value = (int32_t)proxy;
uint64_t(minval), uint64_t(maxval),
uint64_t(default_value))) {
value = int32_t(proxy);
return true;
}
return false;
@@ -294,29 +296,30 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option,
//-----------------------------------------------------------------------------
const struct option_verb mode_bits[] = {{"rdonly", MDBX_RDONLY},
{"mapasync", MDBX_MAPASYNC},
{"nosync-utterly", MDBX_UTTERLY_NOSYNC},
{"nosubdir", MDBX_NOSUBDIR},
{"nosync-safe", MDBX_SAFE_NOSYNC},
{"nometasync", MDBX_NOMETASYNC},
{"writemap", MDBX_WRITEMAP},
{"notls", MDBX_NOTLS},
{"nordahead", MDBX_NORDAHEAD},
{"nomeminit", MDBX_NOMEMINIT},
{"coalesce", MDBX_COALESCE},
{"lifo", MDBX_LIFORECLAIM},
{"perturb", MDBX_PAGEPERTURB},
{"accede", MDBX_ACCEDE},
{nullptr, 0}};
const struct option_verb mode_bits[] = {
{"rdonly", unsigned(MDBX_RDONLY)},
{"mapasync", unsigned(MDBX_MAPASYNC)},
{"nosync-utterly", unsigned(MDBX_UTTERLY_NOSYNC)},
{"nosubdir", unsigned(MDBX_NOSUBDIR)},
{"nosync-safe", unsigned(MDBX_SAFE_NOSYNC)},
{"nometasync", unsigned(MDBX_NOMETASYNC)},
{"writemap", unsigned(MDBX_WRITEMAP)},
{"notls", unsigned(MDBX_NOTLS)},
{"nordahead", unsigned(MDBX_NORDAHEAD)},
{"nomeminit", unsigned(MDBX_NOMEMINIT)},
{"coalesce", unsigned(MDBX_COALESCE)},
{"lifo", unsigned(MDBX_LIFORECLAIM)},
{"perturb", unsigned(MDBX_PAGEPERTURB)},
{"accede", unsigned(MDBX_ACCEDE)},
{nullptr, 0}};
const struct option_verb table_bits[] = {
{"key.reverse", MDBX_REVERSEKEY},
{"key.integer", MDBX_INTEGERKEY},
{"data.integer", MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT},
{"data.fixed", MDBX_DUPFIXED | MDBX_DUPSORT},
{"data.reverse", MDBX_REVERSEDUP | MDBX_DUPSORT},
{"data.dups", MDBX_DUPSORT},
{"key.reverse", unsigned(MDBX_REVERSEKEY)},
{"key.integer", unsigned(MDBX_INTEGERKEY)},
{"data.integer", unsigned(MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT)},
{"data.fixed", unsigned(MDBX_DUPFIXED | MDBX_DUPSORT)},
{"data.reverse", unsigned(MDBX_REVERSEDUP | MDBX_DUPSORT)},
{"data.dups", unsigned(MDBX_DUPSORT)},
{nullptr, 0}};
static void dump_verbs(const char *caption, size_t bits,
@@ -590,7 +593,7 @@ unsigned actor_params::mdbx_keylen_min() const {
}
unsigned actor_params::mdbx_keylen_max() const {
return (unsigned)mdbx_limits_keysize_max(pagesize, table_flags);
return unsigned(mdbx_limits_keysize_max(pagesize, table_flags));
}
unsigned actor_params::mdbx_datalen_min() const {
@@ -598,6 +601,6 @@ unsigned actor_params::mdbx_datalen_min() const {
}
unsigned actor_params::mdbx_datalen_max() const {
return std::min((unsigned)UINT16_MAX,
(unsigned)mdbx_limits_valsize_max(pagesize, table_flags));
return std::min(unsigned(UINT16_MAX),
unsigned(mdbx_limits_valsize_max(pagesize, table_flags)));
}

View File

@@ -79,8 +79,22 @@ struct option_verb {
unsigned mask;
};
template <typename MASK>
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &mask, const option_verb *verbs);
MASK &mask, const option_verb *verbs) {
static_assert(sizeof(MASK) <= sizeof(unsigned), "WTF?");
unsigned u = unsigned(mask);
if (parse_option<unsigned>(argc, argv, narg, option, u, verbs)) {
mask = MASK(u);
return true;
}
return false;
}
template <>
bool parse_option<unsigned>(int argc, char *const argv[], int &narg,
const char *option, unsigned &mask,
const option_verb *verbs);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint64_t &value, const scale_mode scale,
@@ -236,8 +250,8 @@ struct keygen_params_pod {
};
struct actor_params_pod {
unsigned mode_flags{0};
unsigned table_flags{0};
MDBX_env_flags_t mode_flags{MDBX_ENV_DEFAULTS};
MDBX_db_flags_t table_flags{MDBX_DB_DEFAULTS};
intptr_t size_lower{0};
intptr_t size_now{0};
intptr_t size_upper{0};

View File

@@ -78,11 +78,12 @@ void __hot maker::pair(serial_t serial, const buffer &key, buffer &value,
assert(mapping.mesh <= mapping.width);
assert(mapping.rotate <= mapping.width);
assert(mapping.offset <= mask(mapping.width));
assert(
!(key_essentials.flags & ~(essentials::prng_fill_flag | MDBX_INTEGERKEY |
MDBX_REVERSEKEY | MDBX_DUPSORT)));
assert(!(key_essentials.flags &
~(essentials::prng_fill_flag |
unsigned(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT))));
assert(!(value_essentials.flags &
~(essentials::prng_fill_flag | MDBX_INTEGERDUP | MDBX_REVERSEDUP)));
~(essentials::prng_fill_flag |
unsigned(MDBX_INTEGERDUP | MDBX_REVERSEDUP))));
log_trace("keygen-pair: serial %" PRIu64 ", data-age %" PRIu64, serial,
value_age);
@@ -197,8 +198,17 @@ void __hot maker::pair(serial_t serial, const buffer &key, buffer &value,
void maker::setup(const config::actor_params_pod &actor, unsigned actor_id,
unsigned thread_number) {
#if !defined(_MSC_VER) || _MSC_VER > 1900
static_assert(unsigned(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT |
MDBX_INTEGERDUP | MDBX_REVERSEDUP) < UINT16_MAX,
"WTF?");
#else
assert(unsigned(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT |
MDBX_INTEGERDUP | MDBX_REVERSEDUP) < UINT16_MAX);
#endif
key_essentials.flags =
actor.table_flags & (MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT);
actor.table_flags &
uint16_t(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT);
assert(actor.keylen_min <= UINT16_MAX);
key_essentials.minlen = (uint16_t)actor.keylen_min;
assert(actor.keylen_max <= UINT32_MAX);
@@ -207,7 +217,7 @@ void maker::setup(const config::actor_params_pod &actor, unsigned actor_id,
(uint32_t)mdbx_limits_keysize_max(actor.pagesize, key_essentials.flags));
value_essentials.flags =
actor.table_flags & (MDBX_INTEGERDUP | MDBX_REVERSEDUP);
actor.table_flags & uint16_t(MDBX_INTEGERDUP | MDBX_REVERSEDUP);
assert(actor.datalen_min <= UINT16_MAX);
value_essentials.minlen = (uint16_t)actor.datalen_min;
assert(actor.datalen_max <= UINT32_MAX);
@@ -305,10 +315,17 @@ void __hot maker::mk_begin(const serial_t serial, const essentials &params,
void __hot maker::mk_continue(const serial_t serial, const essentials &params,
result &out) {
static_assert((essentials::prng_fill_flag &
(MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERKEY |
MDBX_INTEGERDUP | MDBX_REVERSEKEY | MDBX_REVERSEDUP)) == 0,
"WTF?");
#if !defined(_MSC_VER) || _MSC_VER > 1900
static_assert(
(essentials::prng_fill_flag &
unsigned(MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERKEY |
MDBX_INTEGERDUP | MDBX_REVERSEKEY | MDBX_REVERSEDUP)) == 0,
"WTF?");
#else
assert((essentials::prng_fill_flag &
unsigned(MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERKEY |
MDBX_INTEGERDUP | MDBX_REVERSEKEY | MDBX_REVERSEDUP)) == 0);
#endif
out.value.iov_base = out.bytes;
if (params.flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) {
assert(params.maxlen == params.minlen);
@@ -317,7 +334,7 @@ void __hot maker::mk_continue(const serial_t serial, const essentials &params,
out.u64 = serial;
else
out.u32 = (uint32_t)serial;
} else if (params.flags & (MDBX_REVERSEKEY | MDBX_REVERSEDUP)) {
} else if (params.flags & unsigned(MDBX_REVERSEKEY | MDBX_REVERSEDUP)) {
if (out.value.iov_len > 8) {
if (params.flags & essentials::prng_fill_flag) {
uint64_t state = serial ^ UINT64_C(0x41803711c9b75f19);

View File

@@ -37,8 +37,8 @@ void __noreturn failure_perror(const char *what, int errnum) {
//-----------------------------------------------------------------------------
static void mdbx_logger(int priority, const char *function, int line,
const char *msg, va_list args) {
static void mdbx_logger(MDBX_log_level_t priority, const char *function,
int line, const char *msg, va_list args) {
if (!function)
function = "unknown";
@@ -60,7 +60,7 @@ static FILE *last;
void setlevel(loglevel priority) {
level = priority;
int rc = mdbx_setup_debug(priority,
int rc = mdbx_setup_debug(MDBX_log_level_t(priority),
MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER,
mdbx_logger);
log_trace("set mdbx debug-opts: 0x%02x", rc);

View File

@@ -26,7 +26,7 @@
#include <atomic>
#ifndef MDBX_LOCKING
#error "Opps, MDBX_LOCKING is undefined!"
#error "Oops, MDBX_LOCKING is undefined!"
#endif
#if defined(__APPLE__) && (MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \

View File

@@ -411,7 +411,7 @@ bool testcase::should_continue(bool check_timeout_only) const {
}
void testcase::fetch_canary() {
mdbx_canary canary_now;
MDBX_canary canary_now;
log_trace(">> fetch_canary");
int rc = mdbx_canary_get(txn_guard.get(), &canary_now);
@@ -434,7 +434,7 @@ void testcase::fetch_canary() {
}
void testcase::update_canary(uint64_t increment) {
mdbx_canary canary_now = last.canary;
MDBX_canary canary_now = last.canary;
log_trace(">> update_canary: sequence %" PRIu64 " += %" PRIu64, canary_now.y,
increment);

View File

@@ -155,7 +155,7 @@ protected:
keygen::maker keyvalue_maker;
struct {
mdbx_canary canary;
MDBX_canary canary;
} last;
SET speculum{ItemCompare(this)}, speculum_commited{ItemCompare(this)};