From 8867c2ddc27df750bbc7526f8d7745ca481331f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9B=D0=B5=D0=BE=D0=BD=D0=B8=D0=B4=20=D0=AE=D1=80=D1=8C?= =?UTF-8?q?=D0=B5=D0=B2=20=28Leonid=20Yuriev=29?= Date: Wed, 11 Dec 2024 21:22:04 +0300 Subject: [PATCH] =?UTF-8?q?mdbx:=20=D0=BD=D0=BE=D0=B2=D1=8B=D0=B5=20=D0=BD?= =?UTF-8?q?=D0=B0=D1=81=D1=82=D1=80=D0=BE=D0=B9=D0=BA=D0=B8=20clang-format?= =?UTF-8?q?=20(=D0=BA=D0=BE=D1=81=D0=BC=D0=B5=D1=82=D0=B8=D0=BA=D0=B0).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .clang-format | 4 +- example/example-mdbx.c | 39 +- mdbx.h | 735 ++---- mdbx.h++ | 3149 +++++++++----------------- src/api-cursor.c | 177 +- src/api-env.c | 272 +-- src/api-extra.c | 36 +- src/api-key-transform.c | 84 +- src/api-txn.c | 96 +- src/atomics-ops.h | 112 +- src/atomics-types.h | 26 +- src/audit.c | 46 +- src/chk.c | 1007 +++----- src/cogs.c | 122 +- src/cogs.h | 208 +- src/coherency.c | 118 +- src/cold.c | 228 +- src/copy.c | 177 +- src/cursor.c | 393 ++-- src/cursor.h | 144 +- src/dbi.c | 238 +- src/dbi.h | 67 +- src/debug_begin.h | 18 +- src/dpl.c | 104 +- src/dpl.h | 33 +- src/dxb.c | 712 ++---- src/env-opts.c | 54 +- src/env.c | 161 +- src/essentials.h | 15 +- src/gc-get.c | 351 +-- src/gc-put.c | 357 +-- src/gc.h | 3 +- src/global.c | 17 +- src/internals.h | 54 +- src/layout-dxb.h | 32 +- src/layout-lck.h | 30 +- src/lck-posix.c | 161 +- src/lck-windows.c | 161 +- src/lck.c | 60 +- src/lck.h | 6 +- src/logging_and_debug.c | 67 +- src/logging_and_debug.h | 124 +- src/mdbx.c++ | 374 ++- src/meta.c | 322 +-- src/meta.h | 106 +- src/misc.c | 44 +- src/mvcc-readers.c | 126 +- src/node.c | 89 +- src/node.h | 53 +- src/options.h | 72 +- src/osal.c | 842 +++---- src/osal.h | 148 +- src/page-get.c | 280 +-- src/page-iov.c | 37 +- src/page-iov.h | 12 +- src/page-ops.c | 141 +- src/page-ops.h | 103 +- src/page-search.c | 19 +- src/pnl.c | 60 +- src/pnl.h | 53 +- src/preface.h | 244 +- src/proto.h | 69 +- src/range-estimate.c | 95 +- src/refund.c | 49 +- src/sort.h | 551 +++-- src/spill.c | 163 +- src/spill.h | 34 +- src/table.c | 33 +- src/tls.c | 204 +- src/tools/chk.c | 209 +- src/tools/copy.c | 52 +- src/tools/drop.c | 15 +- src/tools/dump.c | 97 +- src/tools/load.c | 188 +- src/tools/stat.c | 117 +- src/tools/wingetopt.c | 11 +- src/tree.c | 286 +-- src/txl.c | 29 +- src/txl.h | 6 +- src/txn.c | 539 ++--- src/unaligned.h | 119 +- src/utils.c | 11 +- src/utils.h | 41 +- src/version.c.in | 12 +- src/walk.c | 76 +- src/walk.h | 13 +- src/windows-import.c | 27 +- src/windows-import.h | 55 +- test/append.c++ | 47 +- test/base.h++ | 22 +- test/cases.c++ | 30 +- test/chrono.c++ | 32 +- test/chrono.h++ | 9 +- test/config.c++ | 335 +-- test/config.h++ | 69 +- test/copy.c++ | 41 +- test/dead.c++ | 6 +- test/extra/crunched_delete.c++ | 76 +- test/extra/cursor_closing.c++ | 15 +- test/extra/dbi.c++ | 13 +- test/extra/doubtless_positioning.c++ | 68 +- test/extra/dupfix_addodd.c | 6 +- test/extra/dupfix_multiple.c++ | 158 +- test/extra/early_close_dbi.c++ | 9 +- test/extra/hex_base64_base58.c++ | 56 +- test/extra/maindb_ordinal.c++ | 21 +- test/extra/open.c++ | 22 +- test/extra/pcrf/pcrf_test.c | 129 +- test/extra/probe.c++ | 3 +- test/extra/upsert_alldups.c | 33 +- test/fork.c++ | 83 +- test/hill.c++ | 38 +- test/jitter.c++ | 52 +- test/keygen.c++ | 247 +- test/keygen.h++ | 28 +- test/log.c++ | 48 +- test/log.h++ | 17 +- test/main.c++ | 449 ++-- test/nested.c++ | 81 +- test/osal-unix.c++ | 78 +- test/osal-windows.c++ | 46 +- test/stub/pthread_barrier.c | 6 +- test/stub/pthread_barrier.h | 6 +- test/test.c++ | 492 ++-- test/test.h++ | 121 +- test/try.c++ | 3 +- test/ttl.c++ | 55 +- test/utils.c++ | 24 +- test/utils.h++ | 69 +- 129 files changed, 6727 insertions(+), 12640 deletions(-) diff --git a/.clang-format b/.clang-format index 6c59ef3a..03069f65 100644 --- a/.clang-format +++ b/.clang-format @@ -1,3 +1,3 @@ BasedOnStyle: LLVM -Standard: Cpp11 -ReflowComments: true +Standard: c++20 +ColumnLimit: 120 diff --git a/example/example-mdbx.c b/example/example-mdbx.c index 215a0fca..beae17f5 100644 --- a/example/example-mdbx.c +++ b/example/example-mdbx.c @@ -18,8 +18,7 @@ * . */ -#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \ - !defined(__USE_MINGW_ANSI_STDIO) +#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO) #define __USE_MINGW_ANSI_STDIO 1 #endif /* MinGW */ @@ -59,33 +58,23 @@ int main(int argc, char *argv[]) { pagesize_min, pagesize_max, pagesize_default); printf("\tKey size: minimum %zu, maximum ≈¼ pagesize (%zu bytes for default" " %zuK pagesize, %zu bytes for %zuK pagesize).\n", - (size_t)0, mdbx_limits_keysize_max(-1, MDBX_DB_DEFAULTS), - pagesize_default / 1024, - mdbx_limits_keysize_max(pagesize_max, MDBX_DB_DEFAULTS), - pagesize_max / 1024); + (size_t)0, mdbx_limits_keysize_max(-1, MDBX_DB_DEFAULTS), pagesize_default / 1024, + mdbx_limits_keysize_max(pagesize_max, MDBX_DB_DEFAULTS), pagesize_max / 1024); printf("\tValue size: minimum %zu, maximum %zu (0x%08zX) bytes for maps," " ≈¼ pagesize for multimaps (%zu bytes for default %zuK pagesize," " %zu bytes for %zuK pagesize).\n", (size_t)0, mdbx_limits_valsize_max(pagesize_min, MDBX_DB_DEFAULTS), - mdbx_limits_valsize_max(pagesize_min, MDBX_DB_DEFAULTS), - mdbx_limits_valsize_max(-1, MDBX_DUPSORT), pagesize_default / 1024, - mdbx_limits_valsize_max(pagesize_max, MDBX_DUPSORT), - pagesize_max / 1024); + mdbx_limits_valsize_max(pagesize_min, MDBX_DB_DEFAULTS), mdbx_limits_valsize_max(-1, MDBX_DUPSORT), + pagesize_default / 1024, mdbx_limits_valsize_max(pagesize_max, MDBX_DUPSORT), pagesize_max / 1024); printf("\tWrite transaction size: up to %zu (0x%zX) pages (%f %s for default " "%zuK pagesize, %f %s for %zuK pagesize).\n", - mdbx_limits_txnsize_max(pagesize_min) / pagesize_min, - mdbx_limits_txnsize_max(pagesize_min) / pagesize_min, - mdbx_limits_txnsize_max(-1) / scale_factor, scale_unit, - pagesize_default / 1024, - mdbx_limits_txnsize_max(pagesize_max) / scale_factor, scale_unit, - pagesize_max / 1024); + mdbx_limits_txnsize_max(pagesize_min) / pagesize_min, mdbx_limits_txnsize_max(pagesize_min) / pagesize_min, + mdbx_limits_txnsize_max(-1) / scale_factor, scale_unit, pagesize_default / 1024, + mdbx_limits_txnsize_max(pagesize_max) / scale_factor, scale_unit, pagesize_max / 1024); printf("\tDatabase size: up to %zu pages (%f %s for default %zuK " "pagesize, %f %s for %zuK pagesize).\n", - mdbx_limits_dbsize_max(pagesize_min) / pagesize_min, - mdbx_limits_dbsize_max(-1) / scale_factor, scale_unit, - pagesize_default / 1024, - mdbx_limits_dbsize_max(pagesize_max) / scale_factor, scale_unit, - pagesize_max / 1024); + mdbx_limits_dbsize_max(pagesize_min) / pagesize_min, mdbx_limits_dbsize_max(-1) / scale_factor, scale_unit, + pagesize_default / 1024, mdbx_limits_dbsize_max(pagesize_max) / scale_factor, scale_unit, pagesize_max / 1024); printf("\tMaximum sub-databases: %u.\n", MDBX_MAX_DBI); printf("-----\n"); @@ -94,8 +83,7 @@ int main(int argc, char *argv[]) { fprintf(stderr, "mdbx_env_create: (%d) %s\n", rc, mdbx_strerror(rc)); goto bailout; } - rc = mdbx_env_open(env, "./example-db", MDBX_NOSUBDIR | MDBX_LIFORECLAIM, - 0664); + rc = mdbx_env_open(env, "./example-db", MDBX_NOSUBDIR | MDBX_LIFORECLAIM, 0664); if (rc != MDBX_SUCCESS) { fprintf(stderr, "mdbx_env_open: (%d) %s\n", rc, mdbx_strerror(rc)); goto bailout; @@ -143,9 +131,8 @@ int main(int argc, char *argv[]) { int found = 0; while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == 0) { - printf("key: %p %.*s, data: %p %.*s\n", key.iov_base, (int)key.iov_len, - (char *)key.iov_base, data.iov_base, (int)data.iov_len, - (char *)data.iov_base); + printf("key: %p %.*s, data: %p %.*s\n", key.iov_base, (int)key.iov_len, (char *)key.iov_base, data.iov_base, + (int)data.iov_len, (char *)data.iov_base); found += 1; } if (rc != MDBX_NOTFOUND || found == 0) { diff --git a/mdbx.h b/mdbx.h index 29feb245..9854c6e5 100644 --- a/mdbx.h +++ b/mdbx.h @@ -39,8 +39,7 @@ credits and acknowledgments. #ifndef LIBMDBX_H #define LIBMDBX_H -#if defined(__riscv) || defined(__riscv__) || defined(__RISCV) || \ - defined(__RISCV__) +#if defined(__riscv) || defined(__riscv__) || defined(__RISCV) || defined(__RISCV__) #warning "The RISC-V architecture is intentionally insecure by design. \ Please delete this admonition at your own risk, \ if you make such decision informed and consciously. \ @@ -49,12 +48,12 @@ credits and acknowledgments. #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #endif /* _MSC_VER (warnings) */ @@ -224,8 +223,7 @@ typedef mode_t mdbx_mode_t; #define __has_feature(x) (0) #define __has_exceptions_disabled (0) #elif !defined(__has_exceptions_disabled) -#define __has_exceptions_disabled \ - (__has_feature(cxx_noexcept) && !__has_feature(cxx_exceptions)) +#define __has_exceptions_disabled (__has_feature(cxx_noexcept) && !__has_feature(cxx_exceptions)) #endif /* __has_feature */ #ifndef __has_extension @@ -246,9 +244,9 @@ typedef mode_t mdbx_mode_t; #define MDBX_PURE_FUNCTION [[gnu::pure]] #elif __has_C23_or_CXX_attribute(gnu::pure) #define MDBX_PURE_FUNCTION [[gnu::pure]] -#elif (defined(__GNUC__) || __has_attribute(__pure__)) && \ - (!defined(__clang__) /* https://bugs.llvm.org/show_bug.cgi?id=43275 */ || \ - !defined(__cplusplus) || __has_exceptions_disabled) +#elif (defined(__GNUC__) || __has_attribute(__pure__)) && \ + (!defined(__clang__) /* https://bugs.llvm.org/show_bug.cgi?id=43275 */ || !defined(__cplusplus) || \ + __has_exceptions_disabled) #define MDBX_PURE_FUNCTION __attribute__((__pure__)) #else #define MDBX_PURE_FUNCTION @@ -265,8 +263,7 @@ typedef mode_t mdbx_mode_t; #else #define MDBX_NOTHROW_PURE_FUNCTION [[gnu::pure]] #endif -#elif defined(__GNUC__) || \ - (__has_attribute(__pure__) && __has_attribute(__nothrow__)) +#elif defined(__GNUC__) || (__has_attribute(__pure__) && __has_attribute(__nothrow__)) #define MDBX_NOTHROW_PURE_FUNCTION __attribute__((__pure__, __nothrow__)) #elif __has_cpp_attribute(pure) #define MDBX_NOTHROW_PURE_FUNCTION [[pure]] @@ -288,9 +285,9 @@ typedef mode_t mdbx_mode_t; #define MDBX_CONST_FUNCTION [[gnu::const]] #elif __has_C23_or_CXX_attribute(gnu::const) #define MDBX_CONST_FUNCTION [[gnu::const]] -#elif (defined(__GNUC__) || __has_attribute(__const__)) && \ - (!defined(__clang__) /* https://bugs.llvm.org/show_bug.cgi?id=43275 */ || \ - !defined(__cplusplus) || __has_exceptions_disabled) +#elif (defined(__GNUC__) || __has_attribute(__const__)) && \ + (!defined(__clang__) /* https://bugs.llvm.org/show_bug.cgi?id=43275 */ || !defined(__cplusplus) || \ + __has_exceptions_disabled) #define MDBX_CONST_FUNCTION __attribute__((__const__)) #else #define MDBX_CONST_FUNCTION MDBX_PURE_FUNCTION @@ -307,8 +304,7 @@ typedef mode_t mdbx_mode_t; #else #define MDBX_NOTHROW_CONST_FUNCTION [[gnu::const]] #endif -#elif defined(__GNUC__) || \ - (__has_attribute(__const__) && __has_attribute(__nothrow__)) +#elif defined(__GNUC__) || (__has_attribute(__const__) && __has_attribute(__nothrow__)) #define MDBX_NOTHROW_CONST_FUNCTION __attribute__((__const__, __nothrow__)) #elif __has_cpp_attribute_qualified(const) #define MDBX_NOTHROW_CONST_FUNCTION [[const]] @@ -322,17 +318,13 @@ typedef mode_t mdbx_mode_t; #ifndef MDBX_DEPRECATED #ifdef __deprecated #define MDBX_DEPRECATED __deprecated -#elif defined(DOXYGEN) || \ - ((!defined(__GNUC__) || defined(__clang__) || __GNUC__ > 5) && \ - ((defined(__cplusplus) && __cplusplus >= 201403L && \ - __has_cpp_attribute(deprecated) && \ - __has_cpp_attribute(deprecated) >= 201309L) || \ - (!defined(__cplusplus) && defined(__STDC_VERSION__) && \ - __STDC_VERSION__ >= 202304L))) +#elif defined(DOXYGEN) || ((!defined(__GNUC__) || defined(__clang__) || __GNUC__ > 5) && \ + ((defined(__cplusplus) && __cplusplus >= 201403L && __has_cpp_attribute(deprecated) && \ + __has_cpp_attribute(deprecated) >= 201309L) || \ + (!defined(__cplusplus) && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202304L))) #define MDBX_DEPRECATED [[deprecated]] -#elif (defined(__GNUC__) && __GNUC__ > 5) || \ - (__has_attribute(__deprecated__) && \ - (!defined(__GNUC__) || defined(__clang__) || __GNUC__ > 5)) +#elif (defined(__GNUC__) && __GNUC__ > 5) || \ + (__has_attribute(__deprecated__) && (!defined(__GNUC__) || defined(__clang__) || __GNUC__ > 5)) #define MDBX_DEPRECATED __attribute__((__deprecated__)) #elif defined(_MSC_VER) #define MDBX_DEPRECATED __declspec(deprecated) @@ -344,9 +336,8 @@ typedef mode_t mdbx_mode_t; #ifndef MDBX_DEPRECATED_ENUM #ifdef __deprecated_enum #define MDBX_DEPRECATED_ENUM __deprecated_enum -#elif defined(DOXYGEN) || \ - (!defined(_MSC_VER) || (defined(__cplusplus) && __cplusplus >= 201403L && \ - __has_cpp_attribute(deprecated) && \ +#elif defined(DOXYGEN) || \ + (!defined(_MSC_VER) || (defined(__cplusplus) && __cplusplus >= 201403L && __has_cpp_attribute(deprecated) && \ __has_cpp_attribute(deprecated) >= 201309L)) #define MDBX_DEPRECATED_ENUM MDBX_DEPRECATED #else @@ -355,8 +346,8 @@ typedef mode_t mdbx_mode_t; #endif /* MDBX_DEPRECATED_ENUM */ #ifndef __dll_export -#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) || \ - defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__) +#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) || defined(__MINGW__) || defined(__MINGW32__) || \ + defined(__MINGW64__) #if defined(__GNUC__) || __has_attribute(__dllexport__) #define __dll_export __attribute__((__dllexport__)) #elif defined(_MSC_VER) @@ -372,8 +363,8 @@ typedef mode_t mdbx_mode_t; #endif /* __dll_export */ #ifndef __dll_import -#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) || \ - defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__) +#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) || defined(__MINGW__) || defined(__MINGW32__) || \ + defined(__MINGW64__) #if defined(__GNUC__) || __has_attribute(__dllimport__) #define __dll_import __attribute__((__dllimport__)) #elif defined(_MSC_VER) @@ -391,8 +382,8 @@ typedef mode_t mdbx_mode_t; * with old version of libmdbx, with a strictly ODR-common implementation. Thus, * we emulate __extern_inline for all compilers, including non-GNU ones. */ #if defined(LIBMDBX_INTERNALS) && !defined(LIBMDBX_NO_EXPORTS_LEGACY_API) -#define LIBMDBX_INLINE_API(TYPE, NAME, ARGS) \ - /* proto of exported which uses common impl */ LIBMDBX_API TYPE NAME ARGS; \ +#define LIBMDBX_INLINE_API(TYPE, NAME, ARGS) \ + /* proto of exported which uses common impl */ LIBMDBX_API TYPE NAME ARGS; \ /* definition of common impl */ static __inline TYPE __inline_##NAME ARGS #else #define LIBMDBX_INLINE_API(TYPE, NAME, ARGS) static __inline TYPE NAME ARGS @@ -421,8 +412,7 @@ typedef mode_t mdbx_mode_t; /** Workaround for old compilers without support for C++17 `noexcept`. */ #if defined(DOXYGEN) #define MDBX_CXX17_NOEXCEPT noexcept -#elif !defined(__cpp_noexcept_function_type) || \ - __cpp_noexcept_function_type < 201510L +#elif !defined(__cpp_noexcept_function_type) || __cpp_noexcept_function_type < 201510L #define MDBX_CXX17_NOEXCEPT #else #define MDBX_CXX17_NOEXCEPT noexcept @@ -435,14 +425,11 @@ typedef mode_t mdbx_mode_t; #elif !defined(__cplusplus) #define MDBX_CXX01_CONSTEXPR __inline #define MDBX_CXX01_CONSTEXPR_VAR const -#elif !defined(DOXYGEN) && \ - ((__cplusplus < 201103L && defined(__cpp_constexpr) && \ - __cpp_constexpr < 200704L) || \ - (defined(__LCC__) && __LCC__ < 124) || \ - (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ < 407) && \ - !defined(__clang__) && !defined(__LCC__)) || \ - (defined(_MSC_VER) && _MSC_VER < 1910) || \ - (defined(__clang__) && __clang_major__ < 4)) +#elif !defined(DOXYGEN) && \ + ((__cplusplus < 201103L && defined(__cpp_constexpr) && __cpp_constexpr < 200704L) || \ + (defined(__LCC__) && __LCC__ < 124) || \ + (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ < 407) && !defined(__clang__) && !defined(__LCC__)) || \ + (defined(_MSC_VER) && _MSC_VER < 1910) || (defined(__clang__) && __clang_major__ < 4)) #define MDBX_CXX01_CONSTEXPR inline #define MDBX_CXX01_CONSTEXPR_VAR const #else @@ -458,13 +445,10 @@ typedef mode_t mdbx_mode_t; #elif !defined(__cplusplus) #define MDBX_CXX11_CONSTEXPR __inline #define MDBX_CXX11_CONSTEXPR_VAR const -#elif !defined(DOXYGEN) && \ - (!defined(__cpp_constexpr) || __cpp_constexpr < 201304L || \ - (defined(__LCC__) && __LCC__ < 124) || \ - (defined(__GNUC__) && __GNUC__ < 6 && !defined(__clang__) && \ - !defined(__LCC__)) || \ - (defined(_MSC_VER) && _MSC_VER < 1910) || \ - (defined(__clang__) && __clang_major__ < 5)) +#elif !defined(DOXYGEN) && \ + (!defined(__cpp_constexpr) || __cpp_constexpr < 201304L || (defined(__LCC__) && __LCC__ < 124) || \ + (defined(__GNUC__) && __GNUC__ < 6 && !defined(__clang__) && !defined(__LCC__)) || \ + (defined(_MSC_VER) && _MSC_VER < 1910) || (defined(__clang__) && __clang_major__ < 5)) #define MDBX_CXX11_CONSTEXPR inline #define MDBX_CXX11_CONSTEXPR_VAR const #else @@ -480,12 +464,10 @@ typedef mode_t mdbx_mode_t; #elif !defined(__cplusplus) #define MDBX_CXX14_CONSTEXPR __inline #define MDBX_CXX14_CONSTEXPR_VAR const -#elif defined(DOXYGEN) || \ - defined(__cpp_constexpr) && __cpp_constexpr >= 201304L && \ - ((defined(_MSC_VER) && _MSC_VER >= 1910) || \ - (defined(__clang__) && __clang_major__ > 4) || \ - (defined(__GNUC__) && __GNUC__ > 6) || \ - (!defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER))) +#elif defined(DOXYGEN) || \ + defined(__cpp_constexpr) && __cpp_constexpr >= 201304L && \ + ((defined(_MSC_VER) && _MSC_VER >= 1910) || (defined(__clang__) && __clang_major__ > 4) || \ + (defined(__GNUC__) && __GNUC__ > 6) || (!defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER))) #define MDBX_CXX14_CONSTEXPR constexpr #define MDBX_CXX14_CONSTEXPR_VAR constexpr #else @@ -497,9 +479,8 @@ typedef mode_t mdbx_mode_t; #define MDBX_NORETURN __noreturn #elif defined(_Noreturn) #define MDBX_NORETURN _Noreturn -#elif defined(DOXYGEN) || (defined(__cplusplus) && __cplusplus >= 201103L) || \ - (!defined(__cplusplus) && defined(__STDC_VERSION__) && \ - __STDC_VERSION__ > 202005L) +#elif defined(DOXYGEN) || (defined(__cplusplus) && __cplusplus >= 201103L) || \ + (!defined(__cplusplus) && defined(__STDC_VERSION__) && __STDC_VERSION__ > 202005L) #define MDBX_NORETURN [[noreturn]] #elif defined(__GNUC__) || __has_attribute(__noreturn__) #define MDBX_NORETURN __attribute__((__noreturn__)) @@ -512,23 +493,19 @@ typedef mode_t mdbx_mode_t; #ifndef MDBX_PRINTF_ARGS #if defined(__GNUC__) || __has_attribute(__format__) || defined(DOXYGEN) #if defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__) -#define MDBX_PRINTF_ARGS(format_index, first_arg) \ - __attribute__((__format__(__gnu_printf__, format_index, first_arg))) +#define MDBX_PRINTF_ARGS(format_index, first_arg) __attribute__((__format__(__gnu_printf__, format_index, first_arg))) #else -#define MDBX_PRINTF_ARGS(format_index, first_arg) \ - __attribute__((__format__(__printf__, format_index, first_arg))) +#define MDBX_PRINTF_ARGS(format_index, first_arg) __attribute__((__format__(__printf__, format_index, first_arg))) #endif /* MinGW */ #else #define MDBX_PRINTF_ARGS(format_index, first_arg) #endif #endif /* MDBX_PRINTF_ARGS */ -#if defined(DOXYGEN) || \ - (defined(__cplusplus) && __cplusplus >= 201603L && \ - __has_cpp_attribute(maybe_unused) && \ - __has_cpp_attribute(maybe_unused) >= 201603L) || \ - (!defined(__cplusplus) && defined(__STDC_VERSION__) && \ - __STDC_VERSION__ > 202005L) +#if defined(DOXYGEN) || \ + (defined(__cplusplus) && __cplusplus >= 201603L && __has_cpp_attribute(maybe_unused) && \ + __has_cpp_attribute(maybe_unused) >= 201603L) || \ + (!defined(__cplusplus) && defined(__STDC_VERSION__) && __STDC_VERSION__ > 202005L) #define MDBX_MAYBE_UNUSED [[maybe_unused]] #elif defined(__GNUC__) || __has_attribute(__unused__) #define MDBX_MAYBE_UNUSED __attribute__((__unused__)) @@ -550,12 +527,9 @@ typedef mode_t mdbx_mode_t; #if !defined(DEFINE_ENUM_FLAG_OPERATORS) && !defined(DOXYGEN) #ifdef __cplusplus -#if !defined(__cpp_constexpr) || __cpp_constexpr < 200704L || \ - (defined(__LCC__) && __LCC__ < 124) || \ - (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ < 407) && \ - !defined(__clang__) && !defined(__LCC__)) || \ - (defined(_MSC_VER) && _MSC_VER < 1910) || \ - (defined(__clang__) && __clang_major__ < 4) +#if !defined(__cpp_constexpr) || __cpp_constexpr < 200704L || (defined(__LCC__) && __LCC__ < 124) || \ + (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ < 407) && !defined(__clang__) && !defined(__LCC__)) || \ + (defined(_MSC_VER) && _MSC_VER < 1910) || (defined(__clang__) && __clang_major__ < 4) /* The constexpr feature is not available or (may be) broken */ #define CONSTEXPR_ENUM_FLAGS_OPERATIONS 0 #else @@ -565,42 +539,18 @@ typedef mode_t mdbx_mode_t; /// Define operator overloads to enable bit operations on enum values that are /// used to define flags (based on Microsoft's DEFINE_ENUM_FLAG_OPERATORS). -#define DEFINE_ENUM_FLAG_OPERATORS(ENUM) \ - extern "C++" { \ - MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator|(ENUM a, ENUM b) { \ - return ENUM(unsigned(a) | unsigned(b)); \ - } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator|=(ENUM &a, \ - ENUM b) { \ - return a = a | b; \ - } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator&(ENUM a, ENUM b) { \ - return ENUM(unsigned(a) & unsigned(b)); \ - } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator&(ENUM a, \ - unsigned b) { \ - return ENUM(unsigned(a) & b); \ - } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator&(unsigned a, \ - ENUM b) { \ - return ENUM(a & unsigned(b)); \ - } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator&=(ENUM &a, \ - ENUM b) { \ - return a = a & b; \ - } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator&=(ENUM &a, \ - unsigned b) { \ - return a = a & b; \ - } \ - MDBX_CXX01_CONSTEXPR unsigned operator~(ENUM a) { return ~unsigned(a); } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator^(ENUM a, ENUM b) { \ - return ENUM(unsigned(a) ^ unsigned(b)); \ - } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator^=(ENUM &a, \ - ENUM b) { \ - return a = a ^ b; \ - } \ +#define DEFINE_ENUM_FLAG_OPERATORS(ENUM) \ + extern "C++" { \ + MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator|(ENUM a, ENUM b) { return ENUM(unsigned(a) | unsigned(b)); } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator|=(ENUM &a, ENUM b) { return a = a | b; } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator&(ENUM a, ENUM b) { return ENUM(unsigned(a) & unsigned(b)); } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator&(ENUM a, unsigned b) { return ENUM(unsigned(a) & b); } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator&(unsigned a, ENUM b) { return ENUM(a & unsigned(b)); } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator&=(ENUM &a, ENUM b) { return a = a & b; } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator&=(ENUM &a, unsigned b) { return a = a & b; } \ + MDBX_CXX01_CONSTEXPR unsigned operator~(ENUM a) { return ~unsigned(a); } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator^(ENUM a, ENUM b) { return ENUM(unsigned(a) ^ unsigned(b)); } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator^=(ENUM &a, ENUM b) { return a = a ^ b; } \ } #else /* __cplusplus */ /* nope for C since it always allows these operators for enums */ @@ -732,8 +682,7 @@ extern LIBMDBX_VERINFO_API const struct MDBX_build_info { /* As described above mdbx_module_handler() IS REQUIRED for Windows versions * prior to Windows Vista. */ #define MDBX_MANUAL_MODULE_HANDLER 1 -void LIBMDBX_API NTAPI mdbx_module_handler(PVOID module, DWORD reason, - PVOID reserved); +void LIBMDBX_API NTAPI mdbx_module_handler(PVOID module, DWORD reason, PVOID reserved); #endif #endif /* Windows && !DLL && MDBX_MANUAL_MODULE_HANDLER */ @@ -977,8 +926,7 @@ typedef enum MDBX_debug_flags { MDBX_DBG_DONT_UPGRADE = 64, #ifdef ENABLE_UBSAN - MDBX_DBG_MAX = ((unsigned)MDBX_LOG_MAX) << 16 | - 127 /* avoid UBSAN false-positive trap by a tests */, + MDBX_DBG_MAX = ((unsigned)MDBX_LOG_MAX) << 16 | 127 /* avoid UBSAN false-positive trap by a tests */, #endif /* ENABLE_UBSAN */ /** for mdbx_setup_debug() only: Don't change current settings */ @@ -1000,8 +948,7 @@ DEFINE_ENUM_FLAG_OPERATORS(MDBX_debug_flags) * format-message string passed by `fmt` argument. * Maybe NULL or invalid if the format-message string * don't contain `%`-specification of arguments. */ -typedef void MDBX_debug_func(MDBX_log_level_t loglevel, const char *function, - int line, const char *fmt, +typedef void MDBX_debug_func(MDBX_log_level_t loglevel, const char *function, int line, const char *fmt, va_list args) MDBX_CXX17_NOEXCEPT; /** \brief The "don't change `logger`" value for mdbx_setup_debug() */ @@ -1011,20 +958,13 @@ typedef void MDBX_debug_func(MDBX_log_level_t loglevel, const char *function, /** \brief Setup global log-level, debug options and debug logger. * \returns The previously `debug_flags` in the 0-15 bits * and `log_level` in the 16-31 bits. */ -LIBMDBX_API int mdbx_setup_debug(MDBX_log_level_t log_level, - MDBX_debug_flags_t debug_flags, - MDBX_debug_func *logger); +LIBMDBX_API int mdbx_setup_debug(MDBX_log_level_t log_level, MDBX_debug_flags_t debug_flags, MDBX_debug_func *logger); -typedef void MDBX_debug_func_nofmt(MDBX_log_level_t loglevel, - const char *function, int line, - const char *msg, +typedef void MDBX_debug_func_nofmt(MDBX_log_level_t loglevel, const char *function, int line, const char *msg, unsigned length) MDBX_CXX17_NOEXCEPT; -LIBMDBX_API int mdbx_setup_debug_nofmt(MDBX_log_level_t log_level, - MDBX_debug_flags_t debug_flags, - MDBX_debug_func_nofmt *logger, - char *logger_buffer, - size_t logger_buffer_size); +LIBMDBX_API int mdbx_setup_debug_nofmt(MDBX_log_level_t log_level, MDBX_debug_flags_t debug_flags, + MDBX_debug_func_nofmt *logger, char *logger_buffer, size_t logger_buffer_size); /** \brief A callback function for most MDBX assert() failures, * called before printing the message and aborting. @@ -1036,8 +976,7 @@ LIBMDBX_API int mdbx_setup_debug_nofmt(MDBX_log_level_t log_level, * may be NULL. * \param [in] line The line number in the source file * where the assertion check failed, may be zero. */ -typedef void MDBX_assert_func(const MDBX_env *env, const char *msg, - const char *function, +typedef void MDBX_assert_func(const MDBX_env *env, const char *msg, const char *function, unsigned line) MDBX_CXX17_NOEXCEPT; /** \brief Set or reset the assert() callback of the environment. @@ -1060,19 +999,14 @@ LIBMDBX_API int mdbx_env_set_assert(MDBX_env *env, MDBX_assert_func *func); * - NULL if given buffer size less than 4 bytes; * - pointer to constant string if given value NULL or empty; * - otherwise pointer to given buffer. */ -LIBMDBX_API const char *mdbx_dump_val(const MDBX_val *key, char *const buf, - const size_t bufsize); +LIBMDBX_API const char *mdbx_dump_val(const MDBX_val *key, char *const buf, const size_t bufsize); /** \brief Panics with message and causes abnormal process termination. */ -MDBX_NORETURN LIBMDBX_API void mdbx_panic(const char *fmt, ...) - MDBX_PRINTF_ARGS(1, 2); +MDBX_NORETURN LIBMDBX_API void mdbx_panic(const char *fmt, ...) MDBX_PRINTF_ARGS(1, 2); /** \brief Panics with asserton failed message and causes abnormal process * termination. */ -MDBX_NORETURN LIBMDBX_API void mdbx_assert_fail(const MDBX_env *env, - const char *msg, - const char *func, - unsigned line); +MDBX_NORETURN LIBMDBX_API void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func, unsigned line); /** end of c_debug @} */ /** \brief Environment flags @@ -1630,8 +1564,7 @@ typedef enum MDBX_txn_flags { /** Most operations on the transaction are currently illegal. * \note This is a transaction state flag. Returned from \ref mdbx_txn_flags() * but can't be used with \ref mdbx_txn_begin(). */ - MDBX_TXN_BLOCKED = - MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD | MDBX_TXN_PARKED + MDBX_TXN_BLOCKED = MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD | MDBX_TXN_PARKED } MDBX_txn_flags_t; DEFINE_ENUM_FLAG_OPERATORS(MDBX_txn_flags) @@ -2075,9 +2008,7 @@ typedef enum MDBX_error { * \ingroup c_err * \deprecated Please review your code to use MDBX_UNABLE_EXTEND_MAPSIZE * instead. */ -MDBX_DEPRECATED static __inline int MDBX_MAP_RESIZED_is_deprecated(void) { - return MDBX_UNABLE_EXTEND_MAPSIZE; -} +MDBX_DEPRECATED static __inline int MDBX_MAP_RESIZED_is_deprecated(void) { return MDBX_UNABLE_EXTEND_MAPSIZE; } #define MDBX_MAP_RESIZED MDBX_MAP_RESIZED_is_deprecated() /** \brief Return a string describing a given error code. @@ -2138,8 +2069,7 @@ LIBMDBX_API const char *mdbx_strerror_ANSI2OEM(int errnum); * Windows error-messages in the OEM-encoding for console utilities. * \ingroup c_err * \see mdbx_strerror_ANSI2OEM() */ -LIBMDBX_API const char *mdbx_strerror_r_ANSI2OEM(int errnum, char *buf, - size_t buflen); +LIBMDBX_API const char *mdbx_strerror_r_ANSI2OEM(int errnum, char *buf, size_t buflen); #endif /* Bit of Windows' madness */ /** \brief Create an MDBX environment instance. @@ -2472,8 +2402,7 @@ typedef enum MDBX_option { * \see MDBX_option_t * \see mdbx_env_get_option() * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, - uint64_t value); +LIBMDBX_API int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, uint64_t value); /** \brief Gets the value of extra runtime options from an environment. * \ingroup c_settings @@ -2485,9 +2414,7 @@ LIBMDBX_API int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, * \see MDBX_option_t * \see mdbx_env_get_option() * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_env_get_option(const MDBX_env *env, - const MDBX_option_t option, - uint64_t *pvalue); +LIBMDBX_API int mdbx_env_get_option(const MDBX_env *env, const MDBX_option_t option, uint64_t *pvalue); /** \brief Open an environment instance. * \ingroup c_opening @@ -2561,20 +2488,16 @@ LIBMDBX_API int mdbx_env_get_option(const MDBX_env *env, * \retval MDBX_TOO_LARGE Database is too large for this process, * i.e. 32-bit process tries to open >4Gb database. */ -LIBMDBX_API int mdbx_env_open(MDBX_env *env, const char *pathname, - MDBX_env_flags_t flags, mdbx_mode_t mode); +LIBMDBX_API int mdbx_env_open(MDBX_env *env, const char *pathname, MDBX_env_flags_t flags, mdbx_mode_t mode); #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) /** \copydoc mdbx_env_open() * \note Available only on Windows. * \see mdbx_env_open() */ -LIBMDBX_API int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, - MDBX_env_flags_t flags, mdbx_mode_t mode); -#define mdbx_env_openT(env, pathname, flags, mode) \ - mdbx_env_openW(env, pathname, flags, mode) +LIBMDBX_API int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, MDBX_env_flags_t flags, mdbx_mode_t mode); +#define mdbx_env_openT(env, pathname, flags, mode) mdbx_env_openW(env, pathname, flags, mode) #else -#define mdbx_env_openT(env, pathname, flags, mode) \ - mdbx_env_open(env, pathname, flags, mode) +#define mdbx_env_openT(env, pathname, flags, mode) mdbx_env_open(env, pathname, flags, mode) #endif /* Windows */ /** \brief Deletion modes for \ref mdbx_env_delete(). @@ -2615,16 +2538,14 @@ typedef enum MDBX_env_delete_mode { * some possible errors are: * \retval MDBX_RESULT_TRUE No corresponding files or directories were found, * so no deletion was performed. */ -LIBMDBX_API int mdbx_env_delete(const char *pathname, - MDBX_env_delete_mode_t mode); +LIBMDBX_API int mdbx_env_delete(const char *pathname, MDBX_env_delete_mode_t mode); #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) /** \copydoc mdbx_env_delete() * \ingroup c_extra * \note Available only on Windows. * \see mdbx_env_delete() */ -LIBMDBX_API int mdbx_env_deleteW(const wchar_t *pathname, - MDBX_env_delete_mode_t mode); +LIBMDBX_API int mdbx_env_deleteW(const wchar_t *pathname, MDBX_env_delete_mode_t mode); #define mdbx_env_deleteT(pathname, mode) mdbx_env_deleteW(pathname, mode) #else #define mdbx_env_deleteT(pathname, mode) mdbx_env_delete(pathname, mode) @@ -2683,8 +2604,7 @@ LIBMDBX_API int mdbx_env_deleteW(const wchar_t *pathname, * \see mdbx_txn_park() * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_env_copy(MDBX_env *env, const char *dest, - MDBX_copy_flags_t flags); +LIBMDBX_API int mdbx_env_copy(MDBX_env *env, const char *dest, MDBX_copy_flags_t flags); /** \brief Copy an MDBX environment by given read transaction to the specified * path, with options. @@ -2739,30 +2659,25 @@ LIBMDBX_API int mdbx_env_copy(MDBX_env *env, const char *dest, * \see mdbx_txn_park() * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_txn_copy2pathname(MDBX_txn *txn, const char *dest, - MDBX_copy_flags_t flags); +LIBMDBX_API int mdbx_txn_copy2pathname(MDBX_txn *txn, const char *dest, MDBX_copy_flags_t flags); #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) /** \copydoc mdbx_env_copy() * \ingroup c_extra * \note Available only on Windows. * \see mdbx_env_copy() */ -LIBMDBX_API int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest, - MDBX_copy_flags_t flags); +LIBMDBX_API int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest, MDBX_copy_flags_t flags); #define mdbx_env_copyT(env, dest, flags) mdbx_env_copyW(env, dest, flags) /** \copydoc mdbx_txn_copy2pathname() * \ingroup c_extra * \note Available only on Windows. * \see mdbx_txn_copy2pathname() */ -LIBMDBX_API int mdbx_txn_copy2pathnameW(MDBX_txn *txn, const wchar_t *dest, - MDBX_copy_flags_t flags); -#define mdbx_txn_copy2pathnameT(txn, dest, flags) \ - mdbx_txn_copy2pathnameW(txn, dest, path) +LIBMDBX_API int mdbx_txn_copy2pathnameW(MDBX_txn *txn, const wchar_t *dest, MDBX_copy_flags_t flags); +#define mdbx_txn_copy2pathnameT(txn, dest, flags) mdbx_txn_copy2pathnameW(txn, dest, path) #else #define mdbx_env_copyT(env, dest, flags) mdbx_env_copy(env, dest, flags) -#define mdbx_txn_copy2pathnameT(txn, dest, flags) \ - mdbx_txn_copy2pathname(txn, dest, path) +#define mdbx_txn_copy2pathnameT(txn, dest, flags) mdbx_txn_copy2pathname(txn, dest, path) #endif /* Windows */ /** \brief Copy an environment to the specified file descriptor, with @@ -2789,8 +2704,7 @@ LIBMDBX_API int mdbx_txn_copy2pathnameW(MDBX_txn *txn, const wchar_t *dest, * \param [in] flags Special options for this operation. \see mdbx_env_copy() * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, - MDBX_copy_flags_t flags); +LIBMDBX_API int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, MDBX_copy_flags_t flags); /** \brief Copy an environment by given read transaction to the specified file * descriptor, with options. @@ -2815,21 +2729,20 @@ LIBMDBX_API int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, * \param [in] flags Special options for this operation. \see mdbx_env_copy() * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_txn_copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, - MDBX_copy_flags_t flags); +LIBMDBX_API int mdbx_txn_copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, MDBX_copy_flags_t flags); /** \brief Statistics for a table in the environment * \ingroup c_statinfo * \see mdbx_env_stat_ex() \see mdbx_dbi_stat() */ struct MDBX_stat { - uint32_t ms_psize; /**< Size of a table page. This is the same for all tables - in a database. */ - uint32_t ms_depth; /**< Depth (height) of the B-tree */ + uint32_t ms_psize; /**< Size of a table page. This is the same for all tables + in a database. */ + uint32_t ms_depth; /**< Depth (height) of the B-tree */ uint64_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ uint64_t ms_leaf_pages; /**< Number of leaf pages */ uint64_t ms_overflow_pages; /**< Number of large/overflow pages */ uint64_t ms_entries; /**< Number of data items */ - uint64_t ms_mod_txnid; /**< Transaction ID of committed last modification */ + uint64_t ms_mod_txnid; /**< Transaction ID of committed last modification */ }; #ifndef __cplusplus /** \ingroup c_statinfo */ @@ -2855,15 +2768,12 @@ typedef struct MDBX_stat MDBX_stat; * \param [in] bytes The size of \ref MDBX_stat. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, - MDBX_stat *stat, size_t bytes); +LIBMDBX_API int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, MDBX_stat *stat, size_t bytes); /** \brief Return statistics about the MDBX environment. * \ingroup c_statinfo * \deprecated Please use mdbx_env_stat_ex() instead. */ -MDBX_DEPRECATED LIBMDBX_INLINE_API(int, mdbx_env_stat, - (const MDBX_env *env, MDBX_stat *stat, - size_t bytes)) { +MDBX_DEPRECATED LIBMDBX_INLINE_API(int, mdbx_env_stat, (const MDBX_env *env, MDBX_stat *stat, size_t bytes)) { return mdbx_env_stat_ex(env, NULL, stat, bytes); } @@ -2878,10 +2788,10 @@ struct MDBX_envinfo { uint64_t shrink; /**< Shrink threshold for datafile */ uint64_t grow; /**< Growth step for datafile */ } mi_geo; - uint64_t mi_mapsize; /**< Size of the data memory map */ - uint64_t mi_last_pgno; /**< Number of the last used page */ - uint64_t mi_recent_txnid; /**< ID of the last committed transaction */ - uint64_t mi_latter_reader_txnid; /**< ID of the last reader transaction */ + uint64_t mi_mapsize; /**< Size of the data memory map */ + uint64_t mi_last_pgno; /**< Number of the last used page */ + uint64_t mi_recent_txnid; /**< ID of the last committed transaction */ + uint64_t mi_latter_reader_txnid; /**< ID of the last reader transaction */ uint64_t mi_self_latter_reader_txnid; /**< ID of the last reader transaction of caller process */ uint64_t mi_meta_txnid[3], mi_meta_sign[3]; @@ -2940,10 +2850,8 @@ struct MDBX_envinfo { to a disk */ uint64_t prefault; /**< Number of prefault write operations (not a pages) */ uint64_t mincore; /**< Number of mincore() calls */ - uint64_t - msync; /**< Number of explicit msync-to-disk operations (not a pages) */ - uint64_t - fsync; /**< Number of explicit fsync-to-disk operations (not a pages) */ + uint64_t msync; /**< Number of explicit msync-to-disk operations (not a pages) */ + uint64_t fsync; /**< Number of explicit fsync-to-disk operations (not a pages) */ } mi_pgop_stat; /* GUID of the database DXB file. */ @@ -2976,14 +2884,11 @@ typedef struct MDBX_envinfo MDBX_envinfo; * this value is used to provide ABI compatibility. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, - MDBX_envinfo *info, size_t bytes); +LIBMDBX_API int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *info, size_t bytes); /** \brief Return information about the MDBX environment. * \ingroup c_statinfo * \deprecated Please use mdbx_env_info_ex() instead. */ -MDBX_DEPRECATED LIBMDBX_INLINE_API(int, mdbx_env_info, - (const MDBX_env *env, MDBX_envinfo *info, - size_t bytes)) { +MDBX_DEPRECATED LIBMDBX_INLINE_API(int, mdbx_env_info, (const MDBX_env *env, MDBX_envinfo *info, size_t bytes)) { return mdbx_env_info_ex(env, NULL, info, bytes); } @@ -3028,16 +2933,12 @@ LIBMDBX_API int mdbx_env_sync_ex(MDBX_env *env, bool force, bool nonblock); /** \brief The shortcut to calling \ref mdbx_env_sync_ex() with * the `force=true` and `nonblock=false` arguments. * \ingroup c_extra */ -LIBMDBX_INLINE_API(int, mdbx_env_sync, (MDBX_env * env)) { - return mdbx_env_sync_ex(env, true, false); -} +LIBMDBX_INLINE_API(int, mdbx_env_sync, (MDBX_env * env)) { return mdbx_env_sync_ex(env, true, false); } /** \brief The shortcut to calling \ref mdbx_env_sync_ex() with * the `force=false` and `nonblock=true` arguments. * \ingroup c_extra */ -LIBMDBX_INLINE_API(int, mdbx_env_sync_poll, (MDBX_env * env)) { - return mdbx_env_sync_ex(env, false, true); -} +LIBMDBX_INLINE_API(int, mdbx_env_sync_poll, (MDBX_env * env)) { return mdbx_env_sync_ex(env, false, true); } /** \brief Sets threshold to force flush the data buffers to disk, even any of * \ref MDBX_SAFE_NOSYNC flag in the environment. @@ -3062,8 +2963,7 @@ LIBMDBX_INLINE_API(int, mdbx_env_sync_poll, (MDBX_env * env)) { * a synchronous flush would be made. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_INLINE_API(int, mdbx_env_set_syncbytes, - (MDBX_env * env, size_t threshold)) { +LIBMDBX_INLINE_API(int, mdbx_env_set_syncbytes, (MDBX_env * env, size_t threshold)) { return mdbx_env_set_option(env, MDBX_opt_sync_bytes, threshold); } @@ -3081,8 +2981,7 @@ LIBMDBX_INLINE_API(int, mdbx_env_set_syncbytes, * \returns A non-zero error value on failure and 0 on success, * some possible errors are: * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_INLINE_API(int, mdbx_env_get_syncbytes, - (const MDBX_env *env, size_t *threshold)) { +LIBMDBX_INLINE_API(int, mdbx_env_get_syncbytes, (const MDBX_env *env, size_t *threshold)) { int rc = MDBX_EINVAL; if (threshold) { uint64_t proxy = 0; @@ -3125,8 +3024,7 @@ LIBMDBX_INLINE_API(int, mdbx_env_get_syncbytes, * the last unsteady commit. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_INLINE_API(int, mdbx_env_set_syncperiod, - (MDBX_env * env, unsigned seconds_16dot16)) { +LIBMDBX_INLINE_API(int, mdbx_env_set_syncperiod, (MDBX_env * env, unsigned seconds_16dot16)) { return mdbx_env_set_option(env, MDBX_opt_sync_period, seconds_16dot16); } @@ -3146,8 +3044,7 @@ LIBMDBX_INLINE_API(int, mdbx_env_set_syncperiod, * \returns A non-zero error value on failure and 0 on success, * some possible errors are: * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_INLINE_API(int, mdbx_env_get_syncperiod, - (const MDBX_env *env, unsigned *period_seconds_16dot16)) { +LIBMDBX_INLINE_API(int, mdbx_env_get_syncperiod, (const MDBX_env *env, unsigned *period_seconds_16dot16)) { int rc = MDBX_EINVAL; if (period_seconds_16dot16) { uint64_t proxy = 0; @@ -3204,9 +3101,7 @@ LIBMDBX_API int mdbx_env_close_ex(MDBX_env *env, bool dont_sync); /** \brief The shortcut to calling \ref mdbx_env_close_ex() with * the `dont_sync=false` argument. * \ingroup c_opening */ -LIBMDBX_INLINE_API(int, mdbx_env_close, (MDBX_env * env)) { - return mdbx_env_close_ex(env, false); -} +LIBMDBX_INLINE_API(int, mdbx_env_close, (MDBX_env * env)) { return mdbx_env_close_ex(env, false); } #if defined(DOXYGEN) || !(defined(_WIN32) || defined(_WIN64)) /** \brief Восстанавливает экземпляр среды в дочернем процессе после ветвления @@ -3369,8 +3264,7 @@ DEFINE_ENUM_FLAG_OPERATORS(MDBX_warmup_flags) * * \retval MDBX_RESULT_TRUE The specified timeout is reached during load * data into memory. */ -LIBMDBX_API int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, - MDBX_warmup_flags_t flags, +LIBMDBX_API int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, MDBX_warmup_flags_t flags, unsigned timeout_seconds_16dot16); /** \brief Set environment flags. @@ -3393,8 +3287,7 @@ LIBMDBX_API int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, * \returns A non-zero error value on failure and 0 on success, * some possible errors are: * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags, - bool onoff); +LIBMDBX_API int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags, bool onoff); /** \brief Get environment flags. * \ingroup c_statinfo @@ -3644,16 +3537,12 @@ LIBMDBX_API int mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *fd); * \retval MDBX_TOO_LARGE Specified size is too large, i.e. too many pages for * given size, or a 32-bit process requests too much * bytes for the 32-bit address space. */ -LIBMDBX_API int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, - intptr_t size_now, intptr_t size_upper, - intptr_t growth_step, - intptr_t shrink_threshold, - intptr_t pagesize); +LIBMDBX_API int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t size_now, intptr_t size_upper, + intptr_t growth_step, intptr_t shrink_threshold, intptr_t pagesize); /** \deprecated Please use \ref mdbx_env_set_geometry() instead. * \ingroup c_settings */ -MDBX_DEPRECATED LIBMDBX_INLINE_API(int, mdbx_env_set_mapsize, - (MDBX_env * env, size_t size)) { +MDBX_DEPRECATED LIBMDBX_INLINE_API(int, mdbx_env_set_mapsize, (MDBX_env * env, size_t size)) { return mdbx_env_set_geometry(env, size, size, size, -1, -1, -1); } @@ -3672,81 +3561,66 @@ MDBX_DEPRECATED LIBMDBX_INLINE_API(int, mdbx_env_set_mapsize, * i.e. \ref MDBX_NORDAHEAD is useful to * open environment by \ref mdbx_env_open(). * \retval Otherwise the error code. */ -LIBMDBX_API int mdbx_is_readahead_reasonable(size_t volume, - intptr_t redundancy); +LIBMDBX_API int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy); /** \brief Returns the minimal database page size in bytes. * \ingroup c_statinfo */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(intptr_t, mdbx_limits_pgsize_min, - (void)) { - return MDBX_MIN_PAGESIZE; -} +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(intptr_t, mdbx_limits_pgsize_min, (void)) { return MDBX_MIN_PAGESIZE; } /** \brief Returns the maximal database page size in bytes. * \ingroup c_statinfo */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(intptr_t, mdbx_limits_pgsize_max, - (void)) { - return MDBX_MAX_PAGESIZE; -} +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(intptr_t, mdbx_limits_pgsize_max, (void)) { return MDBX_MAX_PAGESIZE; } /** \brief Returns minimal database size in bytes for given page size, * or -1 if pagesize is invalid. * \ingroup c_statinfo */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_dbsize_min(intptr_t pagesize); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_dbsize_min(intptr_t pagesize); /** \brief Returns maximal database size in bytes for given page size, * or -1 if pagesize is invalid. * \ingroup c_statinfo */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_dbsize_max(intptr_t pagesize); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_dbsize_max(intptr_t pagesize); /** \brief Returns maximal key size in bytes for given page size * and table flags, or -1 if pagesize is invalid. * \ingroup c_statinfo * \see db_flags */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_keysize_max(intptr_t pagesize, MDBX_db_flags_t flags); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_keysize_max(intptr_t pagesize, MDBX_db_flags_t flags); /** \brief Returns minimal key size in bytes for given table flags. * \ingroup c_statinfo * \see db_flags */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_keysize_min(MDBX_db_flags_t flags); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_keysize_min(MDBX_db_flags_t flags); /** \brief Returns maximal data size in bytes for given page size * and table flags, or -1 if pagesize is invalid. * \ingroup c_statinfo * \see db_flags */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_valsize_max(intptr_t pagesize, MDBX_db_flags_t flags); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_valsize_max(intptr_t pagesize, MDBX_db_flags_t flags); /** \brief Returns minimal data size in bytes for given table flags. * \ingroup c_statinfo * \see db_flags */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_valsize_min(MDBX_db_flags_t flags); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_valsize_min(MDBX_db_flags_t flags); /** \brief Returns maximal size of key-value pair to fit in a single page with * the given size and table flags, or -1 if pagesize is invalid. * \ingroup c_statinfo * \see db_flags */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_pairsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_pairsize4page_max(intptr_t pagesize, + MDBX_db_flags_t flags); /** \brief Returns maximal data size in bytes to fit in a leaf-page or * single large/overflow-page with the given page size and table flags, * or -1 if pagesize is invalid. * \ingroup c_statinfo * \see db_flags */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_valsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_valsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags); /** \brief Returns maximal write transaction size (i.e. limit for summary volume * of dirty pages) in bytes for given page size, or -1 if pagesize is invalid. * \ingroup c_statinfo */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_txnsize_max(intptr_t pagesize); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_txnsize_max(intptr_t pagesize); /** \brief Set the maximum number of threads/reader slots for for all processes * interacts with the database. @@ -3771,8 +3645,7 @@ mdbx_limits_txnsize_max(intptr_t pagesize); * some possible errors are: * \retval MDBX_EINVAL An invalid parameter was specified. * \retval MDBX_EPERM The environment is already open. */ -LIBMDBX_INLINE_API(int, mdbx_env_set_maxreaders, - (MDBX_env * env, unsigned readers)) { +LIBMDBX_INLINE_API(int, mdbx_env_set_maxreaders, (MDBX_env * env, unsigned readers)) { return mdbx_env_set_option(env, MDBX_opt_max_readers, readers); } @@ -3787,8 +3660,7 @@ LIBMDBX_INLINE_API(int, mdbx_env_set_maxreaders, * \returns A non-zero error value on failure and 0 on success, * some possible errors are: * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_INLINE_API(int, mdbx_env_get_maxreaders, - (const MDBX_env *env, unsigned *readers)) { +LIBMDBX_INLINE_API(int, mdbx_env_get_maxreaders, (const MDBX_env *env, unsigned *readers)) { int rc = MDBX_EINVAL; if (readers) { uint64_t proxy = 0; @@ -3833,8 +3705,7 @@ LIBMDBX_INLINE_API(int, mdbx_env_set_maxdbs, (MDBX_env * env, MDBX_dbi dbs)) { * \returns A non-zero error value on failure and 0 on success, * some possible errors are: * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_INLINE_API(int, mdbx_env_get_maxdbs, - (const MDBX_env *env, MDBX_dbi *dbs)) { +LIBMDBX_INLINE_API(int, mdbx_env_get_maxdbs, (const MDBX_env *env, MDBX_dbi *dbs)) { int rc = MDBX_EINVAL; if (dbs) { uint64_t proxy = 0; @@ -3864,8 +3735,7 @@ MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API size_t mdbx_default_pagesize(void); * available/free RAM pages will be stored. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, - intptr_t *avail_pages); +LIBMDBX_API int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, intptr_t *avail_pages); /** \brief Returns the maximum size of keys can put. * \ingroup c_statinfo @@ -3876,8 +3746,7 @@ LIBMDBX_API int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, * * \returns The maximum size of a key can write, * or -1 if something is wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_env_get_maxkeysize_ex(const MDBX_env *env, MDBX_db_flags_t flags); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_env_get_maxkeysize_ex(const MDBX_env *env, MDBX_db_flags_t flags); /** \brief Returns the maximum size of data we can put. * \ingroup c_statinfo @@ -3888,14 +3757,12 @@ mdbx_env_get_maxkeysize_ex(const MDBX_env *env, MDBX_db_flags_t flags); * * \returns The maximum size of a data can write, * or -1 if something is wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_env_get_maxvalsize_ex(const MDBX_env *env, MDBX_db_flags_t flags); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_env_get_maxvalsize_ex(const MDBX_env *env, MDBX_db_flags_t flags); /** \deprecated Please use \ref mdbx_env_get_maxkeysize_ex() * and/or \ref mdbx_env_get_maxvalsize_ex() * \ingroup c_statinfo */ -MDBX_NOTHROW_PURE_FUNCTION MDBX_DEPRECATED LIBMDBX_API int -mdbx_env_get_maxkeysize(const MDBX_env *env); +MDBX_NOTHROW_PURE_FUNCTION MDBX_DEPRECATED LIBMDBX_API int mdbx_env_get_maxkeysize(const MDBX_env *env); /** \brief Returns maximal size of key-value pair to fit in a single page * for specified table flags. @@ -3907,8 +3774,7 @@ mdbx_env_get_maxkeysize(const MDBX_env *env); * * \returns The maximum size of a data can write, * or -1 if something is wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_env_get_pairsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_env_get_pairsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags); /** \brief Returns maximal data size in bytes to fit in a leaf-page or * single large/overflow-page for specified table flags. @@ -3920,8 +3786,7 @@ mdbx_env_get_pairsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags); * * \returns The maximum size of a data can write, * or -1 if something is wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_env_get_valsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_env_get_valsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags); /** \brief Sets application information (a context pointer) associated with * the environment. @@ -3942,8 +3807,7 @@ LIBMDBX_API int mdbx_env_set_userctx(MDBX_env *env, void *ctx); * \param [in] env An environment handle returned by \ref mdbx_env_create() * \returns The pointer set by \ref mdbx_env_set_userctx() * or `NULL` if something wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void * -mdbx_env_get_userctx(const MDBX_env *env); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void *mdbx_env_get_userctx(const MDBX_env *env); /** \brief Create a transaction with a user provided context pointer * for use with the environment. @@ -4004,8 +3868,7 @@ mdbx_env_get_userctx(const MDBX_env *env); * \retval MDBX_ENOMEM Out of memory. * \retval MDBX_BUSY The write transaction is already started by the * current thread. */ -LIBMDBX_API int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, - MDBX_txn_flags_t flags, MDBX_txn **txn, +LIBMDBX_API int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, MDBX_txn **txn, void *context); /** \brief Create a transaction for use with the environment. @@ -4062,9 +3925,7 @@ LIBMDBX_API int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, * \retval MDBX_ENOMEM Out of memory. * \retval MDBX_BUSY The write transaction is already started by the * current thread. */ -LIBMDBX_INLINE_API(int, mdbx_txn_begin, - (MDBX_env * env, MDBX_txn *parent, MDBX_txn_flags_t flags, - MDBX_txn **txn)) { +LIBMDBX_INLINE_API(int, mdbx_txn_begin, (MDBX_env * env, MDBX_txn *parent, MDBX_txn_flags_t flags, MDBX_txn **txn)) { return mdbx_txn_begin_ex(env, parent, flags, txn, NULL); } @@ -4090,8 +3951,7 @@ LIBMDBX_API int mdbx_txn_set_userctx(MDBX_txn *txn, void *ctx); * \returns The pointer which was passed via the `context` parameter * of `mdbx_txn_begin_ex()` or set by \ref mdbx_txn_set_userctx(), * or `NULL` if something wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void * -mdbx_txn_get_userctx(const MDBX_txn *txn); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void *mdbx_txn_get_userctx(const MDBX_txn *txn); /** \brief Information about the transaction * \ingroup c_statinfo @@ -4158,15 +4018,13 @@ typedef struct MDBX_txn_info MDBX_txn_info; * See description of \ref MDBX_txn_info. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, - bool scan_rlt); +LIBMDBX_API int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt); /** \brief Returns the transaction's MDBX_env. * \ingroup c_transactions * * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin() */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_env * -mdbx_txn_env(const MDBX_txn *txn); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_env *mdbx_txn_env(const MDBX_txn *txn); /** \brief Return the transaction's flags. * \ingroup c_transactions @@ -4177,8 +4035,7 @@ mdbx_txn_env(const MDBX_txn *txn); * * \returns A transaction flags, valid if input is an valid transaction, * otherwise \ref MDBX_TXN_INVALID. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_txn_flags_t -mdbx_txn_flags(const MDBX_txn *txn); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_txn_flags_t mdbx_txn_flags(const MDBX_txn *txn); /** \brief Return the transaction's ID. * \ingroup c_statinfo @@ -4191,8 +4048,7 @@ mdbx_txn_flags(const MDBX_txn *txn); * * \returns A transaction ID, valid if input is an active transaction, * otherwise 0. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint64_t -mdbx_txn_id(const MDBX_txn *txn); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint64_t mdbx_txn_id(const MDBX_txn *txn); /** \brief Latency of commit stages in 1/65536 of seconds units. * \warning This structure may be changed in future releases. @@ -4335,9 +4191,7 @@ LIBMDBX_API int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency); * \retval MDBX_EIO An error occurred during the flushing/writing * data to a storage medium/disk. * \retval MDBX_ENOMEM Out of memory. */ -LIBMDBX_INLINE_API(int, mdbx_txn_commit, (MDBX_txn * txn)) { - return mdbx_txn_commit_ex(txn, NULL); -} +LIBMDBX_INLINE_API(int, mdbx_txn_commit, (MDBX_txn * txn)) { return mdbx_txn_commit_ex(txn, NULL); } /** \brief Abandon all the operations of the transaction instead of saving them. * \ingroup c_transactions @@ -4602,8 +4456,7 @@ LIBMDBX_API int mdbx_canary_get(const MDBX_txn *txn, MDBX_canary *canary); * You have been warned but still can use custom comparators knowing * about the issues noted above. In this case you should ignore `deprecated` * warnings or define `MDBX_DEPRECATED` macro to empty to avoid ones. */ -typedef int(MDBX_cmp_func)(const MDBX_val *a, - const MDBX_val *b) MDBX_CXX17_NOEXCEPT; +typedef int(MDBX_cmp_func)(const MDBX_val *a, const MDBX_val *b) MDBX_CXX17_NOEXCEPT; /** \brief Open or Create a named table in the environment. * \ingroup c_dbi @@ -4694,12 +4547,10 @@ typedef int(MDBX_cmp_func)(const MDBX_val *a, * opened with a different comparison function(s). * \retval MDBX_THREAD_MISMATCH Given transaction is not owned * by current thread. */ -LIBMDBX_API int mdbx_dbi_open(MDBX_txn *txn, const char *name, - MDBX_db_flags_t flags, MDBX_dbi *dbi); +LIBMDBX_API int mdbx_dbi_open(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, MDBX_dbi *dbi); /** \copydoc mdbx_dbi_open() * \ingroup c_dbi */ -LIBMDBX_API int mdbx_dbi_open2(MDBX_txn *txn, const MDBX_val *name, - MDBX_db_flags_t flags, MDBX_dbi *dbi); +LIBMDBX_API int mdbx_dbi_open2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, MDBX_dbi *dbi); /** \brief Open or Create a named table in the environment * with using custom comparison functions. @@ -4717,14 +4568,12 @@ LIBMDBX_API int mdbx_dbi_open2(MDBX_txn *txn, const MDBX_val *name, * \param [in] datacmp Optional custom data comparison function for a table. * \param [out] dbi Address where the new MDBX_dbi handle will be stored. * \returns A non-zero error value on failure and 0 on success. */ -MDBX_DEPRECATED LIBMDBX_API int -mdbx_dbi_open_ex(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, - MDBX_dbi *dbi, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp); +MDBX_DEPRECATED LIBMDBX_API int mdbx_dbi_open_ex(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, MDBX_dbi *dbi, + MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp); /** \copydoc mdbx_dbi_open_ex() * \ingroup c_dbi */ -MDBX_DEPRECATED LIBMDBX_API int -mdbx_dbi_open_ex2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, - MDBX_dbi *dbi, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp); +MDBX_DEPRECATED LIBMDBX_API int mdbx_dbi_open_ex2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, + MDBX_dbi *dbi, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp); /** \brief Переименовает таблицу по DBI-дескриптору * @@ -4744,8 +4593,7 @@ mdbx_dbi_open_ex2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, LIBMDBX_API int mdbx_dbi_rename(MDBX_txn *txn, MDBX_dbi dbi, const char *name); /** \copydoc mdbx_dbi_rename() * \ingroup c_dbi */ -LIBMDBX_API int mdbx_dbi_rename2(MDBX_txn *txn, MDBX_dbi dbi, - const MDBX_val *name); +LIBMDBX_API int mdbx_dbi_rename2(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *name); /** \brief Функция обратного вызова для перечисления * пользовательских именованных таблиц. @@ -4766,10 +4614,8 @@ LIBMDBX_API int mdbx_dbi_rename2(MDBX_txn *txn, MDBX_dbi dbi, * \returns Ноль при успехе и продолжении перечисления, при возвращении другого * значения оно будет немедленно возвращено вызывающему * без продолжения перечисления. */ -typedef int(MDBX_table_enum_func)(void *ctx, const MDBX_txn *txn, - const MDBX_val *name, MDBX_db_flags_t flags, - const struct MDBX_stat *stat, - MDBX_dbi dbi) MDBX_CXX17_NOEXCEPT; +typedef int(MDBX_table_enum_func)(void *ctx, const MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, + const struct MDBX_stat *stat, MDBX_dbi dbi) MDBX_CXX17_NOEXCEPT; /** \brief Перечисляет пользовательские именнованные таблицы. * @@ -4791,8 +4637,7 @@ typedef int(MDBX_table_enum_func)(void *ctx, const MDBX_txn *txn, * в функцию `func()` как есть. * * \returns Ненулевое значение кода ошибки, либо 0 при успешном выполнении. */ -LIBMDBX_API int mdbx_enumerate_tables(const MDBX_txn *txn, - MDBX_table_enum_func *func, void *ctx); +LIBMDBX_API int mdbx_enumerate_tables(const MDBX_txn *txn, MDBX_table_enum_func *func, void *ctx); /** \defgroup value2key Value-to-Key functions * \brief Value-to-Key functions to @@ -4805,28 +4650,21 @@ LIBMDBX_API int mdbx_enumerate_tables(const MDBX_txn *txn, * and IEEE754 double values in one index for JSON-numbers with restriction for * integer numbers range corresponding to RFC-7159, i.e. \f$[-2^{53}+1, * 2^{53}-1]\f$. See bottom of page 6 at https://tools.ietf.org/html/rfc7159 */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint64_t -mdbx_key_from_jsonInteger(const int64_t json_integer); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint64_t mdbx_key_from_jsonInteger(const int64_t json_integer); -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint64_t -mdbx_key_from_double(const double ieee754_64bit); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint64_t mdbx_key_from_double(const double ieee754_64bit); -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint64_t -mdbx_key_from_ptrdouble(const double *const ieee754_64bit); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint64_t mdbx_key_from_ptrdouble(const double *const ieee754_64bit); -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint32_t -mdbx_key_from_float(const float ieee754_32bit); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint32_t mdbx_key_from_float(const float ieee754_32bit); -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint32_t -mdbx_key_from_ptrfloat(const float *const ieee754_32bit); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint32_t mdbx_key_from_ptrfloat(const float *const ieee754_32bit); -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(uint64_t, mdbx_key_from_int64, - (const int64_t i64)) { +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(uint64_t, mdbx_key_from_int64, (const int64_t i64)) { return UINT64_C(0x8000000000000000) + i64; } -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(uint32_t, mdbx_key_from_int32, - (const int32_t i32)) { +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(uint32_t, mdbx_key_from_int32, (const int32_t i32)) { return UINT32_C(0x80000000) + i32; } /** end of value2key @} */ @@ -4836,20 +4674,15 @@ MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(uint32_t, mdbx_key_from_int32, * \ref avoid_custom_comparators "avoid using custom comparators" * \see value2key * @{ */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int64_t -mdbx_jsonInteger_from_key(const MDBX_val); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int64_t mdbx_jsonInteger_from_key(const MDBX_val); -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API double -mdbx_double_from_key(const MDBX_val); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API double mdbx_double_from_key(const MDBX_val); -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API float -mdbx_float_from_key(const MDBX_val); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API float mdbx_float_from_key(const MDBX_val); -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int32_t -mdbx_int32_from_key(const MDBX_val); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int32_t mdbx_int32_from_key(const MDBX_val); -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int64_t -mdbx_int64_from_key(const MDBX_val); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int64_t mdbx_int64_from_key(const MDBX_val); /** end of value2key @} */ /** \brief Retrieve statistics for a table. @@ -4866,8 +4699,7 @@ mdbx_int64_from_key(const MDBX_val); * \retval MDBX_THREAD_MISMATCH Given transaction is not owned * by current thread. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_dbi_stat(const MDBX_txn *txn, MDBX_dbi dbi, - MDBX_stat *stat, size_t bytes); +LIBMDBX_API int mdbx_dbi_stat(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *stat, size_t bytes); /** \brief Retrieve depth (bitmask) information of nested dupsort (multi-value) * B+trees for given table. @@ -4884,8 +4716,7 @@ LIBMDBX_API int mdbx_dbi_stat(const MDBX_txn *txn, MDBX_dbi dbi, * by current thread. * \retval MDBX_EINVAL An invalid parameter was specified. * \retval MDBX_RESULT_TRUE The dbi isn't a dupsort (multi-value) table. */ -LIBMDBX_API int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi, - uint32_t *mask); +LIBMDBX_API int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi, uint32_t *mask); /** \brief DBI state bits returted by \ref mdbx_dbi_flags_ex() * \ingroup c_statinfo @@ -4913,14 +4744,12 @@ DEFINE_ENUM_FLAG_OPERATORS(MDBX_dbi_state) * \param [out] state Address where the state will be returned. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_dbi_flags_ex(const MDBX_txn *txn, MDBX_dbi dbi, - unsigned *flags, unsigned *state); +LIBMDBX_API int mdbx_dbi_flags_ex(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags, unsigned *state); /** \brief The shortcut to calling \ref mdbx_dbi_flags_ex() with `state=NULL` * for discarding it result. * \ingroup c_statinfo * \see MDBX_db_flags_t */ -LIBMDBX_INLINE_API(int, mdbx_dbi_flags, - (const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags)) { +LIBMDBX_INLINE_API(int, mdbx_dbi_flags, (const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags)) { unsigned state; return mdbx_dbi_flags_ex(txn, dbi, flags, &state); } @@ -4999,8 +4828,7 @@ LIBMDBX_API int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, bool del); * by current thread. * \retval MDBX_NOTFOUND The key was not in the table. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - MDBX_val *data); +LIBMDBX_API int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data); /** \brief Get items from a table * and optionally number of data items for a given key. @@ -5032,8 +4860,7 @@ LIBMDBX_API int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, * by current thread. * \retval MDBX_NOTFOUND The key was not in the table. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, - MDBX_val *data, size_t *values_count); +LIBMDBX_API int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, size_t *values_count); /** \brief Get equal or great item from a table. * \ingroup c_crud @@ -5063,8 +4890,7 @@ LIBMDBX_API int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, * by current thread. * \retval MDBX_NOTFOUND The key was not in the table. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, - MDBX_val *key, MDBX_val *data); +LIBMDBX_API int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data); /** \brief Store items into a table. * \ingroup c_crud @@ -5147,8 +4973,7 @@ LIBMDBX_API int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, * \retval MDBX_EACCES An attempt was made to write * in a read-only transaction. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - MDBX_val *data, MDBX_put_flags_t flags); +LIBMDBX_API int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data, MDBX_put_flags_t flags); /** \brief Replace items in a table. * \ingroup c_crud @@ -5193,16 +5018,12 @@ LIBMDBX_API int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, * \see \ref c_crud_hints "Quick reference for Insert/Update/Delete operations" * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - MDBX_val *new_data, MDBX_val *old_data, +LIBMDBX_API int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *new_data, MDBX_val *old_data, MDBX_put_flags_t flags); -typedef int (*MDBX_preserve_func)(void *context, MDBX_val *target, - const void *src, size_t bytes); -LIBMDBX_API int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, - const MDBX_val *key, MDBX_val *new_data, - MDBX_val *old_data, MDBX_put_flags_t flags, - MDBX_preserve_func preserver, +typedef int (*MDBX_preserve_func)(void *context, MDBX_val *target, const void *src, size_t bytes); +LIBMDBX_API int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *new_data, + MDBX_val *old_data, MDBX_put_flags_t flags, MDBX_preserve_func preserver, void *preserver_context); /** \brief Delete items from a table. @@ -5230,8 +5051,7 @@ LIBMDBX_API int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, * \retval MDBX_EACCES An attempt was made to write * in a read-only transaction. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - const MDBX_val *data); +LIBMDBX_API int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, const MDBX_val *data); /** \brief Create a cursor handle but not bind it to transaction nor DBI-handle. * \ingroup c_cursors @@ -5277,8 +5097,7 @@ LIBMDBX_API int mdbx_cursor_set_userctx(MDBX_cursor *cursor, void *ctx); * \returns The pointer which was passed via the `context` parameter * of `mdbx_cursor_create()` or set by \ref mdbx_cursor_set_userctx(), * or `NULL` if something wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void * -mdbx_cursor_get_userctx(const MDBX_cursor *cursor); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void *mdbx_cursor_get_userctx(const MDBX_cursor *cursor); /** \brief Bind cursor to specified transaction and DBI-handle. * \ingroup c_cursors @@ -5305,8 +5124,7 @@ mdbx_cursor_get_userctx(const MDBX_cursor *cursor); * \retval MDBX_THREAD_MISMATCH Given transaction is not owned * by current thread. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_cursor_bind(const MDBX_txn *txn, MDBX_cursor *cursor, - MDBX_dbi dbi); +LIBMDBX_API int mdbx_cursor_bind(const MDBX_txn *txn, MDBX_cursor *cursor, MDBX_dbi dbi); /** \brief Unbind cursor from a transaction. * \ingroup c_cursors @@ -5375,8 +5193,7 @@ LIBMDBX_API int mdbx_cursor_reset(MDBX_cursor *cursor); * \retval MDBX_THREAD_MISMATCH Given transaction is not owned * by current thread. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_cursor_open(const MDBX_txn *txn, MDBX_dbi dbi, - MDBX_cursor **cursor); +LIBMDBX_API int mdbx_cursor_open(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_cursor **cursor); /** \brief Close a cursor handle. * \ingroup c_cursors @@ -5445,8 +5262,7 @@ LIBMDBX_API int mdbx_cursor_renew(const MDBX_txn *txn, MDBX_cursor *cursor); * \ingroup c_cursors * * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_txn * -mdbx_cursor_txn(const MDBX_cursor *cursor); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_txn *mdbx_cursor_txn(const MDBX_cursor *cursor); /** \brief Return the cursor's table handle. * \ingroup c_cursors @@ -5487,9 +5303,7 @@ LIBMDBX_API int mdbx_cursor_copy(const MDBX_cursor *src, MDBX_cursor *dest); * * \retval Значение со знаком в семантике оператора `<=>` (меньше нуля, ноль, * либо больше нуля) как результат сравнения позиций курсоров. */ -LIBMDBX_API int mdbx_cursor_compare(const MDBX_cursor *left, - const MDBX_cursor *right, - bool ignore_multival); +LIBMDBX_API int mdbx_cursor_compare(const MDBX_cursor *left, const MDBX_cursor *right, bool ignore_multival); /** \brief Retrieve by cursor. * \ingroup c_crud @@ -5522,8 +5336,7 @@ LIBMDBX_API int mdbx_cursor_compare(const MDBX_cursor *left, * by current thread. * \retval MDBX_NOTFOUND No matching key found. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_cursor_get(MDBX_cursor *cursor, MDBX_val *key, - MDBX_val *data, MDBX_cursor_op op); +LIBMDBX_API int mdbx_cursor_get(MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op); /** \brief Служебная функция для использования в утилитах. * \ingroup c_extra @@ -5569,8 +5382,7 @@ LIBMDBX_API int mdbx_cursor_ignord(MDBX_cursor *cursor); * * \see mdbx_cursor_scan() * \see mdbx_cursor_scan_from() */ -typedef int(MDBX_predicate_func)(void *context, MDBX_val *key, MDBX_val *value, - void *arg) MDBX_CXX17_NOEXCEPT; +typedef int(MDBX_predicate_func)(void *context, MDBX_val *key, MDBX_val *value, void *arg) MDBX_CXX17_NOEXCEPT; /** \brief Сканирует таблицу с использованием передаваемого предиката, * с уменьшением сопутствующих накладных расходов. @@ -5640,10 +5452,8 @@ typedef int(MDBX_predicate_func)(void *context, MDBX_val *key, MDBX_val *value, * и \ref MDBX_RESULT_FALSE, является кодом ошибки при позиционировании * курса, либо определяемым пользователем кодом остановки поиска * или ошибочной ситуации. */ -LIBMDBX_API int mdbx_cursor_scan(MDBX_cursor *cursor, - MDBX_predicate_func *predicate, void *context, - MDBX_cursor_op start_op, - MDBX_cursor_op turn_op, void *arg); +LIBMDBX_API int mdbx_cursor_scan(MDBX_cursor *cursor, MDBX_predicate_func *predicate, void *context, + MDBX_cursor_op start_op, MDBX_cursor_op turn_op, void *arg); /** Сканирует таблицу с использованием передаваемого предиката, * начиная с передаваемой пары ключ-значение, @@ -5729,10 +5539,8 @@ LIBMDBX_API int mdbx_cursor_scan(MDBX_cursor *cursor, * и \ref MDBX_RESULT_FALSE, является кодом ошибки при позиционировании * курса, либо определяемым пользователем кодом остановки поиска * или ошибочной ситуации. */ -LIBMDBX_API int mdbx_cursor_scan_from(MDBX_cursor *cursor, - MDBX_predicate_func *predicate, - void *context, MDBX_cursor_op from_op, - MDBX_val *from_key, MDBX_val *from_value, +LIBMDBX_API int mdbx_cursor_scan_from(MDBX_cursor *cursor, MDBX_predicate_func *predicate, void *context, + MDBX_cursor_op from_op, MDBX_val *from_key, MDBX_val *from_value, MDBX_cursor_op turn_op, void *arg); /** \brief Retrieve multiple non-dupsort key/value pairs by cursor. @@ -5776,8 +5584,7 @@ LIBMDBX_API int mdbx_cursor_scan_from(MDBX_cursor *cursor, * \retval MDBX_RESULT_TRUE The returned chunk is the last one, * and there are no pairs left. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_cursor_get_batch(MDBX_cursor *cursor, size_t *count, - MDBX_val *pairs, size_t limit, +LIBMDBX_API int mdbx_cursor_get_batch(MDBX_cursor *cursor, size_t *count, MDBX_val *pairs, size_t limit, MDBX_cursor_op op); /** \brief Store by cursor. @@ -5860,8 +5667,7 @@ LIBMDBX_API int mdbx_cursor_get_batch(MDBX_cursor *cursor, size_t *count, * \retval MDBX_EACCES An attempt was made to write in a read-only * transaction. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_cursor_put(MDBX_cursor *cursor, const MDBX_val *key, - MDBX_val *data, MDBX_put_flags_t flags); +LIBMDBX_API int mdbx_cursor_put(MDBX_cursor *cursor, const MDBX_val *key, MDBX_val *data, MDBX_put_flags_t flags); /** \brief Delete current key/data pair. * \ingroup c_crud @@ -5924,8 +5730,7 @@ LIBMDBX_API int mdbx_cursor_count(const MDBX_cursor *cursor, size_t *pcount); * positioned * \retval MDBX_RESULT_FALSE A data is available * \retval Otherwise the error code */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_cursor_eof(const MDBX_cursor *cursor); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cursor_eof(const MDBX_cursor *cursor); /** \brief Determines whether the cursor is pointed to the first key-value pair * or not. @@ -5938,8 +5743,7 @@ mdbx_cursor_eof(const MDBX_cursor *cursor); * \retval MDBX_RESULT_TRUE Cursor positioned to the first key-value pair * \retval MDBX_RESULT_FALSE Cursor NOT positioned to the first key-value * pair \retval Otherwise the error code */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_cursor_on_first(const MDBX_cursor *cursor); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cursor_on_first(const MDBX_cursor *cursor); /** \brief Определяет стоит ли курсор на первом или единственном * мульти-значении соответствующем ключу. @@ -5952,8 +5756,7 @@ mdbx_cursor_on_first(const MDBX_cursor *cursor); * \retval MDBX_RESULT_FALSE курсор НЕ установлен на первом или единственном * мульти-значении соответствующем ключу. * \retval ИНАЧЕ код ошибки. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_cursor_on_first_dup(const MDBX_cursor *cursor); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cursor_on_first_dup(const MDBX_cursor *cursor); /** \brief Determines whether the cursor is pointed to the last key-value pair * or not. @@ -5966,8 +5769,7 @@ mdbx_cursor_on_first_dup(const MDBX_cursor *cursor); * \retval MDBX_RESULT_TRUE Cursor positioned to the last key-value pair * \retval MDBX_RESULT_FALSE Cursor NOT positioned to the last key-value pair * \retval Otherwise the error code */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_cursor_on_last(const MDBX_cursor *cursor); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cursor_on_last(const MDBX_cursor *cursor); /** \brief Определяет стоит ли курсор на последнем или единственном * мульти-значении соответствующем ключу. @@ -5980,8 +5782,7 @@ mdbx_cursor_on_last(const MDBX_cursor *cursor); * \retval MDBX_RESULT_FALSE курсор НЕ установлен на последнем или единственном * мульти-значении соответствующем ключу. * \retval ИНАЧЕ код ошибки. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_cursor_on_last_dup(const MDBX_cursor *cursor); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cursor_on_last_dup(const MDBX_cursor *cursor); /** \addtogroup c_rqest * \details \note The estimation result varies greatly depending on the filling @@ -6027,9 +5828,7 @@ mdbx_cursor_on_last_dup(const MDBX_cursor *cursor); * i.e. `*distance_items = distance(first, last)`. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_estimate_distance(const MDBX_cursor *first, - const MDBX_cursor *last, - ptrdiff_t *distance_items); +LIBMDBX_API int mdbx_estimate_distance(const MDBX_cursor *first, const MDBX_cursor *last, ptrdiff_t *distance_items); /** \brief Estimates the move distance. * \ingroup c_rqest @@ -6051,8 +5850,7 @@ LIBMDBX_API int mdbx_estimate_distance(const MDBX_cursor *first, * as the number of elements. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, - MDBX_val *data, MDBX_cursor_op move_op, +LIBMDBX_API int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, MDBX_cursor_op move_op, ptrdiff_t *distance_items); /** \brief Estimates the size of a range as a number of elements. @@ -6079,11 +5877,8 @@ LIBMDBX_API int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, * \param [out] distance_items A pointer to store range estimation result. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, - const MDBX_val *begin_key, - const MDBX_val *begin_data, - const MDBX_val *end_key, - const MDBX_val *end_data, +LIBMDBX_API int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *begin_key, + const MDBX_val *begin_data, const MDBX_val *end_key, const MDBX_val *end_data, ptrdiff_t *distance_items); /** \brief The EPSILON value for mdbx_estimate_range() @@ -6123,8 +5918,7 @@ LIBMDBX_API int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, * \retval MDBX_RESULT_TRUE Given address is on the dirty page. * \retval MDBX_RESULT_FALSE Given address is NOT on the dirty page. * \retval Otherwise the error code. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_is_dirty(const MDBX_txn *txn, - const void *ptr); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr); /** \brief Sequence generation for a table. * \ingroup c_crud @@ -6147,8 +5941,7 @@ MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_is_dirty(const MDBX_txn *txn, * some possible errors are: * \retval MDBX_RESULT_TRUE Increasing the sequence has resulted in an * overflow and therefore cannot be executed. */ -LIBMDBX_API int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, - uint64_t increment); +LIBMDBX_API int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, uint64_t increment); /** \brief Compare two keys according to a particular table. * \ingroup c_crud @@ -6165,15 +5958,12 @@ LIBMDBX_API int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, * \param [in] b The second item to compare. * * \returns < 0 if a < b, 0 if a == b, > 0 if a > b */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cmp(const MDBX_txn *txn, - MDBX_dbi dbi, - const MDBX_val *a, +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, const MDBX_val *b); /** \brief Returns default internal key's comparator for given table flags. * \ingroup c_extra */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API MDBX_cmp_func * -mdbx_get_keycmp(MDBX_db_flags_t flags); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API MDBX_cmp_func *mdbx_get_keycmp(MDBX_db_flags_t flags); /** \brief Compare two data items according to a particular table. * \ingroup c_crud @@ -6190,15 +5980,12 @@ mdbx_get_keycmp(MDBX_db_flags_t flags); * \param [in] b The second item to compare. * * \returns < 0 if a < b, 0 if a == b, > 0 if a > b */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_dcmp(const MDBX_txn *txn, - MDBX_dbi dbi, - const MDBX_val *a, +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_dcmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, const MDBX_val *b); /** \brief Returns default internal data's comparator for given table flags * \ingroup c_extra */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API MDBX_cmp_func * -mdbx_get_datacmp(MDBX_db_flags_t flags); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API MDBX_cmp_func *mdbx_get_datacmp(MDBX_db_flags_t flags); /** \brief A callback function used to enumerate the reader lock table. * \ingroup c_statinfo @@ -6225,10 +6012,8 @@ mdbx_get_datacmp(MDBX_db_flags_t flags); * for reuse by completion read transaction. * * \returns < 0 on failure, >= 0 on success. \see mdbx_reader_list() */ -typedef int(MDBX_reader_list_func)(void *ctx, int num, int slot, mdbx_pid_t pid, - mdbx_tid_t thread, uint64_t txnid, - uint64_t lag, size_t bytes_used, - size_t bytes_retained) MDBX_CXX17_NOEXCEPT; +typedef int(MDBX_reader_list_func)(void *ctx, int num, int slot, mdbx_pid_t pid, mdbx_tid_t thread, uint64_t txnid, + uint64_t lag, size_t bytes_used, size_t bytes_retained) MDBX_CXX17_NOEXCEPT; /** \brief Enumerate the entries in the reader lock table. * @@ -6241,8 +6026,7 @@ typedef int(MDBX_reader_list_func)(void *ctx, int num, int slot, mdbx_pid_t pid, * * \returns A non-zero error value on failure and 0 on success, * or \ref MDBX_RESULT_TRUE if the reader lock table is empty. */ -LIBMDBX_API int mdbx_reader_list(const MDBX_env *env, - MDBX_reader_list_func *func, void *ctx); +LIBMDBX_API int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func, void *ctx); /** \brief Check for stale entries in the reader lock table. * \ingroup c_extra @@ -6266,8 +6050,7 @@ LIBMDBX_API int mdbx_reader_check(MDBX_env *env, int *dead); * * \returns Number of transactions committed after the given was started for * read, or negative value on failure. */ -MDBX_DEPRECATED LIBMDBX_API int mdbx_txn_straggler(const MDBX_txn *txn, - int *percent); +MDBX_DEPRECATED LIBMDBX_API int mdbx_txn_straggler(const MDBX_txn *txn, int *percent); /** \brief Registers the current thread as a reader for the environment. * \ingroup c_extra @@ -6377,10 +6160,8 @@ LIBMDBX_API int mdbx_thread_unregister(const MDBX_env *env); * \retval 2 or great The reader process was terminated or killed, * and libmdbx should entirely reset reader registration. */ -typedef int(MDBX_hsr_func)(const MDBX_env *env, const MDBX_txn *txn, - mdbx_pid_t pid, mdbx_tid_t tid, uint64_t laggard, - unsigned gap, size_t space, - int retry) MDBX_CXX17_NOEXCEPT; +typedef int(MDBX_hsr_func)(const MDBX_env *env, const MDBX_txn *txn, mdbx_pid_t pid, mdbx_tid_t tid, uint64_t laggard, + unsigned gap, size_t space, int retry) MDBX_CXX17_NOEXCEPT; /** \brief Sets a Handle-Slow-Readers callback to resolve database full/overflow * issue due to a reader(s) which prevents the old data from being recycled. @@ -6414,8 +6195,7 @@ LIBMDBX_API int mdbx_env_set_hsr(MDBX_env *env, MDBX_hsr_func *hsr_callback); * * \returns A MDBX_hsr_func function or NULL if disabled * or something wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_hsr_func * -mdbx_env_get_hsr(const MDBX_env *env); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_hsr_func *mdbx_env_get_hsr(const MDBX_env *env); /** \defgroup chk Checking and Recovery * Basically this is internal API for `mdbx_chk` tool, etc. @@ -6442,23 +6222,19 @@ LIBMDBX_API int mdbx_txn_unlock(MDBX_env *env); * * \note On Windows the \ref mdbx_env_open_for_recoveryW() is recommended * to use. */ -LIBMDBX_API int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname, - unsigned target_meta, - bool writeable); +LIBMDBX_API int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname, unsigned target_meta, bool writeable); #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) /** \copydoc mdbx_env_open_for_recovery() * \ingroup c_extra * \note Available only on Windows. * \see mdbx_env_open_for_recovery() */ -LIBMDBX_API int mdbx_env_open_for_recoveryW(MDBX_env *env, - const wchar_t *pathname, - unsigned target_meta, +LIBMDBX_API int mdbx_env_open_for_recoveryW(MDBX_env *env, const wchar_t *pathname, unsigned target_meta, bool writeable); -#define mdbx_env_open_for_recoveryT(env, pathname, target_mets, writeable) \ +#define mdbx_env_open_for_recoveryT(env, pathname, target_mets, writeable) \ mdbx_env_open_for_recoveryW(env, pathname, target_mets, writeable) #else -#define mdbx_env_open_for_recoveryT(env, pathname, target_mets, writeable) \ +#define mdbx_env_open_for_recoveryT(env, pathname, target_mets, writeable) \ mdbx_env_open_for_recovery(env, pathname, target_mets, writeable) #endif /* Windows */ @@ -6500,20 +6276,16 @@ LIBMDBX_API int mdbx_env_turn_for_recovery(MDBX_env *env, unsigned target_meta); * другим размером страницы и/или изменением любых других параметров. * * \returns Ненулевое значение кода ошибки, либо 0 при успешном выполнении. */ -LIBMDBX_API int mdbx_preopen_snapinfo(const char *pathname, MDBX_envinfo *info, - size_t bytes); +LIBMDBX_API int mdbx_preopen_snapinfo(const char *pathname, MDBX_envinfo *info, size_t bytes); #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) /** \copydoc mdbx_preopen_snapinfo() * \ingroup c_opening * \note Available only on Windows. * \see mdbx_preopen_snapinfo() */ -LIBMDBX_API int mdbx_preopen_snapinfoW(const wchar_t *pathname, - MDBX_envinfo *info, size_t bytes); -#define mdbx_preopen_snapinfoT(pathname, info, bytes) \ - mdbx_preopen_snapinfoW(pathname, info, bytes) +LIBMDBX_API int mdbx_preopen_snapinfoW(const wchar_t *pathname, MDBX_envinfo *info, size_t bytes); +#define mdbx_preopen_snapinfoT(pathname, info, bytes) mdbx_preopen_snapinfoW(pathname, info, bytes) #else -#define mdbx_preopen_snapinfoT(pathname, info, bytes) \ - mdbx_preopen_snapinfo(pathname, info, bytes) +#define mdbx_preopen_snapinfoT(pathname, info, bytes) mdbx_preopen_snapinfo(pathname, info, bytes) #endif /* Windows */ /** \brief Флаги/опции для проверки целостности базы данных. @@ -6673,10 +6445,8 @@ typedef struct MDBX_chk_context { size_t total_payload_bytes; size_t table_total, table_processed; size_t total_unused_bytes, unused_pages; - size_t processed_pages, reclaimable_pages, gc_pages, alloc_pages, - backed_pages; - size_t problems_meta, tree_problems, gc_tree_problems, kv_tree_problems, - problems_gc, problems_kv, total_problems; + size_t processed_pages, reclaimable_pages, gc_pages, alloc_pages, backed_pages; + size_t problems_meta, tree_problems, gc_tree_problems, kv_tree_problems, problems_gc, problems_kv, total_problems; uint64_t steady_txnid, recent_txnid; /** Указатель на массив размером table_total с указателями на экземпляры * структур MDBX_chk_table_t с информацией о всех таблицах ключ-значение, @@ -6702,37 +6472,28 @@ typedef struct MDBX_chk_context { * \see mdbx_env_chk() */ typedef struct MDBX_chk_callbacks { bool (*check_break)(MDBX_chk_context_t *ctx); - int (*scope_push)(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *outer, - MDBX_chk_scope_t *inner, const char *fmt, va_list args); - int (*scope_conclude)(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *outer, - MDBX_chk_scope_t *inner, int err); - void (*scope_pop)(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *outer, - MDBX_chk_scope_t *inner); - void (*issue)(MDBX_chk_context_t *ctx, const char *object, - uint64_t entry_number, const char *issue, const char *extra_fmt, - va_list extra_args); - MDBX_chk_user_table_cookie_t *(*table_filter)(MDBX_chk_context_t *ctx, - const MDBX_val *name, - MDBX_db_flags_t flags); - int (*table_conclude)(MDBX_chk_context_t *ctx, const MDBX_chk_table_t *table, - MDBX_cursor *cursor, int err); + int (*scope_push)(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *outer, MDBX_chk_scope_t *inner, const char *fmt, + va_list args); + int (*scope_conclude)(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *outer, MDBX_chk_scope_t *inner, int err); + void (*scope_pop)(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *outer, MDBX_chk_scope_t *inner); + void (*issue)(MDBX_chk_context_t *ctx, const char *object, uint64_t entry_number, const char *issue, + const char *extra_fmt, va_list extra_args); + MDBX_chk_user_table_cookie_t *(*table_filter)(MDBX_chk_context_t *ctx, const MDBX_val *name, MDBX_db_flags_t flags); + int (*table_conclude)(MDBX_chk_context_t *ctx, const MDBX_chk_table_t *table, MDBX_cursor *cursor, int err); void (*table_dispose)(MDBX_chk_context_t *ctx, const MDBX_chk_table_t *table); - int (*table_handle_kv)(MDBX_chk_context_t *ctx, const MDBX_chk_table_t *table, - size_t entry_number, const MDBX_val *key, - const MDBX_val *value); + int (*table_handle_kv)(MDBX_chk_context_t *ctx, const MDBX_chk_table_t *table, size_t entry_number, + const MDBX_val *key, const MDBX_val *value); int (*stage_begin)(MDBX_chk_context_t *ctx, MDBX_chk_stage_t); int (*stage_end)(MDBX_chk_context_t *ctx, MDBX_chk_stage_t, int err); - MDBX_chk_line_t *(*print_begin)(MDBX_chk_context_t *ctx, - MDBX_chk_severity_t severity); + MDBX_chk_line_t *(*print_begin)(MDBX_chk_context_t *ctx, MDBX_chk_severity_t severity); void (*print_flush)(MDBX_chk_line_t *); void (*print_done)(MDBX_chk_line_t *); void (*print_chars)(MDBX_chk_line_t *, const char *str, size_t len); void (*print_format)(MDBX_chk_line_t *, const char *fmt, va_list args); - void (*print_size)(MDBX_chk_line_t *, const char *prefix, - const uint64_t value, const char *suffix); + void (*print_size)(MDBX_chk_line_t *, const char *prefix, const uint64_t value, const char *suffix); } MDBX_chk_callbacks_t; /** \brief Проверяет целостность базы данных. @@ -6764,10 +6525,8 @@ typedef struct MDBX_chk_callbacks { * секунды для выполнения проверки, * либо 0 при отсутствии ограничения. * \returns Нулевое значение в случае успеха, иначе код ошибки. */ -LIBMDBX_API int mdbx_env_chk(MDBX_env *env, const MDBX_chk_callbacks_t *cb, - MDBX_chk_context_t *ctx, - const MDBX_chk_flags_t flags, - MDBX_chk_severity_t verbosity, +LIBMDBX_API int mdbx_env_chk(MDBX_env *env, const MDBX_chk_callbacks_t *cb, MDBX_chk_context_t *ctx, + const MDBX_chk_flags_t flags, MDBX_chk_severity_t verbosity, unsigned timeout_seconds_16dot16); /** \brief Вспомогательная функция для подсчета проблем детектируемых diff --git a/mdbx.h++ b/mdbx.h++ index 1166dc0a..b5e28262 100644 --- a/mdbx.h++ +++ b/mdbx.h++ @@ -27,8 +27,7 @@ #pragma once /* Workaround for modern libstdc++ with CLANG < 4.x */ -#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && \ - defined(__clang__) && __clang_major__ < 4 +#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && defined(__clang__) && __clang_major__ < 4 #define __GLIBCXX_BITSIZE_INT_N_0 128 #define __GLIBCXX_TYPE_INT_N_0 __int128 #endif /* Workaround for modern libstdc++ with CLANG < 4.x */ @@ -37,14 +36,12 @@ #if !defined(_MSC_VER) || _MSC_VER < 1900 #error "C++11 compiler or better is required" #elif _MSC_VER >= 1910 -#error \ - "Please add `/Zc:__cplusplus` to MSVC compiler options to enforce it conform ISO C++" +#error "Please add `/Zc:__cplusplus` to MSVC compiler options to enforce it conform ISO C++" #endif /* MSVC is mad and don't define __cplusplus properly */ #endif /* __cplusplus < 201103L */ #if (defined(_WIN32) || defined(_WIN64)) && MDBX_WITHOUT_MSVC_CRT -#error \ - "CRT is required for C++ API, the MDBX_WITHOUT_MSVC_CRT option must be disabled" +#error "CRT is required for C++ API, the MDBX_WITHOUT_MSVC_CRT option must be disabled" #endif /* Windows */ #ifndef __has_include @@ -84,18 +81,13 @@ #ifndef MDBX_USING_CXX_EXPERIMETAL_FILESYSTEM #ifdef INCLUDE_STD_FILESYSTEM_EXPERIMENTAL #define MDBX_USING_CXX_EXPERIMETAL_FILESYSTEM 1 -#elif defined(__cpp_lib_filesystem) && __cpp_lib_filesystem >= 201703L && \ - __cplusplus >= 201703L +#elif defined(__cpp_lib_filesystem) && __cpp_lib_filesystem >= 201703L && __cplusplus >= 201703L #define MDBX_USING_CXX_EXPERIMETAL_FILESYSTEM 0 -#elif (!defined(_MSC_VER) || __cplusplus >= 201403L || \ - (defined(_MSC_VER) && \ - defined(_SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING) && \ - __cplusplus >= 201403L)) -#if defined(__cpp_lib_experimental_filesystem) && \ - __cpp_lib_experimental_filesystem >= 201406L +#elif (!defined(_MSC_VER) || __cplusplus >= 201403L || \ + (defined(_MSC_VER) && defined(_SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING) && __cplusplus >= 201403L)) +#if defined(__cpp_lib_experimental_filesystem) && __cpp_lib_experimental_filesystem >= 201406L #define MDBX_USING_CXX_EXPERIMETAL_FILESYSTEM 1 -#elif defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L && \ - __has_include() +#elif defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L && __has_include() #define MDBX_USING_CXX_EXPERIMETAL_FILESYSTEM 1 #else #define MDBX_USING_CXX_EXPERIMETAL_FILESYSTEM 0 @@ -122,13 +114,12 @@ #include "mdbx.h" -#if (defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L) || \ - (defined(__cpp_lib_endian) && __cpp_lib_endian >= 201907L) || \ - (defined(__cpp_lib_bitops) && __cpp_lib_bitops >= 201907L) || \ +#if (defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L) || \ + (defined(__cpp_lib_endian) && __cpp_lib_endian >= 201907L) || \ + (defined(__cpp_lib_bitops) && __cpp_lib_bitops >= 201907L) || \ (defined(__cpp_lib_int_pow2) && __cpp_lib_int_pow2 >= 202002L) #include -#elif !(defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \ - defined(__ORDER_BIG_ENDIAN__)) +#elif !(defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && defined(__ORDER_BIG_ENDIAN__)) #if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) #define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN #define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN @@ -140,26 +131,18 @@ #else #define __ORDER_LITTLE_ENDIAN__ 1234 #define __ORDER_BIG_ENDIAN__ 4321 -#if defined(__LITTLE_ENDIAN__) || \ - (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ - defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ - defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ - defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ - defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ - defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ - defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ - defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ - defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ - defined(__WINDOWS__) +#if defined(__LITTLE_ENDIAN__) || (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || defined(__ARMEL__) || \ + defined(__THUMBEL__) || defined(__AARCH64EL__) || defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || defined(__elbrus_4c__) || defined(__elbrus_8c__) || \ + defined(__bfin__) || defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || defined(__IA64__) || \ + defined(__ia64) || defined(_M_IA64) || defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || defined(__WINDOWS__) #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#elif defined(__BIG_ENDIAN__) || \ - (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ - defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ - defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ - defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ - defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ - defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ - defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#elif defined(__BIG_ENDIAN__) || (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || defined(__ARMEB__) || \ + defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \ + defined(__sparc__) || defined(__sparc) || defined(__370__) || defined(__THW_370__) || defined(__s390__) || \ + defined(__s390x__) || defined(__SYSC_ZARCH__) #define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ #endif #endif @@ -169,11 +152,9 @@ */ #if defined(DOXYGEN) #define MDBX_CXX17_CONSTEXPR constexpr -#elif defined(__cpp_constexpr) && __cpp_constexpr >= 201603L && \ - ((defined(_MSC_VER) && _MSC_VER >= 1915) || \ - (defined(__clang__) && __clang_major__ > 5) || \ - (defined(__GNUC__) && __GNUC__ > 7) || \ - (!defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER))) +#elif defined(__cpp_constexpr) && __cpp_constexpr >= 201603L && \ + ((defined(_MSC_VER) && _MSC_VER >= 1915) || (defined(__clang__) && __clang_major__ > 5) || \ + (defined(__GNUC__) && __GNUC__ > 7) || (!defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER))) #define MDBX_CXX17_CONSTEXPR constexpr #else #define MDBX_CXX17_CONSTEXPR inline @@ -183,10 +164,8 @@ */ #if defined(DOXYGEN) #define MDBX_CXX20_CONSTEXPR constexpr -#elif defined(__cpp_lib_is_constant_evaluated) && \ - __cpp_lib_is_constant_evaluated >= 201811L && \ - defined(__cpp_lib_constexpr_string) && \ - __cpp_lib_constexpr_string >= 201907L +#elif defined(__cpp_lib_is_constant_evaluated) && __cpp_lib_is_constant_evaluated >= 201811L && \ + defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L #define MDBX_CXX20_CONSTEXPR constexpr #else #define MDBX_CXX20_CONSTEXPR inline @@ -213,14 +192,11 @@ #elif defined NDEBUG #define MDBX_CONSTEXPR_ASSERT(expr) void(0) #else -#define MDBX_CONSTEXPR_ASSERT(expr) \ - ((expr) ? void(0) : [] { assert(!#expr); }()) +#define MDBX_CONSTEXPR_ASSERT(expr) ((expr) ? void(0) : [] { assert(!#expr); }()) #endif /* MDBX_CONSTEXPR_ASSERT */ #ifndef MDBX_LIKELY -#if defined(DOXYGEN) || \ - (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if defined(DOXYGEN) || (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define MDBX_LIKELY(cond) __builtin_expect(!!(cond), 1) #else #define MDBX_LIKELY(x) (x) @@ -228,9 +204,7 @@ #endif /* MDBX_LIKELY */ #ifndef MDBX_UNLIKELY -#if defined(DOXYGEN) || \ - (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if defined(DOXYGEN) || (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define MDBX_UNLIKELY(cond) __builtin_expect(!!(cond), 0) #else #define MDBX_UNLIKELY(x) (x) @@ -247,25 +221,21 @@ #define MDBX_IF_CONSTEXPR #endif /* MDBX_IF_CONSTEXPR */ -#if defined(DOXYGEN) || \ - (__has_cpp_attribute(fallthrough) && \ - (!defined(__clang__) || __clang__ > 4)) || \ +#if defined(DOXYGEN) || (__has_cpp_attribute(fallthrough) && (!defined(__clang__) || __clang__ > 4)) || \ __cplusplus >= 201703L #define MDBX_CXX17_FALLTHROUGH [[fallthrough]] #else #define MDBX_CXX17_FALLTHROUGH #endif /* MDBX_CXX17_FALLTHROUGH */ -#if defined(DOXYGEN) || (__has_cpp_attribute(likely) >= 201803L && \ - (!defined(__GNUC__) || __GNUC__ > 9)) +#if defined(DOXYGEN) || (__has_cpp_attribute(likely) >= 201803L && (!defined(__GNUC__) || __GNUC__ > 9)) #define MDBX_CXX20_LIKELY [[likely]] #else #define MDBX_CXX20_LIKELY #endif /* MDBX_CXX20_LIKELY */ #ifndef MDBX_CXX20_UNLIKELY -#if defined(DOXYGEN) || (__has_cpp_attribute(unlikely) >= 201803L && \ - (!defined(__GNUC__) || __GNUC__ > 9)) +#if defined(DOXYGEN) || (__has_cpp_attribute(unlikely) >= 201803L && (!defined(__GNUC__) || __GNUC__ > 9)) #define MDBX_CXX20_UNLIKELY [[unlikely]] #else #define MDBX_CXX20_UNLIKELY @@ -293,10 +263,9 @@ #ifndef MDBX_ASSERT_CXX20_CONCEPT_SATISFIED #if MDBX_HAVE_CXX20_CONCEPTS || defined(DOXYGEN) -#define MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(CONCEPT, TYPE) \ - static_assert(CONCEPT) +#define MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(CONCEPT, TYPE) static_assert(CONCEPT) #else -#define MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(CONCEPT, NAME) \ +#define MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(CONCEPT, NAME) \ static_assert(true, MDBX_STRINGIFY(CONCEPT) "<" MDBX_STRINGIFY(TYPE) ">") #endif #endif /* MDBX_ASSERT_CXX20_CONCEPT_SATISFIED */ @@ -304,9 +273,9 @@ #ifdef _MSC_VER #pragma warning(push, 4) #pragma warning(disable : 4127) /* conditional expression is constant */ -#pragma warning(disable : 4251) /* 'std::FOO' needs to have dll-interface to \ +#pragma warning(disable : 4251) /* 'std::FOO' needs to have dll-interface to \ be used by clients of 'mdbx::BAR' */ -#pragma warning(disable : 4275) /* non dll-interface 'std::FOO' used as \ +#pragma warning(disable : 4275) /* non dll-interface 'std::FOO' used as \ base for dll-interface 'mdbx::BAR' */ /* MSVC is mad and can generate this warning for its own intermediate * automatically generated code, which becomes unreachable after some kinds of @@ -317,9 +286,9 @@ #if defined(__LCC__) && __LCC__ >= 126 #pragma diagnostic push #if __LCC__ < 127 -#pragma diag_suppress 3058 /* workaround: call to is_constant_evaluated() \ +#pragma diag_suppress 3058 /* workaround: call to is_constant_evaluated() \ appearing in a constant expression `true` */ -#pragma diag_suppress 3060 /* workaround: call to is_constant_evaluated() \ +#pragma diag_suppress 3060 /* workaround: call to is_constant_evaluated() \ appearing in a constant expression `false` */ #endif #endif /* E2K LCC (warnings) */ @@ -349,16 +318,10 @@ using byte = unsigned char; #if defined(__cpp_lib_endian) && __cpp_lib_endian >= 201907L using endian = ::std::endian; -#elif defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \ - defined(__ORDER_BIG_ENDIAN__) -enum class endian { - little = __ORDER_LITTLE_ENDIAN__, - big = __ORDER_BIG_ENDIAN__, - native = __BYTE_ORDER__ -}; +#elif defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && defined(__ORDER_BIG_ENDIAN__) +enum class endian { little = __ORDER_LITTLE_ENDIAN__, big = __ORDER_BIG_ENDIAN__, native = __BYTE_ORDER__ }; #else -#error \ - "Please use a C++ compiler provides byte order information or C++20 support" +#error "Please use a C++ compiler provides byte order information or C++20 support" #endif /* Byte Order enum */ /// \copydoc MDBX_version_info @@ -374,19 +337,16 @@ MDBX_CXX11_CONSTEXPR const build_info &get_build() noexcept; static MDBX_CXX17_CONSTEXPR size_t strlen(const char *c_str) noexcept; /// \brief constexpr-enabled memcpy(). -static MDBX_CXX20_CONSTEXPR void *memcpy(void *dest, const void *src, - size_t bytes) noexcept; +static MDBX_CXX20_CONSTEXPR void *memcpy(void *dest, const void *src, size_t bytes) noexcept; /// \brief constexpr-enabled memcmp(). -static MDBX_CXX20_CONSTEXPR int memcmp(const void *a, const void *b, - size_t bytes) noexcept; +static MDBX_CXX20_CONSTEXPR int memcmp(const void *a, const void *b, size_t bytes) noexcept; /// \brief Legacy allocator /// but it is recommended to use \ref polymorphic_allocator. using legacy_allocator = ::std::string::allocator_type; -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_memory_resource) && \ - __cpp_lib_memory_resource >= 201603L && _GLIBCXX_USE_CXX11_ABI) +#if defined(DOXYGEN) || \ + (defined(__cpp_lib_memory_resource) && __cpp_lib_memory_resource >= 201603L && _GLIBCXX_USE_CXX11_ABI) /// \brief Default polymorphic allocator for modern code. using polymorphic_allocator = ::std::pmr::string::allocator_type; using default_allocator = polymorphic_allocator; @@ -396,9 +356,7 @@ using default_allocator = legacy_allocator; struct slice; struct default_capacity_policy; -template -class buffer; +template class buffer; class env; class env_managed; class txn; @@ -421,13 +379,11 @@ namespace filesystem = ::std::experimental::filesystem::v1; namespace filesystem = ::std::experimental::filesystem; #endif #define MDBX_STD_FILESYSTEM_PATH ::mdbx::filesystem::path -#elif defined(DOXYGEN) || \ - (defined(__cpp_lib_filesystem) && __cpp_lib_filesystem >= 201703L && \ - defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L && \ - (!defined(__MAC_OS_X_VERSION_MIN_REQUIRED) || \ - __MAC_OS_X_VERSION_MIN_REQUIRED >= 101500) && \ - (!defined(__IPHONE_OS_VERSION_MIN_REQUIRED) || \ - __IPHONE_OS_VERSION_MIN_REQUIRED >= 130100)) && \ +#elif defined(DOXYGEN) || \ + (defined(__cpp_lib_filesystem) && __cpp_lib_filesystem >= 201703L && defined(__cpp_lib_string_view) && \ + __cpp_lib_string_view >= 201606L && \ + (!defined(__MAC_OS_X_VERSION_MIN_REQUIRED) || __MAC_OS_X_VERSION_MIN_REQUIRED >= 101500) && \ + (!defined(__IPHONE_OS_VERSION_MIN_REQUIRED) || __IPHONE_OS_VERSION_MIN_REQUIRED >= 130100)) && \ (!defined(_MSC_VER) || __cplusplus >= 201703L) namespace filesystem = ::std::filesystem; /// \brief Defined if `mdbx::filesystem::path` is available. @@ -447,8 +403,7 @@ using path = ::std::wstring; using path = ::std::string; #endif /* mdbx::path */ -#if defined(__SIZEOF_INT128__) || \ - (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) +#if defined(__SIZEOF_INT128__) || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) #ifndef MDBX_U128_TYPE #define MDBX_U128_TYPE __uint128_t #endif /* MDBX_U128_TYPE */ @@ -495,10 +450,8 @@ public: error &operator=(const error &) = default; error &operator=(error &&) = default; - MDBX_CXX11_CONSTEXPR friend bool operator==(const error &a, - const error &b) noexcept; - MDBX_CXX11_CONSTEXPR friend bool operator!=(const error &a, - const error &b) noexcept; + MDBX_CXX11_CONSTEXPR friend bool operator==(const error &a, const error &b) noexcept; + MDBX_CXX11_CONSTEXPR friend bool operator!=(const error &a, const error &b) noexcept; MDBX_CXX11_CONSTEXPR bool is_success() const noexcept; MDBX_CXX11_CONSTEXPR bool is_result_true() const noexcept; @@ -517,30 +470,23 @@ public: /// \brief Returns true for MDBX's errors. MDBX_CXX11_CONSTEXPR bool is_mdbx_error() const noexcept; /// \brief Panics on unrecoverable errors inside destructors etc. - [[noreturn]] void panic(const char *context_where_when, - const char *func_who_what) const noexcept; + [[noreturn]] void panic(const char *context_where_when, const char *func_who_what) const noexcept; [[noreturn]] void throw_exception() const; [[noreturn]] static inline void throw_exception(int error_code); inline void throw_on_failure() const; inline void success_or_throw() const; inline void success_or_throw(const exception_thunk &) const; - inline void panic_on_failure(const char *context_where, - const char *func_who) const noexcept; - inline void success_or_panic(const char *context_where, - const char *func_who) const noexcept; + inline void panic_on_failure(const char *context_where, const char *func_who) const noexcept; + inline void success_or_panic(const char *context_where, const char *func_who) const noexcept; static inline void throw_on_nullptr(const void *ptr, MDBX_error_t error_code); static inline void success_or_throw(MDBX_error_t error_code); - static void success_or_throw(int error_code) { - success_or_throw(static_cast(error_code)); - } + static void success_or_throw(int error_code) { success_or_throw(static_cast(error_code)); } static inline void throw_on_failure(int error_code); static inline bool boolean_or_throw(int error_code); static inline void success_or_throw(int error_code, const exception_thunk &); static inline bool boolean_or_throw(int error_code, const exception_thunk &); - static inline void panic_on_failure(int error_code, const char *context_where, - const char *func_who) noexcept; - static inline void success_or_panic(int error_code, const char *context_where, - const char *func_who) noexcept; + static inline void panic_on_failure(int error_code, const char *context_where, const char *func_who) noexcept; + static inline void success_or_panic(int error_code, const char *context_where, const char *func_who) noexcept; }; /// \brief Base class for all libmdbx's exceptions that are corresponds @@ -577,10 +523,10 @@ public: virtual ~fatal() noexcept; }; -#define MDBX_DECLARE_EXCEPTION(NAME) \ - struct LIBMDBX_API_TYPE NAME : public exception { \ - NAME(const ::mdbx::error &); \ - virtual ~NAME() noexcept; \ +#define MDBX_DECLARE_EXCEPTION(NAME) \ + struct LIBMDBX_API_TYPE NAME : public exception { \ + NAME(const ::mdbx::error &); \ + virtual ~NAME() noexcept; \ } MDBX_DECLARE_EXCEPTION(bad_map_id); MDBX_DECLARE_EXCEPTION(bad_transaction); @@ -623,10 +569,8 @@ MDBX_DECLARE_EXCEPTION(mvcc_retarded); [[noreturn]] LIBMDBX_API void throw_bad_value_size(); [[noreturn]] LIBMDBX_API void throw_incomparable_cursors(); static MDBX_CXX14_CONSTEXPR size_t check_length(size_t bytes); -static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, - size_t payload); -static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, size_t payload, - size_t tailroom); +static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, size_t payload); +static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, size_t payload, size_t tailroom); /// end of cxx_exceptions @} @@ -661,35 +605,27 @@ concept ImmutableByteProducer = requires(const T &a, char array[42]) { * \interface SliceTranscoder * \brief SliceTranscoder C++20 concept */ template -concept SliceTranscoder = - ImmutableByteProducer && requires(const slice &source, const T &a) { - T(source); - { a.is_erroneous() } -> std::same_as; - }; +concept SliceTranscoder = ImmutableByteProducer && requires(const slice &source, const T &a) { + T(source); + { a.is_erroneous() } -> std::same_as; +}; #endif /* MDBX_HAVE_CXX20_CONCEPTS */ -template -inline buffer -make_buffer(PRODUCER &producer, const ALLOCATOR &allocator = ALLOCATOR()); +inline buffer make_buffer(PRODUCER &producer, const ALLOCATOR &allocator = ALLOCATOR()); -template -inline buffer -make_buffer(const PRODUCER &producer, const ALLOCATOR &allocator = ALLOCATOR()); +inline buffer make_buffer(const PRODUCER &producer, + const ALLOCATOR &allocator = ALLOCATOR()); -template -inline string make_string(PRODUCER &producer, - const ALLOCATOR &allocator = ALLOCATOR()); +template +inline string make_string(PRODUCER &producer, const ALLOCATOR &allocator = ALLOCATOR()); -template -inline string make_string(const PRODUCER &producer, - const ALLOCATOR &allocator = ALLOCATOR()); +template +inline string make_string(const PRODUCER &producer, const ALLOCATOR &allocator = ALLOCATOR()); /// \brief References a data located outside the slice. /// @@ -716,8 +652,7 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { MDBX_CXX14_CONSTEXPR slice(const void *begin, const void *end); /// \brief Create a slice that refers to text[0,strlen(text)-1]. - template - MDBX_CXX14_CONSTEXPR slice(const char (&text)[SIZE]) : slice(text, SIZE - 1) { + template MDBX_CXX14_CONSTEXPR slice(const char (&text)[SIZE]) : slice(text, SIZE - 1) { MDBX_CONSTEXPR_ASSERT(SIZE > 0 && text[SIZE - 1] == '\0'); } /// \brief Create a slice that refers to c_str[0,strlen(c_str)-1]. @@ -726,8 +661,7 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { /// \brief Create a slice that refers to the contents of "str". /// \note 'explicit' to avoid reference to the temporary std::string instance. template - explicit MDBX_CXX20_CONSTEXPR - slice(const ::std::basic_string &str) + explicit MDBX_CXX20_CONSTEXPR slice(const ::std::basic_string &str) : slice(str.data(), str.length() * sizeof(CHAR)) {} MDBX_CXX14_CONSTEXPR slice(const MDBX_val &src); @@ -736,29 +670,22 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { MDBX_CXX14_CONSTEXPR slice(slice &&src) noexcept; #if defined(DOXYGEN) || (defined(__cpp_lib_span) && __cpp_lib_span >= 202002L) - template - MDBX_CXX14_CONSTEXPR slice(const ::std::span &span) - : slice(span.begin(), span.end()) { - static_assert(::std::is_standard_layout::value && - !::std::is_pointer::value, + template MDBX_CXX14_CONSTEXPR slice(const ::std::span &span) : slice(span.begin(), span.end()) { + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value, "Must be a standard layout type!"); } - template - MDBX_CXX14_CONSTEXPR ::std::span as_span() const { - static_assert(::std::is_standard_layout::value && - !::std::is_pointer::value, + template MDBX_CXX14_CONSTEXPR ::std::span as_span() const { + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value, "Must be a standard layout type!"); if (MDBX_LIKELY(size() % sizeof(POD) == 0)) MDBX_CXX20_LIKELY - return ::std::span(static_cast(data()), - size() / sizeof(POD)); + return ::std::span(static_cast(data()), size() / sizeof(POD)); throw_bad_value_size(); } template MDBX_CXX14_CONSTEXPR ::std::span as_span() { - static_assert(::std::is_standard_layout::value && - !::std::is_pointer::value, + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value, "Must be a standard layout type!"); if (MDBX_LIKELY(size() % sizeof(POD) == 0)) MDBX_CXX20_LIKELY @@ -766,38 +693,24 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { throw_bad_value_size(); } - MDBX_CXX14_CONSTEXPR ::std::span bytes() const { - return as_span(); - } + MDBX_CXX14_CONSTEXPR ::std::span bytes() const { return as_span(); } MDBX_CXX14_CONSTEXPR ::std::span bytes() { return as_span(); } - MDBX_CXX14_CONSTEXPR ::std::span chars() const { - return as_span(); - } + MDBX_CXX14_CONSTEXPR ::std::span chars() const { return as_span(); } MDBX_CXX14_CONSTEXPR ::std::span chars() { return as_span(); } #endif /* __cpp_lib_span >= 202002L */ -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) /// \brief Create a slice that refers to the same contents as "string_view" template - MDBX_CXX14_CONSTEXPR slice(const ::std::basic_string_view &sv) - : slice(sv.data(), sv.data() + sv.length()) {} + MDBX_CXX14_CONSTEXPR slice(const ::std::basic_string_view &sv) : slice(sv.data(), sv.data() + sv.length()) {} - template - slice(::std::basic_string_view &&sv) : slice(sv) { - sv = {}; - } + template slice(::std::basic_string_view &&sv) : slice(sv) { sv = {}; } #endif /* __cpp_lib_string_view >= 201606L */ - template - static MDBX_CXX14_CONSTEXPR slice wrap(const char (&text)[SIZE]) { - return slice(text); - } + template static MDBX_CXX14_CONSTEXPR slice wrap(const char (&text)[SIZE]) { return slice(text); } - template - MDBX_CXX14_CONSTEXPR static slice wrap(const POD &pod) { - static_assert(::std::is_standard_layout::value && - !::std::is_pointer::value, + template MDBX_CXX14_CONSTEXPR static slice wrap(const POD &pod) { + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value, "Must be a standard layout type!"); return slice(&pod, sizeof(pod)); } @@ -808,19 +721,15 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { inline slice &assign(slice &&src) noexcept; inline slice &assign(::MDBX_val &&src); inline slice &assign(const void *begin, const void *end); - template - slice &assign(const ::std::basic_string &str) { + template slice &assign(const ::std::basic_string &str) { return assign(str.data(), str.length() * sizeof(CHAR)); } inline slice &assign(const char *c_str); -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) - template - slice &assign(const ::std::basic_string_view &view) { +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template slice &assign(const ::std::basic_string_view &view) { return assign(view.begin(), view.end()); } - template - slice &assign(::std::basic_string_view &&view) { + template slice &assign(::std::basic_string_view &&view) { assign(view); view = {}; return *this; @@ -833,152 +742,119 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { operator MDBX_val *() noexcept { return this; } operator const MDBX_val *() const noexcept { return this; } -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) - template - slice &operator=(const ::std::basic_string_view &view) { +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template slice &operator=(const ::std::basic_string_view &view) { return assign(view); } - template - slice &operator=(::std::basic_string_view &&view) { - return assign(view); - } + template slice &operator=(::std::basic_string_view &&view) { return assign(view); } /// \brief Return a string_view that references the same data as this slice. template > - MDBX_CXX11_CONSTEXPR ::std::basic_string_view - string_view() const noexcept { + MDBX_CXX11_CONSTEXPR ::std::basic_string_view string_view() const noexcept { static_assert(sizeof(CHAR) == 1, "Must be single byte characters"); return ::std::basic_string_view(char_ptr(), length()); } /// \brief Return a string_view that references the same data as this slice. template - MDBX_CXX11_CONSTEXPR explicit - operator ::std::basic_string_view() const noexcept { + MDBX_CXX11_CONSTEXPR explicit operator ::std::basic_string_view() const noexcept { return this->string_view(); } #endif /* __cpp_lib_string_view >= 201606L */ - template , - class ALLOCATOR = default_allocator> + template , class ALLOCATOR = default_allocator> MDBX_CXX20_CONSTEXPR ::std::basic_string as_string(const ALLOCATOR &allocator = ALLOCATOR()) const { static_assert(sizeof(CHAR) == 1, "Must be single byte characters"); - return ::std::basic_string(char_ptr(), length(), - allocator); + return ::std::basic_string(char_ptr(), length(), allocator); } template - MDBX_CXX20_CONSTEXPR explicit - operator ::std::basic_string() const { + MDBX_CXX20_CONSTEXPR explicit operator ::std::basic_string() const { return as_string(); } /// \brief Returns a string with a hexadecimal dump of the slice content. template - inline string - as_hex_string(bool uppercase = false, unsigned wrap_width = 0, - const ALLOCATOR &allocator = ALLOCATOR()) const; + inline string as_hex_string(bool uppercase = false, unsigned wrap_width = 0, + const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Returns a string with a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of the slice content. template - inline string - as_base58_string(unsigned wrap_width = 0, - const ALLOCATOR &allocator = ALLOCATOR()) const; + inline string as_base58_string(unsigned wrap_width = 0, const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Returns a string with a /// [Base58](https://en.wikipedia.org/wiki/Base64) dump of the slice content. template - inline string - as_base64_string(unsigned wrap_width = 0, - const ALLOCATOR &allocator = ALLOCATOR()) const; + inline string as_base64_string(unsigned wrap_width = 0, const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Returns a buffer with a hexadecimal dump of the slice content. - template - inline buffer - encode_hex(bool uppercase = false, unsigned wrap_width = 0, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline buffer encode_hex(bool uppercase = false, unsigned wrap_width = 0, + const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Returns a buffer with a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of the slice content. - template - inline buffer - encode_base58(unsigned wrap_width = 0, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline buffer encode_base58(unsigned wrap_width = 0, + const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Returns a buffer with a /// [Base64](https://en.wikipedia.org/wiki/Base64) dump of the slice content. - template - inline buffer - encode_base64(unsigned wrap_width = 0, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline buffer encode_base64(unsigned wrap_width = 0, + const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Decodes hexadecimal dump from the slice content to returned buffer. - template - inline buffer - hex_decode(bool ignore_spaces = false, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline buffer hex_decode(bool ignore_spaces = false, + const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Decodes [Base58](https://en.wikipedia.org/wiki/Base58) dump /// from the slice content to returned buffer. - template - inline buffer - base58_decode(bool ignore_spaces = false, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline buffer base58_decode(bool ignore_spaces = false, + const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Decodes [Base64](https://en.wikipedia.org/wiki/Base64) dump /// from the slice content to returned buffer. - template - inline buffer - base64_decode(bool ignore_spaces = false, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline buffer base64_decode(bool ignore_spaces = false, + const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Checks whether the content of the slice is printable. /// \param [in] disable_utf8 By default if `disable_utf8` is `false` function /// checks that content bytes are printable ASCII-7 characters or a valid UTF8 /// sequences. Otherwise, if `disable_utf8` is `true` function checks that /// content bytes are printable extended 8-bit ASCII codes. - MDBX_NOTHROW_PURE_FUNCTION bool - is_printable(bool disable_utf8 = false) const noexcept; + MDBX_NOTHROW_PURE_FUNCTION bool is_printable(bool disable_utf8 = false) const noexcept; /// \brief Checks whether the content of the slice is a hexadecimal dump. /// \param [in] ignore_spaces If `true` function will skips spaces surrounding /// (before, between and after) a encoded bytes. However, spaces should not /// break a pair of characters encoding a single byte. - MDBX_NOTHROW_PURE_FUNCTION inline bool - is_hex(bool ignore_spaces = false) const noexcept; + MDBX_NOTHROW_PURE_FUNCTION inline bool is_hex(bool ignore_spaces = false) const noexcept; /// \brief Checks whether the content of the slice is a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump. /// \param [in] ignore_spaces If `true` function will skips spaces surrounding /// (before, between and after) a encoded bytes. However, spaces should not /// break a code group of characters. - MDBX_NOTHROW_PURE_FUNCTION inline bool - is_base58(bool ignore_spaces = false) const noexcept; + MDBX_NOTHROW_PURE_FUNCTION inline bool is_base58(bool ignore_spaces = false) const noexcept; /// \brief Checks whether the content of the slice is a /// [Base64](https://en.wikipedia.org/wiki/Base64) dump. /// \param [in] ignore_spaces If `true` function will skips spaces surrounding /// (before, between and after) a encoded bytes. However, spaces should not /// break a code group of characters. - MDBX_NOTHROW_PURE_FUNCTION inline bool - is_base64(bool ignore_spaces = false) const noexcept; + MDBX_NOTHROW_PURE_FUNCTION inline bool is_base64(bool ignore_spaces = false) const noexcept; inline void swap(slice &other) noexcept; -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) - template - void swap(::std::basic_string_view &view) noexcept { +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template void swap(::std::basic_string_view &view) noexcept { static_assert(sizeof(CHAR) == 1, "Must be single byte characters"); const auto temp = ::std::basic_string_view(*this); *this = view; @@ -1054,12 +930,10 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { inline void safe_remove_suffix(size_t n); /// \brief Checks if the data starts with the given prefix. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool - starts_with(const slice &prefix) const noexcept; + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool starts_with(const slice &prefix) const noexcept; /// \brief Checks if the data ends with the given suffix. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool - ends_with(const slice &suffix) const noexcept; + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool ends_with(const slice &suffix) const noexcept; /// \brief Returns the nth byte in the referenced data. /// \pre REQUIRES: `n < size()` @@ -1097,8 +971,7 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { /// \attention Function implementation and returned hash values may changed /// version to version, and in future the t1ha3 will be used here. Therefore /// values obtained from this function shouldn't be persisted anywhere. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t - hash_value() const noexcept; + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t hash_value() const noexcept; /// \brief Three-way fast non-lexicographically length-based comparison. /// \details Firstly compares length and if it equal then compare content @@ -1108,43 +981,31 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { /// or the same length and lexicographically less than `b`; /// `> 0` if `a` longer than `b`, /// or the same length and lexicographically great than `b`. - MDBX_NOTHROW_PURE_FUNCTION static MDBX_CXX14_CONSTEXPR intptr_t - compare_fast(const slice &a, const slice &b) noexcept; + MDBX_NOTHROW_PURE_FUNCTION static MDBX_CXX14_CONSTEXPR intptr_t compare_fast(const slice &a, const slice &b) noexcept; /// \brief Three-way lexicographically comparison. /// \return value: /// `== 0` if `a` lexicographically equal `b`; /// `< 0` if `a` lexicographically less than `b`; /// `> 0` if `a` lexicographically great than `b`. - MDBX_NOTHROW_PURE_FUNCTION static MDBX_CXX14_CONSTEXPR intptr_t - compare_lexicographically(const slice &a, const slice &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator==(const slice &a, - const slice &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator<(const slice &a, - const slice &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator>(const slice &a, - const slice &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator<=(const slice &a, - const slice &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator>=(const slice &a, - const slice &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator!=(const slice &a, - const slice &b) noexcept; + MDBX_NOTHROW_PURE_FUNCTION static MDBX_CXX14_CONSTEXPR intptr_t compare_lexicographically(const slice &a, + const slice &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator==(const slice &a, const slice &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator<(const slice &a, const slice &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator>(const slice &a, const slice &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator<=(const slice &a, const slice &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator>=(const slice &a, const slice &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator!=(const slice &a, const slice &b) noexcept; /// \brief Checks the slice is not refers to null address or has zero length. - MDBX_CXX11_CONSTEXPR bool is_valid() const noexcept { - return !(iov_base == nullptr && iov_len != 0); - } + MDBX_CXX11_CONSTEXPR bool is_valid() const noexcept { return !(iov_base == nullptr && iov_len != 0); } /// \brief Build an invalid slice which non-zero length and refers to null /// address. - MDBX_CXX14_CONSTEXPR static slice invalid() noexcept { - return slice(size_t(-1)); - } + MDBX_CXX14_CONSTEXPR static slice invalid() noexcept { return slice(size_t(-1)); } template MDBX_CXX14_CONSTEXPR POD as_pod() const { - static_assert(::std::is_standard_layout::value && - !::std::is_pointer::value, + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value, "Must be a standard layout type!"); if (MDBX_LIKELY(size() == sizeof(POD))) MDBX_CXX20_LIKELY { @@ -1156,9 +1017,7 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { } #ifdef MDBX_U128_TYPE - MDBX_CXX14_CONSTEXPR MDBX_U128_TYPE as_uint128() const { - return as_pod(); - } + MDBX_CXX14_CONSTEXPR MDBX_U128_TYPE as_uint128() const { return as_pod(); } #endif /* MDBX_U128_TYPE */ MDBX_CXX14_CONSTEXPR uint64_t as_uint64() const { return as_pod(); } MDBX_CXX14_CONSTEXPR uint32_t as_uint32() const { return as_pod(); } @@ -1166,9 +1025,7 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { MDBX_CXX14_CONSTEXPR uint8_t as_uint8() const { return as_pod(); } #ifdef MDBX_I128_TYPE - MDBX_CXX14_CONSTEXPR MDBX_I128_TYPE as_int128() const { - return as_pod(); - } + MDBX_CXX14_CONSTEXPR MDBX_I128_TYPE as_int128() const { return as_pod(); } #endif /* MDBX_I128_TYPE */ MDBX_CXX14_CONSTEXPR int64_t as_int64() const { return as_pod(); } MDBX_CXX14_CONSTEXPR int32_t as_int32() const { return as_pod(); } @@ -1192,8 +1049,7 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { int8_t as_int8_adapt() const; protected: - MDBX_CXX11_CONSTEXPR slice(size_t invalid_length) noexcept - : ::MDBX_val({nullptr, invalid_length}) {} + MDBX_CXX11_CONSTEXPR slice(size_t invalid_length) noexcept : ::MDBX_val({nullptr, invalid_length}) {} }; //------------------------------------------------------------------------------ @@ -1201,8 +1057,7 @@ protected: namespace allocation_aware_details { template constexpr bool allocator_is_always_equal() noexcept { -#if defined(__cpp_lib_allocator_traits_is_always_equal) && \ - __cpp_lib_allocator_traits_is_always_equal >= 201411L +#if defined(__cpp_lib_allocator_traits_is_always_equal) && __cpp_lib_allocator_traits_is_always_equal >= 201411L return ::std::allocator_traits::is_always_equal::value; #else return ::std::is_empty::value; @@ -1210,17 +1065,13 @@ template constexpr bool allocator_is_always_equal() noexcept { } template ::propagate_on_container_move_assignment::value> + bool PoCMA = ::std::allocator_traits::propagate_on_container_move_assignment::value> struct move_assign_alloc; template struct move_assign_alloc { - static constexpr bool is_nothrow() noexcept { - return allocator_is_always_equal(); - } + static constexpr bool is_nothrow() noexcept { return allocator_is_always_equal(); } static MDBX_CXX20_CONSTEXPR bool is_moveable(T *target, T &source) noexcept { - return allocator_is_always_equal() || - target->get_allocator() == source.get_allocator(); + return allocator_is_always_equal() || target->get_allocator() == source.get_allocator(); } static MDBX_CXX20_CONSTEXPR void propagate(T *target, T &source) noexcept { assert(target->get_allocator() != source.get_allocator()); @@ -1231,8 +1082,7 @@ template struct move_assign_alloc { template struct move_assign_alloc { static constexpr bool is_nothrow() noexcept { - return allocator_is_always_equal() || - ::std::is_nothrow_move_assignable::value; + return allocator_is_always_equal() || ::std::is_nothrow_move_assignable::value; } static constexpr bool is_moveable(T *, T &) noexcept { return true; } static MDBX_CXX20_CONSTEXPR void propagate(T *target, T &source) { @@ -1242,14 +1092,12 @@ template struct move_assign_alloc { }; template ::propagate_on_container_copy_assignment::value> + bool PoCCA = ::std::allocator_traits::propagate_on_container_copy_assignment::value> struct copy_assign_alloc; template struct copy_assign_alloc { static constexpr bool is_nothrow() noexcept { return false; } - static MDBX_CXX20_CONSTEXPR void propagate(T *target, - const T &source) noexcept { + static MDBX_CXX20_CONSTEXPR void propagate(T *target, const T &source) noexcept { assert(target->get_allocator() != source.get_allocator()); (void)target; (void)source; @@ -1258,16 +1106,13 @@ template struct copy_assign_alloc { template struct copy_assign_alloc { static constexpr bool is_nothrow() noexcept { - return allocator_is_always_equal() || - ::std::is_nothrow_copy_assignable::value; + return allocator_is_always_equal() || ::std::is_nothrow_copy_assignable::value; } - static MDBX_CXX20_CONSTEXPR void - propagate(T *target, const T &source) noexcept(is_nothrow()) { + static MDBX_CXX20_CONSTEXPR void propagate(T *target, const T &source) noexcept(is_nothrow()) { if MDBX_IF_CONSTEXPR (!allocator_is_always_equal()) { if (MDBX_UNLIKELY(target->get_allocator() != source.get_allocator())) MDBX_CXX20_UNLIKELY target->get_allocator() = - ::std::allocator_traits::select_on_container_copy_construction( - source.get_allocator()); + ::std::allocator_traits::select_on_container_copy_construction(source.get_allocator()); } else { /* gag for buggy compilers */ (void)target; @@ -1277,16 +1122,12 @@ template struct copy_assign_alloc { }; template ::propagate_on_container_swap::value> + bool PoCS = ::std::allocator_traits::propagate_on_container_swap::value> struct swap_alloc; template struct swap_alloc { - static constexpr bool is_nothrow() noexcept { - return allocator_is_always_equal(); - } - static MDBX_CXX20_CONSTEXPR void propagate(T *target, - T &source) noexcept(is_nothrow()) { + static constexpr bool is_nothrow() noexcept { return allocator_is_always_equal(); } + static MDBX_CXX20_CONSTEXPR void propagate(T *target, T &source) noexcept(is_nothrow()) { if MDBX_IF_CONSTEXPR (!allocator_is_always_equal()) { if (MDBX_UNLIKELY(target->get_allocator() != source.get_allocator())) MDBX_CXX20_UNLIKELY throw_allocators_mismatch(); @@ -1304,11 +1145,9 @@ template struct swap_alloc { #if defined(__cpp_lib_is_swappable) && __cpp_lib_is_swappable >= 201603L ::std::is_nothrow_swappable() || #endif /* __cpp_lib_is_swappable >= 201603L */ - (::std::is_nothrow_move_constructible::value && - ::std::is_nothrow_move_assignable::value); + (::std::is_nothrow_move_constructible::value && ::std::is_nothrow_move_assignable::value); } - static MDBX_CXX20_CONSTEXPR void propagate(T *target, - T &source) noexcept(is_nothrow()) { + static MDBX_CXX20_CONSTEXPR void propagate(T *target, T &source) noexcept(is_nothrow()) { if MDBX_IF_CONSTEXPR (!allocator_is_always_equal()) { if (MDBX_UNLIKELY(target->get_allocator() != source.get_allocator())) MDBX_CXX20_UNLIKELY ::std::swap(*target, source); @@ -1323,29 +1162,19 @@ template struct swap_alloc { } // namespace allocation_aware_details struct default_capacity_policy { - enum : size_t { - extra_inplace_storage = 0, - pettiness_threshold = 64, - max_reserve = 65536 - }; + enum : size_t { extra_inplace_storage = 0, pettiness_threshold = 64, max_reserve = 65536 }; static MDBX_CXX11_CONSTEXPR size_t round(const size_t value) { - static_assert((pettiness_threshold & (pettiness_threshold - 1)) == 0, - "pettiness_threshold must be a power of 2"); - static_assert(pettiness_threshold % 2 == 0, - "pettiness_threshold must be even"); - static_assert(pettiness_threshold >= sizeof(uint64_t), - "pettiness_threshold must be > 7"); + static_assert((pettiness_threshold & (pettiness_threshold - 1)) == 0, "pettiness_threshold must be a power of 2"); + static_assert(pettiness_threshold % 2 == 0, "pettiness_threshold must be even"); + static_assert(pettiness_threshold >= sizeof(uint64_t), "pettiness_threshold must be > 7"); constexpr const auto pettiness_mask = ~size_t(pettiness_threshold - 1); return (value + pettiness_threshold - 1) & pettiness_mask; } - static MDBX_CXX11_CONSTEXPR size_t advise(const size_t current, - const size_t wanna) { - static_assert(max_reserve % pettiness_threshold == 0, - "max_reserve must be a multiple of pettiness_threshold"); - static_assert(max_reserve / 3 > pettiness_threshold, - "max_reserve must be > pettiness_threshold * 3"); + static MDBX_CXX11_CONSTEXPR size_t advise(const size_t current, const size_t wanna) { + static_assert(max_reserve % pettiness_threshold == 0, "max_reserve must be a multiple of pettiness_threshold"); + static_assert(max_reserve / 3 > pettiness_threshold, "max_reserve must be > pettiness_threshold * 3"); if (wanna > current) /* doubling capacity, but don't made reserve more than max_reserve */ return round(wanna + ::std::min(size_t(max_reserve), current)); @@ -1366,8 +1195,7 @@ struct LIBMDBX_API to_hex { const slice source; const bool uppercase = false; const unsigned wrap_width = 0; - MDBX_CXX11_CONSTEXPR to_hex(const slice &source, bool uppercase = false, - unsigned wrap_width = 0) noexcept + MDBX_CXX11_CONSTEXPR to_hex(const slice &source, bool uppercase = false, unsigned wrap_width = 0) noexcept : source(source), uppercase(uppercase), wrap_width(wrap_width) { MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(SliceTranscoder, to_hex); } @@ -1379,10 +1207,8 @@ struct LIBMDBX_API to_hex { } /// \brief Returns a buffer with a hexadecimal dump of a passed slice. - template - buffer - as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { + template + buffer as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_buffer(*this, allocator); } @@ -1417,8 +1243,7 @@ struct LIBMDBX_API to_base58 { const slice source; const unsigned wrap_width = 0; MDBX_CXX11_CONSTEXPR - to_base58(const slice &source, unsigned wrap_width = 0) noexcept - : source(source), wrap_width(wrap_width) { + to_base58(const slice &source, unsigned wrap_width = 0) noexcept : source(source), wrap_width(wrap_width) { MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(SliceTranscoder, to_base58); } @@ -1431,10 +1256,8 @@ struct LIBMDBX_API to_base58 { /// \brief Returns a buffer with a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of a passed slice. - template - buffer - as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { + template + buffer as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_buffer(*this, allocator); } @@ -1471,8 +1294,7 @@ struct LIBMDBX_API to_base64 { const slice source; const unsigned wrap_width = 0; MDBX_CXX11_CONSTEXPR - to_base64(const slice &source, unsigned wrap_width = 0) noexcept - : source(source), wrap_width(wrap_width) { + to_base64(const slice &source, unsigned wrap_width = 0) noexcept : source(source), wrap_width(wrap_width) { MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(SliceTranscoder, to_base64); } @@ -1485,10 +1307,8 @@ struct LIBMDBX_API to_base64 { /// \brief Returns a buffer with a /// [Base64](https://en.wikipedia.org/wiki/Base64) dump of a passed slice. - template - buffer - as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { + template + buffer as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_buffer(*this, allocator); } @@ -1519,24 +1339,15 @@ struct LIBMDBX_API to_base64 { bool is_erroneous() const noexcept { return false; } }; -inline ::std::ostream &operator<<(::std::ostream &out, const to_hex &wrapper) { - return wrapper.output(out); -} -inline ::std::ostream &operator<<(::std::ostream &out, - const to_base58 &wrapper) { - return wrapper.output(out); -} -inline ::std::ostream &operator<<(::std::ostream &out, - const to_base64 &wrapper) { - return wrapper.output(out); -} +inline ::std::ostream &operator<<(::std::ostream &out, const to_hex &wrapper) { return wrapper.output(out); } +inline ::std::ostream &operator<<(::std::ostream &out, const to_base58 &wrapper) { return wrapper.output(out); } +inline ::std::ostream &operator<<(::std::ostream &out, const to_base64 &wrapper) { return wrapper.output(out); } /// \brief Hexadecimal decoder which satisfy \ref SliceTranscoder concept. struct LIBMDBX_API from_hex { const slice source; const bool ignore_spaces = false; - MDBX_CXX11_CONSTEXPR from_hex(const slice &source, - bool ignore_spaces = false) noexcept + MDBX_CXX11_CONSTEXPR from_hex(const slice &source, bool ignore_spaces = false) noexcept : source(source), ignore_spaces(ignore_spaces) { MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(SliceTranscoder, from_hex); } @@ -1548,18 +1359,14 @@ struct LIBMDBX_API from_hex { } /// \brief Decodes hexadecimal dump from a passed slice to returned buffer. - template - buffer - as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { + template + buffer as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_buffer(*this, allocator); } /// \brief Returns the number of bytes needed for conversion /// hexadecimal dump from a passed slice to decoded data. - MDBX_CXX11_CONSTEXPR size_t envisage_result_length() const noexcept { - return source.length() >> 1; - } + MDBX_CXX11_CONSTEXPR size_t envisage_result_length() const noexcept { return source.length() >> 1; } /// \brief Fills the destination with data decoded from hexadecimal dump /// from a passed slice. @@ -1580,8 +1387,7 @@ struct LIBMDBX_API from_hex { struct LIBMDBX_API from_base58 { const slice source; const bool ignore_spaces = false; - MDBX_CXX11_CONSTEXPR from_base58(const slice &source, - bool ignore_spaces = false) noexcept + MDBX_CXX11_CONSTEXPR from_base58(const slice &source, bool ignore_spaces = false) noexcept : source(source), ignore_spaces(ignore_spaces) { MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(SliceTranscoder, from_base58); } @@ -1595,10 +1401,8 @@ struct LIBMDBX_API from_base58 { /// \brief Decodes [Base58](https://en.wikipedia.org/wiki/Base58) dump from a /// passed slice to returned buffer. - template - buffer - as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { + template + buffer as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_buffer(*this, allocator); } @@ -1629,8 +1433,7 @@ struct LIBMDBX_API from_base58 { struct LIBMDBX_API from_base64 { const slice source; const bool ignore_spaces = false; - MDBX_CXX11_CONSTEXPR from_base64(const slice &source, - bool ignore_spaces = false) noexcept + MDBX_CXX11_CONSTEXPR from_base64(const slice &source, bool ignore_spaces = false) noexcept : source(source), ignore_spaces(ignore_spaces) { MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(SliceTranscoder, from_base64); } @@ -1644,19 +1447,15 @@ struct LIBMDBX_API from_base64 { /// \brief Decodes [Base64](https://en.wikipedia.org/wiki/Base64) dump from a /// passed slice to returned buffer. - template - buffer - as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { + template + buffer as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_buffer(*this, allocator); } /// \brief Returns the number of bytes needed for conversion /// [Base64](https://en.wikipedia.org/wiki/Base64) dump from a passed slice to /// decoded data. - MDBX_CXX11_CONSTEXPR size_t envisage_result_length() const noexcept { - return (source.length() + 3) / 4 * 3; - } + MDBX_CXX11_CONSTEXPR size_t envisage_result_length() const noexcept { return (source.length() + 3) / 4 * 3; } /// \brief Fills the destination with data decoded from /// [Base64](https://en.wikipedia.org/wiki/Base64) dump from a passed slice. @@ -1677,8 +1476,7 @@ struct LIBMDBX_API from_base64 { template class buffer { public: #if !defined(_MSC_VER) || _MSC_VER > 1900 - using allocator_type = typename ::std::allocator_traits< - ALLOCATOR>::template rebind_alloc; + using allocator_type = typename ::std::allocator_traits::template rebind_alloc; #else using allocator_type = typename ALLOCATOR::template rebind::other; #endif /* MSVC is mad */ @@ -1696,44 +1494,34 @@ private: struct silo; using swap_alloc = allocation_aware_details::swap_alloc; struct silo /* Empty Base Class Optimization */ : public allocator_type { - MDBX_CXX20_CONSTEXPR const allocator_type &get_allocator() const noexcept { - return *this; - } - MDBX_CXX20_CONSTEXPR allocator_type &get_allocator() noexcept { - return *this; - } + MDBX_CXX20_CONSTEXPR const allocator_type &get_allocator() const noexcept { return *this; } + MDBX_CXX20_CONSTEXPR allocator_type &get_allocator() noexcept { return *this; } using allocator_pointer = typename allocator_traits::pointer; using allocator_const_pointer = typename allocator_traits::const_pointer; - MDBX_CXX20_CONSTEXPR ::std::pair - allocate_storage(size_t bytes) { + MDBX_CXX20_CONSTEXPR ::std::pair allocate_storage(size_t bytes) { assert(bytes >= sizeof(bin)); constexpr size_t unit = sizeof(typename allocator_type::value_type); - static_assert((unit & (unit - 1)) == 0, - "size of ALLOCATOR::value_type should be a power of 2"); + static_assert((unit & (unit - 1)) == 0, "size of ALLOCATOR::value_type should be a power of 2"); static_assert(unit > 0, "size of ALLOCATOR::value_type must be > 0"); const size_t n = (bytes + unit - 1) / unit; - return ::std::make_pair(allocator_traits::allocate(get_allocator(), n), - n * unit); + return ::std::make_pair(allocator_traits::allocate(get_allocator(), n), n * unit); } - MDBX_CXX20_CONSTEXPR void deallocate_storage(allocator_pointer ptr, - size_t bytes) { + MDBX_CXX20_CONSTEXPR void deallocate_storage(allocator_pointer ptr, size_t bytes) { constexpr size_t unit = sizeof(typename allocator_type::value_type); assert(ptr && bytes >= sizeof(bin) && bytes >= unit && bytes % unit == 0); allocator_traits::deallocate(get_allocator(), ptr, bytes / unit); } - static MDBX_CXX17_CONSTEXPR void * - to_address(allocator_pointer ptr) noexcept { + static MDBX_CXX17_CONSTEXPR void *to_address(allocator_pointer ptr) noexcept { #if defined(__cpp_lib_to_address) && __cpp_lib_to_address >= 201711L return static_cast(::std::to_address(ptr)); #else return static_cast(::std::addressof(*ptr)); #endif /* __cpp_lib_to_address */ } - static MDBX_CXX17_CONSTEXPR const void * - to_address(allocator_const_pointer ptr) noexcept { + static MDBX_CXX17_CONSTEXPR const void *to_address(allocator_const_pointer ptr) noexcept { #if defined(__cpp_lib_to_address) && __cpp_lib_to_address >= 201711L return static_cast(::std::to_address(ptr)); #else @@ -1745,55 +1533,41 @@ private: struct allocated { allocator_pointer ptr_; size_t capacity_bytes_; - constexpr allocated(allocator_pointer ptr, size_t bytes) noexcept - : ptr_(ptr), capacity_bytes_(bytes) {} + constexpr allocated(allocator_pointer ptr, size_t bytes) noexcept : ptr_(ptr), capacity_bytes_(bytes) {} constexpr allocated(const allocated &) noexcept = default; constexpr allocated(allocated &&) noexcept = default; - MDBX_CXX17_CONSTEXPR allocated & - operator=(const allocated &) noexcept = default; - MDBX_CXX17_CONSTEXPR allocated & - operator=(allocated &&) noexcept = default; + MDBX_CXX17_CONSTEXPR allocated &operator=(const allocated &) noexcept = default; + MDBX_CXX17_CONSTEXPR allocated &operator=(allocated &&) noexcept = default; }; allocated allocated_; uint64_t align_hint_; - byte inplace_[(sizeof(allocated) + extra_inplace_storage + 7u) & - ~size_t(7)]; + byte inplace_[(sizeof(allocated) + extra_inplace_storage + 7u) & ~size_t(7)]; - static constexpr bool - is_suitable_for_inplace(size_t capacity_bytes) noexcept { + static constexpr bool is_suitable_for_inplace(size_t capacity_bytes) noexcept { static_assert(sizeof(bin) == sizeof(inplace_), "WTF?"); return capacity_bytes < sizeof(bin); } enum : byte { lastbyte_inplace_signature = byte(~byte(0)) }; enum : size_t { - inplace_signature_limit = - size_t(lastbyte_inplace_signature) - << (sizeof(size_t /* allocated::capacity_bytes_ */) - 1) * CHAR_BIT + inplace_signature_limit = size_t(lastbyte_inplace_signature) + << (sizeof(size_t /* allocated::capacity_bytes_ */) - 1) * CHAR_BIT }; - constexpr byte inplace_lastbyte() const noexcept { - return inplace_[sizeof(bin) - 1]; - } - MDBX_CXX17_CONSTEXPR byte &inplace_lastbyte() noexcept { - return inplace_[sizeof(bin) - 1]; - } + constexpr byte inplace_lastbyte() const noexcept { return inplace_[sizeof(bin) - 1]; } + MDBX_CXX17_CONSTEXPR byte &inplace_lastbyte() noexcept { return inplace_[sizeof(bin) - 1]; } constexpr bool is_inplace() const noexcept { - static_assert(size_t(inplace_signature_limit) > size_t(max_capacity), + static_assert(size_t(inplace_signature_limit) > size_t(max_capacity), "WTF?"); + static_assert(std::numeric_limits::max() - (std::numeric_limits::max() >> CHAR_BIT) == + inplace_signature_limit, "WTF?"); - static_assert( - std::numeric_limits::max() - - (std::numeric_limits::max() >> CHAR_BIT) == - inplace_signature_limit, - "WTF?"); return inplace_lastbyte() == lastbyte_inplace_signature; } constexpr bool is_allocated() const noexcept { return !is_inplace(); } - template - MDBX_CXX17_CONSTEXPR byte *make_inplace() noexcept { + template MDBX_CXX17_CONSTEXPR byte *make_inplace() noexcept { if (destroy_ptr) { MDBX_CONSTEXPR_ASSERT(is_allocated()); /* properly destroy allocator::pointer */ @@ -1803,14 +1577,12 @@ private: /* workaround for "uninitialized" warning from some compilers */ memset(&allocated_.ptr_, 0, sizeof(allocated_.ptr_)); inplace_lastbyte() = lastbyte_inplace_signature; - MDBX_CONSTEXPR_ASSERT(is_inplace() && address() == inplace_ && - is_suitable_for_inplace(capacity())); + MDBX_CONSTEXPR_ASSERT(is_inplace() && address() == inplace_ && is_suitable_for_inplace(capacity())); return address(); } template - MDBX_CXX17_CONSTEXPR byte * - make_allocated(allocator_pointer ptr, size_t capacity_bytes) noexcept { + MDBX_CXX17_CONSTEXPR byte *make_allocated(allocator_pointer ptr, size_t capacity_bytes) noexcept { MDBX_CONSTEXPR_ASSERT(inplace_signature_limit > capacity_bytes); if (construct_ptr) /* properly construct allocator::pointer */ @@ -1820,8 +1592,7 @@ private: allocated_.ptr_ = ptr; allocated_.capacity_bytes_ = capacity_bytes; } - MDBX_CONSTEXPR_ASSERT(is_allocated() && address() == to_address(ptr) && - capacity() == capacity_bytes); + MDBX_CONSTEXPR_ASSERT(is_allocated() && address() == to_address(ptr) && capacity() == capacity_bytes); return address(); } @@ -1830,8 +1601,7 @@ private: make_inplace(); (void)capacity_bytes; } - MDBX_CXX20_CONSTEXPR bin(allocator_pointer ptr, - size_t capacity_bytes) noexcept { + MDBX_CXX20_CONSTEXPR bin(allocator_pointer ptr, size_t capacity_bytes) noexcept { MDBX_CONSTEXPR_ASSERT(!is_suitable_for_inplace(capacity_bytes)); make_allocated(ptr, capacity_bytes); } @@ -1863,11 +1633,9 @@ private: memcpy(inplace_, ditto.inplace_, sizeof(inplace_)); MDBX_CONSTEXPR_ASSERT(is_inplace()); } else if (is_inplace()) - make_allocated(ditto.allocated_.ptr_, - ditto.allocated_.capacity_bytes_); + make_allocated(ditto.allocated_.ptr_, ditto.allocated_.capacity_bytes_); else - make_allocated(ditto.allocated_.ptr_, - ditto.allocated_.capacity_bytes_); + make_allocated(ditto.allocated_.ptr_, ditto.allocated_.capacity_bytes_); return *this; } @@ -1878,29 +1646,22 @@ private: return *this; } - static MDBX_CXX20_CONSTEXPR size_t advise_capacity(const size_t current, - const size_t wanna) { + static MDBX_CXX20_CONSTEXPR size_t advise_capacity(const size_t current, const size_t wanna) { if (MDBX_UNLIKELY(wanna > max_capacity)) MDBX_CXX20_UNLIKELY throw_max_length_exceeded(); const size_t advised = reservation_policy::advise(current, wanna); assert(advised >= wanna); - return ::std::min(size_t(max_capacity), - ::std::max(sizeof(bin) - 1, advised)); + return ::std::min(size_t(max_capacity), ::std::max(sizeof(bin) - 1, advised)); } constexpr const byte *address() const noexcept { - return is_inplace() - ? inplace_ - : static_cast(to_address(allocated_.ptr_)); + return is_inplace() ? inplace_ : static_cast(to_address(allocated_.ptr_)); } MDBX_CXX17_CONSTEXPR byte *address() noexcept { - return is_inplace() ? inplace_ - : static_cast(to_address(allocated_.ptr_)); - } - constexpr size_t capacity() const noexcept { - return is_inplace() ? sizeof(bin) - 1 : allocated_.capacity_bytes_; + return is_inplace() ? inplace_ : static_cast(to_address(allocated_.ptr_)); } + constexpr size_t capacity() const noexcept { return is_inplace() ? sizeof(bin) - 1 : allocated_.capacity_bytes_; } } bin_; MDBX_CXX20_CONSTEXPR void *init(size_t capacity) { @@ -1917,36 +1678,30 @@ private: MDBX_CXX20_CONSTEXPR void release() noexcept { if (bin_.is_allocated()) { - deallocate_storage(bin_.allocated_.ptr_, - bin_.allocated_.capacity_bytes_); + deallocate_storage(bin_.allocated_.ptr_, bin_.allocated_.capacity_bytes_); bin_.template make_inplace(); } } template - MDBX_CXX20_CONSTEXPR void * - reshape(const size_t wanna_capacity, const size_t wanna_headroom, - const void *const content, const size_t length) { + MDBX_CXX20_CONSTEXPR void *reshape(const size_t wanna_capacity, const size_t wanna_headroom, + const void *const content, const size_t length) { assert(wanna_capacity >= wanna_headroom + length); const size_t old_capacity = bin_.capacity(); - const size_t new_capacity = - bin::advise_capacity(old_capacity, wanna_capacity); + const size_t new_capacity = bin::advise_capacity(old_capacity, wanna_capacity); if (MDBX_LIKELY(new_capacity == old_capacity)) MDBX_CXX20_LIKELY { - assert(bin_.is_inplace() == - bin::is_suitable_for_inplace(new_capacity)); + assert(bin_.is_inplace() == bin::is_suitable_for_inplace(new_capacity)); byte *const new_place = bin_.address() + wanna_headroom; if (MDBX_LIKELY(length)) MDBX_CXX20_LIKELY { if (external_content) memcpy(new_place, content, length); else { - const size_t old_headroom = - bin_.address() - static_cast(content); + const size_t old_headroom = bin_.address() - static_cast(content); assert(old_capacity >= old_headroom + length); if (MDBX_UNLIKELY(old_headroom != wanna_headroom)) - MDBX_CXX20_UNLIKELY ::std::memmove(new_place, content, - length); + MDBX_CXX20_UNLIKELY ::std::memmove(new_place, content, length); } } return new_place; @@ -1955,8 +1710,7 @@ private: if (bin::is_suitable_for_inplace(new_capacity)) { assert(bin_.is_allocated()); const auto old_allocated = ::std::move(bin_.allocated_.ptr_); - byte *const new_place = - bin_.template make_inplace() + wanna_headroom; + byte *const new_place = bin_.template make_inplace() + wanna_headroom; if (MDBX_LIKELY(length)) MDBX_CXX20_LIKELY memcpy(new_place, content, length); deallocate_storage(old_allocated, old_capacity); @@ -1966,8 +1720,7 @@ private: if (!bin_.is_allocated()) { const auto pair = allocate_storage(new_capacity); assert(pair.second >= new_capacity); - byte *const new_place = - static_cast(to_address(pair.first)) + wanna_headroom; + byte *const new_place = static_cast(to_address(pair.first)) + wanna_headroom; if (MDBX_LIKELY(length)) MDBX_CXX20_LIKELY memcpy(new_place, content, length); bin_.template make_allocated(pair.first, pair.second); @@ -1979,9 +1732,7 @@ private: deallocate_storage(old_allocated, old_capacity); const auto pair = allocate_storage(new_capacity); assert(pair.second >= new_capacity); - byte *const new_place = - bin_.template make_allocated(pair.first, pair.second) + - wanna_headroom; + byte *const new_place = bin_.template make_allocated(pair.first, pair.second) + wanna_headroom; if (MDBX_LIKELY(length)) MDBX_CXX20_LIKELY memcpy(new_place, content, length); if (!external_content) @@ -1997,8 +1748,7 @@ private: assert(capacity() >= offset); return bin_.address() + offset; } - MDBX_CXX20_CONSTEXPR byte *put(size_t offset, const void *ptr, - size_t length) { + MDBX_CXX20_CONSTEXPR byte *put(size_t offset, const void *ptr, size_t length) { assert(capacity() >= offset + length); return static_cast(memcpy(get(offset), ptr, length)); } @@ -2008,128 +1758,92 @@ private: MDBX_CXX20_CONSTEXPR silo() noexcept : allocator_type() { init(0); } MDBX_CXX20_CONSTEXPR - silo(const allocator_type &alloc) noexcept : allocator_type(alloc) { - init(0); - } + silo(const allocator_type &alloc) noexcept : allocator_type(alloc) { init(0); } MDBX_CXX20_CONSTEXPR silo(size_t capacity) { init(capacity); } - MDBX_CXX20_CONSTEXPR silo(size_t capacity, const allocator_type &alloc) - : silo(alloc) { - init(capacity); - } + MDBX_CXX20_CONSTEXPR silo(size_t capacity, const allocator_type &alloc) : silo(alloc) { init(capacity); } - MDBX_CXX20_CONSTEXPR silo(silo &&ditto) noexcept( - ::std::is_nothrow_move_constructible::value) - : allocator_type(::std::move(ditto.get_allocator())), - bin_(::std::move(ditto.bin_)) {} + MDBX_CXX20_CONSTEXPR silo(silo &&ditto) noexcept(::std::is_nothrow_move_constructible::value) + : allocator_type(::std::move(ditto.get_allocator())), bin_(::std::move(ditto.bin_)) {} - MDBX_CXX20_CONSTEXPR silo(size_t capacity, size_t headroom, const void *ptr, - size_t length) - : silo(capacity) { + MDBX_CXX20_CONSTEXPR silo(size_t capacity, size_t headroom, const void *ptr, size_t length) : silo(capacity) { assert(capacity >= headroom + length); if (length) put(headroom, ptr, length); } // select_on_container_copy_construction() - MDBX_CXX20_CONSTEXPR silo(size_t capacity, size_t headroom, const void *ptr, - size_t length, const allocator_type &alloc) + MDBX_CXX20_CONSTEXPR silo(size_t capacity, size_t headroom, const void *ptr, size_t length, + const allocator_type &alloc) : silo(capacity, alloc) { assert(capacity >= headroom + length); if (length) put(headroom, ptr, length); } - MDBX_CXX20_CONSTEXPR silo(const void *ptr, size_t length) - : silo(length, 0, ptr, length) {} - MDBX_CXX20_CONSTEXPR silo(const void *ptr, size_t length, - const allocator_type &alloc) + MDBX_CXX20_CONSTEXPR silo(const void *ptr, size_t length) : silo(length, 0, ptr, length) {} + MDBX_CXX20_CONSTEXPR silo(const void *ptr, size_t length, const allocator_type &alloc) : silo(length, 0, ptr, length, alloc) {} ~silo() { release(); } //-------------------------------------------------------------------------- - MDBX_CXX20_CONSTEXPR void *assign(size_t headroom, const void *ptr, - size_t length, size_t tailroom) { + MDBX_CXX20_CONSTEXPR void *assign(size_t headroom, const void *ptr, size_t length, size_t tailroom) { return reshape(headroom + length + tailroom, headroom, ptr, length); } - MDBX_CXX20_CONSTEXPR void *assign(const void *ptr, size_t length) { - return assign(0, ptr, length, 0); - } + MDBX_CXX20_CONSTEXPR void *assign(const void *ptr, size_t length) { return assign(0, ptr, length, 0); } - MDBX_CXX20_CONSTEXPR silo &assign(const silo &ditto, size_t headroom, - slice &content) { + MDBX_CXX20_CONSTEXPR silo &assign(const silo &ditto, size_t headroom, slice &content) { assert(ditto.get() + headroom == content.byte_ptr()); - if MDBX_IF_CONSTEXPR (!allocation_aware_details:: - allocator_is_always_equal()) { + if MDBX_IF_CONSTEXPR (!allocation_aware_details::allocator_is_always_equal()) { if (MDBX_UNLIKELY(get_allocator() != ditto.get_allocator())) MDBX_CXX20_UNLIKELY { release(); - allocation_aware_details::copy_assign_alloc< - silo, allocator_type>::propagate(this, ditto); + allocation_aware_details::copy_assign_alloc::propagate(this, ditto); } } - content.iov_base = reshape(ditto.capacity(), headroom, - content.data(), content.length()); + content.iov_base = reshape(ditto.capacity(), headroom, content.data(), content.length()); return *this; } MDBX_CXX20_CONSTEXPR silo & - assign(silo &&ditto, size_t headroom, slice &content) noexcept( - allocation_aware_details::move_assign_alloc< - silo, allocator_type>::is_nothrow()) { + assign(silo &&ditto, size_t headroom, + slice &content) noexcept(allocation_aware_details::move_assign_alloc::is_nothrow()) { assert(ditto.get() + headroom == content.byte_ptr()); - if (allocation_aware_details::move_assign_alloc< - silo, allocator_type>::is_moveable(this, ditto)) { + if (allocation_aware_details::move_assign_alloc::is_moveable(this, ditto)) { release(); - allocation_aware_details::move_assign_alloc< - silo, allocator_type>::propagate(this, ditto); + allocation_aware_details::move_assign_alloc::propagate(this, ditto); /* no reallocation nor copying required */ bin_ = ::std::move(ditto.bin_); assert(get() + headroom == content.byte_ptr()); } else { /* copy content since allocators are different */ - content.iov_base = reshape(ditto.capacity(), headroom, - content.data(), content.length()); + content.iov_base = reshape(ditto.capacity(), headroom, content.data(), content.length()); ditto.release(); } return *this; } - MDBX_CXX20_CONSTEXPR void *clear() { - return reshape(0, 0, nullptr, 0); - } - MDBX_CXX20_CONSTEXPR void *clear_and_reserve(size_t whole_capacity, - size_t headroom) { + MDBX_CXX20_CONSTEXPR void *clear() { return reshape(0, 0, nullptr, 0); } + MDBX_CXX20_CONSTEXPR void *clear_and_reserve(size_t whole_capacity, size_t headroom) { return reshape(whole_capacity, headroom, nullptr, 0); } - MDBX_CXX20_CONSTEXPR void resize(size_t capacity, size_t headroom, - slice &content) { - content.iov_base = - reshape(capacity, headroom, content.iov_base, content.iov_len); + MDBX_CXX20_CONSTEXPR void resize(size_t capacity, size_t headroom, slice &content) { + content.iov_base = reshape(capacity, headroom, content.iov_base, content.iov_len); } - MDBX_CXX20_CONSTEXPR void swap(silo &ditto) noexcept( - allocation_aware_details::swap_alloc::is_nothrow()) { - allocation_aware_details::swap_alloc::propagate( - this, ditto); + MDBX_CXX20_CONSTEXPR void + swap(silo &ditto) noexcept(allocation_aware_details::swap_alloc::is_nothrow()) { + allocation_aware_details::swap_alloc::propagate(this, ditto); ::std::swap(bin_, ditto.bin_); } /* MDBX_CXX20_CONSTEXPR void shrink_to_fit() { TODO } */ - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX11_CONSTEXPR size_t - capacity() const noexcept { - return bin_.capacity(); - } - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX11_CONSTEXPR const void * - data(size_t offset = 0) const noexcept { - return get(offset); - } - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX11_CONSTEXPR void * - data(size_t offset = 0) noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX11_CONSTEXPR size_t capacity() const noexcept { return bin_.capacity(); } + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX11_CONSTEXPR const void *data(size_t offset = 0) const noexcept { return get(offset); } + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX11_CONSTEXPR void *data(size_t offset = 0) noexcept { return get(offset); } }; silo silo_; @@ -2141,21 +1855,18 @@ private: slice_.iov_base = silo_.data(); } - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR const byte * - silo_begin() const noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR const byte *silo_begin() const noexcept { return static_cast(silo_.data()); } - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR const byte * - silo_end() const noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR const byte *silo_end() const noexcept { return silo_begin() + silo_.capacity(); } struct data_preserver : public exception_thunk { buffer data; data_preserver(allocator_type &allocator) : data(allocator) {} - static int callback(void *context, MDBX_val *target, const void *src, - size_t bytes) noexcept { + static int callback(void *context, MDBX_val *target, const void *src, size_t bytes) noexcept { auto self = static_cast(context); assert(self->is_clean()); assert(&self->data.slice_ == target); @@ -2168,12 +1879,8 @@ private: return MDBX_RESULT_TRUE; } } - MDBX_CXX11_CONSTEXPR operator MDBX_preserve_func() const noexcept { - return callback; - } - MDBX_CXX11_CONSTEXPR operator const buffer &() const noexcept { - return data; - } + MDBX_CXX11_CONSTEXPR operator MDBX_preserve_func() const noexcept { return callback; } + MDBX_CXX11_CONSTEXPR operator const buffer &() const noexcept { return data; } MDBX_CXX11_CONSTEXPR operator buffer &() noexcept { return data; } }; @@ -2182,61 +1889,46 @@ public: /// \todo buffer& operator>>(buffer&, ...) for reading (delegated to slice) /// \todo template key(X) for encoding keys while writing - using move_assign_alloc = - allocation_aware_details::move_assign_alloc; - using copy_assign_alloc = - allocation_aware_details::copy_assign_alloc; + using move_assign_alloc = allocation_aware_details::move_assign_alloc; + using copy_assign_alloc = allocation_aware_details::copy_assign_alloc; /// \brief Returns the associated allocator. - MDBX_CXX20_CONSTEXPR allocator_type get_allocator() const { - return silo_.get_allocator(); - } + MDBX_CXX20_CONSTEXPR allocator_type get_allocator() const { return silo_.get_allocator(); } /// \brief Checks whether data chunk stored inside the buffer, otherwise /// buffer just refers to data located outside the buffer. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool - is_freestanding() const noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool is_freestanding() const noexcept { static_assert(size_t(-long(max_length)) > max_length, "WTF?"); return size_t(byte_ptr() - silo_begin()) < silo_.capacity(); } /// \brief Checks whether the buffer just refers to data located outside /// the buffer, rather than stores it. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool - is_reference() const noexcept { - return !is_freestanding(); - } + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool is_reference() const noexcept { return !is_freestanding(); } /// \brief Returns the number of bytes that can be held in currently allocated /// storage. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t - capacity() const noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t capacity() const noexcept { return is_freestanding() ? silo_.capacity() : 0; } /// \brief Returns the number of bytes that available in currently allocated /// storage ahead the currently beginning of data. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t - headroom() const noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t headroom() const noexcept { return is_freestanding() ? slice_.byte_ptr() - silo_begin() : 0; } /// \brief Returns the number of bytes that available in currently allocated /// storage after the currently data end. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t - tailroom() const noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t tailroom() const noexcept { return is_freestanding() ? capacity() - headroom() - slice_.length() : 0; } /// \brief Returns casted to const pointer to byte an address of data. - MDBX_CXX11_CONSTEXPR const byte *byte_ptr() const noexcept { - return slice_.byte_ptr(); - } + MDBX_CXX11_CONSTEXPR const byte *byte_ptr() const noexcept { return slice_.byte_ptr(); } /// \brief Returns casted to const pointer to byte an end of data. - MDBX_CXX11_CONSTEXPR const byte *end_byte_ptr() const noexcept { - return slice_.end_byte_ptr(); - } + MDBX_CXX11_CONSTEXPR const byte *end_byte_ptr() const noexcept { return slice_.end_byte_ptr(); } /// \brief Returns casted to pointer to byte an address of data. /// \pre REQUIRES: The buffer should store data chunk, but not referenced to @@ -2255,14 +1947,10 @@ public: } /// \brief Returns casted to const pointer to char an address of data. - MDBX_CXX11_CONSTEXPR const char *char_ptr() const noexcept { - return slice_.char_ptr(); - } + MDBX_CXX11_CONSTEXPR const char *char_ptr() const noexcept { return slice_.char_ptr(); } /// \brief Returns casted to const pointer to char an end of data. - MDBX_CXX11_CONSTEXPR const char *end_char_ptr() const noexcept { - return slice_.end_char_ptr(); - } + MDBX_CXX11_CONSTEXPR const char *end_char_ptr() const noexcept { return slice_.end_char_ptr(); } /// \brief Returns casted to pointer to char an address of data. /// \pre REQUIRES: The buffer should store data chunk, but not referenced to @@ -2281,9 +1969,7 @@ public: } /// \brief Return a const pointer to the beginning of the referenced data. - MDBX_CXX11_CONSTEXPR const void *data() const noexcept { - return slice_.data(); - } + MDBX_CXX11_CONSTEXPR const void *data() const noexcept { return slice_.data(); } /// \brief Return a const pointer to the end of the referenced data. MDBX_CXX11_CONSTEXPR const void *end() const noexcept { return slice_.end(); } @@ -2305,18 +1991,13 @@ public: } /// \brief Returns the number of bytes. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t - length() const noexcept { - return MDBX_CONSTEXPR_ASSERT(is_reference() || - slice_.length() + headroom() <= - silo_.capacity()), - slice_.length(); + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t length() const noexcept { + return MDBX_CONSTEXPR_ASSERT(is_reference() || slice_.length() + headroom() <= silo_.capacity()), slice_.length(); } /// \brief Set length of data. MDBX_CXX14_CONSTEXPR buffer &set_length(size_t bytes) { - MDBX_CONSTEXPR_ASSERT(is_reference() || - bytes + headroom() <= silo_.capacity()); + MDBX_CONSTEXPR_ASSERT(is_reference() || bytes + headroom() <= silo_.capacity()); slice_.set_length(bytes); return *this; } @@ -2336,87 +2017,66 @@ public: } MDBX_CXX20_CONSTEXPR buffer() noexcept = default; - MDBX_CXX20_CONSTEXPR buffer(const allocator_type &allocator) noexcept - : silo_(allocator) {} + MDBX_CXX20_CONSTEXPR buffer(const allocator_type &allocator) noexcept : silo_(allocator) {} - buffer(const struct slice &src, bool make_reference, - const allocator_type &allocator = allocator_type()) + buffer(const struct slice &src, bool make_reference, const allocator_type &allocator = allocator_type()) : silo_(allocator), slice_(src) { if (!make_reference) insulate(); } - buffer(const buffer &src, bool make_reference, - const allocator_type &allocator = allocator_type()) + buffer(const buffer &src, bool make_reference, const allocator_type &allocator = allocator_type()) : buffer(src.slice_, make_reference, allocator) {} - buffer(const void *ptr, size_t bytes, bool make_reference, - const allocator_type &allocator = allocator_type()) + buffer(const void *ptr, size_t bytes, bool make_reference, const allocator_type &allocator = allocator_type()) : buffer(::mdbx::slice(ptr, bytes), make_reference, allocator) {} - template - buffer(const ::std::basic_string &) = delete; - template - buffer(const ::std::basic_string &&) = delete; + template buffer(const ::std::basic_string &) = delete; + template buffer(const ::std::basic_string &&) = delete; - buffer(const char *c_str, bool make_reference, + buffer(const char *c_str, bool make_reference, const allocator_type &allocator = allocator_type()) + : buffer(::mdbx::slice(c_str), make_reference, allocator) {} + +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template + buffer(const ::std::basic_string_view &view, bool make_reference, const allocator_type &allocator = allocator_type()) - : buffer(::mdbx::slice(c_str), make_reference, allocator){} - -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) - template - buffer(const ::std::basic_string_view &view, - bool make_reference, - const allocator_type &allocator = allocator_type()) - : buffer(::mdbx::slice(view), make_reference, allocator) { - } + : buffer(::mdbx::slice(view), make_reference, allocator) {} #endif /* __cpp_lib_string_view >= 201606L */ MDBX_CXX20_CONSTEXPR - buffer(const struct slice &src, - const allocator_type &allocator = allocator_type()) - : silo_(src.data(), src.length(), allocator), - slice_(silo_.data(), src.length()) {} + buffer(const struct slice &src, const allocator_type &allocator = allocator_type()) + : silo_(src.data(), src.length(), allocator), slice_(silo_.data(), src.length()) {} MDBX_CXX20_CONSTEXPR - buffer(const buffer &src, const allocator_type &allocator = allocator_type()) - : buffer(src.slice_, allocator) {} + buffer(const buffer &src, const allocator_type &allocator = allocator_type()) : buffer(src.slice_, allocator) {} MDBX_CXX20_CONSTEXPR - buffer(const void *ptr, size_t bytes, - const allocator_type &allocator = allocator_type()) + buffer(const void *ptr, size_t bytes, const allocator_type &allocator = allocator_type()) : buffer(::mdbx::slice(ptr, bytes), allocator) {} template - MDBX_CXX20_CONSTEXPR - buffer(const ::std::basic_string &str, - const allocator_type &allocator = allocator_type()) + MDBX_CXX20_CONSTEXPR buffer(const ::std::basic_string &str, + const allocator_type &allocator = allocator_type()) : buffer(::mdbx::slice(str), allocator) {} MDBX_CXX20_CONSTEXPR buffer(const char *c_str, const allocator_type &allocator = allocator_type()) - : buffer(::mdbx::slice(c_str), allocator){} + : buffer(::mdbx::slice(c_str), allocator) {} -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) - template - MDBX_CXX20_CONSTEXPR - buffer(const ::std::basic_string_view &view, - const allocator_type &allocator = allocator_type()) - : buffer(::mdbx::slice(view), allocator) { - } +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template + MDBX_CXX20_CONSTEXPR buffer(const ::std::basic_string_view &view, + const allocator_type &allocator = allocator_type()) + : buffer(::mdbx::slice(view), allocator) {} #endif /* __cpp_lib_string_view >= 201606L */ - buffer(size_t head_room, size_t tail_room, - const allocator_type &allocator = allocator_type()) - : silo_(allocator) { + buffer(size_t head_room, size_t tail_room, const allocator_type &allocator = allocator_type()) : silo_(allocator) { slice_.iov_base = silo_.init(check_length(head_room, tail_room)); assert(slice_.iov_len == 0); } - buffer(size_t capacity, const allocator_type &allocator = allocator_type()) - : silo_(allocator) { + buffer(size_t capacity, const allocator_type &allocator = allocator_type()) : silo_(allocator) { slice_.iov_base = silo_.init(check_length(capacity)); assert(slice_.iov_len == 0); } @@ -2424,87 +2084,57 @@ public: buffer(size_t head_room, const struct slice &src, size_t tail_room, const allocator_type &allocator = allocator_type()) : silo_(allocator) { - slice_.iov_base = - silo_.init(check_length(head_room, src.length(), tail_room)); + slice_.iov_base = silo_.init(check_length(head_room, src.length(), tail_room)); slice_.iov_len = src.length(); memcpy(slice_.iov_base, src.data(), src.length()); } - buffer(size_t head_room, const buffer &src, size_t tail_room, - const allocator_type &allocator = allocator_type()) + buffer(size_t head_room, const buffer &src, size_t tail_room, const allocator_type &allocator = allocator_type()) : buffer(head_room, src.slice_, tail_room, allocator) {} - inline buffer(const ::mdbx::txn &txn, const struct slice &src, - const allocator_type &allocator = allocator_type()); + inline buffer(const ::mdbx::txn &txn, const struct slice &src, const allocator_type &allocator = allocator_type()); buffer(buffer &&src) noexcept(move_assign_alloc::is_nothrow()) : silo_(::std::move(src.silo_)), slice_(::std::move(src.slice_)) {} - MDBX_CXX11_CONSTEXPR const struct slice &slice() const noexcept { - return slice_; - } + MDBX_CXX11_CONSTEXPR const struct slice &slice() const noexcept { return slice_; } - MDBX_CXX11_CONSTEXPR operator const struct slice &() const noexcept { - return slice_; - } + MDBX_CXX11_CONSTEXPR operator const struct slice &() const noexcept { return slice_; } #if defined(DOXYGEN) || (defined(__cpp_lib_span) && __cpp_lib_span >= 202002L) - template - MDBX_CXX14_CONSTEXPR buffer(const ::std::span &span) - : buffer(span.begin(), span.end()) { - static_assert(::std::is_standard_layout::value && - !::std::is_pointer::value, + template MDBX_CXX14_CONSTEXPR buffer(const ::std::span &span) : buffer(span.begin(), span.end()) { + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value, "Must be a standard layout type!"); } - template - MDBX_CXX14_CONSTEXPR ::std::span as_span() const { + template MDBX_CXX14_CONSTEXPR ::std::span as_span() const { return slice_.template as_span(); } - template MDBX_CXX14_CONSTEXPR ::std::span as_span() { - return slice_.template as_span(); - } + template MDBX_CXX14_CONSTEXPR ::std::span as_span() { return slice_.template as_span(); } - MDBX_CXX14_CONSTEXPR ::std::span bytes() const { - return as_span(); - } + MDBX_CXX14_CONSTEXPR ::std::span bytes() const { return as_span(); } MDBX_CXX14_CONSTEXPR ::std::span bytes() { return as_span(); } - MDBX_CXX14_CONSTEXPR ::std::span chars() const { - return as_span(); - } + MDBX_CXX14_CONSTEXPR ::std::span chars() const { return as_span(); } MDBX_CXX14_CONSTEXPR ::std::span chars() { return as_span(); } #endif /* __cpp_lib_span >= 202002L */ template - static buffer wrap(const POD &pod, bool make_reference = false, - const allocator_type &allocator = allocator_type()) { + static buffer wrap(const POD &pod, bool make_reference = false, const allocator_type &allocator = allocator_type()) { return buffer(::mdbx::slice::wrap(pod), make_reference, allocator); } - template MDBX_CXX14_CONSTEXPR POD as_pod() const { - return slice_.as_pod(); - } + template MDBX_CXX14_CONSTEXPR POD as_pod() const { return slice_.as_pod(); } #ifdef MDBX_U128_TYPE - MDBX_CXX14_CONSTEXPR MDBX_U128_TYPE as_uint128() const { - return slice().as_uint128(); - } + MDBX_CXX14_CONSTEXPR MDBX_U128_TYPE as_uint128() const { return slice().as_uint128(); } #endif /* MDBX_U128_TYPE */ - MDBX_CXX14_CONSTEXPR uint64_t as_uint64() const { - return slice().as_uint64(); - } - MDBX_CXX14_CONSTEXPR uint32_t as_uint32() const { - return slice().as_uint32(); - } - MDBX_CXX14_CONSTEXPR uint16_t as_uint16() const { - return slice().as_uint16(); - } + MDBX_CXX14_CONSTEXPR uint64_t as_uint64() const { return slice().as_uint64(); } + MDBX_CXX14_CONSTEXPR uint32_t as_uint32() const { return slice().as_uint32(); } + MDBX_CXX14_CONSTEXPR uint16_t as_uint16() const { return slice().as_uint16(); } MDBX_CXX14_CONSTEXPR uint8_t as_uint8() const { return slice().as_uint8(); } #ifdef MDBX_I128_TYPE - MDBX_CXX14_CONSTEXPR MDBX_I128_TYPE as_int128() const { - return slice().as_int128(); - } + MDBX_CXX14_CONSTEXPR MDBX_I128_TYPE as_int128() const { return slice().as_int128(); } #endif /* MDBX_I128_TYPE */ MDBX_CXX14_CONSTEXPR int64_t as_int64() const { return slice().as_int64(); } MDBX_CXX14_CONSTEXPR int32_t as_int32() const { return slice().as_int32(); } @@ -2528,32 +2158,27 @@ public: int8_t as_int8_adapt() const { return slice().as_int8_adapt(); } /// \brief Returns a new buffer with a hexadecimal dump of the slice content. - static buffer hex(const ::mdbx::slice &source, bool uppercase = false, - unsigned wrap_width = 0, + static buffer hex(const ::mdbx::slice &source, bool uppercase = false, unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) { - return source.template encode_hex( - uppercase, wrap_width, allocator); + return source.template encode_hex(uppercase, wrap_width, allocator); } /// \brief Returns a new buffer with a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of the slice content. static buffer base58(const ::mdbx::slice &source, unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) { - return source.template encode_base58(wrap_width, - allocator); + return source.template encode_base58(wrap_width, allocator); } /// \brief Returns a new buffer with a /// [Base64](https://en.wikipedia.org/wiki/Base64) dump of the slice content. static buffer base64(const ::mdbx::slice &source, unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) { - return source.template encode_base64(wrap_width, - allocator); + return source.template encode_base64(wrap_width, allocator); } /// \brief Returns a new buffer with a hexadecimal dump of the given pod. template - static buffer hex(const POD &pod, bool uppercase = false, - unsigned wrap_width = 0, + static buffer hex(const POD &pod, bool uppercase = false, unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) { return hex(mdbx::slice::wrap(pod), uppercase, wrap_width, allocator); } @@ -2561,105 +2186,80 @@ public: /// \brief Returns a new buffer with a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of the given pod. template - static buffer base58(const POD &pod, unsigned wrap_width = 0, - const allocator_type &allocator = allocator_type()) { + static buffer base58(const POD &pod, unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) { return base58(mdbx::slice::wrap(pod), wrap_width, allocator); } /// \brief Returns a new buffer with a /// [Base64](https://en.wikipedia.org/wiki/Base64) dump of the given pod. template - static buffer base64(const POD &pod, unsigned wrap_width = 0, - const allocator_type &allocator = allocator_type()) { + static buffer base64(const POD &pod, unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) { return base64(mdbx::slice::wrap(pod), wrap_width, allocator); } /// \brief Returns a new buffer with a hexadecimal dump of the slice content. buffer encode_hex(bool uppercase = false, unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) const { - return slice().template encode_hex( - uppercase, wrap_width, allocator); + return slice().template encode_hex(uppercase, wrap_width, allocator); } /// \brief Returns a new buffer with a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of the slice content. - buffer - encode_base58(unsigned wrap_width = 0, - const allocator_type &allocator = allocator_type()) const { - return slice().template encode_base58( - wrap_width, allocator); + buffer encode_base58(unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) const { + return slice().template encode_base58(wrap_width, allocator); } /// \brief Returns a new buffer with a /// [Base64](https://en.wikipedia.org/wiki/Base64) dump of the slice content. - buffer - encode_base64(unsigned wrap_width = 0, - const allocator_type &allocator = allocator_type()) const { - return slice().template encode_base64( - wrap_width, allocator); + buffer encode_base64(unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) const { + return slice().template encode_base64(wrap_width, allocator); } /// \brief Decodes hexadecimal dump from the slice content to returned buffer. - static buffer hex_decode(const ::mdbx::slice &source, - bool ignore_spaces = false, + static buffer hex_decode(const ::mdbx::slice &source, bool ignore_spaces = false, const allocator_type &allocator = allocator_type()) { - return source.template hex_decode(ignore_spaces, - allocator); + return source.template hex_decode(ignore_spaces, allocator); } /// \brief Decodes [Base58](https://en.wikipedia.org/wiki/Base58) dump /// from the slice content to returned buffer. - static buffer - base58_decode(const ::mdbx::slice &source, bool ignore_spaces = false, - const allocator_type &allocator = allocator_type()) { - return source.template base58_decode( - ignore_spaces, allocator); + static buffer base58_decode(const ::mdbx::slice &source, bool ignore_spaces = false, + const allocator_type &allocator = allocator_type()) { + return source.template base58_decode(ignore_spaces, allocator); } /// \brief Decodes [Base64](https://en.wikipedia.org/wiki/Base64) dump /// from the slice content to returned buffer. - static buffer - base64_decode(const ::mdbx::slice &source, bool ignore_spaces = false, - const allocator_type &allocator = allocator_type()) { - return source.template base64_decode( - ignore_spaces, allocator); + static buffer base64_decode(const ::mdbx::slice &source, bool ignore_spaces = false, + const allocator_type &allocator = allocator_type()) { + return source.template base64_decode(ignore_spaces, allocator); } /// \brief Decodes hexadecimal dump /// from the buffer content to new returned buffer. - buffer hex_decode(bool ignore_spaces = false, - const allocator_type &allocator = allocator_type()) const { + buffer hex_decode(bool ignore_spaces = false, const allocator_type &allocator = allocator_type()) const { return hex_decode(slice(), ignore_spaces, allocator); } /// \brief Decodes [Base58](https://en.wikipedia.org/wiki/Base58) dump /// from the buffer content to new returned buffer. - buffer - base58_decode(bool ignore_spaces = false, - const allocator_type &allocator = allocator_type()) const { + buffer base58_decode(bool ignore_spaces = false, const allocator_type &allocator = allocator_type()) const { return base58_decode(slice(), ignore_spaces, allocator); } /// \brief Decodes [Base64](https://en.wikipedia.org/wiki/Base64) dump /// from the buffer content to new returned buffer. - buffer - base64_decode(bool ignore_spaces = false, - const allocator_type &allocator = allocator_type()) const { + buffer base64_decode(bool ignore_spaces = false, const allocator_type &allocator = allocator_type()) const { return base64_decode(slice(), ignore_spaces, allocator); } /// \brief Reserves storage space. void reserve(size_t wanna_headroom, size_t wanna_tailroom) { - wanna_headroom = ::std::min(::std::max(headroom(), wanna_headroom), - wanna_headroom + pettiness_threshold); - wanna_tailroom = ::std::min(::std::max(tailroom(), wanna_tailroom), - wanna_tailroom + pettiness_threshold); - const size_t wanna_capacity = - check_length(wanna_headroom, slice_.length(), wanna_tailroom); + wanna_headroom = ::std::min(::std::max(headroom(), wanna_headroom), wanna_headroom + pettiness_threshold); + wanna_tailroom = ::std::min(::std::max(tailroom(), wanna_tailroom), wanna_tailroom + pettiness_threshold); + const size_t wanna_capacity = check_length(wanna_headroom, slice_.length(), wanna_tailroom); silo_.resize(wanna_capacity, wanna_headroom, slice_); - assert(headroom() >= wanna_headroom && - headroom() <= wanna_headroom + pettiness_threshold); - assert(tailroom() >= wanna_tailroom && - tailroom() <= wanna_tailroom + pettiness_threshold); + assert(headroom() >= wanna_headroom && headroom() <= wanna_headroom + pettiness_threshold); + assert(tailroom() >= wanna_tailroom && tailroom() <= wanna_tailroom + pettiness_threshold); } /// \brief Reserves space before the payload. @@ -2675,30 +2275,24 @@ public: } buffer &assign_freestanding(const void *ptr, size_t bytes) { - silo_.assign(static_cast(ptr), - check_length(bytes)); + silo_.assign(static_cast(ptr), check_length(bytes)); slice_.assign(silo_.data(), bytes); return *this; } - MDBX_CXX20_CONSTEXPR void - swap(buffer &other) noexcept(swap_alloc::is_nothrow()) { + MDBX_CXX20_CONSTEXPR void swap(buffer &other) noexcept(swap_alloc::is_nothrow()) { silo_.swap(other.silo_); slice_.swap(other.slice_); } - static buffer clone(const buffer &src, - const allocator_type &allocator = allocator_type()) { + static buffer clone(const buffer &src, const allocator_type &allocator = allocator_type()) { return buffer(src.headroom(), src.slice_, src.tailroom(), allocator); } - buffer &assign(const buffer &src, bool make_reference = false) { - return assign(src.slice_, make_reference); - } + buffer &assign(const buffer &src, bool make_reference = false) { return assign(src.slice_, make_reference); } buffer &assign(const void *ptr, size_t bytes, bool make_reference = false) { - return make_reference ? assign_reference(ptr, bytes) - : assign_freestanding(ptr, bytes); + return make_reference ? assign_reference(ptr, bytes) : assign_freestanding(ptr, bytes); } buffer &assign(const struct slice &src, bool make_reference = false) { @@ -2721,17 +2315,12 @@ public: return *this; } - buffer &assign(const void *begin, const void *end, - bool make_reference = false) { - return assign(begin, - static_cast(end) - - static_cast(begin), - make_reference); + buffer &assign(const void *begin, const void *end, bool make_reference = false) { + return assign(begin, static_cast(end) - static_cast(begin), make_reference); } template - buffer &assign(const ::std::basic_string &str, - bool make_reference = false) { + buffer &assign(const ::std::basic_string &str, bool make_reference = false) { return assign(str.data(), str.length(), make_reference); } @@ -2741,14 +2330,11 @@ public: #if defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L template - buffer &assign(const ::std::basic_string_view &view, - bool make_reference = false) { + buffer &assign(const ::std::basic_string_view &view, bool make_reference = false) { return assign(view.data(), view.length(), make_reference); } - template - buffer &assign(::std::basic_string_view &&view, - bool make_reference = false) { + template buffer &assign(::std::basic_string_view &&view, bool make_reference = false) { assign(view.data(), view.length(), make_reference); view = {}; return *this; @@ -2757,18 +2343,14 @@ public: buffer &operator=(const buffer &src) { return assign(src); } - buffer &operator=(buffer &&src) noexcept(move_assign_alloc::is_nothrow()) { - return assign(::std::move(src)); - } + buffer &operator=(buffer &&src) noexcept(move_assign_alloc::is_nothrow()) { return assign(::std::move(src)); } buffer &operator=(const struct slice &src) { return assign(src); } buffer &operator=(struct slice &&src) { return assign(::std::move(src)); } -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) - template - buffer &operator=(const ::std::basic_string_view &view) noexcept { +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template buffer &operator=(const ::std::basic_string_view &view) noexcept { return assign(view); } @@ -2779,58 +2361,43 @@ public: } /// \brief Return a string_view that references the data of this buffer. - template - operator ::std::basic_string_view() const noexcept { + template operator ::std::basic_string_view() const noexcept { return string_view(); } #endif /* __cpp_lib_string_view >= 201606L */ /// \brief Checks whether the string is empty. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool empty() const noexcept { - return length() == 0; - } + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool empty() const noexcept { return length() == 0; } /// \brief Checks whether the data pointer of the buffer is nullptr. - MDBX_CXX11_CONSTEXPR bool is_null() const noexcept { - return data() == nullptr; - } + MDBX_CXX11_CONSTEXPR bool is_null() const noexcept { return data() == nullptr; } /// \brief Returns the number of bytes. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t size() const noexcept { - return length(); - } + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t size() const noexcept { return length(); } /// \brief Returns the hash value of the data. /// \attention Function implementation and returned hash values may changed /// version to version, and in future the t1ha3 will be used here. Therefore /// values obtained from this function shouldn't be persisted anywhere. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t - hash_value() const noexcept { - return slice_.hash_value(); - } + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t hash_value() const noexcept { return slice_.hash_value(); } - template , - class A = legacy_allocator> - MDBX_CXX20_CONSTEXPR ::std::basic_string - as_string(const A &allocator = A()) const { + template , class A = legacy_allocator> + MDBX_CXX20_CONSTEXPR ::std::basic_string as_string(const A &allocator = A()) const { return slice_.as_string(allocator); } template - MDBX_CXX20_CONSTEXPR explicit - operator ::std::basic_string() const { + MDBX_CXX20_CONSTEXPR explicit operator ::std::basic_string() const { return as_string(); } /// \brief Checks if the data starts with the given prefix. - MDBX_NOTHROW_PURE_FUNCTION bool - starts_with(const struct slice &prefix) const noexcept { + MDBX_NOTHROW_PURE_FUNCTION bool starts_with(const struct slice &prefix) const noexcept { return slice_.starts_with(prefix); } /// \brief Checks if the data ends with the given suffix. - MDBX_NOTHROW_PURE_FUNCTION bool - ends_with(const struct slice &suffix) const noexcept { + MDBX_NOTHROW_PURE_FUNCTION bool ends_with(const struct slice &suffix) const noexcept { return slice_.ends_with(suffix); } @@ -2889,40 +2456,27 @@ public: /// \brief Returns the first "n" bytes of the data chunk. /// \pre REQUIRES: `n <= size()` - MDBX_CXX14_CONSTEXPR struct slice head(size_t n) const noexcept { - return slice_.head(n); - } + MDBX_CXX14_CONSTEXPR struct slice head(size_t n) const noexcept { return slice_.head(n); } /// \brief Returns the last "n" bytes of the data chunk. /// \pre REQUIRES: `n <= size()` - MDBX_CXX14_CONSTEXPR struct slice tail(size_t n) const noexcept { - return slice_.tail(n); - } + MDBX_CXX14_CONSTEXPR struct slice tail(size_t n) const noexcept { return slice_.tail(n); } /// \brief Returns the middle "n" bytes of the data chunk. /// \pre REQUIRES: `from + n <= size()` - MDBX_CXX14_CONSTEXPR struct slice middle(size_t from, - size_t n) const noexcept { - return slice_.middle(from, n); - } + MDBX_CXX14_CONSTEXPR struct slice middle(size_t from, size_t n) const noexcept { return slice_.middle(from, n); } /// \brief Returns the first "n" bytes of the data chunk. /// \throws std::out_of_range if `n >= size()` - MDBX_CXX14_CONSTEXPR struct slice safe_head(size_t n) const { - return slice_.safe_head(n); - } + MDBX_CXX14_CONSTEXPR struct slice safe_head(size_t n) const { return slice_.safe_head(n); } /// \brief Returns the last "n" bytes of the data chunk. /// \throws std::out_of_range if `n >= size()` - MDBX_CXX14_CONSTEXPR struct slice safe_tail(size_t n) const { - return slice_.safe_tail(n); - } + MDBX_CXX14_CONSTEXPR struct slice safe_tail(size_t n) const { return slice_.safe_tail(n); } /// \brief Returns the middle "n" bytes of the data chunk. /// \throws std::out_of_range if `from + n >= size()` - MDBX_CXX14_CONSTEXPR struct slice safe_middle(size_t from, size_t n) const { - return slice_.safe_middle(from, n); - } + MDBX_CXX14_CONSTEXPR struct slice safe_middle(size_t from, size_t n) const { return slice_.safe_middle(from, n); } buffer &append(const void *src, size_t bytes) { if (MDBX_UNLIKELY(tailroom() < check_length(bytes))) @@ -2932,41 +2486,33 @@ public: return *this; } - buffer &append(const struct slice &chunk) { - return append(chunk.data(), chunk.size()); - } + buffer &append(const struct slice &chunk) { return append(chunk.data(), chunk.size()); } buffer &add_header(const void *src, size_t bytes) { if (MDBX_UNLIKELY(headroom() < check_length(bytes))) MDBX_CXX20_UNLIKELY reserve_headroom(bytes); - slice_.iov_base = - memcpy(static_cast(slice_.iov_base) - bytes, src, bytes); + slice_.iov_base = memcpy(static_cast(slice_.iov_base) - bytes, src, bytes); slice_.iov_len += bytes; return *this; } - buffer &add_header(const struct slice &chunk) { - return add_header(chunk.data(), chunk.size()); - } + buffer &add_header(const struct slice &chunk) { return add_header(chunk.data(), chunk.size()); } - template - buffer &append_producer(PRODUCER &producer) { + template buffer &append_producer(PRODUCER &producer) { const size_t wanna_bytes = producer.envisage_result_length(); if (MDBX_UNLIKELY(tailroom() < check_length(wanna_bytes))) MDBX_CXX20_UNLIKELY reserve_tailroom(wanna_bytes); return set_end(producer.write_bytes(end_char_ptr(), tailroom())); } - template - buffer &append_producer(const PRODUCER &producer) { + template buffer &append_producer(const PRODUCER &producer) { const size_t wanna_bytes = producer.envisage_result_length(); if (MDBX_UNLIKELY(tailroom() < check_length(wanna_bytes))) MDBX_CXX20_UNLIKELY reserve_tailroom(wanna_bytes); return set_end(producer.write_bytes(end_char_ptr(), tailroom())); } - buffer &append_hex(const struct slice &data, bool uppercase = false, - unsigned wrap_width = 0) { + buffer &append_hex(const struct slice &data, bool uppercase = false, unsigned wrap_width = 0) { return append_producer(to_hex(data, uppercase, wrap_width)); } @@ -2978,18 +2524,15 @@ public: return append_producer(to_base64(data, wrap_width)); } - buffer &append_decoded_hex(const struct slice &data, - bool ignore_spaces = false) { + buffer &append_decoded_hex(const struct slice &data, bool ignore_spaces = false) { return append_producer(from_hex(data, ignore_spaces)); } - buffer &append_decoded_base58(const struct slice &data, - bool ignore_spaces = false) { + buffer &append_decoded_base58(const struct slice &data, bool ignore_spaces = false) { return append_producer(from_base58(data, ignore_spaces)); } - buffer &append_decoded_base64(const struct slice &data, - bool ignore_spaces = false) { + buffer &append_decoded_base64(const struct slice &data, bool ignore_spaces = false) { return append_producer(from_base64(data, ignore_spaces)); } @@ -3068,149 +2611,99 @@ public: //---------------------------------------------------------------------------- - template - static buffer key_from(const char (&text)[SIZE], bool make_reference = true) { + template static buffer key_from(const char (&text)[SIZE], bool make_reference = true) { return buffer(::mdbx::slice(text), make_reference); } -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) template - static buffer key_from(const ::std::basic_string_view &src, - bool make_reference = false) { + static buffer key_from(const ::std::basic_string_view &src, bool make_reference = false) { return buffer(src, make_reference); } #endif /* __cpp_lib_string_view >= 201606L */ - static buffer key_from(const char *src, bool make_reference = false) { - return buffer(src, make_reference); - } + static buffer key_from(const char *src, bool make_reference = false) { return buffer(src, make_reference); } template - static buffer key_from(const ::std::basic_string &src, - bool make_reference = false) { + static buffer key_from(const ::std::basic_string &src, bool make_reference = false) { return buffer(src, make_reference); } - static buffer key_from(silo &&src) noexcept { - return buffer(::std::move(src)); - } + static buffer key_from(silo &&src) noexcept { return buffer(::std::move(src)); } - static buffer key_from_double(const double ieee754_64bit) { - return wrap(::mdbx_key_from_double(ieee754_64bit)); - } + static buffer key_from_double(const double ieee754_64bit) { return wrap(::mdbx_key_from_double(ieee754_64bit)); } - static buffer key_from(const double ieee754_64bit) { - return key_from_double(ieee754_64bit); - } + static buffer key_from(const double ieee754_64bit) { return key_from_double(ieee754_64bit); } - static buffer key_from(const double *ieee754_64bit) { - return wrap(::mdbx_key_from_ptrdouble(ieee754_64bit)); - } + static buffer key_from(const double *ieee754_64bit) { return wrap(::mdbx_key_from_ptrdouble(ieee754_64bit)); } - static buffer key_from_u64(const uint64_t unsigned_int64) { - return wrap(unsigned_int64); - } + static buffer key_from_u64(const uint64_t unsigned_int64) { return wrap(unsigned_int64); } - static buffer key_from(const uint64_t unsigned_int64) { - return key_from_u64(unsigned_int64); - } + static buffer key_from(const uint64_t unsigned_int64) { return key_from_u64(unsigned_int64); } - static buffer key_from_i64(const int64_t signed_int64) { - return wrap(::mdbx_key_from_int64(signed_int64)); - } + static buffer key_from_i64(const int64_t signed_int64) { return wrap(::mdbx_key_from_int64(signed_int64)); } - static buffer key_from(const int64_t signed_int64) { - return key_from_i64(signed_int64); - } + static buffer key_from(const int64_t signed_int64) { return key_from_i64(signed_int64); } static buffer key_from_jsonInteger(const int64_t json_integer) { return wrap(::mdbx_key_from_jsonInteger(json_integer)); } - static buffer key_from_float(const float ieee754_32bit) { - return wrap(::mdbx_key_from_float(ieee754_32bit)); - } + static buffer key_from_float(const float ieee754_32bit) { return wrap(::mdbx_key_from_float(ieee754_32bit)); } - static buffer key_from(const float ieee754_32bit) { - return key_from_float(ieee754_32bit); - } + static buffer key_from(const float ieee754_32bit) { return key_from_float(ieee754_32bit); } - static buffer key_from(const float *ieee754_32bit) { - return wrap(::mdbx_key_from_ptrfloat(ieee754_32bit)); - } + static buffer key_from(const float *ieee754_32bit) { return wrap(::mdbx_key_from_ptrfloat(ieee754_32bit)); } - static buffer key_from_u32(const uint32_t unsigned_int32) { - return wrap(unsigned_int32); - } + static buffer key_from_u32(const uint32_t unsigned_int32) { return wrap(unsigned_int32); } - static buffer key_from(const uint32_t unsigned_int32) { - return key_from_u32(unsigned_int32); - } + static buffer key_from(const uint32_t unsigned_int32) { return key_from_u32(unsigned_int32); } - static buffer key_from_i32(const int32_t signed_int32) { - return wrap(::mdbx_key_from_int32(signed_int32)); - } + static buffer key_from_i32(const int32_t signed_int32) { return wrap(::mdbx_key_from_int32(signed_int32)); } - static buffer key_from(const int32_t signed_int32) { - return key_from_i32(signed_int32); - } + static buffer key_from(const int32_t signed_int32) { return key_from_i32(signed_int32); } }; -template -inline buffer -make_buffer(PRODUCER &producer, const ALLOCATOR &allocator) { +template +inline buffer make_buffer(PRODUCER &producer, const ALLOCATOR &allocator) { if (MDBX_LIKELY(!producer.is_empty())) MDBX_CXX20_LIKELY { - buffer result( - producer.envisage_result_length(), allocator); - result.set_end( - producer.write_bytes(result.end_char_ptr(), result.tailroom())); + buffer result(producer.envisage_result_length(), allocator); + result.set_end(producer.write_bytes(result.end_char_ptr(), result.tailroom())); return result; } return buffer(allocator); } -template -inline buffer -make_buffer(const PRODUCER &producer, const ALLOCATOR &allocator) { +template +inline buffer make_buffer(const PRODUCER &producer, const ALLOCATOR &allocator) { if (MDBX_LIKELY(!producer.is_empty())) MDBX_CXX20_LIKELY { - buffer result( - producer.envisage_result_length(), allocator); - result.set_end( - producer.write_bytes(result.end_char_ptr(), result.tailroom())); + buffer result(producer.envisage_result_length(), allocator); + result.set_end(producer.write_bytes(result.end_char_ptr(), result.tailroom())); return result; } return buffer(allocator); } template -inline string make_string(PRODUCER &producer, - const ALLOCATOR &allocator) { +inline string make_string(PRODUCER &producer, const ALLOCATOR &allocator) { string result(allocator); if (MDBX_LIKELY(!producer.is_empty())) MDBX_CXX20_LIKELY { result.resize(producer.envisage_result_length()); - result.resize(producer.write_bytes(const_cast(result.data()), - result.capacity()) - - result.data()); + result.resize(producer.write_bytes(const_cast(result.data()), result.capacity()) - result.data()); } return result; } template -inline string make_string(const PRODUCER &producer, - const ALLOCATOR &allocator) { +inline string make_string(const PRODUCER &producer, const ALLOCATOR &allocator) { string result(allocator); if (MDBX_LIKELY(!producer.is_empty())) MDBX_CXX20_LIKELY { result.resize(producer.envisage_result_length()); - result.resize(producer.write_bytes(const_cast(result.data()), - result.capacity()) - - result.data()); + result.resize(producer.write_bytes(const_cast(result.data()), result.capacity()) - result.data()); } return result; } @@ -3220,8 +2713,7 @@ inline string make_string(const PRODUCER &producer, struct value_result { slice value; bool done; - value_result(const slice &value, bool done) noexcept - : value(value), done(done) {} + value_result(const slice &value, bool done) noexcept : value(value), done(done) {} value_result(const value_result &) noexcept = default; value_result &operator=(const value_result &) noexcept = default; MDBX_CXX14_CONSTEXPR operator bool() const noexcept { @@ -3235,52 +2727,37 @@ struct value_result { struct pair { using stl_pair = std::pair; slice key, value; - MDBX_CXX11_CONSTEXPR pair(const slice &key, const slice &value) noexcept - : key(key), value(value) {} - MDBX_CXX11_CONSTEXPR pair(const stl_pair &couple) noexcept - : key(couple.first), value(couple.second) {} - MDBX_CXX11_CONSTEXPR operator stl_pair() const noexcept { - return stl_pair(key, value); - } + MDBX_CXX11_CONSTEXPR pair(const slice &key, const slice &value) noexcept : key(key), value(value) {} + MDBX_CXX11_CONSTEXPR pair(const stl_pair &couple) noexcept : key(couple.first), value(couple.second) {} + MDBX_CXX11_CONSTEXPR operator stl_pair() const noexcept { return stl_pair(key, value); } pair(const pair &) noexcept = default; pair &operator=(const pair &) noexcept = default; MDBX_CXX14_CONSTEXPR operator bool() const noexcept { assert(bool(key) == bool(value)); return key; } - MDBX_CXX14_CONSTEXPR static pair invalid() noexcept { - return pair(slice::invalid(), slice::invalid()); - } + MDBX_CXX14_CONSTEXPR static pair invalid() noexcept { return pair(slice::invalid(), slice::invalid()); } /// \brief Three-way fast non-lexicographically length-based comparison. - MDBX_NOTHROW_PURE_FUNCTION static MDBX_CXX14_CONSTEXPR intptr_t - compare_fast(const pair &a, const pair &b) noexcept; + MDBX_NOTHROW_PURE_FUNCTION static MDBX_CXX14_CONSTEXPR intptr_t compare_fast(const pair &a, const pair &b) noexcept; /// \brief Three-way lexicographically comparison. - MDBX_NOTHROW_PURE_FUNCTION static MDBX_CXX14_CONSTEXPR intptr_t - compare_lexicographically(const pair &a, const pair &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator==(const pair &a, - const pair &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator<(const pair &a, - const pair &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator>(const pair &a, - const pair &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator<=(const pair &a, - const pair &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator>=(const pair &a, - const pair &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator!=(const pair &a, - const pair &b) noexcept; + MDBX_NOTHROW_PURE_FUNCTION static MDBX_CXX14_CONSTEXPR intptr_t compare_lexicographically(const pair &a, + const pair &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator==(const pair &a, const pair &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator<(const pair &a, const pair &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator>(const pair &a, const pair &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator<=(const pair &a, const pair &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator>=(const pair &a, const pair &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator!=(const pair &a, const pair &b) noexcept; }; /// \brief Combines pair of slices for key and value with boolean flag to /// represent result of certain operations. struct pair_result : public pair { bool done; - MDBX_CXX11_CONSTEXPR pair_result() noexcept - : pair(pair::invalid()), done(false) {} - MDBX_CXX11_CONSTEXPR pair_result(const slice &key, const slice &value, - bool done) noexcept + MDBX_CXX11_CONSTEXPR pair_result() noexcept : pair(pair::invalid()), done(false) {} + MDBX_CXX11_CONSTEXPR pair_result(const slice &key, const slice &value, bool done) noexcept : pair(key, value), done(done) {} pair_result(const pair_result &) noexcept = default; pair_result &operator=(const pair_result &) noexcept = default; @@ -3290,8 +2767,7 @@ struct pair_result : public pair { } }; -template -struct buffer_pair_spec { +template struct buffer_pair_spec { using buffer_type = buffer; using allocator_type = typename buffer_type::allocator_type; using allocator_traits = typename buffer_type::allocator_traits; @@ -3301,64 +2777,49 @@ struct buffer_pair_spec { MDBX_CXX20_CONSTEXPR buffer_pair_spec() noexcept = default; MDBX_CXX20_CONSTEXPR - buffer_pair_spec(const allocator_type &allocator) noexcept - : key(allocator), value(allocator) {} + buffer_pair_spec(const allocator_type &allocator) noexcept : key(allocator), value(allocator) {} - buffer_pair_spec(const buffer_type &key, const buffer_type &value, - const allocator_type &allocator = allocator_type()) + buffer_pair_spec(const buffer_type &key, const buffer_type &value, const allocator_type &allocator = allocator_type()) : key(key, allocator), value(value, allocator) {} - buffer_pair_spec(const buffer_type &key, const buffer_type &value, - bool make_reference, + buffer_pair_spec(const buffer_type &key, const buffer_type &value, bool make_reference, const allocator_type &allocator = allocator_type()) - : key(key, make_reference, allocator), - value(value, make_reference, allocator) {} + : key(key, make_reference, allocator), value(value, make_reference, allocator) {} - buffer_pair_spec(const stl_pair &pair, - const allocator_type &allocator = allocator_type()) + buffer_pair_spec(const stl_pair &pair, const allocator_type &allocator = allocator_type()) : buffer_pair_spec(pair.first, pair.second, allocator) {} - buffer_pair_spec(const stl_pair &pair, bool make_reference, - const allocator_type &allocator = allocator_type()) + buffer_pair_spec(const stl_pair &pair, bool make_reference, const allocator_type &allocator = allocator_type()) : buffer_pair_spec(pair.first, pair.second, make_reference, allocator) {} - buffer_pair_spec(const slice &key, const slice &value, - const allocator_type &allocator = allocator_type()) + buffer_pair_spec(const slice &key, const slice &value, const allocator_type &allocator = allocator_type()) : key(key, allocator), value(value, allocator) {} buffer_pair_spec(const slice &key, const slice &value, bool make_reference, const allocator_type &allocator = allocator_type()) - : key(key, make_reference, allocator), - value(value, make_reference, allocator) {} + : key(key, make_reference, allocator), value(value, make_reference, allocator) {} - buffer_pair_spec(const pair &pair, - const allocator_type &allocator = allocator_type()) + buffer_pair_spec(const pair &pair, const allocator_type &allocator = allocator_type()) : buffer_pair_spec(pair.key, pair.value, allocator) {} - buffer_pair_spec(const pair &pair, bool make_reference, - const allocator_type &allocator = allocator_type()) + buffer_pair_spec(const pair &pair, bool make_reference, const allocator_type &allocator = allocator_type()) : buffer_pair_spec(pair.key, pair.value, make_reference, allocator) {} buffer_pair_spec(const txn &txn, const slice &key, const slice &value, const allocator_type &allocator = allocator_type()) : key(txn, key, allocator), value(txn, value, allocator) {} - buffer_pair_spec(const txn &txn, const pair &pair, - const allocator_type &allocator = allocator_type()) + buffer_pair_spec(const txn &txn, const pair &pair, const allocator_type &allocator = allocator_type()) : buffer_pair_spec(txn, pair.key, pair.value, allocator) {} - buffer_pair_spec(buffer_type &&key, buffer_type &&value) noexcept( - buffer_type::move_assign_alloc::is_nothrow()) + buffer_pair_spec(buffer_type &&key, buffer_type &&value) noexcept(buffer_type::move_assign_alloc::is_nothrow()) : key(::std::move(key)), value(::std::move(value)) {} - buffer_pair_spec(buffer_pair_spec &&pair) noexcept( - buffer_type::move_assign_alloc::is_nothrow()) + buffer_pair_spec(buffer_pair_spec &&pair) noexcept(buffer_type::move_assign_alloc::is_nothrow()) : buffer_pair_spec(::std::move(pair.key), ::std::move(pair.value)) {} /// \brief Checks whether data chunk stored inside the buffers both, otherwise /// at least one of buffers just refers to data located outside. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool - is_freestanding() const noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool is_freestanding() const noexcept { return key.is_freestanding() && value.is_freestanding(); } /// \brief Checks whether one of the buffers just refers to data located /// outside the buffer, rather than stores it. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool - is_reference() const noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool is_reference() const noexcept { return key.is_reference() || value.is_reference(); } /// \brief Makes buffers owning the data. @@ -3373,8 +2834,7 @@ struct buffer_pair_spec { }; template -using buffer_pair = buffer_pair_spec; +using buffer_pair = buffer_pair_spec; /// end of cxx_data @} @@ -3396,9 +2856,9 @@ enum class key_mode { ///< sorted as such. The keys must all be of the ///< same size and must be aligned while passing ///< as arguments. - msgpack = -1 ///< Keys are in [MessagePack](https://msgpack.org/) - ///< format with appropriate comparison. - ///< \note Not yet implemented and PRs are welcome. + msgpack = -1 ///< Keys are in [MessagePack](https://msgpack.org/) + ///< format with appropriate comparison. + ///< \note Not yet implemented and PRs are welcome. }; MDBX_CXX01_CONSTEXPR_ENUM bool is_usual(key_mode mode) noexcept { @@ -3417,70 +2877,57 @@ MDBX_CXX01_CONSTEXPR_ENUM bool is_reverse(key_mode mode) noexcept { return (MDBX_db_flags_t(mode) & MDBX_REVERSEKEY) != 0; } -MDBX_CXX01_CONSTEXPR_ENUM bool is_msgpack(key_mode mode) noexcept { - return mode == key_mode::msgpack; -} +MDBX_CXX01_CONSTEXPR_ENUM bool is_msgpack(key_mode mode) noexcept { return mode == key_mode::msgpack; } /// \brief Kind of the values and sorted multi-values with corresponding /// comparison. enum class value_mode { single = MDBX_DB_DEFAULTS, ///< Usual single value for each key. In terms of ///< keys, they are unique. - multi = - MDBX_DUPSORT, ///< A more than one data value could be associated with - ///< each key. Internally each key is stored once, and the - ///< corresponding data values are sorted by byte-by-byte - ///< lexicographic comparison like `std::memcmp()`. - ///< In terms of keys, they are not unique, i.e. has - ///< duplicates which are sorted by associated data values. + multi = MDBX_DUPSORT, ///< A more than one data value could be associated with + ///< each key. Internally each key is stored once, and the + ///< corresponding data values are sorted by byte-by-byte + ///< lexicographic comparison like `std::memcmp()`. + ///< In terms of keys, they are not unique, i.e. has + ///< duplicates which are sorted by associated data values. #if CONSTEXPR_ENUM_FLAGS_OPERATIONS || defined(DOXYGEN) - multi_reverse = - MDBX_DUPSORT | - MDBX_REVERSEDUP, ///< A more than one data value could be associated with - ///< each key. Internally each key is stored once, and - ///< the corresponding data values are sorted by - ///< byte-by-byte lexicographic comparison in reverse - ///< order, from the end of the keys to the beginning. - ///< In terms of keys, they are not unique, i.e. has - ///< duplicates which are sorted by associated data - ///< values. - multi_samelength = - MDBX_DUPSORT | - MDBX_DUPFIXED, ///< A more than one data value could be associated with - ///< each key, and all data values must be same length. - ///< Internally each key is stored once, and the - ///< corresponding data values are sorted by byte-by-byte - ///< lexicographic comparison like `std::memcmp()`. In - ///< terms of keys, they are not unique, i.e. has - ///< duplicates which are sorted by associated data values. - multi_ordinal = - MDBX_DUPSORT | MDBX_DUPFIXED | - MDBX_INTEGERDUP, ///< A more than one data value could be associated with - ///< each key, and all data values are binary integers in - ///< native byte order, either `uint32_t` or `uint64_t`, - ///< and will be sorted as such. Internally each key is - ///< stored once, and the corresponding data values are - ///< sorted. In terms of keys, they are not unique, i.e. - ///< has duplicates which are sorted by associated data - ///< values. + multi_reverse = MDBX_DUPSORT | MDBX_REVERSEDUP, ///< A more than one data value could be associated with + ///< each key. Internally each key is stored once, and + ///< the corresponding data values are sorted by + ///< byte-by-byte lexicographic comparison in reverse + ///< order, from the end of the keys to the beginning. + ///< In terms of keys, they are not unique, i.e. has + ///< duplicates which are sorted by associated data + ///< values. + multi_samelength = MDBX_DUPSORT | MDBX_DUPFIXED, ///< A more than one data value could be associated with + ///< each key, and all data values must be same length. + ///< Internally each key is stored once, and the + ///< corresponding data values are sorted by byte-by-byte + ///< lexicographic comparison like `std::memcmp()`. In + ///< terms of keys, they are not unique, i.e. has + ///< duplicates which are sorted by associated data values. + multi_ordinal = MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP, ///< A more than one data value could be associated + ///< with each key, and all data values are binary + ///< integers in native byte order, either `uint32_t` + ///< or `uint64_t`, and will be sorted as such. + ///< Internally each key is stored once, and the + ///< corresponding data values are sorted. In terms of + ///< keys, they are not unique, i.e. has duplicates + ///< which are sorted by associated data values. multi_reverse_samelength = - MDBX_DUPSORT | MDBX_REVERSEDUP | - MDBX_DUPFIXED, ///< A more than one data value could be associated with - ///< each key, and all data values must be same length. - ///< Internally each key is stored once, and the - ///< corresponding data values are sorted by byte-by-byte - ///< lexicographic comparison in reverse order, from the - ///< end of the keys to the beginning. In terms of keys, - ///< they are not unique, i.e. has duplicates which are - ///< sorted by associated data values. + MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED, ///< A more than one data value could be associated with + ///< each key, and all data values must be same length. + ///< Internally each key is stored once, and the + ///< corresponding data values are sorted by byte-by-byte + ///< lexicographic comparison in reverse order, from the + ///< end of the keys to the beginning. In terms of keys, + ///< they are not unique, i.e. has duplicates which are + ///< sorted by associated data values. #else multi_reverse = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_REVERSEDUP), multi_samelength = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_DUPFIXED), - multi_ordinal = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_DUPFIXED) | - uint32_t(MDBX_INTEGERDUP), - multi_reverse_samelength = uint32_t(MDBX_DUPSORT) | - uint32_t(MDBX_REVERSEDUP) | - uint32_t(MDBX_DUPFIXED), + multi_ordinal = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_DUPFIXED) | uint32_t(MDBX_INTEGERDUP), + multi_reverse_samelength = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_REVERSEDUP) | uint32_t(MDBX_DUPFIXED), #endif msgpack = -1 ///< A more than one data value could be associated with each ///< key. Values are in [MessagePack](https://msgpack.org/) @@ -3492,8 +2939,7 @@ enum class value_mode { }; MDBX_CXX01_CONSTEXPR_ENUM bool is_usual(value_mode mode) noexcept { - return (MDBX_db_flags_t(mode) & (MDBX_DUPSORT | MDBX_INTEGERDUP | - MDBX_DUPFIXED | MDBX_REVERSEDUP)) == 0; + return (MDBX_db_flags_t(mode) & (MDBX_DUPSORT | MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_REVERSEDUP)) == 0; } MDBX_CXX01_CONSTEXPR_ENUM bool is_multi(value_mode mode) noexcept { @@ -3512,9 +2958,7 @@ MDBX_CXX01_CONSTEXPR_ENUM bool is_reverse(value_mode mode) noexcept { return (MDBX_db_flags_t(mode) & MDBX_REVERSEDUP) != 0; } -MDBX_CXX01_CONSTEXPR_ENUM bool is_msgpack(value_mode mode) noexcept { - return mode == value_mode::msgpack; -} +MDBX_CXX01_CONSTEXPR_ENUM bool is_msgpack(value_mode mode) noexcept { return mode == value_mode::msgpack; } /// \brief A handle for an individual table (aka key-value space, maps or /// sub-database) in the environment. @@ -3537,8 +2981,7 @@ struct LIBMDBX_API_TYPE map_handle { struct LIBMDBX_API_TYPE info { map_handle::flags flags; map_handle::state state; - MDBX_CXX11_CONSTEXPR info(map_handle::flags flags, - map_handle::state state) noexcept; + MDBX_CXX11_CONSTEXPR info(map_handle::flags flags, map_handle::state state) noexcept; info(const info &) noexcept = default; info &operator=(const info &) noexcept = default; MDBX_CXX11_CONSTEXPR_ENUM mdbx::key_mode key_mode() const noexcept; @@ -3587,10 +3030,8 @@ public: MDBX_CXX14_CONSTEXPR operator bool() const noexcept; MDBX_CXX14_CONSTEXPR operator const MDBX_env *() const; MDBX_CXX14_CONSTEXPR operator MDBX_env *(); - friend MDBX_CXX11_CONSTEXPR bool operator==(const env &a, - const env &b) noexcept; - friend MDBX_CXX11_CONSTEXPR bool operator!=(const env &a, - const env &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator==(const env &a, const env &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator!=(const env &a, const env &b) noexcept; //---------------------------------------------------------------------------- @@ -3664,20 +3105,15 @@ public: intptr_t pagesize{default_value}; inline geometry &make_fixed(intptr_t size) noexcept; - inline geometry &make_dynamic(intptr_t lower = minimal_value, - intptr_t upper = maximal_value) noexcept; + inline geometry &make_dynamic(intptr_t lower = minimal_value, intptr_t upper = maximal_value) noexcept; MDBX_CXX11_CONSTEXPR geometry() noexcept {} MDBX_CXX11_CONSTEXPR geometry(const geometry &) noexcept = default; - MDBX_CXX11_CONSTEXPR geometry(intptr_t size_lower, - intptr_t size_now = default_value, - intptr_t size_upper = maximal_value, - intptr_t growth_step = default_value, - intptr_t shrink_threshold = default_value, - intptr_t pagesize = default_value) noexcept - : size_lower(size_lower), size_now(size_now), size_upper(size_upper), - growth_step(growth_step), shrink_threshold(shrink_threshold), - pagesize(pagesize) {} + MDBX_CXX11_CONSTEXPR geometry(intptr_t size_lower, intptr_t size_now = default_value, + intptr_t size_upper = maximal_value, intptr_t growth_step = default_value, + intptr_t shrink_threshold = default_value, intptr_t pagesize = default_value) noexcept + : size_lower(size_lower), size_now(size_now), size_upper(size_upper), growth_step(growth_step), + shrink_threshold(shrink_threshold), pagesize(pagesize) {} }; /// \brief Operation mode. @@ -3705,8 +3141,7 @@ public: MDBX_CXX11_CONSTEXPR reclaiming_options() noexcept {} MDBX_CXX11_CONSTEXPR reclaiming_options(const reclaiming_options &) noexcept = default; - MDBX_CXX14_CONSTEXPR reclaiming_options & - operator=(const reclaiming_options &) noexcept = default; + MDBX_CXX14_CONSTEXPR reclaiming_options &operator=(const reclaiming_options &) noexcept = default; reclaiming_options(MDBX_env_flags_t) noexcept; }; @@ -3728,8 +3163,7 @@ public: MDBX_CXX11_CONSTEXPR operate_options() noexcept {} MDBX_CXX11_CONSTEXPR operate_options(const operate_options &) noexcept = default; - MDBX_CXX14_CONSTEXPR operate_options & - operator=(const operate_options &) noexcept = default; + MDBX_CXX14_CONSTEXPR operate_options &operator=(const operate_options &) noexcept = default; operate_options(MDBX_env_flags_t) noexcept; }; @@ -3748,31 +3182,25 @@ public: MDBX_CXX11_CONSTEXPR operate_parameters() noexcept {} MDBX_CXX11_CONSTEXPR - operate_parameters( - const unsigned max_maps, const unsigned max_readers = 0, - const env::mode mode = env::mode::write_mapped_io, - env::durability durability = env::durability::robust_synchronous, - const env::reclaiming_options &reclaiming = env::reclaiming_options(), - const env::operate_options &options = env::operate_options()) noexcept - : max_maps(max_maps), max_readers(max_readers), mode(mode), - durability(durability), reclaiming(reclaiming), options(options) {} + operate_parameters(const unsigned max_maps, const unsigned max_readers = 0, + const env::mode mode = env::mode::write_mapped_io, + env::durability durability = env::durability::robust_synchronous, + const env::reclaiming_options &reclaiming = env::reclaiming_options(), + const env::operate_options &options = env::operate_options()) noexcept + : max_maps(max_maps), max_readers(max_readers), mode(mode), durability(durability), reclaiming(reclaiming), + options(options) {} MDBX_CXX11_CONSTEXPR operate_parameters(const operate_parameters &) noexcept = default; - MDBX_CXX14_CONSTEXPR operate_parameters & - operator=(const operate_parameters &) noexcept = default; - MDBX_env_flags_t make_flags( - bool accede = true, ///< Allows accepting incompatible operating options - ///< in case the database is already being used by - ///< another process(es) \see MDBX_ACCEDE - bool use_subdirectory = - false ///< use subdirectory to place the DB files + MDBX_CXX14_CONSTEXPR operate_parameters &operator=(const operate_parameters &) noexcept = default; + MDBX_env_flags_t make_flags(bool accede = true, ///< Allows accepting incompatible operating options + ///< in case the database is already being used by + ///< another process(es) \see MDBX_ACCEDE + bool use_subdirectory = false ///< use subdirectory to place the DB files ) const; static env::mode mode_from_flags(MDBX_env_flags_t) noexcept; static env::durability durability_from_flags(MDBX_env_flags_t) noexcept; - inline static env::reclaiming_options - reclaiming_from_flags(MDBX_env_flags_t flags) noexcept; - inline static env::operate_options - options_from_flags(MDBX_env_flags_t flags) noexcept; + inline static env::reclaiming_options reclaiming_from_flags(MDBX_env_flags_t flags) noexcept; + inline static env::operate_options options_from_flags(MDBX_env_flags_t flags) noexcept; }; /// \brief Returns current operation parameters. @@ -3794,9 +3222,7 @@ public: bool is_empty() const; /// \brief Returns default page size for current system/platform. - static size_t default_pagesize() noexcept { - return ::mdbx_default_pagesize(); - } + static size_t default_pagesize() noexcept { return ::mdbx_default_pagesize(); } struct limits { limits() = delete; @@ -3849,8 +3275,7 @@ public: /// \brief Returns maximal size of key-value pair to fit in a single page /// for specified size and table flags. - static inline size_t pairsize4page_max(intptr_t pagesize, - MDBX_db_flags_t flags); + static inline size_t pairsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags); /// \brief Returns maximal size of key-value pair to fit in a single page /// for specified page size and values mode. static inline size_t pairsize4page_max(intptr_t pagesize, value_mode); @@ -3863,8 +3288,7 @@ public: /// \brief Returns maximal data size in bytes to fit in a leaf-page or /// single large/overflow-page for specified size and table flags. - static inline size_t valsize4page_max(intptr_t pagesize, - MDBX_db_flags_t flags); + static inline size_t valsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags); /// \brief Returns maximal data size in bytes to fit in a leaf-page or /// single large/overflow-page for specified page size and values mode. static inline size_t valsize4page_max(intptr_t pagesize, value_mode); @@ -3892,35 +3316,24 @@ public: /// \brief Returns the maximal key size in bytes for specified keys mode. size_t key_max(key_mode mode) const { return limits::key_max(*this, mode); } /// \brief Returns the minimal value size in bytes for specified values mode. - size_t value_min(value_mode mode) const noexcept { - return limits::value_min(mode); - } + size_t value_min(value_mode mode) const noexcept { return limits::value_min(mode); } /// \brief Returns the maximal value size in bytes for specified values mode. - size_t value_max(value_mode mode) const { - return limits::value_max(*this, mode); - } + size_t value_max(value_mode mode) const { return limits::value_max(*this, mode); } /// \brief Returns the maximal write transaction size (i.e. limit for summary /// volume of dirty pages) in bytes. - size_t transaction_size_max() const { - return limits::transaction_size_max(this->get_pagesize()); - } + size_t transaction_size_max() const { return limits::transaction_size_max(this->get_pagesize()); } /// \brief Make a copy (backup) of an existing environment to the specified /// path. #ifdef MDBX_STD_FILESYSTEM_PATH - env ©(const MDBX_STD_FILESYSTEM_PATH &destination, bool compactify, - bool force_dynamic_size = false); + env ©(const MDBX_STD_FILESYSTEM_PATH &destination, bool compactify, bool force_dynamic_size = false); #endif /* MDBX_STD_FILESYSTEM_PATH */ #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) - env ©(const ::std::wstring &destination, bool compactify, - bool force_dynamic_size = false); - env ©(const wchar_t *destination, bool compactify, - bool force_dynamic_size = false); + env ©(const ::std::wstring &destination, bool compactify, bool force_dynamic_size = false); + env ©(const wchar_t *destination, bool compactify, bool force_dynamic_size = false); #endif /* Windows */ - env ©(const ::std::string &destination, bool compactify, - bool force_dynamic_size = false); - env ©(const char *destination, bool compactify, - bool force_dynamic_size = false); + env ©(const ::std::string &destination, bool compactify, bool force_dynamic_size = false); + env ©(const char *destination, bool compactify, bool force_dynamic_size = false); /// \brief Copy an environment to the specified file descriptor. env ©(filehandle fd, bool compactify, bool force_dynamic_size = false); @@ -3945,19 +3358,14 @@ public: /// \brief Removes the environment's files in a proper and multiprocess-safe /// way. #ifdef MDBX_STD_FILESYSTEM_PATH - static bool remove(const MDBX_STD_FILESYSTEM_PATH &pathname, - const remove_mode mode = just_remove); + static bool remove(const MDBX_STD_FILESYSTEM_PATH &pathname, const remove_mode mode = just_remove); #endif /* MDBX_STD_FILESYSTEM_PATH */ #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) - static bool remove(const ::std::wstring &pathname, - const remove_mode mode = just_remove); - static bool remove(const wchar_t *pathname, - const remove_mode mode = just_remove); + static bool remove(const ::std::wstring &pathname, const remove_mode mode = just_remove); + static bool remove(const wchar_t *pathname, const remove_mode mode = just_remove); #endif /* Windows */ - static bool remove(const ::std::string &pathname, - const remove_mode mode = just_remove); - static bool remove(const char *pathname, - const remove_mode mode = just_remove); + static bool remove(const ::std::string &pathname, const remove_mode mode = just_remove); + static bool remove(const char *pathname, const remove_mode mode = just_remove); /// \brief Statistics for a database in the MDBX environment. using stat = ::MDBX_stat; @@ -4168,20 +3576,18 @@ public: ///< i.e. the number of committed write /// transactions since the current read /// transaction started. - size_t bytes_used; ///< The number of last used page in the MVCC-snapshot - ///< which being read, i.e. database file can't be shrunk - ///< beyond this. - size_t bytes_retained; ///< The total size of the database pages that - ///< were retired by committed write transactions - ///< after the reader's MVCC-snapshot, i.e. the space - ///< which would be freed after the Reader releases - ///< the MVCC-snapshot for reuse by completion read - ///< transaction. + size_t bytes_used; ///< The number of last used page in the MVCC-snapshot + ///< which being read, i.e. database file can't be shrunk + ///< beyond this. + size_t bytes_retained; ///< The total size of the database pages that + ///< were retired by committed write transactions + ///< after the reader's MVCC-snapshot, i.e. the space + ///< which would be freed after the Reader releases + ///< the MVCC-snapshot for reuse by completion read + ///< transaction. - MDBX_CXX11_CONSTEXPR reader_info(int slot, mdbx_pid_t pid, - mdbx_tid_t thread, uint64_t txnid, - uint64_t lag, size_t used, - size_t retained) noexcept; + MDBX_CXX11_CONSTEXPR reader_info(int slot, mdbx_pid_t pid, mdbx_tid_t thread, uint64_t txnid, uint64_t lag, + size_t used, size_t retained) noexcept; }; /// \brief Enumerate readers. @@ -4257,19 +3663,14 @@ public: /// \brief Open existing database. #ifdef MDBX_STD_FILESYSTEM_PATH - env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, - const operate_parameters &, bool accede = true); + env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, const operate_parameters &, bool accede = true); #endif /* MDBX_STD_FILESYSTEM_PATH */ #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) - env_managed(const ::std::wstring &pathname, const operate_parameters &, - bool accede = true); - explicit env_managed(const wchar_t *pathname, const operate_parameters &, - bool accede = true); + env_managed(const ::std::wstring &pathname, const operate_parameters &, bool accede = true); + explicit env_managed(const wchar_t *pathname, const operate_parameters &, bool accede = true); #endif /* Windows */ - env_managed(const ::std::string &pathname, const operate_parameters &, - bool accede = true); - explicit env_managed(const char *pathname, const operate_parameters &, - bool accede = true); + env_managed(const ::std::string &pathname, const operate_parameters &, bool accede = true); + explicit env_managed(const char *pathname, const operate_parameters &, bool accede = true); /// \brief Additional parameters for creating a new database. /// \see env_managed(const ::std::string &pathname, const create_parameters &, @@ -4284,20 +3685,17 @@ public: /// \brief Create new or open existing database. #ifdef MDBX_STD_FILESYSTEM_PATH - env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, - const create_parameters &, const operate_parameters &, + env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, const create_parameters &, const operate_parameters &, bool accede = true); #endif /* MDBX_STD_FILESYSTEM_PATH */ #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) - env_managed(const ::std::wstring &pathname, const create_parameters &, - const operate_parameters &, bool accede = true); - explicit env_managed(const wchar_t *pathname, const create_parameters &, - const operate_parameters &, bool accede = true); + env_managed(const ::std::wstring &pathname, const create_parameters &, const operate_parameters &, + bool accede = true); + explicit env_managed(const wchar_t *pathname, const create_parameters &, const operate_parameters &, + bool accede = true); #endif /* Windows */ - env_managed(const ::std::string &pathname, const create_parameters &, - const operate_parameters &, bool accede = true); - explicit env_managed(const char *pathname, const create_parameters &, - const operate_parameters &, bool accede = true); + env_managed(const ::std::string &pathname, const create_parameters &, const operate_parameters &, bool accede = true); + explicit env_managed(const char *pathname, const create_parameters &, const operate_parameters &, bool accede = true); /// \brief Explicitly closes the environment and release the memory map. /// @@ -4353,10 +3751,8 @@ public: MDBX_CXX14_CONSTEXPR operator bool() const noexcept; MDBX_CXX14_CONSTEXPR operator const MDBX_txn *() const; MDBX_CXX14_CONSTEXPR operator MDBX_txn *(); - friend MDBX_CXX11_CONSTEXPR bool operator==(const txn &a, - const txn &b) noexcept; - friend MDBX_CXX11_CONSTEXPR bool operator!=(const txn &a, - const txn &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator==(const txn &a, const txn &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator!=(const txn &a, const txn &b) noexcept; /// \brief Returns the transaction's environment. inline ::mdbx::env env() const noexcept; @@ -4426,20 +3822,14 @@ public: inline size_t unbind_all_cursors() const { return release_all_cursors(true); } /// \brief Open existing key-value map. - inline map_handle open_map( - const char *name, - const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, - const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const; + inline map_handle open_map(const char *name, const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const; /// \brief Open existing key-value map. - inline map_handle open_map( - const ::std::string &name, - const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, - const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const; + inline map_handle open_map(const ::std::string &name, const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const; /// \brief Open existing key-value map. - inline map_handle open_map( - const ::mdbx::slice &name, - const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, - const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const; + inline map_handle open_map(const ::mdbx::slice &name, const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const; /// \brief Open existing key-value map. inline map_handle open_map_accede(const char *name) const; @@ -4449,20 +3839,14 @@ public: inline map_handle open_map_accede(const ::mdbx::slice &name) const; /// \brief Create new or open existing key-value map. - inline map_handle - create_map(const char *name, - const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, - const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single); + inline map_handle create_map(const char *name, const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single); /// \brief Create new or open existing key-value map. - inline map_handle - create_map(const ::std::string &name, - const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, - const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single); + inline map_handle create_map(const ::std::string &name, const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single); /// \brief Create new or open existing key-value map. - inline map_handle - create_map(const ::mdbx::slice &name, - const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, - const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single); + inline map_handle create_map(const ::mdbx::slice &name, const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single); /// \brief Drops key-value map using handle. inline void drop_map(map_handle map); @@ -4486,8 +3870,7 @@ public: bool clear_map(const char *name, bool throw_if_absent = false); /// \return `True` if the key-value map existed and was cleared, either /// `false` if the key-value map did not exist and there is nothing to clear. - inline bool clear_map(const ::std::string &name, - bool throw_if_absent = false); + inline bool clear_map(const ::std::string &name, bool throw_if_absent = false); /// \return `True` if the key-value map existed and was cleared, either /// `false` if the key-value map did not exist and there is nothing to clear. bool clear_map(const ::mdbx::slice &name, bool throw_if_absent = false); @@ -4501,36 +3884,29 @@ public: /// \brief Переименовывает таблицу ключ-значение. /// \return `True` если таблица существует и была переименована, либо /// `false` в случае отсутствия исходной таблицы. - bool rename_map(const char *old_name, const char *new_name, - bool throw_if_absent = false); + bool rename_map(const char *old_name, const char *new_name, bool throw_if_absent = false); /// \brief Переименовывает таблицу ключ-значение. /// \return `True` если таблица существует и была переименована, либо /// `false` в случае отсутствия исходной таблицы. - bool rename_map(const ::std::string &old_name, const ::std::string &new_name, - bool throw_if_absent = false); + bool rename_map(const ::std::string &old_name, const ::std::string &new_name, bool throw_if_absent = false); /// \brief Переименовывает таблицу ключ-значение. /// \return `True` если таблица существует и была переименована, либо /// `false` в случае отсутствия исходной таблицы. - bool rename_map(const ::mdbx::slice &old_name, const ::mdbx::slice &new_name, - bool throw_if_absent = false); + bool rename_map(const ::mdbx::slice &old_name, const ::mdbx::slice &new_name, bool throw_if_absent = false); -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) /// \brief Open existing key-value map. - inline map_handle open_map( - const ::std::string_view &name, - const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, - const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const { + inline map_handle open_map(const ::std::string_view &name, const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const { return open_map(::mdbx::slice(name), key_mode, value_mode); } /// \brief Open existing key-value map. inline map_handle open_map_accede(const ::std::string_view &name) const; /// \brief Create new or open existing key-value map. - inline map_handle - create_map(const ::std::string_view &name, - const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, - const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) { + inline map_handle create_map(const ::std::string_view &name, + const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) { return create_map(::mdbx::slice(name), key_mode, value_mode); } /// \brief Drop key-value map. @@ -4549,11 +3925,9 @@ public: /// \brief Переименовывает таблицу ключ-значение. /// \return `True` если таблица существует и была переименована, либо /// `false` в случае отсутствия исходной таблицы. - bool rename_map(const ::std::string_view &old_name, - const ::std::string_view &new_name, + bool rename_map(const ::std::string_view &old_name, const ::std::string_view &new_name, bool throw_if_absent = false) { - return rename_map(::mdbx::slice(old_name), ::mdbx::slice(new_name), - throw_if_absent); + return rename_map(::mdbx::slice(old_name), ::mdbx::slice(new_name), throw_if_absent); } #endif /* __cpp_lib_string_view >= 201606L */ @@ -4583,20 +3957,16 @@ public: /// \brief Compare two keys according to a particular key-value map (aka /// table). - inline int compare_keys(map_handle map, const slice &a, - const slice &b) const noexcept; + inline int compare_keys(map_handle map, const slice &a, const slice &b) const noexcept; /// \brief Compare two values according to a particular key-value map (aka /// table). - inline int compare_values(map_handle map, const slice &a, - const slice &b) const noexcept; + inline int compare_values(map_handle map, const slice &a, const slice &b) const noexcept; /// \brief Compare keys of two pairs according to a particular key-value map /// (aka table). - inline int compare_keys(map_handle map, const pair &a, - const pair &b) const noexcept; + inline int compare_keys(map_handle map, const pair &a, const pair &b) const noexcept; /// \brief Compare values of two pairs according to a particular key-value map /// (aka table). - inline int compare_values(map_handle map, const pair &a, - const pair &b) const noexcept; + inline int compare_values(map_handle map, const pair &a, const pair &b) const noexcept; /// \brief Get value by key from a key-value map (aka table). inline slice get(map_handle map, const slice &key) const; @@ -4604,12 +3974,10 @@ public: /// multimap (aka table). inline slice get(map_handle map, slice key, size_t &values_count) const; /// \brief Get value by key from a key-value map (aka table). - inline slice get(map_handle map, const slice &key, - const slice &value_at_absence) const; + inline slice get(map_handle map, const slice &key, const slice &value_at_absence) const; /// \brief Get first of multi-value and values count by key from a key-value /// multimap (aka table). - inline slice get(map_handle map, slice key, size_t &values_count, - const slice &value_at_absence) const; + inline slice get(map_handle map, slice key, size_t &values_count, const slice &value_at_absence) const; /// \brief Get value for equal or great key from a table. /// \return Bundle of key-value pair and boolean flag, /// which will be `true` if the exact key was found and `false` otherwise. @@ -4617,42 +3985,27 @@ public: /// \brief Get value for equal or great key from a table. /// \return Bundle of key-value pair and boolean flag, /// which will be `true` if the exact key was found and `false` otherwise. - inline pair_result get_equal_or_great(map_handle map, const slice &key, - const slice &value_at_absence) const; + inline pair_result get_equal_or_great(map_handle map, const slice &key, const slice &value_at_absence) const; - inline MDBX_error_t put(map_handle map, const slice &key, slice *value, - MDBX_put_flags_t flags) noexcept; + inline MDBX_error_t put(map_handle map, const slice &key, slice *value, MDBX_put_flags_t flags) noexcept; inline void put(map_handle map, const slice &key, slice value, put_mode mode); inline void insert(map_handle map, const slice &key, slice value); inline value_result try_insert(map_handle map, const slice &key, slice value); - inline slice insert_reserve(map_handle map, const slice &key, - size_t value_length); - inline value_result try_insert_reserve(map_handle map, const slice &key, - size_t value_length); + inline slice insert_reserve(map_handle map, const slice &key, size_t value_length); + inline value_result try_insert_reserve(map_handle map, const slice &key, size_t value_length); inline void upsert(map_handle map, const slice &key, const slice &value); - inline slice upsert_reserve(map_handle map, const slice &key, - size_t value_length); + inline slice upsert_reserve(map_handle map, const slice &key, size_t value_length); inline void update(map_handle map, const slice &key, const slice &value); inline bool try_update(map_handle map, const slice &key, const slice &value); - inline slice update_reserve(map_handle map, const slice &key, - size_t value_length); - inline value_result try_update_reserve(map_handle map, const slice &key, - size_t value_length); + inline slice update_reserve(map_handle map, const slice &key, size_t value_length); + inline value_result try_update_reserve(map_handle map, const slice &key, size_t value_length); - void put(map_handle map, const pair &kv, put_mode mode) { - return put(map, kv.key, kv.value, mode); - } - void insert(map_handle map, const pair &kv) { - return insert(map, kv.key, kv.value); - } - value_result try_insert(map_handle map, const pair &kv) { - return try_insert(map, kv.key, kv.value); - } - void upsert(map_handle map, const pair &kv) { - return upsert(map, kv.key, kv.value); - } + void put(map_handle map, const pair &kv, put_mode mode) { return put(map, kv.key, kv.value, mode); } + void insert(map_handle map, const pair &kv) { return insert(map, kv.key, kv.value); } + value_result try_insert(map_handle map, const pair &kv) { return try_insert(map, kv.key, kv.value); } + void upsert(map_handle map, const pair &kv) { return upsert(map, kv.key, kv.value); } /// \brief Removes all values for given key. inline bool erase(map_handle map, const slice &key); @@ -4661,28 +4014,27 @@ public: inline bool erase(map_handle map, const slice &key, const slice &value); /// \brief Replaces the particular multi-value of the key with a new value. - inline void replace(map_handle map, const slice &key, slice old_value, - const slice &new_value); + inline void replace(map_handle map, const slice &key, slice old_value, const slice &new_value); /// \brief Removes and return a value of the key. template inline buffer extract(map_handle map, const slice &key, - const typename buffer::allocator_type & - allocator = buffer::allocator_type()); + const typename buffer::allocator_type &allocator = + buffer::allocator_type()); /// \brief Replaces and returns a value of the key with new one. template inline buffer replace(map_handle map, const slice &key, const slice &new_value, - const typename buffer::allocator_type & - allocator = buffer::allocator_type()); + const typename buffer::allocator_type &allocator = + buffer::allocator_type()); template - inline buffer replace_reserve( - map_handle map, const slice &key, slice &new_value, - const typename buffer::allocator_type - &allocator = buffer::allocator_type()); + inline buffer + replace_reserve(map_handle map, const slice &key, slice &new_value, + const typename buffer::allocator_type &allocator = + buffer::allocator_type()); /// \brief Adding a key-value pair, provided that ascending order of the keys /// and (optionally) values are preserved. @@ -4700,39 +4052,28 @@ public: /// \param [in] multivalue_order_preserved /// If `multivalue_order_preserved == true` then the same rules applied for /// to pages of nested b+tree of multimap's values. - inline void append(map_handle map, const slice &key, const slice &value, - bool multivalue_order_preserved = true); - inline void append(map_handle map, const pair &kv, - bool multivalue_order_preserved = true) { + inline void append(map_handle map, const slice &key, const slice &value, bool multivalue_order_preserved = true); + inline void append(map_handle map, const pair &kv, bool multivalue_order_preserved = true) { return append(map, kv.key, kv.value, multivalue_order_preserved); } - size_t put_multiple_samelength(map_handle map, const slice &key, - const size_t value_length, - const void *values_array, size_t values_count, - put_mode mode, bool allow_partial = false); + size_t put_multiple_samelength(map_handle map, const slice &key, const size_t value_length, const void *values_array, + size_t values_count, put_mode mode, bool allow_partial = false); template - size_t put_multiple_samelength(map_handle map, const slice &key, - const VALUE *values_array, size_t values_count, + size_t put_multiple_samelength(map_handle map, const slice &key, const VALUE *values_array, size_t values_count, put_mode mode, bool allow_partial = false) { - static_assert(::std::is_standard_layout::value && - !::std::is_pointer::value && + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value && !::std::is_array::value, "Must be a standard layout type!"); - return put_multiple_samelength(map, key, sizeof(VALUE), values_array, - values_count, mode, allow_partial); + return put_multiple_samelength(map, key, sizeof(VALUE), values_array, values_count, mode, allow_partial); } template - void put_multiple_samelength(map_handle map, const slice &key, - const ::std::vector &vector, - put_mode mode) { + void put_multiple_samelength(map_handle map, const slice &key, const ::std::vector &vector, put_mode mode) { put_multiple_samelength(map, key, vector.data(), vector.size(), mode); } - inline ptrdiff_t estimate(map_handle map, const pair &from, - const pair &to) const; - inline ptrdiff_t estimate(map_handle map, const slice &from, - const slice &to) const; + inline ptrdiff_t estimate(map_handle map, const pair &from, const pair &to) const; + inline ptrdiff_t estimate(map_handle map, const slice &from, const slice &to) const; inline ptrdiff_t estimate_from_first(map_handle map, const slice &to) const; inline ptrdiff_t estimate_to_last(map_handle map, const slice &from) const; }; @@ -4823,23 +4164,17 @@ public: MDBX_CXX14_CONSTEXPR operator bool() const noexcept; MDBX_CXX14_CONSTEXPR operator const MDBX_cursor *() const; MDBX_CXX14_CONSTEXPR operator MDBX_cursor *(); - friend MDBX_CXX11_CONSTEXPR bool operator==(const cursor &a, - const cursor &b) noexcept; - friend MDBX_CXX11_CONSTEXPR bool operator!=(const cursor &a, - const cursor &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator==(const cursor &a, const cursor &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator!=(const cursor &a, const cursor &b) noexcept; - friend inline int compare_position_nothrow(const cursor &left, - const cursor &right, - bool ignore_nested) noexcept; - friend inline int compare_position(const cursor &left, const cursor &right, - bool ignore_nested); + friend inline int compare_position_nothrow(const cursor &left, const cursor &right, bool ignore_nested) noexcept; + friend inline int compare_position(const cursor &left, const cursor &right, bool ignore_nested); bool is_before_than(const cursor &other, bool ignore_nested = false) const { return compare_position(*this, other, ignore_nested) < 0; } - bool is_same_or_before_than(const cursor &other, - bool ignore_nested = false) const { + bool is_same_or_before_than(const cursor &other, bool ignore_nested = false) const { return compare_position(*this, other, ignore_nested) <= 0; } @@ -4851,8 +4186,7 @@ public: return compare_position(*this, other, ignore_nested) > 0; } - bool is_same_or_after_than(const cursor &other, - bool ignore_nested = false) const { + bool is_same_or_after_than(const cursor &other, bool ignore_nested = false) const { return compare_position(*this, other, ignore_nested) >= 0; } @@ -4893,11 +4227,9 @@ public: /* Doubtless cursor positioning at a specified key-value pair * for dupsort/multi-value hives. */ multi_exactkey_value_lesser_than = MDBX_TO_EXACT_KEY_VALUE_LESSER_THAN, - multi_exactkey_value_lesser_or_equal = - MDBX_TO_EXACT_KEY_VALUE_LESSER_OR_EQUAL, + multi_exactkey_value_lesser_or_equal = MDBX_TO_EXACT_KEY_VALUE_LESSER_OR_EQUAL, multi_exactkey_value_equal = MDBX_TO_EXACT_KEY_VALUE_EQUAL, - multi_exactkey_value_greater_or_equal = - MDBX_TO_EXACT_KEY_VALUE_GREATER_OR_EQUAL, + multi_exactkey_value_greater_or_equal = MDBX_TO_EXACT_KEY_VALUE_GREATER_OR_EQUAL, multi_exactkey_value_greater = MDBX_TO_EXACT_KEY_VALUE_GREATER_THAN, pair_lesser_than = MDBX_TO_PAIR_LESSER_THAN, @@ -4915,14 +4247,10 @@ public: struct move_result : public pair_result { inline move_result(const cursor &cursor, bool throw_notfound); move_result(cursor &cursor, move_operation operation, bool throw_notfound) - : move_result(cursor, operation, slice::invalid(), slice::invalid(), - throw_notfound) {} - move_result(cursor &cursor, move_operation operation, const slice &key, - bool throw_notfound) - : move_result(cursor, operation, key, slice::invalid(), - throw_notfound) {} - inline move_result(cursor &cursor, move_operation operation, - const slice &key, const slice &value, + : move_result(cursor, operation, slice::invalid(), slice::invalid(), throw_notfound) {} + move_result(cursor &cursor, move_operation operation, const slice &key, bool throw_notfound) + : move_result(cursor, operation, key, slice::invalid(), throw_notfound) {} + inline move_result(cursor &cursor, move_operation operation, const slice &key, const slice &value, bool throw_notfound); move_result(const move_result &) noexcept = default; move_result &operator=(const move_result &) noexcept = default; @@ -4931,38 +4259,30 @@ public: struct estimate_result : public pair { ptrdiff_t approximate_quantity; estimate_result(const cursor &cursor, move_operation operation) - : estimate_result(cursor, operation, slice::invalid(), - slice::invalid()) {} - estimate_result(const cursor &cursor, move_operation operation, - const slice &key) + : estimate_result(cursor, operation, slice::invalid(), slice::invalid()) {} + estimate_result(const cursor &cursor, move_operation operation, const slice &key) : estimate_result(cursor, operation, key, slice::invalid()) {} - inline estimate_result(const cursor &cursor, move_operation operation, - const slice &key, const slice &value); + inline estimate_result(const cursor &cursor, move_operation operation, const slice &key, const slice &value); estimate_result(const estimate_result &) noexcept = default; estimate_result &operator=(const estimate_result &) noexcept = default; }; protected: /* fake const, i.e. for some move/get operations */ - inline bool move(move_operation operation, MDBX_val *key, MDBX_val *value, - bool throw_notfound) const; + inline bool move(move_operation operation, MDBX_val *key, MDBX_val *value, bool throw_notfound) const; - inline ptrdiff_t estimate(move_operation operation, MDBX_val *key, - MDBX_val *value) const; + inline ptrdiff_t estimate(move_operation operation, MDBX_val *key, MDBX_val *value) const; public: template - bool scan(CALLABLE_PREDICATE predicate, move_operation start = first, - move_operation turn = next) { + bool scan(CALLABLE_PREDICATE predicate, move_operation start = first, move_operation turn = next) { struct wrapper : public exception_thunk { - static int probe(void *context, MDBX_val *key, MDBX_val *value, - void *arg) noexcept { + static int probe(void *context, MDBX_val *key, MDBX_val *value, void *arg) noexcept { auto thunk = static_cast(context); assert(thunk->is_clean()); auto &predicate = *static_cast(arg); try { - return predicate(pair(*key, *value)) ? MDBX_RESULT_TRUE - : MDBX_RESULT_FALSE; + return predicate(pair(*key, *value)) ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE; } catch (... /* capture any exception to rethrow it over C code */) { thunk->capture(); return MDBX_RESULT_TRUE; @@ -4970,92 +4290,71 @@ public: } } thunk; return error::boolean_or_throw( - ::mdbx_cursor_scan(handle_, wrapper::probe, &thunk, - MDBX_cursor_op(start), MDBX_cursor_op(turn), - &predicate), + ::mdbx_cursor_scan(handle_, wrapper::probe, &thunk, MDBX_cursor_op(start), MDBX_cursor_op(turn), &predicate), thunk); } - template - bool fullscan(CALLABLE_PREDICATE predicate, bool backward = false) { - return scan(std::move(predicate), backward ? last : first, - backward ? previous : next); + template bool fullscan(CALLABLE_PREDICATE predicate, bool backward = false) { + return scan(std::move(predicate), backward ? last : first, backward ? previous : next); } template - bool scan_from(CALLABLE_PREDICATE predicate, slice &from, - move_operation start = key_greater_or_equal, + bool scan_from(CALLABLE_PREDICATE predicate, slice &from, move_operation start = key_greater_or_equal, move_operation turn = next) { struct wrapper : public exception_thunk { - static int probe(void *context, MDBX_val *key, MDBX_val *value, - void *arg) noexcept { + static int probe(void *context, MDBX_val *key, MDBX_val *value, void *arg) noexcept { auto thunk = static_cast(context); assert(thunk->is_clean()); auto &predicate = *static_cast(arg); try { - return predicate(pair(*key, *value)) ? MDBX_RESULT_TRUE - : MDBX_RESULT_FALSE; + return predicate(pair(*key, *value)) ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE; } catch (... /* capture any exception to rethrow it over C code */) { thunk->capture(); return MDBX_RESULT_TRUE; } } } thunk; - return error::boolean_or_throw( - ::mdbx_cursor_scan_from(handle_, wrapper::probe, &thunk, - MDBX_cursor_op(start), &from, nullptr, - MDBX_cursor_op(turn), &predicate), - thunk); + return error::boolean_or_throw(::mdbx_cursor_scan_from(handle_, wrapper::probe, &thunk, MDBX_cursor_op(start), + &from, nullptr, MDBX_cursor_op(turn), &predicate), + thunk); } template - bool scan_from(CALLABLE_PREDICATE predicate, pair &from, - move_operation start = pair_greater_or_equal, + bool scan_from(CALLABLE_PREDICATE predicate, pair &from, move_operation start = pair_greater_or_equal, move_operation turn = next) { struct wrapper : public exception_thunk { - static int probe(void *context, MDBX_val *key, MDBX_val *value, - void *arg) noexcept { + static int probe(void *context, MDBX_val *key, MDBX_val *value, void *arg) noexcept { auto thunk = static_cast(context); assert(thunk->is_clean()); auto &predicate = *static_cast(arg); try { - return predicate(pair(*key, *value)) ? MDBX_RESULT_TRUE - : MDBX_RESULT_FALSE; + return predicate(pair(*key, *value)) ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE; } catch (... /* capture any exception to rethrow it over C code */) { thunk->capture(); return MDBX_RESULT_TRUE; } } } thunk; - return error::boolean_or_throw( - ::mdbx_cursor_scan_from(handle_, wrapper::probe, &thunk, - MDBX_cursor_op(start), &from.key, &from.value, - MDBX_cursor_op(turn), &predicate), - thunk); + return error::boolean_or_throw(::mdbx_cursor_scan_from(handle_, wrapper::probe, &thunk, MDBX_cursor_op(start), + &from.key, &from.value, MDBX_cursor_op(turn), &predicate), + thunk); } move_result move(move_operation operation, bool throw_notfound) { return move_result(*this, operation, throw_notfound); } - move_result move(move_operation operation, const slice &key, - bool throw_notfound) { + move_result move(move_operation operation, const slice &key, bool throw_notfound) { return move_result(*this, operation, key, slice::invalid(), throw_notfound); } - move_result move(move_operation operation, const slice &key, - const slice &value, bool throw_notfound) { + move_result move(move_operation operation, const slice &key, const slice &value, bool throw_notfound) { return move_result(*this, operation, key, value, throw_notfound); } - bool move(move_operation operation, slice &key, slice &value, - bool throw_notfound) { + bool move(move_operation operation, slice &key, slice &value, bool throw_notfound) { return move(operation, &key, &value, throw_notfound); } - move_result to_first(bool throw_notfound = true) { - return move(first, throw_notfound); - } - move_result to_previous(bool throw_notfound = true) { - return move(previous, throw_notfound); - } + move_result to_first(bool throw_notfound = true) { return move(first, throw_notfound); } + move_result to_previous(bool throw_notfound = true) { return move(previous, throw_notfound); } move_result to_previous_last_multi(bool throw_notfound = true) { return move(multi_prevkey_lastvalue, throw_notfound); } @@ -5065,30 +4364,21 @@ public: move_result to_current_prev_multi(bool throw_notfound = true) { return move(multi_currentkey_prevvalue, throw_notfound); } - move_result current(bool throw_notfound = true) const { - return move_result(*this, throw_notfound); - } + move_result current(bool throw_notfound = true) const { return move_result(*this, throw_notfound); } move_result to_current_next_multi(bool throw_notfound = true) { return move(multi_currentkey_nextvalue, throw_notfound); } move_result to_current_last_multi(bool throw_notfound = true) { return move(multi_currentkey_lastvalue, throw_notfound); } - move_result to_next_first_multi(bool throw_notfound = true) { - return move(multi_nextkey_firstvalue, throw_notfound); - } - move_result to_next(bool throw_notfound = true) { - return move(next, throw_notfound); - } - move_result to_last(bool throw_notfound = true) { - return move(last, throw_notfound); - } + move_result to_next_first_multi(bool throw_notfound = true) { return move(multi_nextkey_firstvalue, throw_notfound); } + move_result to_next(bool throw_notfound = true) { return move(next, throw_notfound); } + move_result to_last(bool throw_notfound = true) { return move(last, throw_notfound); } move_result to_key_lesser_than(const slice &key, bool throw_notfound = true) { return move(key_lesser_than, key, throw_notfound); } - move_result to_key_lesser_or_equal(const slice &key, - bool throw_notfound = true) { + move_result to_key_lesser_or_equal(const slice &key, bool throw_notfound = true) { return move(key_lesser_or_equal, key, throw_notfound); } move_result to_key_equal(const slice &key, bool throw_notfound = true) { @@ -5097,64 +4387,45 @@ public: move_result to_key_exact(const slice &key, bool throw_notfound = true) { return move(key_exact, key, throw_notfound); } - move_result to_key_greater_or_equal(const slice &key, - bool throw_notfound = true) { + move_result to_key_greater_or_equal(const slice &key, bool throw_notfound = true) { return move(key_greater_or_equal, key, throw_notfound); } - move_result to_key_greater_than(const slice &key, - bool throw_notfound = true) { + move_result to_key_greater_than(const slice &key, bool throw_notfound = true) { return move(key_greater_than, key, throw_notfound); } - move_result to_exact_key_value_lesser_than(const slice &key, - const slice &value, - bool throw_notfound = true) { + move_result to_exact_key_value_lesser_than(const slice &key, const slice &value, bool throw_notfound = true) { return move(multi_exactkey_value_lesser_than, key, value, throw_notfound); } - move_result to_exact_key_value_lesser_or_equal(const slice &key, - const slice &value, - bool throw_notfound = true) { - return move(multi_exactkey_value_lesser_or_equal, key, value, - throw_notfound); + move_result to_exact_key_value_lesser_or_equal(const slice &key, const slice &value, bool throw_notfound = true) { + return move(multi_exactkey_value_lesser_or_equal, key, value, throw_notfound); } - move_result to_exact_key_value_equal(const slice &key, const slice &value, - bool throw_notfound = true) { + move_result to_exact_key_value_equal(const slice &key, const slice &value, bool throw_notfound = true) { return move(multi_exactkey_value_equal, key, value, throw_notfound); } - move_result to_exact_key_value_greater_or_equal(const slice &key, - const slice &value, - bool throw_notfound = true) { - return move(multi_exactkey_value_greater_or_equal, key, value, - throw_notfound); + move_result to_exact_key_value_greater_or_equal(const slice &key, const slice &value, bool throw_notfound = true) { + return move(multi_exactkey_value_greater_or_equal, key, value, throw_notfound); } - move_result to_exact_key_value_greater_than(const slice &key, - const slice &value, - bool throw_notfound = true) { + move_result to_exact_key_value_greater_than(const slice &key, const slice &value, bool throw_notfound = true) { return move(multi_exactkey_value_greater, key, value, throw_notfound); } - move_result to_pair_lesser_than(const slice &key, const slice &value, - bool throw_notfound = true) { + move_result to_pair_lesser_than(const slice &key, const slice &value, bool throw_notfound = true) { return move(pair_lesser_than, key, value, throw_notfound); } - move_result to_pair_lesser_or_equal(const slice &key, const slice &value, - bool throw_notfound = true) { + move_result to_pair_lesser_or_equal(const slice &key, const slice &value, bool throw_notfound = true) { return move(pair_lesser_or_equal, key, value, throw_notfound); } - move_result to_pair_equal(const slice &key, const slice &value, - bool throw_notfound = true) { + move_result to_pair_equal(const slice &key, const slice &value, bool throw_notfound = true) { return move(pair_equal, key, value, throw_notfound); } - move_result to_pair_exact(const slice &key, const slice &value, - bool throw_notfound = true) { + move_result to_pair_exact(const slice &key, const slice &value, bool throw_notfound = true) { return move(pair_exact, key, value, throw_notfound); } - move_result to_pair_greater_or_equal(const slice &key, const slice &value, - bool throw_notfound = true) { + move_result to_pair_greater_or_equal(const slice &key, const slice &value, bool throw_notfound = true) { return move(pair_greater_or_equal, key, value, throw_notfound); } - move_result to_pair_greater_than(const slice &key, const slice &value, - bool throw_notfound = true) { + move_result to_pair_greater_than(const slice &key, const slice &value, bool throw_notfound = true) { return move(pair_greater_than, key, value, throw_notfound); } @@ -5166,17 +4437,11 @@ public: /// \brief Return count of duplicates for current key. inline size_t count_multivalue() const; - inline move_result find_multivalue(const slice &key, const slice &value, - bool throw_notfound = true); - inline move_result lower_bound_multivalue(const slice &key, - const slice &value, - bool throw_notfound = false); - inline move_result upper_bound_multivalue(const slice &key, - const slice &value, - bool throw_notfound = false); + inline move_result find_multivalue(const slice &key, const slice &value, bool throw_notfound = true); + inline move_result lower_bound_multivalue(const slice &key, const slice &value, bool throw_notfound = false); + inline move_result upper_bound_multivalue(const slice &key, const slice &value, bool throw_notfound = false); - inline move_result get_multiple_samelength(const slice &key, - bool throw_notfound = true) { + inline move_result get_multiple_samelength(const slice &key, bool throw_notfound = true) { return move(batch_samelength, key, throw_notfound); } @@ -5222,8 +4487,7 @@ public: inline operator ::mdbx::txn() const { return txn(); } inline operator ::mdbx::map_handle() const { return map(); } - inline MDBX_error_t put(const slice &key, slice *value, - MDBX_put_flags_t flags) noexcept; + inline MDBX_error_t put(const slice &key, slice *value, MDBX_put_flags_t flags) noexcept; inline void put(const slice &key, slice value, put_mode mode); inline void insert(const slice &key, slice value); inline value_result try_insert(const slice &key, slice value); @@ -5238,13 +4502,9 @@ public: inline slice update_reserve(const slice &key, size_t value_length); inline value_result try_update_reserve(const slice &key, size_t value_length); - void put(const pair &kv, put_mode mode) { - return put(kv.key, kv.value, mode); - } + void put(const pair &kv, put_mode mode) { return put(kv.key, kv.value, mode); } void insert(const pair &kv) { return insert(kv.key, kv.value); } - value_result try_insert(const pair &kv) { - return try_insert(kv.key, kv.value); - } + value_result try_insert(const pair &kv) { return try_insert(kv.key, kv.value); } void upsert(const pair &kv) { return upsert(kv.key, kv.value); } /// \brief Removes single key-value pair or all multi-values at the current @@ -5272,13 +4532,11 @@ class LIBMDBX_API_TYPE cursor_managed : public cursor { using inherited = cursor; friend class txn; /// delegated constructor for RAII - MDBX_CXX11_CONSTEXPR cursor_managed(MDBX_cursor *ptr) noexcept - : inherited(ptr) {} + MDBX_CXX11_CONSTEXPR cursor_managed(MDBX_cursor *ptr) noexcept : inherited(ptr) {} public: /// \brief Creates a new managed cursor with underlying object. - cursor_managed(void *your_context = nullptr) - : cursor_managed(::mdbx_cursor_create(your_context)) { + cursor_managed(void *your_context = nullptr) : cursor_managed(::mdbx_cursor_create(your_context)) { if (MDBX_UNLIKELY(!handle_)) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_ENOMEM); } @@ -5308,53 +4566,33 @@ LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const slice &); LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const pair &); LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const pair_result &); template -inline ::std::ostream & -operator<<(::std::ostream &out, const buffer &it) { - return (it.is_freestanding() - ? out << "buf-" << it.headroom() << "." << it.tailroom() - : out << "ref-") - << it.slice(); +inline ::std::ostream &operator<<(::std::ostream &out, const buffer &it) { + return (it.is_freestanding() ? out << "buf-" << it.headroom() << "." << it.tailroom() : out << "ref-") << it.slice(); } -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const env::geometry::size &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::geometry::size &); LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::geometry &); -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const env::operate_parameters &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::operate_parameters &); LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::mode &); -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const env::durability &); -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const env::reclaiming_options &); -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const env::operate_options &); -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const env_managed::create_parameters &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::durability &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::reclaiming_options &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::operate_options &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env_managed::create_parameters &); -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const MDBX_log_level_t &); -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const MDBX_debug_flags_t &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const MDBX_log_level_t &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const MDBX_debug_flags_t &); LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const error &); -inline ::std::ostream &operator<<(::std::ostream &out, - const MDBX_error_t &errcode) { - return out << error(errcode); -} +inline ::std::ostream &operator<<(::std::ostream &out, const MDBX_error_t &errcode) { return out << error(errcode); } //============================================================================== // // Inline body of the libmdbx C++ API // -MDBX_CXX11_CONSTEXPR const version_info &get_version() noexcept { - return ::mdbx_version; -} -MDBX_CXX11_CONSTEXPR const build_info &get_build() noexcept { - return ::mdbx_build; -} +MDBX_CXX11_CONSTEXPR const version_info &get_version() noexcept { return ::mdbx_version; } +MDBX_CXX11_CONSTEXPR const build_info &get_build() noexcept { return ::mdbx_build; } static MDBX_CXX17_CONSTEXPR size_t strlen(const char *c_str) noexcept { -#if defined(__cpp_lib_is_constant_evaluated) && \ - __cpp_lib_is_constant_evaluated >= 201811L +#if defined(__cpp_lib_is_constant_evaluated) && __cpp_lib_is_constant_evaluated >= 201811L if (::std::is_constant_evaluated()) { for (size_t i = 0; c_str; ++i) if (!c_str[i]) @@ -5369,10 +4607,8 @@ static MDBX_CXX17_CONSTEXPR size_t strlen(const char *c_str) noexcept { #endif } -MDBX_MAYBE_UNUSED static MDBX_CXX20_CONSTEXPR void * -memcpy(void *dest, const void *src, size_t bytes) noexcept { -#if defined(__cpp_lib_is_constant_evaluated) && \ - __cpp_lib_is_constant_evaluated >= 201811L +MDBX_MAYBE_UNUSED static MDBX_CXX20_CONSTEXPR void *memcpy(void *dest, const void *src, size_t bytes) noexcept { +#if defined(__cpp_lib_is_constant_evaluated) && __cpp_lib_is_constant_evaluated >= 201811L if (::std::is_constant_evaluated()) { for (size_t i = 0; i < bytes; ++i) static_cast(dest)[i] = static_cast(src)[i]; @@ -5382,14 +4618,11 @@ memcpy(void *dest, const void *src, size_t bytes) noexcept { return ::std::memcpy(dest, src, bytes); } -static MDBX_CXX20_CONSTEXPR int memcmp(const void *a, const void *b, - size_t bytes) noexcept { -#if defined(__cpp_lib_is_constant_evaluated) && \ - __cpp_lib_is_constant_evaluated >= 201811L +static MDBX_CXX20_CONSTEXPR int memcmp(const void *a, const void *b, size_t bytes) noexcept { +#if defined(__cpp_lib_is_constant_evaluated) && __cpp_lib_is_constant_evaluated >= 201811L if (::std::is_constant_evaluated()) { for (size_t i = 0; i < bytes; ++i) { - const int diff = int(static_cast(a)[i]) - - int(static_cast(b)[i]); + const int diff = int(static_cast(a)[i]) - int(static_cast(b)[i]); if (diff) return diff; } @@ -5405,13 +4638,11 @@ static MDBX_CXX14_CONSTEXPR size_t check_length(size_t bytes) { return bytes; } -static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, - size_t payload) { +static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, size_t payload) { return check_length(check_length(headroom) + check_length(payload)); } -MDBX_MAYBE_UNUSED static MDBX_CXX14_CONSTEXPR size_t -check_length(size_t headroom, size_t payload, size_t tailroom) { +MDBX_MAYBE_UNUSED static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, size_t payload, size_t tailroom) { return check_length(check_length(headroom, payload) + check_length(tailroom)); } @@ -5429,33 +4660,22 @@ inline void exception_thunk::rethrow_captured() const { //------------------------------------------------------------------------------ -MDBX_CXX11_CONSTEXPR error::error(MDBX_error_t error_code) noexcept - : code_(error_code) {} +MDBX_CXX11_CONSTEXPR error::error(MDBX_error_t error_code) noexcept : code_(error_code) {} inline error &error::operator=(MDBX_error_t error_code) noexcept { code_ = error_code; return *this; } -MDBX_CXX11_CONSTEXPR bool operator==(const error &a, const error &b) noexcept { - return a.code_ == b.code_; -} +MDBX_CXX11_CONSTEXPR bool operator==(const error &a, const error &b) noexcept { return a.code_ == b.code_; } -MDBX_CXX11_CONSTEXPR bool operator!=(const error &a, const error &b) noexcept { - return !(a == b); -} +MDBX_CXX11_CONSTEXPR bool operator!=(const error &a, const error &b) noexcept { return !(a == b); } -MDBX_CXX11_CONSTEXPR bool error::is_success() const noexcept { - return code_ == MDBX_SUCCESS; -} +MDBX_CXX11_CONSTEXPR bool error::is_success() const noexcept { return code_ == MDBX_SUCCESS; } -MDBX_CXX11_CONSTEXPR bool error::is_result_true() const noexcept { - return code_ == MDBX_RESULT_FALSE; -} +MDBX_CXX11_CONSTEXPR bool error::is_result_true() const noexcept { return code_ == MDBX_RESULT_FALSE; } -MDBX_CXX11_CONSTEXPR bool error::is_result_false() const noexcept { - return code_ == MDBX_RESULT_TRUE; -} +MDBX_CXX11_CONSTEXPR bool error::is_result_false() const noexcept { return code_ == MDBX_RESULT_TRUE; } MDBX_CXX11_CONSTEXPR bool error::is_failure() const noexcept { return code_ != MDBX_SUCCESS && code_ != MDBX_RESULT_TRUE; @@ -5464,10 +4684,8 @@ MDBX_CXX11_CONSTEXPR bool error::is_failure() const noexcept { MDBX_CXX11_CONSTEXPR MDBX_error_t error::code() const noexcept { return code_; } MDBX_CXX11_CONSTEXPR bool error::is_mdbx_error() const noexcept { - return (code() >= MDBX_FIRST_LMDB_ERRCODE && - code() <= MDBX_LAST_LMDB_ERRCODE) || - (code() >= MDBX_FIRST_ADDED_ERRCODE && - code() <= MDBX_LAST_ADDED_ERRCODE); + return (code() >= MDBX_FIRST_LMDB_ERRCODE && code() <= MDBX_LAST_LMDB_ERRCODE) || + (code() >= MDBX_FIRST_ADDED_ERRCODE && code() <= MDBX_LAST_ADDED_ERRCODE); } inline void error::throw_exception(int error_code) { @@ -5488,20 +4706,17 @@ inline void error::success_or_throw() const { inline void error::success_or_throw(const exception_thunk &thunk) const { assert(thunk.is_clean() || code() != MDBX_SUCCESS); if (MDBX_UNLIKELY(!is_success())) { - MDBX_CXX20_UNLIKELY if (MDBX_UNLIKELY(!thunk.is_clean())) - thunk.rethrow_captured(); + MDBX_CXX20_UNLIKELY if (MDBX_UNLIKELY(!thunk.is_clean())) thunk.rethrow_captured(); else throw_exception(); } } -inline void error::panic_on_failure(const char *context_where, - const char *func_who) const noexcept { +inline void error::panic_on_failure(const char *context_where, const char *func_who) const noexcept { if (MDBX_UNLIKELY(is_failure())) MDBX_CXX20_UNLIKELY panic(context_where, func_who); } -inline void error::success_or_panic(const char *context_where, - const char *func_who) const noexcept { +inline void error::success_or_panic(const char *context_where, const char *func_who) const noexcept { if (MDBX_UNLIKELY(!is_success())) MDBX_CXX20_UNLIKELY panic(context_where, func_who); } @@ -5532,26 +4747,22 @@ inline bool error::boolean_or_throw(int error_code) { } } -inline void error::success_or_throw(int error_code, - const exception_thunk &thunk) { +inline void error::success_or_throw(int error_code, const exception_thunk &thunk) { error rc(static_cast(error_code)); rc.success_or_throw(thunk); } -inline void error::panic_on_failure(int error_code, const char *context_where, - const char *func_who) noexcept { +inline void error::panic_on_failure(int error_code, const char *context_where, const char *func_who) noexcept { error rc(static_cast(error_code)); rc.panic_on_failure(context_where, func_who); } -inline void error::success_or_panic(int error_code, const char *context_where, - const char *func_who) noexcept { +inline void error::success_or_panic(int error_code, const char *context_where, const char *func_who) noexcept { error rc(static_cast(error_code)); rc.success_or_panic(context_where, func_who); } -inline bool error::boolean_or_throw(int error_code, - const exception_thunk &thunk) { +inline bool error::boolean_or_throw(int error_code, const exception_thunk &thunk) { if (MDBX_UNLIKELY(!thunk.is_clean())) MDBX_CXX20_UNLIKELY thunk.rethrow_captured(); return boolean_or_throw(error_code); @@ -5565,22 +4776,15 @@ MDBX_CXX14_CONSTEXPR slice::slice(const void *ptr, size_t bytes) : ::MDBX_val({const_cast(ptr), check_length(bytes)}) {} MDBX_CXX14_CONSTEXPR slice::slice(const void *begin, const void *end) - : slice(begin, static_cast(end) - - static_cast(begin)) {} + : slice(begin, static_cast(end) - static_cast(begin)) {} -MDBX_CXX17_CONSTEXPR slice::slice(const char *c_str) - : slice(c_str, ::mdbx::strlen(c_str)) {} +MDBX_CXX17_CONSTEXPR slice::slice(const char *c_str) : slice(c_str, ::mdbx::strlen(c_str)) {} -MDBX_CXX14_CONSTEXPR slice::slice(const MDBX_val &src) - : slice(src.iov_base, src.iov_len) {} +MDBX_CXX14_CONSTEXPR slice::slice(const MDBX_val &src) : slice(src.iov_base, src.iov_len) {} -MDBX_CXX14_CONSTEXPR slice::slice(MDBX_val &&src) : slice(src) { - src.iov_base = nullptr; -} +MDBX_CXX14_CONSTEXPR slice::slice(MDBX_val &&src) : slice(src) { src.iov_base = nullptr; } -MDBX_CXX14_CONSTEXPR slice::slice(slice &&src) noexcept : slice(src) { - src.invalidate(); -} +MDBX_CXX14_CONSTEXPR slice::slice(slice &&src) noexcept : slice(src) { src.invalidate(); } inline slice &slice::assign(const void *ptr, size_t bytes) { iov_base = const_cast(ptr); @@ -5594,9 +4798,7 @@ inline slice &slice::assign(const slice &src) noexcept { return *this; } -inline slice &slice::assign(const ::MDBX_val &src) { - return assign(src.iov_base, src.iov_len); -} +inline slice &slice::assign(const ::MDBX_val &src) { return assign(src.iov_base, src.iov_len); } slice &slice::assign(slice &&src) noexcept { assign(src); @@ -5611,21 +4813,14 @@ inline slice &slice::assign(::MDBX_val &&src) { } inline slice &slice::assign(const void *begin, const void *end) { - return assign(begin, static_cast(end) - - static_cast(begin)); + return assign(begin, static_cast(end) - static_cast(begin)); } -inline slice &slice::assign(const char *c_str) { - return assign(c_str, ::mdbx::strlen(c_str)); -} +inline slice &slice::assign(const char *c_str) { return assign(c_str, ::mdbx::strlen(c_str)); } -inline slice &slice::operator=(slice &&src) noexcept { - return assign(::std::move(src)); -} +inline slice &slice::operator=(slice &&src) noexcept { return assign(::std::move(src)); } -inline slice &slice::operator=(::MDBX_val &&src) { - return assign(::std::move(src)); -} +inline slice &slice::operator=(::MDBX_val &&src) { return assign(::std::move(src)); } inline void slice::swap(slice &other) noexcept { const auto temp = *this; @@ -5637,47 +4832,27 @@ MDBX_CXX11_CONSTEXPR const ::mdbx::byte *slice::byte_ptr() const noexcept { return static_cast(iov_base); } -MDBX_CXX11_CONSTEXPR const ::mdbx::byte *slice::end_byte_ptr() const noexcept { - return byte_ptr() + length(); -} +MDBX_CXX11_CONSTEXPR const ::mdbx::byte *slice::end_byte_ptr() const noexcept { return byte_ptr() + length(); } -MDBX_CXX11_CONSTEXPR ::mdbx::byte *slice::byte_ptr() noexcept { - return static_cast(iov_base); -} +MDBX_CXX11_CONSTEXPR ::mdbx::byte *slice::byte_ptr() noexcept { return static_cast(iov_base); } -MDBX_CXX11_CONSTEXPR ::mdbx::byte *slice::end_byte_ptr() noexcept { - return byte_ptr() + length(); -} +MDBX_CXX11_CONSTEXPR ::mdbx::byte *slice::end_byte_ptr() noexcept { return byte_ptr() + length(); } -MDBX_CXX11_CONSTEXPR const char *slice::char_ptr() const noexcept { - return static_cast(iov_base); -} +MDBX_CXX11_CONSTEXPR const char *slice::char_ptr() const noexcept { return static_cast(iov_base); } -MDBX_CXX11_CONSTEXPR const char *slice::end_char_ptr() const noexcept { - return char_ptr() + length(); -} +MDBX_CXX11_CONSTEXPR const char *slice::end_char_ptr() const noexcept { return char_ptr() + length(); } -MDBX_CXX11_CONSTEXPR char *slice::char_ptr() noexcept { - return static_cast(iov_base); -} +MDBX_CXX11_CONSTEXPR char *slice::char_ptr() noexcept { return static_cast(iov_base); } -MDBX_CXX11_CONSTEXPR char *slice::end_char_ptr() noexcept { - return char_ptr() + length(); -} +MDBX_CXX11_CONSTEXPR char *slice::end_char_ptr() noexcept { return char_ptr() + length(); } -MDBX_CXX11_CONSTEXPR const void *slice::data() const noexcept { - return iov_base; -} +MDBX_CXX11_CONSTEXPR const void *slice::data() const noexcept { return iov_base; } -MDBX_CXX11_CONSTEXPR const void *slice::end() const noexcept { - return static_cast(end_byte_ptr()); -} +MDBX_CXX11_CONSTEXPR const void *slice::end() const noexcept { return static_cast(end_byte_ptr()); } MDBX_CXX11_CONSTEXPR void *slice::data() noexcept { return iov_base; } -MDBX_CXX11_CONSTEXPR void *slice::end() noexcept { - return static_cast(end_byte_ptr()); -} +MDBX_CXX11_CONSTEXPR void *slice::end() noexcept { return static_cast(end_byte_ptr()); } MDBX_CXX11_CONSTEXPR size_t slice::length() const noexcept { return iov_len; } @@ -5691,19 +4866,13 @@ MDBX_CXX14_CONSTEXPR slice &slice::set_end(const void *ptr) { return set_length(static_cast(ptr) - char_ptr()); } -MDBX_CXX11_CONSTEXPR bool slice::empty() const noexcept { - return length() == 0; -} +MDBX_CXX11_CONSTEXPR bool slice::empty() const noexcept { return length() == 0; } -MDBX_CXX11_CONSTEXPR bool slice::is_null() const noexcept { - return data() == nullptr; -} +MDBX_CXX11_CONSTEXPR bool slice::is_null() const noexcept { return data() == nullptr; } MDBX_CXX11_CONSTEXPR size_t slice::size() const noexcept { return length(); } -MDBX_CXX11_CONSTEXPR slice::operator bool() const noexcept { - return !is_null(); -} +MDBX_CXX11_CONSTEXPR slice::operator bool() const noexcept { return !is_null(); } MDBX_CXX14_CONSTEXPR void slice::invalidate() noexcept { iov_base = nullptr; } @@ -5735,20 +4904,16 @@ inline void slice::safe_remove_suffix(size_t n) { remove_suffix(n); } -MDBX_CXX14_CONSTEXPR bool -slice::starts_with(const slice &prefix) const noexcept { - return length() >= prefix.length() && - memcmp(data(), prefix.data(), prefix.length()) == 0; +MDBX_CXX14_CONSTEXPR bool slice::starts_with(const slice &prefix) const noexcept { + return length() >= prefix.length() && memcmp(data(), prefix.data(), prefix.length()) == 0; } MDBX_CXX14_CONSTEXPR bool slice::ends_with(const slice &suffix) const noexcept { return length() >= suffix.length() && - memcmp(byte_ptr() + length() - suffix.length(), suffix.data(), - suffix.length()) == 0; + memcmp(byte_ptr() + length() - suffix.length(), suffix.data(), suffix.length()) == 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t -slice::hash_value() const noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t slice::hash_value() const noexcept { size_t h = length() * 3977471; for (size_t i = 0; i < length(); ++i) h = (h ^ static_cast(data())[i]) * 1664525 + 1013904223; @@ -5801,17 +4966,14 @@ MDBX_CXX14_CONSTEXPR slice slice::safe_middle(size_t from, size_t n) const { return middle(from, n); } -MDBX_CXX14_CONSTEXPR intptr_t slice::compare_fast(const slice &a, - const slice &b) noexcept { +MDBX_CXX14_CONSTEXPR intptr_t slice::compare_fast(const slice &a, const slice &b) noexcept { const intptr_t diff = intptr_t(a.length()) - intptr_t(b.length()); - return diff ? diff - : MDBX_UNLIKELY(a.length() == 0 || a.data() == b.data()) - ? 0 - : memcmp(a.data(), b.data(), a.length()); + return diff ? diff + : MDBX_UNLIKELY(a.length() == 0 || a.data() == b.data()) ? 0 + : memcmp(a.data(), b.data(), a.length()); } -MDBX_CXX14_CONSTEXPR intptr_t -slice::compare_lexicographically(const slice &a, const slice &b) noexcept { +MDBX_CXX14_CONSTEXPR intptr_t slice::compare_lexicographically(const slice &a, const slice &b) noexcept { const size_t shortest = ::std::min(a.length(), b.length()); if (MDBX_LIKELY(shortest > 0)) MDBX_CXX20_LIKELY { @@ -5822,159 +4984,124 @@ slice::compare_lexicographically(const slice &a, const slice &b) noexcept { return intptr_t(a.length()) - intptr_t(b.length()); } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator==(const slice &a, const slice &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator==(const slice &a, const slice &b) noexcept { return slice::compare_fast(a, b) == 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator<(const slice &a, const slice &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator<(const slice &a, const slice &b) noexcept { return slice::compare_lexicographically(a, b) < 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator>(const slice &a, const slice &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator>(const slice &a, const slice &b) noexcept { return slice::compare_lexicographically(a, b) > 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator<=(const slice &a, const slice &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator<=(const slice &a, const slice &b) noexcept { return slice::compare_lexicographically(a, b) <= 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator>=(const slice &a, const slice &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator>=(const slice &a, const slice &b) noexcept { return slice::compare_lexicographically(a, b) >= 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator!=(const slice &a, const slice &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator!=(const slice &a, const slice &b) noexcept { return slice::compare_fast(a, b) != 0; } template -inline string -slice::as_hex_string(bool uppercase, unsigned wrap_width, - const ALLOCATOR &allocator) const { +inline string slice::as_hex_string(bool uppercase, unsigned wrap_width, const ALLOCATOR &allocator) const { return to_hex(*this, uppercase, wrap_width).as_string(allocator); } template -inline string -slice::as_base58_string(unsigned wrap_width, const ALLOCATOR &allocator) const { +inline string slice::as_base58_string(unsigned wrap_width, const ALLOCATOR &allocator) const { return to_base58(*this, wrap_width).as_string(allocator); } template -inline string -slice::as_base64_string(unsigned wrap_width, const ALLOCATOR &allocator) const { +inline string slice::as_base64_string(unsigned wrap_width, const ALLOCATOR &allocator) const { return to_base64(*this, wrap_width).as_string(allocator); } template -inline buffer -slice::encode_hex(bool uppercase, unsigned wrap_width, - const ALLOCATOR &allocator) const { - return to_hex(*this, uppercase, wrap_width) - .as_buffer(allocator); +inline buffer slice::encode_hex(bool uppercase, unsigned wrap_width, + const ALLOCATOR &allocator) const { + return to_hex(*this, uppercase, wrap_width).as_buffer(allocator); } template -inline buffer -slice::encode_base58(unsigned wrap_width, const ALLOCATOR &allocator) const { - return to_base58(*this, wrap_width) - .as_buffer(allocator); +inline buffer slice::encode_base58(unsigned wrap_width, const ALLOCATOR &allocator) const { + return to_base58(*this, wrap_width).as_buffer(allocator); } template -inline buffer -slice::encode_base64(unsigned wrap_width, const ALLOCATOR &allocator) const { - return to_base64(*this, wrap_width) - .as_buffer(allocator); +inline buffer slice::encode_base64(unsigned wrap_width, const ALLOCATOR &allocator) const { + return to_base64(*this, wrap_width).as_buffer(allocator); } template -inline buffer -slice::hex_decode(bool ignore_spaces, const ALLOCATOR &allocator) const { - return from_hex(*this, ignore_spaces) - .as_buffer(allocator); +inline buffer slice::hex_decode(bool ignore_spaces, const ALLOCATOR &allocator) const { + return from_hex(*this, ignore_spaces).as_buffer(allocator); } template -inline buffer -slice::base58_decode(bool ignore_spaces, const ALLOCATOR &allocator) const { - return from_base58(*this, ignore_spaces) - .as_buffer(allocator); +inline buffer slice::base58_decode(bool ignore_spaces, const ALLOCATOR &allocator) const { + return from_base58(*this, ignore_spaces).as_buffer(allocator); } template -inline buffer -slice::base64_decode(bool ignore_spaces, const ALLOCATOR &allocator) const { - return from_base64(*this, ignore_spaces) - .as_buffer(allocator); +inline buffer slice::base64_decode(bool ignore_spaces, const ALLOCATOR &allocator) const { + return from_base64(*this, ignore_spaces).as_buffer(allocator); } -MDBX_NOTHROW_PURE_FUNCTION inline bool -slice::is_hex(bool ignore_spaces) const noexcept { +MDBX_NOTHROW_PURE_FUNCTION inline bool slice::is_hex(bool ignore_spaces) const noexcept { return !from_hex(*this, ignore_spaces).is_erroneous(); } -MDBX_NOTHROW_PURE_FUNCTION inline bool -slice::is_base58(bool ignore_spaces) const noexcept { +MDBX_NOTHROW_PURE_FUNCTION inline bool slice::is_base58(bool ignore_spaces) const noexcept { return !from_base58(*this, ignore_spaces).is_erroneous(); } -MDBX_NOTHROW_PURE_FUNCTION inline bool -slice::is_base64(bool ignore_spaces) const noexcept { +MDBX_NOTHROW_PURE_FUNCTION inline bool slice::is_base64(bool ignore_spaces) const noexcept { return !from_base64(*this, ignore_spaces).is_erroneous(); } //------------------------------------------------------------------------------ -MDBX_CXX14_CONSTEXPR intptr_t pair::compare_fast(const pair &a, - const pair &b) noexcept { +MDBX_CXX14_CONSTEXPR intptr_t pair::compare_fast(const pair &a, const pair &b) noexcept { const auto diff = slice::compare_fast(a.key, b.key); return diff ? diff : slice::compare_fast(a.value, b.value); } -MDBX_CXX14_CONSTEXPR intptr_t -pair::compare_lexicographically(const pair &a, const pair &b) noexcept { +MDBX_CXX14_CONSTEXPR intptr_t pair::compare_lexicographically(const pair &a, const pair &b) noexcept { const auto diff = slice::compare_lexicographically(a.key, b.key); return diff ? diff : slice::compare_lexicographically(a.value, b.value); } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator==(const pair &a, const pair &b) noexcept { - return a.key.length() == b.key.length() && - a.value.length() == b.value.length() && +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator==(const pair &a, const pair &b) noexcept { + return a.key.length() == b.key.length() && a.value.length() == b.value.length() && memcmp(a.key.data(), b.key.data(), a.key.length()) == 0 && memcmp(a.value.data(), b.value.data(), a.value.length()) == 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator<(const pair &a, const pair &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator<(const pair &a, const pair &b) noexcept { return pair::compare_lexicographically(a, b) < 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator>(const pair &a, const pair &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator>(const pair &a, const pair &b) noexcept { return pair::compare_lexicographically(a, b) > 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator<=(const pair &a, const pair &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator<=(const pair &a, const pair &b) noexcept { return pair::compare_lexicographically(a, b) <= 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator>=(const pair &a, const pair &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator>=(const pair &a, const pair &b) noexcept { return pair::compare_lexicographically(a, b) >= 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator!=(const pair &a, const pair &b) noexcept { - return a.key.length() != b.key.length() || - a.value.length() != b.value.length() || +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator!=(const pair &a, const pair &b) noexcept { + return a.key.length() != b.key.length() || a.value.length() != b.value.length() || memcmp(a.key.data(), b.key.data(), a.key.length()) != 0 || memcmp(a.value.data(), b.value.data(), a.value.length()) != 0; } @@ -5982,25 +5109,21 @@ operator!=(const pair &a, const pair &b) noexcept { //------------------------------------------------------------------------------ template -inline buffer::buffer( - const txn &txn, const struct slice &src, const allocator_type &allocator) +inline buffer::buffer(const txn &txn, const struct slice &src, + const allocator_type &allocator) : buffer(src, !txn.is_dirty(src.data()), allocator) {} //------------------------------------------------------------------------------ -MDBX_CXX11_CONSTEXPR map_handle::info::info(map_handle::flags flags, - map_handle::state state) noexcept +MDBX_CXX11_CONSTEXPR map_handle::info::info(map_handle::flags flags, map_handle::state state) noexcept : flags(flags), state(state) {} -MDBX_CXX11_CONSTEXPR_ENUM mdbx::key_mode -map_handle::info::key_mode() const noexcept { +MDBX_CXX11_CONSTEXPR_ENUM mdbx::key_mode map_handle::info::key_mode() const noexcept { return ::mdbx::key_mode(flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)); } -MDBX_CXX11_CONSTEXPR_ENUM mdbx::value_mode -map_handle::info::value_mode() const noexcept { - return ::mdbx::value_mode(flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | - MDBX_DUPFIXED | MDBX_INTEGERDUP)); +MDBX_CXX11_CONSTEXPR_ENUM mdbx::value_mode map_handle::info::value_mode() const noexcept { + return ::mdbx::value_mode(flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED | MDBX_INTEGERDUP)); } //------------------------------------------------------------------------------ @@ -6013,9 +5136,7 @@ inline env &env::operator=(env &&other) noexcept { return *this; } -inline env::env(env &&other) noexcept : handle_(other.handle_) { - other.handle_ = nullptr; -} +inline env::env(env &&other) noexcept : handle_(other.handle_) { other.handle_ = nullptr; } inline env::~env() noexcept { #ifndef NDEBUG @@ -6023,21 +5144,15 @@ inline env::~env() noexcept { #endif } -MDBX_CXX14_CONSTEXPR env::operator bool() const noexcept { - return handle_ != nullptr; -} +MDBX_CXX14_CONSTEXPR env::operator bool() const noexcept { return handle_ != nullptr; } MDBX_CXX14_CONSTEXPR env::operator const MDBX_env *() const { return handle_; } MDBX_CXX14_CONSTEXPR env::operator MDBX_env *() { return handle_; } -MDBX_CXX11_CONSTEXPR bool operator==(const env &a, const env &b) noexcept { - return a.handle_ == b.handle_; -} +MDBX_CXX11_CONSTEXPR bool operator==(const env &a, const env &b) noexcept { return a.handle_ == b.handle_; } -MDBX_CXX11_CONSTEXPR bool operator!=(const env &a, const env &b) noexcept { - return a.handle_ != b.handle_; -} +MDBX_CXX11_CONSTEXPR bool operator!=(const env &a, const env &b) noexcept { return a.handle_ != b.handle_; } inline env::geometry &env::geometry::make_fixed(intptr_t size) noexcept { size_lower = size_now = size_upper = size; @@ -6045,21 +5160,18 @@ inline env::geometry &env::geometry::make_fixed(intptr_t size) noexcept { return *this; } -inline env::geometry &env::geometry::make_dynamic(intptr_t lower, - intptr_t upper) noexcept { +inline env::geometry &env::geometry::make_dynamic(intptr_t lower, intptr_t upper) noexcept { size_now = size_lower = lower; size_upper = upper; growth_step = shrink_threshold = default_value; return *this; } -inline env::reclaiming_options env::operate_parameters::reclaiming_from_flags( - MDBX_env_flags_t flags) noexcept { +inline env::reclaiming_options env::operate_parameters::reclaiming_from_flags(MDBX_env_flags_t flags) noexcept { return reclaiming_options(flags); } -inline env::operate_options -env::operate_parameters::options_from_flags(MDBX_env_flags_t flags) noexcept { +inline env::operate_options env::operate_parameters::options_from_flags(MDBX_env_flags_t flags) noexcept { return operate_options(flags); } @@ -6081,13 +5193,9 @@ inline size_t env::limits::dbsize_max(intptr_t pagesize) { return static_cast(result); } -inline size_t env::limits::key_min(MDBX_db_flags_t flags) noexcept { - return (flags & MDBX_INTEGERKEY) ? 4 : 0; -} +inline size_t env::limits::key_min(MDBX_db_flags_t flags) noexcept { return (flags & MDBX_INTEGERKEY) ? 4 : 0; } -inline size_t env::limits::key_min(key_mode mode) noexcept { - return key_min(MDBX_db_flags_t(mode)); -} +inline size_t env::limits::key_min(key_mode mode) noexcept { return key_min(MDBX_db_flags_t(mode)); } inline size_t env::limits::key_max(intptr_t pagesize, MDBX_db_flags_t flags) { const intptr_t result = mdbx_limits_keysize_max(pagesize, flags); @@ -6107,17 +5215,11 @@ inline size_t env::limits::key_max(const env &env, MDBX_db_flags_t flags) { return static_cast(result); } -inline size_t env::limits::key_max(const env &env, key_mode mode) { - return key_max(env, MDBX_db_flags_t(mode)); -} +inline size_t env::limits::key_max(const env &env, key_mode mode) { return key_max(env, MDBX_db_flags_t(mode)); } -inline size_t env::limits::value_min(MDBX_db_flags_t flags) noexcept { - return (flags & MDBX_INTEGERDUP) ? 4 : 0; -} +inline size_t env::limits::value_min(MDBX_db_flags_t flags) noexcept { return (flags & MDBX_INTEGERDUP) ? 4 : 0; } -inline size_t env::limits::value_min(value_mode mode) noexcept { - return value_min(MDBX_db_flags_t(mode)); -} +inline size_t env::limits::value_min(value_mode mode) noexcept { return value_min(MDBX_db_flags_t(mode)); } inline size_t env::limits::value_max(intptr_t pagesize, MDBX_db_flags_t flags) { const intptr_t result = mdbx_limits_valsize_max(pagesize, flags); @@ -6137,25 +5239,20 @@ inline size_t env::limits::value_max(const env &env, MDBX_db_flags_t flags) { return static_cast(result); } -inline size_t env::limits::value_max(const env &env, value_mode mode) { - return value_max(env, MDBX_db_flags_t(mode)); -} +inline size_t env::limits::value_max(const env &env, value_mode mode) { return value_max(env, MDBX_db_flags_t(mode)); } -inline size_t env::limits::pairsize4page_max(intptr_t pagesize, - MDBX_db_flags_t flags) { +inline size_t env::limits::pairsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags) { const intptr_t result = mdbx_limits_pairsize4page_max(pagesize, flags); if (result < 0) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); return static_cast(result); } -inline size_t env::limits::pairsize4page_max(intptr_t pagesize, - value_mode mode) { +inline size_t env::limits::pairsize4page_max(intptr_t pagesize, value_mode mode) { return pairsize4page_max(pagesize, MDBX_db_flags_t(mode)); } -inline size_t env::limits::pairsize4page_max(const env &env, - MDBX_db_flags_t flags) { +inline size_t env::limits::pairsize4page_max(const env &env, MDBX_db_flags_t flags) { const intptr_t result = mdbx_env_get_pairsize4page_max(env, flags); if (result < 0) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); @@ -6166,21 +5263,18 @@ inline size_t env::limits::pairsize4page_max(const env &env, value_mode mode) { return pairsize4page_max(env, MDBX_db_flags_t(mode)); } -inline size_t env::limits::valsize4page_max(intptr_t pagesize, - MDBX_db_flags_t flags) { +inline size_t env::limits::valsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags) { const intptr_t result = mdbx_limits_valsize4page_max(pagesize, flags); if (result < 0) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); return static_cast(result); } -inline size_t env::limits::valsize4page_max(intptr_t pagesize, - value_mode mode) { +inline size_t env::limits::valsize4page_max(intptr_t pagesize, value_mode mode) { return valsize4page_max(pagesize, MDBX_db_flags_t(mode)); } -inline size_t env::limits::valsize4page_max(const env &env, - MDBX_db_flags_t flags) { +inline size_t env::limits::valsize4page_max(const env &env, MDBX_db_flags_t flags) { const intptr_t result = mdbx_env_get_valsize4page_max(env, flags); if (result < 0) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); @@ -6202,16 +5296,13 @@ inline size_t env::limits::max_map_handles(void) { return MDBX_MAX_DBI; } inline env::operate_parameters env::get_operation_parameters() const { const auto flags = get_flags(); - return operate_parameters(max_maps(), max_readers(), - operate_parameters::mode_from_flags(flags), + return operate_parameters(max_maps(), max_readers(), operate_parameters::mode_from_flags(flags), operate_parameters::durability_from_flags(flags), operate_parameters::reclaiming_from_flags(flags), operate_parameters::options_from_flags(flags)); } -inline env::mode env::get_mode() const { - return operate_parameters::mode_from_flags(get_flags()); -} +inline env::mode env::get_mode() const { return operate_parameters::mode_from_flags(get_flags()); } inline env::durability env::get_durability() const { return env::operate_parameters::durability_from_flags(get_flags()); @@ -6273,9 +5364,7 @@ inline unsigned env::max_maps() const { return r; } -inline void *env::get_context() const noexcept { - return mdbx_env_get_userctx(handle_); -} +inline void *env::get_context() const noexcept { return mdbx_env_get_userctx(handle_); } inline env &env::set_context(void *ptr) { error::success_or_throw(::mdbx_env_set_userctx(handle_, ptr)); @@ -6308,31 +5397,22 @@ inline env &env::set_sync_period__seconds_double(double seconds) { return set_sync_period__seconds_16dot16(unsigned(seconds * 65536)); } -inline double env::sync_period__seconds_double() const { - return sync_period__seconds_16dot16() / 65536.0; -} +inline double env::sync_period__seconds_double() const { return sync_period__seconds_16dot16() / 65536.0; } #if __cplusplus >= 201103L -inline env &env::set_sync_period(const duration &period) { - return set_sync_period__seconds_16dot16(period.count()); -} +inline env &env::set_sync_period(const duration &period) { return set_sync_period__seconds_16dot16(period.count()); } -inline duration env::sync_period() const { - return duration(sync_period__seconds_16dot16()); -} +inline duration env::sync_period() const { return duration(sync_period__seconds_16dot16()); } #endif -inline env &env::set_extra_option(enum env::extra_runtime_option option, - uint64_t value) { - error::success_or_throw( - ::mdbx_env_set_option(handle_, ::MDBX_option_t(option), value)); +inline env &env::set_extra_option(enum env::extra_runtime_option option, uint64_t value) { + error::success_or_throw(::mdbx_env_set_option(handle_, ::MDBX_option_t(option), value)); return *this; } inline uint64_t env::extra_option(enum env::extra_runtime_option option) const { uint64_t value; - error::success_or_throw( - ::mdbx_env_get_option(handle_, ::MDBX_option_t(option), &value)); + error::success_or_throw(::mdbx_env_get_option(handle_, ::MDBX_option_t(option), &value)); return value; } @@ -6342,9 +5422,8 @@ inline env &env::alter_flags(MDBX_env_flags_t flags, bool on_off) { } inline env &env::set_geometry(const geometry &geo) { - error::success_or_throw(::mdbx_env_set_geometry( - handle_, geo.size_lower, geo.size_now, geo.size_upper, geo.growth_step, - geo.shrink_threshold, geo.pagesize)); + error::success_or_throw(::mdbx_env_set_geometry(handle_, geo.size_lower, geo.size_now, geo.size_upper, + geo.growth_step, geo.shrink_threshold, geo.pagesize)); return *this; } @@ -6361,24 +5440,19 @@ inline bool env::sync_to_disk(bool force, bool nonblock) { } } -inline void env::close_map(const map_handle &handle) { - error::success_or_throw(::mdbx_dbi_close(*this, handle.dbi)); -} +inline void env::close_map(const map_handle &handle) { error::success_or_throw(::mdbx_dbi_close(*this, handle.dbi)); } MDBX_CXX11_CONSTEXPR -env::reader_info::reader_info(int slot, mdbx_pid_t pid, mdbx_tid_t thread, - uint64_t txnid, uint64_t lag, size_t used, +env::reader_info::reader_info(int slot, mdbx_pid_t pid, mdbx_tid_t thread, uint64_t txnid, uint64_t lag, size_t used, size_t retained) noexcept - : slot(slot), pid(pid), thread(thread), transaction_id(txnid), - transaction_lag(lag), bytes_used(used), bytes_retained(retained) {} + : slot(slot), pid(pid), thread(thread), transaction_id(txnid), transaction_lag(lag), bytes_used(used), + bytes_retained(retained) {} -template -inline int env::enumerate_readers(VISITOR &visitor) { +template inline int env::enumerate_readers(VISITOR &visitor) { struct reader_visitor_thunk : public exception_thunk { VISITOR &visitor_; - static int cb(void *ctx, int number, int slot, mdbx_pid_t pid, - mdbx_tid_t thread, uint64_t txnid, uint64_t lag, size_t used, - size_t retained) noexcept { + static int cb(void *ctx, int number, int slot, mdbx_pid_t pid, mdbx_tid_t thread, uint64_t txnid, uint64_t lag, + size_t used, size_t retained) noexcept { reader_visitor_thunk *thunk = static_cast(ctx); assert(thunk->is_clean()); try { @@ -6389,8 +5463,7 @@ inline int env::enumerate_readers(VISITOR &visitor) { return loop_control::exit_loop; } } - MDBX_CXX11_CONSTEXPR reader_visitor_thunk(VISITOR &visitor) noexcept - : visitor_(visitor) {} + MDBX_CXX11_CONSTEXPR reader_visitor_thunk(VISITOR &visitor) noexcept : visitor_(visitor) {} }; reader_visitor_thunk thunk(visitor); const auto rc = ::mdbx_reader_list(*this, thunk.cb, &thunk); @@ -6410,38 +5483,32 @@ inline env &env::set_HandleSlowReaders(MDBX_hsr_func *cb) { return *this; } -inline MDBX_hsr_func *env::get_HandleSlowReaders() const noexcept { - return ::mdbx_env_get_hsr(handle_); -} +inline MDBX_hsr_func *env::get_HandleSlowReaders() const noexcept { return ::mdbx_env_get_hsr(handle_); } inline txn_managed env::start_read() const { ::MDBX_txn *ptr; - error::success_or_throw( - ::mdbx_txn_begin(handle_, nullptr, MDBX_TXN_RDONLY, &ptr)); + error::success_or_throw(::mdbx_txn_begin(handle_, nullptr, MDBX_TXN_RDONLY, &ptr)); assert(ptr != nullptr); return txn_managed(ptr); } inline txn_managed env::prepare_read() const { ::MDBX_txn *ptr; - error::success_or_throw( - ::mdbx_txn_begin(handle_, nullptr, MDBX_TXN_RDONLY_PREPARE, &ptr)); + error::success_or_throw(::mdbx_txn_begin(handle_, nullptr, MDBX_TXN_RDONLY_PREPARE, &ptr)); assert(ptr != nullptr); return txn_managed(ptr); } inline txn_managed env::start_write(bool dont_wait) { ::MDBX_txn *ptr; - error::success_or_throw(::mdbx_txn_begin( - handle_, nullptr, dont_wait ? MDBX_TXN_TRY : MDBX_TXN_READWRITE, &ptr)); + error::success_or_throw(::mdbx_txn_begin(handle_, nullptr, dont_wait ? MDBX_TXN_TRY : MDBX_TXN_READWRITE, &ptr)); assert(ptr != nullptr); return txn_managed(ptr); } inline txn_managed env::start_write(txn &parent) { ::MDBX_txn *ptr; - error::success_or_throw( - ::mdbx_txn_begin(handle_, parent, MDBX_TXN_READWRITE, &ptr)); + error::success_or_throw(::mdbx_txn_begin(handle_, parent, MDBX_TXN_READWRITE, &ptr)); assert(ptr != nullptr); return txn_managed(ptr); } @@ -6458,9 +5525,7 @@ inline txn &txn::operator=(txn &&other) noexcept { return *this; } -inline txn::txn(txn &&other) noexcept : handle_(other.handle_) { - other.handle_ = nullptr; -} +inline txn::txn(txn &&other) noexcept : handle_(other.handle_) { other.handle_ = nullptr; } inline txn::~txn() noexcept { #ifndef NDEBUG @@ -6468,25 +5533,17 @@ inline txn::~txn() noexcept { #endif } -MDBX_CXX14_CONSTEXPR txn::operator bool() const noexcept { - return handle_ != nullptr; -} +MDBX_CXX14_CONSTEXPR txn::operator bool() const noexcept { return handle_ != nullptr; } MDBX_CXX14_CONSTEXPR txn::operator const MDBX_txn *() const { return handle_; } MDBX_CXX14_CONSTEXPR txn::operator MDBX_txn *() { return handle_; } -MDBX_CXX11_CONSTEXPR bool operator==(const txn &a, const txn &b) noexcept { - return a.handle_ == b.handle_; -} +MDBX_CXX11_CONSTEXPR bool operator==(const txn &a, const txn &b) noexcept { return a.handle_ == b.handle_; } -MDBX_CXX11_CONSTEXPR bool operator!=(const txn &a, const txn &b) noexcept { - return a.handle_ != b.handle_; -} +MDBX_CXX11_CONSTEXPR bool operator!=(const txn &a, const txn &b) noexcept { return a.handle_ != b.handle_; } -inline void *txn::get_context() const noexcept { - return mdbx_txn_get_userctx(handle_); -} +inline void *txn::get_context() const noexcept { return mdbx_txn_get_userctx(handle_); } inline txn &txn::set_context(void *ptr) { error::success_or_throw(::mdbx_txn_set_userctx(handle_, ptr)); @@ -6519,17 +5576,11 @@ inline uint64_t txn::id() const { return txnid; } -inline void txn::reset_reading() { - error::success_or_throw(::mdbx_txn_reset(handle_)); -} +inline void txn::reset_reading() { error::success_or_throw(::mdbx_txn_reset(handle_)); } -inline void txn::renew_reading() { - error::success_or_throw(::mdbx_txn_renew(handle_)); -} +inline void txn::renew_reading() { error::success_or_throw(::mdbx_txn_renew(handle_)); } -inline void txn::park_reading(bool autounpark) { - error::success_or_throw(::mdbx_txn_park(handle_, autounpark)); -} +inline void txn::park_reading(bool autounpark) { error::success_or_throw(::mdbx_txn_park(handle_, autounpark)); } inline bool txn::unpark_reading(bool restart_if_ousted) { return error::boolean_or_throw(::mdbx_txn_unpark(handle_, restart_if_ousted)); @@ -6554,76 +5605,59 @@ inline size_t txn::release_all_cursors(bool unbind) const { return size_t(err); } -inline ::mdbx::map_handle -txn::open_map(const ::mdbx::slice &name, const ::mdbx::key_mode key_mode, - const ::mdbx::value_mode value_mode) const { - ::mdbx::map_handle map; - error::success_or_throw(::mdbx_dbi_open2( - handle_, name, MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), - &map.dbi)); - assert(map.dbi != 0); - return map; -} - -inline ::mdbx::map_handle -txn::open_map(const char *name, const ::mdbx::key_mode key_mode, - const ::mdbx::value_mode value_mode) const { - ::mdbx::map_handle map; - error::success_or_throw(::mdbx_dbi_open( - handle_, name, MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), - &map.dbi)); - assert(map.dbi != 0); - return map; -} - -inline ::mdbx::map_handle -txn::open_map_accede(const ::mdbx::slice &name) const { +inline ::mdbx::map_handle txn::open_map(const ::mdbx::slice &name, const ::mdbx::key_mode key_mode, + const ::mdbx::value_mode value_mode) const { ::mdbx::map_handle map; error::success_or_throw( - ::mdbx_dbi_open2(handle_, name, MDBX_DB_ACCEDE, &map.dbi)); + ::mdbx_dbi_open2(handle_, name, MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), &map.dbi)); + assert(map.dbi != 0); + return map; +} + +inline ::mdbx::map_handle txn::open_map(const char *name, const ::mdbx::key_mode key_mode, + const ::mdbx::value_mode value_mode) const { + ::mdbx::map_handle map; + error::success_or_throw( + ::mdbx_dbi_open(handle_, name, MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), &map.dbi)); + assert(map.dbi != 0); + return map; +} + +inline ::mdbx::map_handle txn::open_map_accede(const ::mdbx::slice &name) const { + ::mdbx::map_handle map; + error::success_or_throw(::mdbx_dbi_open2(handle_, name, MDBX_DB_ACCEDE, &map.dbi)); assert(map.dbi != 0); return map; } inline ::mdbx::map_handle txn::open_map_accede(const char *name) const { ::mdbx::map_handle map; + error::success_or_throw(::mdbx_dbi_open(handle_, name, MDBX_DB_ACCEDE, &map.dbi)); + assert(map.dbi != 0); + return map; +} + +inline ::mdbx::map_handle txn::create_map(const ::mdbx::slice &name, const ::mdbx::key_mode key_mode, + const ::mdbx::value_mode value_mode) { + ::mdbx::map_handle map; error::success_or_throw( - ::mdbx_dbi_open(handle_, name, MDBX_DB_ACCEDE, &map.dbi)); + ::mdbx_dbi_open2(handle_, name, MDBX_CREATE | MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), &map.dbi)); assert(map.dbi != 0); return map; } -inline ::mdbx::map_handle txn::create_map(const ::mdbx::slice &name, - const ::mdbx::key_mode key_mode, +inline ::mdbx::map_handle txn::create_map(const char *name, const ::mdbx::key_mode key_mode, const ::mdbx::value_mode value_mode) { ::mdbx::map_handle map; - error::success_or_throw(::mdbx_dbi_open2( - handle_, name, - MDBX_CREATE | MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), - &map.dbi)); + error::success_or_throw( + ::mdbx_dbi_open(handle_, name, MDBX_CREATE | MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), &map.dbi)); assert(map.dbi != 0); return map; } -inline ::mdbx::map_handle txn::create_map(const char *name, - const ::mdbx::key_mode key_mode, - const ::mdbx::value_mode value_mode) { - ::mdbx::map_handle map; - error::success_or_throw(::mdbx_dbi_open( - handle_, name, - MDBX_CREATE | MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), - &map.dbi)); - assert(map.dbi != 0); - return map; -} +inline void txn::drop_map(map_handle map) { error::success_or_throw(::mdbx_drop(handle_, map.dbi, true)); } -inline void txn::drop_map(map_handle map) { - error::success_or_throw(::mdbx_drop(handle_, map.dbi, true)); -} - -inline void txn::clear_map(map_handle map) { - error::success_or_throw(::mdbx_drop(handle_, map.dbi, false)); -} +inline void txn::clear_map(map_handle map) { error::success_or_throw(::mdbx_drop(handle_, map.dbi, false)); } inline void txn::rename_map(map_handle map, const char *new_name) { error::success_or_throw(::mdbx_dbi_rename(handle_, map, new_name)); @@ -6633,19 +5667,16 @@ inline void txn::rename_map(map_handle map, const ::mdbx::slice &new_name) { error::success_or_throw(::mdbx_dbi_rename2(handle_, map, new_name)); } -inline ::mdbx::map_handle -txn::open_map(const ::std::string &name, const ::mdbx::key_mode key_mode, - const ::mdbx::value_mode value_mode) const { +inline ::mdbx::map_handle txn::open_map(const ::std::string &name, const ::mdbx::key_mode key_mode, + const ::mdbx::value_mode value_mode) const { return open_map(::mdbx::slice(name), key_mode, value_mode); } -inline ::mdbx::map_handle -txn::open_map_accede(const ::std::string &name) const { +inline ::mdbx::map_handle txn::open_map_accede(const ::std::string &name) const { return open_map_accede(::mdbx::slice(name)); } -inline ::mdbx::map_handle txn::create_map(const ::std::string &name, - const ::mdbx::key_mode key_mode, +inline ::mdbx::map_handle txn::create_map(const ::std::string &name, const ::mdbx::key_mode key_mode, const ::mdbx::value_mode value_mode) { return create_map(::mdbx::slice(name), key_mode, value_mode); } @@ -6676,8 +5707,7 @@ inline uint32_t txn::get_tree_deepmask(map_handle map) const { inline map_handle::info txn::get_handle_info(map_handle map) const { unsigned flags, state; - error::success_or_throw( - ::mdbx_dbi_flags_ex(handle_, map.dbi, &flags, &state)); + error::success_or_throw(::mdbx_dbi_flags_ex(handle_, map.dbi, &flags, &state)); return map_handle::info(MDBX_db_flags_t(flags), MDBX_dbi_state_t(state)); } @@ -6700,28 +5730,23 @@ inline uint64_t txn::sequence(map_handle map) const { inline uint64_t txn::sequence(map_handle map, uint64_t increment) { uint64_t result; - error::success_or_throw( - ::mdbx_dbi_sequence(handle_, map.dbi, &result, increment)); + error::success_or_throw(::mdbx_dbi_sequence(handle_, map.dbi, &result, increment)); return result; } -inline int txn::compare_keys(map_handle map, const slice &a, - const slice &b) const noexcept { +inline int txn::compare_keys(map_handle map, const slice &a, const slice &b) const noexcept { return ::mdbx_cmp(handle_, map.dbi, &a, &b); } -inline int txn::compare_values(map_handle map, const slice &a, - const slice &b) const noexcept { +inline int txn::compare_values(map_handle map, const slice &a, const slice &b) const noexcept { return ::mdbx_dcmp(handle_, map.dbi, &a, &b); } -inline int txn::compare_keys(map_handle map, const pair &a, - const pair &b) const noexcept { +inline int txn::compare_keys(map_handle map, const pair &a, const pair &b) const noexcept { return compare_keys(map, a.key, b.key); } -inline int txn::compare_values(map_handle map, const pair &a, - const pair &b) const noexcept { +inline int txn::compare_values(map_handle map, const pair &a, const pair &b) const noexcept { return compare_values(map, a.value, b.value); } @@ -6733,13 +5758,11 @@ inline slice txn::get(map_handle map, const slice &key) const { inline slice txn::get(map_handle map, slice key, size_t &values_count) const { slice result; - error::success_or_throw( - ::mdbx_get_ex(handle_, map.dbi, &key, &result, &values_count)); + error::success_or_throw(::mdbx_get_ex(handle_, map.dbi, &key, &result, &values_count)); return result; } -inline slice txn::get(map_handle map, const slice &key, - const slice &value_at_absence) const { +inline slice txn::get(map_handle map, const slice &key, const slice &value_at_absence) const { slice result; const int err = ::mdbx_get(handle_, map.dbi, &key, &result); switch (err) { @@ -6752,8 +5775,7 @@ inline slice txn::get(map_handle map, const slice &key, } } -inline slice txn::get(map_handle map, slice key, size_t &values_count, - const slice &value_at_absence) const { +inline slice txn::get(map_handle map, slice key, size_t &values_count, const slice &value_at_absence) const { slice result; const int err = ::mdbx_get_ex(handle_, map.dbi, &key, &result, &values_count); switch (err) { @@ -6766,20 +5788,15 @@ inline slice txn::get(map_handle map, slice key, size_t &values_count, } } -inline pair_result txn::get_equal_or_great(map_handle map, - const slice &key) const { +inline pair_result txn::get_equal_or_great(map_handle map, const slice &key) const { pair result(key, slice()); - bool exact = !error::boolean_or_throw( - ::mdbx_get_equal_or_great(handle_, map.dbi, &result.key, &result.value)); + bool exact = !error::boolean_or_throw(::mdbx_get_equal_or_great(handle_, map.dbi, &result.key, &result.value)); return pair_result(result.key, result.value, exact); } -inline pair_result -txn::get_equal_or_great(map_handle map, const slice &key, - const slice &value_at_absence) const { +inline pair_result txn::get_equal_or_great(map_handle map, const slice &key, const slice &value_at_absence) const { pair result{key, slice()}; - const int err = - ::mdbx_get_equal_or_great(handle_, map.dbi, &result.key, &result.value); + const int err = ::mdbx_get_equal_or_great(handle_, map.dbi, &result.key, &result.value); switch (err) { case MDBX_SUCCESS: return pair_result{result.key, result.value, true}; @@ -6792,27 +5809,22 @@ txn::get_equal_or_great(map_handle map, const slice &key, } } -inline MDBX_error_t txn::put(map_handle map, const slice &key, slice *value, - MDBX_put_flags_t flags) noexcept { +inline MDBX_error_t txn::put(map_handle map, const slice &key, slice *value, MDBX_put_flags_t flags) noexcept { return MDBX_error_t(::mdbx_put(handle_, map.dbi, &key, value, flags)); } -inline void txn::put(map_handle map, const slice &key, slice value, - put_mode mode) { +inline void txn::put(map_handle map, const slice &key, slice value, put_mode mode) { error::success_or_throw(put(map, key, &value, MDBX_put_flags_t(mode))); } inline void txn::insert(map_handle map, const slice &key, slice value) { - error::success_or_throw( - put(map, key, &value /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique))); + error::success_or_throw(put(map, key, &value /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert_unique))); } -inline value_result txn::try_insert(map_handle map, const slice &key, - slice value) { - const int err = - put(map, key, &value /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique)); +inline value_result txn::try_insert(map_handle map, const slice &key, slice value) { + const int err = put(map, key, &value /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert_unique)); switch (err) { case MDBX_SUCCESS: return value_result{slice(), true}; @@ -6823,21 +5835,17 @@ inline value_result txn::try_insert(map_handle map, const slice &key, } } -inline slice txn::insert_reserve(map_handle map, const slice &key, - size_t value_length) { +inline slice txn::insert_reserve(map_handle map, const slice &key, size_t value_length) { slice result(nullptr, value_length); - error::success_or_throw( - put(map, key, &result /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE)); + error::success_or_throw(put(map, key, &result /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE)); return result; } -inline value_result txn::try_insert_reserve(map_handle map, const slice &key, - size_t value_length) { +inline value_result txn::try_insert_reserve(map_handle map, const slice &key, size_t value_length) { slice result(nullptr, value_length); - const int err = - put(map, key, &result /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE); + const int err = put(map, key, &result /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE); switch (err) { case MDBX_SUCCESS: return value_result{result, true}; @@ -6849,27 +5857,21 @@ inline value_result txn::try_insert_reserve(map_handle map, const slice &key, } inline void txn::upsert(map_handle map, const slice &key, const slice &value) { - error::success_or_throw(put(map, key, const_cast(&value), - MDBX_put_flags_t(put_mode::upsert))); + error::success_or_throw(put(map, key, const_cast(&value), MDBX_put_flags_t(put_mode::upsert))); } -inline slice txn::upsert_reserve(map_handle map, const slice &key, - size_t value_length) { +inline slice txn::upsert_reserve(map_handle map, const slice &key, size_t value_length) { slice result(nullptr, value_length); - error::success_or_throw(put( - map, key, &result, MDBX_put_flags_t(put_mode::upsert) | MDBX_RESERVE)); + error::success_or_throw(put(map, key, &result, MDBX_put_flags_t(put_mode::upsert) | MDBX_RESERVE)); return result; } inline void txn::update(map_handle map, const slice &key, const slice &value) { - error::success_or_throw(put(map, key, const_cast(&value), - MDBX_put_flags_t(put_mode::update))); + error::success_or_throw(put(map, key, const_cast(&value), MDBX_put_flags_t(put_mode::update))); } -inline bool txn::try_update(map_handle map, const slice &key, - const slice &value) { - const int err = put(map, key, const_cast(&value), - MDBX_put_flags_t(put_mode::update)); +inline bool txn::try_update(map_handle map, const slice &key, const slice &value) { + const int err = put(map, key, const_cast(&value), MDBX_put_flags_t(put_mode::update)); switch (err) { case MDBX_SUCCESS: return true; @@ -6880,19 +5882,15 @@ inline bool txn::try_update(map_handle map, const slice &key, } } -inline slice txn::update_reserve(map_handle map, const slice &key, - size_t value_length) { +inline slice txn::update_reserve(map_handle map, const slice &key, size_t value_length) { slice result(nullptr, value_length); - error::success_or_throw(put( - map, key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE)); + error::success_or_throw(put(map, key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE)); return result; } -inline value_result txn::try_update_reserve(map_handle map, const slice &key, - size_t value_length) { +inline value_result txn::try_update_reserve(map_handle map, const slice &key, size_t value_length) { slice result(nullptr, value_length); - const int err = - put(map, key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE); + const int err = put(map, key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE); switch (err) { case MDBX_SUCCESS: return value_result{result, true}; @@ -6927,69 +5925,53 @@ inline bool txn::erase(map_handle map, const slice &key, const slice &value) { } } -inline void txn::replace(map_handle map, const slice &key, slice old_value, - const slice &new_value) { - error::success_or_throw(::mdbx_replace_ex( - handle_, map.dbi, &key, const_cast(&new_value), &old_value, - MDBX_CURRENT | MDBX_NOOVERWRITE, nullptr, nullptr)); +inline void txn::replace(map_handle map, const slice &key, slice old_value, const slice &new_value) { + error::success_or_throw(::mdbx_replace_ex(handle_, map.dbi, &key, const_cast(&new_value), &old_value, + MDBX_CURRENT | MDBX_NOOVERWRITE, nullptr, nullptr)); } template inline buffer txn::extract(map_handle map, const slice &key, - const typename buffer::allocator_type - &allocator) { + const typename buffer::allocator_type &allocator) { typename buffer::data_preserver result(allocator); - error::success_or_throw(::mdbx_replace_ex(handle_, map.dbi, &key, nullptr, - &result.slice_, MDBX_CURRENT, - result, &result), - result); + error::success_or_throw( + ::mdbx_replace_ex(handle_, map.dbi, &key, nullptr, &result.slice_, MDBX_CURRENT, result, &result), result); return result; } template inline buffer txn::replace(map_handle map, const slice &key, const slice &new_value, - const typename buffer::allocator_type - &allocator) { + const typename buffer::allocator_type &allocator) { typename buffer::data_preserver result(allocator); - error::success_or_throw( - ::mdbx_replace_ex(handle_, map.dbi, &key, const_cast(&new_value), - &result.slice_, MDBX_CURRENT, result, &result), - result); + error::success_or_throw(::mdbx_replace_ex(handle_, map.dbi, &key, const_cast(&new_value), &result.slice_, + MDBX_CURRENT, result, &result), + result); return result; } template -inline buffer txn::replace_reserve( - map_handle map, const slice &key, slice &new_value, - const typename buffer::allocator_type - &allocator) { +inline buffer +txn::replace_reserve(map_handle map, const slice &key, slice &new_value, + const typename buffer::allocator_type &allocator) { typename buffer::data_preserver result(allocator); - error::success_or_throw( - ::mdbx_replace_ex(handle_, map.dbi, &key, &new_value, &result.slice_, - MDBX_CURRENT | MDBX_RESERVE, result, &result), - result); + error::success_or_throw(::mdbx_replace_ex(handle_, map.dbi, &key, &new_value, &result.slice_, + MDBX_CURRENT | MDBX_RESERVE, result, &result), + result); return result; } -inline void txn::append(map_handle map, const slice &key, const slice &value, - bool multivalue_order_preserved) { - error::success_or_throw(::mdbx_put( - handle_, map.dbi, const_cast(&key), const_cast(&value), - multivalue_order_preserved ? (MDBX_APPEND | MDBX_APPENDDUP) - : MDBX_APPEND)); +inline void txn::append(map_handle map, const slice &key, const slice &value, bool multivalue_order_preserved) { + error::success_or_throw(::mdbx_put(handle_, map.dbi, const_cast(&key), const_cast(&value), + multivalue_order_preserved ? (MDBX_APPEND | MDBX_APPENDDUP) : MDBX_APPEND)); } -inline size_t txn::put_multiple_samelength(map_handle map, const slice &key, - const size_t value_length, - const void *values_array, - size_t values_count, put_mode mode, +inline size_t txn::put_multiple_samelength(map_handle map, const slice &key, const size_t value_length, + const void *values_array, size_t values_count, put_mode mode, bool allow_partial) { - MDBX_val args[2] = {{const_cast(values_array), value_length}, - {nullptr, values_count}}; - const int err = ::mdbx_put(handle_, map.dbi, const_cast(&key), args, - MDBX_put_flags_t(mode) | MDBX_MULTIPLE); + MDBX_val args[2] = {{const_cast(values_array), value_length}, {nullptr, values_count}}; + const int err = ::mdbx_put(handle_, map.dbi, const_cast(&key), args, MDBX_put_flags_t(mode) | MDBX_MULTIPLE); switch (err) { case MDBX_SUCCESS: MDBX_CXX20_LIKELY break; @@ -7004,35 +5986,27 @@ inline size_t txn::put_multiple_samelength(map_handle map, const slice &key, return args[1].iov_len /* done item count */; } -inline ptrdiff_t txn::estimate(map_handle map, const pair &from, - const pair &to) const { +inline ptrdiff_t txn::estimate(map_handle map, const pair &from, const pair &to) const { ptrdiff_t result; - error::success_or_throw(mdbx_estimate_range( - handle_, map.dbi, &from.key, &from.value, &to.key, &to.value, &result)); + error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, &from.key, &from.value, &to.key, &to.value, &result)); return result; } -inline ptrdiff_t txn::estimate(map_handle map, const slice &from, - const slice &to) const { +inline ptrdiff_t txn::estimate(map_handle map, const slice &from, const slice &to) const { ptrdiff_t result; - error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, &from, nullptr, - &to, nullptr, &result)); + error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, &from, nullptr, &to, nullptr, &result)); return result; } -inline ptrdiff_t txn::estimate_from_first(map_handle map, - const slice &to) const { +inline ptrdiff_t txn::estimate_from_first(map_handle map, const slice &to) const { ptrdiff_t result; - error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, nullptr, - nullptr, &to, nullptr, &result)); + error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, nullptr, nullptr, &to, nullptr, &result)); return result; } -inline ptrdiff_t txn::estimate_to_last(map_handle map, - const slice &from) const { +inline ptrdiff_t txn::estimate_to_last(map_handle map, const slice &from) const { ptrdiff_t result; - error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, &from, nullptr, - nullptr, nullptr, &result)); + error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, &from, nullptr, nullptr, nullptr, &result)); return result; } @@ -7046,9 +6020,7 @@ inline cursor_managed cursor::clone(void *your_context) const { return clone; } -inline void *cursor::get_context() const noexcept { - return mdbx_cursor_get_userctx(handle_); -} +inline void *cursor::get_context() const noexcept { return mdbx_cursor_get_userctx(handle_); } inline cursor &cursor::set_context(void *ptr) { error::success_or_throw(::mdbx_cursor_set_userctx(handle_, ptr)); @@ -7061,9 +6033,7 @@ inline cursor &cursor::operator=(cursor &&other) noexcept { return *this; } -inline cursor::cursor(cursor &&other) noexcept : handle_(other.handle_) { - other.handle_ = nullptr; -} +inline cursor::cursor(cursor &&other) noexcept : handle_(other.handle_) { other.handle_ = nullptr; } inline cursor::~cursor() noexcept { #ifndef NDEBUG @@ -7071,33 +6041,21 @@ inline cursor::~cursor() noexcept { #endif } -MDBX_CXX14_CONSTEXPR cursor::operator bool() const noexcept { - return handle_ != nullptr; -} +MDBX_CXX14_CONSTEXPR cursor::operator bool() const noexcept { return handle_ != nullptr; } -MDBX_CXX14_CONSTEXPR cursor::operator const MDBX_cursor *() const { - return handle_; -} +MDBX_CXX14_CONSTEXPR cursor::operator const MDBX_cursor *() const { return handle_; } MDBX_CXX14_CONSTEXPR cursor::operator MDBX_cursor *() { return handle_; } -MDBX_CXX11_CONSTEXPR bool operator==(const cursor &a, - const cursor &b) noexcept { - return a.handle_ == b.handle_; -} +MDBX_CXX11_CONSTEXPR bool operator==(const cursor &a, const cursor &b) noexcept { return a.handle_ == b.handle_; } -MDBX_CXX11_CONSTEXPR bool operator!=(const cursor &a, - const cursor &b) noexcept { - return a.handle_ != b.handle_; -} +MDBX_CXX11_CONSTEXPR bool operator!=(const cursor &a, const cursor &b) noexcept { return a.handle_ != b.handle_; } -inline int compare_position_nothrow(const cursor &left, const cursor &right, - bool ignore_nested = false) noexcept { +inline int compare_position_nothrow(const cursor &left, const cursor &right, bool ignore_nested = false) noexcept { return mdbx_cursor_compare(left.handle_, right.handle_, ignore_nested); } -inline int compare_position(const cursor &left, const cursor &right, - bool ignore_nested = false) { +inline int compare_position(const cursor &left, const cursor &right, bool ignore_nested = false) { const auto diff = compare_position_nothrow(left, right, ignore_nested); assert(compare_position_nothrow(right, left, ignore_nested) == -diff); if (MDBX_LIKELY(int16_t(diff) == diff)) @@ -7106,24 +6064,18 @@ inline int compare_position(const cursor &left, const cursor &right, throw_incomparable_cursors(); } -inline cursor::move_result::move_result(const cursor &cursor, - bool throw_notfound) - : pair_result() { +inline cursor::move_result::move_result(const cursor &cursor, bool throw_notfound) : pair_result() { done = cursor.move(get_current, &this->key, &this->value, throw_notfound); } -inline cursor::move_result::move_result(cursor &cursor, - move_operation operation, - const slice &key, const slice &value, +inline cursor::move_result::move_result(cursor &cursor, move_operation operation, const slice &key, const slice &value, bool throw_notfound) : pair_result(key, value, false) { this->done = cursor.move(operation, &this->key, &this->value, throw_notfound); } -inline bool cursor::move(move_operation operation, MDBX_val *key, - MDBX_val *value, bool throw_notfound) const { - const int err = - ::mdbx_cursor_get(handle_, key, value, MDBX_cursor_op(operation)); +inline bool cursor::move(move_operation operation, MDBX_val *key, MDBX_val *value, bool throw_notfound) const { + const int err = ::mdbx_cursor_get(handle_, key, value, MDBX_cursor_op(operation)); switch (err) { case MDBX_SUCCESS: MDBX_CXX20_LIKELY return true; @@ -7138,19 +6090,15 @@ inline bool cursor::move(move_operation operation, MDBX_val *key, } } -inline cursor::estimate_result::estimate_result(const cursor &cursor, - move_operation operation, - const slice &key, +inline cursor::estimate_result::estimate_result(const cursor &cursor, move_operation operation, const slice &key, const slice &value) : pair(key, value), approximate_quantity(PTRDIFF_MIN) { approximate_quantity = cursor.estimate(operation, &this->key, &this->value); } -inline ptrdiff_t cursor::estimate(move_operation operation, MDBX_val *key, - MDBX_val *value) const { +inline ptrdiff_t cursor::estimate(move_operation operation, MDBX_val *key, MDBX_val *value) const { ptrdiff_t result; - error::success_or_throw(::mdbx_estimate_move( - *this, key, value, MDBX_cursor_op(operation), &result)); + error::success_or_throw(::mdbx_estimate_move(*this, key, value, MDBX_cursor_op(operation), &result)); return result; } @@ -7164,37 +6112,27 @@ inline cursor::move_result cursor::find(const slice &key, bool throw_notfound) { return move(key_exact, key, throw_notfound); } -inline cursor::move_result cursor::lower_bound(const slice &key, - bool throw_notfound) { +inline cursor::move_result cursor::lower_bound(const slice &key, bool throw_notfound) { return move(key_lowerbound, key, throw_notfound); } -inline cursor::move_result cursor::upper_bound(const slice &key, - bool throw_notfound) { +inline cursor::move_result cursor::upper_bound(const slice &key, bool throw_notfound) { return move(key_greater_than, key, throw_notfound); } -inline cursor::move_result cursor::find_multivalue(const slice &key, - const slice &value, - bool throw_notfound) { +inline cursor::move_result cursor::find_multivalue(const slice &key, const slice &value, bool throw_notfound) { return move(multi_find_pair, key, value, throw_notfound); } -inline cursor::move_result cursor::lower_bound_multivalue(const slice &key, - const slice &value, - bool throw_notfound) { +inline cursor::move_result cursor::lower_bound_multivalue(const slice &key, const slice &value, bool throw_notfound) { return move(multi_exactkey_lowerboundvalue, key, value, throw_notfound); } -inline cursor::move_result cursor::upper_bound_multivalue(const slice &key, - const slice &value, - bool throw_notfound) { +inline cursor::move_result cursor::upper_bound_multivalue(const slice &key, const slice &value, bool throw_notfound) { return move(multi_exactkey_value_greater, key, value, throw_notfound); } -inline bool cursor::seek(const slice &key) { - return move(seek_key, const_cast(&key), nullptr, false); -} +inline bool cursor::seek(const slice &key) { return move(seek_key, const_cast(&key), nullptr, false); } inline size_t cursor::count_multivalue() const { size_t result; @@ -7202,28 +6140,17 @@ inline size_t cursor::count_multivalue() const { return result; } -inline bool cursor::eof() const { - return error::boolean_or_throw(::mdbx_cursor_eof(*this)); -} +inline bool cursor::eof() const { return error::boolean_or_throw(::mdbx_cursor_eof(*this)); } -inline bool cursor::on_first() const { - return error::boolean_or_throw(::mdbx_cursor_on_first(*this)); -} +inline bool cursor::on_first() const { return error::boolean_or_throw(::mdbx_cursor_on_first(*this)); } -inline bool cursor::on_last() const { - return error::boolean_or_throw(::mdbx_cursor_on_last(*this)); -} +inline bool cursor::on_last() const { return error::boolean_or_throw(::mdbx_cursor_on_last(*this)); } -inline bool cursor::on_first_multival() const { - return error::boolean_or_throw(::mdbx_cursor_on_first_dup(*this)); -} +inline bool cursor::on_first_multival() const { return error::boolean_or_throw(::mdbx_cursor_on_first_dup(*this)); } -inline bool cursor::on_last_multival() const { - return error::boolean_or_throw(::mdbx_cursor_on_last_dup(*this)); -} +inline bool cursor::on_last_multival() const { return error::boolean_or_throw(::mdbx_cursor_on_last_dup(*this)); } -inline cursor::estimate_result cursor::estimate(const slice &key, - const slice &value) const { +inline cursor::estimate_result cursor::estimate(const slice &key, const slice &value) const { return estimate_result(*this, multi_exactkey_lowerboundvalue, key, value); } @@ -7231,23 +6158,17 @@ inline cursor::estimate_result cursor::estimate(const slice &key) const { return estimate_result(*this, key_lowerbound, key); } -inline cursor::estimate_result -cursor::estimate(move_operation operation) const { +inline cursor::estimate_result cursor::estimate(move_operation operation) const { return estimate_result(*this, operation); } -inline void cursor::renew(const ::mdbx::txn &txn) { - error::success_or_throw(::mdbx_cursor_renew(txn, handle_)); -} +inline void cursor::renew(const ::mdbx::txn &txn) { error::success_or_throw(::mdbx_cursor_renew(txn, handle_)); } -inline void cursor::bind(const ::mdbx::txn &txn, - ::mdbx::map_handle map_handle) { +inline void cursor::bind(const ::mdbx::txn &txn, ::mdbx::map_handle map_handle) { error::success_or_throw(::mdbx_cursor_bind(txn, handle_, map_handle.dbi)); } -inline void cursor::unbind() { - error::success_or_throw(::mdbx_cursor_unbind(handle_)); -} +inline void cursor::unbind() { error::success_or_throw(::mdbx_cursor_unbind(handle_)); } inline txn cursor::txn() const { MDBX_txn *txn = ::mdbx_cursor_txn(handle_); @@ -7262,8 +6183,7 @@ inline map_handle cursor::map() const { return map_handle(dbi); } -inline MDBX_error_t cursor::put(const slice &key, slice *value, - MDBX_put_flags_t flags) noexcept { +inline MDBX_error_t cursor::put(const slice &key, slice *value, MDBX_put_flags_t flags) noexcept { return MDBX_error_t(::mdbx_cursor_put(handle_, &key, value, flags)); } @@ -7273,14 +6193,12 @@ inline void cursor::put(const slice &key, slice value, put_mode mode) { inline void cursor::insert(const slice &key, slice value) { error::success_or_throw( - put(key, &value /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique))); + put(key, &value /* takes the present value in case MDBX_KEYEXIST */, MDBX_put_flags_t(put_mode::insert_unique))); } inline value_result cursor::try_insert(const slice &key, slice value) { const int err = - put(key, &value /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique)); + put(key, &value /* takes the present value in case MDBX_KEYEXIST */, MDBX_put_flags_t(put_mode::insert_unique)); switch (err) { case MDBX_SUCCESS: return value_result{slice(), true}; @@ -7293,18 +6211,15 @@ inline value_result cursor::try_insert(const slice &key, slice value) { inline slice cursor::insert_reserve(const slice &key, size_t value_length) { slice result(nullptr, value_length); - error::success_or_throw( - put(key, &result /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE)); + error::success_or_throw(put(key, &result /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE)); return result; } -inline value_result cursor::try_insert_reserve(const slice &key, - size_t value_length) { +inline value_result cursor::try_insert_reserve(const slice &key, size_t value_length) { slice result(nullptr, value_length); - const int err = - put(key, &result /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE); + const int err = put(key, &result /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE); switch (err) { case MDBX_SUCCESS: return value_result{result, true}; @@ -7316,25 +6231,21 @@ inline value_result cursor::try_insert_reserve(const slice &key, } inline void cursor::upsert(const slice &key, const slice &value) { - error::success_or_throw(put(key, const_cast(&value), - MDBX_put_flags_t(put_mode::upsert))); + error::success_or_throw(put(key, const_cast(&value), MDBX_put_flags_t(put_mode::upsert))); } inline slice cursor::upsert_reserve(const slice &key, size_t value_length) { slice result(nullptr, value_length); - error::success_or_throw( - put(key, &result, MDBX_put_flags_t(put_mode::upsert) | MDBX_RESERVE)); + error::success_or_throw(put(key, &result, MDBX_put_flags_t(put_mode::upsert) | MDBX_RESERVE)); return result; } inline void cursor::update(const slice &key, const slice &value) { - error::success_or_throw(put(key, const_cast(&value), - MDBX_put_flags_t(put_mode::update))); + error::success_or_throw(put(key, const_cast(&value), MDBX_put_flags_t(put_mode::update))); } inline bool cursor::try_update(const slice &key, const slice &value) { - const int err = - put(key, const_cast(&value), MDBX_put_flags_t(put_mode::update)); + const int err = put(key, const_cast(&value), MDBX_put_flags_t(put_mode::update)); switch (err) { case MDBX_SUCCESS: return true; @@ -7347,16 +6258,13 @@ inline bool cursor::try_update(const slice &key, const slice &value) { inline slice cursor::update_reserve(const slice &key, size_t value_length) { slice result(nullptr, value_length); - error::success_or_throw( - put(key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE)); + error::success_or_throw(put(key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE)); return result; } -inline value_result cursor::try_update_reserve(const slice &key, - size_t value_length) { +inline value_result cursor::try_update_reserve(const slice &key, size_t value_length) { slice result(nullptr, value_length); - const int err = - put(key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE); + const int err = put(key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE); switch (err) { case MDBX_SUCCESS: return value_result{result, true}; @@ -7368,8 +6276,7 @@ inline value_result cursor::try_update_reserve(const slice &key, } inline bool cursor::erase(bool whole_multivalue) { - const int err = ::mdbx_cursor_del(handle_, whole_multivalue ? MDBX_ALLDUPS - : MDBX_CURRENT); + const int err = ::mdbx_cursor_del(handle_, whole_multivalue ? MDBX_ALLDUPS : MDBX_CURRENT); switch (err) { case MDBX_SUCCESS: MDBX_CXX20_LIKELY return true; @@ -7409,8 +6316,7 @@ inline string to_string(const ::mdbx::slice &value) { } template -inline string -to_string(const ::mdbx::buffer &buffer) { +inline string to_string(const ::mdbx::buffer &buffer) { ostringstream out; out << buffer; return out.str(); @@ -7482,15 +6388,10 @@ inline string to_string(const ::mdbx::error &value) { return out.str(); } -inline string to_string(const ::MDBX_error_t &errcode) { - return to_string(::mdbx::error(errcode)); -} +inline string to_string(const ::MDBX_error_t &errcode) { return to_string(::mdbx::error(errcode)); } template <> struct hash<::mdbx::slice> { - MDBX_CXX14_CONSTEXPR size_t - operator()(::mdbx::slice const &slice) const noexcept { - return slice.hash_value(); - } + MDBX_CXX14_CONSTEXPR size_t operator()(::mdbx::slice const &slice) const noexcept { return slice.hash_value(); } }; /// end cxx_api @} diff --git a/src/api-cursor.c b/src/api-cursor.c index 6bb89cfa..24d21fce 100644 --- a/src/api-cursor.c +++ b/src/api-cursor.c @@ -14,30 +14,24 @@ MDBX_cursor *mdbx_cursor_create(void *context) { couple->userctx = context; couple->outer.top_and_flags = z_poor_mark; couple->inner.cursor.top_and_flags = z_poor_mark | z_inner; - VALGRIND_MAKE_MEM_DEFINED(&couple->outer.backup, - sizeof(couple->outer.backup)); + VALGRIND_MAKE_MEM_DEFINED(&couple->outer.backup, sizeof(couple->outer.backup)); VALGRIND_MAKE_MEM_DEFINED(&couple->outer.tree, sizeof(couple->outer.tree)); VALGRIND_MAKE_MEM_DEFINED(&couple->outer.clc, sizeof(couple->outer.clc)); - VALGRIND_MAKE_MEM_DEFINED(&couple->outer.dbi_state, - sizeof(couple->outer.dbi_state)); - VALGRIND_MAKE_MEM_DEFINED(&couple->outer.subcur, - sizeof(couple->outer.subcur)); + VALGRIND_MAKE_MEM_DEFINED(&couple->outer.dbi_state, sizeof(couple->outer.dbi_state)); + VALGRIND_MAKE_MEM_DEFINED(&couple->outer.subcur, sizeof(couple->outer.subcur)); VALGRIND_MAKE_MEM_DEFINED(&couple->outer.txn, sizeof(couple->outer.txn)); return &couple->outer; } int mdbx_cursor_renew(const MDBX_txn *txn, MDBX_cursor *mc) { - return likely(mc) - ? mdbx_cursor_bind(txn, mc, (kvx_t *)mc->clc - txn->env->kvs) - : LOG_IFERR(MDBX_EINVAL); + return likely(mc) ? mdbx_cursor_bind(txn, mc, (kvx_t *)mc->clc - txn->env->kvs) : LOG_IFERR(MDBX_EINVAL); } int mdbx_cursor_reset(MDBX_cursor *mc) { if (unlikely(!mc)) return LOG_IFERR(MDBX_EINVAL); - if (unlikely(mc->signature != cur_signature_ready4dispose && - mc->signature != cur_signature_live)) + if (unlikely(mc->signature != cur_signature_ready4dispose && mc->signature != cur_signature_live)) return LOG_IFERR(MDBX_EBADSIGN); cursor_couple_t *couple = (cursor_couple_t *)mc; @@ -50,8 +44,7 @@ int mdbx_cursor_bind(const MDBX_txn *txn, MDBX_cursor *mc, MDBX_dbi dbi) { if (unlikely(!mc)) return LOG_IFERR(MDBX_EINVAL); - if (unlikely(mc->signature != cur_signature_ready4dispose && - mc->signature != cur_signature_live)) + if (unlikely(mc->signature != cur_signature_ready4dispose && mc->signature != cur_signature_live)) return LOG_IFERR(MDBX_EBADSIGN); int rc = check_txn(txn, MDBX_TXN_BLOCKED); @@ -68,16 +61,14 @@ int mdbx_cursor_bind(const MDBX_txn *txn, MDBX_cursor *mc, MDBX_dbi dbi) { if (unlikely(mc->backup)) /* Cursor from parent transaction */ { cASSERT(mc, mc->signature == cur_signature_live); if (unlikely(cursor_dbi(mc) != dbi || - /* paranoia */ mc->signature != cur_signature_live || - mc->txn != txn)) + /* paranoia */ mc->signature != cur_signature_live || mc->txn != txn)) return LOG_IFERR(MDBX_EINVAL); cASSERT(mc, mc->tree == &txn->dbs[dbi]); cASSERT(mc, mc->clc == &txn->env->kvs[dbi].clc); cASSERT(mc, cursor_dbi(mc) == dbi); return likely(cursor_dbi(mc) == dbi && - /* paranoia */ mc->signature == cur_signature_live && - mc->txn == txn) + /* paranoia */ mc->signature == cur_signature_live && mc->txn == txn) ? MDBX_SUCCESS : LOG_IFERR(MDBX_EINVAL) /* Disallow change DBI in nested transactions */ @@ -105,9 +96,7 @@ int mdbx_cursor_unbind(MDBX_cursor *mc) { return LOG_IFERR(MDBX_EINVAL); if (unlikely(mc->signature != cur_signature_live)) - return (mc->signature == cur_signature_ready4dispose) - ? MDBX_SUCCESS - : LOG_IFERR(MDBX_EBADSIGN); + return (mc->signature == cur_signature_ready4dispose) ? MDBX_SUCCESS : LOG_IFERR(MDBX_EBADSIGN); if (unlikely(mc->backup)) /* Cursor from parent transaction */ return LOG_IFERR(MDBX_EINVAL); @@ -116,9 +105,7 @@ int mdbx_cursor_unbind(MDBX_cursor *mc) { cASSERT(mc, mc->signature == cur_signature_live); cASSERT(mc, !mc->backup); if (unlikely(!mc->txn || mc->txn->signature != txn_signature)) { - ERROR("Wrong cursor's transaction %p 0x%x", - __Wpedantic_format_voidptr(mc->txn), - mc->txn ? mc->txn->signature : 0); + ERROR("Wrong cursor's transaction %p 0x%x", __Wpedantic_format_voidptr(mc->txn), mc->txn ? mc->txn->signature : 0); return LOG_IFERR(MDBX_PROBLEM); } if (mc->next != mc) { @@ -160,8 +147,7 @@ int mdbx_cursor_open(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_cursor **ret) { void mdbx_cursor_close(MDBX_cursor *mc) { if (likely(mc)) { - ENSURE(nullptr, mc->signature == cur_signature_live || - mc->signature == cur_signature_ready4dispose); + ENSURE(nullptr, mc->signature == cur_signature_live || mc->signature == cur_signature_ready4dispose); MDBX_txn *const txn = mc->txn; if (!mc->backup) { mc->txn = nullptr; @@ -194,9 +180,7 @@ int mdbx_cursor_copy(const MDBX_cursor *src, MDBX_cursor *dest) { if (unlikely(!src)) return LOG_IFERR(MDBX_EINVAL); if (unlikely(src->signature != cur_signature_live)) - return LOG_IFERR((src->signature == cur_signature_ready4dispose) - ? MDBX_EINVAL - : MDBX_EBADSIGN); + return LOG_IFERR((src->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN); int rc = mdbx_cursor_bind(src->txn, dest, cursor_dbi(src)); if (unlikely(rc != MDBX_SUCCESS)) @@ -229,8 +213,7 @@ int mdbx_txn_release_all_cursors(const MDBX_txn *txn, bool unbind) { TXN_FOREACH_DBI_FROM(txn, i, MAIN_DBI) { while (txn->cursors[i]) { MDBX_cursor *mc = txn->cursors[i]; - ENSURE(nullptr, mc->signature == cur_signature_live && - (mc->next != mc) && !mc->backup); + ENSURE(nullptr, mc->signature == cur_signature_live && (mc->next != mc) && !mc->backup); rc = likely(rc < INT_MAX) ? rc + 1 : rc; txn->cursors[i] = mc->next; mc->next = mc; @@ -250,8 +233,7 @@ int mdbx_txn_release_all_cursors(const MDBX_txn *txn, bool unbind) { return rc; } -int mdbx_cursor_compare(const MDBX_cursor *l, const MDBX_cursor *r, - bool ignore_multival) { +int mdbx_cursor_compare(const MDBX_cursor *l, const MDBX_cursor *r, bool ignore_multival) { const int incomparable = INT16_MAX + 1; if (unlikely(!l)) return r ? -incomparable * 9 : 0; @@ -267,8 +249,7 @@ int mdbx_cursor_compare(const MDBX_cursor *l, const MDBX_cursor *r, if (l->txn->env != r->txn->env) return (l->txn->env > r->txn->env) ? incomparable * 7 : -incomparable * 7; if (l->txn->txnid != r->txn->txnid) - return (l->txn->txnid > r->txn->txnid) ? incomparable * 6 - : -incomparable * 6; + return (l->txn->txnid > r->txn->txnid) ? incomparable * 6 : -incomparable * 6; return (l->clc > r->clc) ? incomparable * 5 : -incomparable * 5; } assert(cursor_dbi(l) == cursor_dbi(r)); @@ -333,9 +314,7 @@ int mdbx_cursor_count(const MDBX_cursor *mc, size_t *countp) { return LOG_IFERR(MDBX_EINVAL); if (unlikely(mc->signature != cur_signature_live)) - return LOG_IFERR((mc->signature == cur_signature_ready4dispose) - ? MDBX_EINVAL - : MDBX_EBADSIGN); + return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN); int rc = check_txn(mc->txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) @@ -349,9 +328,8 @@ int mdbx_cursor_count(const MDBX_cursor *mc, size_t *countp) { const page_t *mp = mc->pg[mc->top]; const node_t *node = page_node(mp, mc->ki[mc->top]); cASSERT(mc, node_flags(node) & N_DUP); - *countp = unlikely(mc->subcur->nested_tree.items > PTRDIFF_MAX) - ? PTRDIFF_MAX - : (size_t)mc->subcur->nested_tree.items; + *countp = + unlikely(mc->subcur->nested_tree.items > PTRDIFF_MAX) ? PTRDIFF_MAX : (size_t)mc->subcur->nested_tree.items; } } return MDBX_SUCCESS; @@ -362,9 +340,7 @@ int mdbx_cursor_on_first(const MDBX_cursor *mc) { return LOG_IFERR(MDBX_EINVAL); if (unlikely(mc->signature != cur_signature_live)) - return LOG_IFERR((mc->signature == cur_signature_ready4dispose) - ? MDBX_EINVAL - : MDBX_EBADSIGN); + return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN); for (intptr_t i = 0; i <= mc->top; ++i) { if (mc->ki[i]) @@ -379,9 +355,7 @@ int mdbx_cursor_on_first_dup(const MDBX_cursor *mc) { return LOG_IFERR(MDBX_EINVAL); if (unlikely(mc->signature != cur_signature_live)) - return LOG_IFERR((mc->signature == cur_signature_ready4dispose) - ? MDBX_EINVAL - : MDBX_EBADSIGN); + return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN); if (is_filled(mc) && mc->subcur) { mc = &mc->subcur->cursor; @@ -399,9 +373,7 @@ int mdbx_cursor_on_last(const MDBX_cursor *mc) { return LOG_IFERR(MDBX_EINVAL); if (unlikely(mc->signature != cur_signature_live)) - return LOG_IFERR((mc->signature == cur_signature_ready4dispose) - ? MDBX_EINVAL - : MDBX_EBADSIGN); + return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN); for (intptr_t i = 0; i <= mc->top; ++i) { size_t nkeys = page_numkeys(mc->pg[i]); @@ -417,9 +389,7 @@ int mdbx_cursor_on_last_dup(const MDBX_cursor *mc) { return LOG_IFERR(MDBX_EINVAL); if (unlikely(mc->signature != cur_signature_live)) - return LOG_IFERR((mc->signature == cur_signature_ready4dispose) - ? MDBX_EINVAL - : MDBX_EBADSIGN); + return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN); if (is_filled(mc) && mc->subcur) { mc = &mc->subcur->cursor; @@ -438,22 +408,17 @@ int mdbx_cursor_eof(const MDBX_cursor *mc) { return LOG_IFERR(MDBX_EINVAL); if (unlikely(mc->signature != cur_signature_live)) - return LOG_IFERR((mc->signature == cur_signature_ready4dispose) - ? MDBX_EINVAL - : MDBX_EBADSIGN); + return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN); return is_eof(mc) ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE; } -int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, - MDBX_cursor_op op) { +int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) { if (unlikely(mc == nullptr)) return LOG_IFERR(MDBX_EINVAL); if (unlikely(mc->signature != cur_signature_live)) - return LOG_IFERR((mc->signature == cur_signature_ready4dispose) - ? MDBX_EINVAL - : MDBX_EBADSIGN); + return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN); int rc = check_txn(mc->txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) @@ -465,8 +430,7 @@ int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, return LOG_IFERR(cursor_ops(mc, key, data, op)); } -__hot static int scan_confinue(MDBX_cursor *mc, MDBX_predicate_func *predicate, - void *context, void *arg, MDBX_val *key, +__hot static int scan_confinue(MDBX_cursor *mc, MDBX_predicate_func *predicate, void *context, void *arg, MDBX_val *key, MDBX_val *value, MDBX_cursor_op turn_op) { int rc; switch (turn_op) { @@ -528,22 +492,19 @@ __hot static int scan_confinue(MDBX_cursor *mc, MDBX_predicate_func *predicate, } } -int mdbx_cursor_scan(MDBX_cursor *mc, MDBX_predicate_func *predicate, - void *context, MDBX_cursor_op start_op, +int mdbx_cursor_scan(MDBX_cursor *mc, MDBX_predicate_func *predicate, void *context, MDBX_cursor_op start_op, MDBX_cursor_op turn_op, void *arg) { if (unlikely(!predicate)) return LOG_IFERR(MDBX_EINVAL); - const unsigned valid_start_mask = - 1 << MDBX_FIRST | 1 << MDBX_FIRST_DUP | 1 << MDBX_LAST | - 1 << MDBX_LAST_DUP | 1 << MDBX_GET_CURRENT | 1 << MDBX_GET_MULTIPLE; + const unsigned valid_start_mask = 1 << MDBX_FIRST | 1 << MDBX_FIRST_DUP | 1 << MDBX_LAST | 1 << MDBX_LAST_DUP | + 1 << MDBX_GET_CURRENT | 1 << MDBX_GET_MULTIPLE; if (unlikely(start_op > 30 || ((1 << start_op) & valid_start_mask) == 0)) return LOG_IFERR(MDBX_EINVAL); - const unsigned valid_turn_mask = - 1 << MDBX_NEXT | 1 << MDBX_NEXT_DUP | 1 << MDBX_NEXT_NODUP | - 1 << MDBX_PREV | 1 << MDBX_PREV_DUP | 1 << MDBX_PREV_NODUP | - 1 << MDBX_NEXT_MULTIPLE | 1 << MDBX_PREV_MULTIPLE; + const unsigned valid_turn_mask = 1 << MDBX_NEXT | 1 << MDBX_NEXT_DUP | 1 << MDBX_NEXT_NODUP | 1 << MDBX_PREV | + 1 << MDBX_PREV_DUP | 1 << MDBX_PREV_NODUP | 1 << MDBX_NEXT_MULTIPLE | + 1 << MDBX_PREV_MULTIPLE; if (unlikely(turn_op > 30 || ((1 << turn_op) & valid_turn_mask) == 0)) return LOG_IFERR(MDBX_EINVAL); @@ -551,28 +512,22 @@ int mdbx_cursor_scan(MDBX_cursor *mc, MDBX_predicate_func *predicate, int rc = mdbx_cursor_get(mc, &key, &value, start_op); if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); - return LOG_IFERR( - scan_confinue(mc, predicate, context, arg, &key, &value, turn_op)); + return LOG_IFERR(scan_confinue(mc, predicate, context, arg, &key, &value, turn_op)); } -int mdbx_cursor_scan_from(MDBX_cursor *mc, MDBX_predicate_func *predicate, - void *context, MDBX_cursor_op from_op, MDBX_val *key, - MDBX_val *value, MDBX_cursor_op turn_op, void *arg) { +int mdbx_cursor_scan_from(MDBX_cursor *mc, MDBX_predicate_func *predicate, void *context, MDBX_cursor_op from_op, + MDBX_val *key, MDBX_val *value, MDBX_cursor_op turn_op, void *arg) { if (unlikely(!predicate || !key)) return LOG_IFERR(MDBX_EINVAL); - const unsigned valid_start_mask = - 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY | - 1 << MDBX_GET_MULTIPLE | 1 << MDBX_SET_LOWERBOUND | - 1 << MDBX_SET_UPPERBOUND; - if (unlikely(from_op < MDBX_TO_KEY_LESSER_THAN && - ((1 << from_op) & valid_start_mask) == 0)) + const unsigned valid_start_mask = 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY | + 1 << MDBX_GET_MULTIPLE | 1 << MDBX_SET_LOWERBOUND | 1 << MDBX_SET_UPPERBOUND; + if (unlikely(from_op < MDBX_TO_KEY_LESSER_THAN && ((1 << from_op) & valid_start_mask) == 0)) return LOG_IFERR(MDBX_EINVAL); - const unsigned valid_turn_mask = - 1 << MDBX_NEXT | 1 << MDBX_NEXT_DUP | 1 << MDBX_NEXT_NODUP | - 1 << MDBX_PREV | 1 << MDBX_PREV_DUP | 1 << MDBX_PREV_NODUP | - 1 << MDBX_NEXT_MULTIPLE | 1 << MDBX_PREV_MULTIPLE; + const unsigned valid_turn_mask = 1 << MDBX_NEXT | 1 << MDBX_NEXT_DUP | 1 << MDBX_NEXT_NODUP | 1 << MDBX_PREV | + 1 << MDBX_PREV_DUP | 1 << MDBX_PREV_NODUP | 1 << MDBX_NEXT_MULTIPLE | + 1 << MDBX_PREV_MULTIPLE; if (unlikely(turn_op > 30 || ((1 << turn_op) & valid_turn_mask) == 0)) return LOG_IFERR(MDBX_EINVAL); @@ -588,12 +543,10 @@ int mdbx_cursor_scan_from(MDBX_cursor *mc, MDBX_predicate_func *predicate, if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); } - return LOG_IFERR( - scan_confinue(mc, predicate, context, arg, key, value, turn_op)); + return LOG_IFERR(scan_confinue(mc, predicate, context, arg, key, value, turn_op)); } -int mdbx_cursor_get_batch(MDBX_cursor *mc, size_t *count, MDBX_val *pairs, - size_t limit, MDBX_cursor_op op) { +int mdbx_cursor_get_batch(MDBX_cursor *mc, size_t *count, MDBX_val *pairs, size_t limit, MDBX_cursor_op op) { if (unlikely(!count)) return LOG_IFERR(MDBX_EINVAL); @@ -602,9 +555,7 @@ int mdbx_cursor_get_batch(MDBX_cursor *mc, size_t *count, MDBX_val *pairs, return LOG_IFERR(MDBX_EINVAL); if (unlikely(mc->signature != cur_signature_live)) - return LOG_IFERR((mc->signature == cur_signature_ready4dispose) - ? MDBX_EINVAL - : MDBX_EBADSIGN); + return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN); int rc = check_txn(mc->txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) @@ -661,11 +612,9 @@ int mdbx_cursor_get_batch(MDBX_cursor *mc, size_t *count, MDBX_val *pairs, } mp = mc->pg[mc->top]; - DEBUG("next page is %" PRIaPGNO ", key index %u", mp->pgno, - mc->ki[mc->top]); + DEBUG("next page is %" PRIaPGNO ", key index %u", mp->pgno, mc->ki[mc->top]); if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->pgno, mp->flags); + ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags); rc = MDBX_CORRUPTED; goto bailout; } @@ -686,8 +635,7 @@ int mdbx_cursor_set_userctx(MDBX_cursor *mc, void *ctx) { if (unlikely(!mc)) return LOG_IFERR(MDBX_EINVAL); - if (unlikely(mc->signature != cur_signature_ready4dispose && - mc->signature != cur_signature_live)) + if (unlikely(mc->signature != cur_signature_ready4dispose && mc->signature != cur_signature_live)) return LOG_IFERR(MDBX_EBADSIGN); cursor_couple_t *couple = container_of(mc, cursor_couple_t, outer); @@ -699,8 +647,7 @@ void *mdbx_cursor_get_userctx(const MDBX_cursor *mc) { if (unlikely(!mc)) return nullptr; - if (unlikely(mc->signature != cur_signature_ready4dispose && - mc->signature != cur_signature_live)) + if (unlikely(mc->signature != cur_signature_ready4dispose && mc->signature != cur_signature_live)) return nullptr; cursor_couple_t *couple = container_of(mc, cursor_couple_t, outer); @@ -726,15 +673,12 @@ MDBX_dbi mdbx_cursor_dbi(const MDBX_cursor *mc) { /*----------------------------------------------------------------------------*/ -int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, - MDBX_put_flags_t flags) { +int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, MDBX_put_flags_t flags) { if (unlikely(mc == nullptr || key == nullptr || data == nullptr)) return LOG_IFERR(MDBX_EINVAL); if (unlikely(mc->signature != cur_signature_live)) - return LOG_IFERR((mc->signature == cur_signature_ready4dispose) - ? MDBX_EINVAL - : MDBX_EBADSIGN); + return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN); int rc = check_txn_rw(mc->txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) @@ -754,12 +698,9 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, const size_t dcount = data[1].iov_len; if (unlikely(dcount < 2 || data->iov_len == 0)) return LOG_IFERR(MDBX_BAD_VALSIZE); - if (unlikely(mc->tree->dupfix_size != data->iov_len) && - mc->tree->dupfix_size) + if (unlikely(mc->tree->dupfix_size != data->iov_len) && mc->tree->dupfix_size) return LOG_IFERR(MDBX_BAD_VALSIZE); - if (unlikely(dcount > - MAX_MAPSIZE / 2 / - (BRANCH_NODE_MAX(MDBX_MAX_PAGESIZE) - NODESIZE))) { + if (unlikely(dcount > MAX_MAPSIZE / 2 / (BRANCH_NODE_MAX(MDBX_MAX_PAGESIZE) - NODESIZE))) { /* checking for multiplication overflow */ if (unlikely(dcount > MAX_MAPSIZE / 2 / data->iov_len)) return LOG_IFERR(MDBX_TOO_LARGE); @@ -767,15 +708,13 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, } if (flags & MDBX_RESERVE) { - if (unlikely(mc->tree->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | - MDBX_INTEGERDUP | MDBX_DUPFIXED))) + if (unlikely(mc->tree->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_INTEGERDUP | MDBX_DUPFIXED))) return LOG_IFERR(MDBX_INCOMPATIBLE); data->iov_base = nullptr; } if (unlikely(mc->txn->flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED))) - return LOG_IFERR((mc->txn->flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS - : MDBX_BAD_TXN); + return LOG_IFERR((mc->txn->flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN); return LOG_IFERR(cursor_put_checklen(mc, key, data, flags)); } @@ -785,9 +724,7 @@ int mdbx_cursor_del(MDBX_cursor *mc, MDBX_put_flags_t flags) { return LOG_IFERR(MDBX_EINVAL); if (unlikely(mc->signature != cur_signature_live)) - return LOG_IFERR((mc->signature == cur_signature_ready4dispose) - ? MDBX_EINVAL - : MDBX_EBADSIGN); + return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN); int rc = check_txn_rw(mc->txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) @@ -804,9 +741,7 @@ __cold int mdbx_cursor_ignord(MDBX_cursor *mc) { return LOG_IFERR(MDBX_EINVAL); if (unlikely(mc->signature != cur_signature_live)) - return LOG_IFERR((mc->signature == cur_signature_ready4dispose) - ? MDBX_EINVAL - : MDBX_EBADSIGN); + return LOG_IFERR((mc->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN); mc->checking |= z_ignord; if (mc->subcur) diff --git a/src/api-env.c b/src/api-env.c index be47d566..b9d0488c 100644 --- a/src/api-env.c +++ b/src/api-env.c @@ -7,8 +7,7 @@ __cold static intptr_t reasonable_db_maxsize(void) { static intptr_t cached_result; if (cached_result == 0) { intptr_t pagesize, total_ram_pages; - if (unlikely(mdbx_get_sysraminfo(&pagesize, &total_ram_pages, nullptr) != - MDBX_SUCCESS)) + if (unlikely(mdbx_get_sysraminfo(&pagesize, &total_ram_pages, nullptr) != MDBX_SUCCESS)) /* the 32-bit limit is good enough for fallback */ return cached_result = MAX_MAPSIZE32; @@ -24,8 +23,7 @@ __cold static intptr_t reasonable_db_maxsize(void) { const size_t floor = floor_powerof2(cached_result, unit); const size_t ceil = ceil_powerof2(cached_result, unit); const size_t threshold = (size_t)cached_result >> 4; - const bool down = - cached_result - floor < ceil - cached_result || ceil > MAX_MAPSIZE; + const bool down = cached_result - floor < ceil - cached_result || ceil > MAX_MAPSIZE; if (threshold < (down ? cached_result - floor : ceil - cached_result)) break; cached_result = down ? floor : ceil; @@ -39,14 +37,12 @@ __cold static int check_alternative_lck_absent(const pathchar_t *lck_pathname) { if (unlikely(err != MDBX_RESULT_FALSE)) { if (err == MDBX_RESULT_TRUE) err = MDBX_DUPLICATED_CLK; - ERROR("Alternative/Duplicate LCK-file '%" MDBX_PRIsPATH "' error %d", - lck_pathname, err); + ERROR("Alternative/Duplicate LCK-file '%" MDBX_PRIsPATH "' error %d", lck_pathname, err); } return err; } -__cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname, - const mdbx_mode_t mode) { +__cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname, const mdbx_mode_t mode) { memset(&env->pathname, 0, sizeof(env->pathname)); if (unlikely(!pathname || !*pathname)) return MDBX_EINVAL; @@ -63,8 +59,7 @@ __cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname, return rc; /* auto-create directory if requested */ - if ((env->flags & MDBX_NOSUBDIR) == 0 && - !CreateDirectoryW(pathname, nullptr)) { + if ((env->flags & MDBX_NOSUBDIR) == 0 && !CreateDirectoryW(pathname, nullptr)) { rc = GetLastError(); if (rc != ERROR_ALREADY_EXISTS) return rc; @@ -87,8 +82,7 @@ __cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname, /* auto-create directory if requested */ const mdbx_mode_t dir_mode = - (/* inherit read/write permissions for group and others */ mode & - (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) | + (/* inherit read/write permissions for group and others */ mode & (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) | /* always add read/write/search for owner */ S_IRWXU | ((mode & S_IRGRP) ? /* +search if readable by group */ S_IXGRP : 0) | ((mode & S_IROTH) ? /* +search if readable by others */ S_IXOTH : 0); @@ -120,15 +114,11 @@ __cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname, size_t base_len = pathname_len; static const size_t dxb_name_len = ARRAY_LENGTH(dxb_name) - 1; if (env->flags & MDBX_NOSUBDIR) { - if (base_len > dxb_name_len && - osal_pathequal(pathname + base_len - dxb_name_len, dxb_name, - dxb_name_len)) { + if (base_len > dxb_name_len && osal_pathequal(pathname + base_len - dxb_name_len, dxb_name, dxb_name_len)) { env->flags -= MDBX_NOSUBDIR; base_len -= dxb_name_len; - } else if (base_len == dxb_name_len - 1 && osal_isdirsep(dxb_name[0]) && - osal_isdirsep(lck_name[0]) && - osal_pathequal(pathname + base_len - dxb_name_len + 1, - dxb_name + 1, dxb_name_len - 1)) { + } else if (base_len == dxb_name_len - 1 && osal_isdirsep(dxb_name[0]) && osal_isdirsep(lck_name[0]) && + osal_pathequal(pathname + base_len - dxb_name_len + 1, dxb_name + 1, dxb_name_len - 1)) { env->flags -= MDBX_NOSUBDIR; base_len -= dxb_name_len - 1; } @@ -136,11 +126,9 @@ __cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname, const size_t suflen_with_NOSUBDIR = sizeof(lock_suffix) + sizeof(pathchar_t); const size_t suflen_without_NOSUBDIR = sizeof(lck_name) + sizeof(dxb_name); - const size_t enough4any = (suflen_with_NOSUBDIR > suflen_without_NOSUBDIR) - ? suflen_with_NOSUBDIR - : suflen_without_NOSUBDIR; - const size_t bytes_needed = - sizeof(pathchar_t) * (base_len * 2 + pathname_len + 1) + enough4any; + const size_t enough4any = + (suflen_with_NOSUBDIR > suflen_without_NOSUBDIR) ? suflen_with_NOSUBDIR : suflen_without_NOSUBDIR; + const size_t bytes_needed = sizeof(pathchar_t) * (base_len * 2 + pathname_len + 1) + enough4any; env->pathname.buffer = osal_malloc(bytes_needed); if (!env->pathname.buffer) return MDBX_ENOMEM; @@ -153,8 +141,7 @@ __cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname, if (base_len) { memcpy(buf, pathname, sizeof(pathchar_t) * pathname_len); if (env->flags & MDBX_NOSUBDIR) { - const pathchar_t *const lck_ext = - osal_fileext(lck_name, ARRAY_LENGTH(lck_name)); + const pathchar_t *const lck_ext = osal_fileext(lck_name, ARRAY_LENGTH(lck_name)); if (lck_ext) { pathchar_t *pathname_ext = osal_fileext(buf, pathname_len); memcpy(pathname_ext ? pathname_ext : buf + pathname_len, lck_ext, @@ -181,14 +168,11 @@ __cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname, memcpy(buf + dxb_name_len - 1, lock_suffix, sizeof(lock_suffix)); rc = check_alternative_lck_absent(buf); - memcpy(env->pathname.dxb, dxb_name + 1, - sizeof(dxb_name) - sizeof(pathchar_t)); - memcpy(env->pathname.lck, lck_name + 1, - sizeof(lck_name) - sizeof(pathchar_t)); + memcpy(env->pathname.dxb, dxb_name + 1, sizeof(dxb_name) - sizeof(pathchar_t)); + memcpy(env->pathname.lck, lck_name + 1, sizeof(lck_name) - sizeof(pathchar_t)); } - memcpy(env->pathname.specified, pathname, - sizeof(pathchar_t) * (pathname_len + 1)); + memcpy(env->pathname.specified, pathname, sizeof(pathchar_t) * (pathname_len + 1)); return rc; } @@ -212,8 +196,7 @@ __cold int mdbx_env_create(MDBX_env **penv) { #endif /* MDBX_64BIT_ATOMIC */ #endif /* MDBX_HAVE_C11ATOMICS */ - if (unlikely(!is_powerof2(globals.sys_pagesize) || - globals.sys_pagesize < MDBX_MIN_PAGESIZE)) { + if (unlikely(!is_powerof2(globals.sys_pagesize) || globals.sys_pagesize < MDBX_MIN_PAGESIZE)) { ERROR("unsuitable system pagesize %u", globals.sys_pagesize); return LOG_IFERR(MDBX_INCOMPATIBLE); } @@ -222,10 +205,8 @@ __cold int mdbx_env_create(MDBX_env **penv) { if (unlikely(globals.linux_kernel_version < 0x04000000)) { /* 2022-09-01: Прошло уже более двух лет после окончания какой-либо * поддержки самого "долгоиграющего" ядра 3.16.85 ветки 3.x */ - ERROR("too old linux kernel %u.%u.%u.%u, the >= 4.0.0 is required", - globals.linux_kernel_version >> 24, - (globals.linux_kernel_version >> 16) & 255, - (globals.linux_kernel_version >> 8) & 255, + ERROR("too old linux kernel %u.%u.%u.%u, the >= 4.0.0 is required", globals.linux_kernel_version >> 24, + (globals.linux_kernel_version >> 16) & 255, (globals.linux_kernel_version >> 8) & 255, globals.linux_kernel_version & 255); return LOG_IFERR(MDBX_INCOMPATIBLE); } @@ -237,14 +218,11 @@ __cold int mdbx_env_create(MDBX_env **penv) { env->max_readers = DEFAULT_READERS; env->max_dbi = env->n_dbi = CORE_DBS; - env->lazy_fd = env->dsync_fd = env->fd4meta = env->lck_mmap.fd = - INVALID_HANDLE_VALUE; + env->lazy_fd = env->dsync_fd = env->fd4meta = env->lck_mmap.fd = INVALID_HANDLE_VALUE; env->stuck_meta = -1; env_options_init(env); - env_setup_pagesize(env, (globals.sys_pagesize < MDBX_MAX_PAGESIZE) - ? globals.sys_pagesize - : MDBX_MAX_PAGESIZE); + env_setup_pagesize(env, (globals.sys_pagesize < MDBX_MAX_PAGESIZE) ? globals.sys_pagesize : MDBX_MAX_PAGESIZE); int rc = osal_fastmutex_init(&env->dbi_lock); if (unlikely(rc != MDBX_SUCCESS)) @@ -318,8 +296,7 @@ __cold int mdbx_env_turn_for_recovery(MDBX_env *env, unsigned target) { return LOG_IFERR(meta_override(env, target, new_txnid, target_meta)); } -__cold int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname, - unsigned target_meta, bool writeable) { +__cold int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname, unsigned target_meta, bool writeable) { #if defined(_WIN32) || defined(_WIN64) wchar_t *pathnameW = nullptr; int rc = osal_mb2w(pathname, &pathnameW); @@ -330,8 +307,7 @@ __cold int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname, return LOG_IFERR(rc); } -__cold int mdbx_env_open_for_recoveryW(MDBX_env *env, const wchar_t *pathname, - unsigned target_meta, bool writeable) { +__cold int mdbx_env_open_for_recoveryW(MDBX_env *env, const wchar_t *pathname, unsigned target_meta, bool writeable) { #endif /* Windows */ if (unlikely(target_meta >= NUM_METAS)) @@ -349,8 +325,7 @@ __cold int mdbx_env_open_for_recoveryW(MDBX_env *env, const wchar_t *pathname, #else mdbx_env_open #endif /* Windows */ - (env, pathname, writeable ? MDBX_EXCLUSIVE : MDBX_EXCLUSIVE | MDBX_RDONLY, - 0); + (env, pathname, writeable ? MDBX_EXCLUSIVE : MDBX_EXCLUSIVE | MDBX_RDONLY, 0); } __cold int mdbx_env_delete(const char *pathname, MDBX_env_delete_mode_t mode) { @@ -364,8 +339,7 @@ __cold int mdbx_env_delete(const char *pathname, MDBX_env_delete_mode_t mode) { return LOG_IFERR(rc); } -__cold int mdbx_env_deleteW(const wchar_t *pathname, - MDBX_env_delete_mode_t mode) { +__cold int mdbx_env_deleteW(const wchar_t *pathname, MDBX_env_delete_mode_t mode) { #endif /* Windows */ switch (mode) { @@ -383,22 +357,18 @@ __cold int mdbx_env_deleteW(const wchar_t *pathname, MDBX_env dummy_env_silo, *const dummy_env = &dummy_env_silo; #endif memset(dummy_env, 0, sizeof(*dummy_env)); - dummy_env->flags = - (mode == MDBX_ENV_ENSURE_UNUSED) ? MDBX_EXCLUSIVE : MDBX_ENV_DEFAULTS; + dummy_env->flags = (mode == MDBX_ENV_ENSURE_UNUSED) ? MDBX_EXCLUSIVE : MDBX_ENV_DEFAULTS; dummy_env->ps = (unsigned)mdbx_default_pagesize(); STATIC_ASSERT(sizeof(dummy_env->flags) == sizeof(MDBX_env_flags_t)); int rc = MDBX_RESULT_TRUE, err = env_handle_pathname(dummy_env, pathname, 0); if (likely(err == MDBX_SUCCESS)) { - mdbx_filehandle_t clk_handle = INVALID_HANDLE_VALUE, - dxb_handle = INVALID_HANDLE_VALUE; + mdbx_filehandle_t clk_handle = INVALID_HANDLE_VALUE, dxb_handle = INVALID_HANDLE_VALUE; if (mode > MDBX_ENV_JUST_DELETE) { - err = osal_openfile(MDBX_OPEN_DELETE, dummy_env, dummy_env->pathname.dxb, - &dxb_handle, 0); + err = osal_openfile(MDBX_OPEN_DELETE, dummy_env, dummy_env->pathname.dxb, &dxb_handle, 0); err = (err == MDBX_ENOFILE) ? MDBX_SUCCESS : err; if (err == MDBX_SUCCESS) { - err = osal_openfile(MDBX_OPEN_DELETE, dummy_env, - dummy_env->pathname.lck, &clk_handle, 0); + err = osal_openfile(MDBX_OPEN_DELETE, dummy_env, dummy_env->pathname.lck, &clk_handle, 0); err = (err == MDBX_ENOFILE) ? MDBX_SUCCESS : err; } if (err == MDBX_SUCCESS && clk_handle != INVALID_HANDLE_VALUE) @@ -425,8 +395,7 @@ __cold int mdbx_env_deleteW(const wchar_t *pathname, if (err == MDBX_SUCCESS && !(dummy_env->flags & MDBX_NOSUBDIR) && (/* pathname != "." */ pathname[0] != '.' || pathname[1] != 0) && - (/* pathname != ".." */ pathname[0] != '.' || pathname[1] != '.' || - pathname[2] != 0)) { + (/* pathname != ".." */ pathname[0] != '.' || pathname[1] != '.' || pathname[2] != 0)) { err = osal_removedirectory(pathname); if (err == MDBX_SUCCESS) rc = MDBX_SUCCESS; @@ -445,8 +414,7 @@ __cold int mdbx_env_deleteW(const wchar_t *pathname, return LOG_IFERR((err == MDBX_SUCCESS) ? rc : err); } -__cold int mdbx_env_open(MDBX_env *env, const char *pathname, - MDBX_env_flags_t flags, mdbx_mode_t mode) { +__cold int mdbx_env_open(MDBX_env *env, const char *pathname, MDBX_env_flags_t flags, mdbx_mode_t mode) { #if defined(_WIN32) || defined(_WIN64) wchar_t *pathnameW = nullptr; int rc = osal_mb2w(pathname, &pathnameW); @@ -460,8 +428,7 @@ __cold int mdbx_env_open(MDBX_env *env, const char *pathname, return LOG_IFERR(rc); } -__cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, - MDBX_env_flags_t flags, mdbx_mode_t mode) { +__cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, MDBX_env_flags_t flags, mdbx_mode_t mode) { #endif /* Windows */ int rc = check_env(env, false); @@ -471,8 +438,7 @@ __cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, if (unlikely(flags & ~ENV_USABLE_FLAGS)) return LOG_IFERR(MDBX_EINVAL); - if (unlikely(env->lazy_fd != INVALID_HANDLE_VALUE || - (env->flags & ENV_ACTIVE) != 0 || env->dxb_mmap.base)) + if (unlikely(env->lazy_fd != INVALID_HANDLE_VALUE || (env->flags & ENV_ACTIVE) != 0 || env->dxb_mmap.base)) return LOG_IFERR(MDBX_EPERM); /* Pickup previously mdbx_env_set_flags(), @@ -482,9 +448,8 @@ __cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, if (flags & MDBX_RDONLY) { /* Silently ignore irrelevant flags when we're only getting read access */ - flags &= ~(MDBX_WRITEMAP | DEPRECATED_MAPASYNC | MDBX_SAFE_NOSYNC | - MDBX_NOMETASYNC | DEPRECATED_COALESCE | MDBX_LIFORECLAIM | - MDBX_NOMEMINIT | MDBX_ACCEDE); + flags &= ~(MDBX_WRITEMAP | DEPRECATED_MAPASYNC | MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | DEPRECATED_COALESCE | + MDBX_LIFORECLAIM | MDBX_NOMEMINIT | MDBX_ACCEDE); mode = 0; } else { #if MDBX_MMAP_INCOHERENT_FILE_WRITE @@ -520,16 +485,14 @@ __cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, MDBX_txn *txn = nullptr; const intptr_t bitmap_bytes = #if MDBX_ENABLE_DBI_SPARSE - ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) / - CHAR_BIT; + ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) / CHAR_BIT; #else 0; #endif /* MDBX_ENABLE_DBI_SPARSE */ const size_t base = sizeof(MDBX_txn) + sizeof(cursor_couple_t); - const size_t size = - base + bitmap_bytes + - env->max_dbi * (sizeof(txn->dbs[0]) + sizeof(txn->cursors[0]) + - sizeof(txn->dbi_seqs[0]) + sizeof(txn->dbi_state[0])); + const size_t size = base + bitmap_bytes + + env->max_dbi * (sizeof(txn->dbs[0]) + sizeof(txn->cursors[0]) + sizeof(txn->dbi_seqs[0]) + + sizeof(txn->dbi_state[0])); txn = osal_calloc(1, size); if (unlikely(!txn)) { @@ -538,10 +501,8 @@ __cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, } txn->dbs = ptr_disp(txn, base); txn->cursors = ptr_disp(txn->dbs, env->max_dbi * sizeof(txn->dbs[0])); - txn->dbi_seqs = - ptr_disp(txn->cursors, env->max_dbi * sizeof(txn->cursors[0])); - txn->dbi_state = - ptr_disp(txn, size - env->max_dbi * sizeof(txn->dbi_state[0])); + txn->dbi_seqs = ptr_disp(txn->cursors, env->max_dbi * sizeof(txn->cursors[0])); + txn->dbi_state = ptr_disp(txn, size - env->max_dbi * sizeof(txn->dbi_state[0])); #if MDBX_ENABLE_DBI_SPARSE txn->dbi_sparse = ptr_disp(txn->dbi_state, -bitmap_bytes); #endif /* MDBX_ENABLE_DBI_SPARSE */ @@ -566,10 +527,9 @@ __cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, const meta_ptr_t head = meta_recent(env, &troika); const tree_t *db = &head.ptr_c->trees.main; - DEBUG("opened database version %u, pagesize %u", - (uint8_t)unaligned_peek_u64(4, head.ptr_c->magic_and_version), env->ps); - DEBUG("using meta page %" PRIaPGNO ", txn %" PRIaTXN, - data_page(head.ptr_c)->pgno, head.txnid); + DEBUG("opened database version %u, pagesize %u", (uint8_t)unaligned_peek_u64(4, head.ptr_c->magic_and_version), + env->ps); + DEBUG("using meta page %" PRIaPGNO ", txn %" PRIaTXN, data_page(head.ptr_c)->pgno, head.txnid); DEBUG("depth: %u", db->height); DEBUG("entries: %" PRIu64, db->items); DEBUG("branch pages: %" PRIaPGNO, db->branch_pages); @@ -651,8 +611,7 @@ __cold int mdbx_env_close_ex(MDBX_env *env, bool dont_sync) { env->flags |= ENV_FATAL_ERROR; #endif /* MDBX_ENV_CHECKPID */ - if (env->dxb_mmap.base && - (env->flags & (MDBX_RDONLY | ENV_FATAL_ERROR)) == 0 && env->basal_txn) { + if (env->dxb_mmap.base && (env->flags & (MDBX_RDONLY | ENV_FATAL_ERROR)) == 0 && env->basal_txn) { if (env->basal_txn->owner && env->basal_txn->owner != osal_thread_self()) return LOG_IFERR(MDBX_BUSY); } else @@ -675,8 +634,8 @@ __cold int mdbx_env_close_ex(MDBX_env *env, bool dont_sync) { rc = errno; else if (st.st_nlink > 0 /* don't sync deleted files */) { rc = env_sync(env, true, true); - rc = (rc == MDBX_BUSY || rc == EAGAIN || rc == EACCES || rc == EBUSY || - rc == EWOULDBLOCK || rc == MDBX_RESULT_TRUE) + rc = (rc == MDBX_BUSY || rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || + rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; } @@ -717,8 +676,7 @@ __cold int mdbx_env_close_ex(MDBX_env *env, bool dont_sync) { /*----------------------------------------------------------------------------*/ -static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn, - MDBX_envinfo *out, const size_t bytes, +static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out, const size_t bytes, troika_t *const troika) { const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid); const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat); @@ -752,8 +710,7 @@ static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn, #endif } - *troika = - (txn && !(txn->flags & MDBX_TXN_RDONLY)) ? txn->tw.troika : meta_tap(env); + *troika = (txn && !(txn->flags & MDBX_TXN_RDONLY)) ? txn->tw.troika : meta_tap(env); const meta_ptr_t head = meta_recent(env, troika); const meta_t *const meta0 = METAPAGE(env, 0); const meta_t *const meta1 = METAPAGE(env, 1); @@ -780,9 +737,7 @@ static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn, out->mi_last_pgno = txn->geo.first_unallocated - 1; out->mi_geo.current = pgno2bytes(env, txn->geo.end_pgno); - const txnid_t wanna_meta_txnid = (txn->flags & MDBX_TXN_RDONLY) - ? txn->txnid - : txn->txnid - xMDBX_TXNID_STEP; + const txnid_t wanna_meta_txnid = (txn->flags & MDBX_TXN_RDONLY) ? txn->txnid : txn->txnid - xMDBX_TXNID_STEP; txn_meta = (out->mi_meta_txnid[0] == wanna_meta_txnid) ? meta0 : txn_meta; txn_meta = (out->mi_meta_txnid[1] == wanna_meta_txnid) ? meta1 : txn_meta; txn_meta = (out->mi_meta_txnid[2] == wanna_meta_txnid) ? meta2 : txn_meta; @@ -795,30 +750,23 @@ static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn, const lck_t *const lck = env->lck; out->mi_maxreaders = env->max_readers; - out->mi_numreaders = env->lck_mmap.lck - ? atomic_load32(&lck->rdt_length, mo_Relaxed) - : INT32_MAX; + out->mi_numreaders = env->lck_mmap.lck ? atomic_load32(&lck->rdt_length, mo_Relaxed) : INT32_MAX; out->mi_dxb_pagesize = env->ps; out->mi_sys_pagesize = globals.sys_pagesize; if (likely(bytes > size_before_bootid)) { const uint64_t unsynced_pages = atomic_load64(&lck->unsynced_pages, mo_Relaxed) + - ((uint32_t)out->mi_recent_txnid != - atomic_load32(&lck->meta_sync_txnid, mo_Relaxed)); + ((uint32_t)out->mi_recent_txnid != atomic_load32(&lck->meta_sync_txnid, mo_Relaxed)); out->mi_unsync_volume = pgno2bytes(env, (size_t)unsynced_pages); const uint64_t monotime_now = osal_monotime(); uint64_t ts = atomic_load64(&lck->eoos_timestamp, mo_Relaxed); - out->mi_since_sync_seconds16dot16 = - ts ? osal_monotime_to_16dot16_noUnderflow(monotime_now - ts) : 0; + out->mi_since_sync_seconds16dot16 = ts ? osal_monotime_to_16dot16_noUnderflow(monotime_now - ts) : 0; ts = atomic_load64(&lck->readers_check_timestamp, mo_Relaxed); - out->mi_since_reader_check_seconds16dot16 = - ts ? osal_monotime_to_16dot16_noUnderflow(monotime_now - ts) : 0; - out->mi_autosync_threshold = - pgno2bytes(env, atomic_load32(&lck->autosync_threshold, mo_Relaxed)); + out->mi_since_reader_check_seconds16dot16 = ts ? osal_monotime_to_16dot16_noUnderflow(monotime_now - ts) : 0; + out->mi_autosync_threshold = pgno2bytes(env, atomic_load32(&lck->autosync_threshold, mo_Relaxed)); out->mi_autosync_period_seconds16dot16 = - osal_monotime_to_16dot16_noUnderflow( - atomic_load64(&lck->autosync_period, mo_Relaxed)); + osal_monotime_to_16dot16_noUnderflow(atomic_load64(&lck->autosync_period, mo_Relaxed)); out->mi_bootid.current.x = globals.bootid.x; out->mi_bootid.current.y = globals.bootid.y; out->mi_mode = env->lck_mmap.lck ? lck->envmode.weak : env->flags; @@ -834,8 +782,7 @@ static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn, out->mi_pgop_stat.spill = atomic_load64(&lck->pgops.spill, mo_Relaxed); out->mi_pgop_stat.unspill = atomic_load64(&lck->pgops.unspill, mo_Relaxed); out->mi_pgop_stat.wops = atomic_load64(&lck->pgops.wops, mo_Relaxed); - out->mi_pgop_stat.prefault = - atomic_load64(&lck->pgops.prefault, mo_Relaxed); + out->mi_pgop_stat.prefault = atomic_load64(&lck->pgops.prefault, mo_Relaxed); out->mi_pgop_stat.mincore = atomic_load64(&lck->pgops.mincore, mo_Relaxed); out->mi_pgop_stat.msync = atomic_load64(&lck->pgops.msync, mo_Relaxed); out->mi_pgop_stat.fsync = atomic_load64(&lck->pgops.fsync, mo_Relaxed); @@ -865,8 +812,7 @@ static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn, return MDBX_SUCCESS; } -__cold int env_info(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out, - size_t bytes, troika_t *troika) { +__cold int env_info(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out, size_t bytes, troika_t *troika) { MDBX_envinfo snap; int rc = env_info_snap(env, txn, &snap, sizeof(snap), troika); if (unlikely(rc != MDBX_SUCCESS)) @@ -878,24 +824,22 @@ __cold int env_info(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out, if (unlikely(rc != MDBX_SUCCESS)) return rc; snap.mi_since_sync_seconds16dot16 = out->mi_since_sync_seconds16dot16; - snap.mi_since_reader_check_seconds16dot16 = - out->mi_since_reader_check_seconds16dot16; + snap.mi_since_reader_check_seconds16dot16 = out->mi_since_reader_check_seconds16dot16; if (likely(memcmp(&snap, out, bytes) == 0)) return MDBX_SUCCESS; memcpy(&snap, out, bytes); } } -__cold int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, - MDBX_envinfo *arg, size_t bytes) { +__cold int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *arg, size_t bytes) { if (unlikely((env == nullptr && txn == nullptr) || arg == nullptr)) return LOG_IFERR(MDBX_EINVAL); const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid); const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat); const size_t size_before_dxbid = offsetof(MDBX_envinfo, mi_dxbid); - if (unlikely(bytes != sizeof(MDBX_envinfo)) && bytes != size_before_bootid && - bytes != size_before_pgop_stat && bytes != size_before_dxbid) + if (unlikely(bytes != sizeof(MDBX_envinfo)) && bytes != size_before_bootid && bytes != size_before_pgop_stat && + bytes != size_before_dxbid) return LOG_IFERR(MDBX_EINVAL); if (txn) { @@ -917,8 +861,7 @@ __cold int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, return LOG_IFERR(env_info(env, txn, arg, bytes, &troika)); } -__cold int mdbx_preopen_snapinfo(const char *pathname, MDBX_envinfo *out, - size_t bytes) { +__cold int mdbx_preopen_snapinfo(const char *pathname, MDBX_envinfo *out, size_t bytes) { #if defined(_WIN32) || defined(_WIN64) wchar_t *pathnameW = nullptr; int rc = osal_mb2w(pathname, &pathnameW); @@ -929,8 +872,7 @@ __cold int mdbx_preopen_snapinfo(const char *pathname, MDBX_envinfo *out, return LOG_IFERR(rc); } -__cold int mdbx_preopen_snapinfoW(const wchar_t *pathname, MDBX_envinfo *out, - size_t bytes) { +__cold int mdbx_preopen_snapinfoW(const wchar_t *pathname, MDBX_envinfo *out, size_t bytes) { #endif /* Windows */ if (unlikely(!out)) return LOG_IFERR(MDBX_EINVAL); @@ -938,8 +880,8 @@ __cold int mdbx_preopen_snapinfoW(const wchar_t *pathname, MDBX_envinfo *out, const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid); const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat); const size_t size_before_dxbid = offsetof(MDBX_envinfo, mi_dxbid); - if (unlikely(bytes != sizeof(MDBX_envinfo)) && bytes != size_before_bootid && - bytes != size_before_pgop_stat && bytes != size_before_dxbid) + if (unlikely(bytes != sizeof(MDBX_envinfo)) && bytes != size_before_bootid && bytes != size_before_pgop_stat && + bytes != size_before_dxbid) return LOG_IFERR(MDBX_EINVAL); memset(out, 0, bytes); @@ -951,8 +893,7 @@ __cold int mdbx_preopen_snapinfoW(const wchar_t *pathname, MDBX_envinfo *out, MDBX_env env; memset(&env, 0, sizeof(env)); env.pid = osal_getpid(); - if (unlikely(!is_powerof2(globals.sys_pagesize) || - globals.sys_pagesize < MDBX_MIN_PAGESIZE)) { + if (unlikely(!is_powerof2(globals.sys_pagesize) || globals.sys_pagesize < MDBX_MIN_PAGESIZE)) { ERROR("unsuitable system pagesize %u", globals.sys_pagesize); return LOG_IFERR(MDBX_INCOMPATIBLE); } @@ -972,8 +913,7 @@ __cold int mdbx_preopen_snapinfoW(const wchar_t *pathname, MDBX_envinfo *out, int rc = env_handle_pathname(&env, pathname, 0); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; - rc = osal_openfile(MDBX_OPEN_DXB_READ, &env, env.pathname.dxb, &env.lazy_fd, - 0); + rc = osal_openfile(MDBX_OPEN_DXB_READ, &env, env.pathname.dxb, &env.lazy_fd, 0); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; @@ -1006,10 +946,8 @@ bailout: /*----------------------------------------------------------------------------*/ -__cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, - intptr_t size_now, intptr_t size_upper, - intptr_t growth_step, - intptr_t shrink_threshold, intptr_t pagesize) { +__cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t size_now, intptr_t size_upper, + intptr_t growth_step, intptr_t shrink_threshold, intptr_t pagesize) { int rc = check_env(env, false); if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); @@ -1038,8 +976,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, should_unlock = true; env->basal_txn->tw.troika = meta_tap(env); eASSERT(env, !env->txn && !env->basal_txn->nested); - env->basal_txn->txnid = - env->basal_txn->tw.troika.txnid[env->basal_txn->tw.troika.recent]; + env->basal_txn->txnid = env->basal_txn->tw.troika.txnid[env->basal_txn->tw.troika.recent]; txn_snapshot_oldest(env->basal_txn); } @@ -1047,9 +984,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, if (pagesize <= 0 || pagesize >= INT_MAX) pagesize = env->ps; const geo_t *const geo = - inside_txn - ? &env->txn->geo - : &meta_recent(env, &env->basal_txn->tw.troika).ptr_c->geometry; + inside_txn ? &env->txn->geo : &meta_recent(env, &env->basal_txn->tw.troika).ptr_c->geometry; if (size_lower < 0) size_lower = pgno2bytes(env, geo->lower); if (size_now < 0) @@ -1065,8 +1000,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, rc = MDBX_EINVAL; goto bailout; } - const size_t usedbytes = - pgno2bytes(env, mvcc_snapshot_largest(env, geo->first_unallocated)); + const size_t usedbytes = pgno2bytes(env, mvcc_snapshot_largest(env, geo->first_unallocated)); if ((size_t)size_upper < usedbytes) { rc = MDBX_MAP_FULL; goto bailout; @@ -1101,14 +1035,12 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, else if (top >= (intptr_t)MAX_MAPSIZE /* maximal */) top = MAX_MAPSIZE; - while (top > pagesize * (int64_t)(MAX_PAGENO + 1) && - pagesize < MDBX_MAX_PAGESIZE) + while (top > pagesize * (int64_t)(MAX_PAGENO + 1) && pagesize < MDBX_MAX_PAGESIZE) pagesize <<= 1; } } - if (pagesize < (intptr_t)MDBX_MIN_PAGESIZE || - pagesize > (intptr_t)MDBX_MAX_PAGESIZE || !is_powerof2(pagesize)) { + if (pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || !is_powerof2(pagesize)) { rc = MDBX_EINVAL; goto bailout; } @@ -1140,13 +1072,10 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, size_upper = size_now; else if (size_now >= reasonable_db_maxsize() / 2) size_upper = reasonable_db_maxsize(); - else if ((size_t)size_now >= MAX_MAPSIZE32 / 2 && - (size_t)size_now <= MAX_MAPSIZE32 / 4 * 3) + else if ((size_t)size_now >= MAX_MAPSIZE32 / 2 && (size_t)size_now <= MAX_MAPSIZE32 / 4 * 3) size_upper = MAX_MAPSIZE32; else { - size_upper = ceil_powerof2(((size_t)size_now < MAX_MAPSIZE / 4) - ? size_now + size_now - : size_now + size_now / 2, + size_upper = ceil_powerof2(((size_t)size_now < MAX_MAPSIZE / 4) ? size_now + size_now : size_now + size_now / 2, MEGABYTE * MDBX_WORDBITS * MDBX_WORDBITS / 32); if ((size_t)size_upper > MAX_MAPSIZE) size_upper = MAX_MAPSIZE; @@ -1174,15 +1103,12 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, size_now = size_lower; } - if (unlikely((size_t)size_upper > MAX_MAPSIZE || - (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) { + if (unlikely((size_t)size_upper > MAX_MAPSIZE || (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) { rc = MDBX_TOO_LARGE; goto bailout; } - const size_t unit = (globals.sys_pagesize > (size_t)pagesize) - ? globals.sys_pagesize - : (size_t)pagesize; + const size_t unit = (globals.sys_pagesize > (size_t)pagesize) ? globals.sys_pagesize : (size_t)pagesize; size_lower = ceil_powerof2(size_lower, unit); size_upper = ceil_powerof2(size_upper, unit); size_now = ceil_powerof2(size_now, unit); @@ -1190,10 +1116,8 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, /* LY: подбираем значение size_upper: * - кратное размеру страницы * - без нарушения MAX_MAPSIZE и MAX_PAGENO */ - while (unlikely((size_t)size_upper > MAX_MAPSIZE || - (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) { - if ((size_t)size_upper < unit + MIN_MAPSIZE || - (size_t)size_upper < (size_t)pagesize * (MIN_PAGENO + 1)) { + while (unlikely((size_t)size_upper > MAX_MAPSIZE || (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) { + if ((size_t)size_upper < unit + MIN_MAPSIZE || (size_t)size_upper < (size_t)pagesize * (MIN_PAGENO + 1)) { /* паранойа на случай переполнения при невероятных значениях */ rc = MDBX_EINVAL; goto bailout; @@ -1235,10 +1159,8 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, env->geo_in_bytes.lower = size_lower; env->geo_in_bytes.now = size_now; env->geo_in_bytes.upper = size_upper; - env->geo_in_bytes.grow = - pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, growth_step)))); - env->geo_in_bytes.shrink = - pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, shrink_threshold)))); + env->geo_in_bytes.grow = pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, growth_step)))); + env->geo_in_bytes.shrink = pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, shrink_threshold)))); env_options_adjust_defaults(env); ENSURE(env, env->geo_in_bytes.lower >= MIN_MAPSIZE); @@ -1290,15 +1212,13 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, meta_set_txnid(env, &meta, txnid); } - const geo_t *const current_geo = - &(env->txn ? env->txn : env->basal_txn)->geo; + const geo_t *const current_geo = &(env->txn ? env->txn : env->basal_txn)->geo; /* update env-geo to avoid influences */ env->geo_in_bytes.now = pgno2bytes(env, current_geo->now); env->geo_in_bytes.lower = pgno2bytes(env, current_geo->lower); env->geo_in_bytes.upper = pgno2bytes(env, current_geo->upper); env->geo_in_bytes.grow = pgno2bytes(env, pv2pages(current_geo->grow_pv)); - env->geo_in_bytes.shrink = - pgno2bytes(env, pv2pages(current_geo->shrink_pv)); + env->geo_in_bytes.shrink = pgno2bytes(env, pv2pages(current_geo->shrink_pv)); geo_t new_geo; new_geo.lower = bytes2pgno(env, size_lower); @@ -1326,8 +1246,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, #if defined(_WIN32) || defined(_WIN64) /* Was DB shrinking disabled before and now it will be enabled? */ if (new_geo.lower < new_geo.upper && new_geo.shrink_pv && - !(current_geo->lower < current_geo->upper && - current_geo->shrink_pv)) { + !(current_geo->lower < current_geo->upper && current_geo->shrink_pv)) { if (!env->lck_mmap.lck) { rc = MDBX_EPERM; goto bailout; @@ -1341,9 +1260,7 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, /* Check if there are any reading threads that do not use the SRWL */ const size_t CurrentTid = GetCurrentThreadId(); const reader_slot_t *const begin = env->lck_mmap.lck->rdt; - const reader_slot_t *const end = - begin + - atomic_load32(&env->lck_mmap.lck->rdt_length, mo_AcquireRelease); + const reader_slot_t *const end = begin + atomic_load32(&env->lck_mmap.lck->rdt_length, mo_AcquireRelease); for (const reader_slot_t *reader = begin; reader < end; ++reader) { if (reader->pid.weak == env->pid && reader->tid.weak != CurrentTid) { /* At least one thread may don't use SRWL */ @@ -1358,10 +1275,8 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, } #endif /* Windows */ - if (new_geo.now != current_geo->now || - new_geo.upper != current_geo->upper) { - rc = dxb_resize(env, current_geo->first_unallocated, new_geo.now, - new_geo.upper, explicit_resize); + if (new_geo.now != current_geo->now || new_geo.upper != current_geo->upper) { + rc = dxb_resize(env, current_geo->first_unallocated, new_geo.now, new_geo.upper, explicit_resize); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; } @@ -1370,13 +1285,10 @@ __cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, env->txn->flags |= MDBX_TXN_DIRTY; } else { meta.geometry = new_geo; - rc = - dxb_sync_locked(env, env->flags, &meta, &env->basal_txn->tw.troika); + rc = dxb_sync_locked(env, env->flags, &meta, &env->basal_txn->tw.troika); if (likely(rc == MDBX_SUCCESS)) { - env->geo_in_bytes.now = - pgno2bytes(env, new_geo.now = meta.geometry.now); - env->geo_in_bytes.upper = - pgno2bytes(env, new_geo.upper = meta.geometry.upper); + env->geo_in_bytes.now = pgno2bytes(env, new_geo.now = meta.geometry.now); + env->geo_in_bytes.upper = pgno2bytes(env, new_geo.upper = meta.geometry.upper); } } } diff --git a/src/api-extra.c b/src/api-extra.c index 8c6a6301..e74c3bbc 100644 --- a/src/api-extra.c +++ b/src/api-extra.c @@ -6,8 +6,7 @@ /*------------------------------------------------------------------------------ * Readers API */ -__cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func, - void *ctx) { +__cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func, void *ctx) { int rc = check_env(env, true); if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); @@ -19,8 +18,7 @@ __cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func, int serial = 0; lck_t *const lck = env->lck_mmap.lck; if (likely(lck)) { - const size_t snap_nreaders = - atomic_load32(&lck->rdt_length, mo_AcquireRelease); + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); for (size_t i = 0; i < snap_nreaders; i++) { const reader_slot_t *r = lck->rdt + i; retry_reader:; @@ -29,17 +27,12 @@ __cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func, continue; txnid_t txnid = safe64_read(&r->txnid); const uint64_t tid = atomic_load64(&r->tid, mo_Relaxed); - const pgno_t pages_used = - atomic_load32(&r->snapshot_pages_used, mo_Relaxed); - const uint64_t reader_pages_retired = - atomic_load64(&r->snapshot_pages_retired, mo_Relaxed); - if (unlikely(txnid != safe64_read(&r->txnid) || - pid != atomic_load32(&r->pid, mo_AcquireRelease) || + const pgno_t pages_used = atomic_load32(&r->snapshot_pages_used, mo_Relaxed); + const uint64_t reader_pages_retired = atomic_load64(&r->snapshot_pages_retired, mo_Relaxed); + if (unlikely(txnid != safe64_read(&r->txnid) || pid != atomic_load32(&r->pid, mo_AcquireRelease) || tid != atomic_load64(&r->tid, mo_Relaxed) || - pages_used != - atomic_load32(&r->snapshot_pages_used, mo_Relaxed) || - reader_pages_retired != - atomic_load64(&r->snapshot_pages_retired, mo_Relaxed))) + pages_used != atomic_load32(&r->snapshot_pages_used, mo_Relaxed) || + reader_pages_retired != atomic_load64(&r->snapshot_pages_retired, mo_Relaxed))) goto retry_reader; eASSERT(env, txnid > 0); @@ -53,22 +46,18 @@ __cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func, troika_t troika = meta_tap(env); retry_header:; const meta_ptr_t head = meta_recent(env, &troika); - const uint64_t head_pages_retired = - unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired); + const uint64_t head_pages_retired = unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired); if (unlikely(meta_should_retry(env, &troika) || - head_pages_retired != unaligned_peek_u64_volatile( - 4, head.ptr_v->pages_retired))) + head_pages_retired != unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired))) goto retry_header; lag = (head.txnid - txnid) / xMDBX_TXNID_STEP; bytes_used = pgno2bytes(env, pages_used); bytes_retained = (head_pages_retired > reader_pages_retired) - ? pgno2bytes(env, (pgno_t)(head_pages_retired - - reader_pages_retired)) + ? pgno2bytes(env, (pgno_t)(head_pages_retired - reader_pages_retired)) : 0; } - rc = func(ctx, ++serial, (unsigned)i, pid, (mdbx_tid_t)((intptr_t)tid), - txnid, lag, bytes_used, bytes_retained); + rc = func(ctx, ++serial, (unsigned)i, pid, (mdbx_tid_t)((intptr_t)tid), txnid, lag, bytes_used, bytes_retained); if (unlikely(rc != MDBX_SUCCESS)) break; } @@ -93,8 +82,7 @@ int mdbx_txn_lock(MDBX_env *env, bool dont_wait) { if (unlikely(env->flags & MDBX_RDONLY)) return LOG_IFERR(MDBX_EACCESS); - if (unlikely(env->basal_txn->owner || - (env->basal_txn->flags & MDBX_TXN_FINISHED) == 0)) + if (unlikely(env->basal_txn->owner || (env->basal_txn->flags & MDBX_TXN_FINISHED) == 0)) return LOG_IFERR(MDBX_BUSY); return LOG_IFERR(lck_txn_lock(env, dont_wait)); diff --git a/src/api-key-transform.c b/src/api-key-transform.c index e28f8de9..6143a62e 100644 --- a/src/api-key-transform.c +++ b/src/api-key-transform.c @@ -9,16 +9,14 @@ static inline double key2double(const int64_t key) { double f; } casting; - casting.u = (key < 0) ? key + UINT64_C(0x8000000000000000) - : UINT64_C(0xffffFFFFffffFFFF) - key; + casting.u = (key < 0) ? key + UINT64_C(0x8000000000000000) : UINT64_C(0xffffFFFFffffFFFF) - key; return casting.f; } static inline uint64_t double2key(const double *const ptr) { STATIC_ASSERT(sizeof(double) == sizeof(int64_t)); const int64_t i = *(const int64_t *)ptr; - const uint64_t u = (i < 0) ? UINT64_C(0xffffFFFFffffFFFF) - i - : i + UINT64_C(0x8000000000000000); + const uint64_t u = (i < 0) ? UINT64_C(0xffffFFFFffffFFFF) - i : i + UINT64_C(0x8000000000000000); if (ASSERT_ENABLED()) { const double f = key2double(u); assert(memcmp(&f, ptr, sizeof(double)) == 0); @@ -32,16 +30,14 @@ static inline float key2float(const int32_t key) { float f; } casting; - casting.u = - (key < 0) ? key + UINT32_C(0x80000000) : UINT32_C(0xffffFFFF) - key; + casting.u = (key < 0) ? key + UINT32_C(0x80000000) : UINT32_C(0xffffFFFF) - key; return casting.f; } static inline uint32_t float2key(const float *const ptr) { STATIC_ASSERT(sizeof(float) == sizeof(int32_t)); const int32_t i = *(const int32_t *)ptr; - const uint32_t u = - (i < 0) ? UINT32_C(0xffffFFFF) - i : i + UINT32_C(0x80000000); + const uint32_t u = (i < 0) ? UINT32_C(0xffffFFFF) - i : i + UINT32_C(0x80000000); if (ASSERT_ENABLED()) { const float f = key2float(u); assert(memcmp(&f, ptr, sizeof(float)) == 0); @@ -49,21 +45,13 @@ static inline uint32_t float2key(const float *const ptr) { return u; } -uint64_t mdbx_key_from_double(const double ieee754_64bit) { - return double2key(&ieee754_64bit); -} +uint64_t mdbx_key_from_double(const double ieee754_64bit) { return double2key(&ieee754_64bit); } -uint64_t mdbx_key_from_ptrdouble(const double *const ieee754_64bit) { - return double2key(ieee754_64bit); -} +uint64_t mdbx_key_from_ptrdouble(const double *const ieee754_64bit) { return double2key(ieee754_64bit); } -uint32_t mdbx_key_from_float(const float ieee754_32bit) { - return float2key(&ieee754_32bit); -} +uint32_t mdbx_key_from_float(const float ieee754_32bit) { return float2key(&ieee754_32bit); } -uint32_t mdbx_key_from_ptrfloat(const float *const ieee754_32bit) { - return float2key(ieee754_32bit); -} +uint32_t mdbx_key_from_ptrfloat(const float *const ieee754_32bit) { return float2key(ieee754_32bit); } #define IEEE754_DOUBLE_MANTISSA_SIZE 52 #define IEEE754_DOUBLE_EXPONENTA_BIAS 0x3FF @@ -78,8 +66,7 @@ static inline int clz64(uint64_t value) { return __builtin_clz(value); if (sizeof(value) == sizeof(long)) return __builtin_clzl(value); -#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || \ - __has_builtin(__builtin_clzll) +#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || __has_builtin(__builtin_clzll) return __builtin_clzll(value); #endif /* have(long long) && long long == uint64_t */ #endif /* GNU C */ @@ -105,11 +92,10 @@ static inline int clz64(uint64_t value) { value |= value >> 8; value |= value >> 16; value |= value >> 32; - static const uint8_t debruijn_clz64[64] = { - 63, 16, 62, 7, 15, 36, 61, 3, 6, 14, 22, 26, 35, 47, 60, 2, - 9, 5, 28, 11, 13, 21, 42, 19, 25, 31, 34, 40, 46, 52, 59, 1, - 17, 8, 37, 4, 23, 27, 48, 10, 29, 12, 43, 20, 32, 41, 53, 18, - 38, 24, 49, 30, 44, 33, 54, 39, 50, 45, 55, 51, 56, 57, 58, 0}; + static const uint8_t debruijn_clz64[64] = {63, 16, 62, 7, 15, 36, 61, 3, 6, 14, 22, 26, 35, 47, 60, 2, + 9, 5, 28, 11, 13, 21, 42, 19, 25, 31, 34, 40, 46, 52, 59, 1, + 17, 8, 37, 4, 23, 27, 48, 10, 29, 12, 43, 20, 32, 41, 53, 18, + 38, 24, 49, 30, 44, 33, 54, 39, 50, 45, 55, 51, 56, 57, 58, 0}; return debruijn_clz64[value * UINT64_C(0x03F79D71B4CB0A89) >> 58]; } @@ -134,17 +120,12 @@ uint64_t mdbx_key_from_jsonInteger(const int64_t json_integer) { mantissa = round_mantissa(u64, --shift); } - assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD && - mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX); - const uint64_t exponent = (uint64_t)IEEE754_DOUBLE_EXPONENTA_BIAS + - IEEE754_DOUBLE_MANTISSA_SIZE - shift; + assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD && mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX); + const uint64_t exponent = (uint64_t)IEEE754_DOUBLE_EXPONENTA_BIAS + IEEE754_DOUBLE_MANTISSA_SIZE - shift; assert(exponent > 0 && exponent <= IEEE754_DOUBLE_EXPONENTA_MAX); - const uint64_t key = bias + (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) + - (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD); -#if !defined(_MSC_VER) || \ - defined( \ - _DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \ - symbol __except1 referenced in function __ftol3_except */ + const uint64_t key = bias + (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) + (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD); +#if !defined(_MSC_VER) || defined(_DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \ + symbol __except1 referenced in function __ftol3_except */ assert(key == mdbx_key_from_double((double)json_integer)); #endif /* Workaround for MSVC */ return key; @@ -160,17 +141,13 @@ uint64_t mdbx_key_from_jsonInteger(const int64_t json_integer) { mantissa = round_mantissa(u64, --shift); } - assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD && - mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX); - const uint64_t exponent = (uint64_t)IEEE754_DOUBLE_EXPONENTA_BIAS + - IEEE754_DOUBLE_MANTISSA_SIZE - shift; + assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD && mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX); + const uint64_t exponent = (uint64_t)IEEE754_DOUBLE_EXPONENTA_BIAS + IEEE754_DOUBLE_MANTISSA_SIZE - shift; assert(exponent > 0 && exponent <= IEEE754_DOUBLE_EXPONENTA_MAX); - const uint64_t key = bias - 1 - (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) - - (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD); -#if !defined(_MSC_VER) || \ - defined( \ - _DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \ - symbol __except1 referenced in function __ftol3_except */ + const uint64_t key = + bias - 1 - (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) - (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD); +#if !defined(_MSC_VER) || defined(_DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \ + symbol __except1 referenced in function __ftol3_except */ assert(key == mdbx_key_from_double((double)json_integer)); #endif /* Workaround for MSVC */ return key; @@ -185,21 +162,17 @@ int64_t mdbx_jsonInteger_from_key(const MDBX_val v) { const uint64_t bias = UINT64_C(0x8000000000000000); const uint64_t covalent = (key > bias) ? key - bias : bias - key - 1; const int shift = IEEE754_DOUBLE_EXPONENTA_BIAS + 63 - - (IEEE754_DOUBLE_EXPONENTA_MAX & - (int)(covalent >> IEEE754_DOUBLE_MANTISSA_SIZE)); + (IEEE754_DOUBLE_EXPONENTA_MAX & (int)(covalent >> IEEE754_DOUBLE_MANTISSA_SIZE)); if (unlikely(shift < 1)) return (key < bias) ? INT64_MIN : INT64_MAX; if (unlikely(shift > 63)) return 0; - const uint64_t unscaled = ((covalent & IEEE754_DOUBLE_MANTISSA_MASK) - << (63 - IEEE754_DOUBLE_MANTISSA_SIZE)) + - bias; + const uint64_t unscaled = ((covalent & IEEE754_DOUBLE_MANTISSA_MASK) << (63 - IEEE754_DOUBLE_MANTISSA_SIZE)) + bias; const int64_t absolute = unscaled >> shift; const int64_t value = (key < bias) ? -absolute : absolute; assert(key == mdbx_key_from_jsonInteger(value) || - (mdbx_key_from_jsonInteger(value - 1) < key && - key < mdbx_key_from_jsonInteger(value + 1))); + (mdbx_key_from_jsonInteger(value - 1) < key && key < mdbx_key_from_jsonInteger(value + 1))); return value; } @@ -220,6 +193,5 @@ int32_t mdbx_int32_from_key(const MDBX_val v) { int64_t mdbx_int64_from_key(const MDBX_val v) { assert(v.iov_len == 8); - return (int64_t)(unaligned_peek_u64(2, v.iov_base) - - UINT64_C(0x8000000000000000)); + return (int64_t)(unaligned_peek_u64(2, v.iov_base) - UINT64_C(0x8000000000000000)); } diff --git a/src/api-txn.c b/src/api-txn.c index d75f05ff..2b3ba0d8 100644 --- a/src/api-txn.c +++ b/src/api-txn.c @@ -16,9 +16,7 @@ int mdbx_txn_straggler(const MDBX_txn *txn, int *percent) MDBX_env *env = txn->env; if (unlikely((txn->flags & MDBX_TXN_RDONLY) == 0)) { if (percent) - *percent = (int)((txn->geo.first_unallocated * UINT64_C(100) + - txn->geo.end_pgno / 2) / - txn->geo.end_pgno); + *percent = (int)((txn->geo.first_unallocated * UINT64_C(100) + txn->geo.end_pgno / 2) / txn->geo.end_pgno); return 0; } @@ -28,9 +26,7 @@ int mdbx_txn_straggler(const MDBX_txn *txn, int *percent) const meta_ptr_t head = meta_recent(env, &troika); if (percent) { const pgno_t maxpg = head.ptr_v->geometry.now; - *percent = (int)((head.ptr_v->geometry.first_unallocated * UINT64_C(100) + - maxpg / 2) / - maxpg); + *percent = (int)((head.ptr_v->geometry.first_unallocated * UINT64_C(100) + maxpg / 2) / maxpg); } lag = (head.txnid - txn->txnid) / xMDBX_TXNID_STEP; } while (unlikely(meta_should_retry(env, &troika))); @@ -38,8 +34,7 @@ int mdbx_txn_straggler(const MDBX_txn *txn, int *percent) return (lag > INT_MAX) ? INT_MAX : (int)lag; } -__cold int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi, - uint32_t *mask) { +__cold int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi, uint32_t *mask) { if (unlikely(!mask)) return LOG_IFERR(MDBX_EINVAL); @@ -58,8 +53,7 @@ __cold int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val key, data; rc = outer_first(&cx.outer, &key, &data); while (rc == MDBX_SUCCESS) { - const node_t *node = - page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]); + const node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]); const tree_t *db = node_data(node); const unsigned flags = node_flags(node); switch (flags) { @@ -77,8 +71,7 @@ __cold int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi, *mask |= 1 << UNALIGNED_PEEK_16(db, tree_t, height); break; default: - ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid node-size", flags); + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid node-size", flags); return LOG_IFERR(MDBX_CORRUPTED); } rc = outer_next(&cx.outer, &key, &data, MDBX_NEXT_NODUP); @@ -101,8 +94,7 @@ int mdbx_canary_get(const MDBX_txn *txn, MDBX_canary *canary) { return MDBX_SUCCESS; } -int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - MDBX_val *data) { +int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data) { DKBUF_DEBUG; DEBUG("===> get db %u key [%s]", dbi, DKEY_DEBUG(key)); @@ -121,8 +113,7 @@ int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, return LOG_IFERR(cursor_seek(&cx.outer, (MDBX_val *)key, data, MDBX_SET).err); } -int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, - MDBX_val *data) { +int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data) { int rc = check_txn(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); @@ -141,8 +132,7 @@ int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, return LOG_IFERR(cursor_ops(&cx.outer, key, data, MDBX_SET_LOWERBOUND)); } -int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, - MDBX_val *data, size_t *values_count) { +int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, size_t *values_count) { DKBUF_DEBUG; DEBUG("===> get db %u key [%s]", dbi, DKEY_DEBUG(key)); @@ -169,8 +159,7 @@ int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, *values_count = 1; if (inner_pointed(&cx.outer)) *values_count = - (sizeof(*values_count) >= sizeof(cx.inner.nested_tree.items) || - cx.inner.nested_tree.items <= PTRDIFF_MAX) + (sizeof(*values_count) >= sizeof(cx.inner.nested_tree.items) || cx.inner.nested_tree.items <= PTRDIFF_MAX) ? (size_t)cx.inner.nested_tree.items : PTRDIFF_MAX; } @@ -185,8 +174,7 @@ int mdbx_canary_put(MDBX_txn *txn, const MDBX_canary *canary) { return LOG_IFERR(rc); if (likely(canary)) { - if (txn->canary.x == canary->x && txn->canary.y == canary->y && - txn->canary.z == canary->z) + if (txn->canary.x == canary->x && txn->canary.y == canary->y && txn->canary.z == canary->z) return MDBX_SUCCESS; txn->canary.x = canary->x; txn->canary.y = canary->y; @@ -236,17 +224,14 @@ int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr) { * not to the beginning of a data. */ return LOG_IFERR(MDBX_EINVAL); } - return ((txn->flags & MDBX_TXN_RDONLY) || !is_modifable(txn, page)) - ? MDBX_RESULT_FALSE - : MDBX_RESULT_TRUE; + return ((txn->flags & MDBX_TXN_RDONLY) || !is_modifable(txn, page)) ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE; } if ((size_t)offset < env->dxb_mmap.limit) { /* Указатель адресует что-то в пределах mmap, но за границей * распределенных страниц. Такое может случится если mdbx_is_dirty() * вызывается после операции, в ходе которой грязная страница была * возвращена в нераспределенное пространство. */ - return (txn->flags & MDBX_TXN_RDONLY) ? LOG_IFERR(MDBX_EINVAL) - : MDBX_RESULT_TRUE; + return (txn->flags & MDBX_TXN_RDONLY) ? LOG_IFERR(MDBX_EINVAL) : MDBX_RESULT_TRUE; } } @@ -256,13 +241,10 @@ int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr) { * * Для режима MDBX_WRITE_MAP режима страница однозначно "не грязная", * а для режимов без MDBX_WRITE_MAP однозначно "не чистая". */ - return (txn->flags & (MDBX_WRITEMAP | MDBX_TXN_RDONLY)) - ? LOG_IFERR(MDBX_EINVAL) - : MDBX_RESULT_TRUE; + return (txn->flags & (MDBX_WRITEMAP | MDBX_TXN_RDONLY)) ? LOG_IFERR(MDBX_EINVAL) : MDBX_RESULT_TRUE; } -int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - const MDBX_val *data) { +int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, const MDBX_val *data) { int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); @@ -274,8 +256,7 @@ int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, return LOG_IFERR(MDBX_BAD_DBI); if (unlikely(txn->flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED))) - return LOG_IFERR((txn->flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS - : MDBX_BAD_TXN); + return LOG_IFERR((txn->flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN); cursor_couple_t cx; rc = cursor_init(&cx.outer, txn, dbi); @@ -302,8 +283,7 @@ int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, return LOG_IFERR(rc); } -int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data, - MDBX_put_flags_t flags) { +int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data, MDBX_put_flags_t flags) { int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); @@ -314,14 +294,12 @@ int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data, if (unlikely(dbi <= FREE_DBI)) return LOG_IFERR(MDBX_BAD_DBI); - if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS | - MDBX_ALLDUPS | MDBX_RESERVE | MDBX_APPEND | + if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS | MDBX_ALLDUPS | MDBX_RESERVE | MDBX_APPEND | MDBX_APPENDDUP | MDBX_CURRENT | MDBX_MULTIPLE))) return LOG_IFERR(MDBX_EINVAL); if (unlikely(txn->flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED))) - return LOG_IFERR((txn->flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS - : MDBX_BAD_TXN); + return LOG_IFERR((txn->flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN); cursor_couple_t cx; rc = cursor_init(&cx.outer, txn, dbi); @@ -333,14 +311,11 @@ int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data, /* LY: support for update (explicit overwrite) */ if (flags & MDBX_CURRENT) { rc = cursor_seek(&cx.outer, (MDBX_val *)key, nullptr, MDBX_SET).err; - if (likely(rc == MDBX_SUCCESS) && (txn->dbs[dbi].flags & MDBX_DUPSORT) && - (flags & MDBX_ALLDUPS) == 0) { + if (likely(rc == MDBX_SUCCESS) && (txn->dbs[dbi].flags & MDBX_DUPSORT) && (flags & MDBX_ALLDUPS) == 0) { /* LY: allows update (explicit overwrite) only for unique keys */ - node_t *node = - page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]); + node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]); if (node_flags(node) & N_DUP) { - tASSERT(txn, inner_pointed(&cx.outer) && - cx.outer.subcur->nested_tree.items > 1); + tASSERT(txn, inner_pointed(&cx.outer) && cx.outer.subcur->nested_tree.items > 1); rc = MDBX_EMULTIVAL; if ((flags & MDBX_NOOVERWRITE) == 0) { flags -= MDBX_CURRENT; @@ -383,10 +358,8 @@ int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data, * - получения dirty-статуса страницы по адресу (знать о MUTABLE/WRITEABLE). */ -int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - MDBX_val *new_data, MDBX_val *old_data, - MDBX_put_flags_t flags, MDBX_preserve_func preserver, - void *preserver_context) { +int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *new_data, MDBX_val *old_data, + MDBX_put_flags_t flags, MDBX_preserve_func preserver, void *preserver_context) { int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); @@ -397,16 +370,14 @@ int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, if (unlikely(old_data->iov_base == nullptr && old_data->iov_len)) return LOG_IFERR(MDBX_EINVAL); - if (unlikely(new_data == nullptr && - (flags & (MDBX_CURRENT | MDBX_RESERVE)) != MDBX_CURRENT)) + if (unlikely(new_data == nullptr && (flags & (MDBX_CURRENT | MDBX_RESERVE)) != MDBX_CURRENT)) return LOG_IFERR(MDBX_EINVAL); if (unlikely(dbi <= FREE_DBI)) return LOG_IFERR(MDBX_BAD_DBI); - if (unlikely(flags & - ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS | - MDBX_RESERVE | MDBX_APPEND | MDBX_APPENDDUP | MDBX_CURRENT))) + if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS | MDBX_RESERVE | MDBX_APPEND | + MDBX_APPENDDUP | MDBX_CURRENT))) return LOG_IFERR(MDBX_EINVAL); cursor_couple_t cx; @@ -452,8 +423,7 @@ int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, /* disallow update/delete for multi-values */ node_t *node = page_node(page, cx.outer.ki[cx.outer.top]); if (node_flags(node) & N_DUP) { - tASSERT(txn, inner_pointed(&cx.outer) && - cx.outer.subcur->nested_tree.items > 1); + tASSERT(txn, inner_pointed(&cx.outer) && cx.outer.subcur->nested_tree.items > 1); if (cx.outer.subcur->nested_tree.items > 1) { rc = MDBX_EMULTIVAL; goto bailout; @@ -472,8 +442,7 @@ int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, *old_data = *new_data; goto bailout; } - rc = preserver ? preserver(preserver_context, old_data, - present_data.iov_base, present_data.iov_len) + rc = preserver ? preserver(preserver_context, old_data, present_data.iov_base, present_data.iov_len) : MDBX_SUCCESS; if (unlikely(rc != MDBX_SUCCESS)) goto bailout; @@ -494,8 +463,7 @@ bailout: return LOG_IFERR(rc); } -static int default_value_preserver(void *context, MDBX_val *target, - const void *src, size_t bytes) { +static int default_value_preserver(void *context, MDBX_val *target, const void *src, size_t bytes) { (void)context; if (unlikely(target->iov_len < bytes)) { target->iov_base = nullptr; @@ -506,9 +474,7 @@ static int default_value_preserver(void *context, MDBX_val *target, return MDBX_SUCCESS; } -int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - MDBX_val *new_data, MDBX_val *old_data, +int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *new_data, MDBX_val *old_data, MDBX_put_flags_t flags) { - return mdbx_replace_ex(txn, dbi, key, new_data, old_data, flags, - default_value_preserver, nullptr); + return mdbx_replace_ex(txn, dbi, key, new_data, old_data, flags, default_value_preserver, nullptr); } diff --git a/src/atomics-ops.h b/src/atomics-ops.h index 0b29cb84..c61c055d 100644 --- a/src/atomics-ops.h +++ b/src/atomics-ops.h @@ -8,43 +8,36 @@ #ifndef __cplusplus #ifdef MDBX_HAVE_C11ATOMICS -#define osal_memory_fence(order, write) \ - atomic_thread_fence((write) ? mo_c11_store(order) : mo_c11_load(order)) +#define osal_memory_fence(order, write) atomic_thread_fence((write) ? mo_c11_store(order) : mo_c11_load(order)) #else /* MDBX_HAVE_C11ATOMICS */ -#define osal_memory_fence(order, write) \ - do { \ - osal_compiler_barrier(); \ - if (write && order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed \ - : mo_AcquireRelease)) \ - osal_memory_barrier(); \ +#define osal_memory_fence(order, write) \ + do { \ + osal_compiler_barrier(); \ + if (write && order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed : mo_AcquireRelease)) \ + osal_memory_barrier(); \ } while (0) #endif /* MDBX_HAVE_C11ATOMICS */ #if defined(MDBX_HAVE_C11ATOMICS) && defined(__LCC__) -#define atomic_store32(p, value, order) \ - ({ \ - const uint32_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ +#define atomic_store32(p, value, order) \ + ({ \ + const uint32_t value_to_store = (value); \ + atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value_to_store, mo_c11_store(order)); \ + value_to_store; \ }) -#define atomic_load32(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)) -#define atomic_store64(p, value, order) \ - ({ \ - const uint64_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ +#define atomic_load32(p, order) atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)) +#define atomic_store64(p, value, order) \ + ({ \ + const uint64_t value_to_store = (value); \ + atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value_to_store, mo_c11_store(order)); \ + value_to_store; \ }) -#define atomic_load64(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)) +#define atomic_load64(p, order) atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)) #endif /* LCC && MDBX_HAVE_C11ATOMICS */ #ifndef atomic_store32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t -atomic_store32(mdbx_atomic_uint32_t *p, const uint32_t value, - enum mdbx_memory_order order) { +MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_store32(mdbx_atomic_uint32_t *p, const uint32_t value, + enum mdbx_memory_order order) { STATIC_ASSERT(sizeof(mdbx_atomic_uint32_t) == 4); #ifdef MDBX_HAVE_C11ATOMICS assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); @@ -60,8 +53,8 @@ atomic_store32(mdbx_atomic_uint32_t *p, const uint32_t value, #endif /* atomic_store32 */ #ifndef atomic_load32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( - const volatile mdbx_atomic_uint32_t *p, enum mdbx_memory_order order) { +MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32(const volatile mdbx_atomic_uint32_t *p, + enum mdbx_memory_order order) { STATIC_ASSERT(sizeof(mdbx_atomic_uint32_t) == 4); #ifdef MDBX_HAVE_C11ATOMICS assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p))); @@ -90,9 +83,8 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( #endif /* xMDBX_TXNID_STEP */ #ifndef atomic_store64 -MDBX_MAYBE_UNUSED static __always_inline uint64_t -atomic_store64(mdbx_atomic_uint64_t *p, const uint64_t value, - enum mdbx_memory_order order) { +MDBX_MAYBE_UNUSED static __always_inline uint64_t atomic_store64(mdbx_atomic_uint64_t *p, const uint64_t value, + enum mdbx_memory_order order) { STATIC_ASSERT(sizeof(mdbx_atomic_uint64_t) == 8); #if MDBX_64BIT_ATOMIC #if __GNUC_PREREQ(11, 0) @@ -124,8 +116,7 @@ MDBX_MAYBE_UNUSED static __always_inline #endif /* MDBX_64BIT_ATOMIC */ uint64_t - atomic_load64(const volatile mdbx_atomic_uint64_t *p, - enum mdbx_memory_order order) { + atomic_load64(const volatile mdbx_atomic_uint64_t *p, enum mdbx_memory_order order) { STATIC_ASSERT(sizeof(mdbx_atomic_uint64_t) == 8); #if MDBX_64BIT_ATOMIC #ifdef MDBX_HAVE_C11ATOMICS @@ -142,15 +133,13 @@ MDBX_MAYBE_UNUSED static osal_compiler_barrier(); uint64_t value = (uint64_t)atomic_load32(&p->high, order) << 32; jitter4testing(true); - value |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed - : mo_AcquireRelease); + value |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed : mo_AcquireRelease); jitter4testing(true); for (;;) { osal_compiler_barrier(); uint64_t again = (uint64_t)atomic_load32(&p->high, order) << 32; jitter4testing(true); - again |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed - : mo_AcquireRelease); + again |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed : mo_AcquireRelease); jitter4testing(true); if (likely(value == again)) return value; @@ -171,19 +160,16 @@ MDBX_MAYBE_UNUSED static __always_inline void atomic_yield(void) { #else __asm__ __volatile__("hint @pause"); #endif -#elif defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 6) || \ - defined(__ARM_ARCH_6K__) +#elif defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 6) || defined(__ARM_ARCH_6K__) #ifdef __CC_ARM __yield(); #else __asm__ __volatile__("yield"); #endif -#elif (defined(__mips64) || defined(__mips64__)) && defined(__mips_isa_rev) && \ - __mips_isa_rev >= 2 +#elif (defined(__mips64) || defined(__mips64__)) && defined(__mips_isa_rev) && __mips_isa_rev >= 2 __asm__ __volatile__("pause"); -#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) __asm__ __volatile__(".word 0x00000140"); #elif defined(__linux__) || defined(__gnu_linux__) || defined(_UNIX03_SOURCE) sched_yield(); @@ -193,8 +179,7 @@ MDBX_MAYBE_UNUSED static __always_inline void atomic_yield(void) { } #if MDBX_64BIT_CAS -MDBX_MAYBE_UNUSED static __always_inline bool -atomic_cas64(mdbx_atomic_uint64_t *p, uint64_t c, uint64_t v) { +MDBX_MAYBE_UNUSED static __always_inline bool atomic_cas64(mdbx_atomic_uint64_t *p, uint64_t c, uint64_t v) { #ifdef MDBX_HAVE_C11ATOMICS STATIC_ASSERT(sizeof(long long) >= sizeof(uint64_t)); assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p))); @@ -202,8 +187,7 @@ atomic_cas64(mdbx_atomic_uint64_t *p, uint64_t c, uint64_t v) { #elif defined(__GNUC__) || defined(__clang__) return __sync_bool_compare_and_swap(&p->weak, c, v); #elif defined(_MSC_VER) - return c == (uint64_t)_InterlockedCompareExchange64( - (volatile __int64 *)&p->weak, v, c); + return c == (uint64_t)_InterlockedCompareExchange64((volatile __int64 *)&p->weak, v, c); #elif defined(__APPLE__) return OSAtomicCompareAndSwap64Barrier(c, v, &p->weak); #else @@ -212,8 +196,7 @@ atomic_cas64(mdbx_atomic_uint64_t *p, uint64_t c, uint64_t v) { } #endif /* MDBX_64BIT_CAS */ -MDBX_MAYBE_UNUSED static __always_inline bool -atomic_cas32(mdbx_atomic_uint32_t *p, uint32_t c, uint32_t v) { +MDBX_MAYBE_UNUSED static __always_inline bool atomic_cas32(mdbx_atomic_uint32_t *p, uint32_t c, uint32_t v) { #ifdef MDBX_HAVE_C11ATOMICS STATIC_ASSERT(sizeof(int) >= sizeof(uint32_t)); assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); @@ -222,8 +205,7 @@ atomic_cas32(mdbx_atomic_uint32_t *p, uint32_t c, uint32_t v) { return __sync_bool_compare_and_swap(&p->weak, c, v); #elif defined(_MSC_VER) STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t)); - return c == - (uint32_t)_InterlockedCompareExchange((volatile long *)&p->weak, v, c); + return c == (uint32_t)_InterlockedCompareExchange((volatile long *)&p->weak, v, c); #elif defined(__APPLE__) return OSAtomicCompareAndSwap32Barrier(c, v, &p->weak); #else @@ -231,8 +213,7 @@ atomic_cas32(mdbx_atomic_uint32_t *p, uint32_t c, uint32_t v) { #endif } -MDBX_MAYBE_UNUSED static __always_inline uint32_t -atomic_add32(mdbx_atomic_uint32_t *p, uint32_t v) { +MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_add32(mdbx_atomic_uint32_t *p, uint32_t v) { #ifdef MDBX_HAVE_C11ATOMICS STATIC_ASSERT(sizeof(int) >= sizeof(uint32_t)); assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); @@ -251,8 +232,7 @@ atomic_add32(mdbx_atomic_uint32_t *p, uint32_t v) { #define atomic_sub32(p, v) atomic_add32(p, 0 - (v)) -MDBX_MAYBE_UNUSED static __always_inline uint64_t -safe64_txnid_next(uint64_t txnid) { +MDBX_MAYBE_UNUSED static __always_inline uint64_t safe64_txnid_next(uint64_t txnid) { txnid += xMDBX_TXNID_STEP; #if !MDBX_64BIT_CAS /* avoid overflow of low-part in safe64_reset() */ @@ -262,8 +242,7 @@ safe64_txnid_next(uint64_t txnid) { } /* Atomically make target value >= SAFE64_INVALID_THRESHOLD */ -MDBX_MAYBE_UNUSED static __always_inline void -safe64_reset(mdbx_atomic_uint64_t *p, bool single_writer) { +MDBX_MAYBE_UNUSED static __always_inline void safe64_reset(mdbx_atomic_uint64_t *p, bool single_writer) { if (single_writer) { #if MDBX_64BIT_ATOMIC && MDBX_WORDBITS >= 64 atomic_store64(p, UINT64_MAX, mo_AcquireRelease); @@ -290,8 +269,7 @@ safe64_reset(mdbx_atomic_uint64_t *p, bool single_writer) { jitter4testing(true); } -MDBX_MAYBE_UNUSED static __always_inline bool -safe64_reset_compare(mdbx_atomic_uint64_t *p, uint64_t compare) { +MDBX_MAYBE_UNUSED static __always_inline bool safe64_reset_compare(mdbx_atomic_uint64_t *p, uint64_t compare) { /* LY: This function is used to reset `txnid` from hsr-handler in case * the asynchronously cancellation of read transaction. Therefore, * there may be a collision between the cleanup performed here and @@ -307,8 +285,7 @@ safe64_reset_compare(mdbx_atomic_uint64_t *p, uint64_t compare) { bool rc = false; if (likely(atomic_load32(&p->low, mo_AcquireRelease) == (uint32_t)compare && atomic_cas32(&p->high, (uint32_t)(compare >> 32), UINT32_MAX))) { - if (unlikely(atomic_load32(&p->low, mo_AcquireRelease) != - (uint32_t)compare)) + if (unlikely(atomic_load32(&p->low, mo_AcquireRelease) != (uint32_t)compare)) atomic_cas32(&p->high, UINT32_MAX, (uint32_t)(compare >> 32)); else rc = true; @@ -318,8 +295,7 @@ safe64_reset_compare(mdbx_atomic_uint64_t *p, uint64_t compare) { return rc; } -MDBX_MAYBE_UNUSED static __always_inline void -safe64_write(mdbx_atomic_uint64_t *p, const uint64_t v) { +MDBX_MAYBE_UNUSED static __always_inline void safe64_write(mdbx_atomic_uint64_t *p, const uint64_t v) { assert(p->weak >= SAFE64_INVALID_THRESHOLD); #if MDBX_64BIT_ATOMIC && MDBX_64BIT_CAS atomic_store64(p, v, mo_AcquireRelease); @@ -336,8 +312,7 @@ safe64_write(mdbx_atomic_uint64_t *p, const uint64_t v) { jitter4testing(true); } -MDBX_MAYBE_UNUSED static __always_inline uint64_t -safe64_read(const mdbx_atomic_uint64_t *p) { +MDBX_MAYBE_UNUSED static __always_inline uint64_t safe64_read(const mdbx_atomic_uint64_t *p) { jitter4testing(true); uint64_t v; do @@ -366,8 +341,7 @@ MDBX_MAYBE_UNUSED static __always_inline bool #endif /* unused for now */ /* non-atomic write with safety for reading a half-updated value */ -MDBX_MAYBE_UNUSED static __always_inline void -safe64_update(mdbx_atomic_uint64_t *p, const uint64_t v) { +MDBX_MAYBE_UNUSED static __always_inline void safe64_update(mdbx_atomic_uint64_t *p, const uint64_t v) { #if MDBX_64BIT_ATOMIC atomic_store64(p, v, mo_Relaxed); #else diff --git a/src/atomics-types.h b/src/atomics-types.h index 8e3e4b9b..fe977b94 100644 --- a/src/atomics-types.h +++ b/src/atomics-types.h @@ -16,21 +16,19 @@ #if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() #include #define MDBX_HAVE_C11ATOMICS -#elif !defined(__cplusplus) && \ - (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ - !defined(__STDC_NO_ATOMICS__) && \ - (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ - !(defined(__GNUC__) || defined(__clang__))) +#elif !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || !(defined(__GNUC__) || defined(__clang__))) #include #define MDBX_HAVE_C11ATOMICS #elif defined(__GNUC__) || defined(__clang__) #elif defined(_MSC_VER) #pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ -#pragma warning(disable : 4133) /* 'function': incompatible types - from \ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ 'size_t' to 'LONGLONG' */ -#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ 'std::size_t', possible loss of data */ -#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ 'long', possible loss of data */ #pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) #pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) @@ -85,13 +83,13 @@ typedef union { #define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a) #endif /* Crutches for C11 atomic compiler's bugs */ -#define mo_c11_store(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_release \ +#define mo_c11_store(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_release \ : memory_order_seq_cst) -#define mo_c11_load(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ +#define mo_c11_load(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ : memory_order_seq_cst) #endif /* MDBX_HAVE_C11ATOMICS */ diff --git a/src/audit.c b/src/audit.c index 34a0f68f..05c69632 100644 --- a/src/audit.c +++ b/src/audit.c @@ -8,29 +8,23 @@ struct audit_ctx { uint8_t *const done_bitmap; }; -static int audit_dbi(void *ctx, const MDBX_txn *txn, const MDBX_val *name, - MDBX_db_flags_t flags, const struct MDBX_stat *stat, - MDBX_dbi dbi) { +static int audit_dbi(void *ctx, const MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, + const struct MDBX_stat *stat, MDBX_dbi dbi) { struct audit_ctx *audit_ctx = ctx; (void)name; (void)txn; (void)flags; - audit_ctx->used += (size_t)stat->ms_branch_pages + - (size_t)stat->ms_leaf_pages + - (size_t)stat->ms_overflow_pages; + audit_ctx->used += (size_t)stat->ms_branch_pages + (size_t)stat->ms_leaf_pages + (size_t)stat->ms_overflow_pages; if (dbi) audit_ctx->done_bitmap[dbi / CHAR_BIT] |= 1 << dbi % CHAR_BIT; return MDBX_SUCCESS; } static size_t audit_db_used(const tree_t *db) { - return db ? (size_t)db->branch_pages + (size_t)db->leaf_pages + - (size_t)db->large_pages - : 0; + return db ? (size_t)db->branch_pages + (size_t)db->leaf_pages + (size_t)db->large_pages : 0; } -__cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored, - bool dont_filter_gc) { +__cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored, bool dont_filter_gc) { const MDBX_env *const env = txn->env; size_t pending = 0; if ((txn->flags & MDBX_TXN_RDONLY) == 0) @@ -48,8 +42,7 @@ __cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored, while (rc == MDBX_SUCCESS) { if (!dont_filter_gc) { if (unlikely(key.iov_len != sizeof(txnid_t))) { - ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid GC-key size", (unsigned)key.iov_len); + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-key size", (unsigned)key.iov_len); return MDBX_CORRUPTED; } txnid_t id = unaligned_peek_u64(4, key.iov_base); @@ -68,18 +61,16 @@ __cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored, const size_t done_bitmap_size = (txn->n_dbi + CHAR_BIT - 1) / CHAR_BIT; if (txn->parent) { - tASSERT(txn, txn->n_dbi == txn->parent->n_dbi && - txn->n_dbi == txn->env->txn->n_dbi); + tASSERT(txn, txn->n_dbi == txn->parent->n_dbi && txn->n_dbi == txn->env->txn->n_dbi); #if MDBX_ENABLE_DBI_SPARSE - tASSERT(txn, txn->dbi_sparse == txn->parent->dbi_sparse && - txn->dbi_sparse == txn->env->txn->dbi_sparse); + tASSERT(txn, txn->dbi_sparse == txn->parent->dbi_sparse && txn->dbi_sparse == txn->env->txn->dbi_sparse); #endif /* MDBX_ENABLE_DBI_SPARSE */ } struct audit_ctx ctx = {0, alloca(done_bitmap_size)}; memset(ctx.done_bitmap, 0, done_bitmap_size); - ctx.used = NUM_METAS + audit_db_used(dbi_dig(txn, FREE_DBI, nullptr)) + - audit_db_used(dbi_dig(txn, MAIN_DBI, nullptr)); + ctx.used = + NUM_METAS + audit_db_used(dbi_dig(txn, FREE_DBI, nullptr)) + audit_db_used(dbi_dig(txn, MAIN_DBI, nullptr)); rc = mdbx_enumerate_tables(txn, audit_dbi, &ctx); tASSERT(txn, rc == MDBX_SUCCESS); @@ -91,11 +82,9 @@ __cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored, if (db) ctx.used += audit_db_used(db); else if (dbi_state(txn, dbi)) - WARNING("audit %s@%" PRIaTXN - ": unable account dbi %zd / \"%*s\", state 0x%02x", - txn->parent ? "nested-" : "", txn->txnid, dbi, - (int)env->kvs[dbi].name.iov_len, - (const char *)env->kvs[dbi].name.iov_base, dbi_state(txn, dbi)); + WARNING("audit %s@%" PRIaTXN ": unable account dbi %zd / \"%*s\", state 0x%02x", txn->parent ? "nested-" : "", + txn->txnid, dbi, (int)env->kvs[dbi].name.iov_len, (const char *)env->kvs[dbi].name.iov_base, + dbi_state(txn, dbi)); } if (pending + gc + ctx.used == txn->geo.first_unallocated) @@ -104,15 +93,12 @@ __cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored, if ((txn->flags & MDBX_TXN_RDONLY) == 0) ERROR("audit @%" PRIaTXN ": %zu(pending) = %zu(loose) + " "%zu(reclaimed) + %zu(retired-pending) - %zu(retired-stored)", - txn->txnid, pending, txn->tw.loose_count, - MDBX_PNL_GETSIZE(txn->tw.relist), - txn->tw.retired_pages ? MDBX_PNL_GETSIZE(txn->tw.retired_pages) : 0, - retired_stored); + txn->txnid, pending, txn->tw.loose_count, MDBX_PNL_GETSIZE(txn->tw.relist), + txn->tw.retired_pages ? MDBX_PNL_GETSIZE(txn->tw.retired_pages) : 0, retired_stored); ERROR("audit @%" PRIaTXN ": %zu(pending) + %zu" "(gc) + %zu(count) = %zu(total) <> %zu" "(allocated)", - txn->txnid, pending, gc, ctx.used, pending + gc + ctx.used, - (size_t)txn->geo.first_unallocated); + txn->txnid, pending, gc, ctx.used, pending + gc + ctx.used, (size_t)txn->geo.first_unallocated); return MDBX_PROBLEM; } diff --git a/src/chk.c b/src/chk.c index 83d7d74c..4197b29c 100644 --- a/src/chk.c +++ b/src/chk.c @@ -28,8 +28,7 @@ typedef struct MDBX_chk_internal { __cold static int chk_check_break(MDBX_chk_scope_t *const scope) { MDBX_chk_internal_t *const chk = scope->internal; - return (chk->got_break || (chk->cb->check_break && - (chk->got_break = chk->cb->check_break(chk->usr)))) + return (chk->got_break || (chk->cb->check_break && (chk->got_break = chk->cb->check_break(chk->usr)))) ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE; } @@ -37,15 +36,14 @@ __cold static int chk_check_break(MDBX_chk_scope_t *const scope) { __cold static void chk_line_end(MDBX_chk_line_t *line) { if (likely(line)) { MDBX_chk_internal_t *chk = line->ctx->internal; - assert(line->begin <= line->end && line->begin <= line->out && - line->out <= line->end); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); if (likely(chk->cb->print_done)) chk->cb->print_done(line); } } -__cold __must_check_result static MDBX_chk_line_t * -chk_line_begin(MDBX_chk_scope_t *const scope, enum MDBX_chk_severity severity) { +__cold __must_check_result static MDBX_chk_line_t *chk_line_begin(MDBX_chk_scope_t *const scope, + enum MDBX_chk_severity severity) { MDBX_chk_internal_t *const chk = scope->internal; if (severity < MDBX_chk_warning) mdbx_env_chk_encount_problem(chk->usr); @@ -54,8 +52,7 @@ chk_line_begin(MDBX_chk_scope_t *const scope, enum MDBX_chk_severity severity) { line = chk->cb->print_begin(chk->usr, severity); if (likely(line)) { assert(line->ctx == nullptr || (line->ctx == chk->usr && line->empty)); - assert(line->begin <= line->end && line->begin <= line->out && - line->out <= line->end); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); line->ctx = chk->usr; } } @@ -75,12 +72,10 @@ __cold static MDBX_chk_line_t *chk_line_feed(MDBX_chk_line_t *line) { __cold static MDBX_chk_line_t *chk_flush(MDBX_chk_line_t *line) { if (likely(line)) { MDBX_chk_internal_t *chk = line->ctx->internal; - assert(line->begin <= line->end && line->begin <= line->out && - line->out <= line->end); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); if (likely(chk->cb->print_flush)) { chk->cb->print_flush(line); - assert(line->begin <= line->end && line->begin <= line->out && - line->out <= line->end); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); line->out = line->begin; } } @@ -90,8 +85,7 @@ __cold static MDBX_chk_line_t *chk_flush(MDBX_chk_line_t *line) { __cold static size_t chk_print_wanna(MDBX_chk_line_t *line, size_t need) { if (likely(line && need)) { size_t have = line->end - line->out; - assert(line->begin <= line->end && line->begin <= line->out && - line->out <= line->end); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); if (need > have) { line = chk_flush(line); have = line->end - line->out; @@ -101,17 +95,14 @@ __cold static size_t chk_print_wanna(MDBX_chk_line_t *line, size_t need) { return 0; } -__cold static MDBX_chk_line_t *chk_puts(MDBX_chk_line_t *line, - const char *str) { +__cold static MDBX_chk_line_t *chk_puts(MDBX_chk_line_t *line, const char *str) { if (likely(line && str && *str)) { MDBX_chk_internal_t *chk = line->ctx->internal; size_t left = strlen(str); - assert(line->begin <= line->end && line->begin <= line->out && - line->out <= line->end); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); if (chk->cb->print_chars) { chk->cb->print_chars(line, str, left); - assert(line->begin <= line->end && line->begin <= line->out && - line->out <= line->end); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); } else do { size_t chunk = chk_print_wanna(line, left); @@ -120,8 +111,7 @@ __cold static MDBX_chk_line_t *chk_puts(MDBX_chk_line_t *line, break; memcpy(line->out, str, chunk); line->out += chunk; - assert(line->begin <= line->end && line->begin <= line->out && - line->out <= line->end); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); str += chunk; left -= chunk; } while (left); @@ -130,16 +120,13 @@ __cold static MDBX_chk_line_t *chk_puts(MDBX_chk_line_t *line, return line; } -__cold static MDBX_chk_line_t *chk_print_va(MDBX_chk_line_t *line, - const char *fmt, va_list args) { +__cold static MDBX_chk_line_t *chk_print_va(MDBX_chk_line_t *line, const char *fmt, va_list args) { if (likely(line)) { MDBX_chk_internal_t *chk = line->ctx->internal; - assert(line->begin <= line->end && line->begin <= line->out && - line->out <= line->end); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); if (chk->cb->print_format) { chk->cb->print_format(line, fmt, args); - assert(line->begin <= line->end && line->begin <= line->out && - line->out <= line->end); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); } else { va_list ones; va_copy(ones, args); @@ -151,8 +138,7 @@ __cold static MDBX_chk_line_t *chk_print_va(MDBX_chk_line_t *line, int written = vsnprintf(line->out, have, fmt, args); if (likely(written > 0)) line->out += written; - assert(line->begin <= line->end && line->begin <= line->out && - line->out <= line->end); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); } } } @@ -161,8 +147,7 @@ __cold static MDBX_chk_line_t *chk_print_va(MDBX_chk_line_t *line, return line; } -__cold static MDBX_chk_line_t *MDBX_PRINTF_ARGS(2, 3) - chk_print(MDBX_chk_line_t *line, const char *fmt, ...) { +__cold static MDBX_chk_line_t *MDBX_PRINTF_ARGS(2, 3) chk_print(MDBX_chk_line_t *line, const char *fmt, ...) { if (likely(line)) { // MDBX_chk_internal_t *chk = line->ctx->internal; va_list args; @@ -174,12 +159,9 @@ __cold static MDBX_chk_line_t *MDBX_PRINTF_ARGS(2, 3) return line; } -__cold static MDBX_chk_line_t *chk_print_size(MDBX_chk_line_t *line, - const char *prefix, - const uint64_t value, +__cold static MDBX_chk_line_t *chk_print_size(MDBX_chk_line_t *line, const char *prefix, const uint64_t value, const char *suffix) { - static const char sf[] = - "KMGTPEZY"; /* LY: Kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta! */ + static const char sf[] = "KMGTPEZY"; /* LY: Kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta! */ if (likely(line)) { MDBX_chk_internal_t *chk = line->ctx->internal; prefix = prefix ? prefix : ""; @@ -191,33 +173,27 @@ __cold static MDBX_chk_line_t *chk_print_size(MDBX_chk_line_t *line, const unsigned scale = 10 + i * 10; const uint64_t rounded = value + (UINT64_C(5) << (scale - 10)); const uint64_t integer = rounded >> scale; - const uint64_t fractional = - (rounded - (integer << scale)) * 100u >> scale; + const uint64_t fractional = (rounded - (integer << scale)) * 100u >> scale; if ((rounded >> scale) <= 1000) - return chk_print(line, "%s%" PRIu64 " (%u.%02u %ciB)%s", prefix, - value, (unsigned)integer, (unsigned)fractional, - sf[i], suffix); + return chk_print(line, "%s%" PRIu64 " (%u.%02u %ciB)%s", prefix, value, (unsigned)integer, + (unsigned)fractional, sf[i], suffix); } line->empty = false; } return line; } -__cold static int chk_error_rc(MDBX_chk_scope_t *const scope, int err, - const char *subj) { +__cold static int chk_error_rc(MDBX_chk_scope_t *const scope, int err, const char *subj) { MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_error); if (line) - chk_line_end(chk_flush(chk_print(line, "%s() failed, error %s (%d)", subj, - mdbx_strerror(err), err))); + chk_line_end(chk_flush(chk_print(line, "%s() failed, error %s (%d)", subj, mdbx_strerror(err), err))); else - debug_log(MDBX_LOG_ERROR, "mdbx_env_chk", 0, "%s() failed, error %s (%d)", - subj, mdbx_strerror(err), err); + debug_log(MDBX_LOG_ERROR, "mdbx_env_chk", 0, "%s() failed, error %s (%d)", subj, mdbx_strerror(err), err); return err; } __cold static void MDBX_PRINTF_ARGS(5, 6) - chk_object_issue(MDBX_chk_scope_t *const scope, const char *object, - uint64_t entry_number, const char *caption, + chk_object_issue(MDBX_chk_scope_t *const scope, const char *object, uint64_t entry_number, const char *caption, const char *extra_fmt, ...) { MDBX_chk_internal_t *const chk = scope->internal; MDBX_chk_issue_t *issue = chk->usr->scope->issues; @@ -258,8 +234,7 @@ __cold static void MDBX_PRINTF_ARGS(5, 6) va_end(args); } -__cold static void MDBX_PRINTF_ARGS(2, 3) - chk_scope_issue(MDBX_chk_scope_t *const scope, const char *fmt, ...) { +__cold static void MDBX_PRINTF_ARGS(2, 3) chk_scope_issue(MDBX_chk_scope_t *const scope, const char *fmt, ...) { MDBX_chk_internal_t *const chk = scope->internal; va_list args; va_start(args, fmt); @@ -267,8 +242,7 @@ __cold static void MDBX_PRINTF_ARGS(2, 3) mdbx_env_chk_encount_problem(chk->usr); chk->cb->issue(chk->usr, nullptr, 0, nullptr, fmt, args); } else - chk_line_end( - chk_print_va(chk_line_begin(scope, MDBX_chk_error), fmt, args)); + chk_line_end(chk_print_va(chk_line_begin(scope, MDBX_chk_error), fmt, args)); va_end(args); } @@ -306,26 +280,19 @@ __cold static int chk_scope_end(MDBX_chk_internal_t *chk, int err) { return err; } -__cold static int chk_scope_begin_args(MDBX_chk_internal_t *chk, - int verbosity_adjustment, - enum MDBX_chk_stage stage, - const void *object, size_t *problems, - const char *fmt, va_list args) { +__cold static int chk_scope_begin_args(MDBX_chk_internal_t *chk, int verbosity_adjustment, enum MDBX_chk_stage stage, + const void *object, size_t *problems, const char *fmt, va_list args) { if (unlikely(chk->scope_depth + 1u >= ARRAY_LENGTH(chk->scope_stack))) return MDBX_BACKLOG_DEPLETED; MDBX_chk_scope_t *const outer = chk->scope_stack + chk->scope_depth; - const int verbosity = - outer->verbosity + - (verbosity_adjustment - 1) * (1 << MDBX_chk_severity_prio_shift); + const int verbosity = outer->verbosity + (verbosity_adjustment - 1) * (1 << MDBX_chk_severity_prio_shift); MDBX_chk_scope_t *const inner = outer + 1; memset(inner, 0, sizeof(*inner)); inner->internal = outer->internal; inner->stage = stage ? stage : (stage = outer->stage); inner->object = object; - inner->verbosity = (verbosity < MDBX_chk_warning) - ? MDBX_chk_warning - : (enum MDBX_chk_severity)verbosity; + inner->verbosity = (verbosity < MDBX_chk_warning) ? MDBX_chk_warning : (enum MDBX_chk_severity)verbosity; if (problems) chk->problem_counter = problems; else if (!chk->problem_counter || outer->stage != stage) @@ -351,13 +318,11 @@ __cold static int chk_scope_begin_args(MDBX_chk_internal_t *chk, } __cold static int MDBX_PRINTF_ARGS(6, 7) - chk_scope_begin(MDBX_chk_internal_t *chk, int verbosity_adjustment, - enum MDBX_chk_stage stage, const void *object, + chk_scope_begin(MDBX_chk_internal_t *chk, int verbosity_adjustment, enum MDBX_chk_stage stage, const void *object, size_t *problems, const char *fmt, ...) { va_list args; va_start(args, fmt); - int rc = chk_scope_begin_args(chk, verbosity_adjustment, stage, object, - problems, fmt, args); + int rc = chk_scope_begin_args(chk, verbosity_adjustment, stage, object, problems, fmt, args); va_end(args); return rc; } @@ -376,19 +341,16 @@ __cold void chk_scope_pop(MDBX_chk_scope_t *const inner) { } __cold static MDBX_chk_scope_t *MDBX_PRINTF_ARGS(3, 4) - chk_scope_push(MDBX_chk_scope_t *const scope, int verbosity_adjustment, - const char *fmt, ...) { + chk_scope_push(MDBX_chk_scope_t *const scope, int verbosity_adjustment, const char *fmt, ...) { chk_scope_restore(scope, MDBX_SUCCESS); va_list args; va_start(args, fmt); - int err = chk_scope_begin_args(scope->internal, verbosity_adjustment, - scope->stage, nullptr, nullptr, fmt, args); + int err = chk_scope_begin_args(scope->internal, verbosity_adjustment, scope->stage, nullptr, nullptr, fmt, args); va_end(args); return err ? nullptr : scope + 1; } -__cold static const char *chk_v2a(MDBX_chk_internal_t *chk, - const MDBX_val *val) { +__cold static const char *chk_v2a(MDBX_chk_internal_t *chk, const MDBX_val *val) { if (val == MDBX_CHK_MAIN) return "@MAIN"; if (val == MDBX_CHK_GC) @@ -418,8 +380,7 @@ __cold static const char *chk_v2a(MDBX_chk_internal_t *chk, chk->v2a_buf.iov_base = ptr; chk->v2a_buf.iov_len = enough; } - snprintf(chk->v2a_buf.iov_base, chk->v2a_buf.iov_len, - "", len); + snprintf(chk->v2a_buf.iov_base, chk->v2a_buf.iov_len, "", len); return chk->v2a_buf.iov_base; } @@ -428,8 +389,7 @@ __cold static const char *chk_v2a(MDBX_chk_internal_t *chk, size_t xchars = 0; for (size_t i = 0; i < len && printable; ++i) { quoting = quoting || !(data[i] == '_' || isalnum(data[i])); - printable = - isprint(data[i]) || (data[i] < ' ' && ++xchars < 4 && len > xchars * 4); + printable = isprint(data[i]) || (data[i] < ' ' && ++xchars < 4 && len > xchars * 4); } size_t need = len + 1; @@ -525,10 +485,8 @@ static void histogram_reduce(struct MDBX_chk_histogram *p) { // ищем пару для слияния с минимальной ошибкой size_t min_err = SIZE_MAX, min_i = last - 1; for (size_t i = 0; i < last; ++i) { - const size_t b1 = p->ranges[i].begin, e1 = p->ranges[i].end, - s1 = p->ranges[i].amount; - const size_t b2 = p->ranges[i + 1].begin, e2 = p->ranges[i + 1].end, - s2 = p->ranges[i + 1].amount; + const size_t b1 = p->ranges[i].begin, e1 = p->ranges[i].end, s1 = p->ranges[i].amount; + const size_t b2 = p->ranges[i + 1].begin, e2 = p->ranges[i + 1].end, s2 = p->ranges[i + 1].amount; const size_t l1 = e1 - b1, l2 = e2 - b2, lx = e2 - b1, sx = s1 + s2; assert(s1 > 0 && b1 > 0 && b1 < e1); assert(s2 > 0 && b2 > 0 && b2 < e2); @@ -550,8 +508,7 @@ static void histogram_reduce(struct MDBX_chk_histogram *p) { p->ranges[min_i].count += p->ranges[min_i + 1].count; if (min_i < last) // перемещаем хвост - memmove(p->ranges + min_i, p->ranges + min_i + 1, - (last - min_i) * sizeof(p->ranges[0])); + memmove(p->ranges + min_i, p->ranges + min_i + 1, (last - min_i) * sizeof(p->ranges[0])); // обнуляем последний элемент и продолжаем p->ranges[last].count = 0; } @@ -585,8 +542,7 @@ static void histogram_acc(const size_t n, struct MDBX_chk_histogram *p) { #ifdef __COVERITY__ if (i < last) /* avoid Coverity false-positive issue */ #endif /* __COVERITY__ */ - memmove(p->ranges + i + 1, p->ranges + i, - (last - i) * sizeof(p->ranges[0])); + memmove(p->ranges + i + 1, p->ranges + i, (last - i) * sizeof(p->ranges[0])); } p->ranges[i].begin = n; p->ranges[i].end = n + 1; @@ -598,10 +554,8 @@ static void histogram_acc(const size_t n, struct MDBX_chk_histogram *p) { } } -__cold static MDBX_chk_line_t * -histogram_dist(MDBX_chk_line_t *line, - const struct MDBX_chk_histogram *histogram, const char *prefix, - const char *first, bool amount) { +__cold static MDBX_chk_line_t *histogram_dist(MDBX_chk_line_t *line, const struct MDBX_chk_histogram *histogram, + const char *prefix, const char *first, bool amount) { line = chk_print(line, "%s:", prefix); const char *comma = ""; const size_t first_val = amount ? histogram->ones : histogram->pad; @@ -614,36 +568,28 @@ histogram_dist(MDBX_chk_line_t *line, chk_print(line, "%s %" PRIuSIZE, comma, histogram->ranges[n].begin); if (histogram->ranges[n].begin != histogram->ranges[n].end - 1) chk_print(line, "-%" PRIuSIZE, histogram->ranges[n].end - 1); - line = chk_print(line, "=%" PRIuSIZE, - amount ? histogram->ranges[n].amount - : histogram->ranges[n].count); + line = chk_print(line, "=%" PRIuSIZE, amount ? histogram->ranges[n].amount : histogram->ranges[n].count); comma = ","; } return line; } -__cold static MDBX_chk_line_t * -histogram_print(MDBX_chk_scope_t *scope, MDBX_chk_line_t *line, - const struct MDBX_chk_histogram *histogram, const char *prefix, - const char *first, bool amount) { +__cold static MDBX_chk_line_t *histogram_print(MDBX_chk_scope_t *scope, MDBX_chk_line_t *line, + const struct MDBX_chk_histogram *histogram, const char *prefix, + const char *first, bool amount) { if (histogram->count) { - line = chk_print(line, "%s %" PRIuSIZE, prefix, - amount ? histogram->amount : histogram->count); + line = chk_print(line, "%s %" PRIuSIZE, prefix, amount ? histogram->amount : histogram->count); if (scope->verbosity > MDBX_chk_info) - line = chk_puts( - histogram_dist(line, histogram, " (distribution", first, amount), - ")"); + line = chk_puts(histogram_dist(line, histogram, " (distribution", first, amount), ")"); } return line; } //----------------------------------------------------------------------------- -__cold static int chk_get_tbl(MDBX_chk_scope_t *const scope, - const walk_tbl_t *in, MDBX_chk_table_t **out) { +__cold static int chk_get_tbl(MDBX_chk_scope_t *const scope, const walk_tbl_t *in, MDBX_chk_table_t **out) { MDBX_chk_internal_t *const chk = scope->internal; - if (chk->last_lookup && - chk->last_lookup->name.iov_base == in->name.iov_base) { + if (chk->last_lookup && chk->last_lookup->name.iov_base == in->name.iov_base) { *out = chk->last_lookup; return MDBX_SUCCESS; } @@ -665,34 +611,27 @@ __cold static int chk_get_tbl(MDBX_chk_scope_t *const scope, if (tbl->id < 0) { tbl->id = (int)i; tbl->cookie = - chk->cb->table_filter - ? chk->cb->table_filter(chk->usr, &tbl->name, tbl->flags) - : (void *)(intptr_t)-1; + chk->cb->table_filter ? chk->cb->table_filter(chk->usr, &tbl->name, tbl->flags) : (void *)(intptr_t)-1; } *out = (chk->last_lookup = tbl); return MDBX_SUCCESS; } } - chk_scope_issue(scope, "too many tables > %u", - (unsigned)ARRAY_LENGTH(chk->table) - CORE_DBS - /* meta */ 1); + chk_scope_issue(scope, "too many tables > %u", (unsigned)ARRAY_LENGTH(chk->table) - CORE_DBS - /* meta */ 1); *out = nullptr; return MDBX_PROBLEM; } //------------------------------------------------------------------------------ -__cold static void chk_verbose_meta(MDBX_chk_scope_t *const scope, - const unsigned num) { +__cold static void chk_verbose_meta(MDBX_chk_scope_t *const scope, const unsigned num) { MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_verbose); MDBX_chk_internal_t *const chk = scope->internal; if (line) { MDBX_env *const env = chk->usr->env; - const bool have_bootid = (chk->envinfo.mi_bootid.current.x | - chk->envinfo.mi_bootid.current.y) != 0; - const bool bootid_match = - have_bootid && memcmp(&chk->envinfo.mi_bootid.meta[num], - &chk->envinfo.mi_bootid.current, - sizeof(chk->envinfo.mi_bootid.current)) == 0; + const bool have_bootid = (chk->envinfo.mi_bootid.current.x | chk->envinfo.mi_bootid.current.y) != 0; + const bool bootid_match = have_bootid && memcmp(&chk->envinfo.mi_bootid.meta[num], &chk->envinfo.mi_bootid.current, + sizeof(chk->envinfo.mi_bootid.current)) == 0; const char *status = "stay"; if (num == chk->troika.recent) @@ -707,9 +646,7 @@ __cold static void chk_verbose_meta(MDBX_chk_scope_t *const scope, break; case DATASIGN_WEAK: line = chk_print(line, "weak-%s", - have_bootid - ? (bootid_match ? "intact (same boot-id)" : "dead") - : "unknown (no boot-id)"); + have_bootid ? (bootid_match ? "intact (same boot-id)" : "dead") : "unknown (no boot-id)"); break; default: line = chk_puts(line, "steady"); @@ -718,10 +655,8 @@ __cold static void chk_verbose_meta(MDBX_chk_scope_t *const scope, const txnid_t meta_txnid = chk->envinfo.mi_meta_txnid[num]; line = chk_print(line, " txn#%" PRIaTXN ", ", meta_txnid); if (chk->envinfo.mi_bootid.meta[num].x | chk->envinfo.mi_bootid.meta[num].y) - line = chk_print(line, "boot-id %" PRIx64 "-%" PRIx64 " (%s)", - chk->envinfo.mi_bootid.meta[num].x, - chk->envinfo.mi_bootid.meta[num].y, - bootid_match ? "live" : "not match"); + line = chk_print(line, "boot-id %" PRIx64 "-%" PRIx64 " (%s)", chk->envinfo.mi_bootid.meta[num].x, + chk->envinfo.mi_bootid.meta[num].y, bootid_match ? "live" : "not match"); else line = chk_puts(line, "no boot-id"); @@ -730,22 +665,16 @@ __cold static void chk_verbose_meta(MDBX_chk_scope_t *const scope, line = chk_print(line, ", %s", "forced for checking"); } else if (meta_txnid > chk->envinfo.mi_recent_txnid && (env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == MDBX_EXCLUSIVE) - line = chk_print(line, - ", rolled-back %" PRIu64 " commit(s) (%" PRIu64 - " >>> %" PRIu64 ")", - meta_txnid - chk->envinfo.mi_recent_txnid, meta_txnid, - chk->envinfo.mi_recent_txnid); + line = chk_print(line, ", rolled-back %" PRIu64 " commit(s) (%" PRIu64 " >>> %" PRIu64 ")", + meta_txnid - chk->envinfo.mi_recent_txnid, meta_txnid, chk->envinfo.mi_recent_txnid); chk_line_end(line); } } -__cold static int -chk_pgvisitor(const size_t pgno, const unsigned npages, void *const ctx, - const int deep, const walk_tbl_t *tbl_info, - const size_t page_size, const page_type_t pagetype, - const MDBX_error_t page_err, const size_t nentries, - const size_t payload_bytes, const size_t header_bytes, - const size_t unused_bytes) { +__cold static int chk_pgvisitor(const size_t pgno, const unsigned npages, void *const ctx, const int deep, + const walk_tbl_t *tbl_info, const size_t page_size, const page_type_t pagetype, + const MDBX_error_t page_err, const size_t nentries, const size_t payload_bytes, + const size_t header_bytes, const size_t unused_bytes) { MDBX_chk_scope_t *const scope = ctx; MDBX_chk_internal_t *const chk = scope->internal; MDBX_chk_context_t *const usr = chk->usr; @@ -772,9 +701,8 @@ chk_pgvisitor(const size_t pgno, const unsigned npages, void *const ctx, if (tbl->flags & MDBX_DUPSORT) height -= tbl_info->internal->height; else { - chk_object_issue(scope, "nested tree", pgno, "unexpected", - "table %s flags 0x%x, deep %i", chk_v2a(chk, &tbl->name), - tbl->flags, deep); + chk_object_issue(scope, "nested tree", pgno, "unexpected", "table %s flags 0x%x, deep %i", + chk_v2a(chk, &tbl->name), tbl->flags, deep); nested = nullptr; } } else @@ -784,8 +712,7 @@ chk_pgvisitor(const size_t pgno, const unsigned npages, void *const ctx, bool branch = false; switch (pagetype) { default: - chk_object_issue(scope, "page", pgno, "unknown page-type", - "type %u, deep %i", (unsigned)pagetype, deep); + chk_object_issue(scope, "page", pgno, "unknown page-type", "type %u, deep %i", (unsigned)pagetype, deep); pagetype_caption = "unknown"; tbl->pages.other += npages; break; @@ -803,10 +730,8 @@ chk_pgvisitor(const size_t pgno, const unsigned npages, void *const ctx, pagetype_caption = "large"; histogram_acc(npages, &tbl->histogram.large_pages); if (tbl->flags & MDBX_DUPSORT) - chk_object_issue(scope, "page", pgno, "unexpected", - "type %u, table %s flags 0x%x, deep %i", - (unsigned)pagetype, chk_v2a(chk, &tbl->name), tbl->flags, - deep); + chk_object_issue(scope, "page", pgno, "unexpected", "type %u, table %s flags 0x%x, deep %i", (unsigned)pagetype, + chk_v2a(chk, &tbl->name), tbl->flags, deep); break; case page_branch: branch = true; @@ -820,10 +745,8 @@ chk_pgvisitor(const size_t pgno, const unsigned npages, void *const ctx, break; case page_dupfix_leaf: if (!nested) - chk_object_issue(scope, "page", pgno, "unexpected", - "type %u, table %s flags 0x%x, deep %i", - (unsigned)pagetype, chk_v2a(chk, &tbl->name), tbl->flags, - deep); + chk_object_issue(scope, "page", pgno, "unexpected", "type %u, table %s flags 0x%x, deep %i", (unsigned)pagetype, + chk_v2a(chk, &tbl->name), tbl->flags, deep); /* fall through */ __fallthrough; case page_leaf: @@ -831,33 +754,27 @@ chk_pgvisitor(const size_t pgno, const unsigned npages, void *const ctx, pagetype_caption = "leaf"; tbl->pages.leaf += 1; if (height != tbl_info->internal->height) - chk_object_issue(scope, "page", pgno, "wrong tree height", - "actual %i != %i table %s", height, + chk_object_issue(scope, "page", pgno, "wrong tree height", "actual %i != %i table %s", height, tbl_info->internal->height, chk_v2a(chk, &tbl->name)); } else { - pagetype_caption = - (pagetype == page_leaf) ? "nested-leaf" : "nested-leaf-dupfix"; + pagetype_caption = (pagetype == page_leaf) ? "nested-leaf" : "nested-leaf-dupfix"; tbl->pages.nested_leaf += 1; if (chk->last_nested != nested) { histogram_acc(height, &tbl->histogram.nested_tree); chk->last_nested = nested; } if (height != nested->height) - chk_object_issue(scope, "page", pgno, "wrong nested-tree height", - "actual %i != %i dupsort-node %s", height, + chk_object_issue(scope, "page", pgno, "wrong nested-tree height", "actual %i != %i dupsort-node %s", height, nested->height, chk_v2a(chk, &tbl->name)); } break; case page_sub_dupfix_leaf: case page_sub_leaf: - pagetype_caption = - (pagetype == page_sub_leaf) ? "subleaf-dupsort" : "subleaf-dupfix"; + pagetype_caption = (pagetype == page_sub_leaf) ? "subleaf-dupsort" : "subleaf-dupfix"; tbl->pages.nested_subleaf += 1; if ((tbl->flags & MDBX_DUPSORT) == 0 || nested) - chk_object_issue(scope, "page", pgno, "unexpected", - "type %u, table %s flags 0x%x, deep %i", - (unsigned)pagetype, chk_v2a(chk, &tbl->name), tbl->flags, - deep); + chk_object_issue(scope, "page", pgno, "unexpected", "type %u, table %s flags 0x%x, deep %i", (unsigned)pagetype, + chk_v2a(chk, &tbl->name), tbl->flags, deep); break; } @@ -867,33 +784,24 @@ chk_pgvisitor(const size_t pgno, const unsigned npages, void *const ctx, if (npages == 1) chk_print(line, "%s-page %" PRIuSIZE, pagetype_caption, pgno); else - chk_print(line, "%s-span %" PRIuSIZE "[%u]", pagetype_caption, pgno, - npages); - chk_line_end(chk_print(line, - " of %s: header %" PRIiPTR ", %s %" PRIiPTR - ", payload %" PRIiPTR ", unused %" PRIiPTR - ", deep %i", - chk_v2a(chk, &tbl->name), header_bytes, - (pagetype == page_branch) ? "keys" : "entries", - nentries, payload_bytes, unused_bytes, deep)); + chk_print(line, "%s-span %" PRIuSIZE "[%u]", pagetype_caption, pgno, npages); + chk_line_end(chk_print( + line, " of %s: header %" PRIiPTR ", %s %" PRIiPTR ", payload %" PRIiPTR ", unused %" PRIiPTR ", deep %i", + chk_v2a(chk, &tbl->name), header_bytes, (pagetype == page_branch) ? "keys" : "entries", nentries, + payload_bytes, unused_bytes, deep)); } bool already_used = false; for (unsigned n = 0; n < npages; ++n) { const size_t spanpgno = pgno + n; if (spanpgno >= usr->result.alloc_pages) { - chk_object_issue(scope, "page", spanpgno, "wrong page-no", - "%s-page: %" PRIuSIZE " > %" PRIuSIZE ", deep %i", - pagetype_caption, spanpgno, usr->result.alloc_pages, - deep); + chk_object_issue(scope, "page", spanpgno, "wrong page-no", "%s-page: %" PRIuSIZE " > %" PRIuSIZE ", deep %i", + pagetype_caption, spanpgno, usr->result.alloc_pages, deep); tbl->pages.all += 1; } else if (chk->pagemap[spanpgno]) { - const MDBX_chk_table_t *const rival = - chk->table[chk->pagemap[spanpgno] - 1]; - chk_object_issue(scope, "page", spanpgno, - (branch && rival == tbl) ? "loop" : "already used", - "%s-page: by %s, deep %i", pagetype_caption, - chk_v2a(chk, &rival->name), deep); + const MDBX_chk_table_t *const rival = chk->table[chk->pagemap[spanpgno] - 1]; + chk_object_issue(scope, "page", spanpgno, (branch && rival == tbl) ? "loop" : "already used", + "%s-page: by %s, deep %i", pagetype_caption, chk_v2a(chk, &rival->name), deep); already_used = true; } else { chk->pagemap[spanpgno] = (int16_t)tbl->id + 1; @@ -907,36 +815,30 @@ chk_pgvisitor(const size_t pgno, const unsigned npages, void *const ctx, } if (MDBX_IS_ERROR(page_err)) { - chk_object_issue(scope, "page", pgno, "invalid/corrupted", "%s-page", - pagetype_caption); + chk_object_issue(scope, "page", pgno, "invalid/corrupted", "%s-page", pagetype_caption); } else { if (unused_bytes > page_size) - chk_object_issue(scope, "page", pgno, "illegal unused-bytes", - "%s-page: %u < %" PRIuSIZE " < %u", pagetype_caption, 0, - unused_bytes, env->ps); + chk_object_issue(scope, "page", pgno, "illegal unused-bytes", "%s-page: %u < %" PRIuSIZE " < %u", + pagetype_caption, 0, unused_bytes, env->ps); - if (header_bytes < (int)sizeof(long) || - (size_t)header_bytes >= env->ps - sizeof(long)) { + if (header_bytes < (int)sizeof(long) || (size_t)header_bytes >= env->ps - sizeof(long)) { chk_object_issue(scope, "page", pgno, "illegal header-length", - "%s-page: %" PRIuSIZE " < %" PRIuSIZE " < %" PRIuSIZE, - pagetype_caption, sizeof(long), header_bytes, - env->ps - sizeof(long)); + "%s-page: %" PRIuSIZE " < %" PRIuSIZE " < %" PRIuSIZE, pagetype_caption, sizeof(long), + header_bytes, env->ps - sizeof(long)); } if (nentries < 1 || (pagetype == page_branch && nentries < 2)) { chk_object_issue(scope, "page", pgno, nentries ? "half-empty" : "empty", - "%s-page: payload %" PRIuSIZE " bytes, %" PRIuSIZE - " entries, deep %i", - pagetype_caption, payload_bytes, nentries, deep); + "%s-page: payload %" PRIuSIZE " bytes, %" PRIuSIZE " entries, deep %i", pagetype_caption, + payload_bytes, nentries, deep); tbl->pages.empty += 1; } if (npages) { if (page_bytes != page_size) { chk_object_issue(scope, "page", pgno, "misused", - "%s-page: %" PRIuPTR " != %" PRIuPTR " (%" PRIuPTR - "h + %" PRIuPTR "p + %" PRIuPTR "u), deep %i", - pagetype_caption, page_size, page_bytes, header_bytes, - payload_bytes, unused_bytes, deep); + "%s-page: %" PRIuPTR " != %" PRIuPTR " (%" PRIuPTR "h + %" PRIuPTR "p + %" PRIuPTR + "u), deep %i", + pagetype_caption, page_size, page_bytes, header_bytes, payload_bytes, unused_bytes, deep); if (page_size > page_bytes) tbl->lost_bytes += page_size - page_bytes; } else { @@ -1001,76 +903,53 @@ __cold static int chk_tree(MDBX_chk_scope_t *const scope) { chk_line_end(chk_print(chk_line_begin(usr->scope, MDBX_chk_resolution), "walked %zu pages, left/unused %zu" ", %" PRIuSIZE " problem(s)", - usr->result.processed_pages, - usr->result.unused_pages, - usr->scope->subtotal_issues)); + usr->result.processed_pages, usr->result.unused_pages, usr->scope->subtotal_issues)); err = chk_scope_restore(scope, err); if (scope->verbosity > MDBX_chk_info) { for (size_t i = 0; i < ARRAY_LENGTH(chk->table) && chk->table[i]; ++i) { MDBX_chk_table_t *const tbl = chk->table[i]; - MDBX_chk_scope_t *inner = - chk_scope_push(scope, 0, "tree %s:", chk_v2a(chk, &tbl->name)); + MDBX_chk_scope_t *inner = chk_scope_push(scope, 0, "tree %s:", chk_v2a(chk, &tbl->name)); if (tbl->pages.all == 0) - chk_line_end( - chk_print(chk_line_begin(inner, MDBX_chk_resolution), "empty")); + chk_line_end(chk_print(chk_line_begin(inner, MDBX_chk_resolution), "empty")); else { MDBX_chk_line_t *line = chk_line_begin(inner, MDBX_chk_info); if (line) { - line = chk_print(line, "page usage: subtotal %" PRIuSIZE, - tbl->pages.all); - const size_t branch_pages = - tbl->pages.branch + tbl->pages.nested_branch; - const size_t leaf_pages = tbl->pages.leaf + tbl->pages.nested_leaf + - tbl->pages.nested_subleaf; + line = chk_print(line, "page usage: subtotal %" PRIuSIZE, tbl->pages.all); + const size_t branch_pages = tbl->pages.branch + tbl->pages.nested_branch; + const size_t leaf_pages = tbl->pages.leaf + tbl->pages.nested_leaf + tbl->pages.nested_subleaf; if (tbl->pages.other) line = chk_print(line, ", other %" PRIuSIZE, tbl->pages.other); - if (tbl->pages.other == 0 || - (branch_pages | leaf_pages | tbl->histogram.large_pages.count) != - 0) { - line = chk_print(line, ", branch %" PRIuSIZE ", leaf %" PRIuSIZE, - branch_pages, leaf_pages); - if (tbl->histogram.large_pages.count || - (tbl->flags & MDBX_DUPSORT) == 0) { - line = chk_print(line, ", large %" PRIuSIZE, - tbl->histogram.large_pages.count); - if (tbl->histogram.large_pages.amount | - tbl->histogram.large_pages.count) - line = histogram_print(inner, line, &tbl->histogram.large_pages, - " amount", "single", true); + if (tbl->pages.other == 0 || (branch_pages | leaf_pages | tbl->histogram.large_pages.count) != 0) { + line = chk_print(line, ", branch %" PRIuSIZE ", leaf %" PRIuSIZE, branch_pages, leaf_pages); + if (tbl->histogram.large_pages.count || (tbl->flags & MDBX_DUPSORT) == 0) { + line = chk_print(line, ", large %" PRIuSIZE, tbl->histogram.large_pages.count); + if (tbl->histogram.large_pages.amount | tbl->histogram.large_pages.count) + line = histogram_print(inner, line, &tbl->histogram.large_pages, " amount", "single", true); } } - line = histogram_dist(chk_line_feed(line), &tbl->histogram.deep, - "tree deep density", "1", false); + line = histogram_dist(chk_line_feed(line), &tbl->histogram.deep, "tree deep density", "1", false); if (tbl != &chk->table_gc && tbl->histogram.nested_tree.count) { - line = chk_print(chk_line_feed(line), "nested tree(s) %" PRIuSIZE, - tbl->histogram.nested_tree.count); - line = histogram_dist(line, &tbl->histogram.nested_tree, " density", - "1", false); + line = chk_print(chk_line_feed(line), "nested tree(s) %" PRIuSIZE, tbl->histogram.nested_tree.count); + line = histogram_dist(line, &tbl->histogram.nested_tree, " density", "1", false); line = chk_print(chk_line_feed(line), - "nested tree(s) pages %" PRIuSIZE - ": branch %" PRIuSIZE ", leaf %" PRIuSIZE + "nested tree(s) pages %" PRIuSIZE ": branch %" PRIuSIZE ", leaf %" PRIuSIZE ", subleaf %" PRIuSIZE, - tbl->pages.nested_branch + tbl->pages.nested_leaf, - tbl->pages.nested_branch, tbl->pages.nested_leaf, - tbl->pages.nested_subleaf); + tbl->pages.nested_branch + tbl->pages.nested_leaf, tbl->pages.nested_branch, + tbl->pages.nested_leaf, tbl->pages.nested_subleaf); } const size_t bytes = pgno2bytes(env, tbl->pages.all); - line = chk_print( - chk_line_feed(line), - "page filling: subtotal %" PRIuSIZE - " bytes (%.1f%%), payload %" PRIuSIZE - " (%.1f%%), unused %" PRIuSIZE " (%.1f%%)", - bytes, bytes * 100.0 / total_page_bytes, tbl->payload_bytes, - tbl->payload_bytes * 100.0 / bytes, bytes - tbl->payload_bytes, - (bytes - tbl->payload_bytes) * 100.0 / bytes); + line = + chk_print(chk_line_feed(line), + "page filling: subtotal %" PRIuSIZE " bytes (%.1f%%), payload %" PRIuSIZE + " (%.1f%%), unused %" PRIuSIZE " (%.1f%%)", + bytes, bytes * 100.0 / total_page_bytes, tbl->payload_bytes, tbl->payload_bytes * 100.0 / bytes, + bytes - tbl->payload_bytes, (bytes - tbl->payload_bytes) * 100.0 / bytes); if (tbl->pages.empty) - line = chk_print(line, ", %" PRIuSIZE " empty pages", - tbl->pages.empty); + line = chk_print(line, ", %" PRIuSIZE " empty pages", tbl->pages.empty); if (tbl->lost_bytes) - line = - chk_print(line, ", %" PRIuSIZE " bytes lost", tbl->lost_bytes); + line = chk_print(line, ", %" PRIuSIZE " bytes lost", tbl->lost_bytes); chk_line_end(line); } } @@ -1080,14 +959,12 @@ __cold static int chk_tree(MDBX_chk_scope_t *const scope) { MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_resolution); line = chk_print(line, - "summary: total %" PRIuSIZE " bytes, payload %" PRIuSIZE - " (%.1f%%), unused %" PRIuSIZE " (%.1f%%)," + "summary: total %" PRIuSIZE " bytes, payload %" PRIuSIZE " (%.1f%%), unused %" PRIuSIZE " (%.1f%%)," " average fill %.1f%%", total_page_bytes, usr->result.total_payload_bytes, usr->result.total_payload_bytes * 100.0 / total_page_bytes, total_page_bytes - usr->result.total_payload_bytes, - (total_page_bytes - usr->result.total_payload_bytes) * - 100.0 / total_page_bytes, + (total_page_bytes - usr->result.total_payload_bytes) * 100.0 / total_page_bytes, usr->result.total_payload_bytes * 100.0 / total_page_bytes); if (total.pages.empty) line = chk_print(line, ", %" PRIuSIZE " empty pages", total.pages.empty); @@ -1097,14 +974,11 @@ __cold static int chk_tree(MDBX_chk_scope_t *const scope) { return err; } -typedef int(chk_kv_visitor)(MDBX_chk_scope_t *const scope, - MDBX_chk_table_t *tbl, const size_t record_number, +typedef int(chk_kv_visitor)(MDBX_chk_scope_t *const scope, MDBX_chk_table_t *tbl, const size_t record_number, const MDBX_val *key, const MDBX_val *data); -__cold static int chk_handle_kv(MDBX_chk_scope_t *const scope, - MDBX_chk_table_t *tbl, - const size_t record_number, const MDBX_val *key, - const MDBX_val *data) { +__cold static int chk_handle_kv(MDBX_chk_scope_t *const scope, MDBX_chk_table_t *tbl, const size_t record_number, + const MDBX_val *key, const MDBX_val *data) { MDBX_chk_internal_t *const chk = scope->internal; int err = MDBX_SUCCESS; assert(tbl->cookie); @@ -1113,8 +987,7 @@ __cold static int chk_handle_kv(MDBX_chk_scope_t *const scope, return err ? err : chk_check_break(scope); } -__cold static int chk_db(MDBX_chk_scope_t *const scope, MDBX_dbi dbi, - MDBX_chk_table_t *tbl, chk_kv_visitor *handler) { +__cold static int chk_db(MDBX_chk_scope_t *const scope, MDBX_dbi dbi, MDBX_chk_table_t *tbl, chk_kv_visitor *handler) { MDBX_chk_internal_t *const chk = scope->internal; MDBX_chk_context_t *const usr = chk->usr; MDBX_env *const env = usr->env; @@ -1124,27 +997,22 @@ __cold static int chk_db(MDBX_chk_scope_t *const scope, MDBX_dbi dbi, int err; if ((MDBX_TXN_FINISHED | MDBX_TXN_ERROR) & txn->flags) { - chk_line_end( - chk_flush(chk_print(chk_line_begin(scope, MDBX_chk_error), - "abort processing %s due to a previous error", - chk_v2a(chk, &tbl->name)))); + chk_line_end(chk_flush(chk_print(chk_line_begin(scope, MDBX_chk_error), + "abort processing %s due to a previous error", chk_v2a(chk, &tbl->name)))); err = MDBX_BAD_TXN; goto bailout; } if (0 > (int)dbi) { - err = dbi_open( - txn, &tbl->name, MDBX_DB_ACCEDE, &dbi, - (chk->flags & MDBX_CHK_IGNORE_ORDER) ? cmp_equal_or_greater : nullptr, - (chk->flags & MDBX_CHK_IGNORE_ORDER) ? cmp_equal_or_greater : nullptr); + err = dbi_open(txn, &tbl->name, MDBX_DB_ACCEDE, &dbi, + (chk->flags & MDBX_CHK_IGNORE_ORDER) ? cmp_equal_or_greater : nullptr, + (chk->flags & MDBX_CHK_IGNORE_ORDER) ? cmp_equal_or_greater : nullptr); if (unlikely(err)) { - tASSERT(txn, dbi >= txn->env->n_dbi || - (txn->env->dbs_flags[dbi] & DB_VALID) == 0); + tASSERT(txn, dbi >= txn->env->n_dbi || (txn->env->dbs_flags[dbi] & DB_VALID) == 0); chk_error_rc(scope, err, "mdbx_dbi_open"); goto bailout; } - tASSERT(txn, dbi < txn->env->n_dbi && - (txn->env->dbs_flags[dbi] & DB_VALID) != 0); + tASSERT(txn, dbi < txn->env->n_dbi && (txn->env->dbs_flags[dbi] & DB_VALID) != 0); } const tree_t *const db = txn->dbs + dbi; @@ -1165,13 +1033,11 @@ __cold static int chk_db(MDBX_chk_scope_t *const scope, MDBX_dbi dbi, break; default: key_mode = "inconsistent"; - chk_scope_issue(scope, "wrong key-mode (0x%x)", - tbl->flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)); + chk_scope_issue(scope, "wrong key-mode (0x%x)", tbl->flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)); } const char *value_mode = nullptr; - switch (tbl->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED | - MDBX_INTEGERDUP)) { + switch (tbl->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED | MDBX_INTEGERDUP)) { case 0: value_mode = "single"; break; @@ -1199,69 +1065,51 @@ __cold static int chk_db(MDBX_chk_scope_t *const scope, MDBX_dbi dbi, default: value_mode = "inconsistent"; chk_scope_issue(scope, "wrong value-mode (0x%x)", - tbl->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | - MDBX_DUPFIXED | MDBX_INTEGERDUP)); + tbl->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED | MDBX_INTEGERDUP)); } MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_info); - line = chk_print(line, "key-value kind: %s-key => %s-value", key_mode, - value_mode); + line = chk_print(line, "key-value kind: %s-key => %s-value", key_mode, value_mode); line = chk_print(line, ", flags:"); if (!tbl->flags) line = chk_print(line, " none"); else { - const uint8_t f[] = {MDBX_DUPSORT, - MDBX_INTEGERKEY, - MDBX_REVERSEKEY, - MDBX_DUPFIXED, - MDBX_REVERSEDUP, - MDBX_INTEGERDUP, - 0}; - const char *const t[] = {"dupsort", "integerkey", "reversekey", - "dupfix", "reversedup", "integerdup"}; + const uint8_t f[] = { + MDBX_DUPSORT, MDBX_INTEGERKEY, MDBX_REVERSEKEY, MDBX_DUPFIXED, MDBX_REVERSEDUP, MDBX_INTEGERDUP, 0}; + const char *const t[] = {"dupsort", "integerkey", "reversekey", "dupfix", "reversedup", "integerdup"}; for (size_t i = 0; f[i]; i++) if (tbl->flags & f[i]) line = chk_print(line, " %s", t[i]); } chk_line_end(chk_print(line, " (0x%02X)", tbl->flags)); - line = chk_print(chk_line_begin(scope, MDBX_chk_verbose), - "entries %" PRIu64 ", sequence %" PRIu64, db->items, + line = chk_print(chk_line_begin(scope, MDBX_chk_verbose), "entries %" PRIu64 ", sequence %" PRIu64, db->items, db->sequence); if (db->mod_txnid) - line = - chk_print(line, ", last modification txn#%" PRIaTXN, db->mod_txnid); + line = chk_print(line, ", last modification txn#%" PRIaTXN, db->mod_txnid); if (db->root != P_INVALID) line = chk_print(line, ", root #%" PRIaPGNO, db->root); chk_line_end(line); chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_verbose), - "b-tree depth %u, pages: branch %" PRIaPGNO - ", leaf %" PRIaPGNO ", large %" PRIaPGNO, - db->height, db->branch_pages, db->leaf_pages, - db->large_pages)); + "b-tree depth %u, pages: branch %" PRIaPGNO ", leaf %" PRIaPGNO ", large %" PRIaPGNO, + db->height, db->branch_pages, db->leaf_pages, db->large_pages)); if ((chk->flags & MDBX_CHK_SKIP_BTREE_TRAVERSAL) == 0) { const size_t branch_pages = tbl->pages.branch + tbl->pages.nested_branch; const size_t leaf_pages = tbl->pages.leaf + tbl->pages.nested_leaf; - const size_t subtotal_pages = - db->branch_pages + db->leaf_pages + db->large_pages; + const size_t subtotal_pages = db->branch_pages + db->leaf_pages + db->large_pages; if (subtotal_pages != tbl->pages.all) - chk_scope_issue( - scope, "%s pages mismatch (%" PRIuSIZE " != walked %" PRIuSIZE ")", - "subtotal", subtotal_pages, tbl->pages.all); + chk_scope_issue(scope, "%s pages mismatch (%" PRIuSIZE " != walked %" PRIuSIZE ")", "subtotal", subtotal_pages, + tbl->pages.all); if (db->branch_pages != branch_pages) - chk_scope_issue( - scope, "%s pages mismatch (%" PRIaPGNO " != walked %" PRIuSIZE ")", - "branch", db->branch_pages, branch_pages); + chk_scope_issue(scope, "%s pages mismatch (%" PRIaPGNO " != walked %" PRIuSIZE ")", "branch", db->branch_pages, + branch_pages); if (db->leaf_pages != leaf_pages) - chk_scope_issue( - scope, "%s pages mismatch (%" PRIaPGNO " != walked %" PRIuSIZE ")", - "all-leaf", db->leaf_pages, leaf_pages); + chk_scope_issue(scope, "%s pages mismatch (%" PRIaPGNO " != walked %" PRIuSIZE ")", "all-leaf", db->leaf_pages, + leaf_pages); if (db->large_pages != tbl->histogram.large_pages.amount) - chk_scope_issue( - scope, "%s pages mismatch (%" PRIaPGNO " != walked %" PRIuSIZE ")", - "large/overlow", db->large_pages, - tbl->histogram.large_pages.amount); + chk_scope_issue(scope, "%s pages mismatch (%" PRIaPGNO " != walked %" PRIuSIZE ")", "large/overlow", + db->large_pages, tbl->histogram.large_pages.amount); } } @@ -1287,31 +1135,24 @@ __cold static int chk_db(MDBX_chk_scope_t *const scope, MDBX_dbi dbi, bool bad_key = false; if (key.iov_len > maxkeysize) { - chk_object_issue(scope, "entry", record_count, - "key length exceeds max-key-size", - "%" PRIuPTR " > %" PRIuPTR, key.iov_len, maxkeysize); + chk_object_issue(scope, "entry", record_count, "key length exceeds max-key-size", "%" PRIuPTR " > %" PRIuPTR, + key.iov_len, maxkeysize); bad_key = true; - } else if ((tbl->flags & MDBX_INTEGERKEY) && key.iov_len != 8 && - key.iov_len != 4) { - chk_object_issue(scope, "entry", record_count, "wrong key length", - "%" PRIuPTR " != 4or8", key.iov_len); + } else if ((tbl->flags & MDBX_INTEGERKEY) && key.iov_len != 8 && key.iov_len != 4) { + chk_object_issue(scope, "entry", record_count, "wrong key length", "%" PRIuPTR " != 4or8", key.iov_len); bad_key = true; } bool bad_data = false; - if ((tbl->flags & MDBX_INTEGERDUP) && data.iov_len != 8 && - data.iov_len != 4) { - chk_object_issue(scope, "entry", record_count, "wrong data length", - "%" PRIuPTR " != 4or8", data.iov_len); + if ((tbl->flags & MDBX_INTEGERDUP) && data.iov_len != 8 && data.iov_len != 4) { + chk_object_issue(scope, "entry", record_count, "wrong data length", "%" PRIuPTR " != 4or8", data.iov_len); bad_data = true; } if (prev_key.iov_base) { - if (prev_data.iov_base && !bad_data && (tbl->flags & MDBX_DUPFIXED) && - prev_data.iov_len != data.iov_len) { - chk_object_issue(scope, "entry", record_count, "different data length", - "%" PRIuPTR " != %" PRIuPTR, prev_data.iov_len, - data.iov_len); + if (prev_data.iov_base && !bad_data && (tbl->flags & MDBX_DUPFIXED) && prev_data.iov_len != data.iov_len) { + chk_object_issue(scope, "entry", record_count, "different data length", "%" PRIuPTR " != %" PRIuPTR, + prev_data.iov_len, data.iov_len); bad_data = true; } @@ -1320,38 +1161,30 @@ __cold static int chk_db(MDBX_chk_scope_t *const scope, MDBX_dbi dbi, if (cmp == 0) { ++dups; if ((tbl->flags & MDBX_DUPSORT) == 0) { - chk_object_issue(scope, "entry", record_count, "duplicated entries", - nullptr); + chk_object_issue(scope, "entry", record_count, "duplicated entries", nullptr); if (prev_data.iov_base && data.iov_len == prev_data.iov_len && memcmp(data.iov_base, prev_data.iov_base, data.iov_len) == 0) - chk_object_issue(scope, "entry", record_count, - "complete duplicate", nullptr); + chk_object_issue(scope, "entry", record_count, "complete duplicate", nullptr); } else if (!bad_data && prev_data.iov_base) { cmp = mdbx_dcmp(txn, dbi, &data, &prev_data); if (cmp == 0) - chk_object_issue(scope, "entry", record_count, - "complete duplicate", nullptr); + chk_object_issue(scope, "entry", record_count, "complete duplicate", nullptr); else if (cmp < 0 && !(chk->flags & MDBX_CHK_IGNORE_ORDER)) - chk_object_issue(scope, "entry", record_count, - "wrong order of multi-values", nullptr); + chk_object_issue(scope, "entry", record_count, "wrong order of multi-values", nullptr); } } else if (cmp < 0 && !(chk->flags & MDBX_CHK_IGNORE_ORDER)) - chk_object_issue(scope, "entry", record_count, - "wrong order of entries", nullptr); + chk_object_issue(scope, "entry", record_count, "wrong order of entries", nullptr); } } if (!bad_key) { if (!prev_key.iov_base && (tbl->flags & MDBX_INTEGERKEY)) - chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_info), - "fixed key-size %" PRIuSIZE, key.iov_len)); + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_info), "fixed key-size %" PRIuSIZE, key.iov_len)); prev_key = key; } if (!bad_data) { - if (!prev_data.iov_base && - (tbl->flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED))) - chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_info), - "fixed data-size %" PRIuSIZE, data.iov_len)); + if (!prev_data.iov_base && (tbl->flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED))) + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_info), "fixed data-size %" PRIuSIZE, data.iov_len)); prev_data = data; } @@ -1359,17 +1192,13 @@ __cold static int chk_db(MDBX_chk_scope_t *const scope, MDBX_dbi dbi, histogram_acc(key.iov_len, &tbl->histogram.key_len); histogram_acc(data.iov_len, &tbl->histogram.val_len); - const node_t *const node = - page_node(cursor->pg[cursor->top], cursor->ki[cursor->top]); + const node_t *const node = page_node(cursor->pg[cursor->top], cursor->ki[cursor->top]); if (node_flags(node) == N_TREE) { - if (dbi != MAIN_DBI || (tbl->flags & (MDBX_DUPSORT | MDBX_DUPFIXED | - MDBX_REVERSEDUP | MDBX_INTEGERDUP))) - chk_object_issue(scope, "entry", record_count, "unexpected table", - "node-flags 0x%x", node_flags(node)); + if (dbi != MAIN_DBI || (tbl->flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP))) + chk_object_issue(scope, "entry", record_count, "unexpected table", "node-flags 0x%x", node_flags(node)); else if (data.iov_len != sizeof(tree_t)) - chk_object_issue(scope, "entry", record_count, "wrong table node size", - "node-size %" PRIuSIZE " != %" PRIuSIZE, data.iov_len, - sizeof(tree_t)); + chk_object_issue(scope, "entry", record_count, "wrong table node size", "node-size %" PRIuSIZE " != %" PRIuSIZE, + data.iov_len, sizeof(tree_t)); else if (scope->stage == MDBX_chk_maindb) /* подсчитываем table при первом проходе */ sub_databases += 1; @@ -1384,9 +1213,8 @@ __cold static int chk_db(MDBX_chk_scope_t *const scope, MDBX_dbi dbi, if (unlikely(err)) goto bailout; if (table->cookie) { - err = chk_scope_begin( - chk, 0, MDBX_chk_tables, table, &usr->result.problems_kv, - "Processing table %s...", chk_v2a(chk, &table->name)); + err = chk_scope_begin(chk, 0, MDBX_chk_tables, table, &usr->result.problems_kv, "Processing table %s...", + chk_v2a(chk, &table->name)); if (likely(!err)) { err = chk_db(usr->scope, (MDBX_dbi)-1, table, chk_handle_kv); if (err != MDBX_EINTR && err != MDBX_RESULT_TRUE) @@ -1396,9 +1224,8 @@ __cold static int chk_db(MDBX_chk_scope_t *const scope, MDBX_dbi dbi, if (unlikely(err)) goto bailout; } else - chk_line_end(chk_flush( - chk_print(chk_line_begin(scope, MDBX_chk_processing), - "Skip processing %s...", chk_v2a(chk, &table->name)))); + chk_line_end(chk_flush(chk_print(chk_line_begin(scope, MDBX_chk_processing), "Skip processing %s...", + chk_v2a(chk, &table->name)))); } } else if (handler) { err = handler(scope, tbl, record_count, &key, &data); @@ -1409,22 +1236,17 @@ __cold static int chk_db(MDBX_chk_scope_t *const scope, MDBX_dbi dbi, err = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT); } - err = (err != MDBX_NOTFOUND) ? chk_error_rc(scope, err, "mdbx_cursor_get") - : MDBX_SUCCESS; + err = (err != MDBX_NOTFOUND) ? chk_error_rc(scope, err, "mdbx_cursor_get") : MDBX_SUCCESS; if (err == MDBX_SUCCESS && record_count != db->items) - chk_scope_issue(scope, - "different number of entries %" PRIuSIZE " != %" PRIu64, - record_count, db->items); + chk_scope_issue(scope, "different number of entries %" PRIuSIZE " != %" PRIu64, record_count, db->items); bailout: if (cursor) { if (handler) { if (tbl->histogram.key_len.count) { MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_info); - line = histogram_dist(line, &tbl->histogram.key_len, - "key length density", "0/1", false); + line = histogram_dist(line, &tbl->histogram.key_len, "key length density", "0/1", false); chk_line_feed(line); - line = histogram_dist(line, &tbl->histogram.val_len, - "value length density", "0/1", false); + line = histogram_dist(line, &tbl->histogram.val_len, "value length density", "0/1", false); chk_line_end(line); } if (scope->stage == MDBX_chk_maindb) @@ -1433,8 +1255,7 @@ bailout: err = chk->cb->table_conclude(usr, tbl, cursor, err); MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_resolution); line = chk_print(line, "summary: %" PRIuSIZE " records,", record_count); - if (dups || (tbl->flags & (MDBX_DUPSORT | MDBX_DUPFIXED | - MDBX_REVERSEDUP | MDBX_INTEGERDUP))) + if (dups || (tbl->flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP))) line = chk_print(line, " %" PRIuSIZE " dups,", dups); if (sub_databases || dbi == MAIN_DBI) line = chk_print(line, " %" PRIuSIZE " tables,", sub_databases); @@ -1442,8 +1263,7 @@ bailout: " %" PRIuSIZE " key's bytes," " %" PRIuSIZE " data's bytes," " %" PRIuSIZE " problem(s)", - tbl->histogram.key_len.amount, - tbl->histogram.val_len.amount, scope->subtotal_issues); + tbl->histogram.key_len.amount, tbl->histogram.val_len.amount, scope->subtotal_issues); chk_line_end(chk_flush(line)); } @@ -1454,10 +1274,8 @@ bailout: return err; } -__cold static int chk_handle_gc(MDBX_chk_scope_t *const scope, - MDBX_chk_table_t *tbl, - const size_t record_number, const MDBX_val *key, - const MDBX_val *data) { +__cold static int chk_handle_gc(MDBX_chk_scope_t *const scope, MDBX_chk_table_t *tbl, const size_t record_number, + const MDBX_val *key, const MDBX_val *data) { MDBX_chk_internal_t *const chk = scope->internal; MDBX_chk_context_t *const usr = chk->usr; assert(tbl == &chk->table_gc); @@ -1466,25 +1284,20 @@ __cold static int chk_handle_gc(MDBX_chk_scope_t *const scope, pgno_t *iptr = data->iov_base; if (key->iov_len != sizeof(txnid_t)) - chk_object_issue(scope, "entry", record_number, "wrong txn-id size", - "key-size %" PRIuSIZE, key->iov_len); + chk_object_issue(scope, "entry", record_number, "wrong txn-id size", "key-size %" PRIuSIZE, key->iov_len); else { txnid_t txnid; memcpy(&txnid, key->iov_base, sizeof(txnid)); if (txnid < 1 || txnid > usr->txn->txnid) - chk_object_issue(scope, "entry", record_number, "wrong txn-id", - "%" PRIaTXN, txnid); + chk_object_issue(scope, "entry", record_number, "wrong txn-id", "%" PRIaTXN, txnid); else { if (data->iov_len < sizeof(pgno_t) || data->iov_len % sizeof(pgno_t)) - chk_object_issue(scope, "entry", txnid, "wrong idl size", "%" PRIuPTR, - data->iov_len); + chk_object_issue(scope, "entry", txnid, "wrong idl size", "%" PRIuPTR, data->iov_len); size_t number = (data->iov_len >= sizeof(pgno_t)) ? *iptr++ : 0; if (number > PAGELIST_LIMIT) - chk_object_issue(scope, "entry", txnid, "wrong idl length", "%" PRIuPTR, - number); + chk_object_issue(scope, "entry", txnid, "wrong idl length", "%" PRIuPTR, number); else if ((number + 1) * sizeof(pgno_t) > data->iov_len) { - chk_object_issue(scope, "entry", txnid, "trimmed idl", - "%" PRIuSIZE " > %" PRIuSIZE " (corruption)", + chk_object_issue(scope, "entry", txnid, "trimmed idl", "%" PRIuSIZE " > %" PRIuSIZE " (corruption)", (number + 1) * sizeof(pgno_t), data->iov_len); number = data->iov_len / sizeof(pgno_t) - 1; } else if (data->iov_len - (number + 1) * sizeof(pgno_t) >= @@ -1492,38 +1305,31 @@ __cold static int chk_handle_gc(MDBX_chk_scope_t *const scope, * and better than shink-and-retry inside gc_update() */ usr->env->ps) chk_object_issue(scope, "entry", txnid, "extra idl space", - "%" PRIuSIZE " < %" PRIuSIZE " (minor, not a trouble)", - (number + 1) * sizeof(pgno_t), data->iov_len); + "%" PRIuSIZE " < %" PRIuSIZE " (minor, not a trouble)", (number + 1) * sizeof(pgno_t), + data->iov_len); usr->result.gc_pages += number; if (chk->envinfo.mi_latter_reader_txnid > txnid) usr->result.reclaimable_pages += number; - size_t prev = - MDBX_PNL_ASCENDING ? NUM_METAS - 1 : usr->txn->geo.first_unallocated; + size_t prev = MDBX_PNL_ASCENDING ? NUM_METAS - 1 : usr->txn->geo.first_unallocated; size_t span = 1; for (size_t i = 0; i < number; ++i) { const size_t pgno = iptr[i]; if (pgno < NUM_METAS) - chk_object_issue(scope, "entry", txnid, "wrong idl entry", - "pgno %" PRIuSIZE " < meta-pages %u", pgno, + chk_object_issue(scope, "entry", txnid, "wrong idl entry", "pgno %" PRIuSIZE " < meta-pages %u", pgno, NUM_METAS); else if (pgno >= usr->result.backed_pages) - chk_object_issue(scope, "entry", txnid, "wrong idl entry", - "pgno %" PRIuSIZE " > backed-pages %" PRIuSIZE, pgno, - usr->result.backed_pages); + chk_object_issue(scope, "entry", txnid, "wrong idl entry", "pgno %" PRIuSIZE " > backed-pages %" PRIuSIZE, + pgno, usr->result.backed_pages); else if (pgno >= usr->result.alloc_pages) - chk_object_issue(scope, "entry", txnid, "wrong idl entry", - "pgno %" PRIuSIZE " > alloc-pages %" PRIuSIZE, pgno, - usr->result.alloc_pages - 1); + chk_object_issue(scope, "entry", txnid, "wrong idl entry", "pgno %" PRIuSIZE " > alloc-pages %" PRIuSIZE, + pgno, usr->result.alloc_pages - 1); else { if (MDBX_PNL_DISORDERED(prev, pgno)) { bad = " [bad sequence]"; - chk_object_issue( - scope, "entry", txnid, "bad sequence", - "%" PRIuSIZE " %c [%" PRIuSIZE "].%" PRIuSIZE, prev, - (prev == pgno) ? '=' : (MDBX_PNL_ASCENDING ? '>' : '<'), i, - pgno); + chk_object_issue(scope, "entry", txnid, "bad sequence", "%" PRIuSIZE " %c [%" PRIuSIZE "].%" PRIuSIZE, prev, + (prev == pgno) ? '=' : (MDBX_PNL_ASCENDING ? '>' : '<'), i, pgno); } if (chk->pagemap) { const intptr_t id = chk->pagemap[pgno]; @@ -1531,38 +1337,31 @@ __cold static int chk_handle_gc(MDBX_chk_scope_t *const scope, chk->pagemap[pgno] = -1 /* mark the pgno listed in GC */; else if (id > 0) { assert(id - 1 <= (intptr_t)ARRAY_LENGTH(chk->table)); - chk_object_issue(scope, "page", pgno, "already used", "by %s", - chk_v2a(chk, &chk->table[id - 1]->name)); + chk_object_issue(scope, "page", pgno, "already used", "by %s", chk_v2a(chk, &chk->table[id - 1]->name)); } else - chk_object_issue(scope, "page", pgno, "already listed in GC", - nullptr); + chk_object_issue(scope, "page", pgno, "already listed in GC", nullptr); } } prev = pgno; while (i + span < number && - iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span) - : pgno_sub(pgno, span))) + iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span) : pgno_sub(pgno, span))) ++span; } if (tbl->cookie) { chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_details), - "transaction %" PRIaTXN ", %" PRIuSIZE - " pages, maxspan %" PRIuSIZE "%s", - txnid, number, span, bad)); + "transaction %" PRIaTXN ", %" PRIuSIZE " pages, maxspan %" PRIuSIZE "%s", txnid, number, + span, bad)); for (size_t i = 0; i < number; i += span) { const size_t pgno = iptr[i]; - for (span = 1; - i + span < number && - iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span) - : pgno_sub(pgno, span)); + for (span = 1; i + span < number && + iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span) : pgno_sub(pgno, span)); ++span) ; histogram_acc(span, &tbl->histogram.nested_tree); MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_extra); if (line) { if (span > 1) - line = - chk_print(line, "%9" PRIuSIZE "[%" PRIuSIZE "]", pgno, span); + line = chk_print(line, "%9" PRIuSIZE "[%" PRIuSIZE "]", pgno, span); else line = chk_print(line, "%9" PRIuSIZE, pgno); chk_line_end(line); @@ -1582,26 +1381,21 @@ __cold static int env_chk(MDBX_chk_scope_t *const scope) { MDBX_chk_context_t *const usr = chk->usr; MDBX_env *const env = usr->env; MDBX_txn *const txn = usr->txn; - int err = - env_info(env, txn, &chk->envinfo, sizeof(chk->envinfo), &chk->troika); + int err = env_info(env, txn, &chk->envinfo, sizeof(chk->envinfo), &chk->troika); if (unlikely(err)) return chk_error_rc(scope, err, "env_info"); MDBX_chk_line_t *line = - chk_puts(chk_line_begin(scope, MDBX_chk_info - - (1 << MDBX_chk_severity_prio_shift)), - "dxb-id "); + chk_puts(chk_line_begin(scope, MDBX_chk_info - (1 << MDBX_chk_severity_prio_shift)), "dxb-id "); if (chk->envinfo.mi_dxbid.x | chk->envinfo.mi_dxbid.y) - line = chk_print(line, "%016" PRIx64 "-%016" PRIx64, - chk->envinfo.mi_dxbid.x, chk->envinfo.mi_dxbid.y); + line = chk_print(line, "%016" PRIx64 "-%016" PRIx64, chk->envinfo.mi_dxbid.x, chk->envinfo.mi_dxbid.y); else line = chk_puts(line, "is absent"); chk_line_end(line); line = chk_puts(chk_line_begin(scope, MDBX_chk_info), "current boot-id "); if (chk->envinfo.mi_bootid.current.x | chk->envinfo.mi_bootid.current.y) - line = chk_print(line, "%016" PRIx64 "-%016" PRIx64, - chk->envinfo.mi_bootid.current.x, + line = chk_print(line, "%016" PRIx64 "-%016" PRIx64, chk->envinfo.mi_bootid.current.x, chk->envinfo.mi_bootid.current.y); else line = chk_puts(line, "is unavailable"); @@ -1613,103 +1407,79 @@ __cold static int env_chk(MDBX_chk_scope_t *const scope) { //-------------------------------------------------------------------------- - err = chk_scope_begin(chk, 1, MDBX_chk_meta, nullptr, - &usr->result.problems_meta, "Peek the meta-pages..."); + err = chk_scope_begin(chk, 1, MDBX_chk_meta, nullptr, &usr->result.problems_meta, "Peek the meta-pages..."); if (likely(!err)) { MDBX_chk_scope_t *const inner = usr->scope; const uint64_t dxbfile_pages = env->dxb_mmap.filesize >> env->ps2ln; usr->result.alloc_pages = txn->geo.first_unallocated; usr->result.backed_pages = bytes2pgno(env, env->dxb_mmap.current); if (unlikely(usr->result.backed_pages > dxbfile_pages)) - chk_scope_issue(inner, "backed-pages %zu > file-pages %" PRIu64, - usr->result.backed_pages, dxbfile_pages); + chk_scope_issue(inner, "backed-pages %zu > file-pages %" PRIu64, usr->result.backed_pages, dxbfile_pages); if (unlikely(dxbfile_pages < NUM_METAS)) - chk_scope_issue(inner, "file-pages %" PRIu64 " < %u", dxbfile_pages, - NUM_METAS); + chk_scope_issue(inner, "file-pages %" PRIu64 " < %u", dxbfile_pages, NUM_METAS); if (unlikely(usr->result.backed_pages < NUM_METAS)) - chk_scope_issue(inner, "backed-pages %zu < %u", usr->result.backed_pages, - NUM_METAS); + chk_scope_issue(inner, "backed-pages %zu < %u", usr->result.backed_pages, NUM_METAS); if (unlikely(usr->result.backed_pages < NUM_METAS)) { - chk_scope_issue(inner, "backed-pages %zu < num-metas %u", - usr->result.backed_pages, NUM_METAS); + chk_scope_issue(inner, "backed-pages %zu < num-metas %u", usr->result.backed_pages, NUM_METAS); return MDBX_CORRUPTED; } if (unlikely(dxbfile_pages < NUM_METAS)) { - chk_scope_issue(inner, "backed-pages %zu < num-metas %u", - usr->result.backed_pages, NUM_METAS); + chk_scope_issue(inner, "backed-pages %zu < num-metas %u", usr->result.backed_pages, NUM_METAS); return MDBX_CORRUPTED; } if (unlikely(usr->result.backed_pages > (size_t)MAX_PAGENO + 1)) { - chk_scope_issue(inner, "backed-pages %zu > max-pages %zu", - usr->result.backed_pages, (size_t)MAX_PAGENO + 1); + chk_scope_issue(inner, "backed-pages %zu > max-pages %zu", usr->result.backed_pages, (size_t)MAX_PAGENO + 1); usr->result.backed_pages = MAX_PAGENO + 1; } if ((env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) { if (unlikely(usr->result.backed_pages > dxbfile_pages)) { - chk_scope_issue(inner, "backed-pages %zu > file-pages %" PRIu64, - usr->result.backed_pages, dxbfile_pages); + chk_scope_issue(inner, "backed-pages %zu > file-pages %" PRIu64, usr->result.backed_pages, dxbfile_pages); usr->result.backed_pages = (size_t)dxbfile_pages; } if (unlikely(usr->result.alloc_pages > usr->result.backed_pages)) { - chk_scope_issue(scope, "alloc-pages %zu > backed-pages %zu", - usr->result.alloc_pages, usr->result.backed_pages); + chk_scope_issue(scope, "alloc-pages %zu > backed-pages %zu", usr->result.alloc_pages, usr->result.backed_pages); usr->result.alloc_pages = usr->result.backed_pages; } } else { /* DB may be shrunk by writer down to the allocated (but unused) pages. */ if (unlikely(usr->result.alloc_pages > usr->result.backed_pages)) { - chk_scope_issue(inner, "alloc-pages %zu > backed-pages %zu", - usr->result.alloc_pages, usr->result.backed_pages); + chk_scope_issue(inner, "alloc-pages %zu > backed-pages %zu", usr->result.alloc_pages, usr->result.backed_pages); usr->result.alloc_pages = usr->result.backed_pages; } if (unlikely(usr->result.alloc_pages > dxbfile_pages)) { - chk_scope_issue(inner, "alloc-pages %zu > file-pages %" PRIu64, - usr->result.alloc_pages, dxbfile_pages); + chk_scope_issue(inner, "alloc-pages %zu > file-pages %" PRIu64, usr->result.alloc_pages, dxbfile_pages); usr->result.alloc_pages = (size_t)dxbfile_pages; } if (unlikely(usr->result.backed_pages > dxbfile_pages)) usr->result.backed_pages = (size_t)dxbfile_pages; } - line = chk_line_feed(chk_print( - chk_line_begin(inner, MDBX_chk_info), - "pagesize %u (%u system), max keysize %u..%u" - ", max readers %u", - env->ps, globals.sys_pagesize, - mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT), - mdbx_env_get_maxkeysize_ex(env, MDBX_DB_DEFAULTS), env->max_readers)); - line = chk_line_feed( - chk_print_size(line, "mapsize ", env->dxb_mmap.current, nullptr)); + line = chk_line_feed(chk_print(chk_line_begin(inner, MDBX_chk_info), + "pagesize %u (%u system), max keysize %u..%u" + ", max readers %u", + env->ps, globals.sys_pagesize, mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT), + mdbx_env_get_maxkeysize_ex(env, MDBX_DB_DEFAULTS), env->max_readers)); + line = chk_line_feed(chk_print_size(line, "mapsize ", env->dxb_mmap.current, nullptr)); if (txn->geo.lower == txn->geo.upper) - line = chk_print_size( - line, "fixed datafile: ", chk->envinfo.mi_geo.current, nullptr); + line = chk_print_size(line, "fixed datafile: ", chk->envinfo.mi_geo.current, nullptr); else { - line = chk_print_size( - line, "dynamic datafile: ", chk->envinfo.mi_geo.lower, nullptr); + line = chk_print_size(line, "dynamic datafile: ", chk->envinfo.mi_geo.lower, nullptr); line = chk_print_size(line, " .. ", chk->envinfo.mi_geo.upper, ", "); line = chk_print_size(line, "+", chk->envinfo.mi_geo.grow, ", "); - line = chk_line_feed( - chk_print_size(line, "-", chk->envinfo.mi_geo.shrink, nullptr)); - line = chk_print_size( - line, "current datafile: ", chk->envinfo.mi_geo.current, nullptr); + line = chk_line_feed(chk_print_size(line, "-", chk->envinfo.mi_geo.shrink, nullptr)); + line = chk_print_size(line, "current datafile: ", chk->envinfo.mi_geo.current, nullptr); } - tASSERT(txn, txn->geo.now == chk->envinfo.mi_geo.current / - chk->envinfo.mi_dxb_pagesize); + tASSERT(txn, txn->geo.now == chk->envinfo.mi_geo.current / chk->envinfo.mi_dxb_pagesize); chk_line_end(chk_print(line, ", %u pages", txn->geo.now)); #if defined(_WIN32) || defined(_WIN64) || MDBX_DEBUG - if (txn->geo.shrink_pv && txn->geo.now != txn->geo.upper && - scope->verbosity >= MDBX_chk_verbose) { + if (txn->geo.shrink_pv && txn->geo.now != txn->geo.upper && scope->verbosity >= MDBX_chk_verbose) { line = chk_line_begin(inner, MDBX_chk_notice); - chk_line_feed(chk_print( - line, " > WARNING: Due Windows system limitations a file couldn't")); - chk_line_feed(chk_print( - line, " > be truncated while the database is opened. So, the size")); - chk_line_feed(chk_print( - line, " > database file of may by large than the database itself,")); - chk_line_end(chk_print( - line, " > until it will be closed or reopened in read-write mode.")); + chk_line_feed(chk_print(line, " > WARNING: Due Windows system limitations a file couldn't")); + chk_line_feed(chk_print(line, " > be truncated while the database is opened. So, the size")); + chk_line_feed(chk_print(line, " > database file of may by large than the database itself,")); + chk_line_end(chk_print(line, " > until it will be closed or reopened in read-write mode.")); } #endif /* Windows || Debug */ chk_verbose_meta(inner, 0); @@ -1721,18 +1491,14 @@ __cold static int env_chk(MDBX_chk_scope_t *const scope) { "skip checking meta-pages since the %u" " is selected for verification", env->stuck_meta)); - line = chk_line_feed( - chk_print(chk_line_begin(inner, MDBX_chk_resolution), - "transactions: recent %" PRIu64 ", " - "selected for verification %" PRIu64 ", lag %" PRIi64, - chk->envinfo.mi_recent_txnid, - chk->envinfo.mi_meta_txnid[env->stuck_meta], - chk->envinfo.mi_recent_txnid - - chk->envinfo.mi_meta_txnid[env->stuck_meta])); + line = chk_line_feed(chk_print(chk_line_begin(inner, MDBX_chk_resolution), + "transactions: recent %" PRIu64 ", " + "selected for verification %" PRIu64 ", lag %" PRIi64, + chk->envinfo.mi_recent_txnid, chk->envinfo.mi_meta_txnid[env->stuck_meta], + chk->envinfo.mi_recent_txnid - chk->envinfo.mi_meta_txnid[env->stuck_meta])); chk_line_end(line); } else { - chk_line_end(chk_puts(chk_line_begin(inner, MDBX_chk_verbose), - "performs check for meta-pages clashes")); + chk_line_end(chk_puts(chk_line_begin(inner, MDBX_chk_verbose), "performs check for meta-pages clashes")); const unsigned meta_clash_mask = meta_eq_mask(&chk->troika); if (meta_clash_mask & 1) chk_scope_issue(inner, "meta-%d and meta-%d are clashed", 0, 1); @@ -1742,62 +1508,45 @@ __cold static int env_chk(MDBX_chk_scope_t *const scope) { chk_scope_issue(inner, "meta-%d and meta-%d are clashed", 2, 0); const unsigned prefer_steady_metanum = chk->troika.prefer_steady; - const uint64_t prefer_steady_txnid = - chk->troika.txnid[prefer_steady_metanum]; + const uint64_t prefer_steady_txnid = chk->troika.txnid[prefer_steady_metanum]; const unsigned recent_metanum = chk->troika.recent; const uint64_t recent_txnid = chk->troika.txnid[recent_metanum]; if (env->flags & MDBX_EXCLUSIVE) { chk_line_end( - chk_puts(chk_line_begin(inner, MDBX_chk_verbose), - "performs full check recent-txn-id with meta-pages")); + chk_puts(chk_line_begin(inner, MDBX_chk_verbose), "performs full check recent-txn-id with meta-pages")); eASSERT(env, recent_txnid == chk->envinfo.mi_recent_txnid); if (prefer_steady_txnid != recent_txnid) { - if ((chk->flags & MDBX_CHK_READWRITE) != 0 && - (env->flags & MDBX_RDONLY) == 0 && + if ((chk->flags & MDBX_CHK_READWRITE) != 0 && (env->flags & MDBX_RDONLY) == 0 && recent_txnid > prefer_steady_txnid && - (chk->envinfo.mi_bootid.current.x | - chk->envinfo.mi_bootid.current.y) != 0 && - chk->envinfo.mi_bootid.current.x == - chk->envinfo.mi_bootid.meta[recent_metanum].x && - chk->envinfo.mi_bootid.current.y == - chk->envinfo.mi_bootid.meta[recent_metanum].y) { - chk_line_end( - chk_print(chk_line_begin(inner, MDBX_chk_verbose), - "recent meta-%u is weak, but boot-id match current" - " (will synced upon successful check)", - recent_metanum)); + (chk->envinfo.mi_bootid.current.x | chk->envinfo.mi_bootid.current.y) != 0 && + chk->envinfo.mi_bootid.current.x == chk->envinfo.mi_bootid.meta[recent_metanum].x && + chk->envinfo.mi_bootid.current.y == chk->envinfo.mi_bootid.meta[recent_metanum].y) { + chk_line_end(chk_print(chk_line_begin(inner, MDBX_chk_verbose), + "recent meta-%u is weak, but boot-id match current" + " (will synced upon successful check)", + recent_metanum)); } else - chk_scope_issue( - inner, - "steady meta-%d txn-id mismatch recent-txn-id (%" PRIi64 - " != %" PRIi64 ")", - prefer_steady_metanum, prefer_steady_txnid, recent_txnid); + chk_scope_issue(inner, "steady meta-%d txn-id mismatch recent-txn-id (%" PRIi64 " != %" PRIi64 ")", + prefer_steady_metanum, prefer_steady_txnid, recent_txnid); } } else if (chk->write_locked) { - chk_line_end( - chk_puts(chk_line_begin(inner, MDBX_chk_verbose), - "performs lite check recent-txn-id with meta-pages (not a " - "monopolistic mode)")); + chk_line_end(chk_puts(chk_line_begin(inner, MDBX_chk_verbose), + "performs lite check recent-txn-id with meta-pages (not a " + "monopolistic mode)")); if (recent_txnid != chk->envinfo.mi_recent_txnid) { - chk_scope_issue(inner, - "weak meta-%d txn-id mismatch recent-txn-id (%" PRIi64 - " != %" PRIi64 ")", - recent_metanum, recent_txnid, - chk->envinfo.mi_recent_txnid); + chk_scope_issue(inner, "weak meta-%d txn-id mismatch recent-txn-id (%" PRIi64 " != %" PRIi64 ")", + recent_metanum, recent_txnid, chk->envinfo.mi_recent_txnid); } } else { - chk_line_end(chk_puts( - chk_line_begin(inner, MDBX_chk_verbose), - "skip check recent-txn-id with meta-pages (monopolistic or " - "read-write mode only)")); + chk_line_end(chk_puts(chk_line_begin(inner, MDBX_chk_verbose), + "skip check recent-txn-id with meta-pages (monopolistic or " + "read-write mode only)")); } - chk_line_end(chk_print( - chk_line_begin(inner, MDBX_chk_resolution), - "transactions: recent %" PRIu64 ", latter reader %" PRIu64 - ", lag %" PRIi64, - chk->envinfo.mi_recent_txnid, chk->envinfo.mi_latter_reader_txnid, - chk->envinfo.mi_recent_txnid - chk->envinfo.mi_latter_reader_txnid)); + chk_line_end(chk_print(chk_line_begin(inner, MDBX_chk_resolution), + "transactions: recent %" PRIu64 ", latter reader %" PRIu64 ", lag %" PRIi64, + chk->envinfo.mi_recent_txnid, chk->envinfo.mi_latter_reader_txnid, + chk->envinfo.mi_recent_txnid - chk->envinfo.mi_latter_reader_txnid)); } } err = chk_scope_restore(scope, err); @@ -1806,12 +1555,10 @@ __cold static int env_chk(MDBX_chk_scope_t *const scope) { const char *const subj_tree = "B-Trees"; if (chk->flags & MDBX_CHK_SKIP_BTREE_TRAVERSAL) - chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), - "Skipping %s traversal...", subj_tree)); + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), "Skipping %s traversal...", subj_tree)); else { - err = chk_scope_begin( - chk, -1, MDBX_chk_tree, nullptr, &usr->result.tree_problems, - "Traversal %s by txn#%" PRIaTXN "...", subj_tree, txn->txnid); + err = chk_scope_begin(chk, -1, MDBX_chk_tree, nullptr, &usr->result.tree_problems, + "Traversal %s by txn#%" PRIaTXN "...", subj_tree, txn->txnid); if (likely(!err)) err = chk_tree(usr->scope); if (usr->result.tree_problems && usr->result.gc_tree_problems == 0) @@ -1823,35 +1570,26 @@ __cold static int env_chk(MDBX_chk_scope_t *const scope) { const char *const subj_gc = chk_v2a(chk, MDBX_CHK_GC); if (usr->result.gc_tree_problems > 0) - chk_line_end(chk_print( - chk_line_begin(scope, MDBX_chk_processing), - "Skip processing %s since %s is corrupted (%" PRIuSIZE " problem(s))", - subj_gc, subj_tree, - usr->result.problems_gc = usr->result.gc_tree_problems)); + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), + "Skip processing %s since %s is corrupted (%" PRIuSIZE " problem(s))", subj_gc, subj_tree, + usr->result.problems_gc = usr->result.gc_tree_problems)); else { - err = chk_scope_begin( - chk, -1, MDBX_chk_gc, &chk->table_gc, &usr->result.problems_gc, - "Processing %s by txn#%" PRIaTXN "...", subj_gc, txn->txnid); + err = chk_scope_begin(chk, -1, MDBX_chk_gc, &chk->table_gc, &usr->result.problems_gc, + "Processing %s by txn#%" PRIaTXN "...", subj_gc, txn->txnid); if (likely(!err)) err = chk_db(usr->scope, FREE_DBI, &chk->table_gc, chk_handle_gc); line = chk_line_begin(scope, MDBX_chk_info); if (line) { - histogram_print(scope, line, &chk->table_gc.histogram.nested_tree, - "span(s)", "single", false); + histogram_print(scope, line, &chk->table_gc.histogram.nested_tree, "span(s)", "single", false); chk_line_end(line); } - if (usr->result.problems_gc == 0 && - (chk->flags & MDBX_CHK_SKIP_BTREE_TRAVERSAL) == 0) { + if (usr->result.problems_gc == 0 && (chk->flags & MDBX_CHK_SKIP_BTREE_TRAVERSAL) == 0) { const size_t used_pages = usr->result.alloc_pages - usr->result.gc_pages; if (usr->result.processed_pages != used_pages) - chk_scope_issue(usr->scope, - "used pages mismatch (%" PRIuSIZE - "(walked) != %" PRIuSIZE "(allocated - GC))", + chk_scope_issue(usr->scope, "used pages mismatch (%" PRIuSIZE "(walked) != %" PRIuSIZE "(allocated - GC))", usr->result.processed_pages, used_pages); if (usr->result.unused_pages != usr->result.gc_pages) - chk_scope_issue(usr->scope, - "GC pages mismatch (%" PRIuSIZE - "(expected) != %" PRIuSIZE "(GC))", + chk_scope_issue(usr->scope, "GC pages mismatch (%" PRIuSIZE "(expected) != %" PRIuSIZE "(GC))", usr->result.unused_pages, usr->result.gc_pages); } } @@ -1859,99 +1597,70 @@ __cold static int env_chk(MDBX_chk_scope_t *const scope) { //-------------------------------------------------------------------------- - err = chk_scope_begin(chk, 1, MDBX_chk_space, nullptr, nullptr, - "Page allocation:"); + err = chk_scope_begin(chk, 1, MDBX_chk_space, nullptr, nullptr, "Page allocation:"); const double percent_boundary_reciprocal = 100.0 / txn->geo.upper; const double percent_backed_reciprocal = 100.0 / usr->result.backed_pages; const size_t detained = usr->result.gc_pages - usr->result.reclaimable_pages; - const size_t available2boundary = - txn->geo.upper - usr->result.alloc_pages + usr->result.reclaimable_pages; - const size_t available2backed = usr->result.backed_pages - - usr->result.alloc_pages + - usr->result.reclaimable_pages; + const size_t available2boundary = txn->geo.upper - usr->result.alloc_pages + usr->result.reclaimable_pages; + const size_t available2backed = usr->result.backed_pages - usr->result.alloc_pages + usr->result.reclaimable_pages; const size_t remained2boundary = txn->geo.upper - usr->result.alloc_pages; - const size_t remained2backed = - usr->result.backed_pages - usr->result.alloc_pages; + const size_t remained2backed = usr->result.backed_pages - usr->result.alloc_pages; - const size_t used = (chk->flags & MDBX_CHK_SKIP_BTREE_TRAVERSAL) - ? usr->result.alloc_pages - usr->result.gc_pages - : usr->result.processed_pages; + const size_t used = (chk->flags & MDBX_CHK_SKIP_BTREE_TRAVERSAL) ? usr->result.alloc_pages - usr->result.gc_pages + : usr->result.processed_pages; line = chk_line_begin(usr->scope, MDBX_chk_info); line = chk_print(line, "backed by file: %" PRIuSIZE " pages (%.1f%%)" ", %" PRIuSIZE " left to boundary (%.1f%%)", - usr->result.backed_pages, - usr->result.backed_pages * percent_boundary_reciprocal, + usr->result.backed_pages, usr->result.backed_pages * percent_boundary_reciprocal, txn->geo.upper - usr->result.backed_pages, - (txn->geo.upper - usr->result.backed_pages) * - percent_boundary_reciprocal); + (txn->geo.upper - usr->result.backed_pages) * percent_boundary_reciprocal); line = chk_line_feed(line); - line = chk_print( - line, "%s: %" PRIuSIZE " page(s), %.1f%% of backed, %.1f%% of boundary", - "used", used, used * percent_backed_reciprocal, - used * percent_boundary_reciprocal); + line = chk_print(line, "%s: %" PRIuSIZE " page(s), %.1f%% of backed, %.1f%% of boundary", "used", used, + used * percent_backed_reciprocal, used * percent_boundary_reciprocal); line = chk_line_feed(line); - line = chk_print( - line, - "%s: %" PRIuSIZE " page(s) (%.1f%%) of backed, %" PRIuSIZE - " to boundary (%.1f%% of boundary)", - "remained", remained2backed, remained2backed * percent_backed_reciprocal, - remained2boundary, remained2boundary * percent_boundary_reciprocal); + line = chk_print(line, "%s: %" PRIuSIZE " page(s) (%.1f%%) of backed, %" PRIuSIZE " to boundary (%.1f%% of boundary)", + "remained", remained2backed, remained2backed * percent_backed_reciprocal, remained2boundary, + remained2boundary * percent_boundary_reciprocal); line = chk_line_feed(line); - line = chk_print( - line, - "reclaimable: %" PRIuSIZE " (%.1f%% of backed, %.1f%% of boundary)" - ", GC %" PRIuSIZE " (%.1f%% of backed, %.1f%% of boundary)", - usr->result.reclaimable_pages, - usr->result.reclaimable_pages * percent_backed_reciprocal, - usr->result.reclaimable_pages * percent_boundary_reciprocal, - usr->result.gc_pages, usr->result.gc_pages * percent_backed_reciprocal, - usr->result.gc_pages * percent_boundary_reciprocal); - line = chk_line_feed(line); - - line = chk_print( - line, - "detained by reader(s): %" PRIuSIZE - " (%.1f%% of backed, %.1f%% of boundary)" - ", %u reader(s), lag %" PRIi64, - detained, detained * percent_backed_reciprocal, - detained * percent_boundary_reciprocal, chk->envinfo.mi_numreaders, - chk->envinfo.mi_recent_txnid - chk->envinfo.mi_latter_reader_txnid); - line = chk_line_feed(line); - - line = chk_print( - line, "%s: %" PRIuSIZE " page(s), %.1f%% of backed, %.1f%% of boundary", - "allocated", usr->result.alloc_pages, - usr->result.alloc_pages * percent_backed_reciprocal, - usr->result.alloc_pages * percent_boundary_reciprocal); + line = + chk_print(line, + "reclaimable: %" PRIuSIZE " (%.1f%% of backed, %.1f%% of boundary)" + ", GC %" PRIuSIZE " (%.1f%% of backed, %.1f%% of boundary)", + usr->result.reclaimable_pages, usr->result.reclaimable_pages * percent_backed_reciprocal, + usr->result.reclaimable_pages * percent_boundary_reciprocal, usr->result.gc_pages, + usr->result.gc_pages * percent_backed_reciprocal, usr->result.gc_pages * percent_boundary_reciprocal); line = chk_line_feed(line); line = chk_print(line, - "%s: %" PRIuSIZE " page(s) (%.1f%%) of backed, %" PRIuSIZE - " to boundary (%.1f%% of boundary)", - "available", available2backed, - available2backed * percent_backed_reciprocal, - available2boundary, + "detained by reader(s): %" PRIuSIZE " (%.1f%% of backed, %.1f%% of boundary)" + ", %u reader(s), lag %" PRIi64, + detained, detained * percent_backed_reciprocal, detained * percent_boundary_reciprocal, + chk->envinfo.mi_numreaders, chk->envinfo.mi_recent_txnid - chk->envinfo.mi_latter_reader_txnid); + line = chk_line_feed(line); + + line = chk_print(line, "%s: %" PRIuSIZE " page(s), %.1f%% of backed, %.1f%% of boundary", "allocated", + usr->result.alloc_pages, usr->result.alloc_pages * percent_backed_reciprocal, + usr->result.alloc_pages * percent_boundary_reciprocal); + line = chk_line_feed(line); + + line = chk_print(line, "%s: %" PRIuSIZE " page(s) (%.1f%%) of backed, %" PRIuSIZE " to boundary (%.1f%% of boundary)", + "available", available2backed, available2backed * percent_backed_reciprocal, available2boundary, available2boundary * percent_boundary_reciprocal); chk_line_end(line); line = chk_line_begin(usr->scope, MDBX_chk_resolution); - line = chk_print(line, "%s %" PRIaPGNO " pages", - (txn->geo.upper == txn->geo.now) ? "total" : "upto", - txn->geo.upper); - line = chk_print(line, ", backed %" PRIuSIZE " (%.1f%%)", - usr->result.backed_pages, + line = chk_print(line, "%s %" PRIaPGNO " pages", (txn->geo.upper == txn->geo.now) ? "total" : "upto", txn->geo.upper); + line = chk_print(line, ", backed %" PRIuSIZE " (%.1f%%)", usr->result.backed_pages, usr->result.backed_pages * percent_boundary_reciprocal); - line = chk_print(line, ", allocated %" PRIuSIZE " (%.1f%%)", - usr->result.alloc_pages, + line = chk_print(line, ", allocated %" PRIuSIZE " (%.1f%%)", usr->result.alloc_pages, usr->result.alloc_pages * percent_boundary_reciprocal); - line = - chk_print(line, ", available %" PRIuSIZE " (%.1f%%)", available2boundary, - available2boundary * percent_boundary_reciprocal); + line = chk_print(line, ", available %" PRIuSIZE " (%.1f%%)", available2boundary, + available2boundary * percent_boundary_reciprocal); chk_line_end(line); chk_scope_restore(scope, err); @@ -1959,17 +1668,13 @@ __cold static int env_chk(MDBX_chk_scope_t *const scope) { const char *const subj_main = chk_v2a(chk, MDBX_CHK_MAIN); if (chk->flags & MDBX_CHK_SKIP_KV_TRAVERSAL) - chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), - "Skip processing %s...", subj_main)); + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), "Skip processing %s...", subj_main)); else if ((usr->result.problems_kv = usr->result.kv_tree_problems) > 0) - chk_line_end(chk_print( - chk_line_begin(scope, MDBX_chk_processing), - "Skip processing %s since %s is corrupted (%" PRIuSIZE " problem(s))", - subj_main, subj_tree, - usr->result.problems_kv = usr->result.kv_tree_problems)); + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), + "Skip processing %s since %s is corrupted (%" PRIuSIZE " problem(s))", subj_main, subj_tree, + usr->result.problems_kv = usr->result.kv_tree_problems)); else { - err = chk_scope_begin(chk, 0, MDBX_chk_maindb, &chk->table_main, - &usr->result.problems_kv, "Processing %s...", + err = chk_scope_begin(chk, 0, MDBX_chk_maindb, &chk->table_main, &usr->result.problems_kv, "Processing %s...", subj_main); if (likely(!err)) err = chk_db(usr->scope, MAIN_DBI, &chk->table_main, chk_handle_kv); @@ -1977,35 +1682,28 @@ __cold static int env_chk(MDBX_chk_scope_t *const scope) { const char *const subj_tables = "table(s)"; if (usr->result.problems_kv && usr->result.table_total) - chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), - "Skip processing %s", subj_tables)); + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), "Skip processing %s", subj_tables)); else if (usr->result.problems_kv == 0 && usr->result.table_total == 0) - chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_info), "No %s", - subj_tables)); + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_info), "No %s", subj_tables)); else if (usr->result.problems_kv == 0 && usr->result.table_total) { - err = chk_scope_begin( - chk, 1, MDBX_chk_tables, nullptr, &usr->result.problems_kv, - "Processing %s by txn#%" PRIaTXN "...", subj_tables, txn->txnid); + err = chk_scope_begin(chk, 1, MDBX_chk_tables, nullptr, &usr->result.problems_kv, + "Processing %s by txn#%" PRIaTXN "...", subj_tables, txn->txnid); if (!err) err = chk_db(usr->scope, MAIN_DBI, &chk->table_main, nullptr); if (usr->scope->subtotal_issues) chk_line_end(chk_print(chk_line_begin(usr->scope, MDBX_chk_resolution), - "processed %" PRIuSIZE " of %" PRIuSIZE - " %s, %" PRIuSIZE " problems(s)", - usr->result.table_processed, - usr->result.table_total, subj_tables, + "processed %" PRIuSIZE " of %" PRIuSIZE " %s, %" PRIuSIZE " problems(s)", + usr->result.table_processed, usr->result.table_total, subj_tables, usr->scope->subtotal_issues)); } chk_scope_restore(scope, err); } - return chk_scope_end(chk, chk_scope_begin(chk, 0, MDBX_chk_conclude, nullptr, - nullptr, nullptr)); + return chk_scope_end(chk, chk_scope_begin(chk, 0, MDBX_chk_conclude, nullptr, nullptr, nullptr)); } __cold int mdbx_env_chk_encount_problem(MDBX_chk_context_t *ctx) { - if (likely(ctx && ctx->internal && ctx->internal->usr == ctx && - ctx->internal->problem_counter && ctx->scope)) { + if (likely(ctx && ctx->internal && ctx->internal->usr == ctx && ctx->internal->problem_counter && ctx->scope)) { *ctx->internal->problem_counter += 1; ctx->scope->subtotal_issues += 1; return MDBX_SUCCESS; @@ -2013,10 +1711,8 @@ __cold int mdbx_env_chk_encount_problem(MDBX_chk_context_t *ctx) { return MDBX_EINVAL; } -__cold int mdbx_env_chk(MDBX_env *env, const struct MDBX_chk_callbacks *cb, - MDBX_chk_context_t *ctx, const MDBX_chk_flags_t flags, - MDBX_chk_severity_t verbosity, - unsigned timeout_seconds_16dot16) { +__cold int mdbx_env_chk(MDBX_env *env, const struct MDBX_chk_callbacks *cb, MDBX_chk_context_t *ctx, + const MDBX_chk_flags_t flags, MDBX_chk_severity_t verbosity, unsigned timeout_seconds_16dot16) { int err, rc = check_env(env, false); if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); @@ -2042,9 +1738,7 @@ __cold int mdbx_env_chk(MDBX_env *env, const struct MDBX_chk_callbacks *cb, chk->table[MAIN_DBI] = &chk->table_main; chk->monotime_timeout = - timeout_seconds_16dot16 - ? osal_16dot16_to_monotime(timeout_seconds_16dot16) + osal_monotime() - : 0; + timeout_seconds_16dot16 ? osal_16dot16_to_monotime(timeout_seconds_16dot16) + osal_monotime() : 0; chk->usr->scope_nesting = 0; chk->usr->result.tables = (const void *)&chk->table; @@ -2053,16 +1747,13 @@ __cold int mdbx_env_chk(MDBX_env *env, const struct MDBX_chk_callbacks *cb, top->internal = chk; // init - rc = chk_scope_end( - chk, chk_scope_begin(chk, 0, MDBX_chk_init, nullptr, nullptr, nullptr)); + rc = chk_scope_end(chk, chk_scope_begin(chk, 0, MDBX_chk_init, nullptr, nullptr, nullptr)); // lock if (likely(!rc)) - rc = chk_scope_begin( - chk, 0, MDBX_chk_lock, nullptr, nullptr, "Taking %slock...", - (env->flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) ? "" : "read "); - if (likely(!rc) && (env->flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == 0 && - (flags & MDBX_CHK_READWRITE)) { + rc = chk_scope_begin(chk, 0, MDBX_chk_lock, nullptr, nullptr, "Taking %slock...", + (env->flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) ? "" : "read "); + if (likely(!rc) && (env->flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == 0 && (flags & MDBX_CHK_READWRITE)) { rc = mdbx_txn_lock(env, false); if (unlikely(rc)) chk_error_rc(ctx->scope, rc, "mdbx_txn_lock"); diff --git a/src/cogs.c b/src/cogs.c index 2c505fbe..2e7d18c6 100644 --- a/src/cogs.c +++ b/src/cogs.c @@ -74,27 +74,22 @@ __cold bool pv2pages_verify(void) { /*----------------------------------------------------------------------------*/ -MDBX_NOTHROW_PURE_FUNCTION size_t bytes_align2os_bytes(const MDBX_env *env, - size_t bytes) { - return ceil_powerof2( - bytes, (env->ps > globals.sys_pagesize) ? env->ps : globals.sys_pagesize); +MDBX_NOTHROW_PURE_FUNCTION size_t bytes_align2os_bytes(const MDBX_env *env, size_t bytes) { + return ceil_powerof2(bytes, (env->ps > globals.sys_pagesize) ? env->ps : globals.sys_pagesize); } -MDBX_NOTHROW_PURE_FUNCTION size_t pgno_align2os_bytes(const MDBX_env *env, - size_t pgno) { +MDBX_NOTHROW_PURE_FUNCTION size_t pgno_align2os_bytes(const MDBX_env *env, size_t pgno) { return ceil_powerof2(pgno2bytes(env, pgno), globals.sys_pagesize); } -MDBX_NOTHROW_PURE_FUNCTION pgno_t pgno_align2os_pgno(const MDBX_env *env, - size_t pgno) { +MDBX_NOTHROW_PURE_FUNCTION pgno_t pgno_align2os_pgno(const MDBX_env *env, size_t pgno) { return bytes2pgno(env, pgno_align2os_bytes(env, pgno)); } /*----------------------------------------------------------------------------*/ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline int -cmp_int_inline(const size_t expected_alignment, const MDBX_val *a, - const MDBX_val *b) { +MDBX_NOTHROW_PURE_FUNCTION static __always_inline int cmp_int_inline(const size_t expected_alignment, const MDBX_val *a, + const MDBX_val *b) { if (likely(a->iov_len == b->iov_len)) { if (sizeof(size_t) > 7 && likely(a->iov_len == 8)) return CMP2INT(unaligned_peek_u64(expected_alignment, a->iov_base), @@ -106,35 +101,31 @@ cmp_int_inline(const size_t expected_alignment, const MDBX_val *a, return CMP2INT(unaligned_peek_u64(expected_alignment, a->iov_base), unaligned_peek_u64(expected_alignment, b->iov_base)); } - ERROR("mismatch and/or invalid size %p.%zu/%p.%zu for INTEGERKEY/INTEGERDUP", - a->iov_base, a->iov_len, b->iov_base, b->iov_len); + ERROR("mismatch and/or invalid size %p.%zu/%p.%zu for INTEGERKEY/INTEGERDUP", a->iov_base, a->iov_len, b->iov_base, + b->iov_len); return 0; } -MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_unaligned(const MDBX_val *a, - const MDBX_val *b) { +MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_unaligned(const MDBX_val *a, const MDBX_val *b) { return cmp_int_inline(1, a, b); } #ifndef cmp_int_align2 /* Compare two items pointing at 2-byte aligned unsigned int's. */ -MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_align2(const MDBX_val *a, - const MDBX_val *b) { +MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_align2(const MDBX_val *a, const MDBX_val *b) { return cmp_int_inline(2, a, b); } #endif /* cmp_int_align2 */ #ifndef cmp_int_align4 /* Compare two items pointing at 4-byte aligned unsigned int's. */ -MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_align4(const MDBX_val *a, - const MDBX_val *b) { +MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_align4(const MDBX_val *a, const MDBX_val *b) { return cmp_int_inline(4, a, b); } #endif /* cmp_int_align4 */ /* Compare two items lexically */ -MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lexical(const MDBX_val *a, - const MDBX_val *b) { +MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lexical(const MDBX_val *a, const MDBX_val *b) { if (a->iov_len == b->iov_len) return a->iov_len ? memcmp(a->iov_base, b->iov_base, a->iov_len) : 0; @@ -144,8 +135,7 @@ MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lexical(const MDBX_val *a, return likely(diff_data) ? diff_data : diff_len; } -MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned -tail3le(const uint8_t *p, size_t l) { +MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned tail3le(const uint8_t *p, size_t l) { STATIC_ASSERT(sizeof(unsigned) > 2); // 1: 0 0 0 // 2: 0 1 1 @@ -154,8 +144,7 @@ tail3le(const uint8_t *p, size_t l) { } /* Compare two items in reverse byte order */ -MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_reverse(const MDBX_val *a, - const MDBX_val *b) { +MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_reverse(const MDBX_val *a, const MDBX_val *b) { size_t left = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len; if (likely(left)) { const uint8_t *pa = ptr_disp(a->iov_base, a->iov_len); @@ -209,25 +198,19 @@ MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_reverse(const MDBX_val *a, } /* Fast non-lexically comparator */ -MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lenfast(const MDBX_val *a, - const MDBX_val *b) { +MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lenfast(const MDBX_val *a, const MDBX_val *b) { int diff = CMP2INT(a->iov_len, b->iov_len); - return (likely(diff) || a->iov_len == 0) - ? diff - : memcmp(a->iov_base, b->iov_base, a->iov_len); + return (likely(diff) || a->iov_len == 0) ? diff : memcmp(a->iov_base, b->iov_base, a->iov_len); } -MDBX_NOTHROW_PURE_FUNCTION __hot bool -eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l) { +MDBX_NOTHROW_PURE_FUNCTION __hot bool eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l) { if (likely(l > 3)) { if (MDBX_UNALIGNED_OK >= 4 && likely(l < 9)) return ((unaligned_peek_u32(1, a) - unaligned_peek_u32(1, b)) | - (unaligned_peek_u32(1, a + l - 4) - - unaligned_peek_u32(1, b + l - 4))) == 0; + (unaligned_peek_u32(1, a + l - 4) - unaligned_peek_u32(1, b + l - 4))) == 0; if (MDBX_UNALIGNED_OK >= 8 && sizeof(size_t) > 7 && likely(l < 17)) return ((unaligned_peek_u64(1, a) - unaligned_peek_u64(1, b)) | - (unaligned_peek_u64(1, a + l - 8) - - unaligned_peek_u64(1, b + l - 8))) == 0; + (unaligned_peek_u64(1, a + l - 8) - unaligned_peek_u64(1, b + l - 8))) == 0; return memcmp(a, b, l) == 0; } if (likely(l)) @@ -235,31 +218,21 @@ eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l) { return true; } -int cmp_equal_or_greater(const MDBX_val *a, const MDBX_val *b) { - return eq_fast(a, b) ? 0 : 1; -} +int cmp_equal_or_greater(const MDBX_val *a, const MDBX_val *b) { return eq_fast(a, b) ? 0 : 1; } -int cmp_equal_or_wrong(const MDBX_val *a, const MDBX_val *b) { - return eq_fast(a, b) ? 0 : -1; -} +int cmp_equal_or_wrong(const MDBX_val *a, const MDBX_val *b) { return eq_fast(a, b) ? 0 : -1; } /*----------------------------------------------------------------------------*/ -__cold void update_mlcnt(const MDBX_env *env, - const pgno_t new_aligned_mlocked_pgno, - const bool lock_not_release) { +__cold void update_mlcnt(const MDBX_env *env, const pgno_t new_aligned_mlocked_pgno, const bool lock_not_release) { for (;;) { - const pgno_t mlock_pgno_before = - atomic_load32(&env->mlocked_pgno, mo_AcquireRelease); - eASSERT(env, - pgno_align2os_pgno(env, mlock_pgno_before) == mlock_pgno_before); - eASSERT(env, pgno_align2os_pgno(env, new_aligned_mlocked_pgno) == - new_aligned_mlocked_pgno); + const pgno_t mlock_pgno_before = atomic_load32(&env->mlocked_pgno, mo_AcquireRelease); + eASSERT(env, pgno_align2os_pgno(env, mlock_pgno_before) == mlock_pgno_before); + eASSERT(env, pgno_align2os_pgno(env, new_aligned_mlocked_pgno) == new_aligned_mlocked_pgno); if (lock_not_release ? (mlock_pgno_before >= new_aligned_mlocked_pgno) : (mlock_pgno_before <= new_aligned_mlocked_pgno)) break; - if (likely(atomic_cas32(&((MDBX_env *)env)->mlocked_pgno, mlock_pgno_before, - new_aligned_mlocked_pgno))) + if (likely(atomic_cas32(&((MDBX_env *)env)->mlocked_pgno, mlock_pgno_before, new_aligned_mlocked_pgno))) for (;;) { mdbx_atomic_uint32_t *const mlcnt = env->lck->mlcnt; const int32_t snap_locked = atomic_load32(mlcnt + 0, mo_Relaxed); @@ -269,52 +242,39 @@ __cold void update_mlcnt(const MDBX_env *env, if (unlikely(!atomic_cas32(mlcnt + 0, snap_locked, snap_locked + 1))) continue; } - if (new_aligned_mlocked_pgno == 0 && - (snap_locked - snap_unlocked) > 0) { + if (new_aligned_mlocked_pgno == 0 && (snap_locked - snap_unlocked) > 0) { eASSERT(env, !lock_not_release); - if (unlikely( - !atomic_cas32(mlcnt + 1, snap_unlocked, snap_unlocked + 1))) + if (unlikely(!atomic_cas32(mlcnt + 1, snap_unlocked, snap_unlocked + 1))) continue; } - NOTICE("%s-pages %u..%u, mlocked-process(es) %u -> %u", - lock_not_release ? "lock" : "unlock", + NOTICE("%s-pages %u..%u, mlocked-process(es) %u -> %u", lock_not_release ? "lock" : "unlock", lock_not_release ? mlock_pgno_before : new_aligned_mlocked_pgno, - lock_not_release ? new_aligned_mlocked_pgno : mlock_pgno_before, - snap_locked - snap_unlocked, - atomic_load32(mlcnt + 0, mo_Relaxed) - - atomic_load32(mlcnt + 1, mo_Relaxed)); + lock_not_release ? new_aligned_mlocked_pgno : mlock_pgno_before, snap_locked - snap_unlocked, + atomic_load32(mlcnt + 0, mo_Relaxed) - atomic_load32(mlcnt + 1, mo_Relaxed)); return; } } } -__cold void munlock_after(const MDBX_env *env, const pgno_t aligned_pgno, - const size_t end_bytes) { +__cold void munlock_after(const MDBX_env *env, const pgno_t aligned_pgno, const size_t end_bytes) { if (atomic_load32(&env->mlocked_pgno, mo_AcquireRelease) > aligned_pgno) { int err = MDBX_ENOSYS; const size_t munlock_begin = pgno2bytes(env, aligned_pgno); const size_t munlock_size = end_bytes - munlock_begin; - eASSERT(env, end_bytes % globals.sys_pagesize == 0 && - munlock_begin % globals.sys_pagesize == 0 && + eASSERT(env, end_bytes % globals.sys_pagesize == 0 && munlock_begin % globals.sys_pagesize == 0 && munlock_size % globals.sys_pagesize == 0); #if defined(_WIN32) || defined(_WIN64) - err = - VirtualUnlock(ptr_disp(env->dxb_mmap.base, munlock_begin), munlock_size) - ? MDBX_SUCCESS - : (int)GetLastError(); + err = VirtualUnlock(ptr_disp(env->dxb_mmap.base, munlock_begin), munlock_size) ? MDBX_SUCCESS : (int)GetLastError(); if (err == ERROR_NOT_LOCKED) err = MDBX_SUCCESS; #elif defined(_POSIX_MEMLOCK_RANGE) - err = munlock(ptr_disp(env->dxb_mmap.base, munlock_begin), munlock_size) - ? errno - : MDBX_SUCCESS; + err = munlock(ptr_disp(env->dxb_mmap.base, munlock_begin), munlock_size) ? errno : MDBX_SUCCESS; #endif if (likely(err == MDBX_SUCCESS)) update_mlcnt(env, aligned_pgno, false); else { #if defined(_WIN32) || defined(_WIN64) - WARNING("VirtualUnlock(%zu, %zu) error %d", munlock_begin, munlock_size, - err); + WARNING("VirtualUnlock(%zu, %zu) error %d", munlock_begin, munlock_size, err); #else WARNING("munlock(%zu, %zu) error %d", munlock_begin, munlock_size, err); #endif @@ -332,13 +292,11 @@ uint32_t combine_durability_flags(const uint32_t a, const uint32_t b) { uint32_t r = a | b; /* avoid false MDBX_UTTERLY_NOSYNC */ - if (F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && - !F_ISSET(b, MDBX_UTTERLY_NOSYNC)) + if (F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && !F_ISSET(b, MDBX_UTTERLY_NOSYNC)) r = (r - MDBX_UTTERLY_NOSYNC) | MDBX_SAFE_NOSYNC; /* convert DEPRECATED_MAPASYNC to MDBX_SAFE_NOSYNC */ - if ((r & (MDBX_WRITEMAP | DEPRECATED_MAPASYNC)) == - (MDBX_WRITEMAP | DEPRECATED_MAPASYNC) && + if ((r & (MDBX_WRITEMAP | DEPRECATED_MAPASYNC)) == (MDBX_WRITEMAP | DEPRECATED_MAPASYNC) && !F_ISSET(r, MDBX_UTTERLY_NOSYNC)) r = (r - DEPRECATED_MAPASYNC) | MDBX_SAFE_NOSYNC; @@ -346,8 +304,6 @@ uint32_t combine_durability_flags(const uint32_t a, const uint32_t b) { if (r & (MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC)) r |= MDBX_NOMETASYNC; - assert(!(F_ISSET(r, MDBX_UTTERLY_NOSYNC) && - !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && - !F_ISSET(b, MDBX_UTTERLY_NOSYNC))); + assert(!(F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && !F_ISSET(b, MDBX_UTTERLY_NOSYNC))); return r; } diff --git a/src/cogs.h b/src/cogs.h index 705900bc..41d79b40 100644 --- a/src/cogs.h +++ b/src/cogs.h @@ -51,19 +51,15 @@ MDBX_MAYBE_UNUSED MDBX_INTERNAL bool pv2pages_verify(void); #define PAGESPACE(pagesize) ((pagesize) - PAGEHDRSZ) -#define BRANCH_NODE_MAX(pagesize) \ - (EVEN_FLOOR((PAGESPACE(pagesize) - sizeof(indx_t) - NODESIZE) / (3 - 1) - \ - sizeof(indx_t))) +#define BRANCH_NODE_MAX(pagesize) \ + (EVEN_FLOOR((PAGESPACE(pagesize) - sizeof(indx_t) - NODESIZE) / (3 - 1) - sizeof(indx_t))) -#define LEAF_NODE_MAX(pagesize) \ - (EVEN_FLOOR(PAGESPACE(pagesize) / 2) - sizeof(indx_t)) +#define LEAF_NODE_MAX(pagesize) (EVEN_FLOOR(PAGESPACE(pagesize) / 2) - sizeof(indx_t)) #define MAX_GC1OVPAGE(pagesize) (PAGESPACE(pagesize) / sizeof(pgno_t) - 1) -MDBX_NOTHROW_CONST_FUNCTION static inline size_t -keysize_max(size_t pagesize, MDBX_db_flags_t flags) { - assert(pagesize >= MDBX_MIN_PAGESIZE && pagesize <= MDBX_MAX_PAGESIZE && - is_powerof2(pagesize)); +MDBX_NOTHROW_CONST_FUNCTION static inline size_t keysize_max(size_t pagesize, MDBX_db_flags_t flags) { + assert(pagesize >= MDBX_MIN_PAGESIZE && pagesize <= MDBX_MAX_PAGESIZE && is_powerof2(pagesize)); STATIC_ASSERT(BRANCH_NODE_MAX(MDBX_MIN_PAGESIZE) - NODESIZE >= 8); if (flags & MDBX_INTEGERKEY) return 8 /* sizeof(uint64_t) */; @@ -72,18 +68,14 @@ keysize_max(size_t pagesize, MDBX_db_flags_t flags) { STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) - NODESIZE - /* sizeof(uint64) as a key */ 8 > sizeof(tree_t)); - if (flags & - (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) { - const intptr_t max_dupsort_leaf_key = - LEAF_NODE_MAX(pagesize) - NODESIZE - sizeof(tree_t); - return (max_branch_key < max_dupsort_leaf_key) ? max_branch_key - : max_dupsort_leaf_key; + if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) { + const intptr_t max_dupsort_leaf_key = LEAF_NODE_MAX(pagesize) - NODESIZE - sizeof(tree_t); + return (max_branch_key < max_dupsort_leaf_key) ? max_branch_key : max_dupsort_leaf_key; } return max_branch_key; } -MDBX_NOTHROW_CONST_FUNCTION static inline size_t -env_keysize_max(const MDBX_env *env, MDBX_db_flags_t flags) { +MDBX_NOTHROW_CONST_FUNCTION static inline size_t env_keysize_max(const MDBX_env *env, MDBX_db_flags_t flags) { size_t size_max; if (flags & MDBX_INTEGERKEY) size_max = 8 /* sizeof(uint64_t) */; @@ -92,12 +84,9 @@ env_keysize_max(const MDBX_env *env, MDBX_db_flags_t flags) { STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) - NODESIZE - /* sizeof(uint64) as a key */ 8 > sizeof(tree_t)); - if (flags & - (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) { - const intptr_t max_dupsort_leaf_key = - env->leaf_nodemax - NODESIZE - sizeof(tree_t); - size_max = (max_branch_key < max_dupsort_leaf_key) ? max_branch_key - : max_dupsort_leaf_key; + if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) { + const intptr_t max_dupsort_leaf_key = env->leaf_nodemax - NODESIZE - sizeof(tree_t); + size_max = (max_branch_key < max_dupsort_leaf_key) ? max_branch_key : max_dupsort_leaf_key; } else size_max = max_branch_key; } @@ -105,13 +94,11 @@ env_keysize_max(const MDBX_env *env, MDBX_db_flags_t flags) { return size_max; } -MDBX_NOTHROW_CONST_FUNCTION static inline size_t -keysize_min(MDBX_db_flags_t flags) { +MDBX_NOTHROW_CONST_FUNCTION static inline size_t keysize_min(MDBX_db_flags_t flags) { return (flags & MDBX_INTEGERKEY) ? 4 /* sizeof(uint32_t) */ : 0; } -MDBX_NOTHROW_CONST_FUNCTION static inline size_t -valsize_min(MDBX_db_flags_t flags) { +MDBX_NOTHROW_CONST_FUNCTION static inline size_t valsize_min(MDBX_db_flags_t flags) { if (flags & MDBX_INTEGERDUP) return 4 /* sizeof(uint32_t) */; else if (flags & MDBX_DUPFIXED) @@ -120,10 +107,8 @@ valsize_min(MDBX_db_flags_t flags) { return 0; } -MDBX_NOTHROW_CONST_FUNCTION static inline size_t -valsize_max(size_t pagesize, MDBX_db_flags_t flags) { - assert(pagesize >= MDBX_MIN_PAGESIZE && pagesize <= MDBX_MAX_PAGESIZE && - is_powerof2(pagesize)); +MDBX_NOTHROW_CONST_FUNCTION static inline size_t valsize_max(size_t pagesize, MDBX_db_flags_t flags) { + assert(pagesize >= MDBX_MIN_PAGESIZE && pagesize <= MDBX_MAX_PAGESIZE && is_powerof2(pagesize)); if (flags & MDBX_INTEGERDUP) return 8 /* sizeof(uint64_t) */; @@ -136,13 +121,11 @@ valsize_max(size_t pagesize, MDBX_db_flags_t flags) { const size_t hard_pages = hard >> page_ln2; STATIC_ASSERT(PAGELIST_LIMIT <= MAX_PAGENO); const size_t pages_limit = PAGELIST_LIMIT / 4; - const size_t limit = - (hard_pages < pages_limit) ? hard : (pages_limit << page_ln2); + const size_t limit = (hard_pages < pages_limit) ? hard : (pages_limit << page_ln2); return (limit < MAX_MAPSIZE / 2) ? limit : MAX_MAPSIZE / 2; } -MDBX_NOTHROW_CONST_FUNCTION static inline size_t -env_valsize_max(const MDBX_env *env, MDBX_db_flags_t flags) { +MDBX_NOTHROW_CONST_FUNCTION static inline size_t env_valsize_max(const MDBX_env *env, MDBX_db_flags_t flags) { size_t size_max; if (flags & MDBX_INTEGERDUP) size_max = 8 /* sizeof(uint64_t) */; @@ -153,8 +136,7 @@ env_valsize_max(const MDBX_env *env, MDBX_db_flags_t flags) { const size_t hard_pages = hard >> env->ps2ln; STATIC_ASSERT(PAGELIST_LIMIT <= MAX_PAGENO); const size_t pages_limit = PAGELIST_LIMIT / 4; - const size_t limit = - (hard_pages < pages_limit) ? hard : (pages_limit << env->ps2ln); + const size_t limit = (hard_pages < pages_limit) ? hard : (pages_limit << env->ps2ln); size_max = (limit < MAX_MAPSIZE / 2) ? limit : MAX_MAPSIZE / 2; } eASSERT(env, size_max == valsize_max(env->ps, flags)); @@ -163,8 +145,8 @@ env_valsize_max(const MDBX_env *env, MDBX_db_flags_t flags) { /*----------------------------------------------------------------------------*/ -MDBX_NOTHROW_PURE_FUNCTION static inline size_t -leaf_size(const MDBX_env *env, const MDBX_val *key, const MDBX_val *data) { +MDBX_NOTHROW_PURE_FUNCTION static inline size_t leaf_size(const MDBX_env *env, const MDBX_val *key, + const MDBX_val *data) { size_t node_bytes = node_size(key, data); if (node_bytes > env->leaf_nodemax) /* put on large/overflow page */ @@ -173,35 +155,30 @@ leaf_size(const MDBX_env *env, const MDBX_val *key, const MDBX_val *data) { return node_bytes + sizeof(indx_t); } -MDBX_NOTHROW_PURE_FUNCTION static inline size_t -branch_size(const MDBX_env *env, const MDBX_val *key) { +MDBX_NOTHROW_PURE_FUNCTION static inline size_t branch_size(const MDBX_env *env, const MDBX_val *key) { /* Size of a node in a branch page with a given key. * This is just the node header plus the key, there is no data. */ size_t node_bytes = node_size(key, nullptr); if (unlikely(node_bytes > env->branch_nodemax)) { /* put on large/overflow page, not implemented */ - mdbx_panic("node_size(key) %zu > %u branch_nodemax", node_bytes, - env->branch_nodemax); + mdbx_panic("node_size(key) %zu > %u branch_nodemax", node_bytes, env->branch_nodemax); node_bytes = node_size(key, nullptr) + sizeof(pgno_t); } return node_bytes + sizeof(indx_t); } -MDBX_NOTHROW_CONST_FUNCTION static inline uint16_t -flags_db2sub(uint16_t db_flags) { +MDBX_NOTHROW_CONST_FUNCTION static inline uint16_t flags_db2sub(uint16_t db_flags) { uint16_t sub_flags = db_flags & MDBX_DUPFIXED; /* MDBX_INTEGERDUP => MDBX_INTEGERKEY */ #define SHIFT_INTEGERDUP_TO_INTEGERKEY 2 - STATIC_ASSERT((MDBX_INTEGERDUP >> SHIFT_INTEGERDUP_TO_INTEGERKEY) == - MDBX_INTEGERKEY); + STATIC_ASSERT((MDBX_INTEGERDUP >> SHIFT_INTEGERDUP_TO_INTEGERKEY) == MDBX_INTEGERKEY); sub_flags |= (db_flags & MDBX_INTEGERDUP) >> SHIFT_INTEGERDUP_TO_INTEGERKEY; /* MDBX_REVERSEDUP => MDBX_REVERSEKEY */ #define SHIFT_REVERSEDUP_TO_REVERSEKEY 5 - STATIC_ASSERT((MDBX_REVERSEDUP >> SHIFT_REVERSEDUP_TO_REVERSEKEY) == - MDBX_REVERSEKEY); + STATIC_ASSERT((MDBX_REVERSEDUP >> SHIFT_REVERSEDUP_TO_REVERSEKEY) == MDBX_REVERSEKEY); sub_flags |= (db_flags & MDBX_REVERSEDUP) >> SHIFT_REVERSEDUP_TO_REVERSEKEY; return sub_flags; @@ -219,41 +196,33 @@ static inline bool check_table_flags(unsigned flags) { case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP: case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP: case MDBX_DB_DEFAULTS: - return (flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)) != - (MDBX_REVERSEKEY | MDBX_INTEGERKEY); + return (flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)) != (MDBX_REVERSEKEY | MDBX_INTEGERKEY); } } /*----------------------------------------------------------------------------*/ -MDBX_NOTHROW_PURE_FUNCTION static inline size_t pgno2bytes(const MDBX_env *env, - size_t pgno) { +MDBX_NOTHROW_PURE_FUNCTION static inline size_t pgno2bytes(const MDBX_env *env, size_t pgno) { eASSERT(env, (1u << env->ps2ln) == env->ps); return ((size_t)pgno) << env->ps2ln; } -MDBX_NOTHROW_PURE_FUNCTION static inline page_t *pgno2page(const MDBX_env *env, - size_t pgno) { +MDBX_NOTHROW_PURE_FUNCTION static inline page_t *pgno2page(const MDBX_env *env, size_t pgno) { return ptr_disp(env->dxb_mmap.base, pgno2bytes(env, pgno)); } -MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t bytes2pgno(const MDBX_env *env, - size_t bytes) { +MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t bytes2pgno(const MDBX_env *env, size_t bytes) { eASSERT(env, (env->ps >> env->ps2ln) == 1); return (pgno_t)(bytes >> env->ps2ln); } -MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t -bytes_align2os_bytes(const MDBX_env *env, size_t bytes); +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t bytes_align2os_bytes(const MDBX_env *env, size_t bytes); -MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t -pgno_align2os_bytes(const MDBX_env *env, size_t pgno); +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t pgno_align2os_bytes(const MDBX_env *env, size_t pgno); -MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL pgno_t -pgno_align2os_pgno(const MDBX_env *env, size_t pgno); +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL pgno_t pgno_align2os_pgno(const MDBX_env *env, size_t pgno); -MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t -largechunk_npages(const MDBX_env *env, size_t bytes) { +MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t largechunk_npages(const MDBX_env *env, size_t bytes) { return bytes2pgno(env, PAGEHDRSZ - 1 + bytes) + 1; } @@ -264,69 +233,53 @@ MDBX_NOTHROW_PURE_FUNCTION static inline MDBX_val get_key(const node_t *node) { return key; } -static inline void get_key_optional(const node_t *node, - MDBX_val *keyptr /* __may_null */) { +static inline void get_key_optional(const node_t *node, MDBX_val *keyptr /* __may_null */) { if (keyptr) *keyptr = get_key(node); } -MDBX_NOTHROW_PURE_FUNCTION static inline void *page_data(const page_t *mp) { - return ptr_disp(mp, PAGEHDRSZ); -} +MDBX_NOTHROW_PURE_FUNCTION static inline void *page_data(const page_t *mp) { return ptr_disp(mp, PAGEHDRSZ); } -MDBX_NOTHROW_PURE_FUNCTION static inline const page_t * -data_page(const void *data) { +MDBX_NOTHROW_PURE_FUNCTION static inline const page_t *data_page(const void *data) { return container_of(data, page_t, entries); } -MDBX_NOTHROW_PURE_FUNCTION static inline meta_t *page_meta(page_t *mp) { - return (meta_t *)page_data(mp); -} +MDBX_NOTHROW_PURE_FUNCTION static inline meta_t *page_meta(page_t *mp) { return (meta_t *)page_data(mp); } -MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_numkeys(const page_t *mp) { - return mp->lower >> 1; -} +MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_numkeys(const page_t *mp) { return mp->lower >> 1; } -MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_room(const page_t *mp) { - return mp->upper - mp->lower; -} +MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_room(const page_t *mp) { return mp->upper - mp->lower; } -MDBX_NOTHROW_PURE_FUNCTION static inline size_t -page_space(const MDBX_env *env) { +MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_space(const MDBX_env *env) { STATIC_ASSERT(PAGEHDRSZ % 2 == 0); return env->ps - PAGEHDRSZ; } -MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_used(const MDBX_env *env, - const page_t *mp) { +MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_used(const MDBX_env *env, const page_t *mp) { return page_space(env) - page_room(mp); } /* The percentage of space used in the page, in a percents. */ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline unsigned -page_fill_percentum_x10(const MDBX_env *env, const page_t *mp) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline unsigned page_fill_percentum_x10(const MDBX_env *env, + const page_t *mp) { const size_t space = page_space(env); return (unsigned)((page_used(env, mp) * 1000 + space / 2) / space); } -MDBX_NOTHROW_PURE_FUNCTION static inline node_t *page_node(const page_t *mp, - size_t i) { +MDBX_NOTHROW_PURE_FUNCTION static inline node_t *page_node(const page_t *mp, size_t i) { assert(page_type_compat(mp) == P_LEAF || page_type(mp) == P_BRANCH); assert(page_numkeys(mp) > i); assert(mp->entries[i] % 2 == 0); return ptr_disp(mp, mp->entries[i] + PAGEHDRSZ); } -MDBX_NOTHROW_PURE_FUNCTION static inline void * -page_dupfix_ptr(const page_t *mp, size_t i, size_t keysize) { - assert(page_type_compat(mp) == (P_LEAF | P_DUPFIX) && i == (indx_t)i && - mp->dupfix_ksize == keysize); +MDBX_NOTHROW_PURE_FUNCTION static inline void *page_dupfix_ptr(const page_t *mp, size_t i, size_t keysize) { + assert(page_type_compat(mp) == (P_LEAF | P_DUPFIX) && i == (indx_t)i && mp->dupfix_ksize == keysize); (void)keysize; return ptr_disp(mp, PAGEHDRSZ + mp->dupfix_ksize * (indx_t)i); } -MDBX_NOTHROW_PURE_FUNCTION static inline MDBX_val -page_dupfix_key(const page_t *mp, size_t i, size_t keysize) { +MDBX_NOTHROW_PURE_FUNCTION static inline MDBX_val page_dupfix_key(const page_t *mp, size_t i, size_t keysize) { MDBX_val r; r.iov_base = page_dupfix_ptr(mp, i, keysize); r.iov_len = mp->dupfix_ksize; @@ -335,11 +288,9 @@ page_dupfix_key(const page_t *mp, size_t i, size_t keysize) { /*----------------------------------------------------------------------------*/ -MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int -cmp_int_unaligned(const MDBX_val *a, const MDBX_val *b); +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_int_unaligned(const MDBX_val *a, const MDBX_val *b); -#if MDBX_UNALIGNED_OK < 2 || \ - (MDBX_DEBUG || MDBX_FORCE_ASSERTIONS || !defined(NDEBUG)) +#if MDBX_UNALIGNED_OK < 2 || (MDBX_DEBUG || MDBX_FORCE_ASSERTIONS || !defined(NDEBUG)) MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int /* Compare two items pointing at 2-byte aligned unsigned int's. */ cmp_int_align2(const MDBX_val *a, const MDBX_val *b); @@ -347,8 +298,7 @@ cmp_int_align2(const MDBX_val *a, const MDBX_val *b); #define cmp_int_align2 cmp_int_unaligned #endif /* !MDBX_UNALIGNED_OK || debug */ -#if MDBX_UNALIGNED_OK < 4 || \ - (MDBX_DEBUG || MDBX_FORCE_ASSERTIONS || !defined(NDEBUG)) +#if MDBX_UNALIGNED_OK < 4 || (MDBX_DEBUG || MDBX_FORCE_ASSERTIONS || !defined(NDEBUG)) MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int /* Compare two items pointing at 4-byte aligned unsigned int's. */ cmp_int_align4(const MDBX_val *a, const MDBX_val *b); @@ -357,50 +307,38 @@ cmp_int_align4(const MDBX_val *a, const MDBX_val *b); #endif /* !MDBX_UNALIGNED_OK || debug */ /* Compare two items lexically */ -MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_lexical(const MDBX_val *a, - const MDBX_val *b); +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_lexical(const MDBX_val *a, const MDBX_val *b); /* Compare two items in reverse byte order */ -MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_reverse(const MDBX_val *a, - const MDBX_val *b); +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_reverse(const MDBX_val *a, const MDBX_val *b); /* Fast non-lexically comparator */ -MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_lenfast(const MDBX_val *a, - const MDBX_val *b); +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_lenfast(const MDBX_val *a, const MDBX_val *b); -MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL bool -eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l); +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL bool eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l); -MDBX_NOTHROW_PURE_FUNCTION static inline bool eq_fast(const MDBX_val *a, - const MDBX_val *b) { - return unlikely(a->iov_len == b->iov_len) && - eq_fast_slowpath(a->iov_base, b->iov_base, a->iov_len); +MDBX_NOTHROW_PURE_FUNCTION static inline bool eq_fast(const MDBX_val *a, const MDBX_val *b) { + return unlikely(a->iov_len == b->iov_len) && eq_fast_slowpath(a->iov_base, b->iov_base, a->iov_len); } -MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int -cmp_equal_or_greater(const MDBX_val *a, const MDBX_val *b); +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_equal_or_greater(const MDBX_val *a, const MDBX_val *b); -MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int -cmp_equal_or_wrong(const MDBX_val *a, const MDBX_val *b); +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_equal_or_wrong(const MDBX_val *a, const MDBX_val *b); static inline MDBX_cmp_func *builtin_keycmp(MDBX_db_flags_t flags) { - return (flags & MDBX_REVERSEKEY) ? cmp_reverse - : (flags & MDBX_INTEGERKEY) ? cmp_int_align2 - : cmp_lexical; + return (flags & MDBX_REVERSEKEY) ? cmp_reverse : (flags & MDBX_INTEGERKEY) ? cmp_int_align2 : cmp_lexical; } static inline MDBX_cmp_func *builtin_datacmp(MDBX_db_flags_t flags) { return !(flags & MDBX_DUPSORT) ? cmp_lenfast - : ((flags & MDBX_INTEGERDUP) - ? cmp_int_unaligned - : ((flags & MDBX_REVERSEDUP) ? cmp_reverse : cmp_lexical)); + : ((flags & MDBX_INTEGERDUP) ? cmp_int_unaligned + : ((flags & MDBX_REVERSEDUP) ? cmp_reverse : cmp_lexical)); } /*----------------------------------------------------------------------------*/ -MDBX_INTERNAL uint32_t combine_durability_flags(const uint32_t a, - const uint32_t b); +MDBX_INTERNAL uint32_t combine_durability_flags(const uint32_t a, const uint32_t b); MDBX_CONST_FUNCTION static inline lck_t *lckless_stub(const MDBX_env *env) { uintptr_t stub = (uintptr_t)&env->lckless_placeholder; @@ -477,12 +415,10 @@ static inline int check_txn(const MDBX_txn *txn, int bad_bits) { } tASSERT(txn, (txn->flags & MDBX_TXN_FINISHED) || - (txn->flags & MDBX_NOSTICKYTHREADS) == - (txn->env->flags & MDBX_NOSTICKYTHREADS)); + (txn->flags & MDBX_NOSTICKYTHREADS) == (txn->env->flags & MDBX_NOSTICKYTHREADS)); #if MDBX_TXN_CHECKOWNER STATIC_ASSERT((long)MDBX_NOSTICKYTHREADS > (long)MDBX_TXN_FINISHED); - if ((txn->flags & (MDBX_NOSTICKYTHREADS | MDBX_TXN_FINISHED)) < - MDBX_TXN_FINISHED && + if ((txn->flags & (MDBX_NOSTICKYTHREADS | MDBX_TXN_FINISHED)) < MDBX_TXN_FINISHED && unlikely(txn->owner != osal_thread_self())) return txn->owner ? MDBX_THREAD_MISMATCH : MDBX_BAD_TXN; #endif /* MDBX_TXN_CHECKOWNER */ @@ -508,12 +444,10 @@ static inline int check_txn_rw(const MDBX_txn *txn, int bad_bits) { MDBX_INTERNAL void mincore_clean_cache(const MDBX_env *const env); -MDBX_INTERNAL void update_mlcnt(const MDBX_env *env, - const pgno_t new_aligned_mlocked_pgno, +MDBX_INTERNAL void update_mlcnt(const MDBX_env *env, const pgno_t new_aligned_mlocked_pgno, const bool lock_not_release); -MDBX_INTERNAL void munlock_after(const MDBX_env *env, const pgno_t aligned_pgno, - const size_t end_bytes); +MDBX_INTERNAL void munlock_after(const MDBX_env *env, const pgno_t aligned_pgno, const size_t end_bytes); MDBX_INTERNAL void munlock_all(const MDBX_env *env); @@ -527,15 +461,13 @@ MDBX_INTERNAL void munlock_all(const MDBX_env *env); #define osal_flush_incoherent_cpu_writeback() osal_compiler_barrier() #endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ -MDBX_MAYBE_UNUSED static inline void -osal_flush_incoherent_mmap(const void *addr, size_t nbytes, - const intptr_t pagesize) { +MDBX_MAYBE_UNUSED static inline void osal_flush_incoherent_mmap(const void *addr, size_t nbytes, + const intptr_t pagesize) { #ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE #error "The MDBX_MMAP_INCOHERENT_FILE_WRITE must be defined before" #elif MDBX_MMAP_INCOHERENT_FILE_WRITE char *const begin = (char *)(-pagesize & (intptr_t)addr); - char *const end = - (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); + char *const end = (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; eASSERT(nullptr, err == 0); (void)err; diff --git a/src/coherency.c b/src/coherency.c index 7701271b..6b9df51d 100644 --- a/src/coherency.c +++ b/src/coherency.c @@ -4,8 +4,7 @@ #include "internals.h" /* check against https://libmdbx.dqdkfa.ru/dead-github/issues/269 */ -static bool coherency_check(const MDBX_env *env, const txnid_t txnid, - const volatile tree_t *trees, +static bool coherency_check(const MDBX_env *env, const txnid_t txnid, const volatile tree_t *trees, const volatile meta_t *meta, bool report) { const txnid_t freedb_mod_txnid = trees[FREE_DBI].mod_txnid; const txnid_t maindb_mod_txnid = trees[MAIN_DBI].mod_txnid; @@ -13,67 +12,42 @@ static bool coherency_check(const MDBX_env *env, const txnid_t txnid, const pgno_t freedb_root_pgno = trees[FREE_DBI].root; const page_t *freedb_root = - (env->dxb_mmap.base && freedb_root_pgno < last_pgno) - ? pgno2page(env, freedb_root_pgno) - : nullptr; + (env->dxb_mmap.base && freedb_root_pgno < last_pgno) ? pgno2page(env, freedb_root_pgno) : nullptr; const pgno_t maindb_root_pgno = trees[MAIN_DBI].root; const page_t *maindb_root = - (env->dxb_mmap.base && maindb_root_pgno < last_pgno) - ? pgno2page(env, maindb_root_pgno) - : nullptr; - const uint64_t magic_and_version = - unaligned_peek_u64_volatile(4, &meta->magic_and_version); + (env->dxb_mmap.base && maindb_root_pgno < last_pgno) ? pgno2page(env, maindb_root_pgno) : nullptr; + const uint64_t magic_and_version = unaligned_peek_u64_volatile(4, &meta->magic_and_version); bool ok = true; - if (freedb_root_pgno != P_INVALID && - unlikely(freedb_root_pgno >= last_pgno)) { + if (freedb_root_pgno != P_INVALID && unlikely(freedb_root_pgno >= last_pgno)) { if (report) - WARNING( - "catch invalid %s-db root %" PRIaPGNO " for meta_txnid %" PRIaTXN - " %s", - "free", freedb_root_pgno, txnid, - (env->stuck_meta < 0) - ? "(workaround for incoherent flaw of unified page/buffer cache)" - : "(wagering meta)"); + WARNING("catch invalid %s-db root %" PRIaPGNO " for meta_txnid %" PRIaTXN " %s", "free", freedb_root_pgno, txnid, + (env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)" + : "(wagering meta)"); ok = false; } - if (maindb_root_pgno != P_INVALID && - unlikely(maindb_root_pgno >= last_pgno)) { + if (maindb_root_pgno != P_INVALID && unlikely(maindb_root_pgno >= last_pgno)) { if (report) - WARNING( - "catch invalid %s-db root %" PRIaPGNO " for meta_txnid %" PRIaTXN - " %s", - "main", maindb_root_pgno, txnid, - (env->stuck_meta < 0) - ? "(workaround for incoherent flaw of unified page/buffer cache)" - : "(wagering meta)"); + WARNING("catch invalid %s-db root %" PRIaPGNO " for meta_txnid %" PRIaTXN " %s", "main", maindb_root_pgno, txnid, + (env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)" + : "(wagering meta)"); ok = false; } if (unlikely(txnid < freedb_mod_txnid || - (!freedb_mod_txnid && freedb_root && - likely(magic_and_version == MDBX_DATA_MAGIC)))) { + (!freedb_mod_txnid && freedb_root && likely(magic_and_version == MDBX_DATA_MAGIC)))) { if (report) WARNING( - "catch invalid %s-db.mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN - " %s", - "free", freedb_mod_txnid, txnid, - (env->stuck_meta < 0) - ? "(workaround for incoherent flaw of unified page/buffer cache)" - : "(wagering meta)"); + "catch invalid %s-db.mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN " %s", "free", freedb_mod_txnid, txnid, + (env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)" : "(wagering meta)"); ok = false; } if (unlikely(txnid < maindb_mod_txnid || - (!maindb_mod_txnid && maindb_root && - likely(magic_and_version == MDBX_DATA_MAGIC)))) { + (!maindb_mod_txnid && maindb_root && likely(magic_and_version == MDBX_DATA_MAGIC)))) { if (report) WARNING( - "catch invalid %s-db.mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN - " %s", - "main", maindb_mod_txnid, txnid, - (env->stuck_meta < 0) - ? "(workaround for incoherent flaw of unified page/buffer cache)" - : "(wagering meta)"); + "catch invalid %s-db.mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN " %s", "main", maindb_mod_txnid, txnid, + (env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)" : "(wagering meta)"); ok = false; } @@ -81,15 +55,13 @@ static bool coherency_check(const MDBX_env *env, const txnid_t txnid, * в пределах текущего отображения. Иначе возможны SIGSEGV до переноса * вызова coherency_check_head() после dxb_resize() внутри txn_renew(). */ if (likely(freedb_root && freedb_mod_txnid && - (size_t)ptr_dist(env->dxb_mmap.base, freedb_root) < - env->dxb_mmap.limit)) { + (size_t)ptr_dist(env->dxb_mmap.base, freedb_root) < env->dxb_mmap.limit)) { VALGRIND_MAKE_MEM_DEFINED(freedb_root, sizeof(freedb_root->txnid)); MDBX_ASAN_UNPOISON_MEMORY_REGION(freedb_root, sizeof(freedb_root->txnid)); const txnid_t root_txnid = freedb_root->txnid; if (unlikely(root_txnid != freedb_mod_txnid)) { if (report) - WARNING("catch invalid root_page %" PRIaPGNO " mod_txnid %" PRIaTXN - " for %s-db.mod_txnid %" PRIaTXN " %s", + WARNING("catch invalid root_page %" PRIaPGNO " mod_txnid %" PRIaTXN " for %s-db.mod_txnid %" PRIaTXN " %s", freedb_root_pgno, root_txnid, "free", freedb_mod_txnid, (env->stuck_meta < 0) ? "(workaround for incoherent flaw of " "unified page/buffer cache)" @@ -98,15 +70,13 @@ static bool coherency_check(const MDBX_env *env, const txnid_t txnid, } } if (likely(maindb_root && maindb_mod_txnid && - (size_t)ptr_dist(env->dxb_mmap.base, maindb_root) < - env->dxb_mmap.limit)) { + (size_t)ptr_dist(env->dxb_mmap.base, maindb_root) < env->dxb_mmap.limit)) { VALGRIND_MAKE_MEM_DEFINED(maindb_root, sizeof(maindb_root->txnid)); MDBX_ASAN_UNPOISON_MEMORY_REGION(maindb_root, sizeof(maindb_root->txnid)); const txnid_t root_txnid = maindb_root->txnid; if (unlikely(root_txnid != maindb_mod_txnid)) { if (report) - WARNING("catch invalid root_page %" PRIaPGNO " mod_txnid %" PRIaTXN - " for %s-db.mod_txnid %" PRIaTXN " %s", + WARNING("catch invalid root_page %" PRIaPGNO " mod_txnid %" PRIaTXN " for %s-db.mod_txnid %" PRIaTXN " %s", maindb_root_pgno, root_txnid, "main", maindb_mod_txnid, (env->stuck_meta < 0) ? "(workaround for incoherent flaw of " "unified page/buffer cache)" @@ -116,24 +86,19 @@ static bool coherency_check(const MDBX_env *env, const txnid_t txnid, } if (unlikely(!ok) && report) env->lck->pgops.incoherence.weak = - (env->lck->pgops.incoherence.weak >= INT32_MAX) - ? INT32_MAX - : env->lck->pgops.incoherence.weak + 1; + (env->lck->pgops.incoherence.weak >= INT32_MAX) ? INT32_MAX : env->lck->pgops.incoherence.weak + 1; return ok; } -__cold int coherency_timeout(uint64_t *timestamp, intptr_t pgno, - const MDBX_env *env) { +__cold int coherency_timeout(uint64_t *timestamp, intptr_t pgno, const MDBX_env *env) { if (likely(timestamp && *timestamp == 0)) *timestamp = osal_monotime(); - else if (unlikely(!timestamp || osal_monotime() - *timestamp > - osal_16dot16_to_monotime(65536 / 10))) { + else if (unlikely(!timestamp || osal_monotime() - *timestamp > osal_16dot16_to_monotime(65536 / 10))) { if (pgno >= 0 && pgno != env->stuck_meta) ERROR("bailout waiting for %" PRIuSIZE " page arrival %s", pgno, "(workaround for incoherent flaw of unified page/buffer cache)"); else if (env->stuck_meta < 0) - ERROR("bailout waiting for valid snapshot (%s)", - "workaround for incoherent flaw of unified page/buffer cache"); + ERROR("bailout waiting for valid snapshot (%s)", "workaround for incoherent flaw of unified page/buffer cache"); return MDBX_PROBLEM; } @@ -152,28 +117,23 @@ __cold int coherency_timeout(uint64_t *timestamp, intptr_t pgno, /* check with timeout as the workaround * for https://libmdbx.dqdkfa.ru/dead-github/issues/269 */ -__hot int coherency_fetch_head(MDBX_txn *txn, const meta_ptr_t head, - uint64_t *timestamp) { +__hot int coherency_fetch_head(MDBX_txn *txn, const meta_ptr_t head, uint64_t *timestamp) { /* Copy the DB info and flags */ txn->txnid = head.txnid; txn->geo = head.ptr_c->geometry; memcpy(txn->dbs, &head.ptr_c->trees, sizeof(head.ptr_c->trees)); STATIC_ASSERT(sizeof(head.ptr_c->trees) == CORE_DBS * sizeof(tree_t)); - VALGRIND_MAKE_MEM_UNDEFINED(txn->dbs + CORE_DBS, - txn->env->max_dbi - CORE_DBS); + VALGRIND_MAKE_MEM_UNDEFINED(txn->dbs + CORE_DBS, txn->env->max_dbi - CORE_DBS); txn->canary = head.ptr_c->canary; - if (unlikely(!coherency_check(txn->env, head.txnid, txn->dbs, head.ptr_v, - *timestamp == 0) || + if (unlikely(!coherency_check(txn->env, head.txnid, txn->dbs, head.ptr_v, *timestamp == 0) || txn->txnid != meta_txnid(head.ptr_v))) return coherency_timeout(timestamp, -1, txn->env); if (unlikely(txn->dbs[FREE_DBI].flags != MDBX_INTEGERKEY)) { if ((txn->dbs[FREE_DBI].flags & DB_PERSISTENT_FLAGS) != MDBX_INTEGERKEY || - unaligned_peek_u64(4, &head.ptr_c->magic_and_version) == - MDBX_DATA_MAGIC) { - ERROR("unexpected/invalid db-flags 0x%x for %s", txn->dbs[FREE_DBI].flags, - "GC/FreeDB"); + unaligned_peek_u64(4, &head.ptr_c->magic_and_version) == MDBX_DATA_MAGIC) { + ERROR("unexpected/invalid db-flags 0x%x for %s", txn->dbs[FREE_DBI].flags, "GC/FreeDB"); return MDBX_INCOMPATIBLE; } txn->dbs[FREE_DBI].flags &= DB_PERSISTENT_FLAGS; @@ -183,23 +143,19 @@ __hot int coherency_fetch_head(MDBX_txn *txn, const meta_ptr_t head, return MDBX_SUCCESS; } -int coherency_check_written(const MDBX_env *env, const txnid_t txnid, - const volatile meta_t *meta, const intptr_t pgno, +int coherency_check_written(const MDBX_env *env, const txnid_t txnid, const volatile meta_t *meta, const intptr_t pgno, uint64_t *timestamp) { const bool report = !(timestamp && *timestamp); const txnid_t head_txnid = meta_txnid(meta); if (likely(head_txnid >= MIN_TXNID && head_txnid >= txnid)) { - if (likely( - coherency_check(env, head_txnid, &meta->trees.gc, meta, report))) { + if (likely(coherency_check(env, head_txnid, &meta->trees.gc, meta, report))) { eASSERT(env, meta->trees.gc.flags == MDBX_INTEGERKEY); eASSERT(env, check_table_flags(meta->trees.main.flags)); return MDBX_SUCCESS; } } else if (report) { env->lck->pgops.incoherence.weak = - (env->lck->pgops.incoherence.weak >= INT32_MAX) - ? INT32_MAX - : env->lck->pgops.incoherence.weak + 1; + (env->lck->pgops.incoherence.weak >= INT32_MAX) ? INT32_MAX : env->lck->pgops.incoherence.weak + 1; WARNING("catch %s txnid %" PRIaTXN " for meta_%" PRIaPGNO " %s", (head_txnid < MIN_TXNID) ? "invalid" : "unexpected", head_txnid, bytes2pgno(env, ptr_dist(meta, env->dxb_mmap.base)), @@ -208,9 +164,7 @@ int coherency_check_written(const MDBX_env *env, const txnid_t txnid, return coherency_timeout(timestamp, pgno, env); } -bool coherency_check_meta(const MDBX_env *env, const volatile meta_t *meta, - bool report) { +bool coherency_check_meta(const MDBX_env *env, const volatile meta_t *meta, bool report) { uint64_t timestamp = 0; - return coherency_check_written(env, 0, meta, -1, - report ? ×tamp : nullptr) == MDBX_SUCCESS; + return coherency_check_written(env, 0, meta, -1, report ? ×tamp : nullptr) == MDBX_SUCCESS; } diff --git a/src/cold.c b/src/cold.c index 35665101..11260ace 100644 --- a/src/cold.c +++ b/src/cold.c @@ -14,8 +14,7 @@ __cold size_t mdbx_default_pagesize(void) { __cold intptr_t mdbx_limits_dbsize_min(intptr_t pagesize) { if (pagesize < 1) pagesize = (intptr_t)mdbx_default_pagesize(); - else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || - pagesize > (intptr_t)MDBX_MAX_PAGESIZE || + else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || !is_powerof2((size_t)pagesize))) return -1; @@ -25,8 +24,7 @@ __cold intptr_t mdbx_limits_dbsize_min(intptr_t pagesize) { __cold intptr_t mdbx_limits_dbsize_max(intptr_t pagesize) { if (pagesize < 1) pagesize = (intptr_t)mdbx_default_pagesize(); - else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || - pagesize > (intptr_t)MDBX_MAX_PAGESIZE || + else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || !is_powerof2((size_t)pagesize))) return -1; @@ -38,112 +36,90 @@ __cold intptr_t mdbx_limits_dbsize_max(intptr_t pagesize) { __cold intptr_t mdbx_limits_txnsize_max(intptr_t pagesize) { if (pagesize < 1) pagesize = (intptr_t)mdbx_default_pagesize(); - else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || - pagesize > (intptr_t)MDBX_MAX_PAGESIZE || + else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || !is_powerof2((size_t)pagesize))) return -1; STATIC_ASSERT(MAX_MAPSIZE < INTPTR_MAX); - const uint64_t pgl_limit = - pagesize * (uint64_t)(PAGELIST_LIMIT / MDBX_GOLD_RATIO_DBL); + const uint64_t pgl_limit = pagesize * (uint64_t)(PAGELIST_LIMIT / MDBX_GOLD_RATIO_DBL); const uint64_t map_limit = (uint64_t)(MAX_MAPSIZE / MDBX_GOLD_RATIO_DBL); return (pgl_limit < map_limit) ? (intptr_t)pgl_limit : (intptr_t)map_limit; } -__cold intptr_t mdbx_limits_keysize_max(intptr_t pagesize, - MDBX_db_flags_t flags) { +__cold intptr_t mdbx_limits_keysize_max(intptr_t pagesize, MDBX_db_flags_t flags) { if (pagesize < 1) pagesize = (intptr_t)mdbx_default_pagesize(); - if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || - pagesize > (intptr_t)MDBX_MAX_PAGESIZE || + if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || !is_powerof2((size_t)pagesize))) return -1; return keysize_max(pagesize, flags); } -__cold int mdbx_env_get_maxkeysize_ex(const MDBX_env *env, - MDBX_db_flags_t flags) { +__cold int mdbx_env_get_maxkeysize_ex(const MDBX_env *env, MDBX_db_flags_t flags) { if (unlikely(!env || env->signature.weak != env_signature)) return -1; return (int)mdbx_limits_keysize_max((intptr_t)env->ps, flags); } -__cold int mdbx_env_get_maxkeysize(const MDBX_env *env) { - return mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT); -} +__cold int mdbx_env_get_maxkeysize(const MDBX_env *env) { return mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT); } -__cold intptr_t mdbx_limits_keysize_min(MDBX_db_flags_t flags) { - return keysize_min(flags); -} +__cold intptr_t mdbx_limits_keysize_min(MDBX_db_flags_t flags) { return keysize_min(flags); } -__cold intptr_t mdbx_limits_valsize_max(intptr_t pagesize, - MDBX_db_flags_t flags) { +__cold intptr_t mdbx_limits_valsize_max(intptr_t pagesize, MDBX_db_flags_t flags) { if (pagesize < 1) pagesize = (intptr_t)mdbx_default_pagesize(); - if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || - pagesize > (intptr_t)MDBX_MAX_PAGESIZE || + if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || !is_powerof2((size_t)pagesize))) return -1; return valsize_max(pagesize, flags); } -__cold int mdbx_env_get_maxvalsize_ex(const MDBX_env *env, - MDBX_db_flags_t flags) { +__cold int mdbx_env_get_maxvalsize_ex(const MDBX_env *env, MDBX_db_flags_t flags) { if (unlikely(!env || env->signature.weak != env_signature)) return -1; return (int)mdbx_limits_valsize_max((intptr_t)env->ps, flags); } -__cold intptr_t mdbx_limits_valsize_min(MDBX_db_flags_t flags) { - return valsize_min(flags); -} +__cold intptr_t mdbx_limits_valsize_min(MDBX_db_flags_t flags) { return valsize_min(flags); } -__cold intptr_t mdbx_limits_pairsize4page_max(intptr_t pagesize, - MDBX_db_flags_t flags) { +__cold intptr_t mdbx_limits_pairsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags) { if (pagesize < 1) pagesize = (intptr_t)mdbx_default_pagesize(); - if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || - pagesize > (intptr_t)MDBX_MAX_PAGESIZE || + if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || !is_powerof2((size_t)pagesize))) return -1; - if (flags & - (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP)) + if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP)) return BRANCH_NODE_MAX(pagesize) - NODESIZE; return LEAF_NODE_MAX(pagesize) - NODESIZE; } -__cold int mdbx_env_get_pairsize4page_max(const MDBX_env *env, - MDBX_db_flags_t flags) { +__cold int mdbx_env_get_pairsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags) { if (unlikely(!env || env->signature.weak != env_signature)) return -1; return (int)mdbx_limits_pairsize4page_max((intptr_t)env->ps, flags); } -__cold intptr_t mdbx_limits_valsize4page_max(intptr_t pagesize, - MDBX_db_flags_t flags) { +__cold intptr_t mdbx_limits_valsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags) { if (pagesize < 1) pagesize = (intptr_t)mdbx_default_pagesize(); - if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || - pagesize > (intptr_t)MDBX_MAX_PAGESIZE || + if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || !is_powerof2((size_t)pagesize))) return -1; - if (flags & - (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP)) + if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP)) return valsize_max(pagesize, flags); return PAGESPACE(pagesize); } -__cold int mdbx_env_get_valsize4page_max(const MDBX_env *env, - MDBX_db_flags_t flags) { +__cold int mdbx_env_get_valsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags) { if (unlikely(!env || env->signature.weak != env_signature)) return -1; @@ -152,17 +128,14 @@ __cold int mdbx_env_get_valsize4page_max(const MDBX_env *env, /*----------------------------------------------------------------------------*/ -__cold static void stat_add(const tree_t *db, MDBX_stat *const st, - const size_t bytes) { +__cold static void stat_add(const tree_t *db, MDBX_stat *const st, const size_t bytes) { st->ms_depth += db->height; st->ms_branch_pages += db->branch_pages; st->ms_leaf_pages += db->leaf_pages; st->ms_overflow_pages += db->large_pages; st->ms_entries += db->items; - if (likely(bytes >= - offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid))) - st->ms_mod_txnid = - (st->ms_mod_txnid > db->mod_txnid) ? st->ms_mod_txnid : db->mod_txnid; + if (likely(bytes >= offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid))) + st->ms_mod_txnid = (st->ms_mod_txnid > db->mod_txnid) ? st->ms_mod_txnid : db->mod_txnid; } __cold static int stat_acc(const MDBX_txn *txn, MDBX_stat *st, size_t bytes) { @@ -179,15 +152,13 @@ __cold static int stat_acc(const MDBX_txn *txn, MDBX_stat *st, size_t bytes) { const MDBX_env *const env = txn->env; st->ms_psize = env->ps; - TXN_FOREACH_DBI_FROM( - txn, dbi, - /* assuming GC is internal and not subject for accounting */ MAIN_DBI) { + TXN_FOREACH_DBI_FROM(txn, dbi, + /* assuming GC is internal and not subject for accounting */ MAIN_DBI) { if ((txn->dbi_state[dbi] & (DBI_VALID | DBI_STALE)) == DBI_VALID) stat_add(txn->dbs + dbi, st, bytes); } - if (!(txn->dbs[MAIN_DBI].flags & MDBX_DUPSORT) && - txn->dbs[MAIN_DBI].items /* TODO: use `md_subs` field */) { + if (!(txn->dbs[MAIN_DBI].flags & MDBX_DUPSORT) && txn->dbs[MAIN_DBI].items /* TODO: use `md_subs` field */) { /* scan and account not opened named tables */ err = tree_search(&cx.outer, nullptr, Z_FIRST); @@ -198,8 +169,7 @@ __cold static int stat_acc(const MDBX_txn *txn, MDBX_stat *st, size_t bytes) { if (node_flags(node) != N_TREE) continue; if (unlikely(node_ds(node) != sizeof(tree_t))) { - ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid table node size", node_ds(node)); + ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid table node size", node_ds(node)); return MDBX_CORRUPTED; } @@ -228,8 +198,7 @@ __cold static int stat_acc(const MDBX_txn *txn, MDBX_stat *st, size_t bytes) { return MDBX_SUCCESS; } -__cold int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, - MDBX_stat *dest, size_t bytes) { +__cold int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, MDBX_stat *dest, size_t bytes) { if (unlikely(!dest)) return LOG_IFERR(MDBX_EINVAL); const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid); @@ -265,18 +234,15 @@ __cold int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, /*----------------------------------------------------------------------------*/ static size_t estimate_rss(size_t database_bytes) { - return database_bytes + database_bytes / 64 + - (512 + MDBX_WORDBITS * 16) * MEGABYTE; + return database_bytes + database_bytes / 64 + (512 + MDBX_WORDBITS * 16) * MEGABYTE; } -__cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, - MDBX_warmup_flags_t flags, +__cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, MDBX_warmup_flags_t flags, unsigned timeout_seconds_16dot16) { if (unlikely(env == nullptr && txn == nullptr)) return LOG_IFERR(MDBX_EINVAL); - if (unlikely(flags > - (MDBX_warmup_force | MDBX_warmup_oomsafe | MDBX_warmup_lock | - MDBX_warmup_touchlimit | MDBX_warmup_release))) + if (unlikely(flags > (MDBX_warmup_force | MDBX_warmup_oomsafe | MDBX_warmup_lock | MDBX_warmup_touchlimit | + MDBX_warmup_release))) return LOG_IFERR(MDBX_EINVAL); if (txn) { @@ -294,10 +260,9 @@ __cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, env = txn->env; } - const uint64_t timeout_monotime = - (timeout_seconds_16dot16 && (flags & MDBX_warmup_force)) - ? osal_monotime() + osal_16dot16_to_monotime(timeout_seconds_16dot16) - : 0; + const uint64_t timeout_monotime = (timeout_seconds_16dot16 && (flags & MDBX_warmup_force)) + ? osal_monotime() + osal_16dot16_to_monotime(timeout_seconds_16dot16) + : 0; if (flags & MDBX_warmup_release) munlock_all(env); @@ -317,18 +282,14 @@ __cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, const size_t estimated_rss = estimate_rss(used_range); #if defined(_WIN32) || defined(_WIN64) SIZE_T current_ws_lower, current_ws_upper; - if (GetProcessWorkingSetSize(GetCurrentProcess(), ¤t_ws_lower, - ¤t_ws_upper) && + if (GetProcessWorkingSetSize(GetCurrentProcess(), ¤t_ws_lower, ¤t_ws_upper) && current_ws_lower < estimated_rss) { const SIZE_T ws_lower = estimated_rss; const SIZE_T ws_upper = - (MDBX_WORDBITS == 32 && ws_lower > MEGABYTE * 2048) - ? ws_lower - : ws_lower + MDBX_WORDBITS * MEGABYTE * 32; + (MDBX_WORDBITS == 32 && ws_lower > MEGABYTE * 2048) ? ws_lower : ws_lower + MDBX_WORDBITS * MEGABYTE * 32; if (!SetProcessWorkingSetSize(GetCurrentProcess(), ws_lower, ws_upper)) { rc = (int)GetLastError(); - WARNING("SetProcessWorkingSetSize(%zu, %zu) error %d", ws_lower, - ws_upper, rc); + WARNING("SetProcessWorkingSetSize(%zu, %zu) error %d", ws_lower, ws_upper, rc); } } #endif /* Windows */ @@ -340,23 +301,21 @@ __cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, rss.rlim_max = estimated_rss; if (setrlimit(RLIMIT_RSS, &rss)) { rc = errno; - WARNING("setrlimit(%s, {%zu, %zu}) error %d", "RLIMIT_RSS", - (size_t)rss.rlim_cur, (size_t)rss.rlim_max, rc); + WARNING("setrlimit(%s, {%zu, %zu}) error %d", "RLIMIT_RSS", (size_t)rss.rlim_cur, (size_t)rss.rlim_max, rc); } } #endif /* RLIMIT_RSS */ #ifdef RLIMIT_MEMLOCK if (flags & MDBX_warmup_lock) { struct rlimit memlock; - if (getrlimit(RLIMIT_MEMLOCK, &memlock) == 0 && - memlock.rlim_cur < estimated_rss) { + if (getrlimit(RLIMIT_MEMLOCK, &memlock) == 0 && memlock.rlim_cur < estimated_rss) { memlock.rlim_cur = estimated_rss; if (memlock.rlim_max < estimated_rss) memlock.rlim_max = estimated_rss; if (setrlimit(RLIMIT_MEMLOCK, &memlock)) { rc = errno; - WARNING("setrlimit(%s, {%zu, %zu}) error %d", "RLIMIT_MEMLOCK", - (size_t)memlock.rlim_cur, (size_t)memlock.rlim_max, rc); + WARNING("setrlimit(%s, {%zu, %zu}) error %d", "RLIMIT_MEMLOCK", (size_t)memlock.rlim_cur, + (size_t)memlock.rlim_max, rc); } } } @@ -364,12 +323,10 @@ __cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, (void)estimated_rss; } -#if defined(MLOCK_ONFAULT) && \ - ((defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 27)) || \ - (defined(__ANDROID_API__) && __ANDROID_API__ >= 30)) && \ +#if defined(MLOCK_ONFAULT) && \ + ((defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 27)) || (defined(__ANDROID_API__) && __ANDROID_API__ >= 30)) && \ (defined(__linux__) || defined(__gnu_linux__)) - if ((flags & MDBX_warmup_lock) != 0 && - globals.linux_kernel_version >= 0x04040000 && + if ((flags & MDBX_warmup_lock) != 0 && globals.linux_kernel_version >= 0x04040000 && atomic_load32(&env->mlocked_pgno, mo_AcquireRelease) < mlock_pgno) { if (mlock2(env->dxb_mmap.base, used_range, MLOCK_ONFAULT)) { rc = errno; @@ -388,8 +345,7 @@ __cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, if (err != MDBX_SUCCESS && rc == MDBX_SUCCESS) rc = err; - if ((flags & MDBX_warmup_force) != 0 && - (rc == MDBX_SUCCESS || rc == MDBX_ENOSYS)) { + if ((flags & MDBX_warmup_force) != 0 && (rc == MDBX_SUCCESS || rc == MDBX_ENOSYS)) { const volatile uint8_t *ptr = env->dxb_mmap.base; size_t offset = 0, unused = 42; #if !(defined(_WIN32) || defined(_WIN64)) @@ -440,8 +396,7 @@ __cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, (void)unused; } - if ((flags & MDBX_warmup_lock) != 0 && - (rc == MDBX_SUCCESS || rc == MDBX_ENOSYS) && + if ((flags & MDBX_warmup_lock) != 0 && (rc == MDBX_SUCCESS || rc == MDBX_ENOSYS) && atomic_load32(&env->mlocked_pgno, mo_AcquireRelease) < mlock_pgno) { #if defined(_WIN32) || defined(_WIN64) if (VirtualLock(env->dxb_mmap.base, used_range)) { @@ -481,14 +436,12 @@ __cold int mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *arg) { return MDBX_SUCCESS; } -__cold int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags, - bool onoff) { +__cold int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags, bool onoff) { int rc = check_env(env, false); if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); - if (unlikely(flags & ((env->flags & ENV_ACTIVE) ? ~ENV_CHANGEABLE_FLAGS - : ~ENV_USABLE_FLAGS))) + if (unlikely(flags & ((env->flags & ENV_ACTIVE) ? ~ENV_CHANGEABLE_FLAGS : ~ENV_USABLE_FLAGS))) return LOG_IFERR(MDBX_EPERM); if (unlikely(env->flags & MDBX_RDONLY)) @@ -536,9 +489,7 @@ __cold int mdbx_env_set_userctx(MDBX_env *env, void *ctx) { return MDBX_SUCCESS; } -__cold void *mdbx_env_get_userctx(const MDBX_env *env) { - return env ? env->userctx : nullptr; -} +__cold void *mdbx_env_get_userctx(const MDBX_env *env) { return env ? env->userctx : nullptr; } __cold int mdbx_env_set_assert(MDBX_env *env, MDBX_assert_func *func) { int rc = check_env(env, false); @@ -564,8 +515,7 @@ __cold int mdbx_env_set_hsr(MDBX_env *env, MDBX_hsr_func *hsr) { } __cold MDBX_hsr_func *mdbx_env_get_hsr(const MDBX_env *env) { - return likely(env && env->signature.weak == env_signature) ? env->hsr_callback - : nullptr; + return likely(env && env->signature.weak == env_signature) ? env->hsr_callback : nullptr; } #if defined(_WIN32) || defined(_WIN64) @@ -595,13 +545,10 @@ __cold int mdbx_env_get_path(const MDBX_env *env, const char **arg) { *arg = nullptr; DWORD flags = /* WC_ERR_INVALID_CHARS */ 0x80; size_t mb_len = - WideCharToMultiByte(CP_THREAD_ACP, flags, env->pathname.specified, -1, - nullptr, 0, nullptr, nullptr); + WideCharToMultiByte(CP_THREAD_ACP, flags, env->pathname.specified, -1, nullptr, 0, nullptr, nullptr); rc = mb_len ? MDBX_SUCCESS : (int)GetLastError(); if (rc == ERROR_INVALID_FLAGS) { - mb_len = - WideCharToMultiByte(CP_THREAD_ACP, flags = 0, env->pathname.specified, - -1, nullptr, 0, nullptr, nullptr); + mb_len = WideCharToMultiByte(CP_THREAD_ACP, flags = 0, env->pathname.specified, -1, nullptr, 0, nullptr, nullptr); rc = mb_len ? MDBX_SUCCESS : (int)GetLastError(); } if (unlikely(rc != MDBX_SUCCESS)) @@ -610,16 +557,14 @@ __cold int mdbx_env_get_path(const MDBX_env *env, const char **arg) { char *const mb_pathname = osal_malloc(mb_len); if (!mb_pathname) return LOG_IFERR(MDBX_ENOMEM); - if (mb_len != (size_t)WideCharToMultiByte( - CP_THREAD_ACP, flags, env->pathname.specified, -1, - mb_pathname, (int)mb_len, nullptr, nullptr)) { + if (mb_len != (size_t)WideCharToMultiByte(CP_THREAD_ACP, flags, env->pathname.specified, -1, mb_pathname, + (int)mb_len, nullptr, nullptr)) { rc = (int)GetLastError(); osal_free(mb_pathname); return LOG_IFERR(rc); } if (env->pathname_char || - InterlockedCompareExchangePointer((PVOID volatile *)&env->pathname_char, - mb_pathname, nullptr)) + InterlockedCompareExchangePointer((PVOID volatile *)&env->pathname_char, mb_pathname, nullptr)) osal_free(mb_pathname); } *arg = env->pathname_char; @@ -634,41 +579,29 @@ __cold int mdbx_env_get_path(const MDBX_env *env, const char **arg) { #ifndef LIBMDBX_NO_EXPORTS_LEGACY_API -LIBMDBX_API int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, - MDBX_txn_flags_t flags, MDBX_txn **ret) { +LIBMDBX_API int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, MDBX_txn **ret) { return __inline_mdbx_txn_begin(env, parent, flags, ret); } -LIBMDBX_API int mdbx_txn_commit(MDBX_txn *txn) { - return __inline_mdbx_txn_commit(txn); -} +LIBMDBX_API int mdbx_txn_commit(MDBX_txn *txn) { return __inline_mdbx_txn_commit(txn); } -LIBMDBX_API __cold int mdbx_env_stat(const MDBX_env *env, MDBX_stat *stat, - size_t bytes) { +LIBMDBX_API __cold int mdbx_env_stat(const MDBX_env *env, MDBX_stat *stat, size_t bytes) { return __inline_mdbx_env_stat(env, stat, bytes); } -LIBMDBX_API __cold int mdbx_env_info(const MDBX_env *env, MDBX_envinfo *info, - size_t bytes) { +LIBMDBX_API __cold int mdbx_env_info(const MDBX_env *env, MDBX_envinfo *info, size_t bytes) { return __inline_mdbx_env_info(env, info, bytes); } -LIBMDBX_API int mdbx_dbi_flags(const MDBX_txn *txn, MDBX_dbi dbi, - unsigned *flags) { +LIBMDBX_API int mdbx_dbi_flags(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags) { return __inline_mdbx_dbi_flags(txn, dbi, flags); } -LIBMDBX_API __cold int mdbx_env_sync(MDBX_env *env) { - return __inline_mdbx_env_sync(env); -} +LIBMDBX_API __cold int mdbx_env_sync(MDBX_env *env) { return __inline_mdbx_env_sync(env); } -LIBMDBX_API __cold int mdbx_env_sync_poll(MDBX_env *env) { - return __inline_mdbx_env_sync_poll(env); -} +LIBMDBX_API __cold int mdbx_env_sync_poll(MDBX_env *env) { return __inline_mdbx_env_sync_poll(env); } -LIBMDBX_API __cold int mdbx_env_close(MDBX_env *env) { - return __inline_mdbx_env_close(env); -} +LIBMDBX_API __cold int mdbx_env_close(MDBX_env *env) { return __inline_mdbx_env_close(env); } LIBMDBX_API __cold int mdbx_env_set_mapsize(MDBX_env *env, size_t size) { return __inline_mdbx_env_set_mapsize(env, size); @@ -682,13 +615,11 @@ LIBMDBX_API __cold int mdbx_env_get_maxdbs(const MDBX_env *env, MDBX_dbi *dbs) { return __inline_mdbx_env_get_maxdbs(env, dbs); } -LIBMDBX_API __cold int mdbx_env_set_maxreaders(MDBX_env *env, - unsigned readers) { +LIBMDBX_API __cold int mdbx_env_set_maxreaders(MDBX_env *env, unsigned readers) { return __inline_mdbx_env_set_maxreaders(env, readers); } -LIBMDBX_API __cold int mdbx_env_get_maxreaders(const MDBX_env *env, - unsigned *readers) { +LIBMDBX_API __cold int mdbx_env_get_maxreaders(const MDBX_env *env, unsigned *readers) { return __inline_mdbx_env_get_maxreaders(env, readers); } @@ -696,35 +627,24 @@ LIBMDBX_API __cold int mdbx_env_set_syncbytes(MDBX_env *env, size_t threshold) { return __inline_mdbx_env_set_syncbytes(env, threshold); } -LIBMDBX_API __cold int mdbx_env_get_syncbytes(const MDBX_env *env, - size_t *threshold) { +LIBMDBX_API __cold int mdbx_env_get_syncbytes(const MDBX_env *env, size_t *threshold) { return __inline_mdbx_env_get_syncbytes(env, threshold); } -LIBMDBX_API __cold int mdbx_env_set_syncperiod(MDBX_env *env, - unsigned seconds_16dot16) { +LIBMDBX_API __cold int mdbx_env_set_syncperiod(MDBX_env *env, unsigned seconds_16dot16) { return __inline_mdbx_env_set_syncperiod(env, seconds_16dot16); } -LIBMDBX_API __cold int mdbx_env_get_syncperiod(const MDBX_env *env, - unsigned *seconds_16dot16) { +LIBMDBX_API __cold int mdbx_env_get_syncperiod(const MDBX_env *env, unsigned *seconds_16dot16) { return __inline_mdbx_env_get_syncperiod(env, seconds_16dot16); } -LIBMDBX_API __cold uint64_t mdbx_key_from_int64(const int64_t i64) { - return __inline_mdbx_key_from_int64(i64); -} +LIBMDBX_API __cold uint64_t mdbx_key_from_int64(const int64_t i64) { return __inline_mdbx_key_from_int64(i64); } -LIBMDBX_API __cold uint32_t mdbx_key_from_int32(const int32_t i32) { - return __inline_mdbx_key_from_int32(i32); -} +LIBMDBX_API __cold uint32_t mdbx_key_from_int32(const int32_t i32) { return __inline_mdbx_key_from_int32(i32); } -LIBMDBX_API __cold intptr_t mdbx_limits_pgsize_min(void) { - return __inline_mdbx_limits_pgsize_min(); -} +LIBMDBX_API __cold intptr_t mdbx_limits_pgsize_min(void) { return __inline_mdbx_limits_pgsize_min(); } -LIBMDBX_API __cold intptr_t mdbx_limits_pgsize_max(void) { - return __inline_mdbx_limits_pgsize_max(); -} +LIBMDBX_API __cold intptr_t mdbx_limits_pgsize_max(void) { return __inline_mdbx_limits_pgsize_max(); } #endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */ diff --git a/src/copy.c b/src/copy.c index c1b7ef7d..3c7a4f2c 100644 --- a/src/copy.c +++ b/src/copy.c @@ -93,8 +93,7 @@ __cold static int compacting_toggle_write_buffers(ctx_t *ctx) { return ctx->error; } -static int compacting_put_bytes(ctx_t *ctx, const void *src, size_t bytes, - pgno_t pgno, pgno_t npages) { +static int compacting_put_bytes(ctx_t *ctx, const void *src, size_t bytes, pgno_t pgno, pgno_t npages) { assert(pgno == 0 || bytes > PAGEHDRSZ); while (bytes > 0) { const size_t side = ctx->head & 1; @@ -130,17 +129,14 @@ static int compacting_put_bytes(ctx_t *ctx, const void *src, size_t bytes, return MDBX_SUCCESS; } -static int compacting_put_page(ctx_t *ctx, const page_t *mp, - const size_t head_bytes, const size_t tail_bytes, +static int compacting_put_page(ctx_t *ctx, const page_t *mp, const size_t head_bytes, const size_t tail_bytes, const pgno_t npages) { if (tail_bytes) { assert(head_bytes + tail_bytes <= ctx->env->ps); - assert(npages == 1 && - (page_type(mp) == P_BRANCH || page_type(mp) == P_LEAF)); + assert(npages == 1 && (page_type(mp) == P_BRANCH || page_type(mp) == P_LEAF)); } else { assert(head_bytes <= pgno2bytes(ctx->env, npages)); - assert((npages == 1 && page_type(mp) == (P_LEAF | P_DUPFIX)) || - page_type(mp) == P_LARGE); + assert((npages == 1 && page_type(mp) == (P_LEAF | P_DUPFIX)) || page_type(mp) == P_LARGE); } const pgno_t pgno = ctx->first_unallocated; @@ -148,18 +144,13 @@ static int compacting_put_page(ctx_t *ctx, const page_t *mp, int err = compacting_put_bytes(ctx, mp, head_bytes, pgno, npages); if (unlikely(err != MDBX_SUCCESS)) return err; - err = compacting_put_bytes( - ctx, nullptr, pgno2bytes(ctx->env, npages) - (head_bytes + tail_bytes), 0, - 0); + err = compacting_put_bytes(ctx, nullptr, pgno2bytes(ctx->env, npages) - (head_bytes + tail_bytes), 0, 0); if (unlikely(err != MDBX_SUCCESS)) return err; - return compacting_put_bytes(ctx, ptr_disp(mp, ctx->env->ps - tail_bytes), - tail_bytes, 0, 0); + return compacting_put_bytes(ctx, ptr_disp(mp, ctx->env->ps - tail_bytes), tail_bytes, 0, 0); } -__cold static int compacting_walk(ctx_t *ctx, MDBX_cursor *mc, - pgno_t *const parent_pgno, - txnid_t parent_txnid) { +__cold static int compacting_walk(ctx_t *ctx, MDBX_cursor *mc, pgno_t *const parent_pgno, txnid_t parent_txnid) { mc->top = 0; mc->ki[0] = 0; int rc = page_get(mc, *parent_pgno, &mc->pg[0], parent_txnid); @@ -201,22 +192,18 @@ __cold static int compacting_walk(ctx_t *ctx, MDBX_cursor *mc, node = page_node(mp, i); } - const pgr_t lp = - page_get_large(mc, node_largedata_pgno(node), mp->txnid); + const pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->txnid); if (unlikely((rc = lp.err) != MDBX_SUCCESS)) goto bailout; const size_t datasize = node_ds(node); const pgno_t npages = largechunk_npages(ctx->env, datasize); poke_pgno(node_data(node), ctx->first_unallocated); - rc = compacting_put_page(ctx, lp.page, PAGEHDRSZ + datasize, 0, - npages); + rc = compacting_put_page(ctx, lp.page, PAGEHDRSZ + datasize, 0, npages); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; } else if (node_flags(node) & N_TREE) { - if (!MDBX_DISABLE_VALIDATION && - unlikely(node_ds(node) != sizeof(tree_t))) { - ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid dupsort sub-tree node size", + if (!MDBX_DISABLE_VALIDATION && unlikely(node_ds(node) != sizeof(tree_t))) { + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid dupsort sub-tree node size", (unsigned)node_ds(node)); rc = MDBX_CORRUPTED; goto bailout; @@ -235,13 +222,11 @@ __cold static int compacting_walk(ctx_t *ctx, MDBX_cursor *mc, rc = cursor_dupsort_setup(mc, node, mp); if (likely(rc == MDBX_SUCCESS)) { nested = &mc->subcur->nested_tree; - rc = compacting_walk(ctx, &mc->subcur->cursor, &nested->root, - mp->txnid); + rc = compacting_walk(ctx, &mc->subcur->cursor, &nested->root, mp->txnid); } } else { cASSERT(mc, (mc->flags & z_inner) == 0 && mc->subcur == 0); - cursor_couple_t *couple = - container_of(mc, cursor_couple_t, outer); + cursor_couple_t *couple = container_of(mc, cursor_couple_t, outer); nested = &couple->inner.nested_tree; memcpy(nested, node_data(node), sizeof(tree_t)); rc = compacting_walk_tree(ctx, nested); @@ -280,11 +265,9 @@ __cold static int compacting_walk(ctx_t *ctx, MDBX_cursor *mc, const pgno_t pgno = ctx->first_unallocated; if (likely(!is_dupfix_leaf(mp))) { - rc = compacting_put_page(ctx, mp, PAGEHDRSZ + mp->lower, - ctx->env->ps - (PAGEHDRSZ + mp->upper), 1); + rc = compacting_put_page(ctx, mp, PAGEHDRSZ + mp->lower, ctx->env->ps - (PAGEHDRSZ + mp->upper), 1); } else { - rc = compacting_put_page( - ctx, mp, PAGEHDRSZ + page_numkeys(mp) * mp->dupfix_ksize, 0, 1); + rc = compacting_put_page(ctx, mp, PAGEHDRSZ + page_numkeys(mp) * mp->dupfix_ksize, 0, 1); } if (unlikely(rc != MDBX_SUCCESS)) goto bailout; @@ -326,19 +309,15 @@ __cold static int compacting_walk_tree(ctx_t *ctx, tree_t *tree) { __cold static void compacting_fixup_meta(MDBX_env *env, meta_t *meta) { eASSERT(env, meta->trees.gc.mod_txnid || meta->trees.gc.root == P_INVALID); - eASSERT(env, - meta->trees.main.mod_txnid || meta->trees.main.root == P_INVALID); + eASSERT(env, meta->trees.main.mod_txnid || meta->trees.main.root == P_INVALID); /* Calculate filesize taking in account shrink/growing thresholds */ if (meta->geometry.first_unallocated != meta->geometry.now) { meta->geometry.now = meta->geometry.first_unallocated; - const size_t aligner = - pv2pages(meta->geometry.grow_pv ? meta->geometry.grow_pv - : meta->geometry.shrink_pv); + const size_t aligner = pv2pages(meta->geometry.grow_pv ? meta->geometry.grow_pv : meta->geometry.shrink_pv); if (aligner) { - const pgno_t aligned = pgno_align2os_pgno( - env, meta->geometry.first_unallocated + aligner - - meta->geometry.first_unallocated % aligner); + const pgno_t aligned = pgno_align2os_pgno(env, meta->geometry.first_unallocated + aligner - + meta->geometry.first_unallocated % aligner); meta->geometry.now = aligned; } } @@ -366,13 +345,10 @@ __cold static void meta_make_sizeable(meta_t *meta) { } } -__cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn, - mdbx_filehandle_t fd, uint8_t *buffer, - const bool dest_is_pipe, - const MDBX_copy_flags_t flags) { +__cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn, mdbx_filehandle_t fd, uint8_t *buffer, + const bool dest_is_pipe, const MDBX_copy_flags_t flags) { const size_t meta_bytes = pgno2bytes(env, NUM_METAS); - uint8_t *const data_buffer = - buffer + ceil_powerof2(meta_bytes, globals.sys_pagesize); + uint8_t *const data_buffer = buffer + ceil_powerof2(meta_bytes, globals.sys_pagesize); meta_t *const meta = meta_init_triplet(env, buffer); meta_set_txnid(env, meta, txn->txnid); @@ -405,22 +381,17 @@ __cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn, int rc = cursor_init(&couple.outer, txn, FREE_DBI); if (unlikely(rc != MDBX_SUCCESS)) return rc; - pgno_t gc_npages = txn->dbs[FREE_DBI].branch_pages + - txn->dbs[FREE_DBI].leaf_pages + - txn->dbs[FREE_DBI].large_pages; + pgno_t gc_npages = txn->dbs[FREE_DBI].branch_pages + txn->dbs[FREE_DBI].leaf_pages + txn->dbs[FREE_DBI].large_pages; MDBX_val key, data; rc = outer_first(&couple.outer, &key, &data); while (rc == MDBX_SUCCESS) { const pnl_t pnl = data.iov_base; - if (unlikely(data.iov_len % sizeof(pgno_t) || - data.iov_len < MDBX_PNL_SIZEOF(pnl))) { - ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid GC-record length", data.iov_len); + if (unlikely(data.iov_len % sizeof(pgno_t) || data.iov_len < MDBX_PNL_SIZEOF(pnl))) { + ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-record length", data.iov_len); return MDBX_CORRUPTED; } if (unlikely(!pnl_check(pnl, txn->geo.first_unallocated))) { - ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid GC-record content"); + ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-record content"); return MDBX_CORRUPTED; } gc_npages += MDBX_PNL_GETSIZE(pnl); @@ -466,21 +437,16 @@ __cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn, /* toggle to flush non-empty buffers */ compacting_toggle_write_buffers(&ctx); - if (likely(rc == MDBX_SUCCESS) && - unlikely(meta->geometry.first_unallocated != ctx.first_unallocated)) { + if (likely(rc == MDBX_SUCCESS) && unlikely(meta->geometry.first_unallocated != ctx.first_unallocated)) { if (ctx.first_unallocated > meta->geometry.first_unallocated) { - ERROR("the source DB %s: post-compactification used pages %" PRIaPGNO - " %c expected %" PRIaPGNO, - "has double-used pages or other corruption", - ctx.first_unallocated, '>', meta->geometry.first_unallocated); + ERROR("the source DB %s: post-compactification used pages %" PRIaPGNO " %c expected %" PRIaPGNO, + "has double-used pages or other corruption", ctx.first_unallocated, '>', + meta->geometry.first_unallocated); rc = MDBX_CORRUPTED; /* corrupted DB */ } if (ctx.first_unallocated < meta->geometry.first_unallocated) { - WARNING( - "the source DB %s: post-compactification used pages %" PRIaPGNO - " %c expected %" PRIaPGNO, - "has page leak(s)", ctx.first_unallocated, '<', - meta->geometry.first_unallocated); + WARNING("the source DB %s: post-compactification used pages %" PRIaPGNO " %c expected %" PRIaPGNO, + "has page leak(s)", ctx.first_unallocated, '<', meta->geometry.first_unallocated); if (dest_is_pipe) /* the root within already written meta-pages is wrong */ rc = MDBX_CORRUPTED; @@ -493,8 +459,7 @@ __cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn, eASSERT(env, (ctx.write_len[ctx.head & 1]) == 0); compacting_toggle_write_buffers(&ctx); thread_err = osal_thread_join(thread); - eASSERT(env, (ctx.tail == ctx.head && ctx.write_len[ctx.head & 1] == 0) || - ctx.error); + eASSERT(env, (ctx.tail == ctx.head && ctx.write_len[ctx.head & 1] == 0) || ctx.error); osal_condpair_destroy(&ctx.condpair); } if (unlikely(thread_err != MDBX_SUCCESS)) @@ -519,9 +484,8 @@ __cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn, const size_t used_size = pgno2bytes(env, meta->geometry.first_unallocated); memset(data_buffer, 0, (size_t)MDBX_ENVCOPY_WRITEBUF); for (size_t offset = used_size; offset < whole_size;) { - const size_t chunk = ((size_t)MDBX_ENVCOPY_WRITEBUF < whole_size - offset) - ? (size_t)MDBX_ENVCOPY_WRITEBUF - : whole_size - offset; + const size_t chunk = + ((size_t)MDBX_ENVCOPY_WRITEBUF < whole_size - offset) ? (size_t)MDBX_ENVCOPY_WRITEBUF : whole_size - offset; int rc = osal_write(fd, data_buffer, chunk); if (unlikely(rc != MDBX_SUCCESS)) return rc; @@ -533,9 +497,8 @@ __cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn, //---------------------------------------------------------------------------- -__cold static int copy_asis(MDBX_env *env, MDBX_txn *txn, mdbx_filehandle_t fd, - uint8_t *buffer, const bool dest_is_pipe, - const MDBX_copy_flags_t flags) { +__cold static int copy_asis(MDBX_env *env, MDBX_txn *txn, mdbx_filehandle_t fd, uint8_t *buffer, + const bool dest_is_pipe, const MDBX_copy_flags_t flags) { bool should_unlock = false; if ((txn->flags & MDBX_TXN_RDONLY) != 0 && (flags & MDBX_CP_RENEW_TXN) != 0) { /* Try temporarily block writers until we snapshot the meta pages */ @@ -566,8 +529,7 @@ retry_snap_meta: rc = MDBX_MVCC_RETARDED; for (size_t n = 0; n < NUM_METAS; ++n) { meta_t *const meta = page_meta(ptr_disp(buffer, pgno2bytes(env, n))); - if (troika.txnid[n] == txn->txnid && - ((/* is_steady */ (troika.fsm >> n) & 1) || rc != MDBX_SUCCESS)) { + if (troika.txnid[n] == txn->txnid && ((/* is_steady */ (troika.fsm >> n) & 1) || rc != MDBX_SUCCESS)) { rc = MDBX_SUCCESS; headcopy = meta; } else if (troika.txnid[n] > txn->txnid) @@ -606,14 +568,12 @@ retry_snap_meta: if (dest_is_pipe) rc = osal_write(fd, buffer, meta_bytes); - uint8_t *const data_buffer = - buffer + ceil_powerof2(meta_bytes, globals.sys_pagesize); + uint8_t *const data_buffer = buffer + ceil_powerof2(meta_bytes, globals.sys_pagesize); #if MDBX_USE_COPYFILERANGE static bool copyfilerange_unavailable; bool not_the_same_filesystem = false; struct statfs statfs_info; - if (fstatfs(fd, &statfs_info) || - statfs_info.f_type == /* ECRYPTFS_SUPER_MAGIC */ 0xf15f) + if (fstatfs(fd, &statfs_info) || statfs_info.f_type == /* ECRYPTFS_SUPER_MAGIC */ 0xf15f) /* avoid use copyfilerange_unavailable() to ecryptfs due bugs */ not_the_same_filesystem = true; #endif /* MDBX_USE_COPYFILERANGE */ @@ -629,8 +589,7 @@ retry_snap_meta: static bool sendfile_unavailable; if (dest_is_pipe && likely(!sendfile_unavailable)) { off_t in_offset = offset; - const ssize_t written = - sendfile(fd, env->lazy_fd, &in_offset, used_size - offset); + const ssize_t written = sendfile(fd, env->lazy_fd, &in_offset, used_size - offset); if (likely(written > 0)) { offset = in_offset; if (flags & MDBX_CP_THROTTLE_MVCC) @@ -645,11 +604,9 @@ retry_snap_meta: #endif /* MDBX_USE_SENDFILE */ #if MDBX_USE_COPYFILERANGE - if (!dest_is_pipe && !not_the_same_filesystem && - likely(!copyfilerange_unavailable)) { + if (!dest_is_pipe && !not_the_same_filesystem && likely(!copyfilerange_unavailable)) { off_t in_offset = offset, out_offset = offset; - ssize_t bytes_copied = copy_file_range( - env->lazy_fd, &in_offset, fd, &out_offset, used_size - offset, 0); + ssize_t bytes_copied = copy_file_range(env->lazy_fd, &in_offset, fd, &out_offset, used_size - offset, 0); if (likely(bytes_copied > 0)) { offset = in_offset; if (flags & MDBX_CP_THROTTLE_MVCC) @@ -672,9 +629,8 @@ retry_snap_meta: #endif /* MDBX_USE_COPYFILERANGE */ /* fallback to portable */ - const size_t chunk = ((size_t)MDBX_ENVCOPY_WRITEBUF < used_size - offset) - ? (size_t)MDBX_ENVCOPY_WRITEBUF - : used_size - offset; + const size_t chunk = + ((size_t)MDBX_ENVCOPY_WRITEBUF < used_size - offset) ? (size_t)MDBX_ENVCOPY_WRITEBUF : used_size - offset; /* copy to avoid EFAULT in case swapped-out */ memcpy(data_buffer, ptr_disp(env->dxb_mmap.base, offset), chunk); if (flags & MDBX_CP_THROTTLE_MVCC) @@ -689,12 +645,9 @@ retry_snap_meta: rc = osal_ftruncate(fd, whole_size); else { memset(data_buffer, 0, (size_t)MDBX_ENVCOPY_WRITEBUF); - for (size_t offset = used_size; - rc == MDBX_SUCCESS && offset < whole_size;) { + for (size_t offset = used_size; rc == MDBX_SUCCESS && offset < whole_size;) { const size_t chunk = - ((size_t)MDBX_ENVCOPY_WRITEBUF < whole_size - offset) - ? (size_t)MDBX_ENVCOPY_WRITEBUF - : whole_size - offset; + ((size_t)MDBX_ENVCOPY_WRITEBUF < whole_size - offset) ? (size_t)MDBX_ENVCOPY_WRITEBUF : whole_size - offset; rc = osal_write(fd, data_buffer, chunk); offset += chunk; } @@ -706,8 +659,7 @@ retry_snap_meta: //---------------------------------------------------------------------------- -__cold static int copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, - MDBX_copy_flags_t flags) { +__cold static int copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, MDBX_copy_flags_t flags) { if (unlikely(txn->flags & MDBX_TXN_DIRTY)) return MDBX_BAD_TXN; @@ -734,9 +686,7 @@ __cold static int copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, MDBX_env *const env = txn->env; const size_t buffer_size = pgno_align2os_bytes(env, NUM_METAS) + - ceil_powerof2(((flags & MDBX_CP_COMPACT) - ? 2 * (size_t)MDBX_ENVCOPY_WRITEBUF - : (size_t)MDBX_ENVCOPY_WRITEBUF), + ceil_powerof2(((flags & MDBX_CP_COMPACT) ? 2 * (size_t)MDBX_ENVCOPY_WRITEBUF : (size_t)MDBX_ENVCOPY_WRITEBUF), globals.sys_pagesize); uint8_t *buffer = nullptr; @@ -755,8 +705,7 @@ __cold static int copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, rc = mdbx_txn_unpark(txn, false); if (likely(rc == MDBX_SUCCESS)) { memset(buffer, 0, pgno2bytes(env, NUM_METAS)); - rc = ((flags & MDBX_CP_COMPACT) ? copy_with_compacting : copy_asis)( - env, txn, fd, buffer, dest_is_pipe, flags); + rc = ((flags & MDBX_CP_COMPACT) ? copy_with_compacting : copy_asis)(env, txn, fd, buffer, dest_is_pipe, flags); if (likely(rc == MDBX_SUCCESS)) rc = mdbx_txn_unpark(txn, false); @@ -785,8 +734,7 @@ __cold static int copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, return rc; } -__cold static int copy2pathname(MDBX_txn *txn, const pathchar_t *dest_path, - MDBX_copy_flags_t flags) { +__cold static int copy2pathname(MDBX_txn *txn, const pathchar_t *dest_path, MDBX_copy_flags_t flags) { if (unlikely(!dest_path || *dest_path == '\0')) return MDBX_EINVAL; @@ -813,7 +761,7 @@ __cold static int copy2pathname(MDBX_txn *txn, const pathchar_t *dest_path, lock_op.l_start = 0; lock_op.l_len = OFF_T_MAX; if (MDBX_FCNTL(newfd, MDBX_F_SETLK, &lock_op) -#if (defined(__linux__) || defined(__gnu_linux__)) && defined(LOCK_EX) && \ +#if (defined(__linux__) || defined(__gnu_linux__)) && defined(LOCK_EX) && \ (!defined(__ANDROID_API__) || __ANDROID_API__ >= 24) || flock(newfd, LOCK_EX | LOCK_NB) #endif /* Linux */ @@ -837,8 +785,7 @@ __cold static int copy2pathname(MDBX_txn *txn, const pathchar_t *dest_path, //---------------------------------------------------------------------------- -__cold int mdbx_txn_copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, - MDBX_copy_flags_t flags) { +__cold int mdbx_txn_copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, MDBX_copy_flags_t flags) { int rc = check_txn(txn, MDBX_TXN_BLOCKED); if (likely(rc == MDBX_SUCCESS)) rc = copy2fd(txn, fd, flags); @@ -847,8 +794,7 @@ __cold int mdbx_txn_copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, return LOG_IFERR(rc); } -__cold int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, - MDBX_copy_flags_t flags) { +__cold int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, MDBX_copy_flags_t flags) { if (unlikely(flags & (MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN))) return LOG_IFERR(MDBX_EINVAL); @@ -866,8 +812,7 @@ __cold int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, return LOG_IFERR(rc); } -__cold int mdbx_txn_copy2pathname(MDBX_txn *txn, const char *dest_path, - MDBX_copy_flags_t flags) { +__cold int mdbx_txn_copy2pathname(MDBX_txn *txn, const char *dest_path, MDBX_copy_flags_t flags) { #if defined(_WIN32) || defined(_WIN64) wchar_t *dest_pathW = nullptr; int rc = osal_mb2w(dest_path, &dest_pathW); @@ -878,8 +823,7 @@ __cold int mdbx_txn_copy2pathname(MDBX_txn *txn, const char *dest_path, return LOG_IFERR(rc); } -__cold int mdbx_txn_copy2pathnameW(MDBX_txn *txn, const wchar_t *dest_path, - MDBX_copy_flags_t flags) { +__cold int mdbx_txn_copy2pathnameW(MDBX_txn *txn, const wchar_t *dest_path, MDBX_copy_flags_t flags) { #endif /* Windows */ int rc = check_txn(txn, MDBX_TXN_BLOCKED); if (likely(rc == MDBX_SUCCESS)) @@ -889,8 +833,7 @@ __cold int mdbx_txn_copy2pathnameW(MDBX_txn *txn, const wchar_t *dest_path, return LOG_IFERR(rc); } -__cold int mdbx_env_copy(MDBX_env *env, const char *dest_path, - MDBX_copy_flags_t flags) { +__cold int mdbx_env_copy(MDBX_env *env, const char *dest_path, MDBX_copy_flags_t flags) { #if defined(_WIN32) || defined(_WIN64) wchar_t *dest_pathW = nullptr; int rc = osal_mb2w(dest_path, &dest_pathW); @@ -901,8 +844,7 @@ __cold int mdbx_env_copy(MDBX_env *env, const char *dest_path, return LOG_IFERR(rc); } -__cold int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest_path, - MDBX_copy_flags_t flags) { +__cold int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest_path, MDBX_copy_flags_t flags) { #endif /* Windows */ if (unlikely(flags & (MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN))) return LOG_IFERR(MDBX_EINVAL); @@ -916,8 +858,7 @@ __cold int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest_path, if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); - rc = copy2pathname(txn, dest_path, - flags | MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN); + rc = copy2pathname(txn, dest_path, flags | MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN); mdbx_txn_abort(txn); return LOG_IFERR(rc); } diff --git a/src/cursor.c b/src/cursor.c index 4371f753..a4bffd76 100644 --- a/src/cursor.c +++ b/src/cursor.c @@ -11,14 +11,11 @@ __cold int cursor_check(const MDBX_cursor *mc) { } else { cASSERT(mc, (mc->txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); cASSERT(mc, mc->txn->tw.dirtyroom + mc->txn->tw.dirtylist->length == - (mc->txn->parent ? mc->txn->parent->tw.dirtyroom - : mc->txn->env->options.dp_limit)); + (mc->txn->parent ? mc->txn->parent->tw.dirtyroom : mc->txn->env->options.dp_limit)); } - cASSERT(mc, (mc->checking & z_updating) ? mc->top + 1 <= mc->tree->height - : mc->top + 1 == mc->tree->height); - if (unlikely((mc->checking & z_updating) ? mc->top + 1 > mc->tree->height - : mc->top + 1 != mc->tree->height)) + cASSERT(mc, (mc->checking & z_updating) ? mc->top + 1 <= mc->tree->height : mc->top + 1 == mc->tree->height); + if (unlikely((mc->checking & z_updating) ? mc->top + 1 > mc->tree->height : mc->top + 1 != mc->tree->height)) return MDBX_CURSOR_FULL; if (is_pointed(mc) && (mc->checking & z_updating) == 0) { @@ -40,17 +37,14 @@ __cold int cursor_check(const MDBX_cursor *mc) { page_t *mp = mc->pg[n]; const size_t nkeys = page_numkeys(mp); const bool expect_branch = (n < mc->tree->height - 1) ? true : false; - const bool expect_nested_leaf = - (n + 1 == mc->tree->height - 1) ? true : false; + const bool expect_nested_leaf = (n + 1 == mc->tree->height - 1) ? true : false; const bool branch = is_branch(mp) ? true : false; cASSERT(mc, branch == expect_branch); if (unlikely(branch != expect_branch)) return MDBX_CURSOR_FULL; if ((mc->checking & z_updating) == 0) { - cASSERT(mc, nkeys > mc->ki[n] || (!branch && nkeys == mc->ki[n] && - (mc->flags & z_hollow) != 0)); - if (unlikely(nkeys <= mc->ki[n] && !(!branch && nkeys == mc->ki[n] && - (mc->flags & z_hollow) != 0))) + cASSERT(mc, nkeys > mc->ki[n] || (!branch && nkeys == mc->ki[n] && (mc->flags & z_hollow) != 0)); + if (unlikely(nkeys <= mc->ki[n] && !(!branch && nkeys == mc->ki[n] && (mc->flags & z_hollow) != 0))) return MDBX_CURSOR_FULL; } else { cASSERT(mc, nkeys + 1 >= mc->ki[n]); @@ -96,8 +90,7 @@ __cold int cursor_check_updating(MDBX_cursor *mc) { } bool cursor_is_tracked(const MDBX_cursor *mc) { - for (MDBX_cursor *scan = mc->txn->cursors[cursor_dbi(mc)]; scan; - scan = scan->next) + for (MDBX_cursor *scan = mc->txn->cursors[cursor_dbi(mc)]; scan; scan = scan->next) if (mc == ((mc->flags & z_inner) ? &scan->subcur->cursor : scan)) return true; return false; @@ -121,16 +114,14 @@ static int touch_dbi(MDBX_cursor *mc) { if (unlikely(rc != MDBX_SUCCESS)) return rc; mc->txn->dbi_state[MAIN_DBI] |= DBI_DIRTY; - rc = tree_search(&cx.outer, &container_of(mc->clc, kvx_t, clc)->name, - Z_MODIFY); + rc = tree_search(&cx.outer, &container_of(mc->clc, kvx_t, clc)->name, Z_MODIFY); if (unlikely(rc != MDBX_SUCCESS)) return rc; } return MDBX_SUCCESS; } -__hot int cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, - const MDBX_val *data) { +__hot int cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, const MDBX_val *data) { cASSERT(mc, (mc->txn->flags & MDBX_TXN_RDONLY) == 0); cASSERT(mc, is_pointed(mc) || mc->tree->height == 0); cASSERT(mc, cursor_is_tracked(mc)); @@ -193,13 +184,10 @@ __hot int cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, /*----------------------------------------------------------------------------*/ -int cursor_shadow(MDBX_cursor *parent_cursor, MDBX_txn *nested_txn, - const size_t dbi) { +int cursor_shadow(MDBX_cursor *parent_cursor, MDBX_txn *nested_txn, const size_t dbi) { tASSERT(nested_txn, dbi > FREE_DBI && dbi < nested_txn->n_dbi); - const size_t size = parent_cursor->subcur - ? sizeof(MDBX_cursor) + sizeof(subcur_t) - : sizeof(MDBX_cursor); + const size_t size = parent_cursor->subcur ? sizeof(MDBX_cursor) + sizeof(subcur_t) : sizeof(MDBX_cursor); for (MDBX_cursor *bk; parent_cursor; parent_cursor = bk->next) { cASSERT(parent_cursor, parent_cursor != parent_cursor->next); bk = parent_cursor; @@ -235,8 +223,7 @@ int cursor_shadow(MDBX_cursor *parent_cursor, MDBX_txn *nested_txn, void cursor_eot(MDBX_cursor *mc, const bool merge) { const unsigned stage = mc->signature; MDBX_cursor *const bk = mc->backup; - ENSURE(mc->txn->env, stage == cur_signature_live || - (stage == cur_signature_wait4eot && bk)); + ENSURE(mc->txn->env, stage == cur_signature_live || (stage == cur_signature_wait4eot && bk)); if (bk) { subcur_t *mx = mc->subcur; cASSERT(mc, mc->txn->parent != nullptr); @@ -274,10 +261,8 @@ void cursor_eot(MDBX_cursor *mc, const bool merge) { /*----------------------------------------------------------------------------*/ -static __always_inline int couple_init(cursor_couple_t *couple, - const MDBX_txn *const txn, - tree_t *const tree, kvx_t *const kvx, - uint8_t *const dbi_state) { +static __always_inline int couple_init(cursor_couple_t *couple, const MDBX_txn *const txn, tree_t *const tree, + kvx_t *const kvx, uint8_t *const dbi_state) { VALGRIND_MAKE_MEM_UNDEFINED(couple, sizeof(cursor_couple_t)); tASSERT(txn, F_ISSET(*dbi_state, DBI_VALID | DBI_LINDO)); @@ -290,12 +275,9 @@ static __always_inline int couple_init(cursor_couple_t *couple, couple->outer.clc = &kvx->clc; couple->outer.dbi_state = dbi_state; couple->outer.top_and_flags = z_fresh_mark; - STATIC_ASSERT((int)z_branch == P_BRANCH && (int)z_leaf == P_LEAF && - (int)z_largepage == P_LARGE && (int)z_dupfix == P_DUPFIX); - couple->outer.checking = - (AUDIT_ENABLED() || (txn->env->flags & MDBX_VALIDATION)) - ? z_pagecheck | z_leaf - : z_leaf; + STATIC_ASSERT((int)z_branch == P_BRANCH && (int)z_leaf == P_LEAF && (int)z_largepage == P_LARGE && + (int)z_dupfix == P_DUPFIX); + couple->outer.checking = (AUDIT_ENABLED() || (txn->env->flags & MDBX_VALIDATION)) ? z_pagecheck | z_leaf : z_leaf; couple->outer.subcur = nullptr; if (tree->flags & MDBX_DUPSORT) { @@ -310,8 +292,7 @@ static __always_inline int couple_init(cursor_couple_t *couple, mx->cursor.dbi_state = dbi_state; mx->cursor.top_and_flags = z_fresh_mark | z_inner; STATIC_ASSERT(MDBX_DUPFIXED * 2 == P_DUPFIX); - mx->cursor.checking = - couple->outer.checking + ((tree->flags & MDBX_DUPFIXED) << 1); + mx->cursor.checking = couple->outer.checking + ((tree->flags & MDBX_DUPFIXED) << 1); } if (unlikely(*dbi_state & DBI_STALE)) @@ -323,8 +304,7 @@ static __always_inline int couple_init(cursor_couple_t *couple, return MDBX_SUCCESS; } -__cold int cursor_init4walk(cursor_couple_t *couple, const MDBX_txn *const txn, - tree_t *const tree, kvx_t *const kvx) { +__cold int cursor_init4walk(cursor_couple_t *couple, const MDBX_txn *const txn, tree_t *const tree, kvx_t *const kvx) { return couple_init(couple, txn, tree, kvx, txn->dbi_state); } @@ -332,21 +312,19 @@ int cursor_init(MDBX_cursor *mc, const MDBX_txn *txn, size_t dbi) { STATIC_ASSERT(offsetof(cursor_couple_t, outer) == 0); int rc = dbi_check(txn, dbi); if (likely(rc == MDBX_SUCCESS)) - rc = couple_init(container_of(mc, cursor_couple_t, outer), txn, - &txn->dbs[dbi], &txn->env->kvs[dbi], &txn->dbi_state[dbi]); + rc = couple_init(container_of(mc, cursor_couple_t, outer), txn, &txn->dbs[dbi], &txn->env->kvs[dbi], + &txn->dbi_state[dbi]); return rc; } __cold static int unexpected_dupsort(MDBX_cursor *mc) { - ERROR("unexpected dupsort-page/node for non-dupsort db/cursor (dbi %zu)", - cursor_dbi(mc)); + ERROR("unexpected dupsort-page/node for non-dupsort db/cursor (dbi %zu)", cursor_dbi(mc)); mc->txn->flags |= MDBX_TXN_ERROR; be_poor(mc); return MDBX_CORRUPTED; } -int cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node, - const page_t *mp) { +int cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node, const page_t *mp) { cASSERT(mc, is_pointed(mc)); subcur_t *mx = mc->subcur; if (!MDBX_DISABLE_VALIDATION && unlikely(mx == nullptr)) @@ -359,16 +337,13 @@ int cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node, goto bailout; case N_DUP | N_TREE: if (!MDBX_DISABLE_VALIDATION && unlikely(node_ds(node) != sizeof(tree_t))) { - ERROR("invalid nested-db record size (%zu, expect %zu)", node_ds(node), - sizeof(tree_t)); + ERROR("invalid nested-db record size (%zu, expect %zu)", node_ds(node), sizeof(tree_t)); goto bailout; } memcpy(&mx->nested_tree, node_data(node), sizeof(tree_t)); const txnid_t pp_txnid = mp->txnid; - if (!MDBX_DISABLE_VALIDATION && - unlikely(mx->nested_tree.mod_txnid > pp_txnid)) { - ERROR("nested-db.mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")", - mx->nested_tree.mod_txnid, pp_txnid); + if (!MDBX_DISABLE_VALIDATION && unlikely(mx->nested_tree.mod_txnid > pp_txnid)) { + ERROR("nested-db.mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")", mx->nested_tree.mod_txnid, pp_txnid); goto bailout; } mx->cursor.top_and_flags = z_fresh_mark | z_inner; @@ -390,25 +365,21 @@ int cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node, mx->cursor.pg[0] = sp; mx->cursor.ki[0] = 0; mx->nested_tree.flags = flags_db2sub(mc->tree->flags); - mx->nested_tree.dupfix_size = - (mc->tree->flags & MDBX_DUPFIXED) ? sp->dupfix_ksize : 0; + mx->nested_tree.dupfix_size = (mc->tree->flags & MDBX_DUPFIXED) ? sp->dupfix_ksize : 0; break; } if (unlikely(mx->nested_tree.dupfix_size != mc->tree->dupfix_size)) { if (!MDBX_DISABLE_VALIDATION && unlikely(mc->tree->dupfix_size != 0)) { - ERROR("cursor mismatched nested-db dupfix_size %u", - mc->tree->dupfix_size); + ERROR("cursor mismatched nested-db dupfix_size %u", mc->tree->dupfix_size); goto bailout; } - if (!MDBX_DISABLE_VALIDATION && - unlikely((mc->tree->flags & MDBX_DUPFIXED) == 0)) { + if (!MDBX_DISABLE_VALIDATION && unlikely((mc->tree->flags & MDBX_DUPFIXED) == 0)) { ERROR("mismatched nested-db flags %u", mc->tree->flags); goto bailout; } if (!MDBX_DISABLE_VALIDATION && - unlikely(mx->nested_tree.dupfix_size < mc->clc->v.lmin || - mx->nested_tree.dupfix_size > mc->clc->v.lmax)) { + unlikely(mx->nested_tree.dupfix_size < mc->clc->v.lmin || mx->nested_tree.dupfix_size > mc->clc->v.lmax)) { ERROR("mismatched nested-db.dupfix_size (%u) <> min/max value-length " "(%zu/%zu)", mx->nested_tree.dupfix_size, mc->clc->v.lmin, mc->clc->v.lmax); @@ -418,8 +389,7 @@ int cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node, mc->clc->v.lmin = mc->clc->v.lmax = mx->nested_tree.dupfix_size; } - DEBUG("Sub-db dbi -%zu root page %" PRIaPGNO, cursor_dbi(&mx->cursor), - mx->nested_tree.root); + DEBUG("Sub-db dbi -%zu root page %" PRIaPGNO, cursor_dbi(&mx->cursor), mx->nested_tree.root); return MDBX_SUCCESS; bailout: @@ -450,14 +420,11 @@ static __always_inline int sibling(MDBX_cursor *mc, bool right) { } cursor_pop(mc); - DEBUG("parent page is page %" PRIaPGNO ", index %u", mc->pg[mc->top]->pgno, - mc->ki[mc->top]); + DEBUG("parent page is page %" PRIaPGNO ", index %u", mc->pg[mc->top]->pgno, mc->ki[mc->top]); int err; - if (right ? (mc->ki[mc->top] + (size_t)1 >= page_numkeys(mc->pg[mc->top])) - : (mc->ki[mc->top] == 0)) { - DEBUG("no more keys aside, moving to next %s sibling", - right ? "right" : "left"); + if (right ? (mc->ki[mc->top] + (size_t)1 >= page_numkeys(mc->pg[mc->top])) : (mc->ki[mc->top] == 0)) { + DEBUG("no more keys aside, moving to next %s sibling", right ? "right" : "left"); err = right ? cursor_sibling_right(mc) : cursor_sibling_left(mc); if (err != MDBX_SUCCESS) { if (likely(err == MDBX_NOTFOUND)) @@ -467,8 +434,7 @@ static __always_inline int sibling(MDBX_cursor *mc, bool right) { } } else { mc->ki[mc->top] += right ? 1 : -1; - DEBUG("just moving to %s index key %u", right ? "right" : "left", - mc->ki[mc->top]); + DEBUG("just moving to %s index key %u", right ? "right" : "left", mc->ki[mc->top]); } cASSERT(mc, is_branch(mc->pg[mc->top])); @@ -515,10 +481,8 @@ __hot int cursor_sibling_right(MDBX_cursor *mc) { /* Функция-шаблон: Приземляет курсор на данные в текущей позиции. * В том числе, загружает данные во вложенный курсор при его наличии. */ -static __always_inline int cursor_bring(const bool inner, const bool tend2first, - MDBX_cursor *__restrict mc, - MDBX_val *__restrict key, - MDBX_val *__restrict data, bool eof) { +static __always_inline int cursor_bring(const bool inner, const bool tend2first, MDBX_cursor *__restrict mc, + MDBX_val *__restrict key, MDBX_val *__restrict data, bool eof) { if (inner) { cASSERT(mc, !data && !mc->subcur && (mc->flags & z_inner) != 0); } else { @@ -527,8 +491,7 @@ static __always_inline int cursor_bring(const bool inner, const bool tend2first, const page_t *mp = mc->pg[mc->top]; if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->pgno, mp->flags); + ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags); return MDBX_CORRUPTED; } @@ -554,8 +517,7 @@ static __always_inline int cursor_bring(const bool inner, const bool tend2first, return err; MDBX_ANALYSIS_ASSUME(mc->subcur != nullptr); if (node_flags(node) & N_TREE) { - err = tend2first ? inner_first(&mc->subcur->cursor, data) - : inner_last(&mc->subcur->cursor, data); + err = tend2first ? inner_first(&mc->subcur->cursor, data) : inner_last(&mc->subcur->cursor, data); if (unlikely(err != MDBX_SUCCESS)) return err; } else { @@ -592,10 +554,8 @@ static __always_inline int cursor_bring(const bool inner, const bool tend2first, } /* Функция-шаблон: Устанавливает курсор в начало или конец. */ -static __always_inline int cursor_brim(const bool inner, const bool tend2first, - MDBX_cursor *__restrict mc, - MDBX_val *__restrict key, - MDBX_val *__restrict data) { +static __always_inline int cursor_brim(const bool inner, const bool tend2first, MDBX_cursor *__restrict mc, + MDBX_val *__restrict key, MDBX_val *__restrict data) { if (mc->top != 0) { int err = tree_search(mc, nullptr, tend2first ? Z_FIRST : Z_LAST); if (unlikely(err != MDBX_SUCCESS)) @@ -607,13 +567,9 @@ static __always_inline int cursor_brim(const bool inner, const bool tend2first, return cursor_bring(inner, tend2first, mc, key, data, !tend2first); } -__hot int inner_first(MDBX_cursor *mc, MDBX_val *data) { - return cursor_brim(true, true, mc, data, nullptr); -} +__hot int inner_first(MDBX_cursor *mc, MDBX_val *data) { return cursor_brim(true, true, mc, data, nullptr); } -__hot int inner_last(MDBX_cursor *mc, MDBX_val *data) { - return cursor_brim(true, false, mc, data, nullptr); -} +__hot int inner_last(MDBX_cursor *mc, MDBX_val *data) { return cursor_brim(true, false, mc, data, nullptr); } __hot int outer_first(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) { return cursor_brim(false, true, mc, key, data); @@ -627,23 +583,18 @@ __hot int outer_last(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) { /* Функция-шаблон: Передвигает курсор на одну позицию. * При необходимости управляет вложенным курсором. */ -static __always_inline int cursor_step(const bool inner, const bool forward, - MDBX_cursor *__restrict mc, - MDBX_val *__restrict key, - MDBX_val *__restrict data, - MDBX_cursor_op op) { +static __always_inline int cursor_step(const bool inner, const bool forward, MDBX_cursor *__restrict mc, + MDBX_val *__restrict key, MDBX_val *__restrict data, MDBX_cursor_op op) { if (forward) { if (inner) cASSERT(mc, op == MDBX_NEXT); else - cASSERT(mc, - op == MDBX_NEXT || op == MDBX_NEXT_DUP || op == MDBX_NEXT_NODUP); + cASSERT(mc, op == MDBX_NEXT || op == MDBX_NEXT_DUP || op == MDBX_NEXT_NODUP); } else { if (inner) cASSERT(mc, op == MDBX_PREV); else - cASSERT(mc, - op == MDBX_PREV || op == MDBX_PREV_DUP || op == MDBX_PREV_NODUP); + cASSERT(mc, op == MDBX_PREV || op == MDBX_PREV_DUP || op == MDBX_PREV_NODUP); } if (inner) { cASSERT(mc, !data && !mc->subcur && (mc->flags & z_inner) != 0); @@ -668,15 +619,13 @@ static __always_inline int cursor_step(const bool inner, const bool forward, cASSERT(mc, nkeys > 0); intptr_t ki = mc->ki[mc->top]; - const uint8_t state = - mc->flags & (z_after_delete | z_hollow | z_eof_hard | z_eof_soft); + const uint8_t state = mc->flags & (z_after_delete | z_hollow | z_eof_hard | z_eof_soft); if (likely(state == 0)) { cASSERT(mc, ki < nkeys); if (!inner && op != (forward ? MDBX_NEXT_NODUP : MDBX_PREV_NODUP)) { int err = MDBX_NOTFOUND; if (inner_pointed(mc)) { - err = forward ? inner_next(&mc->subcur->cursor, data) - : inner_prev(&mc->subcur->cursor, data); + err = forward ? inner_next(&mc->subcur->cursor, data) : inner_prev(&mc->subcur->cursor, data); if (likely(err == MDBX_SUCCESS)) { get_key_optional(page_node(mp, ki), key); return MDBX_SUCCESS; @@ -715,9 +664,8 @@ static __always_inline int cursor_step(const bool inner, const bool forward, } } - DEBUG("turn-%s: top page was %" PRIaPGNO " in cursor %p, ki %zi of %zi", - forward ? "next" : "prev", mp->pgno, __Wpedantic_format_voidptr(mc), ki, - nkeys); + DEBUG("turn-%s: top page was %" PRIaPGNO " in cursor %p, ki %zi of %zi", forward ? "next" : "prev", mp->pgno, + __Wpedantic_format_voidptr(mc), ki, nkeys); if (forward) { if (likely(++ki < nkeys)) mc->ki[mc->top] = (indx_t)ki; @@ -727,8 +675,7 @@ static __always_inline int cursor_step(const bool inner, const bool forward, if (unlikely(err != MDBX_SUCCESS)) return err; mp = mc->pg[mc->top]; - DEBUG("next page is %" PRIaPGNO ", key index %u", mp->pgno, - mc->ki[mc->top]); + DEBUG("next page is %" PRIaPGNO ", key index %u", mp->pgno, mc->ki[mc->top]); } } else { if (likely(--ki >= 0)) @@ -739,47 +686,38 @@ static __always_inline int cursor_step(const bool inner, const bool forward, if (unlikely(err != MDBX_SUCCESS)) return err; mp = mc->pg[mc->top]; - DEBUG("prev page is %" PRIaPGNO ", key index %u", mp->pgno, - mc->ki[mc->top]); + DEBUG("prev page is %" PRIaPGNO ", key index %u", mp->pgno, mc->ki[mc->top]); } } - DEBUG("==> cursor points to page %" PRIaPGNO " with %zu keys, key index %u", - mp->pgno, page_numkeys(mp), mc->ki[mc->top]); + DEBUG("==> cursor points to page %" PRIaPGNO " with %zu keys, key index %u", mp->pgno, page_numkeys(mp), + mc->ki[mc->top]); bring: return cursor_bring(inner, forward, mc, key, data, false); } -__hot int inner_next(MDBX_cursor *mc, MDBX_val *data) { - return cursor_step(true, true, mc, data, nullptr, MDBX_NEXT); -} +__hot int inner_next(MDBX_cursor *mc, MDBX_val *data) { return cursor_step(true, true, mc, data, nullptr, MDBX_NEXT); } -__hot int inner_prev(MDBX_cursor *mc, MDBX_val *data) { - return cursor_step(true, false, mc, data, nullptr, MDBX_PREV); -} +__hot int inner_prev(MDBX_cursor *mc, MDBX_val *data) { return cursor_step(true, false, mc, data, nullptr, MDBX_PREV); } -__hot int outer_next(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, - MDBX_cursor_op op) { +__hot int outer_next(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) { return cursor_step(false, true, mc, key, data, op); } -__hot int outer_prev(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, - MDBX_cursor_op op) { +__hot int outer_prev(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) { return cursor_step(false, false, mc, key, data, op); } /*----------------------------------------------------------------------------*/ -__hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, - unsigned flags) { +__hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags) { int err; DKBUF_DEBUG; MDBX_env *const env = mc->txn->env; if (LOG_ENABLED(MDBX_LOG_DEBUG) && (flags & MDBX_RESERVE)) data->iov_base = nullptr; - DEBUG("==> put db %d key [%s], size %" PRIuPTR ", data [%s] size %" PRIuPTR, - cursor_dbi_dbg(mc), DKEY_DEBUG(key), key->iov_len, DVAL_DEBUG(data), - data->iov_len); + DEBUG("==> put db %d key [%s], size %" PRIuPTR ", data [%s] size %" PRIuPTR, cursor_dbi_dbg(mc), DKEY_DEBUG(key), + key->iov_len, DVAL_DEBUG(data), data->iov_len); if ((flags & MDBX_CURRENT) != 0 && (mc->flags & z_inner) == 0) { if (unlikely(flags & (MDBX_APPEND | MDBX_NOOVERWRITE))) @@ -805,8 +743,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, /* Если за ключом более одного значения, либо если размер данных * отличается, то вместо обновления требуется удаление и * последующая вставка. */ - if (mc->subcur->nested_tree.items > 1 || - current_data.iov_len != data->iov_len) { + if (mc->subcur->nested_tree.items > 1 || current_data.iov_len != data->iov_len) { drop_current: err = cursor_del(mc, flags & MDBX_ALLDUPS); if (unlikely(err != MDBX_SUCCESS)) @@ -826,8 +763,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, goto skip_check_samedata; } } - if (!(flags & MDBX_RESERVE) && - unlikely(cmp_lenfast(¤t_data, data) == 0)) + if (!(flags & MDBX_RESERVE) && unlikely(cmp_lenfast(¤t_data, data) == 0)) return MDBX_SUCCESS /* the same data, nothing to update */; skip_check_samedata:; } @@ -843,8 +779,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, if ((flags & MDBX_APPEND) && mc->tree->items > 0) { old_data.iov_base = nullptr; old_data.iov_len = 0; - rc = (mc->flags & z_inner) ? inner_last(mc, &last_key) - : outer_last(mc, &last_key, &old_data); + rc = (mc->flags & z_inner) ? inner_last(mc, &last_key) : outer_last(mc, &last_key, &old_data); if (likely(rc == MDBX_SUCCESS)) { const int cmp = mc->clc->k.cmp(key, &last_key); if (likely(cmp > 0)) { @@ -875,11 +810,10 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, if (unlikely(mc->flags & z_inner)) { /* nested subtree of DUPSORT-database with the same key, * nothing to update */ - eASSERT(env, - data->iov_len == 0 && (old_data.iov_len == 0 || - /* olddata may not be updated in case - DUPFIX-page of dupfix-table */ - (mc->tree->flags & MDBX_DUPFIXED))); + eASSERT(env, data->iov_len == 0 && (old_data.iov_len == 0 || + /* olddata may not be updated in case + DUPFIX-page of dupfix-table */ + (mc->tree->flags & MDBX_DUPFIXED))); return MDBX_SUCCESS; } if (unlikely(flags & MDBX_ALLDUPS) && inner_pointed(mc)) { @@ -936,13 +870,11 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, mc->tree->root = npr.page->pgno; mc->tree->height++; if (mc->tree->flags & MDBX_INTEGERKEY) { - assert(key->iov_len >= mc->clc->k.lmin && - key->iov_len <= mc->clc->k.lmax); + assert(key->iov_len >= mc->clc->k.lmin && key->iov_len <= mc->clc->k.lmax); mc->clc->k.lmin = mc->clc->k.lmax = key->iov_len; } if (mc->tree->flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) { - assert(data->iov_len >= mc->clc->v.lmin && - data->iov_len <= mc->clc->v.lmax); + assert(data->iov_len >= mc->clc->v.lmin && data->iov_len <= mc->clc->v.lmax); assert(mc->subcur != nullptr); mc->tree->dupfix_size = /* mc->subcur->nested_tree.dupfix_size = */ (unsigned)(mc->clc->v.lmin = mc->clc->v.lmax = data->iov_len); @@ -966,12 +898,10 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, if (insert_key) { /* The key does not exist */ DEBUG("inserting key at index %i", mc->ki[mc->top]); - if ((mc->tree->flags & MDBX_DUPSORT) && - node_size(key, data) > env->leaf_nodemax) { + if ((mc->tree->flags & MDBX_DUPSORT) && node_size(key, data) > env->leaf_nodemax) { /* Too big for a node, insert in sub-DB. Set up an empty * "old sub-page" for convert_to_subtree to expand to a full page. */ - fp->dupfix_ksize = - (mc->tree->flags & MDBX_DUPFIXED) ? (uint16_t)data->iov_len : 0; + fp->dupfix_ksize = (mc->tree->flags & MDBX_DUPFIXED) ? (uint16_t)data->iov_len : 0; fp->lower = fp->upper = 0; old_data.iov_len = PAGEHDRSZ; goto convert_to_subtree; @@ -1022,9 +952,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, /* Large/Overflow page overwrites need special handling */ if (unlikely(node_flags(node) & N_BIG)) { - const size_t dpages = (node_size(key, data) > env->leaf_nodemax) - ? largechunk_npages(env, data->iov_len) - : 0; + const size_t dpages = (node_size(key, data) > env->leaf_nodemax) ? largechunk_npages(env, data->iov_len) : 0; const pgno_t pgno = node_largedata_pgno(node); pgr_t lp = page_get_large(mc, pgno, mc->pg[mc->top]->txnid); @@ -1035,11 +963,8 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, /* Is the ov page from this txn (or a parent) and big enough? */ const size_t ovpages = lp.page->pages; const size_t extra_threshold = - (mc->tree == &mc->txn->dbs[FREE_DBI]) - ? 1 - : /* LY: add configurable threshold to keep reserve space */ 0; - if (!is_frozen(mc->txn, lp.page) && ovpages >= dpages && - ovpages <= dpages + extra_threshold) { + (mc->tree == &mc->txn->dbs[FREE_DBI]) ? 1 : /* LY: add configurable threshold to keep reserve space */ 0; + if (!is_frozen(mc->txn, lp.page) && ovpages >= dpages && ovpages <= dpages + extra_threshold) { /* yes, overwrite it. */ if (!is_modifable(mc->txn, lp.page)) { if (is_spilled(mc->txn, lp.page)) { @@ -1052,10 +977,8 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, if (unlikely(!mc->txn->parent)) { ERROR("Unexpected not frozen/modifiable/spilled but shadowed %s " "page %" PRIaPGNO " mod-txnid %" PRIaTXN "," - " without parent transaction, current txn %" PRIaTXN - " front %" PRIaTXN, - "large/overflow", pgno, lp.page->txnid, mc->txn->txnid, - mc->txn->front_txnid); + " without parent transaction, current txn %" PRIaTXN " front %" PRIaTXN, + "large/overflow", pgno, lp.page->txnid, mc->txn->txnid, mc->txn->front_txnid); return MDBX_PROBLEM; } @@ -1094,8 +1017,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, } else { old_data.iov_len = node_ds(node); old_data.iov_base = node_data(node); - cASSERT(mc, ptr_disp(old_data.iov_base, old_data.iov_len) <= - ptr_disp(mc->pg[mc->top], env->ps)); + cASSERT(mc, ptr_disp(old_data.iov_base, old_data.iov_len) <= ptr_disp(mc->pg[mc->top], env->ps)); /* DB has dups? */ if (mc->tree->flags & MDBX_DUPSORT) { @@ -1133,8 +1055,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, } /* Back up original data item */ - memcpy(old_singledup.iov_base = fp + 1, old_data.iov_base, - old_singledup.iov_len = old_data.iov_len); + memcpy(old_singledup.iov_base = fp + 1, old_data.iov_base, old_singledup.iov_len = old_data.iov_len); /* Make sub-page header for the dup items, with dummy body */ fp->flags = P_LEAF | P_SUBP; @@ -1149,13 +1070,11 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, * не сразу расширять созданную под-страницу. * Резервирование в целом сомнительно (см ниже), но может сработать * в плюс (а если в минус то несущественный) при коротких ключах. */ - xdata.iov_len += page_subleaf2_reserve( - env, page_room(mc->pg[mc->top]) + old_data.iov_len, - xdata.iov_len, data->iov_len); + xdata.iov_len += + page_subleaf2_reserve(env, page_room(mc->pg[mc->top]) + old_data.iov_len, xdata.iov_len, data->iov_len); cASSERT(mc, (xdata.iov_len & 1) == 0); } else { - xdata.iov_len += 2 * (sizeof(indx_t) + NODESIZE) + - (old_data.iov_len & 1) + (data->iov_len & 1); + xdata.iov_len += 2 * (sizeof(indx_t) + NODESIZE) + (old_data.iov_len & 1) + (data->iov_len & 1); } cASSERT(mc, (xdata.iov_len & 1) == 0); fp->upper = (uint16_t)(xdata.iov_len - PAGEHDRSZ); @@ -1169,9 +1088,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, fp = old_data.iov_base; switch (flags) { default: - growth = is_dupfix_leaf(fp) - ? fp->dupfix_ksize - : (node_size(data, nullptr) + sizeof(indx_t)); + growth = is_dupfix_leaf(fp) ? fp->dupfix_ksize : (node_size(data, nullptr) + sizeof(indx_t)); if (page_room(fp) >= growth) { /* На текущей под-странице есть место для добавления элемента. * Оптимальнее продолжить использовать эту страницу, ибо @@ -1241,9 +1158,8 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, * subpage_reserve_prereq = leaf_nodemax (1000‰). */ if (is_dupfix_leaf(fp)) - growth += page_subleaf2_reserve( - env, page_room(mc->pg[mc->top]) + old_data.iov_len, - xdata.iov_len, data->iov_len); + growth += page_subleaf2_reserve(env, page_room(mc->pg[mc->top]) + old_data.iov_len, xdata.iov_len, + data->iov_len); else { /* TODO: Если добавить возможность для пользователя задавать * min/max размеров ключей/данных, то здесь разумно реализовать @@ -1265,13 +1181,10 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, } fp_flags = fp->flags; - if (xdata.iov_len > env->subpage_limit || - node_size_len(node_ks(node), xdata.iov_len) > env->leaf_nodemax || + if (xdata.iov_len > env->subpage_limit || node_size_len(node_ks(node), xdata.iov_len) > env->leaf_nodemax || (env->subpage_room_threshold && - page_room(mc->pg[mc->top]) + - node_size_len(node_ks(node), old_data.iov_len) < - env->subpage_room_threshold + - node_size_len(node_ks(node), xdata.iov_len))) { + page_room(mc->pg[mc->top]) + node_size_len(node_ks(node), old_data.iov_len) < + env->subpage_room_threshold + node_size_len(node_ks(node), xdata.iov_len))) { /* Too big for a sub-page, convert to sub-DB */ convert_to_subtree: fp_flags &= ~P_SUBP; @@ -1310,17 +1223,13 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, cASSERT(mc, fp->upper + growth < UINT16_MAX); mp->upper = fp->upper + (indx_t)growth; if (unlikely(fp_flags & P_DUPFIX)) { - memcpy(page_data(mp), page_data(fp), - page_numkeys(fp) * fp->dupfix_ksize); - cASSERT(mc, (((mp->dupfix_ksize & page_numkeys(mp)) ^ mp->upper) & - 1) == 0); + memcpy(page_data(mp), page_data(fp), page_numkeys(fp) * fp->dupfix_ksize); + cASSERT(mc, (((mp->dupfix_ksize & page_numkeys(mp)) ^ mp->upper) & 1) == 0); } else { cASSERT(mc, (mp->upper & 1) == 0); - memcpy(ptr_disp(mp, mp->upper + PAGEHDRSZ), - ptr_disp(fp, fp->upper + PAGEHDRSZ), + memcpy(ptr_disp(mp, mp->upper + PAGEHDRSZ), ptr_disp(fp, fp->upper + PAGEHDRSZ), old_data.iov_len - fp->upper - PAGEHDRSZ); - memcpy(mp->entries, fp->entries, - page_numkeys(fp) * sizeof(mp->entries[0])); + memcpy(mp->entries, fp->entries, page_numkeys(fp) * sizeof(mp->entries[0])); for (size_t i = 0; i < page_numkeys(fp); i++) { cASSERT(mc, mp->entries[i] + growth <= UINT16_MAX); mp->entries[i] += (indx_t)growth; @@ -1357,8 +1266,7 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, cASSERT(mc, key->iov_len < UINT16_MAX); node_set_ks(node, key->iov_len); memcpy(node_key(node), key->iov_base, key->iov_len); - cASSERT(mc, ptr_disp(node_key(node), node_ds(node)) < - ptr_disp(mc->pg[mc->top], env->ps)); + cASSERT(mc, ptr_disp(node_key(node), node_ds(node)) < ptr_disp(mc->pg[mc->top], env->ps)); goto fix_parent; } @@ -1377,12 +1285,9 @@ __hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, insert_node:; const unsigned naf = flags & NODE_ADD_FLAGS; - size_t nsize = is_dupfix_leaf(mc->pg[mc->top]) - ? key->iov_len - : leaf_size(env, key, ref_data); + size_t nsize = is_dupfix_leaf(mc->pg[mc->top]) ? key->iov_len : leaf_size(env, key, ref_data); if (page_room(mc->pg[mc->top]) < nsize) { - rc = page_split(mc, key, ref_data, P_INVALID, - insert_key ? naf : naf | MDBX_SPLIT_REPLACE); + rc = page_split(mc, key, ref_data, P_INVALID, insert_key ? naf : naf | MDBX_SPLIT_REPLACE); if (rc == MDBX_SUCCESS && AUDIT_ENABLED()) rc = insert_key ? cursor_check(mc) : cursor_check_updating(mc); } else { @@ -1420,12 +1325,8 @@ insert_node:; empty.iov_base = nullptr; node_t *node = page_node(mc->pg[mc->top], mc->ki[mc->top]); #define SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE 1 - STATIC_ASSERT( - (MDBX_NODUPDATA >> SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE) == - MDBX_NOOVERWRITE); - unsigned inner_flags = - MDBX_CURRENT | ((flags & MDBX_NODUPDATA) >> - SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE); + STATIC_ASSERT((MDBX_NODUPDATA >> SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE) == MDBX_NOOVERWRITE); + unsigned inner_flags = MDBX_CURRENT | ((flags & MDBX_NODUPDATA) >> SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE); if ((flags & MDBX_CURRENT) == 0) { inner_flags -= MDBX_CURRENT; rc = cursor_dupsort_setup(mc, node, mc->pg[mc->top]); @@ -1434,8 +1335,7 @@ insert_node:; } subcur_t *const mx = mc->subcur; if (sub_root) { - cASSERT(mc, mx->nested_tree.height == 1 && - mx->nested_tree.root == sub_root->pgno); + cASSERT(mc, mx->nested_tree.height == 1 && mx->nested_tree.root == sub_root->pgno); mx->cursor.flags = z_inner; mx->cursor.top = 0; mx->cursor.pg[0] = sub_root; @@ -1470,9 +1370,7 @@ insert_node:; m2->subcur->cursor.top_and_flags = z_inner; m2->subcur->cursor.ki[0] = 0; } - DEBUG("Sub-dbi -%zu root page %" PRIaPGNO, - cursor_dbi(&m2->subcur->cursor), - m2->subcur->nested_tree.root); + DEBUG("Sub-dbi -%zu root page %" PRIaPGNO, cursor_dbi(&m2->subcur->cursor), m2->subcur->nested_tree.root); } else if (!insert_key && m2->ki[mc->top] < nkeys) cursor_inner_refresh(m2, mp, m2->ki[mc->top]); } @@ -1480,10 +1378,8 @@ insert_node:; cASSERT(mc, mc->subcur->nested_tree.items < PTRDIFF_MAX); const size_t probe = (size_t)mc->subcur->nested_tree.items; #define SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND 1 - STATIC_ASSERT((MDBX_APPENDDUP >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND) == - MDBX_APPEND); - inner_flags |= - (flags & MDBX_APPENDDUP) >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND; + STATIC_ASSERT((MDBX_APPENDDUP >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND) == MDBX_APPEND); + inner_flags |= (flags & MDBX_APPENDDUP) >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND; rc = cursor_put(&mc->subcur->cursor, data, &empty, inner_flags); if (flags & N_TREE) { void *db = node_data(node); @@ -1530,16 +1426,13 @@ insert_node:; return rc; } -__hot int cursor_put_checklen(MDBX_cursor *mc, const MDBX_val *key, - MDBX_val *data, unsigned flags) { +__hot int cursor_put_checklen(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags) { cASSERT(mc, (mc->flags & z_inner) == 0); - if (unlikely(key->iov_len > mc->clc->k.lmax || - key->iov_len < mc->clc->k.lmin)) { + if (unlikely(key->iov_len > mc->clc->k.lmax || key->iov_len < mc->clc->k.lmin)) { cASSERT(mc, !"Invalid key-size"); return MDBX_BAD_VALSIZE; } - if (unlikely(data->iov_len > mc->clc->v.lmax || - data->iov_len < mc->clc->v.lmin)) { + if (unlikely(data->iov_len > mc->clc->v.lmax || data->iov_len < mc->clc->v.lmin)) { cASSERT(mc, !"Invalid data-size"); return MDBX_BAD_VALSIZE; } @@ -1611,8 +1504,7 @@ __hot int cursor_del(MDBX_cursor *mc, unsigned flags) { page_t *mp = mc->pg[mc->top]; cASSERT(mc, is_modifable(mc->txn, mp)); if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->pgno, mp->flags); + ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags); return MDBX_CORRUPTED; } if (is_dupfix_leaf(mp)) @@ -1644,8 +1536,7 @@ __hot int cursor_del(MDBX_cursor *mc, unsigned flags) { node = node_shrink(mp, mc->ki[mc->top], node); mc->subcur->cursor.pg[0] = node_data(node); /* fix other sub-DB cursors pointed at sub-pages on this page */ - for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; - m2 = m2->next) { + for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) { if (!is_related(mc, m2) || m2->pg[mc->top] != mp) continue; const node_t *inner = node; @@ -1664,8 +1555,7 @@ __hot int cursor_del(MDBX_cursor *mc, unsigned flags) { } } mc->tree->items -= 1; - cASSERT(mc, mc->tree->items > 0 && mc->tree->height > 0 && - mc->tree->root != P_INVALID); + cASSERT(mc, mc->tree->items > 0 && mc->tree->height > 0 && mc->tree->root != P_INVALID); return rc; } /* otherwise fall thru and delete the sub-DB */ @@ -1725,9 +1615,7 @@ del_key: /* DB is totally empty now, just bail out. * Other cursors adjustments were already done * by rebalance and aren't needed here. */ - cASSERT(mc, mc->tree->items == 0 && - (mc->tree->root == P_INVALID || - (is_inner(mc) && !mc->tree->root)) && + cASSERT(mc, mc->tree->items == 0 && (mc->tree->root == P_INVALID || (is_inner(mc) && !mc->tree->root)) && mc->flags < 0); return MDBX_SUCCESS; } @@ -1736,9 +1624,7 @@ del_key: mp = mc->pg[mc->top]; cASSERT(mc, is_leaf(mc->pg[mc->top])); size_t nkeys = page_numkeys(mp); - cASSERT(mc, - (mc->tree->items > 0 && nkeys > 0) || - ((mc->flags & z_inner) && mc->tree->items == 0 && nkeys == 0)); + cASSERT(mc, (mc->tree->items > 0 && nkeys > 0) || ((mc->flags & z_inner) && mc->tree->items == 0 && nkeys == 0)); /* Adjust this and other cursors pointing to mp */ const intptr_t top = /* может быть сброшен в -1 */ mc->top; @@ -1777,8 +1663,7 @@ del_key: * нужно установить на первый дубликат. */ if (is_pointed(&m3->subcur->cursor)) { if ((node_flags(node) & N_TREE) == 0) { - cASSERT(m3, m3->subcur->cursor.top == 0 && - m3->subcur->nested_tree.height == 1); + cASSERT(m3, m3->subcur->cursor.top == 0 && m3->subcur->nested_tree.height == 1); m3->subcur->cursor.pg[0] = node_data(node); } } else { @@ -1808,14 +1693,12 @@ fail: /*----------------------------------------------------------------------------*/ -__hot csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, - MDBX_cursor_op op) { +__hot csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) { DKBUF_DEBUG; csr_t ret; ret.exact = false; - if (unlikely(key->iov_len < mc->clc->k.lmin || - key->iov_len > mc->clc->k.lmax)) { + if (unlikely(key->iov_len < mc->clc->k.lmin || key->iov_len > mc->clc->k.lmax)) { cASSERT(mc, !"Invalid key-size"); ret.err = MDBX_BAD_VALSIZE; return ret; @@ -1848,9 +1731,8 @@ __hot csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, const size_t nkeys = page_numkeys(mp); if (unlikely(nkeys == 0)) { /* при создании первой листовой страницы */ - cASSERT(mc, mc->top == 0 && mc->tree->height == 1 && - mc->tree->branch_pages == 0 && - mc->tree->leaf_pages == 1 && mc->ki[0] == 0); + cASSERT(mc, mc->top == 0 && mc->tree->height == 1 && mc->tree->branch_pages == 0 && mc->tree->leaf_pages == 1 && + mc->ki[0] == 0); /* Логически верно, но нет смысла, ибо это мимолетная/временная * ситуация до добавления элемента выше по стеку вызовов: mc->flags |= z_eof_soft | z_hollow; */ @@ -1901,8 +1783,7 @@ __hot csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, * первой/последний и соответственно такое сравнение было выше. */ if (mc->ki[mc->top] > 0 && mc->ki[mc->top] < nkeys - 1) { if (is_dupfix_leaf(mp)) { - nodekey.iov_base = - page_dupfix_ptr(mp, mc->ki[mc->top], nodekey.iov_len); + nodekey.iov_base = page_dupfix_ptr(mp, mc->ki[mc->top], nodekey.iov_len); } else { node = page_node(mp, mc->ki[mc->top]); nodekey = get_key(node); @@ -1928,8 +1809,7 @@ __hot csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, mc->ki[mc->top] = (indx_t)nkeys; if (op < MDBX_SET_RANGE) { target_not_found: - cASSERT(mc, op == MDBX_SET || op == MDBX_SET_KEY || - op == MDBX_GET_BOTH || op == MDBX_GET_BOTH_RANGE); + cASSERT(mc, op == MDBX_SET || op == MDBX_SET_KEY || op == MDBX_GET_BOTH || op == MDBX_GET_BOTH_RANGE); /* Операция предполагает поиск конкретного ключа, который не найден. * Поэтому переводим курсор в неустановленное состояние, но без сброса * top, что позволяет работать fastpath при последующем поиске по дереву @@ -1991,8 +1871,7 @@ got_node: cASSERT(mc, is_pointed(mc) && !inner_pointed(mc)); cASSERT(mc, mc->ki[mc->top] < page_numkeys(mc->pg[mc->top])); if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->pgno, mp->flags); + ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags); ret.err = MDBX_CORRUPTED; return ret; } @@ -2037,8 +1916,7 @@ got_node: } } else if (likely(data)) { if (op <= MDBX_GET_BOTH_RANGE) { - if (unlikely(data->iov_len < mc->clc->v.lmin || - data->iov_len > mc->clc->v.lmax)) { + if (unlikely(data->iov_len < mc->clc->v.lmin || data->iov_len > mc->clc->v.lmax)) { cASSERT(mc, !"Invalid data-size"); ret.err = MDBX_BAD_VALSIZE; return ret; @@ -2049,13 +1927,11 @@ got_node: if (aligned_data.iov_len == 8) { if (unlikely(7 & (uintptr_t)aligned_data.iov_base)) /* copy instead of return error to avoid break compatibility */ - aligned_data.iov_base = - bcopy_8(&aligned_databytes, aligned_data.iov_base); + aligned_data.iov_base = bcopy_8(&aligned_databytes, aligned_data.iov_base); } else if (aligned_data.iov_len == 4) { if (unlikely(3 & (uintptr_t)aligned_data.iov_base)) /* copy instead of return error to avoid break compatibility */ - aligned_data.iov_base = - bcopy_4(&aligned_databytes, aligned_data.iov_base); + aligned_data.iov_base = bcopy_4(&aligned_databytes, aligned_data.iov_base); } else { cASSERT(mc, !"data-size is invalid for MDBX_INTEGERDUP"); ret.err = MDBX_BAD_VALSIZE; @@ -2089,18 +1965,15 @@ got_node: if (op >= MDBX_SET_KEY) get_key_optional(node, key); - DEBUG("==> cursor placed on key [%s], data [%s]", DKEY_DEBUG(key), - DVAL_DEBUG(data)); + DEBUG("==> cursor placed on key [%s], data [%s]", DKEY_DEBUG(key), DVAL_DEBUG(data)); ret.err = MDBX_SUCCESS; be_filled(mc); return ret; } -__hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, - const MDBX_cursor_op op) { +__hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, const MDBX_cursor_op op) { if (op != MDBX_GET_CURRENT) - DEBUG(">> cursor %p(0x%x), ops %u, key %p, value %p", - __Wpedantic_format_voidptr(mc), mc->flags, op, + DEBUG(">> cursor %p(0x%x), ops %u, key %p, value %p", __Wpedantic_format_voidptr(mc), mc->flags, op, __Wpedantic_format_voidptr(key), __Wpedantic_format_voidptr(data)); int rc; @@ -2163,8 +2036,7 @@ __hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, cASSERT(mc, is_filled(mc)); else if (rc == MDBX_NOTFOUND && mc->tree->items) { cASSERT(mc, is_pointed(mc)); - cASSERT(mc, op == MDBX_SET_RANGE || op == MDBX_GET_BOTH_RANGE || - is_hollow(mc)); + cASSERT(mc, op == MDBX_SET_RANGE || op == MDBX_GET_BOTH_RANGE || is_hollow(mc)); cASSERT(mc, op == MDBX_GET_BOTH_RANGE || inner_hollow(mc)); } else cASSERT(mc, is_poor(mc) && !is_filled(mc)); @@ -2271,8 +2143,7 @@ __hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, if ((node_flags(node) & N_DUP) == 0) return node_read(mc, node, data, mc->pg[mc->top]); else if (MDBX_DISABLE_VALIDATION || likely(mc->subcur)) - return ((op == MDBX_FIRST_DUP) ? inner_first - : inner_last)(&mc->subcur->cursor, data); + return ((op == MDBX_FIRST_DUP) ? inner_first : inner_last)(&mc->subcur->cursor, data); else return unexpected_dupsort(mc); } @@ -2338,8 +2209,7 @@ __hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, rc = outer_prev(mc, key, data, MDBX_PREV_NODUP); else if (op == MDBX_TO_KEY_GREATER_THAN) rc = outer_next(mc, key, data, MDBX_NEXT_NODUP); - } else if (op < MDBX_TO_KEY_EQUAL && - (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS)) + } else if (op < MDBX_TO_KEY_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS)) rc = outer_prev(mc, key, data, MDBX_PREV_NODUP); else if (op == MDBX_TO_KEY_EQUAL && rc == MDBX_SUCCESS) rc = MDBX_NOTFOUND; @@ -2371,8 +2241,7 @@ __hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, rc = inner_prev(mx, data); else if (op == MDBX_TO_EXACT_KEY_VALUE_GREATER_THAN) rc = inner_next(mx, data); - } else if (op < MDBX_TO_EXACT_KEY_VALUE_EQUAL && - (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS)) + } else if (op < MDBX_TO_EXACT_KEY_VALUE_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS)) rc = inner_prev(mx, data); else if (op == MDBX_TO_EXACT_KEY_VALUE_EQUAL && rc == MDBX_SUCCESS) rc = MDBX_NOTFOUND; @@ -2425,8 +2294,7 @@ __hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, rc = outer_prev(mc, key, data, MDBX_PREV); else if (op == MDBX_TO_PAIR_GREATER_THAN) rc = outer_next(mc, key, data, MDBX_NEXT); - } else if (op < MDBX_TO_PAIR_EQUAL && - (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS)) + } else if (op < MDBX_TO_PAIR_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS)) rc = outer_prev(mc, key, data, MDBX_PREV); else if (op == MDBX_TO_PAIR_EQUAL && rc == MDBX_SUCCESS) rc = MDBX_NOTFOUND; @@ -2458,8 +2326,7 @@ __hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, break; } } - } else if (op < MDBX_TO_PAIR_EQUAL && - (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS)) + } else if (op < MDBX_TO_PAIR_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS)) rc = outer_prev(mc, key, data, MDBX_PREV_NODUP); else if (op == MDBX_TO_PAIR_EQUAL && rc == MDBX_SUCCESS) rc = MDBX_NOTFOUND; diff --git a/src/cursor.h b/src/cursor.h index 9ecad9d6..4ae45222 100644 --- a/src/cursor.h +++ b/src/cursor.h @@ -125,13 +125,11 @@ enum cursor_state { z_fresh_mark = z_poor_mark | z_fresh }; -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -is_inner(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_inner(const MDBX_cursor *mc) { return (mc->flags & z_inner) != 0; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -is_poor(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_poor(const MDBX_cursor *mc) { const bool r = mc->top < 0; cASSERT(mc, r == (mc->top_and_flags < 0)); if (r && mc->subcur) @@ -139,8 +137,7 @@ is_poor(const MDBX_cursor *mc) { return r; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -is_pointed(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_pointed(const MDBX_cursor *mc) { const bool r = mc->top >= 0; cASSERT(mc, r == (mc->top_and_flags >= 0)); if (!r && mc->subcur) @@ -148,49 +145,41 @@ is_pointed(const MDBX_cursor *mc) { return r; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -is_hollow(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_hollow(const MDBX_cursor *mc) { const bool r = mc->flags < 0; if (!r) { cASSERT(mc, mc->top >= 0); - cASSERT(mc, (mc->flags & z_eof_hard) || - mc->ki[mc->top] < page_numkeys(mc->pg[mc->top])); + cASSERT(mc, (mc->flags & z_eof_hard) || mc->ki[mc->top] < page_numkeys(mc->pg[mc->top])); } else if (mc->subcur) cASSERT(mc, is_poor(&mc->subcur->cursor)); return r; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -is_eof(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_eof(const MDBX_cursor *mc) { const bool r = z_eof_soft <= (uint8_t)mc->flags; return r; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -is_filled(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_filled(const MDBX_cursor *mc) { const bool r = z_eof_hard > (uint8_t)mc->flags; return r; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -inner_filled(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool inner_filled(const MDBX_cursor *mc) { return mc->subcur && is_filled(&mc->subcur->cursor); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -inner_pointed(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool inner_pointed(const MDBX_cursor *mc) { return mc->subcur && is_pointed(&mc->subcur->cursor); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -inner_hollow(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool inner_hollow(const MDBX_cursor *mc) { return !mc->subcur || is_hollow(&mc->subcur->cursor); } MDBX_MAYBE_UNUSED static inline void inner_gone(MDBX_cursor *mc) { if (mc->subcur) { - TRACE("reset inner cursor %p", - __Wpedantic_format_voidptr(&mc->subcur->cursor)); + TRACE("reset inner cursor %p", __Wpedantic_format_voidptr(&mc->subcur->cursor)); mc->subcur->nested_tree.root = 0; mc->subcur->cursor.top_and_flags = z_inner | z_poor_mark; } @@ -218,8 +207,7 @@ MDBX_MAYBE_UNUSED static inline void be_filled(MDBX_cursor *mc) { cASSERT(mc, inner == is_inner(mc)); } -MDBX_MAYBE_UNUSED static inline bool is_related(const MDBX_cursor *base, - const MDBX_cursor *scan) { +MDBX_MAYBE_UNUSED static inline bool is_related(const MDBX_cursor *base, const MDBX_cursor *scan) { cASSERT(base, base->top >= 0); return base->top <= scan->top && base != scan; } @@ -238,36 +226,30 @@ enum cursor_checking { MDBX_INTERNAL int __must_check_result cursor_check(const MDBX_cursor *mc); -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline size_t -cursor_dbi(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline size_t cursor_dbi(const MDBX_cursor *mc) { cASSERT(mc, mc->txn && mc->txn->signature == txn_signature); size_t dbi = mc->dbi_state - mc->txn->dbi_state; cASSERT(mc, dbi < mc->txn->env->n_dbi); return dbi; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -cursor_dbi_changed(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_dbi_changed(const MDBX_cursor *mc) { return dbi_changed(mc->txn, cursor_dbi(mc)); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t * -cursor_dbi_state(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t *cursor_dbi_state(const MDBX_cursor *mc) { return mc->dbi_state; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -cursor_is_gc(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_is_gc(const MDBX_cursor *mc) { return mc->dbi_state == mc->txn->dbi_state + FREE_DBI; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -cursor_is_main(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_is_main(const MDBX_cursor *mc) { return mc->dbi_state == mc->txn->dbi_state + MAIN_DBI; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -cursor_is_core(const MDBX_cursor *mc) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_is_core(const MDBX_cursor *mc) { return mc->dbi_state < mc->txn->dbi_state + CORE_DBS; } @@ -277,10 +259,8 @@ MDBX_MAYBE_UNUSED static inline int cursor_dbi_dbg(const MDBX_cursor *mc) { return (mc->flags & z_inner) ? -dbi : dbi; } -MDBX_MAYBE_UNUSED static inline int __must_check_result -cursor_push(MDBX_cursor *mc, page_t *mp, indx_t ki) { - TRACE("pushing page %" PRIaPGNO " on db %d cursor %p", mp->pgno, - cursor_dbi_dbg(mc), __Wpedantic_format_voidptr(mc)); +MDBX_MAYBE_UNUSED static inline int __must_check_result cursor_push(MDBX_cursor *mc, page_t *mp, indx_t ki) { + TRACE("pushing page %" PRIaPGNO " on db %d cursor %p", mp->pgno, cursor_dbi_dbg(mc), __Wpedantic_format_voidptr(mc)); if (unlikely(mc->top >= CURSOR_STACK_SIZE - 1)) { be_poor(mc); mc->txn->flags |= MDBX_TXN_ERROR; @@ -293,43 +273,32 @@ cursor_push(MDBX_cursor *mc, page_t *mp, indx_t ki) { } MDBX_MAYBE_UNUSED static inline void cursor_pop(MDBX_cursor *mc) { - TRACE("popped page %" PRIaPGNO " off db %d cursor %p", mc->pg[mc->top]->pgno, - cursor_dbi_dbg(mc), __Wpedantic_format_voidptr(mc)); + TRACE("popped page %" PRIaPGNO " off db %d cursor %p", mc->pg[mc->top]->pgno, cursor_dbi_dbg(mc), + __Wpedantic_format_voidptr(mc)); cASSERT(mc, mc->top >= 0); mc->top -= 1; } -MDBX_NOTHROW_PURE_FUNCTION static inline bool -check_leaf_type(const MDBX_cursor *mc, const page_t *mp) { - return (((page_type(mp) ^ mc->checking) & - (z_branch | z_leaf | z_largepage | z_dupfix)) == 0); +MDBX_NOTHROW_PURE_FUNCTION static inline bool check_leaf_type(const MDBX_cursor *mc, const page_t *mp) { + return (((page_type(mp) ^ mc->checking) & (z_branch | z_leaf | z_largepage | z_dupfix)) == 0); } MDBX_INTERNAL void cursor_eot(MDBX_cursor *mc, const bool merge); -MDBX_INTERNAL int cursor_shadow(MDBX_cursor *parent_cursor, - MDBX_txn *nested_txn, const size_t dbi); +MDBX_INTERNAL int cursor_shadow(MDBX_cursor *parent_cursor, MDBX_txn *nested_txn, const size_t dbi); -MDBX_INTERNAL MDBX_cursor *cursor_cpstk(const MDBX_cursor *csrc, - MDBX_cursor *cdst); +MDBX_INTERNAL MDBX_cursor *cursor_cpstk(const MDBX_cursor *csrc, MDBX_cursor *cdst); -MDBX_INTERNAL int __must_check_result cursor_ops(MDBX_cursor *mc, MDBX_val *key, - MDBX_val *data, +MDBX_INTERNAL int __must_check_result cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, const MDBX_cursor_op op); -MDBX_INTERNAL int __must_check_result cursor_put_checklen(MDBX_cursor *mc, - const MDBX_val *key, - MDBX_val *data, +MDBX_INTERNAL int __must_check_result cursor_put_checklen(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags); -MDBX_INTERNAL int __must_check_result cursor_put(MDBX_cursor *mc, - const MDBX_val *key, - MDBX_val *data, - unsigned flags); +MDBX_INTERNAL int __must_check_result cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags); MDBX_INTERNAL int __must_check_result cursor_check_updating(MDBX_cursor *mc); -MDBX_INTERNAL int __must_check_result cursor_del(MDBX_cursor *mc, - unsigned flags); +MDBX_INTERNAL int __must_check_result cursor_del(MDBX_cursor *mc, unsigned flags); MDBX_INTERNAL int __must_check_result cursor_sibling_left(MDBX_cursor *mc); MDBX_INTERNAL int __must_check_result cursor_sibling_right(MDBX_cursor *mc); @@ -339,56 +308,37 @@ typedef struct cursor_set_result { bool exact; } csr_t; -MDBX_INTERNAL csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, - MDBX_cursor_op op); +MDBX_INTERNAL csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op); -MDBX_INTERNAL int __must_check_result inner_first(MDBX_cursor *__restrict mc, +MDBX_INTERNAL int __must_check_result inner_first(MDBX_cursor *__restrict mc, MDBX_val *__restrict data); +MDBX_INTERNAL int __must_check_result inner_last(MDBX_cursor *__restrict mc, MDBX_val *__restrict data); +MDBX_INTERNAL int __must_check_result outer_first(MDBX_cursor *__restrict mc, MDBX_val *__restrict key, MDBX_val *__restrict data); -MDBX_INTERNAL int __must_check_result inner_last(MDBX_cursor *__restrict mc, - MDBX_val *__restrict data); -MDBX_INTERNAL int __must_check_result outer_first(MDBX_cursor *__restrict mc, - MDBX_val *__restrict key, - MDBX_val *__restrict data); -MDBX_INTERNAL int __must_check_result outer_last(MDBX_cursor *__restrict mc, - MDBX_val *__restrict key, +MDBX_INTERNAL int __must_check_result outer_last(MDBX_cursor *__restrict mc, MDBX_val *__restrict key, MDBX_val *__restrict data); -MDBX_INTERNAL int __must_check_result inner_next(MDBX_cursor *__restrict mc, - MDBX_val *__restrict data); -MDBX_INTERNAL int __must_check_result inner_prev(MDBX_cursor *__restrict mc, - MDBX_val *__restrict data); -MDBX_INTERNAL int __must_check_result outer_next(MDBX_cursor *__restrict mc, - MDBX_val *__restrict key, - MDBX_val *__restrict data, - MDBX_cursor_op op); -MDBX_INTERNAL int __must_check_result outer_prev(MDBX_cursor *__restrict mc, - MDBX_val *__restrict key, - MDBX_val *__restrict data, - MDBX_cursor_op op); +MDBX_INTERNAL int __must_check_result inner_next(MDBX_cursor *__restrict mc, MDBX_val *__restrict data); +MDBX_INTERNAL int __must_check_result inner_prev(MDBX_cursor *__restrict mc, MDBX_val *__restrict data); +MDBX_INTERNAL int __must_check_result outer_next(MDBX_cursor *__restrict mc, MDBX_val *__restrict key, + MDBX_val *__restrict data, MDBX_cursor_op op); +MDBX_INTERNAL int __must_check_result outer_prev(MDBX_cursor *__restrict mc, MDBX_val *__restrict key, + MDBX_val *__restrict data, MDBX_cursor_op op); -MDBX_INTERNAL int cursor_init4walk(cursor_couple_t *couple, - const MDBX_txn *const txn, - tree_t *const tree, kvx_t *const kvx); +MDBX_INTERNAL int cursor_init4walk(cursor_couple_t *couple, const MDBX_txn *const txn, tree_t *const tree, + kvx_t *const kvx); -MDBX_INTERNAL int __must_check_result cursor_init(MDBX_cursor *mc, - const MDBX_txn *txn, - size_t dbi); +MDBX_INTERNAL int __must_check_result cursor_init(MDBX_cursor *mc, const MDBX_txn *txn, size_t dbi); -MDBX_INTERNAL int __must_check_result cursor_dupsort_setup(MDBX_cursor *mc, - const node_t *node, - const page_t *mp); +MDBX_INTERNAL int __must_check_result cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node, const page_t *mp); -MDBX_INTERNAL int __must_check_result cursor_touch(MDBX_cursor *const mc, - const MDBX_val *key, - const MDBX_val *data); +MDBX_INTERNAL int __must_check_result cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, const MDBX_val *data); /*----------------------------------------------------------------------------*/ /* Update sub-page pointer, if any, in mc->subcur. * Needed when the node which contains the sub-page may have moved. * Called with mp = mc->pg[mc->top], ki = mc->ki[mc->top]. */ -MDBX_MAYBE_UNUSED static inline void -cursor_inner_refresh(const MDBX_cursor *mc, const page_t *mp, unsigned ki) { +MDBX_MAYBE_UNUSED static inline void cursor_inner_refresh(const MDBX_cursor *mc, const page_t *mp, unsigned ki) { cASSERT(mc, is_leaf(mp)); const node_t *node = page_node(mp, ki); if ((node_flags(node) & (N_DUP | N_TREE)) == N_DUP) diff --git a/src/dbi.c b/src/dbi.c index acadf0e5..60b32d8a 100644 --- a/src/dbi.c +++ b/src/dbi.c @@ -8,16 +8,14 @@ size_t dbi_bitmap_ctz_fallback(const MDBX_txn *txn, intptr_t bmi) { tASSERT(txn, bmi > 0); bmi &= -bmi; if (sizeof(txn->dbi_sparse[0]) > 4) { - static const uint8_t debruijn_ctz64[64] = { - 0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28, - 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11, - 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10, - 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12}; + static const uint8_t debruijn_ctz64[64] = {0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28, + 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11, + 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10, + 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12}; return debruijn_ctz64[(UINT64_C(0x022FDD63CC95386D) * (uint64_t)bmi) >> 58]; } else { - static const uint8_t debruijn_ctz32[32] = { - 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, - 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; + static const uint8_t debruijn_ctz32[32] = {0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; return debruijn_ctz32[(UINT32_C(0x077CB531) * (uint32_t)bmi) >> 27]; } } @@ -45,8 +43,7 @@ __noinline int dbi_import(MDBX_txn *txn, const size_t dbi) { const size_t bitmap_indx = dbi / bitmap_chunk; const size_t bitmap_mask = (size_t)1 << dbi % bitmap_chunk; if (dbi >= txn->n_dbi) { - for (size_t i = (txn->n_dbi + bitmap_chunk - 1) / bitmap_chunk; - bitmap_indx >= i; ++i) + for (size_t i = (txn->n_dbi + bitmap_chunk - 1) / bitmap_chunk; bitmap_indx >= i; ++i) txn->dbi_sparse[i] = 0; eASSERT(env, (txn->dbi_sparse[bitmap_indx] & bitmap_mask) == 0); MDBX_txn *scan = txn; @@ -92,8 +89,7 @@ __noinline int dbi_import(MDBX_txn *txn, const size_t dbi) { int rc = dbi_check(parent, dbi); /* копируем состояние table очищая new-флаги. */ eASSERT(env, txn->dbi_seqs == parent->dbi_seqs); - txn->dbi_state[dbi] = - parent->dbi_state[dbi] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY); + txn->dbi_state[dbi] = parent->dbi_state[dbi] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY); if (likely(rc == MDBX_SUCCESS)) { txn->dbs[dbi] = parent->dbs[dbi]; if (parent->cursors[dbi]) { @@ -111,8 +107,7 @@ __noinline int dbi_import(MDBX_txn *txn, const size_t dbi) { txn->dbi_state[dbi] = DBI_LINDO; } else { eASSERT(env, txn->dbi_seqs[dbi] != env->dbi_seqs[dbi].weak); - if (unlikely((txn->dbi_state[dbi] & (DBI_VALID | DBI_OLDEN)) || - txn->cursors[dbi])) { + if (unlikely((txn->dbi_state[dbi] & (DBI_VALID | DBI_OLDEN)) || txn->cursors[dbi])) { /* хендл уже использовался в транзакции, но был закрыт или переоткрыт, * либо при явном пере-открытии хендла есть висячие курсоры */ eASSERT(env, (txn->dbi_state[dbi] & DBI_STALE) == 0); @@ -137,8 +132,7 @@ __noinline int dbi_import(MDBX_txn *txn, const size_t dbi) { return MDBX_BAD_DBI; } -static int defer_and_release(MDBX_env *const env, - defer_free_item_t *const chain) { +static int defer_and_release(MDBX_env *const env, defer_free_item_t *const chain) { size_t length = 0; defer_free_item_t *obsolete_chain = nullptr; #if MDBX_ENABLE_DBI_LOCKFREE @@ -232,8 +226,7 @@ int dbi_update(MDBX_txn *txn, int keep) { while ((env->dbs_flags[i - 1] & DB_VALID) == 0) { --i; eASSERT(env, i >= CORE_DBS); - eASSERT(env, !env->dbs_flags[i] && !env->kvs[i].name.iov_len && - !env->kvs[i].name.iov_base); + eASSERT(env, !env->dbs_flags[i] && !env->kvs[i].name.iov_len && !env->kvs[i].name.iov_base); } env->n_dbi = (unsigned)i; defer_and_release(env, defer_chain); @@ -241,21 +234,17 @@ int dbi_update(MDBX_txn *txn, int keep) { return MDBX_SUCCESS; } -int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, - MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) { +int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) { const MDBX_env *const env = txn->env; eASSERT(env, dbi < txn->n_dbi && dbi < env->n_dbi); eASSERT(env, dbi_state(txn, dbi) & DBI_LINDO); eASSERT(env, env->dbs_flags[dbi] != DB_POISON); if ((env->dbs_flags[dbi] & DB_VALID) == 0) { - eASSERT(env, !env->kvs[dbi].clc.k.cmp && !env->kvs[dbi].clc.v.cmp && - !env->kvs[dbi].name.iov_len && - !env->kvs[dbi].name.iov_base && - !env->kvs[dbi].clc.k.lmax && !env->kvs[dbi].clc.k.lmin && + eASSERT(env, !env->kvs[dbi].clc.k.cmp && !env->kvs[dbi].clc.v.cmp && !env->kvs[dbi].name.iov_len && + !env->kvs[dbi].name.iov_base && !env->kvs[dbi].clc.k.lmax && !env->kvs[dbi].clc.k.lmin && !env->kvs[dbi].clc.v.lmax && !env->kvs[dbi].clc.v.lmin); } else { - eASSERT(env, !(txn->dbi_state[dbi] & DBI_VALID) || - (txn->dbs[dbi].flags | DB_VALID) == env->dbs_flags[dbi]); + eASSERT(env, !(txn->dbi_state[dbi] & DBI_VALID) || (txn->dbs[dbi].flags | DB_VALID) == env->dbs_flags[dbi]); eASSERT(env, env->kvs[dbi].name.iov_base || dbi < CORE_DBS); } @@ -271,8 +260,7 @@ int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, * 4) user_flags отличаются, но table пустая и задан флаг MDBX_CREATE * = предполагаем что пользователь пересоздает table; */ - if ((user_flags & ~MDBX_CREATE) != - (unsigned)(env->dbs_flags[dbi] & DB_PERSISTENT_FLAGS)) { + if ((user_flags & ~MDBX_CREATE) != (unsigned)(env->dbs_flags[dbi] & DB_PERSISTENT_FLAGS)) { /* flags are differs, check other conditions */ if ((!user_flags && (!keycmp || keycmp == env->kvs[dbi].clc.k.cmp) && (!datacmp || datacmp == env->kvs[dbi].clc.v.cmp)) || @@ -287,11 +275,8 @@ int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, if (unlikely(err == MDBX_SUCCESS)) return err; } - eASSERT(env, ((env->dbs_flags[dbi] ^ txn->dbs[dbi].flags) & - DB_PERSISTENT_FLAGS) == 0); - eASSERT(env, - (txn->dbi_state[dbi] & (DBI_LINDO | DBI_VALID | DBI_STALE)) == - (DBI_LINDO | DBI_VALID)); + eASSERT(env, ((env->dbs_flags[dbi] ^ txn->dbs[dbi].flags) & DB_PERSISTENT_FLAGS) == 0); + eASSERT(env, (txn->dbi_state[dbi] & (DBI_LINDO | DBI_VALID | DBI_STALE)) == (DBI_LINDO | DBI_VALID)); if (unlikely(txn->dbs[dbi].leaf_pages)) return /* FIXME: return extended info */ MDBX_INCOMPATIBLE; @@ -299,13 +284,11 @@ int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, if (unlikely(txn->cursors[dbi])) return MDBX_DANGLING_DBI; env->dbs_flags[dbi] = DB_POISON; - atomic_store32(&env->dbi_seqs[dbi], dbi_seq_next(env, dbi), - mo_AcquireRelease); + atomic_store32(&env->dbi_seqs[dbi], dbi_seq_next(env, dbi), mo_AcquireRelease); const uint32_t seq = dbi_seq_next(env, dbi); const uint16_t db_flags = user_flags & DB_PERSISTENT_FLAGS; - eASSERT(env, txn->dbs[dbi].height == 0 && txn->dbs[dbi].items == 0 && - txn->dbs[dbi].root == P_INVALID); + eASSERT(env, txn->dbs[dbi].height == 0 && txn->dbs[dbi].items == 0 && txn->dbs[dbi].root == P_INVALID); env->kvs[dbi].clc.k.cmp = keycmp ? keycmp : builtin_keycmp(user_flags); env->kvs[dbi].clc.v.cmp = datacmp ? datacmp : builtin_datacmp(user_flags); txn->dbs[dbi].flags = db_flags; @@ -325,8 +308,7 @@ int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, } if (!keycmp) - keycmp = (env->dbs_flags[dbi] & DB_VALID) ? env->kvs[dbi].clc.k.cmp - : builtin_keycmp(user_flags); + keycmp = (env->dbs_flags[dbi] & DB_VALID) ? env->kvs[dbi].clc.k.cmp : builtin_keycmp(user_flags); if (env->kvs[dbi].clc.k.cmp != keycmp) { if (env->dbs_flags[dbi] & DB_VALID) return MDBX_EINVAL; @@ -334,8 +316,7 @@ int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, } if (!datacmp) - datacmp = (env->dbs_flags[dbi] & DB_VALID) ? env->kvs[dbi].clc.v.cmp - : builtin_datacmp(user_flags); + datacmp = (env->dbs_flags[dbi] & DB_VALID) ? env->kvs[dbi].clc.v.cmp : builtin_datacmp(user_flags); if (env->kvs[dbi].clc.v.cmp != datacmp) { if (env->dbs_flags[dbi] & DB_VALID) return MDBX_EINVAL; @@ -346,19 +327,15 @@ int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, } static inline size_t dbi_namelen(const MDBX_val name) { - return (name.iov_len > sizeof(defer_free_item_t)) ? name.iov_len - : sizeof(defer_free_item_t); + return (name.iov_len > sizeof(defer_free_item_t)) ? name.iov_len : sizeof(defer_free_item_t); } -static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi, - MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp, - MDBX_val name) { +static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp, + MDBX_cmp_func *datacmp, MDBX_val name) { MDBX_env *const env = txn->env; /* Cannot mix named table(s) with DUPSORT flags */ - tASSERT(txn, - (txn->dbi_state[MAIN_DBI] & (DBI_LINDO | DBI_VALID | DBI_STALE)) == - (DBI_LINDO | DBI_VALID)); + tASSERT(txn, (txn->dbi_state[MAIN_DBI] & (DBI_LINDO | DBI_VALID | DBI_STALE)) == (DBI_LINDO | DBI_VALID)); if (unlikely(txn->dbs[MAIN_DBI].flags & MDBX_DUPSORT)) { if (unlikely((user_flags & MDBX_CREATE) == 0)) return MDBX_NOTFOUND; @@ -367,18 +344,15 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi, return MDBX_INCOMPATIBLE; /* Пересоздаём MainDB когда там пусто. */ - tASSERT(txn, txn->dbs[MAIN_DBI].height == 0 && - txn->dbs[MAIN_DBI].items == 0 && - txn->dbs[MAIN_DBI].root == P_INVALID); + tASSERT(txn, + txn->dbs[MAIN_DBI].height == 0 && txn->dbs[MAIN_DBI].items == 0 && txn->dbs[MAIN_DBI].root == P_INVALID); if (unlikely(txn->cursors[MAIN_DBI])) return MDBX_DANGLING_DBI; env->dbs_flags[MAIN_DBI] = DB_POISON; - atomic_store32(&env->dbi_seqs[MAIN_DBI], dbi_seq_next(env, MAIN_DBI), - mo_AcquireRelease); + atomic_store32(&env->dbi_seqs[MAIN_DBI], dbi_seq_next(env, MAIN_DBI), mo_AcquireRelease); const uint32_t seq = dbi_seq_next(env, MAIN_DBI); - const uint16_t main_flags = - txn->dbs[MAIN_DBI].flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY); + const uint16_t main_flags = txn->dbs[MAIN_DBI].flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY); env->kvs[MAIN_DBI].clc.k.cmp = builtin_keycmp(main_flags); env->kvs[MAIN_DBI].clc.v.cmp = builtin_datacmp(main_flags); txn->dbs[MAIN_DBI].flags = main_flags; @@ -391,8 +365,7 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi, return err; } env->dbs_flags[MAIN_DBI] = main_flags | DB_VALID; - txn->dbi_seqs[MAIN_DBI] = - atomic_store32(&env->dbi_seqs[MAIN_DBI], seq, mo_AcquireRelease); + txn->dbi_seqs[MAIN_DBI] = atomic_store32(&env->dbi_seqs[MAIN_DBI], seq, mo_AcquireRelease); txn->dbi_state[MAIN_DBI] |= DBI_DIRTY; txn->flags |= MDBX_TXN_DIRTY; } @@ -410,8 +383,7 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi, if (!env->kvs[MAIN_DBI].clc.k.cmp(&name, &env->kvs[scan].name)) { slot = scan; int err = dbi_check(txn, slot); - if (err == MDBX_BAD_DBI && - txn->dbi_state[slot] == (DBI_OLDEN | DBI_LINDO)) { + if (err == MDBX_BAD_DBI && txn->dbi_state[slot] == (DBI_OLDEN | DBI_LINDO)) { /* хендл использовался, стал невалидным, * но теперь явно пере-открывается в этой транзакци */ eASSERT(env, !txn->cursors[slot]); @@ -433,12 +405,10 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi, return MDBX_DBS_FULL; if (env->n_dbi == slot) - eASSERT(env, !env->dbs_flags[slot] && !env->kvs[slot].name.iov_len && - !env->kvs[slot].name.iov_base); + eASSERT(env, !env->dbs_flags[slot] && !env->kvs[slot].name.iov_len && !env->kvs[slot].name.iov_base); env->dbs_flags[slot] = DB_POISON; - atomic_store32(&env->dbi_seqs[slot], dbi_seq_next(env, slot), - mo_AcquireRelease); + atomic_store32(&env->dbi_seqs[slot], dbi_seq_next(env, slot), mo_AcquireRelease); memset(&env->kvs[slot], 0, sizeof(env->kvs[slot])); if (env->n_dbi == slot) env->n_dbi = (unsigned)slot + 1; @@ -461,13 +431,11 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi, return rc; } else { /* make sure this is actually a table */ - node_t *node = - page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]); + node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]); if (unlikely((node_flags(node) & (N_DUP | N_TREE)) != N_TREE)) return MDBX_INCOMPATIBLE; if (!MDBX_DISABLE_VALIDATION && unlikely(body.iov_len != sizeof(tree_t))) { - ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid table node size", body.iov_len); + ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid table node size", body.iov_len); return MDBX_CORRUPTED; } memcpy(&txn->dbs[slot], body.iov_base, sizeof(tree_t)); @@ -490,8 +458,7 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi, txn->dbs[slot].flags = user_flags & DB_PERSISTENT_FLAGS; cx.outer.next = txn->cursors[MAIN_DBI]; txn->cursors[MAIN_DBI] = &cx.outer; - rc = - cursor_put_checklen(&cx.outer, &name, &body, N_TREE | MDBX_NOOVERWRITE); + rc = cursor_put_checklen(&cx.outer, &name, &body, N_TREE | MDBX_NOOVERWRITE); txn->cursors[MAIN_DBI] = cx.outer.next; if (unlikely(rc != MDBX_SUCCESS)) goto bailout; @@ -503,9 +470,8 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi, /* Got info, register DBI in this txn */ const uint32_t seq = dbi_seq_next(env, slot); - eASSERT(env, - env->dbs_flags[slot] == DB_POISON && !txn->cursors[slot] && - (txn->dbi_state[slot] & (DBI_LINDO | DBI_VALID)) == DBI_LINDO); + eASSERT(env, env->dbs_flags[slot] == DB_POISON && !txn->cursors[slot] && + (txn->dbi_state[slot] & (DBI_LINDO | DBI_VALID)) == DBI_LINDO); txn->dbi_state[slot] = dbi_state; memcpy(&txn->dbs[slot], body.iov_base, sizeof(txn->dbs[slot])); env->dbs_flags[slot] = txn->dbs[slot].flags; @@ -515,8 +481,7 @@ static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi, env->kvs[slot].name = name; env->dbs_flags[slot] = txn->dbs[slot].flags | DB_VALID; - txn->dbi_seqs[slot] = - atomic_store32(&env->dbi_seqs[slot], seq, mo_AcquireRelease); + txn->dbi_seqs[slot] = atomic_store32(&env->dbi_seqs[slot], seq, mo_AcquireRelease); done: *dbi = (MDBX_dbi)slot; @@ -525,8 +490,7 @@ done: return MDBX_SUCCESS; bailout: - eASSERT(env, !txn->cursors[slot] && !env->kvs[slot].name.iov_len && - !env->kvs[slot].name.iov_base); + eASSERT(env, !txn->cursors[slot] && !env->kvs[slot].name.iov_len && !env->kvs[slot].name.iov_base); txn->dbi_state[slot] &= DBI_LINDO | DBI_OLDEN; env->dbs_flags[slot] = 0; osal_free(clone); @@ -535,14 +499,13 @@ bailout: return rc; } -int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags, - MDBX_dbi *dbi, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) { +int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp, + MDBX_cmp_func *datacmp) { if (unlikely(!dbi)) return MDBX_EINVAL; *dbi = 0; - if (user_flags != MDBX_ACCEDE && - unlikely(!check_table_flags(user_flags & ~MDBX_CREATE))) + if (user_flags != MDBX_ACCEDE && unlikely(!check_table_flags(user_flags & ~MDBX_CREATE))) return MDBX_EINVAL; int rc = check_txn(txn, MDBX_TXN_BLOCKED); @@ -567,8 +530,7 @@ int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags, } if (unlikely(name == MDBX_CHK_META || name->iov_base == MDBX_CHK_META)) return MDBX_EINVAL; - if (unlikely(name->iov_len > - txn->env->leaf_nodemax - NODESIZE - sizeof(tree_t))) + if (unlikely(name->iov_len > txn->env->leaf_nodemax - NODESIZE - sizeof(tree_t))) return MDBX_EINVAL; #if MDBX_ENABLE_DBI_LOCKFREE @@ -582,31 +544,24 @@ int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags, continue; } - const uint32_t snap_seq = - atomic_load32(&env->dbi_seqs[i], mo_AcquireRelease); + const uint32_t snap_seq = atomic_load32(&env->dbi_seqs[i], mo_AcquireRelease); const uint16_t snap_flags = env->dbs_flags[i]; const MDBX_val snap_name = env->kvs[i].name; if (user_flags != MDBX_ACCEDE && - (((user_flags ^ snap_flags) & DB_PERSISTENT_FLAGS) || - (keycmp && keycmp != env->kvs[i].clc.k.cmp) || + (((user_flags ^ snap_flags) & DB_PERSISTENT_FLAGS) || (keycmp && keycmp != env->kvs[i].clc.k.cmp) || (datacmp && datacmp != env->kvs[i].clc.v.cmp))) continue; - const uint32_t main_seq = - atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease); + const uint32_t main_seq = atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease); MDBX_cmp_func *const snap_cmp = env->kvs[MAIN_DBI].clc.k.cmp; - if (unlikely(!(snap_flags & DB_VALID) || !snap_name.iov_base || - !snap_name.iov_len || !snap_cmp)) + if (unlikely(!(snap_flags & DB_VALID) || !snap_name.iov_base || !snap_name.iov_len || !snap_cmp)) continue; const bool name_match = snap_cmp(&snap_name, name) == 0; osal_flush_incoherent_cpu_writeback(); - if (unlikely( - snap_seq != atomic_load32(&env->dbi_seqs[i], mo_AcquireRelease) || - main_seq != - atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease) || - snap_flags != env->dbs_flags[i] || - snap_name.iov_base != env->kvs[i].name.iov_base || - snap_name.iov_len != env->kvs[i].name.iov_len)) + if (unlikely(snap_seq != atomic_load32(&env->dbi_seqs[i], mo_AcquireRelease) || + main_seq != atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease) || + snap_flags != env->dbs_flags[i] || snap_name.iov_base != env->kvs[i].name.iov_base || + snap_name.iov_len != env->kvs[i].name.iov_len)) goto retry; if (name_match) { rc = dbi_check(txn, i); @@ -634,18 +589,15 @@ int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags, rc = osal_fastmutex_acquire(&txn->env->dbi_lock); if (likely(rc == MDBX_SUCCESS)) { rc = dbi_open_locked(txn, user_flags, dbi, keycmp, datacmp, *name); - ENSURE(txn->env, - osal_fastmutex_release(&txn->env->dbi_lock) == MDBX_SUCCESS); + ENSURE(txn->env, osal_fastmutex_release(&txn->env->dbi_lock) == MDBX_SUCCESS); } return rc; } -static int dbi_open_cstr(MDBX_txn *txn, const char *name_cstr, - MDBX_db_flags_t flags, MDBX_dbi *dbi, +static int dbi_open_cstr(MDBX_txn *txn, const char *name_cstr, MDBX_db_flags_t flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) { MDBX_val thunk, *name; - if (name_cstr == MDBX_CHK_MAIN || name_cstr == MDBX_CHK_GC || - name_cstr == MDBX_CHK_META) + if (name_cstr == MDBX_CHK_MAIN || name_cstr == MDBX_CHK_GC || name_cstr == MDBX_CHK_META) name = (void *)name_cstr; else { thunk.iov_len = strlen(name_cstr); @@ -660,8 +612,7 @@ struct dbi_rename_result { int err; }; -__cold static struct dbi_rename_result -dbi_rename_locked(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val new_name) { +__cold static struct dbi_rename_result dbi_rename_locked(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val new_name) { struct dbi_rename_result pair; pair.defer = nullptr; pair.err = dbi_check(txn, dbi); @@ -670,8 +621,7 @@ dbi_rename_locked(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val new_name) { MDBX_env *const env = txn->env; MDBX_val old_name = env->kvs[dbi].name; - if (env->kvs[MAIN_DBI].clc.k.cmp(&new_name, &old_name) == 0 && - MDBX_DEBUG == 0) + if (env->kvs[MAIN_DBI].clc.k.cmp(&new_name, &old_name) == 0 && MDBX_DEBUG == 0) return pair; cursor_couple_t cx; @@ -695,8 +645,7 @@ dbi_rename_locked(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val new_name) { txn->cursors[MAIN_DBI] = &cx.outer; MDBX_val data = {&txn->dbs[dbi], sizeof(tree_t)}; - pair.err = cursor_put_checklen(&cx.outer, &new_name, &data, - N_TREE | MDBX_NOOVERWRITE); + pair.err = cursor_put_checklen(&cx.outer, &new_name, &data, N_TREE | MDBX_NOOVERWRITE); if (likely(pair.err == MDBX_SUCCESS)) { pair.err = cursor_seek(&cx.outer, &old_name, nullptr, MDBX_SET).err; if (likely(pair.err == MDBX_SUCCESS)) @@ -732,8 +681,7 @@ static defer_free_item_t *dbi_close_locked(MDBX_env *env, MDBX_dbi dbi) { do { --i; eASSERT(env, i >= CORE_DBS); - eASSERT(env, !env->dbs_flags[i] && !env->kvs[i].name.iov_len && - !env->kvs[i].name.iov_base); + eASSERT(env, !env->dbs_flags[i] && !env->kvs[i].name.iov_len && !env->kvs[i].name.iov_base); } while (i > CORE_DBS && !env->kvs[i - 1].name.iov_base); env->n_dbi = (unsigned)i; } @@ -745,25 +693,21 @@ static defer_free_item_t *dbi_close_locked(MDBX_env *env, MDBX_dbi dbi) { /*----------------------------------------------------------------------------*/ /* API */ -int mdbx_dbi_open(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, - MDBX_dbi *dbi) { +int mdbx_dbi_open(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, MDBX_dbi *dbi) { return LOG_IFERR(dbi_open_cstr(txn, name, flags, dbi, nullptr, nullptr)); } -int mdbx_dbi_open2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, - MDBX_dbi *dbi) { +int mdbx_dbi_open2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, MDBX_dbi *dbi) { return LOG_IFERR(dbi_open(txn, name, flags, dbi, nullptr, nullptr)); } -int mdbx_dbi_open_ex(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, - MDBX_dbi *dbi, MDBX_cmp_func *keycmp, +int mdbx_dbi_open_ex(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) { return LOG_IFERR(dbi_open_cstr(txn, name, flags, dbi, keycmp, datacmp)); } -int mdbx_dbi_open_ex2(MDBX_txn *txn, const MDBX_val *name, - MDBX_db_flags_t flags, MDBX_dbi *dbi, - MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) { +int mdbx_dbi_open_ex2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp, + MDBX_cmp_func *datacmp) { return LOG_IFERR(dbi_open(txn, name, flags, dbi, keycmp, datacmp)); } @@ -780,8 +724,7 @@ __cold int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, bool del) { if (txn->dbs[dbi].height) { cx.outer.next = txn->cursors[dbi]; txn->cursors[dbi] = &cx.outer; - rc = tree_drop(&cx.outer, - dbi == MAIN_DBI || (cx.outer.tree->flags & MDBX_DUPSORT)); + rc = tree_drop(&cx.outer, dbi == MAIN_DBI || (cx.outer.tree->flags & MDBX_DUPSORT)); txn->cursors[dbi] = cx.outer.next; if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); @@ -832,8 +775,7 @@ __cold int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, bool del) { __cold int mdbx_dbi_rename(MDBX_txn *txn, MDBX_dbi dbi, const char *name_cstr) { MDBX_val thunk, *name; - if (name_cstr == MDBX_CHK_MAIN || name_cstr == MDBX_CHK_GC || - name_cstr == MDBX_CHK_META) + if (name_cstr == MDBX_CHK_MAIN || name_cstr == MDBX_CHK_GC || name_cstr == MDBX_CHK_META) name = (void *)name_cstr; else { thunk.iov_len = strlen(name_cstr); @@ -860,8 +802,7 @@ int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) { rc = osal_fastmutex_acquire(&env->dbi_lock); if (likely(rc == MDBX_SUCCESS && dbi < env->n_dbi)) { retry: - if (env->basal_txn && (env->dbs_flags[dbi] & DB_VALID) && - (env->basal_txn->flags & MDBX_TXN_FINISHED) == 0) { + if (env->basal_txn && (env->dbs_flags[dbi] & DB_VALID) && (env->basal_txn->flags & MDBX_TXN_FINISHED) == 0) { /* LY: Опасный код, так как env->txn может быть изменено в другом потоке. * К сожалению тут нет надежного решения и может быть падение при неверном * использовании API (вызове mdbx_dbi_close конкурентно с завершением @@ -884,8 +825,7 @@ int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) { * транзакции, и поэтому этот путь потенциально более опасен. */ const MDBX_txn *const hazard = env->txn; osal_compiler_barrier(); - if ((dbi_state(env->basal_txn, dbi) & - (DBI_LINDO | DBI_DIRTY | DBI_CREAT)) > DBI_LINDO) { + if ((dbi_state(env->basal_txn, dbi) & (DBI_LINDO | DBI_DIRTY | DBI_CREAT)) > DBI_LINDO) { bailout_dirty_dbi: osal_fastmutex_release(&env->dbi_lock); return LOG_IFERR(MDBX_DANGLING_DBI); @@ -893,11 +833,9 @@ int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) { osal_memory_barrier(); if (unlikely(hazard != env->txn)) goto retry; - if (hazard != env->basal_txn && hazard && - (hazard->flags & MDBX_TXN_FINISHED) == 0 && + if (hazard != env->basal_txn && hazard && (hazard->flags & MDBX_TXN_FINISHED) == 0 && hazard->signature == txn_signature && - (dbi_state(hazard, dbi) & (DBI_LINDO | DBI_DIRTY | DBI_CREAT)) > - DBI_LINDO) + (dbi_state(hazard, dbi) & (DBI_LINDO | DBI_DIRTY | DBI_CREAT)) > DBI_LINDO) goto bailout_dirty_dbi; osal_compiler_barrier(); if (unlikely(hazard != env->txn)) @@ -908,8 +846,7 @@ int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) { return LOG_IFERR(rc); } -int mdbx_dbi_flags_ex(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags, - unsigned *state) { +int mdbx_dbi_flags_ex(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags, unsigned *state) { if (unlikely(!flags || !state)) return LOG_IFERR(MDBX_EINVAL); @@ -928,21 +865,17 @@ int mdbx_dbi_flags_ex(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags, } *flags = txn->dbs[dbi].flags & DB_PERSISTENT_FLAGS; - *state = - txn->dbi_state[dbi] & (DBI_FRESH | DBI_CREAT | DBI_DIRTY | DBI_STALE); + *state = txn->dbi_state[dbi] & (DBI_FRESH | DBI_CREAT | DBI_DIRTY | DBI_STALE); return MDBX_SUCCESS; } -__cold int mdbx_dbi_rename2(MDBX_txn *txn, MDBX_dbi dbi, - const MDBX_val *new_name) { +__cold int mdbx_dbi_rename2(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *new_name) { int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); - if (unlikely(new_name == MDBX_CHK_MAIN || - new_name->iov_base == MDBX_CHK_MAIN || new_name == MDBX_CHK_GC || - new_name->iov_base == MDBX_CHK_GC || new_name == MDBX_CHK_META || - new_name->iov_base == MDBX_CHK_META)) + if (unlikely(new_name == MDBX_CHK_MAIN || new_name->iov_base == MDBX_CHK_MAIN || new_name == MDBX_CHK_GC || + new_name->iov_base == MDBX_CHK_GC || new_name == MDBX_CHK_META || new_name->iov_base == MDBX_CHK_META)) return LOG_IFERR(MDBX_EINVAL); if (unlikely(dbi < CORE_DBS)) @@ -968,13 +901,11 @@ static void stat_get(const tree_t *db, MDBX_stat *st, size_t bytes) { st->ms_leaf_pages = db->leaf_pages; st->ms_overflow_pages = db->large_pages; st->ms_entries = db->items; - if (likely(bytes >= - offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid))) + if (likely(bytes >= offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid))) st->ms_mod_txnid = db->mod_txnid; } -__cold int mdbx_dbi_stat(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *dest, - size_t bytes) { +__cold int mdbx_dbi_stat(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *dest, size_t bytes) { if (unlikely(!dest)) return LOG_IFERR(MDBX_EINVAL); @@ -1012,8 +943,7 @@ bailout: return LOG_IFERR(rc); } -__cold const tree_t *dbi_dig(const MDBX_txn *txn, const size_t dbi, - tree_t *fallback) { +__cold const tree_t *dbi_dig(const MDBX_txn *txn, const size_t dbi, tree_t *fallback) { const MDBX_txn *dig = txn; do { tASSERT(txn, txn->n_dbi == dig->n_dbi); @@ -1036,8 +966,7 @@ __cold const tree_t *dbi_dig(const MDBX_txn *txn, const size_t dbi, return fallback; } -__cold int mdbx_enumerate_tables(const MDBX_txn *txn, - MDBX_table_enum_func *func, void *ctx) { +__cold int mdbx_enumerate_tables(const MDBX_txn *txn, MDBX_table_enum_func *func, void *ctx) { if (unlikely(!func)) return LOG_IFERR(MDBX_EINVAL); @@ -1054,13 +983,12 @@ __cold int mdbx_enumerate_tables(const MDBX_txn *txn, txn->cursors[MAIN_DBI] = &cx.outer; for (rc = outer_first(&cx.outer, nullptr, nullptr); rc == MDBX_SUCCESS; rc = outer_next(&cx.outer, nullptr, nullptr, MDBX_NEXT_NODUP)) { - node_t *node = - page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]); + node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]); if (node_flags(node) != N_TREE) continue; if (unlikely(node_ds(node) != sizeof(tree_t))) { - ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid dupsort sub-tree node size", (unsigned)node_ds(node)); + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid dupsort sub-tree node size", + (unsigned)node_ds(node)); rc = MDBX_CORRUPTED; break; } diff --git a/src/dbi.h b/src/dbi.h index bfafe3e3..c06f5bd1 100644 --- a/src/dbi.h +++ b/src/dbi.h @@ -7,8 +7,8 @@ #if MDBX_ENABLE_DBI_SPARSE -MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL size_t -dbi_bitmap_ctz_fallback(const MDBX_txn *txn, intptr_t bmi); +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL size_t dbi_bitmap_ctz_fallback(const MDBX_txn *txn, + intptr_t bmi); static inline size_t dbi_bitmap_ctz(const MDBX_txn *txn, intptr_t bmi) { tASSERT(txn, bmi > 0); @@ -18,8 +18,7 @@ static inline size_t dbi_bitmap_ctz(const MDBX_txn *txn, intptr_t bmi) { return __builtin_ctz((int)bmi); if (sizeof(txn->dbi_sparse[0]) == sizeof(long)) return __builtin_ctzl((long)bmi); -#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || \ - __has_builtin(__builtin_ctzll) +#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || __has_builtin(__builtin_ctzll) return __builtin_ctzll(bmi); #endif /* have(long long) && long long == uint64_t */ #endif /* GNU C */ @@ -46,27 +45,27 @@ static inline size_t dbi_bitmap_ctz(const MDBX_txn *txn, intptr_t bmi) { /* LY: Макрос целенаправленно сделан с одним циклом, чтобы сохранить возможность * использования оператора break */ -#define TXN_FOREACH_DBI_FROM(TXN, I, FROM) \ - for (size_t bitmap_chunk = CHAR_BIT * sizeof(TXN->dbi_sparse[0]), \ - bitmap_item = TXN->dbi_sparse[0] >> FROM, I = FROM; \ - I < TXN->n_dbi; ++I) \ - if (bitmap_item == 0) { \ - I = (I - 1) | (bitmap_chunk - 1); \ - bitmap_item = TXN->dbi_sparse[(1 + I) / bitmap_chunk]; \ - if (!bitmap_item) \ - I += bitmap_chunk; \ - continue; \ - } else if ((bitmap_item & 1) == 0) { \ - size_t bitmap_skip = dbi_bitmap_ctz(txn, bitmap_item); \ - bitmap_item >>= bitmap_skip; \ - I += bitmap_skip - 1; \ - continue; \ +#define TXN_FOREACH_DBI_FROM(TXN, I, FROM) \ + for (size_t bitmap_chunk = CHAR_BIT * sizeof(TXN->dbi_sparse[0]), bitmap_item = TXN->dbi_sparse[0] >> FROM, \ + I = FROM; \ + I < TXN->n_dbi; ++I) \ + if (bitmap_item == 0) { \ + I = (I - 1) | (bitmap_chunk - 1); \ + bitmap_item = TXN->dbi_sparse[(1 + I) / bitmap_chunk]; \ + if (!bitmap_item) \ + I += bitmap_chunk; \ + continue; \ + } else if ((bitmap_item & 1) == 0) { \ + size_t bitmap_skip = dbi_bitmap_ctz(txn, bitmap_item); \ + bitmap_item >>= bitmap_skip; \ + I += bitmap_skip - 1; \ + continue; \ } else if (bitmap_item >>= 1, TXN->dbi_state[I]) #else -#define TXN_FOREACH_DBI_FROM(TXN, I, SKIP) \ - for (size_t I = SKIP; I < TXN->n_dbi; ++I) \ +#define TXN_FOREACH_DBI_FROM(TXN, I, SKIP) \ + for (size_t I = SKIP; I < TXN->n_dbi; ++I) \ if (TXN->dbi_state[I]) #endif /* MDBX_ENABLE_DBI_SPARSE */ @@ -80,24 +79,19 @@ struct dbi_snap_result { uint32_t sequence; unsigned flags; }; -MDBX_INTERNAL struct dbi_snap_result dbi_snap(const MDBX_env *env, - const size_t dbi); +MDBX_INTERNAL struct dbi_snap_result dbi_snap(const MDBX_env *env, const size_t dbi); MDBX_INTERNAL int dbi_update(MDBX_txn *txn, int keep); static inline uint8_t dbi_state(const MDBX_txn *txn, const size_t dbi) { - STATIC_ASSERT( - (int)DBI_DIRTY == MDBX_DBI_DIRTY && (int)DBI_STALE == MDBX_DBI_STALE && - (int)DBI_FRESH == MDBX_DBI_FRESH && (int)DBI_CREAT == MDBX_DBI_CREAT); + STATIC_ASSERT((int)DBI_DIRTY == MDBX_DBI_DIRTY && (int)DBI_STALE == MDBX_DBI_STALE && + (int)DBI_FRESH == MDBX_DBI_FRESH && (int)DBI_CREAT == MDBX_DBI_CREAT); #if MDBX_ENABLE_DBI_SPARSE const size_t bitmap_chunk = CHAR_BIT * sizeof(txn->dbi_sparse[0]); const size_t bitmap_indx = dbi / bitmap_chunk; const size_t bitmap_mask = (size_t)1 << dbi % bitmap_chunk; - return likely(dbi < txn->n_dbi && - (txn->dbi_sparse[bitmap_indx] & bitmap_mask) != 0) - ? txn->dbi_state[dbi] - : 0; + return likely(dbi < txn->n_dbi && (txn->dbi_sparse[bitmap_indx] & bitmap_mask) != 0) ? txn->dbi_state[dbi] : 0; #else return likely(dbi < txn->n_dbi) ? txn->dbi_state[dbi] : 0; #endif /* MDBX_ENABLE_DBI_SPARSE */ @@ -106,8 +100,7 @@ static inline uint8_t dbi_state(const MDBX_txn *txn, const size_t dbi) { static inline bool dbi_changed(const MDBX_txn *txn, const size_t dbi) { const MDBX_env *const env = txn->env; eASSERT(env, dbi_state(txn, dbi) & DBI_LINDO); - const uint32_t snap_seq = - atomic_load32(&env->dbi_seqs[dbi], mo_AcquireRelease); + const uint32_t snap_seq = atomic_load32(&env->dbi_seqs[dbi], mo_AcquireRelease); return snap_seq != txn->dbi_seqs[dbi]; } @@ -125,12 +118,10 @@ static inline uint32_t dbi_seq_next(const MDBX_env *const env, size_t dbi) { return v ? v : 1; } -MDBX_INTERNAL int dbi_open(MDBX_txn *txn, const MDBX_val *const name, - unsigned user_flags, MDBX_dbi *dbi, +MDBX_INTERNAL int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp); -MDBX_INTERNAL int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, - MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp); +MDBX_INTERNAL int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, MDBX_cmp_func *keycmp, + MDBX_cmp_func *datacmp); -MDBX_INTERNAL const tree_t *dbi_dig(const MDBX_txn *txn, const size_t dbi, - tree_t *fallback); +MDBX_INTERNAL const tree_t *dbi_dig(const MDBX_txn *txn, const size_t dbi, tree_t *fallback); diff --git a/src/debug_begin.h b/src/debug_begin.h index 521e99cf..09f62cea 100644 --- a/src/debug_begin.h +++ b/src/debug_begin.h @@ -9,28 +9,22 @@ #pragma push_macro("eASSERT") #undef TRACE -#define TRACE(fmt, ...) \ - debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__) +#define TRACE(fmt, ...) debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__) #undef DEBUG -#define DEBUG(fmt, ...) \ - debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__) +#define DEBUG(fmt, ...) debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__) #undef VERBOSE -#define VERBOSE(fmt, ...) \ - debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__) +#define VERBOSE(fmt, ...) debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__) #undef NOTICE -#define NOTICE(fmt, ...) \ - debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__) +#define NOTICE(fmt, ...) debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__) #undef WARNING -#define WARNING(fmt, ...) \ - debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__) +#define WARNING(fmt, ...) debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__) #undef ERROR -#define ERROR(fmt, ...) \ - debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__) +#define ERROR(fmt, ...) debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__) #undef eASSERT #define eASSERT(env, expr) ENSURE(env, expr) diff --git a/src/dpl.c b/src/dpl.c index 5e9f4485..6055ea7b 100644 --- a/src/dpl.c +++ b/src/dpl.c @@ -9,12 +9,10 @@ static inline size_t dpl_size2bytes(ptrdiff_t size) { size += size; #endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(dpl_t) + - (PAGELIST_LIMIT * (MDBX_DPL_PREALLOC_FOR_RADIXSORT + 1)) * - sizeof(dp_t) + + (PAGELIST_LIMIT * (MDBX_DPL_PREALLOC_FOR_RADIXSORT + 1)) * sizeof(dp_t) + MDBX_PNL_GRANULATE * sizeof(void *) * 2 < SIZE_MAX / 4 * 3); - size_t bytes = ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(dpl_t) + - size * sizeof(dp_t), + size_t bytes = ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(dpl_t) + size * sizeof(dp_t), MDBX_PNL_GRANULATE * sizeof(void *) * 2) - MDBX_ASSUME_MALLOC_OVERHEAD; return bytes; @@ -22,8 +20,7 @@ static inline size_t dpl_size2bytes(ptrdiff_t size) { static inline size_t dpl_bytes2size(const ptrdiff_t bytes) { size_t size = (bytes - sizeof(dpl_t)) / sizeof(dp_t); - assert(size > CURSOR_STACK_SIZE && - size <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE); + assert(size > CURSOR_STACK_SIZE && size <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE); #if MDBX_DPL_PREALLOC_FOR_RADIXSORT size >>= 1; #endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ @@ -41,8 +38,7 @@ dpl_t *dpl_reserve(MDBX_txn *txn, size_t size) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - size_t bytes = - dpl_size2bytes((size < PAGELIST_LIMIT) ? size : PAGELIST_LIMIT); + size_t bytes = dpl_size2bytes((size < PAGELIST_LIMIT) ? size : PAGELIST_LIMIT); dpl_t *const dl = osal_realloc(txn->tw.dirtylist, bytes); if (likely(dl)) { #if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) @@ -59,16 +55,13 @@ int dpl_alloc(MDBX_txn *txn) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - const size_t wanna = (txn->env->options.dp_initial < txn->geo.upper) - ? txn->env->options.dp_initial - : txn->geo.upper; + const size_t wanna = (txn->env->options.dp_initial < txn->geo.upper) ? txn->env->options.dp_initial : txn->geo.upper; #if MDBX_FORCE_ASSERTIONS || MDBX_DEBUG if (txn->tw.dirtylist) /* обнуляем чтобы не сработал ассерт внутри dpl_reserve() */ txn->tw.dirtylist->sorted = txn->tw.dirtylist->length = 0; #endif /* asertions enabled */ - if (unlikely(!txn->tw.dirtylist || txn->tw.dirtylist->detent < wanna || - txn->tw.dirtylist->detent > wanna + wanna) && + if (unlikely(!txn->tw.dirtylist || txn->tw.dirtylist->detent < wanna || txn->tw.dirtylist->detent > wanna + wanna) && unlikely(!dpl_reserve(txn, wanna))) return MDBX_ENOMEM; @@ -77,8 +70,7 @@ int dpl_alloc(MDBX_txn *txn) { } #define MDBX_DPL_EXTRACT_KEY(ptr) ((ptr)->pgno) -RADIXSORT_IMPL(dp, dp_t, MDBX_DPL_EXTRACT_KEY, MDBX_DPL_PREALLOC_FOR_RADIXSORT, - 1) +RADIXSORT_IMPL(dp, dp_t, MDBX_DPL_EXTRACT_KEY, MDBX_DPL_PREALLOC_FOR_RADIXSORT, 1) #define DP_SORT_CMP(first, last) ((first).pgno < (last).pgno) SORT_IMPL(dp_sort, false, dp_t, DP_SORT_CMP) @@ -90,16 +82,13 @@ __hot __noinline dpl_t *dpl_sort_slowpath(const MDBX_txn *txn) { dpl_t *dl = txn->tw.dirtylist; assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); const size_t unsorted = dl->length - dl->sorted; - if (likely(unsorted < MDBX_RADIXSORT_THRESHOLD) || - unlikely(!dp_radixsort(dl->items + 1, dl->length))) { + if (likely(unsorted < MDBX_RADIXSORT_THRESHOLD) || unlikely(!dp_radixsort(dl->items + 1, dl->length))) { if (dl->sorted > unsorted / 4 + 4 && - (MDBX_DPL_PREALLOC_FOR_RADIXSORT || - dl->length + unsorted < dl->detent + dpl_gap_mergesort)) { + (MDBX_DPL_PREALLOC_FOR_RADIXSORT || dl->length + unsorted < dl->detent + dpl_gap_mergesort)) { dp_t *const sorted_begin = dl->items + 1; dp_t *const sorted_end = sorted_begin + dl->sorted; - dp_t *const end = dl->items + (MDBX_DPL_PREALLOC_FOR_RADIXSORT - ? dl->length + dl->length + 1 - : dl->detent + dpl_reserve_gap); + dp_t *const end = + dl->items + (MDBX_DPL_PREALLOC_FOR_RADIXSORT ? dl->length + dl->length + 1 : dl->detent + dpl_reserve_gap); dp_t *const tmp = end - unsorted; assert(dl->items + dl->length + 1 < tmp); /* copy unsorted to the end of allocated space and sort it */ @@ -120,19 +109,16 @@ __hot __noinline dpl_t *dpl_sort_slowpath(const MDBX_txn *txn) { #endif } while (likely(--w > l)); assert(r == tmp - 1); - assert(dl->items[0].pgno == 0 && - dl->items[dl->length + 1].pgno == P_INVALID); + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); if (ASSERT_ENABLED()) for (size_t i = 0; i <= dl->length; ++i) assert(dl->items[i].pgno < dl->items[i + 1].pgno); } else { dp_sort(dl->items + 1, dl->items + dl->length + 1); - assert(dl->items[0].pgno == 0 && - dl->items[dl->length + 1].pgno == P_INVALID); + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); } } else { - assert(dl->items[0].pgno == 0 && - dl->items[dl->length + 1].pgno == P_INVALID); + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); } dl->sorted = dl->length; return dl; @@ -143,8 +129,7 @@ __hot __noinline dpl_t *dpl_sort_slowpath(const MDBX_txn *txn) { #define DP_SEARCH_CMP(dp, id) ((dp).pgno < (id)) SEARCH_IMPL(dp_bsearch, dp_t, pgno_t, DP_SEARCH_CMP) -__hot __noinline MDBX_INTERNAL size_t dpl_search(const MDBX_txn *txn, - pgno_t pgno) { +__hot __noinline MDBX_INTERNAL size_t dpl_search(const MDBX_txn *txn, pgno_t pgno) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); @@ -166,10 +151,10 @@ __hot __noinline MDBX_INTERNAL size_t dpl_search(const MDBX_txn *txn, /* whole sorted cases */ break; -#define LINEAR_SEARCH_CASE(N) \ - case N: \ - if (dl->items[dl->length - N + 1].pgno == pgno) \ - return dl->length - N + 1; \ +#define LINEAR_SEARCH_CASE(N) \ + case N: \ + if (dl->items[dl->length - N + 1].pgno == pgno) \ + return dl->length - N + 1; \ __fallthrough /* use linear scan until the threshold */ @@ -193,8 +178,7 @@ const page_t *debug_dpl_find(const MDBX_txn *txn, const pgno_t pgno) { const dpl_t *dl = txn->tw.dirtylist; if (dl) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - assert(dl->items[0].pgno == 0 && - dl->items[dl->length + 1].pgno == P_INVALID); + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); for (size_t i = dl->length; i > dl->sorted; --i) if (dl->items[i].pgno == pgno) return dl->items[i].ptr; @@ -220,13 +204,11 @@ void dpl_remove_ex(const MDBX_txn *txn, size_t i, size_t npages) { dl->pages_including_loose -= npages; dl->sorted -= dl->sorted >= i; dl->length -= 1; - memmove(dl->items + i, dl->items + i + 1, - (dl->length - i + 2) * sizeof(dl->items[0])); + memmove(dl->items + i, dl->items + i + 1, (dl->length - i + 2) * sizeof(dl->items[0])); assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); } -int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, - size_t npages) { +int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, size_t npages) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); const dp_t dp = {page, pgno, (pgno_t)npages}; @@ -237,8 +219,7 @@ int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, dpl_t *dl = txn->tw.dirtylist; tASSERT(txn, dl->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE); - tASSERT(txn, dl->items[0].pgno == 0 && - dl->items[dl->length + 1].pgno == P_INVALID); + tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); if (AUDIT_ENABLED()) { for (size_t i = dl->length; i > 0; --i) { assert(dl->items[i].pgno != dp.pgno); @@ -254,9 +235,7 @@ int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, ERROR("DPL is full (PAGELIST_LIMIT %zu)", PAGELIST_LIMIT); return MDBX_TXN_FULL; } - const size_t size = (dl->detent < MDBX_PNL_INITIAL * 42) - ? dl->detent + dl->detent - : dl->detent + dl->detent / 2; + const size_t size = (dl->detent < MDBX_PNL_INITIAL * 42) ? dl->detent + dl->detent : dl->detent + dl->detent / 2; dl = dpl_reserve(txn, size); if (unlikely(!dl)) return MDBX_ENOMEM; @@ -288,10 +267,7 @@ int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, const ptrdiff_t pivot = (ptrdiff_t)dl->length - dpl_insertion_threshold; #if MDBX_HAVE_CMOV const pgno_t pivot_pgno = - dl->items[(dl->length < dpl_insertion_threshold) - ? 0 - : dl->length - dpl_insertion_threshold] - .pgno; + dl->items[(dl->length < dpl_insertion_threshold) ? 0 : dl->length - dpl_insertion_threshold].pgno; #endif /* MDBX_HAVE_CMOV */ /* copy the stub beyond the end */ @@ -310,9 +286,7 @@ int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, while (i >= dl->items + dl->sorted) { #if !defined(__GNUC__) /* пытаемся избежать вызова memmove() */ i[1] = *i; -#elif MDBX_WORDBITS == 64 && \ - (defined(__SIZEOF_INT128__) || \ - (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)) +#elif MDBX_WORDBITS == 64 && (defined(__SIZEOF_INT128__) || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)) STATIC_ASSERT(sizeof(dp) == sizeof(__uint128_t)); ((__uint128_t *)i)[1] = *(volatile __uint128_t *)i; #else @@ -347,9 +321,8 @@ __cold bool dpl_check(MDBX_txn *txn) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); - tASSERT(txn, txn->tw.dirtyroom + dl->length == - (txn->parent ? txn->parent->tw.dirtyroom - : txn->env->options.dp_limit)); + tASSERT(txn, + txn->tw.dirtyroom + dl->length == (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); if (!AUDIT_ENABLED()) return true; @@ -389,16 +362,12 @@ __cold bool dpl_check(MDBX_txn *txn) { return false; } - const size_t rpa = - pnl_search(txn->tw.relist, dp->pgno, txn->geo.first_unallocated); - tASSERT(txn, rpa > MDBX_PNL_GETSIZE(txn->tw.relist) || - txn->tw.relist[rpa] != dp->pgno); - if (rpa <= MDBX_PNL_GETSIZE(txn->tw.relist) && - unlikely(txn->tw.relist[rpa] == dp->pgno)) + const size_t rpa = pnl_search(txn->tw.relist, dp->pgno, txn->geo.first_unallocated); + tASSERT(txn, rpa > MDBX_PNL_GETSIZE(txn->tw.relist) || txn->tw.relist[rpa] != dp->pgno); + if (rpa <= MDBX_PNL_GETSIZE(txn->tw.relist) && unlikely(txn->tw.relist[rpa] == dp->pgno)) return false; if (num > 1) { - const size_t rpb = pnl_search(txn->tw.relist, dp->pgno + num - 1, - txn->geo.first_unallocated); + const size_t rpb = pnl_search(txn->tw.relist, dp->pgno + num - 1, txn->geo.first_unallocated); tASSERT(txn, rpa == rpb); if (unlikely(rpa != rpb)) return false; @@ -432,8 +401,7 @@ __noinline void dpl_lru_reduce(MDBX_txn *txn) { txn->tw.dirtylru >>= 1; dpl_t *dl = txn->tw.dirtylist; for (size_t i = 1; i <= dl->length; ++i) { - size_t *const ptr = - ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t)); + size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t)); *ptr >>= 1; } txn = txn->parent; @@ -444,8 +412,7 @@ void dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); if (MDBX_PNL_GETSIZE(pl) && txn->tw.dirtylist->length) { - tASSERT(txn, pnl_check_allocated(pl, (size_t)txn->geo.first_unallocated - << spilled)); + tASSERT(txn, pnl_check_allocated(pl, (size_t)txn->geo.first_unallocated << spilled)); dpl_t *dl = dpl_sort(txn); /* Scanning in ascend order */ @@ -501,8 +468,7 @@ void dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled) { dl->sorted = dpl_setlen(dl, w - 1); txn->tw.dirtyroom += r - w; tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom - : txn->env->options.dp_limit)); + (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); return; } } diff --git a/src/dpl.h b/src/dpl.h index 9d2f59c6..0059dc58 100644 --- a/src/dpl.h +++ b/src/dpl.h @@ -49,38 +49,32 @@ static inline dpl_t *dpl_sort(const MDBX_txn *txn) { dpl_t *dl = txn->tw.dirtylist; tASSERT(txn, dl->length <= PAGELIST_LIMIT); tASSERT(txn, dl->sorted <= dl->length); - tASSERT(txn, dl->items[0].pgno == 0 && - dl->items[dl->length + 1].pgno == P_INVALID); + tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); return likely(dl->sorted == dl->length) ? dl : dpl_sort_slowpath(txn); } MDBX_INTERNAL __noinline size_t dpl_search(const MDBX_txn *txn, pgno_t pgno); -MDBX_MAYBE_UNUSED MDBX_INTERNAL const page_t * -debug_dpl_find(const MDBX_txn *txn, const pgno_t pgno); +MDBX_MAYBE_UNUSED MDBX_INTERNAL const page_t *debug_dpl_find(const MDBX_txn *txn, const pgno_t pgno); -MDBX_NOTHROW_PURE_FUNCTION static inline unsigned dpl_npages(const dpl_t *dl, - size_t i) { +MDBX_NOTHROW_PURE_FUNCTION static inline unsigned dpl_npages(const dpl_t *dl, size_t i) { assert(0 <= (intptr_t)i && i <= dl->length); unsigned n = dl->items[i].npages; assert(n == (is_largepage(dl->items[i].ptr) ? dl->items[i].ptr->pages : 1)); return n; } -MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t dpl_endpgno(const dpl_t *dl, - size_t i) { +MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t dpl_endpgno(const dpl_t *dl, size_t i) { return dpl_npages(dl, i) + dl->items[i].pgno; } -static inline bool dpl_intersect(const MDBX_txn *txn, pgno_t pgno, - size_t npages) { +static inline bool dpl_intersect(const MDBX_txn *txn, pgno_t pgno, size_t npages) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); dpl_t *dl = txn->tw.dirtylist; tASSERT(txn, dl->sorted == dl->length); - tASSERT(txn, dl->items[0].pgno == 0 && - dl->items[dl->length + 1].pgno == P_INVALID); + tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); size_t const n = dpl_search(txn, pgno); tASSERT(txn, n >= 1 && n <= dl->length + 1); tASSERT(txn, pgno <= dl->items[n].pgno); @@ -92,8 +86,7 @@ static inline bool dpl_intersect(const MDBX_txn *txn, pgno_t pgno, bool check = false; for (size_t i = 1; i <= dl->length; ++i) { const page_t *const dp = dl->items[i].ptr; - if (!(dp->pgno /* begin */ >= /* end */ pgno + npages || - dpl_endpgno(dl, i) /* end */ <= /* begin */ pgno)) + if (!(dp->pgno /* begin */ >= /* end */ pgno + npages || dpl_endpgno(dl, i) /* end */ <= /* begin */ pgno)) check |= true; } tASSERT(txn, check == rc); @@ -101,8 +94,7 @@ static inline bool dpl_intersect(const MDBX_txn *txn, pgno_t pgno, return rc; } -MDBX_NOTHROW_PURE_FUNCTION static inline size_t dpl_exist(const MDBX_txn *txn, - pgno_t pgno) { +MDBX_NOTHROW_PURE_FUNCTION static inline size_t dpl_exist(const MDBX_txn *txn, pgno_t pgno) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); dpl_t *dl = txn->tw.dirtylist; size_t i = dpl_search(txn, pgno); @@ -116,13 +108,11 @@ static inline void dpl_remove(const MDBX_txn *txn, size_t i) { dpl_remove_ex(txn, i, dpl_npages(txn->tw.dirtylist, i)); } -MDBX_INTERNAL int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, - page_t *page, size_t npages); +MDBX_INTERNAL int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, size_t npages); MDBX_MAYBE_UNUSED MDBX_INTERNAL bool dpl_check(MDBX_txn *txn); -MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t dpl_age(const MDBX_txn *txn, - size_t i) { +MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t dpl_age(const MDBX_txn *txn, size_t i) { tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); const dpl_t *dl = txn->tw.dirtylist; assert((intptr_t)i > 0 && i <= dl->length); @@ -134,8 +124,7 @@ MDBX_INTERNAL void dpl_lru_reduce(MDBX_txn *txn); static inline uint32_t dpl_lru_turn(MDBX_txn *txn) { txn->tw.dirtylru += 1; - if (unlikely(txn->tw.dirtylru > UINT32_MAX / 3) && - (txn->flags & MDBX_WRITEMAP) == 0) + if (unlikely(txn->tw.dirtylru > UINT32_MAX / 3) && (txn->flags & MDBX_WRITEMAP) == 0) dpl_lru_reduce(txn); return txn->tw.dirtylru; } diff --git a/src/dxb.c b/src/dxb.c index 8d50895c..b85258bb 100644 --- a/src/dxb.c +++ b/src/dxb.c @@ -3,8 +3,7 @@ #include "internals.h" -__cold int dxb_read_header(MDBX_env *env, meta_t *dest, const int lck_exclusive, - const mdbx_mode_t mode_bits) { +__cold int dxb_read_header(MDBX_env *env, meta_t *dest, const int lck_exclusive, const mdbx_mode_t mode_bits) { memset(dest, 0, sizeof(meta_t)); int rc = osal_filesize(env->lazy_fd, &env->dxb_mmap.filesize); if (unlikely(rc != MDBX_SUCCESS)) @@ -19,20 +18,17 @@ __cold int dxb_read_header(MDBX_env *env, meta_t *dest, const int lck_exclusive, unsigned guess_pagesize = 0; for (unsigned loop_count = 0; loop_count < loop_limit; ++loop_count) { const unsigned meta_number = loop_count % NUM_METAS; - const unsigned offset = - (guess_pagesize ? guess_pagesize - : (loop_count > NUM_METAS) ? env->ps - : globals.sys_pagesize) * - meta_number; + const unsigned offset = (guess_pagesize ? guess_pagesize + : (loop_count > NUM_METAS) ? env->ps + : globals.sys_pagesize) * + meta_number; char buffer[MDBX_MIN_PAGESIZE]; unsigned retryleft = 42; while (1) { - TRACE("reading meta[%d]: offset %u, bytes %u, retry-left %u", meta_number, - offset, MDBX_MIN_PAGESIZE, retryleft); + TRACE("reading meta[%d]: offset %u, bytes %u, retry-left %u", meta_number, offset, MDBX_MIN_PAGESIZE, retryleft); int err = osal_pread(env->lazy_fd, buffer, MDBX_MIN_PAGESIZE, offset); - if (err == MDBX_ENODATA && offset == 0 && loop_count == 0 && - env->dxb_mmap.filesize == 0 && + if (err == MDBX_ENODATA && offset == 0 && loop_count == 0 && env->dxb_mmap.filesize == 0 && mode_bits /* non-zero for DB creation */ != 0) { NOTICE("read meta: empty file (%d, %s)", err, mdbx_strerror(err)); return err; @@ -42,15 +38,13 @@ __cold int dxb_read_header(MDBX_env *env, meta_t *dest, const int lck_exclusive, SleepEx(0, true); err = osal_pread(env->lazy_fd, buffer, MDBX_MIN_PAGESIZE, offset); if (err == ERROR_LOCK_VIOLATION && --retryleft) { - WARNING("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, - mdbx_strerror(err)); + WARNING("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, mdbx_strerror(err)); continue; } } #endif /* Windows */ if (err != MDBX_SUCCESS) { - ERROR("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, - mdbx_strerror(err)); + ERROR("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, mdbx_strerror(err)); return err; } @@ -61,15 +55,13 @@ __cold int dxb_read_header(MDBX_env *env, meta_t *dest, const int lck_exclusive, SleepEx(0, true); err = osal_pread(env->lazy_fd, again, MDBX_MIN_PAGESIZE, offset); if (err == ERROR_LOCK_VIOLATION && --retryleft) { - WARNING("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, - mdbx_strerror(err)); + WARNING("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, mdbx_strerror(err)); continue; } } #endif /* Windows */ if (err != MDBX_SUCCESS) { - ERROR("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, - mdbx_strerror(err)); + ERROR("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, mdbx_strerror(err)); return err; } @@ -94,13 +86,11 @@ __cold int dxb_read_header(MDBX_env *env, meta_t *dest, const int lck_exclusive, if (env->stuck_meta >= 0) latch = (meta_number == (unsigned)env->stuck_meta); else if (meta_bootid_match(meta)) - latch = meta_choice_recent( - meta->unsafe_txnid, SIGN_IS_STEADY(meta->unsafe_sign), - dest->unsafe_txnid, SIGN_IS_STEADY(dest->unsafe_sign)); + latch = meta_choice_recent(meta->unsafe_txnid, SIGN_IS_STEADY(meta->unsafe_sign), dest->unsafe_txnid, + SIGN_IS_STEADY(dest->unsafe_sign)); else - latch = meta_choice_steady( - meta->unsafe_txnid, SIGN_IS_STEADY(meta->unsafe_sign), - dest->unsafe_txnid, SIGN_IS_STEADY(dest->unsafe_sign)); + latch = meta_choice_steady(meta->unsafe_txnid, SIGN_IS_STEADY(meta->unsafe_sign), dest->unsafe_txnid, + SIGN_IS_STEADY(dest->unsafe_sign)); if (latch) { *dest = *meta; if (!lck_exclusive && !meta_is_steady(dest)) @@ -110,9 +100,7 @@ __cold int dxb_read_header(MDBX_env *env, meta_t *dest, const int lck_exclusive, } if (dest->pagesize == 0 || - (env->stuck_meta < 0 && - !(meta_is_steady(dest) || - meta_weak_acceptable(env, dest, lck_exclusive)))) { + (env->stuck_meta < 0 && !(meta_is_steady(dest) || meta_weak_acceptable(env, dest, lck_exclusive)))) { ERROR("%s", "no usable meta-pages, database is corrupted"); if (rc == MDBX_SUCCESS) { /* TODO: try to restore the database by fully checking b-tree structure @@ -125,8 +113,7 @@ __cold int dxb_read_header(MDBX_env *env, meta_t *dest, const int lck_exclusive, return MDBX_SUCCESS; } -__cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, - const pgno_t size_pgno, pgno_t limit_pgno, +__cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, const pgno_t size_pgno, pgno_t limit_pgno, const enum resize_mode mode) { /* Acquire guard to avoid collision between read and write txns * around geo_in_bytes and dxb_mmap */ @@ -164,14 +151,11 @@ __cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, eASSERT(env, bytes2pgno(env, size_bytes) >= size_pgno); eASSERT(env, bytes2pgno(env, limit_bytes) >= limit_pgno); - unsigned mresize_flags = - env->flags & (MDBX_RDONLY | MDBX_WRITEMAP | MDBX_UTTERLY_NOSYNC); + unsigned mresize_flags = env->flags & (MDBX_RDONLY | MDBX_WRITEMAP | MDBX_UTTERLY_NOSYNC); if (mode >= impilict_shrink) mresize_flags |= txn_shrink_allowed; - if (limit_bytes == env->dxb_mmap.limit && - size_bytes == env->dxb_mmap.current && - size_bytes == env->dxb_mmap.filesize) + if (limit_bytes == env->dxb_mmap.limit && size_bytes == env->dxb_mmap.current && size_bytes == env->dxb_mmap.filesize) goto bailout; /* При использовании MDBX_NOSTICKYTHREADS с транзакциями могут работать любые @@ -179,8 +163,7 @@ __cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, * выполнить remap-действия требующие приостановки работающих с БД потоков. */ if ((env->flags & MDBX_NOSTICKYTHREADS) == 0) { #if defined(_WIN32) || defined(_WIN64) - if ((size_bytes < env->dxb_mmap.current && mode > implicit_grow) || - limit_bytes != env->dxb_mmap.limit) { + if ((size_bytes < env->dxb_mmap.current && mode > implicit_grow) || limit_bytes != env->dxb_mmap.limit) { /* 1) Windows allows only extending a read-write section, but not a * corresponding mapped view. Therefore in other cases we must suspend * the local threads for safe remap. @@ -198,9 +181,8 @@ __cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, ERROR("failed suspend-for-remap: errcode %d", rc); goto bailout; } - mresize_flags |= (mode < explicit_resize) - ? MDBX_MRESIZE_MAY_UNMAP - : MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE; + mresize_flags |= + (mode < explicit_resize) ? MDBX_MRESIZE_MAY_UNMAP : MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE; } #else /* Windows */ lck_t *const lck = env->lck_mmap.lck; @@ -214,12 +196,10 @@ __cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, } /* looking for readers from this process */ - const size_t snap_nreaders = - atomic_load32(&lck->rdt_length, mo_AcquireRelease); + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); eASSERT(env, mode == explicit_resize); for (size_t i = 0; i < snap_nreaders; ++i) { - if (lck->rdt[i].pid.weak == env->pid && - lck->rdt[i].tid.weak != osal_thread_self()) { + if (lck->rdt[i].pid.weak == env->pid && lck->rdt[i].tid.weak != osal_thread_self()) { /* the base address of the mapping can't be changed since * the other reader thread from this process exists. */ lck_rdt_unlock(env); @@ -233,17 +213,14 @@ __cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, } const pgno_t aligned_munlock_pgno = - (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) - ? 0 - : bytes2pgno(env, size_bytes); + (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) ? 0 : bytes2pgno(env, size_bytes); if (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) { mincore_clean_cache(env); if ((env->flags & MDBX_WRITEMAP) && env->lck->unsynced_pages.weak) { #if MDBX_ENABLE_PGOP_STAT env->lck->pgops.msync.weak += 1; #endif /* MDBX_ENABLE_PGOP_STAT */ - rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, used_pgno), - MDBX_SYNC_NONE); + rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, used_pgno), MDBX_SYNC_NONE); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; } @@ -251,51 +228,40 @@ __cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, munlock_after(env, aligned_munlock_pgno, size_bytes); if (size_bytes < prev_size && mode > implicit_grow) { - NOTICE("resize-MADV_%s %u..%u", - (env->flags & MDBX_WRITEMAP) ? "REMOVE" : "DONTNEED", size_pgno, + NOTICE("resize-MADV_%s %u..%u", (env->flags & MDBX_WRITEMAP) ? "REMOVE" : "DONTNEED", size_pgno, bytes2pgno(env, prev_size)); - const uint32_t munlocks_before = - atomic_load32(&env->lck->mlcnt[1], mo_Relaxed); + const uint32_t munlocks_before = atomic_load32(&env->lck->mlcnt[1], mo_Relaxed); rc = MDBX_RESULT_TRUE; #if defined(MADV_REMOVE) if (env->flags & MDBX_WRITEMAP) - rc = madvise(ptr_disp(env->dxb_mmap.base, size_bytes), - prev_size - size_bytes, MADV_REMOVE) - ? ignore_enosys(errno) - : MDBX_SUCCESS; + rc = madvise(ptr_disp(env->dxb_mmap.base, size_bytes), prev_size - size_bytes, MADV_REMOVE) ? ignore_enosys(errno) + : MDBX_SUCCESS; #endif /* MADV_REMOVE */ #if defined(MADV_DONTNEED) if (rc == MDBX_RESULT_TRUE) - rc = madvise(ptr_disp(env->dxb_mmap.base, size_bytes), - prev_size - size_bytes, MADV_DONTNEED) + rc = madvise(ptr_disp(env->dxb_mmap.base, size_bytes), prev_size - size_bytes, MADV_DONTNEED) ? ignore_enosys(errno) : MDBX_SUCCESS; #elif defined(POSIX_MADV_DONTNEED) if (rc == MDBX_RESULT_TRUE) - rc = ignore_enosys(posix_madvise(ptr_disp(env->dxb_mmap.base, size_bytes), - prev_size - size_bytes, - POSIX_MADV_DONTNEED)); + rc = ignore_enosys( + posix_madvise(ptr_disp(env->dxb_mmap.base, size_bytes), prev_size - size_bytes, POSIX_MADV_DONTNEED)); #elif defined(POSIX_FADV_DONTNEED) if (rc == MDBX_RESULT_TRUE) - rc = ignore_enosys(posix_fadvise(env->lazy_fd, size_bytes, - prev_size - size_bytes, - POSIX_FADV_DONTNEED)); + rc = ignore_enosys(posix_fadvise(env->lazy_fd, size_bytes, prev_size - size_bytes, POSIX_FADV_DONTNEED)); #endif /* MADV_DONTNEED */ if (unlikely(MDBX_IS_ERROR(rc))) { - const uint32_t mlocks_after = - atomic_load32(&env->lck->mlcnt[0], mo_Relaxed); + const uint32_t mlocks_after = atomic_load32(&env->lck->mlcnt[0], mo_Relaxed); if (rc == MDBX_EINVAL) { - const int severity = - (mlocks_after - munlocks_before) ? MDBX_LOG_NOTICE : MDBX_LOG_WARN; + const int severity = (mlocks_after - munlocks_before) ? MDBX_LOG_NOTICE : MDBX_LOG_WARN; if (LOG_ENABLED(severity)) debug_log(severity, __func__, __LINE__, "%s-madvise: ignore EINVAL (%d) since some pages maybe " "locked (%u/%u mlcnt-processes)", "resize", rc, mlocks_after, munlocks_before); } else { - ERROR("%s-madvise(%s, %zu, +%zu), %u/%u mlcnt-processes, err %d", - "mresize", "DONTNEED", size_bytes, prev_size - size_bytes, - mlocks_after, munlocks_before, rc); + ERROR("%s-madvise(%s, %zu, +%zu), %u/%u mlcnt-processes, err %d", "mresize", "DONTNEED", size_bytes, + prev_size - size_bytes, mlocks_after, munlocks_before, rc); goto bailout; } } else @@ -314,10 +280,8 @@ __cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, eASSERT(env, size_bytes <= env->dxb_mmap.current); env->lck->discarded_tail.weak = size_pgno; const bool readahead = - !(env->flags & MDBX_NORDAHEAD) && - mdbx_is_readahead_reasonable(size_bytes, -(intptr_t)prev_size); - const bool force = limit_bytes != prev_limit || - env->dxb_mmap.base != prev_map + !(env->flags & MDBX_NORDAHEAD) && mdbx_is_readahead_reasonable(size_bytes, -(intptr_t)prev_size); + const bool force = limit_bytes != prev_limit || env->dxb_mmap.base != prev_map #if defined(_WIN32) || defined(_WIN64) || prev_size > size_bytes #endif /* Windows */ @@ -343,8 +307,7 @@ bailout: VALGRIND_DISCARD(env->valgrind_handle); env->valgrind_handle = 0; if (env->dxb_mmap.limit) - env->valgrind_handle = VALGRIND_CREATE_BLOCK( - env->dxb_mmap.base, env->dxb_mmap.limit, "mdbx"); + env->valgrind_handle = VALGRIND_CREATE_BLOCK(env->dxb_mmap.base, env->dxb_mmap.limit, "mdbx"); } #endif /* ENABLE_MEMCHECK */ } else { @@ -377,8 +340,7 @@ bailout: osal_free(suspended); } #else - if (env->lck_mmap.lck && - (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) != 0) + if (env->lck_mmap.lck && (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) != 0) lck_rdt_unlock(env); int err = osal_fastmutex_release(&env->remap_guard); #endif /* Windows */ @@ -397,10 +359,8 @@ void dxb_sanitize_tail(MDBX_env *env, MDBX_txn *txn) { if (txn) { /* transaction start */ if (env->poison_edge < txn->geo.first_unallocated) env->poison_edge = txn->geo.first_unallocated; - VALGRIND_MAKE_MEM_DEFINED(env->dxb_mmap.base, - pgno2bytes(env, txn->geo.first_unallocated)); - MDBX_ASAN_UNPOISON_MEMORY_REGION( - env->dxb_mmap.base, pgno2bytes(env, txn->geo.first_unallocated)); + VALGRIND_MAKE_MEM_DEFINED(env->dxb_mmap.base, pgno2bytes(env, txn->geo.first_unallocated)); + MDBX_ASAN_UNPOISON_MEMORY_REGION(env->dxb_mmap.base, pgno2bytes(env, txn->geo.first_unallocated)); /* don't touch more, it should be already poisoned */ } else { /* transaction end */ bool should_unlock = false; @@ -410,8 +370,7 @@ void dxb_sanitize_tail(MDBX_env *env, MDBX_txn *txn) { return; } else if (env->txn && env_txn0_owned(env)) { /* inside write-txn */ - last = meta_recent(env, &env->basal_txn->tw.troika) - .ptr_v->geometry.first_unallocated; + last = meta_recent(env, &env->basal_txn->tw.troika).ptr_v->geometry.first_unallocated; } else if (env->flags & MDBX_RDONLY) { /* read-only mode, no write-txn, no wlock mutex */ last = NUM_METAS; @@ -429,12 +388,8 @@ void dxb_sanitize_tail(MDBX_env *env, MDBX_txn *txn) { if (edge > last) { eASSERT(env, last >= NUM_METAS); env->poison_edge = last; - VALGRIND_MAKE_MEM_NOACCESS( - ptr_disp(env->dxb_mmap.base, pgno2bytes(env, last)), - pgno2bytes(env, edge - last)); - MDBX_ASAN_POISON_MEMORY_REGION( - ptr_disp(env->dxb_mmap.base, pgno2bytes(env, last)), - pgno2bytes(env, edge - last)); + VALGRIND_MAKE_MEM_NOACCESS(ptr_disp(env->dxb_mmap.base, pgno2bytes(env, last)), pgno2bytes(env, edge - last)); + MDBX_ASAN_POISON_MEMORY_REGION(ptr_disp(env->dxb_mmap.base, pgno2bytes(env, last)), pgno2bytes(env, edge - last)); } if (should_unlock) lck_txn_unlock(env); @@ -443,22 +398,16 @@ void dxb_sanitize_tail(MDBX_env *env, MDBX_txn *txn) { #endif /* ENABLE_MEMCHECK || __SANITIZE_ADDRESS__ */ /* Turn on/off readahead. It's harmful when the DB is larger than RAM. */ -__cold int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, - const bool enable, const bool force_whole) { +__cold int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, const bool enable, const bool force_whole) { eASSERT(env, edge >= NUM_METAS && edge <= MAX_PAGENO + 1); eASSERT(env, (enable & 1) == (enable != 0)); - const bool toggle = force_whole || - ((enable ^ env->lck->readahead_anchor) & 1) || - !env->lck->readahead_anchor; + const bool toggle = force_whole || ((enable ^ env->lck->readahead_anchor) & 1) || !env->lck->readahead_anchor; const pgno_t prev_edge = env->lck->readahead_anchor >> 1; const size_t limit = env->dxb_mmap.limit; - size_t offset = - toggle ? 0 - : pgno_align2os_bytes(env, (prev_edge < edge) ? prev_edge : edge); + size_t offset = toggle ? 0 : pgno_align2os_bytes(env, (prev_edge < edge) ? prev_edge : edge); offset = (offset < limit) ? offset : limit; - size_t length = - pgno_align2os_bytes(env, (prev_edge < edge) ? edge : prev_edge); + size_t length = pgno_align2os_bytes(env, (prev_edge < edge) ? edge : prev_edge); length = (length < limit) ? length : limit; length -= offset; @@ -466,8 +415,7 @@ __cold int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, if (length == 0) return MDBX_SUCCESS; - NOTICE("readahead %s %u..%u", enable ? "ON" : "OFF", bytes2pgno(env, offset), - bytes2pgno(env, offset + length)); + NOTICE("readahead %s %u..%u", enable ? "ON" : "OFF", bytes2pgno(env, offset), bytes2pgno(env, offset + length)); #if defined(F_RDAHEAD) if (toggle && unlikely(fcntl(env->lazy_fd, F_RDAHEAD, enable) == -1)) @@ -478,8 +426,7 @@ __cold int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, void *const ptr = ptr_disp(env->dxb_mmap.base, offset); if (enable) { #if defined(MADV_NORMAL) - err = - madvise(ptr, length, MADV_NORMAL) ? ignore_enosys(errno) : MDBX_SUCCESS; + err = madvise(ptr, length, MADV_NORMAL) ? ignore_enosys(errno) : MDBX_SUCCESS; if (unlikely(MDBX_IS_ERROR(err))) return err; #elif defined(POSIX_MADV_NORMAL) @@ -487,8 +434,7 @@ __cold int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, if (unlikely(MDBX_IS_ERROR(err))) return err; #elif defined(POSIX_FADV_NORMAL) && defined(POSIX_FADV_WILLNEED) - err = ignore_enosys( - posix_fadvise(env->lazy_fd, offset, length, POSIX_FADV_NORMAL)); + err = ignore_enosys(posix_fadvise(env->lazy_fd, offset, length, POSIX_FADV_NORMAL)); if (unlikely(MDBX_IS_ERROR(err))) return err; #elif defined(_WIN32) || defined(_WIN64) @@ -505,15 +451,10 @@ __cold int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, #if defined(F_RDADVISE) struct radvisory hint; hint.ra_offset = offset; - hint.ra_count = - unlikely(length > INT_MAX && sizeof(length) > sizeof(hint.ra_count)) - ? INT_MAX - : (int)length; - (void)/* Ignore ENOTTY for DB on the ram-disk and so on */ fcntl( - env->lazy_fd, F_RDADVISE, &hint); + hint.ra_count = unlikely(length > INT_MAX && sizeof(length) > sizeof(hint.ra_count)) ? INT_MAX : (int)length; + (void)/* Ignore ENOTTY for DB on the ram-disk and so on */ fcntl(env->lazy_fd, F_RDADVISE, &hint); #elif defined(MADV_WILLNEED) - err = madvise(ptr, length, MADV_WILLNEED) ? ignore_enosys(errno) - : MDBX_SUCCESS; + err = madvise(ptr, length, MADV_WILLNEED) ? ignore_enosys(errno) : MDBX_SUCCESS; if (unlikely(MDBX_IS_ERROR(err))) return err; #elif defined(POSIX_MADV_WILLNEED) @@ -528,8 +469,7 @@ __cold int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, (void)imports.PrefetchVirtualMemory(GetCurrentProcess(), 1, &hint, 0); } #elif defined(POSIX_FADV_WILLNEED) - err = ignore_enosys( - posix_fadvise(env->lazy_fd, offset, length, POSIX_FADV_WILLNEED)); + err = ignore_enosys(posix_fadvise(env->lazy_fd, offset, length, POSIX_FADV_WILLNEED)); if (unlikely(MDBX_IS_ERROR(err))) return err; #else @@ -539,8 +479,7 @@ __cold int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, } else { mincore_clean_cache(env); #if defined(MADV_RANDOM) - err = - madvise(ptr, length, MADV_RANDOM) ? ignore_enosys(errno) : MDBX_SUCCESS; + err = madvise(ptr, length, MADV_RANDOM) ? ignore_enosys(errno) : MDBX_SUCCESS; if (unlikely(MDBX_IS_ERROR(err))) return err; #elif defined(POSIX_MADV_RANDOM) @@ -548,8 +487,7 @@ __cold int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, if (unlikely(MDBX_IS_ERROR(err))) return err; #elif defined(POSIX_FADV_RANDOM) - err = ignore_enosys( - posix_fadvise(env->lazy_fd, offset, length, POSIX_FADV_RANDOM)); + err = ignore_enosys(posix_fadvise(env->lazy_fd, offset, length, POSIX_FADV_RANDOM)); if (unlikely(MDBX_IS_ERROR(err))) return err; #elif defined(_WIN32) || defined(_WIN64) @@ -564,15 +502,13 @@ __cold int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, return err; } -__cold int dxb_setup(MDBX_env *env, const int lck_rc, - const mdbx_mode_t mode_bits) { +__cold int dxb_setup(MDBX_env *env, const int lck_rc, const mdbx_mode_t mode_bits) { meta_t header; eASSERT(env, !(env->flags & ENV_ACTIVE)); int rc = MDBX_RESULT_FALSE; int err = dxb_read_header(env, &header, lck_rc, mode_bits); if (unlikely(err != MDBX_SUCCESS)) { - if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE || err != MDBX_ENODATA || - (env->flags & MDBX_RDONLY) != 0 || + if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE || err != MDBX_ENODATA || (env->flags & MDBX_RDONLY) != 0 || /* recovery mode */ env->stuck_meta >= 0) return err; @@ -591,14 +527,11 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, return err; header = *meta_init_triplet(env, env->page_auxbuf); - err = osal_pwrite(env->lazy_fd, env->page_auxbuf, - env->ps * (size_t)NUM_METAS, 0); + err = osal_pwrite(env->lazy_fd, env->page_auxbuf, env->ps * (size_t)NUM_METAS, 0); if (unlikely(err != MDBX_SUCCESS)) return err; - err = osal_ftruncate(env->lazy_fd, env->dxb_mmap.filesize = - env->dxb_mmap.current = - env->geo_in_bytes.now); + err = osal_ftruncate(env->lazy_fd, env->dxb_mmap.filesize = env->dxb_mmap.current = env->geo_in_bytes.now); if (unlikely(err != MDBX_SUCCESS)) return err; @@ -609,19 +542,14 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, #endif } - VERBOSE("header: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO - "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO " +%u -%u, txn_id %" PRIaTXN - ", %s", - header.trees.main.root, header.trees.gc.root, header.geometry.lower, - header.geometry.first_unallocated, header.geometry.now, - header.geometry.upper, pv2pages(header.geometry.grow_pv), - pv2pages(header.geometry.shrink_pv), - unaligned_peek_u64(4, header.txnid_a), durable_caption(&header)); + VERBOSE("header: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO + " +%u -%u, txn_id %" PRIaTXN ", %s", + header.trees.main.root, header.trees.gc.root, header.geometry.lower, header.geometry.first_unallocated, + header.geometry.now, header.geometry.upper, pv2pages(header.geometry.grow_pv), + pv2pages(header.geometry.shrink_pv), unaligned_peek_u64(4, header.txnid_a), durable_caption(&header)); - if (unlikely((header.trees.gc.flags & DB_PERSISTENT_FLAGS) != - MDBX_INTEGERKEY)) { - ERROR("unexpected/invalid db-flags 0x%x for %s", header.trees.gc.flags, - "GC/FreeDB"); + if (unlikely((header.trees.gc.flags & DB_PERSISTENT_FLAGS) != MDBX_INTEGERKEY)) { + ERROR("unexpected/invalid db-flags 0x%x for %s", header.trees.gc.flags, "GC/FreeDB"); return MDBX_INCOMPATIBLE; } env->dbs_flags[FREE_DBI] = DB_VALID | MDBX_INTEGERKEY; @@ -629,8 +557,7 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, env->kvs[FREE_DBI].clc.k.lmax = env->kvs[FREE_DBI].clc.k.lmin = 8; env->kvs[FREE_DBI].clc.v.cmp = cmp_lenfast; env->kvs[FREE_DBI].clc.v.lmin = 4; - env->kvs[FREE_DBI].clc.v.lmax = - mdbx_env_get_maxvalsize_ex(env, MDBX_INTEGERKEY); + env->kvs[FREE_DBI].clc.v.lmax = mdbx_env_get_maxvalsize_ex(env, MDBX_INTEGERKEY); if (env->ps != header.pagesize) env_setup_pagesize(env, header.pagesize); @@ -641,18 +568,15 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, } const size_t used_bytes = pgno2bytes(env, header.geometry.first_unallocated); - const size_t used_aligned2os_bytes = - ceil_powerof2(used_bytes, globals.sys_pagesize); + const size_t used_aligned2os_bytes = ceil_powerof2(used_bytes, globals.sys_pagesize); if ((env->flags & MDBX_RDONLY) /* readonly */ || lck_rc != MDBX_RESULT_TRUE /* not exclusive */ || /* recovery mode */ env->stuck_meta >= 0) { /* use present params from db */ const size_t pagesize = header.pagesize; - err = mdbx_env_set_geometry( - env, header.geometry.lower * pagesize, header.geometry.now * pagesize, - header.geometry.upper * pagesize, - pv2pages(header.geometry.grow_pv) * pagesize, - pv2pages(header.geometry.shrink_pv) * pagesize, header.pagesize); + err = mdbx_env_set_geometry(env, header.geometry.lower * pagesize, header.geometry.now * pagesize, + header.geometry.upper * pagesize, pv2pages(header.geometry.grow_pv) * pagesize, + pv2pages(header.geometry.shrink_pv) * pagesize, header.pagesize); if (unlikely(err != MDBX_SUCCESS)) { ERROR("%s: err %d", "could not apply geometry from db", err); return (err == MDBX_EINVAL) ? MDBX_INCOMPATIBLE : err; @@ -668,24 +592,17 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, * - upper or lower limit changes * - shrink threshold or growth step * But ignore change just a 'now/current' size. */ - if (bytes_align2os_bytes(env, env->geo_in_bytes.upper) != - pgno2bytes(env, header.geometry.upper) || - bytes_align2os_bytes(env, env->geo_in_bytes.lower) != - pgno2bytes(env, header.geometry.lower) || - bytes_align2os_bytes(env, env->geo_in_bytes.shrink) != - pgno2bytes(env, pv2pages(header.geometry.shrink_pv)) || - bytes_align2os_bytes(env, env->geo_in_bytes.grow) != - pgno2bytes(env, pv2pages(header.geometry.grow_pv))) { + if (bytes_align2os_bytes(env, env->geo_in_bytes.upper) != pgno2bytes(env, header.geometry.upper) || + bytes_align2os_bytes(env, env->geo_in_bytes.lower) != pgno2bytes(env, header.geometry.lower) || + bytes_align2os_bytes(env, env->geo_in_bytes.shrink) != pgno2bytes(env, pv2pages(header.geometry.shrink_pv)) || + bytes_align2os_bytes(env, env->geo_in_bytes.grow) != pgno2bytes(env, pv2pages(header.geometry.grow_pv))) { if (env->geo_in_bytes.shrink && env->geo_in_bytes.now > used_bytes) /* pre-shrink if enabled */ - env->geo_in_bytes.now = used_bytes + env->geo_in_bytes.shrink - - used_bytes % env->geo_in_bytes.shrink; + env->geo_in_bytes.now = used_bytes + env->geo_in_bytes.shrink - used_bytes % env->geo_in_bytes.shrink; - err = mdbx_env_set_geometry( - env, env->geo_in_bytes.lower, env->geo_in_bytes.now, - env->geo_in_bytes.upper, env->geo_in_bytes.grow, - env->geo_in_bytes.shrink, header.pagesize); + err = mdbx_env_set_geometry(env, env->geo_in_bytes.lower, env->geo_in_bytes.now, env->geo_in_bytes.upper, + env->geo_in_bytes.grow, env->geo_in_bytes.shrink, header.pagesize); if (unlikely(err != MDBX_SUCCESS)) { ERROR("%s: err %d", "could not apply preconfigured db-geometry", err); return (err == MDBX_EINVAL) ? MDBX_INCOMPATIBLE : err; @@ -695,20 +612,14 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, header.geometry.now = bytes2pgno(env, env->geo_in_bytes.now); header.geometry.lower = bytes2pgno(env, env->geo_in_bytes.lower); header.geometry.upper = bytes2pgno(env, env->geo_in_bytes.upper); - header.geometry.grow_pv = - pages2pv(bytes2pgno(env, env->geo_in_bytes.grow)); - header.geometry.shrink_pv = - pages2pv(bytes2pgno(env, env->geo_in_bytes.shrink)); + header.geometry.grow_pv = pages2pv(bytes2pgno(env, env->geo_in_bytes.grow)); + header.geometry.shrink_pv = pages2pv(bytes2pgno(env, env->geo_in_bytes.shrink)); - VERBOSE("amended: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO - "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO + VERBOSE("amended: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO " +%u -%u, txn_id %" PRIaTXN ", %s", - header.trees.main.root, header.trees.gc.root, - header.geometry.lower, header.geometry.first_unallocated, - header.geometry.now, header.geometry.upper, - pv2pages(header.geometry.grow_pv), - pv2pages(header.geometry.shrink_pv), - unaligned_peek_u64(4, header.txnid_a), durable_caption(&header)); + header.trees.main.root, header.trees.gc.root, header.geometry.lower, header.geometry.first_unallocated, + header.geometry.now, header.geometry.upper, pv2pages(header.geometry.grow_pv), + pv2pages(header.geometry.shrink_pv), unaligned_peek_u64(4, header.txnid_a), durable_caption(&header)); } else { /* fetch back 'now/current' size, since it was ignored during comparison * and may differ. */ @@ -722,31 +633,25 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, env->geo_in_bytes.lower = pgno2bytes(env, header.geometry.lower); env->geo_in_bytes.upper = pgno2bytes(env, header.geometry.upper); env->geo_in_bytes.grow = pgno2bytes(env, pv2pages(header.geometry.grow_pv)); - env->geo_in_bytes.shrink = - pgno2bytes(env, pv2pages(header.geometry.shrink_pv)); + env->geo_in_bytes.shrink = pgno2bytes(env, pv2pages(header.geometry.shrink_pv)); } - ENSURE(env, pgno_align2os_bytes(env, header.geometry.now) == - env->geo_in_bytes.now); + ENSURE(env, pgno_align2os_bytes(env, header.geometry.now) == env->geo_in_bytes.now); ENSURE(env, env->geo_in_bytes.now >= used_bytes); const uint64_t filesize_before = env->dxb_mmap.filesize; if (unlikely(filesize_before != env->geo_in_bytes.now)) { if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE) { - VERBOSE("filesize mismatch (expect %" PRIuPTR "b/%" PRIaPGNO - "p, have %" PRIu64 "b/%" PRIaPGNO "p), " + VERBOSE("filesize mismatch (expect %" PRIuPTR "b/%" PRIaPGNO "p, have %" PRIu64 "b/%" PRIaPGNO "p), " "assume other process working", - env->geo_in_bytes.now, bytes2pgno(env, env->geo_in_bytes.now), - filesize_before, bytes2pgno(env, (size_t)filesize_before)); - } else { - WARNING("filesize mismatch (expect %" PRIuSIZE "b/%" PRIaPGNO - "p, have %" PRIu64 "b/%" PRIaPGNO "p)", - env->geo_in_bytes.now, bytes2pgno(env, env->geo_in_bytes.now), - filesize_before, bytes2pgno(env, (size_t)filesize_before)); - if (filesize_before < used_bytes) { - ERROR("last-page beyond end-of-file (last %" PRIaPGNO - ", have %" PRIaPGNO ")", - header.geometry.first_unallocated, + env->geo_in_bytes.now, bytes2pgno(env, env->geo_in_bytes.now), filesize_before, bytes2pgno(env, (size_t)filesize_before)); + } else { + WARNING("filesize mismatch (expect %" PRIuSIZE "b/%" PRIaPGNO "p, have %" PRIu64 "b/%" PRIaPGNO "p)", + env->geo_in_bytes.now, bytes2pgno(env, env->geo_in_bytes.now), filesize_before, + bytes2pgno(env, (size_t)filesize_before)); + if (filesize_before < used_bytes) { + ERROR("last-page beyond end-of-file (last %" PRIaPGNO ", have %" PRIaPGNO ")", + header.geometry.first_unallocated, bytes2pgno(env, (size_t)filesize_before)); return MDBX_CORRUPTED; } @@ -757,65 +662,50 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, } WARNING("%s", "ignore filesize mismatch in readonly-mode"); } else { - VERBOSE("will resize datafile to %" PRIuSIZE " bytes, %" PRIaPGNO - " pages", - env->geo_in_bytes.now, bytes2pgno(env, env->geo_in_bytes.now)); + VERBOSE("will resize datafile to %" PRIuSIZE " bytes, %" PRIaPGNO " pages", env->geo_in_bytes.now, + bytes2pgno(env, env->geo_in_bytes.now)); } } } - VERBOSE("current boot-id %" PRIx64 "-%" PRIx64 " (%savailable)", - globals.bootid.x, globals.bootid.y, + VERBOSE("current boot-id %" PRIx64 "-%" PRIx64 " (%savailable)", globals.bootid.x, globals.bootid.y, (globals.bootid.x | globals.bootid.y) ? "" : "not-"); /* calculate readahead hint before mmap with zero redundant pages */ const bool readahead = - !(env->flags & MDBX_NORDAHEAD) && - mdbx_is_readahead_reasonable(used_bytes, 0) == MDBX_RESULT_TRUE; + !(env->flags & MDBX_NORDAHEAD) && mdbx_is_readahead_reasonable(used_bytes, 0) == MDBX_RESULT_TRUE; - err = osal_mmap(env->flags, &env->dxb_mmap, env->geo_in_bytes.now, - env->geo_in_bytes.upper, + err = osal_mmap(env->flags, &env->dxb_mmap, env->geo_in_bytes.now, env->geo_in_bytes.upper, (lck_rc && env->stuck_meta < 0) ? MMAP_OPTION_TRUNCATE : 0); if (unlikely(err != MDBX_SUCCESS)) return err; #if defined(MADV_DONTDUMP) - err = madvise(env->dxb_mmap.base, env->dxb_mmap.limit, MADV_DONTDUMP) - ? ignore_enosys(errno) - : MDBX_SUCCESS; + err = madvise(env->dxb_mmap.base, env->dxb_mmap.limit, MADV_DONTDUMP) ? ignore_enosys(errno) : MDBX_SUCCESS; if (unlikely(MDBX_IS_ERROR(err))) return err; #endif /* MADV_DONTDUMP */ #if defined(MADV_DODUMP) if (globals.runtime_flags & MDBX_DBG_DUMP) { const size_t meta_length_aligned2os = pgno_align2os_bytes(env, NUM_METAS); - err = madvise(env->dxb_mmap.base, meta_length_aligned2os, MADV_DODUMP) - ? ignore_enosys(errno) - : MDBX_SUCCESS; + err = madvise(env->dxb_mmap.base, meta_length_aligned2os, MADV_DODUMP) ? ignore_enosys(errno) : MDBX_SUCCESS; if (unlikely(MDBX_IS_ERROR(err))) return err; } #endif /* MADV_DODUMP */ #ifdef ENABLE_MEMCHECK - env->valgrind_handle = - VALGRIND_CREATE_BLOCK(env->dxb_mmap.base, env->dxb_mmap.limit, "mdbx"); + env->valgrind_handle = VALGRIND_CREATE_BLOCK(env->dxb_mmap.base, env->dxb_mmap.limit, "mdbx"); #endif /* ENABLE_MEMCHECK */ - eASSERT(env, used_bytes >= pgno2bytes(env, NUM_METAS) && - used_bytes <= env->dxb_mmap.limit); + eASSERT(env, used_bytes >= pgno2bytes(env, NUM_METAS) && used_bytes <= env->dxb_mmap.limit); #if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__) - if (env->dxb_mmap.filesize > used_bytes && - env->dxb_mmap.filesize < env->dxb_mmap.limit) { - VALGRIND_MAKE_MEM_NOACCESS(ptr_disp(env->dxb_mmap.base, used_bytes), - env->dxb_mmap.filesize - used_bytes); - MDBX_ASAN_POISON_MEMORY_REGION(ptr_disp(env->dxb_mmap.base, used_bytes), - env->dxb_mmap.filesize - used_bytes); + if (env->dxb_mmap.filesize > used_bytes && env->dxb_mmap.filesize < env->dxb_mmap.limit) { + VALGRIND_MAKE_MEM_NOACCESS(ptr_disp(env->dxb_mmap.base, used_bytes), env->dxb_mmap.filesize - used_bytes); + MDBX_ASAN_POISON_MEMORY_REGION(ptr_disp(env->dxb_mmap.base, used_bytes), env->dxb_mmap.filesize - used_bytes); } env->poison_edge = - bytes2pgno(env, (env->dxb_mmap.filesize < env->dxb_mmap.limit) - ? env->dxb_mmap.filesize - : env->dxb_mmap.limit); + bytes2pgno(env, (env->dxb_mmap.filesize < env->dxb_mmap.limit) ? env->dxb_mmap.filesize : env->dxb_mmap.limit); #endif /* ENABLE_MEMCHECK || __SANITIZE_ADDRESS__ */ troika_t troika = meta_tap(env); @@ -829,8 +719,7 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, meta_t const *const target = METAPAGE(env, env->stuck_meta); err = meta_validate_copy(env, target, &clone); if (unlikely(err != MDBX_SUCCESS)) { - ERROR("target meta[%u] is corrupted", - bytes2pgno(env, ptr_dist(data_page(target), env->dxb_mmap.base))); + ERROR("target meta[%u] is corrupted", bytes2pgno(env, ptr_dist(data_page(target), env->dxb_mmap.base))); meta_troika_dump(env, &troika); return MDBX_CORRUPTED; } @@ -872,9 +761,8 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, err = meta_validate_copy(env, prefer_steady.ptr_c, &clone); if (unlikely(err != MDBX_SUCCESS)) { ERROR("meta[%u] with %s txnid %" PRIaTXN " is corrupted, %s needed", - bytes2pgno(env, - ptr_dist(prefer_steady.ptr_c, env->dxb_mmap.base)), - "steady", prefer_steady.txnid, "manual recovery"); + bytes2pgno(env, ptr_dist(prefer_steady.ptr_c, env->dxb_mmap.base)), "steady", prefer_steady.txnid, + "manual recovery"); meta_troika_dump(env, &troika); return MDBX_CORRUPTED; } @@ -882,23 +770,17 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, break; } - const pgno_t pgno = - bytes2pgno(env, ptr_dist(recent.ptr_c, env->dxb_mmap.base)); - const bool last_valid = - meta_validate_copy(env, recent.ptr_c, &clone) == MDBX_SUCCESS; - eASSERT(env, - !prefer_steady.is_steady || recent.txnid != prefer_steady.txnid); + const pgno_t pgno = bytes2pgno(env, ptr_dist(recent.ptr_c, env->dxb_mmap.base)); + const bool last_valid = meta_validate_copy(env, recent.ptr_c, &clone) == MDBX_SUCCESS; + eASSERT(env, !prefer_steady.is_steady || recent.txnid != prefer_steady.txnid); if (unlikely(!last_valid)) { if (unlikely(!prefer_steady.is_steady)) { - ERROR("%s for open or automatic rollback, %s", - "there are no suitable meta-pages", + ERROR("%s for open or automatic rollback, %s", "there are no suitable meta-pages", "manual recovery is required"); meta_troika_dump(env, &troika); return MDBX_CORRUPTED; } - WARNING("meta[%u] with last txnid %" PRIaTXN - " is corrupted, rollback needed", - pgno, recent.txnid); + WARNING("meta[%u] with last txnid %" PRIaTXN " is corrupted, rollback needed", pgno, recent.txnid); meta_troika_dump(env, &troika); goto purge_meta_head; } @@ -907,15 +789,14 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, if (env->flags & MDBX_RDONLY) { ERROR("%s, but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH: " "rollback NOT needed, steady-sync NEEDED%s", - "opening after an unclean shutdown", globals.bootid.x, - globals.bootid.y, ", but unable in read-only mode"); + "opening after an unclean shutdown", globals.bootid.x, globals.bootid.y, + ", but unable in read-only mode"); meta_troika_dump(env, &troika); return MDBX_WANNA_RECOVERY; } WARNING("%s, but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH: " "rollback NOT needed, steady-sync NEEDED%s", - "opening after an unclean shutdown", globals.bootid.x, - globals.bootid.y, ""); + "opening after an unclean shutdown", globals.bootid.x, globals.bootid.y, ""); header = clone; env->lck->unsynced_pages.weak = header.geometry.first_unallocated; if (!env->lck->eoos_timestamp.weak) @@ -923,18 +804,14 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, break; } if (unlikely(!prefer_steady.is_steady)) { - ERROR("%s, but %s for automatic rollback: %s", - "opening after an unclean shutdown", - "there are no suitable meta-pages", - "manual recovery is required"); + ERROR("%s, but %s for automatic rollback: %s", "opening after an unclean shutdown", + "there are no suitable meta-pages", "manual recovery is required"); meta_troika_dump(env, &troika); return MDBX_CORRUPTED; } if (env->flags & MDBX_RDONLY) { - ERROR("%s and rollback needed: (from head %" PRIaTXN - " to steady %" PRIaTXN ")%s", - "opening after an unclean shutdown", recent.txnid, - prefer_steady.txnid, ", but unable in read-only mode"); + ERROR("%s and rollback needed: (from head %" PRIaTXN " to steady %" PRIaTXN ")%s", + "opening after an unclean shutdown", recent.txnid, prefer_steady.txnid, ", but unable in read-only mode"); meta_troika_dump(env, &troika); return MDBX_WANNA_RECOVERY; } @@ -942,15 +819,13 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, purge_meta_head: NOTICE("%s and doing automatic rollback: " "purge%s meta[%u] with%s txnid %" PRIaTXN, - "opening after an unclean shutdown", last_valid ? "" : " invalid", - pgno, last_valid ? " weak" : "", recent.txnid); + "opening after an unclean shutdown", last_valid ? "" : " invalid", pgno, last_valid ? " weak" : "", + recent.txnid); meta_troika_dump(env, &troika); ENSURE(env, prefer_steady.is_steady); - err = meta_override(env, pgno, 0, - last_valid ? recent.ptr_c : prefer_steady.ptr_c); + err = meta_override(env, pgno, 0, last_valid ? recent.ptr_c : prefer_steady.ptr_c); if (err) { - ERROR("rollback: overwrite meta[%u] with txnid %" PRIaTXN ", error %d", - pgno, recent.txnid, err); + ERROR("rollback: overwrite meta[%u] with txnid %" PRIaTXN ", error %d", pgno, recent.txnid, err); return err; } troika = meta_tap(env); @@ -961,17 +836,14 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, if (lck_rc == /* lck exclusive */ MDBX_RESULT_TRUE) { //-------------------------------------------------- shrink DB & update geo /* re-check size after mmap */ - if ((env->dxb_mmap.current & (globals.sys_pagesize - 1)) != 0 || - env->dxb_mmap.current < used_bytes) { - ERROR("unacceptable/unexpected datafile size %" PRIuPTR, - env->dxb_mmap.current); + if ((env->dxb_mmap.current & (globals.sys_pagesize - 1)) != 0 || env->dxb_mmap.current < used_bytes) { + ERROR("unacceptable/unexpected datafile size %" PRIuPTR, env->dxb_mmap.current); return MDBX_PROBLEM; } if (env->dxb_mmap.current != env->geo_in_bytes.now) { header.geometry.now = bytes2pgno(env, env->dxb_mmap.current); - NOTICE("need update meta-geo to filesize %" PRIuPTR " bytes, %" PRIaPGNO - " pages", - env->dxb_mmap.current, header.geometry.now); + NOTICE("need update meta-geo to filesize %" PRIuPTR " bytes, %" PRIaPGNO " pages", env->dxb_mmap.current, + header.geometry.now); } const meta_ptr_t recent = meta_recent(env, &troika); @@ -979,21 +851,15 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, header.geometry.grow_pv != recent.ptr_c->geometry.grow_pv || header.geometry.shrink_pv != recent.ptr_c->geometry.shrink_pv || header.geometry.lower != recent.ptr_c->geometry.lower || - header.geometry.upper != recent.ptr_c->geometry.upper || - header.geometry.now != recent.ptr_c->geometry.now) { + header.geometry.upper != recent.ptr_c->geometry.upper || header.geometry.now != recent.ptr_c->geometry.now) { if ((env->flags & MDBX_RDONLY) != 0 || /* recovery mode */ env->stuck_meta >= 0) { - WARNING("skipped update meta.geo in %s mode: from l%" PRIaPGNO - "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u, to l%" PRIaPGNO - "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u", - (env->stuck_meta < 0) ? "read-only" : "recovery", - recent.ptr_c->geometry.lower, recent.ptr_c->geometry.now, - recent.ptr_c->geometry.upper, - pv2pages(recent.ptr_c->geometry.shrink_pv), - pv2pages(recent.ptr_c->geometry.grow_pv), header.geometry.lower, - header.geometry.now, header.geometry.upper, - pv2pages(header.geometry.shrink_pv), - pv2pages(header.geometry.grow_pv)); + WARNING("skipped update meta.geo in %s mode: from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO + "/s%u-g%u, to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u", + (env->stuck_meta < 0) ? "read-only" : "recovery", recent.ptr_c->geometry.lower, + recent.ptr_c->geometry.now, recent.ptr_c->geometry.upper, pv2pages(recent.ptr_c->geometry.shrink_pv), + pv2pages(recent.ptr_c->geometry.grow_pv), header.geometry.lower, header.geometry.now, + header.geometry.upper, pv2pages(header.geometry.shrink_pv), pv2pages(header.geometry.grow_pv)); } else { const txnid_t next_txnid = safe64_txnid_next(recent.txnid); if (unlikely(next_txnid > MAX_TXNID)) { @@ -1001,57 +867,41 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, return MDBX_TXN_FULL; } NOTICE("updating meta.geo: " - "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO - "/s%u-g%u (txn#%" PRIaTXN "), " - "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO - "/s%u-g%u (txn#%" PRIaTXN ")", - recent.ptr_c->geometry.lower, recent.ptr_c->geometry.now, - recent.ptr_c->geometry.upper, - pv2pages(recent.ptr_c->geometry.shrink_pv), - pv2pages(recent.ptr_c->geometry.grow_pv), recent.txnid, - header.geometry.lower, header.geometry.now, - header.geometry.upper, pv2pages(header.geometry.shrink_pv), + "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u (txn#%" PRIaTXN "), " + "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u (txn#%" PRIaTXN ")", + recent.ptr_c->geometry.lower, recent.ptr_c->geometry.now, recent.ptr_c->geometry.upper, + pv2pages(recent.ptr_c->geometry.shrink_pv), pv2pages(recent.ptr_c->geometry.grow_pv), recent.txnid, + header.geometry.lower, header.geometry.now, header.geometry.upper, pv2pages(header.geometry.shrink_pv), pv2pages(header.geometry.grow_pv), next_txnid); ENSURE(env, header.unsafe_txnid == recent.txnid); meta_set_txnid(env, &header, next_txnid); - err = dxb_sync_locked(env, env->flags | txn_shrink_allowed, &header, - &troika); + err = dxb_sync_locked(env, env->flags | txn_shrink_allowed, &header, &troika); if (err) { ERROR("error %d, while updating meta.geo: " - "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO - "/s%u-g%u (txn#%" PRIaTXN "), " - "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO - "/s%u-g%u (txn#%" PRIaTXN ")", - err, recent.ptr_c->geometry.lower, recent.ptr_c->geometry.now, - recent.ptr_c->geometry.upper, - pv2pages(recent.ptr_c->geometry.shrink_pv), - pv2pages(recent.ptr_c->geometry.grow_pv), recent.txnid, - header.geometry.lower, header.geometry.now, - header.geometry.upper, pv2pages(header.geometry.shrink_pv), + "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u (txn#%" PRIaTXN "), " + "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u (txn#%" PRIaTXN ")", + err, recent.ptr_c->geometry.lower, recent.ptr_c->geometry.now, recent.ptr_c->geometry.upper, + pv2pages(recent.ptr_c->geometry.shrink_pv), pv2pages(recent.ptr_c->geometry.grow_pv), recent.txnid, + header.geometry.lower, header.geometry.now, header.geometry.upper, pv2pages(header.geometry.shrink_pv), pv2pages(header.geometry.grow_pv), header.unsafe_txnid); return err; } } } - atomic_store32(&env->lck->discarded_tail, - bytes2pgno(env, used_aligned2os_bytes), mo_Relaxed); + atomic_store32(&env->lck->discarded_tail, bytes2pgno(env, used_aligned2os_bytes), mo_Relaxed); if ((env->flags & MDBX_RDONLY) == 0 && env->stuck_meta < 0 && (globals.runtime_flags & MDBX_DBG_DONT_UPGRADE) == 0) { for (unsigned n = 0; n < NUM_METAS; ++n) { meta_t *const meta = METAPAGE(env, n); - if (unlikely(unaligned_peek_u64(4, &meta->magic_and_version) != - MDBX_DATA_MAGIC) || - (meta->dxbid.x | meta->dxbid.y) == 0 || - (meta->gc_flags & ~DB_PERSISTENT_FLAGS)) { - const txnid_t txnid = - meta_is_used(&troika, n) ? constmeta_txnid(meta) : 0; + if (unlikely(unaligned_peek_u64(4, &meta->magic_and_version) != MDBX_DATA_MAGIC) || + (meta->dxbid.x | meta->dxbid.y) == 0 || (meta->gc_flags & ~DB_PERSISTENT_FLAGS)) { + const txnid_t txnid = meta_is_used(&troika, n) ? constmeta_txnid(meta) : 0; NOTICE("%s %s" "meta[%u], txnid %" PRIaTXN, - "updating db-format/guid signature for", - meta_is_steady(meta) ? "stead-" : "weak-", n, txnid); + "updating db-format/guid signature for", meta_is_steady(meta) ? "stead-" : "weak-", n, txnid); err = meta_override(env, n, txnid, meta); if (unlikely(err != MDBX_SUCCESS) && /* Just ignore the MDBX_PROBLEM error, since here it is @@ -1059,8 +909,7 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, * meta-page that is invalid for current state of a DB, * e.g. after shrinking DB file */ err != MDBX_PROBLEM) { - ERROR("%s meta[%u], txnid %" PRIaTXN ", error %d", - "updating db-format signature for", n, txnid, err); + ERROR("%s meta[%u], txnid %" PRIaTXN ", error %d", "updating db-format signature for", n, txnid, err); return err; } troika = meta_tap(env); @@ -1074,11 +923,10 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, #if defined(MADV_REMOVE) if (lck_rc && (env->flags & MDBX_WRITEMAP) != 0 && /* not recovery mode */ env->stuck_meta < 0) { - NOTICE("open-MADV_%s %u..%u", "REMOVE (deallocate file space)", - env->lck->discarded_tail.weak, + NOTICE("open-MADV_%s %u..%u", "REMOVE (deallocate file space)", env->lck->discarded_tail.weak, bytes2pgno(env, env->dxb_mmap.current)); - err = madvise(ptr_disp(env->dxb_mmap.base, used_aligned2os_bytes), - env->dxb_mmap.current - used_aligned2os_bytes, MADV_REMOVE) + err = madvise(ptr_disp(env->dxb_mmap.base, used_aligned2os_bytes), env->dxb_mmap.current - used_aligned2os_bytes, + MADV_REMOVE) ? ignore_enosys(errno) : MDBX_SUCCESS; if (unlikely(MDBX_IS_ERROR(err))) @@ -1086,24 +934,21 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, } #endif /* MADV_REMOVE */ #if defined(MADV_DONTNEED) - NOTICE("open-MADV_%s %u..%u", "DONTNEED", env->lck->discarded_tail.weak, - bytes2pgno(env, env->dxb_mmap.current)); - err = madvise(ptr_disp(env->dxb_mmap.base, used_aligned2os_bytes), - env->dxb_mmap.current - used_aligned2os_bytes, MADV_DONTNEED) + NOTICE("open-MADV_%s %u..%u", "DONTNEED", env->lck->discarded_tail.weak, bytes2pgno(env, env->dxb_mmap.current)); + err = madvise(ptr_disp(env->dxb_mmap.base, used_aligned2os_bytes), env->dxb_mmap.current - used_aligned2os_bytes, + MADV_DONTNEED) ? ignore_enosys(errno) : MDBX_SUCCESS; if (unlikely(MDBX_IS_ERROR(err))) return err; #elif defined(POSIX_MADV_DONTNEED) - err = ignore_enosys(posix_madvise( - ptr_disp(env->dxb_mmap.base, used_aligned2os_bytes), - env->dxb_mmap.current - used_aligned2os_bytes, POSIX_MADV_DONTNEED)); + err = ignore_enosys(posix_madvise(ptr_disp(env->dxb_mmap.base, used_aligned2os_bytes), + env->dxb_mmap.current - used_aligned2os_bytes, POSIX_MADV_DONTNEED)); if (unlikely(MDBX_IS_ERROR(err))) return err; #elif defined(POSIX_FADV_DONTNEED) - err = ignore_enosys(posix_fadvise( - env->lazy_fd, used_aligned2os_bytes, - env->dxb_mmap.current - used_aligned2os_bytes, POSIX_FADV_DONTNEED)); + err = ignore_enosys(posix_fadvise(env->lazy_fd, used_aligned2os_bytes, + env->dxb_mmap.current - used_aligned2os_bytes, POSIX_FADV_DONTNEED)); if (unlikely(MDBX_IS_ERROR(err))) return err; #endif /* MADV_DONTNEED */ @@ -1116,8 +961,7 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, return rc; } -int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, - troika_t *const troika) { +int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, troika_t *const troika) { eASSERT(env, ((env->flags ^ flags) & MDBX_WRITEMAP) == 0); eASSERT(env, pending->trees.gc.flags == MDBX_INTEGERKEY); eASSERT(env, check_table_flags(pending->trees.main.flags)); @@ -1127,107 +971,82 @@ int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, const meta_ptr_t head = meta_recent(env, troika); int rc; - eASSERT(env, - pending < METAPAGE(env, 0) || pending > METAPAGE(env, NUM_METAS)); + eASSERT(env, pending < METAPAGE(env, 0) || pending > METAPAGE(env, NUM_METAS)); eASSERT(env, (env->flags & (MDBX_RDONLY | ENV_FATAL_ERROR)) == 0); eASSERT(env, pending->geometry.first_unallocated <= pending->geometry.now); if (flags & MDBX_SAFE_NOSYNC) { /* Check auto-sync conditions */ - const pgno_t autosync_threshold = - atomic_load32(&env->lck->autosync_threshold, mo_Relaxed); - const uint64_t autosync_period = - atomic_load64(&env->lck->autosync_period, mo_Relaxed); + const pgno_t autosync_threshold = atomic_load32(&env->lck->autosync_threshold, mo_Relaxed); + const uint64_t autosync_period = atomic_load64(&env->lck->autosync_period, mo_Relaxed); uint64_t eoos_timestamp; - if ((autosync_threshold && - atomic_load64(&env->lck->unsynced_pages, mo_Relaxed) >= - autosync_threshold) || - (autosync_period && - (eoos_timestamp = - atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) && + if ((autosync_threshold && atomic_load64(&env->lck->unsynced_pages, mo_Relaxed) >= autosync_threshold) || + (autosync_period && (eoos_timestamp = atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) && osal_monotime() - eoos_timestamp >= autosync_period)) flags &= MDBX_WRITEMAP | txn_shrink_allowed; /* force steady */ } pgno_t shrink = 0; if (flags & txn_shrink_allowed) { - const size_t prev_discarded_pgno = - atomic_load32(&env->lck->discarded_tail, mo_Relaxed); + const size_t prev_discarded_pgno = atomic_load32(&env->lck->discarded_tail, mo_Relaxed); if (prev_discarded_pgno < pending->geometry.first_unallocated) env->lck->discarded_tail.weak = pending->geometry.first_unallocated; - else if (prev_discarded_pgno >= - pending->geometry.first_unallocated + env->madv_threshold) { + else if (prev_discarded_pgno >= pending->geometry.first_unallocated + env->madv_threshold) { /* LY: check conditions to discard unused pages */ - const pgno_t largest_pgno = mvcc_snapshot_largest( - env, (head.ptr_c->geometry.first_unallocated > - pending->geometry.first_unallocated) - ? head.ptr_c->geometry.first_unallocated - : pending->geometry.first_unallocated); + const pgno_t largest_pgno = + mvcc_snapshot_largest(env, (head.ptr_c->geometry.first_unallocated > pending->geometry.first_unallocated) + ? head.ptr_c->geometry.first_unallocated + : pending->geometry.first_unallocated); eASSERT(env, largest_pgno >= NUM_METAS); #if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__) const pgno_t edge = env->poison_edge; if (edge > largest_pgno) { env->poison_edge = largest_pgno; - VALGRIND_MAKE_MEM_NOACCESS( - ptr_disp(env->dxb_mmap.base, pgno2bytes(env, largest_pgno)), - pgno2bytes(env, edge - largest_pgno)); - MDBX_ASAN_POISON_MEMORY_REGION( - ptr_disp(env->dxb_mmap.base, pgno2bytes(env, largest_pgno)), - pgno2bytes(env, edge - largest_pgno)); + VALGRIND_MAKE_MEM_NOACCESS(ptr_disp(env->dxb_mmap.base, pgno2bytes(env, largest_pgno)), + pgno2bytes(env, edge - largest_pgno)); + MDBX_ASAN_POISON_MEMORY_REGION(ptr_disp(env->dxb_mmap.base, pgno2bytes(env, largest_pgno)), + pgno2bytes(env, edge - largest_pgno)); } #endif /* ENABLE_MEMCHECK || __SANITIZE_ADDRESS__ */ #if defined(MADV_DONTNEED) || defined(POSIX_MADV_DONTNEED) const size_t discard_edge_pgno = pgno_align2os_pgno(env, largest_pgno); if (prev_discarded_pgno >= discard_edge_pgno + env->madv_threshold) { - const size_t prev_discarded_bytes = - pgno_align2os_bytes(env, prev_discarded_pgno); + const size_t prev_discarded_bytes = pgno_align2os_bytes(env, prev_discarded_pgno); const size_t discard_edge_bytes = pgno2bytes(env, discard_edge_pgno); /* из-за выравнивания prev_discarded_bytes и discard_edge_bytes * могут быть равны */ if (prev_discarded_bytes > discard_edge_bytes) { - NOTICE("shrink-MADV_%s %zu..%zu", "DONTNEED", discard_edge_pgno, - prev_discarded_pgno); - munlock_after(env, discard_edge_pgno, - bytes_align2os_bytes(env, env->dxb_mmap.current)); - const uint32_t munlocks_before = - atomic_load32(&env->lck->mlcnt[1], mo_Relaxed); + NOTICE("shrink-MADV_%s %zu..%zu", "DONTNEED", discard_edge_pgno, prev_discarded_pgno); + munlock_after(env, discard_edge_pgno, bytes_align2os_bytes(env, env->dxb_mmap.current)); + const uint32_t munlocks_before = atomic_load32(&env->lck->mlcnt[1], mo_Relaxed); #if defined(MADV_DONTNEED) int advise = MADV_DONTNEED; -#if defined(MADV_FREE) && \ - 0 /* MADV_FREE works for only anonymous vma at the moment */ - if ((env->flags & MDBX_WRITEMAP) && - global.linux_kernel_version > 0x04050000) +#if defined(MADV_FREE) && 0 /* MADV_FREE works for only anonymous vma at the moment */ + if ((env->flags & MDBX_WRITEMAP) && global.linux_kernel_version > 0x04050000) advise = MADV_FREE; #endif /* MADV_FREE */ - int err = madvise(ptr_disp(env->dxb_mmap.base, discard_edge_bytes), - prev_discarded_bytes - discard_edge_bytes, advise) + int err = madvise(ptr_disp(env->dxb_mmap.base, discard_edge_bytes), prev_discarded_bytes - discard_edge_bytes, + advise) ? ignore_enosys(errno) : MDBX_SUCCESS; #else - int err = ignore_enosys(posix_madvise( - ptr_disp(env->dxb_mmap.base, discard_edge_bytes), - prev_discarded_bytes - discard_edge_bytes, POSIX_MADV_DONTNEED)); + int err = ignore_enosys(posix_madvise(ptr_disp(env->dxb_mmap.base, discard_edge_bytes), + prev_discarded_bytes - discard_edge_bytes, POSIX_MADV_DONTNEED)); #endif if (unlikely(MDBX_IS_ERROR(err))) { - const uint32_t mlocks_after = - atomic_load32(&env->lck->mlcnt[0], mo_Relaxed); + const uint32_t mlocks_after = atomic_load32(&env->lck->mlcnt[0], mo_Relaxed); if (err == MDBX_EINVAL) { - const int severity = (mlocks_after - munlocks_before) - ? MDBX_LOG_NOTICE - : MDBX_LOG_WARN; + const int severity = (mlocks_after - munlocks_before) ? MDBX_LOG_NOTICE : MDBX_LOG_WARN; if (LOG_ENABLED(severity)) - debug_log( - severity, __func__, __LINE__, - "%s-madvise: ignore EINVAL (%d) since some pages maybe " - "locked (%u/%u mlcnt-processes)", - "shrink", err, mlocks_after, munlocks_before); + debug_log(severity, __func__, __LINE__, + "%s-madvise: ignore EINVAL (%d) since some pages maybe " + "locked (%u/%u mlcnt-processes)", + "shrink", err, mlocks_after, munlocks_before); } else { - ERROR("%s-madvise(%s, %zu, +%zu), %u/%u mlcnt-processes, err %d", - "shrink", "DONTNEED", discard_edge_bytes, - prev_discarded_bytes - discard_edge_bytes, mlocks_after, - munlocks_before, err); + ERROR("%s-madvise(%s, %zu, +%zu), %u/%u mlcnt-processes, err %d", "shrink", "DONTNEED", + discard_edge_bytes, prev_discarded_bytes - discard_edge_bytes, mlocks_after, munlocks_before, err); return err; } } else @@ -1239,23 +1058,15 @@ int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, /* LY: check conditions to shrink datafile */ const pgno_t backlog_gap = 3 + pending->trees.gc.height * 3; pgno_t shrink_step = 0; - if (pending->geometry.shrink_pv && - pending->geometry.now - pending->geometry.first_unallocated > - (shrink_step = pv2pages(pending->geometry.shrink_pv)) + - backlog_gap) { - if (pending->geometry.now > largest_pgno && - pending->geometry.now - largest_pgno > shrink_step + backlog_gap) { + if (pending->geometry.shrink_pv && pending->geometry.now - pending->geometry.first_unallocated > + (shrink_step = pv2pages(pending->geometry.shrink_pv)) + backlog_gap) { + if (pending->geometry.now > largest_pgno && pending->geometry.now - largest_pgno > shrink_step + backlog_gap) { const pgno_t aligner = - pending->geometry.grow_pv - ? /* grow_step */ pv2pages(pending->geometry.grow_pv) - : shrink_step; + pending->geometry.grow_pv ? /* grow_step */ pv2pages(pending->geometry.grow_pv) : shrink_step; const pgno_t with_backlog_gap = largest_pgno + backlog_gap; const pgno_t aligned = - pgno_align2os_pgno(env, (size_t)with_backlog_gap + aligner - - with_backlog_gap % aligner); - const pgno_t bottom = (aligned > pending->geometry.lower) - ? aligned - : pending->geometry.lower; + pgno_align2os_pgno(env, (size_t)with_backlog_gap + aligner - with_backlog_gap % aligner); + const pgno_t bottom = (aligned > pending->geometry.lower) ? aligned : pending->geometry.lower; if (pending->geometry.now > bottom) { if (TROIKA_HAVE_STEADY(troika)) /* force steady, but only if steady-checkpoint is present */ @@ -1264,8 +1075,7 @@ int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, pending->geometry.now = bottom; if (unlikely(head.txnid == pending->unsafe_txnid)) { const txnid_t txnid = safe64_txnid_next(pending->unsafe_txnid); - NOTICE("force-forward pending-txn %" PRIaTXN " -> %" PRIaTXN, - pending->unsafe_txnid, txnid); + NOTICE("force-forward pending-txn %" PRIaTXN " -> %" PRIaTXN, pending->unsafe_txnid, txnid); ENSURE(env, !env->basal_txn || !env->txn); if (unlikely(txnid > MAX_TXNID)) { rc = MDBX_TXN_FULL; @@ -1290,8 +1100,7 @@ int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, if ((flags & MDBX_SAFE_NOSYNC) == 0) { sync_op = 1; mode_bits = MDBX_SYNC_DATA; - if (pending->geometry.first_unallocated > - meta_prefer_steady(env, troika).ptr_c->geometry.now) + if (pending->geometry.first_unallocated > meta_prefer_steady(env, troika).ptr_c->geometry.now) mode_bits |= MDBX_SYNC_SIZE; if (flags & MDBX_NOMETASYNC) mode_bits |= MDBX_SYNC_IODQ; @@ -1303,10 +1112,7 @@ int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, #else (void)sync_op; #endif /* MDBX_ENABLE_PGOP_STAT */ - rc = osal_msync( - &env->dxb_mmap, 0, - pgno_align2os_bytes(env, pending->geometry.first_unallocated), - mode_bits); + rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, pending->geometry.first_unallocated), mode_bits); } else { #if MDBX_ENABLE_PGOP_STAT env->lck->pgops.fsync.weak += sync_op; @@ -1336,12 +1142,10 @@ int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, unaligned_poke_u64(4, pending->sign, DATASIGN_WEAK); } - const bool legal4overwrite = - head.txnid == pending->unsafe_txnid && - !memcmp(&head.ptr_c->trees, &pending->trees, sizeof(pending->trees)) && - !memcmp(&head.ptr_c->canary, &pending->canary, sizeof(pending->canary)) && - !memcmp(&head.ptr_c->geometry, &pending->geometry, - sizeof(pending->geometry)); + const bool legal4overwrite = head.txnid == pending->unsafe_txnid && + !memcmp(&head.ptr_c->trees, &pending->trees, sizeof(pending->trees)) && + !memcmp(&head.ptr_c->canary, &pending->canary, sizeof(pending->canary)) && + !memcmp(&head.ptr_c->geometry, &pending->geometry, sizeof(pending->geometry)); meta_t *target = nullptr; if (head.txnid == pending->unsafe_txnid) { ENSURE(env, legal4overwrite); @@ -1353,51 +1157,40 @@ int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, } } else { const unsigned troika_tail = troika->tail_and_flags & 3; - ENSURE(env, troika_tail < NUM_METAS && troika_tail != troika->recent && - troika_tail != troika->prefer_steady); + ENSURE(env, troika_tail < NUM_METAS && troika_tail != troika->recent && troika_tail != troika->prefer_steady); target = (meta_t *)meta_tail(env, troika).ptr_c; } /* LY: step#2 - update meta-page. */ - DEBUG("writing meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO - ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO - " +%u -%u, txn_id %" PRIaTXN ", %s", - data_page(target)->pgno, pending->trees.main.root, - pending->trees.gc.root, pending->geometry.lower, - pending->geometry.first_unallocated, pending->geometry.now, - pending->geometry.upper, pv2pages(pending->geometry.grow_pv), - pv2pages(pending->geometry.shrink_pv), pending->unsafe_txnid, + DEBUG("writing meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO + "/%" PRIaPGNO " +%u -%u, txn_id %" PRIaTXN ", %s", + data_page(target)->pgno, pending->trees.main.root, pending->trees.gc.root, pending->geometry.lower, + pending->geometry.first_unallocated, pending->geometry.now, pending->geometry.upper, + pv2pages(pending->geometry.grow_pv), pv2pages(pending->geometry.shrink_pv), pending->unsafe_txnid, durable_caption(pending)); DEBUG("meta0: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO "/%" PRIaPGNO, (meta0 == head.ptr_c) ? "head" : (meta0 == target) ? "tail" : "stay", - durable_caption(meta0), constmeta_txnid(meta0), meta0->trees.main.root, - meta0->trees.gc.root); + durable_caption(meta0), constmeta_txnid(meta0), meta0->trees.main.root, meta0->trees.gc.root); DEBUG("meta1: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO "/%" PRIaPGNO, (meta1 == head.ptr_c) ? "head" : (meta1 == target) ? "tail" : "stay", - durable_caption(meta1), constmeta_txnid(meta1), meta1->trees.main.root, - meta1->trees.gc.root); + durable_caption(meta1), constmeta_txnid(meta1), meta1->trees.main.root, meta1->trees.gc.root); DEBUG("meta2: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO "/%" PRIaPGNO, (meta2 == head.ptr_c) ? "head" : (meta2 == target) ? "tail" : "stay", - durable_caption(meta2), constmeta_txnid(meta2), meta2->trees.main.root, - meta2->trees.gc.root); + durable_caption(meta2), constmeta_txnid(meta2), meta2->trees.main.root, meta2->trees.gc.root); - eASSERT(env, pending->unsafe_txnid != constmeta_txnid(meta0) || - (meta_is_steady(pending) && !meta_is_steady(meta0))); - eASSERT(env, pending->unsafe_txnid != constmeta_txnid(meta1) || - (meta_is_steady(pending) && !meta_is_steady(meta1))); - eASSERT(env, pending->unsafe_txnid != constmeta_txnid(meta2) || - (meta_is_steady(pending) && !meta_is_steady(meta2))); + eASSERT(env, pending->unsafe_txnid != constmeta_txnid(meta0) || (meta_is_steady(pending) && !meta_is_steady(meta0))); + eASSERT(env, pending->unsafe_txnid != constmeta_txnid(meta1) || (meta_is_steady(pending) && !meta_is_steady(meta1))); + eASSERT(env, pending->unsafe_txnid != constmeta_txnid(meta2) || (meta_is_steady(pending) && !meta_is_steady(meta2))); eASSERT(env, ((env->flags ^ flags) & MDBX_WRITEMAP) == 0); - ENSURE(env, target == head.ptr_c || - constmeta_txnid(target) < pending->unsafe_txnid); + ENSURE(env, target == head.ptr_c || constmeta_txnid(target) < pending->unsafe_txnid); if (flags & MDBX_WRITEMAP) { jitter4testing(true); if (likely(target != head.ptr_c)) { @@ -1430,8 +1223,7 @@ int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, } else { /* dangerous case (target == head), only sign could * me updated, check assertions once again */ - eASSERT(env, - legal4overwrite && !head.is_steady && meta_is_steady(pending)); + eASSERT(env, legal4overwrite && !head.is_steady && meta_is_steady(pending)); } memcpy(target->sign, pending->sign, 8); osal_flush_incoherent_cpu_writeback(); @@ -1443,19 +1235,15 @@ int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, env->lck->pgops.msync.weak += 1; #endif /* MDBX_ENABLE_PGOP_STAT */ rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), - (flags & MDBX_NOMETASYNC) - ? MDBX_SYNC_NONE - : MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + (flags & MDBX_NOMETASYNC) ? MDBX_SYNC_NONE : MDBX_SYNC_DATA | MDBX_SYNC_IODQ); } else { #if MDBX_ENABLE_PGOP_STAT env->lck->pgops.wops.weak += 1; #endif /* MDBX_ENABLE_PGOP_STAT */ const page_t *page = data_page(target); - rc = osal_pwrite(env->fd4meta, page, env->ps, - ptr_dist(page, env->dxb_mmap.base)); + rc = osal_pwrite(env->fd4meta, page, env->ps, ptr_dist(page, env->dxb_mmap.base)); if (likely(rc == MDBX_SUCCESS)) { - osal_flush_incoherent_mmap(target, sizeof(meta_t), - globals.sys_pagesize); + osal_flush_incoherent_mmap(target, sizeof(meta_t), globals.sys_pagesize); if ((flags & MDBX_NOMETASYNC) == 0 && env->fd4meta == env->lazy_fd) { #if MDBX_ENABLE_PGOP_STAT env->lck->pgops.fsync.weak += 1; @@ -1474,21 +1262,18 @@ int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, const meta_t undo_meta = *target; eASSERT(env, pending->trees.gc.flags == MDBX_INTEGERKEY); eASSERT(env, check_table_flags(pending->trees.main.flags)); - rc = osal_pwrite(env->fd4meta, pending, sizeof(meta_t), - ptr_dist(target, env->dxb_mmap.base)); + rc = osal_pwrite(env->fd4meta, pending, sizeof(meta_t), ptr_dist(target, env->dxb_mmap.base)); if (unlikely(rc != MDBX_SUCCESS)) { undo: DEBUG("%s", "write failed, disk error?"); /* On a failure, the pagecache still contains the new data. * Try write some old data back, to prevent it from being used. */ - osal_pwrite(env->fd4meta, &undo_meta, sizeof(meta_t), - ptr_dist(target, env->dxb_mmap.base)); + osal_pwrite(env->fd4meta, &undo_meta, sizeof(meta_t), ptr_dist(target, env->dxb_mmap.base)); goto fail; } osal_flush_incoherent_mmap(target, sizeof(meta_t), globals.sys_pagesize); /* sync meta-pages */ - if ((flags & MDBX_NOMETASYNC) == 0 && env->fd4meta == env->lazy_fd && - !env->incore) { + if ((flags & MDBX_NOMETASYNC) == 0 && env->fd4meta == env->lazy_fd && !env->incore) { #if MDBX_ENABLE_PGOP_STAT env->lck->pgops.fsync.weak += 1; #endif /* MDBX_ENABLE_PGOP_STAT */ @@ -1500,23 +1285,18 @@ int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, uint64_t timestamp = 0; while ("workaround for https://libmdbx.dqdkfa.ru/dead-github/issues/269") { - rc = coherency_check_written( - env, pending->unsafe_txnid, target, - bytes2pgno(env, ptr_dist(target, env->dxb_mmap.base)), ×tamp); + rc = coherency_check_written(env, pending->unsafe_txnid, target, + bytes2pgno(env, ptr_dist(target, env->dxb_mmap.base)), ×tamp); if (likely(rc == MDBX_SUCCESS)) break; if (unlikely(rc != MDBX_RESULT_TRUE)) goto fail; } - const uint32_t sync_txnid_dist = - ((flags & MDBX_NOMETASYNC) == 0) ? 0 - : ((flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC) - ? MDBX_NOMETASYNC_LAZY_FD - : MDBX_NOMETASYNC_LAZY_WRITEMAP; - env->lck->meta_sync_txnid.weak = - pending->txnid_a[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__].weak - - sync_txnid_dist; + const uint32_t sync_txnid_dist = ((flags & MDBX_NOMETASYNC) == 0) ? 0 + : ((flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC) ? MDBX_NOMETASYNC_LAZY_FD + : MDBX_NOMETASYNC_LAZY_WRITEMAP; + env->lck->meta_sync_txnid.weak = pending->txnid_a[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__].weak - sync_txnid_dist; *troika = meta_tap(env); for (MDBX_txn *txn = env->basal_txn; txn; txn = txn->nested) @@ -1525,10 +1305,8 @@ int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, /* LY: shrink datafile if needed */ if (unlikely(shrink)) { - VERBOSE("shrink to %" PRIaPGNO " pages (-%" PRIaPGNO ")", - pending->geometry.now, shrink); - rc = dxb_resize(env, pending->geometry.first_unallocated, - pending->geometry.now, pending->geometry.upper, + VERBOSE("shrink to %" PRIaPGNO " pages (-%" PRIaPGNO ")", pending->geometry.now, shrink); + rc = dxb_resize(env, pending->geometry.first_unallocated, pending->geometry.now, pending->geometry.upper, impilict_shrink); if (rc != MDBX_SUCCESS && rc != MDBX_EPERM) goto fail; diff --git a/src/env-opts.c b/src/env-opts.c index e22d1a6a..c3bab5f2 100644 --- a/src/env-opts.c +++ b/src/env-opts.c @@ -6,17 +6,11 @@ __cold static unsigned default_rp_augment_limit(const MDBX_env *env) { const size_t timeframe = /* 16 секунд */ 16 << 16; const size_t remain_1sec = - (env->options.gc_time_limit < timeframe) - ? timeframe - (size_t)env->options.gc_time_limit - : 0; - const size_t minimum = (env->maxgc_large1page * 2 > MDBX_PNL_INITIAL) - ? env->maxgc_large1page * 2 - : MDBX_PNL_INITIAL; + (env->options.gc_time_limit < timeframe) ? timeframe - (size_t)env->options.gc_time_limit : 0; + const size_t minimum = (env->maxgc_large1page * 2 > MDBX_PNL_INITIAL) ? env->maxgc_large1page * 2 : MDBX_PNL_INITIAL; const size_t one_third = env->geo_in_bytes.now / 3 >> env->ps2ln; const size_t augment_limit = - (one_third > minimum) - ? minimum + (one_third - minimum) / timeframe * remain_1sec - : minimum; + (one_third > minimum) ? minimum + (one_third - minimum) / timeframe * remain_1sec : minimum; eASSERT(env, augment_limit < PAGELIST_LIMIT); return pnl_bytes2size(pnl_size2bytes(augment_limit)); } @@ -86,29 +80,23 @@ void env_options_adjust_defaults(MDBX_env *env) { const size_t basis = env->geo_in_bytes.now; /* TODO: use options? */ const unsigned factor = 9; - size_t threshold = (basis < ((size_t)65536 << factor)) - ? 65536 /* minimal threshold */ - : (basis > (MEGABYTE * 4 << factor)) - ? MEGABYTE * 4 /* maximal threshold */ - : basis >> factor; + size_t threshold = (basis < ((size_t)65536 << factor)) ? 65536 /* minimal threshold */ + : (basis > (MEGABYTE * 4 << factor)) ? MEGABYTE * 4 /* maximal threshold */ + : basis >> factor; threshold = - (threshold < env->geo_in_bytes.shrink || !env->geo_in_bytes.shrink) - ? threshold - : env->geo_in_bytes.shrink; + (threshold < env->geo_in_bytes.shrink || !env->geo_in_bytes.shrink) ? threshold : env->geo_in_bytes.shrink; env->madv_threshold = bytes2pgno(env, bytes_align2os_bytes(env, threshold)); } //------------------------------------------------------------------------------ -__cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, - uint64_t value) { +__cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, uint64_t value) { int err = check_env(env, false); if (unlikely(err != MDBX_SUCCESS)) return LOG_IFERR(err); - const bool lock_needed = - ((env->flags & ENV_ACTIVE) && env->basal_txn && !env_txn0_owned(env)); + const bool lock_needed = ((env->flags & ENV_ACTIVE) && env->basal_txn && !env_txn0_owned(env)); bool should_unlock = false; switch (option) { case MDBX_opt_sync_bytes: @@ -121,10 +109,8 @@ __cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, if (unlikely(value > SIZE_MAX - 65536)) return LOG_IFERR(MDBX_EINVAL); value = bytes2pgno(env, (size_t)value + env->ps - 1); - if ((uint32_t)value != - atomic_load32(&env->lck->autosync_threshold, mo_AcquireRelease) && - atomic_store32(&env->lck->autosync_threshold, (uint32_t)value, - mo_Relaxed) + if ((uint32_t)value != atomic_load32(&env->lck->autosync_threshold, mo_AcquireRelease) && + atomic_store32(&env->lck->autosync_threshold, (uint32_t)value, mo_Relaxed) /* Дергаем sync(force=off) только если задано новое не-нулевое значение * и мы вне транзакции */ && lock_needed) { @@ -248,8 +234,7 @@ __cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, err = MDBX_EPERM /* unable change during transaction */; else { const pgno_t value32 = (pgno_t)value; - if (option == MDBX_opt_txn_dp_initial && - env->options.dp_initial != value32) { + if (option == MDBX_opt_txn_dp_initial && env->options.dp_initial != value32) { env->options.dp_initial = value32; if (env->options.dp_limit < value32) { env->options.dp_limit = value32; @@ -308,8 +293,7 @@ __cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, #if defined(_WIN32) || defined(_WIN64) /* позволяем "установить" значение по-умолчанию и совпадающее * с поведением соответствующим текущей установке MDBX_NOMETASYNC */ - if (value == /* default */ UINT64_MAX && - value != ((env->flags & MDBX_NOMETASYNC) ? 0 : UINT_MAX)) + if (value == /* default */ UINT64_MAX && value != ((env->flags & MDBX_NOMETASYNC) ? 0 : UINT_MAX)) err = MDBX_EINVAL; #else if (value == /* default */ UINT64_MAX) @@ -335,8 +319,7 @@ __cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, case MDBX_opt_prefer_waf_insteadof_balance: if (value == /* default */ UINT64_MAX) - env->options.prefer_waf_insteadof_balance = - default_prefer_waf_insteadof_balance(env); + env->options.prefer_waf_insteadof_balance = default_prefer_waf_insteadof_balance(env); else if (value > 1) err = MDBX_EINVAL; else @@ -400,8 +383,7 @@ __cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, return LOG_IFERR(err); } -__cold int mdbx_env_get_option(const MDBX_env *env, const MDBX_option_t option, - uint64_t *pvalue) { +__cold int mdbx_env_get_option(const MDBX_env *env, const MDBX_option_t option, uint64_t *pvalue) { int err = check_env(env, false); if (unlikely(err != MDBX_SUCCESS)) return LOG_IFERR(err); @@ -412,15 +394,13 @@ __cold int mdbx_env_get_option(const MDBX_env *env, const MDBX_option_t option, case MDBX_opt_sync_bytes: if (unlikely(!(env->flags & ENV_ACTIVE))) return LOG_IFERR(MDBX_EPERM); - *pvalue = pgno2bytes( - env, atomic_load32(&env->lck->autosync_threshold, mo_Relaxed)); + *pvalue = pgno2bytes(env, atomic_load32(&env->lck->autosync_threshold, mo_Relaxed)); break; case MDBX_opt_sync_period: if (unlikely(!(env->flags & ENV_ACTIVE))) return LOG_IFERR(MDBX_EPERM); - *pvalue = osal_monotime_to_16dot16( - atomic_load64(&env->lck->autosync_period, mo_Relaxed)); + *pvalue = osal_monotime_to_16dot16(atomic_load64(&env->lck->autosync_period, mo_Relaxed)); break; case MDBX_opt_max_db: diff --git a/src/env.c b/src/env.c index c4902765..91af9413 100644 --- a/src/env.c +++ b/src/env.c @@ -4,17 +4,14 @@ #include "internals.h" bool env_txn0_owned(const MDBX_env *env) { - return (env->flags & MDBX_NOSTICKYTHREADS) - ? (env->basal_txn->owner != 0) - : (env->basal_txn->owner == osal_thread_self()); + return (env->flags & MDBX_NOSTICKYTHREADS) ? (env->basal_txn->owner != 0) + : (env->basal_txn->owner == osal_thread_self()); } int env_page_auxbuffer(MDBX_env *env) { - const int err = - env->page_auxbuf - ? MDBX_SUCCESS - : osal_memalign_alloc(globals.sys_pagesize, - env->ps * (size_t)NUM_METAS, &env->page_auxbuf); + const int err = env->page_auxbuf + ? MDBX_SUCCESS + : osal_memalign_alloc(globals.sys_pagesize, env->ps * (size_t)NUM_METAS, &env->page_auxbuf); if (likely(err == MDBX_SUCCESS)) { memset(env->page_auxbuf, -1, env->ps * (size_t)2); memset(ptr_disp(env->page_auxbuf, env->ps * (size_t)2), 0, env->ps); @@ -34,26 +31,19 @@ __cold unsigned env_setup_pagesize(MDBX_env *env, const size_t pagesize) { STATIC_ASSERT(MAX_GC1OVPAGE(MDBX_MIN_PAGESIZE) > 4); STATIC_ASSERT(MAX_GC1OVPAGE(MDBX_MAX_PAGESIZE) < PAGELIST_LIMIT); const intptr_t maxgc_ov1page = (pagesize - PAGEHDRSZ) / sizeof(pgno_t) - 1; - ENSURE(env, - maxgc_ov1page > 42 && maxgc_ov1page < (intptr_t)PAGELIST_LIMIT / 4); + ENSURE(env, maxgc_ov1page > 42 && maxgc_ov1page < (intptr_t)PAGELIST_LIMIT / 4); env->maxgc_large1page = (unsigned)maxgc_ov1page; - env->maxgc_per_branch = - (unsigned)((pagesize - PAGEHDRSZ) / - (sizeof(indx_t) + sizeof(node_t) + sizeof(txnid_t))); + env->maxgc_per_branch = (unsigned)((pagesize - PAGEHDRSZ) / (sizeof(indx_t) + sizeof(node_t) + sizeof(txnid_t))); - STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) > - sizeof(tree_t) + NODESIZE + 42); + STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) > sizeof(tree_t) + NODESIZE + 42); STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MAX_PAGESIZE) < UINT16_MAX); - STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) >= - BRANCH_NODE_MAX(MDBX_MIN_PAGESIZE)); + STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) >= BRANCH_NODE_MAX(MDBX_MIN_PAGESIZE)); STATIC_ASSERT(BRANCH_NODE_MAX(MDBX_MAX_PAGESIZE) > NODESIZE + 42); STATIC_ASSERT(BRANCH_NODE_MAX(MDBX_MAX_PAGESIZE) < UINT16_MAX); const intptr_t branch_nodemax = BRANCH_NODE_MAX(pagesize); const intptr_t leaf_nodemax = LEAF_NODE_MAX(pagesize); - ENSURE(env, branch_nodemax > (intptr_t)(NODESIZE + 42) && - branch_nodemax % 2 == 0 && - leaf_nodemax > (intptr_t)(sizeof(tree_t) + NODESIZE + 42) && - leaf_nodemax >= branch_nodemax && + ENSURE(env, branch_nodemax > (intptr_t)(NODESIZE + 42) && branch_nodemax % 2 == 0 && + leaf_nodemax > (intptr_t)(sizeof(tree_t) + NODESIZE + 42) && leaf_nodemax >= branch_nodemax && leaf_nodemax < (int)UINT16_MAX && leaf_nodemax % 2 == 0); env->leaf_nodemax = (uint16_t)leaf_nodemax; env->branch_nodemax = (uint16_t)branch_nodemax; @@ -71,18 +61,14 @@ __cold unsigned env_setup_pagesize(MDBX_env *env, const size_t pagesize) { if (unlikely(err != MDBX_SUCCESS)) ERROR("mdbx_get_sysraminfo(), rc %d", err); else { - size_t reasonable_dpl_limit = - (size_t)(total_ram_pages + avail_ram_pages) / 42; + size_t reasonable_dpl_limit = (size_t)(total_ram_pages + avail_ram_pages) / 42; if (pagesize > globals.sys_pagesize) reasonable_dpl_limit /= pagesize / globals.sys_pagesize; else if (pagesize < globals.sys_pagesize) reasonable_dpl_limit *= globals.sys_pagesize / pagesize; - reasonable_dpl_limit = (reasonable_dpl_limit < PAGELIST_LIMIT) - ? reasonable_dpl_limit - : PAGELIST_LIMIT; - reasonable_dpl_limit = (reasonable_dpl_limit > CURSOR_STACK_SIZE * 4) - ? reasonable_dpl_limit - : CURSOR_STACK_SIZE * 4; + reasonable_dpl_limit = (reasonable_dpl_limit < PAGELIST_LIMIT) ? reasonable_dpl_limit : PAGELIST_LIMIT; + reasonable_dpl_limit = + (reasonable_dpl_limit > CURSOR_STACK_SIZE * 4) ? reasonable_dpl_limit : CURSOR_STACK_SIZE * 4; env->options.dp_limit = (unsigned)reasonable_dpl_limit; } } @@ -108,46 +94,36 @@ retry:; goto bailout; } - const troika_t troika = - (txn0_owned | should_unlock) ? env->basal_txn->tw.troika : meta_tap(env); + const troika_t troika = (txn0_owned | should_unlock) ? env->basal_txn->tw.troika : meta_tap(env); const meta_ptr_t head = meta_recent(env, &troika); - const uint64_t unsynced_pages = - atomic_load64(&env->lck->unsynced_pages, mo_Relaxed); + const uint64_t unsynced_pages = atomic_load64(&env->lck->unsynced_pages, mo_Relaxed); if (unsynced_pages == 0) { - const uint32_t synched_meta_txnid_u32 = - atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed); + const uint32_t synched_meta_txnid_u32 = atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed); if (synched_meta_txnid_u32 == (uint32_t)head.txnid && head.is_steady) goto bailout; } if (should_unlock && (env->flags & MDBX_WRITEMAP) && - unlikely(head.ptr_c->geometry.first_unallocated > - bytes2pgno(env, env->dxb_mmap.current))) { + unlikely(head.ptr_c->geometry.first_unallocated > bytes2pgno(env, env->dxb_mmap.current))) { - if (unlikely(env->stuck_meta >= 0) && - troika.recent != (uint8_t)env->stuck_meta) { + if (unlikely(env->stuck_meta >= 0) && troika.recent != (uint8_t)env->stuck_meta) { NOTICE("skip %s since wagering meta-page (%u) is mispatch the recent " "meta-page (%u)", "sync datafile", env->stuck_meta, troika.recent); rc = MDBX_RESULT_TRUE; } else { - rc = dxb_resize(env, head.ptr_c->geometry.first_unallocated, - head.ptr_c->geometry.now, head.ptr_c->geometry.upper, + rc = dxb_resize(env, head.ptr_c->geometry.first_unallocated, head.ptr_c->geometry.now, head.ptr_c->geometry.upper, implicit_grow); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; } } - const size_t autosync_threshold = - atomic_load32(&env->lck->autosync_threshold, mo_Relaxed); - const uint64_t autosync_period = - atomic_load64(&env->lck->autosync_period, mo_Relaxed); + const size_t autosync_threshold = atomic_load32(&env->lck->autosync_threshold, mo_Relaxed); + const uint64_t autosync_period = atomic_load64(&env->lck->autosync_period, mo_Relaxed); uint64_t eoos_timestamp; if (force || (autosync_threshold && unsynced_pages >= autosync_threshold) || - (autosync_period && - (eoos_timestamp = - atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) && + (autosync_period && (eoos_timestamp = atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) && osal_monotime() - eoos_timestamp >= autosync_period)) flags &= MDBX_WRITEMAP /* clear flags for full steady sync */; @@ -159,8 +135,7 @@ retry:; int err; /* pre-sync to avoid latency for writer */ - if (unsynced_pages > /* FIXME: define threshold */ 42 && - (flags & MDBX_SAFE_NOSYNC) == 0) { + if (unsynced_pages > /* FIXME: define threshold */ 42 && (flags & MDBX_SAFE_NOSYNC) == 0) { eASSERT(env, ((flags ^ env->flags) & MDBX_WRITEMAP) == 0); if (flags & MDBX_WRITEMAP) { /* Acquire guard to avoid collision with remap */ @@ -171,8 +146,7 @@ retry:; if (unlikely(err != MDBX_SUCCESS)) return err; #endif - const size_t usedbytes = - pgno_align2os_bytes(env, head.ptr_c->geometry.first_unallocated); + const size_t usedbytes = pgno_align2os_bytes(env, head.ptr_c->geometry.first_unallocated); err = osal_msync(&env->dxb_mmap, 0, usedbytes, MDBX_SYNC_DATA); #if defined(_WIN32) || defined(_WIN64) imports.srwl_ReleaseShared(&env->remap_guard); @@ -215,8 +189,7 @@ retry:; eASSERT(env, txn0_owned || should_unlock); eASSERT(env, !txn0_owned || (flags & txn_shrink_allowed) == 0); - if (!head.is_steady && unlikely(env->stuck_meta >= 0) && - troika.recent != (uint8_t)env->stuck_meta) { + if (!head.is_steady && unlikely(env->stuck_meta >= 0) && troika.recent != (uint8_t)env->stuck_meta) { NOTICE("skip %s since wagering meta-page (%u) is mispatch the recent " "meta-page (%u)", "sync datafile", env->stuck_meta, troika.recent); @@ -224,9 +197,8 @@ retry:; goto bailout; } if (!head.is_steady || ((flags & MDBX_SAFE_NOSYNC) == 0 && unsynced_pages)) { - DEBUG("meta-head %" PRIaPGNO ", %s, sync_pending %" PRIu64, - data_page(head.ptr_c)->pgno, durable_caption(head.ptr_c), - unsynced_pages); + DEBUG("meta-head %" PRIaPGNO ", %s, sync_pending %" PRIu64, data_page(head.ptr_c)->pgno, + durable_caption(head.ptr_c), unsynced_pages); meta_t meta = *head.ptr_c; rc = dxb_sync_locked(env, flags, &meta, &env->basal_txn->tw.troika); if (unlikely(rc != MDBX_SUCCESS)) @@ -235,8 +207,7 @@ retry:; /* LY: sync meta-pages if MDBX_NOMETASYNC enabled * and someone was not synced above. */ - if (atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed) != - (uint32_t)head.txnid) + if (atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed) != (uint32_t)head.txnid) rc = meta_sync(env, head); bailout: @@ -334,9 +305,8 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) { */ env->pid = osal_getpid(); - int rc = osal_openfile((env->flags & MDBX_RDONLY) ? MDBX_OPEN_DXB_READ - : MDBX_OPEN_DXB_LAZY, - env, env->pathname.dxb, &env->lazy_fd, mode); + int rc = osal_openfile((env->flags & MDBX_RDONLY) ? MDBX_OPEN_DXB_READ : MDBX_OPEN_DXB_LAZY, env, env->pathname.dxb, + &env->lazy_fd, mode); if (unlikely(rc != MDBX_SUCCESS)) return rc; @@ -355,8 +325,7 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) { #if defined(_WIN32) || defined(_WIN64) eASSERT(env, env->ioring.overlapped_fd == 0); bool ior_direct = false; - if (!(env->flags & - (MDBX_RDONLY | MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_EXCLUSIVE))) { + if (!(env->flags & (MDBX_RDONLY | MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_EXCLUSIVE))) { if (MDBX_AVOID_MSYNC && (env->flags & MDBX_WRITEMAP)) { /* Запрошен режим MDBX_SYNC_DURABLE | MDBX_WRITEMAP при активной опции * MDBX_AVOID_MSYNC. @@ -383,8 +352,7 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) { int err = dxb_read_header(env, &header, MDBX_SUCCESS, true); if ((err == MDBX_SUCCESS && header.pagesize >= globals.sys_pagesize) || (err == MDBX_ENODATA && mode && env->ps >= globals.sys_pagesize && - osal_filesize(env->lazy_fd, &dxb_filesize) == MDBX_SUCCESS && - dxb_filesize == 0)) + osal_filesize(env->lazy_fd, &dxb_filesize) == MDBX_SUCCESS && dxb_filesize == 0)) /* Может быть коллизия, если два процесса пытаются одновременно создать * БД с разным размером страницы, который у одного меньше системной * страницы, а у другого НЕ меньше. Эта допустимая, но очень странная @@ -392,9 +360,8 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) { ior_direct = true; } - rc = osal_openfile(ior_direct ? MDBX_OPEN_DXB_OVERLAPPED_DIRECT - : MDBX_OPEN_DXB_OVERLAPPED, - env, env->pathname.dxb, &env->ioring.overlapped_fd, 0); + rc = osal_openfile(ior_direct ? MDBX_OPEN_DXB_OVERLAPPED_DIRECT : MDBX_OPEN_DXB_OVERLAPPED, env, env->pathname.dxb, + &env->ioring.overlapped_fd, 0); if (unlikely(rc != MDBX_SUCCESS)) return rc; env->dxb_lock_event = CreateEventW(nullptr, true, false, nullptr); @@ -410,8 +377,7 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) { return errno; mode = st.st_mode; } - mode = (/* inherit read permissions for group and others */ mode & - (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) | + mode = (/* inherit read permissions for group and others */ mode & (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) | /* always add read/write for owner */ S_IRUSR | S_IWUSR | ((mode & S_IRGRP) ? /* +write if readable by group */ S_IWGRP : 0) | ((mode & S_IROTH) ? /* +write if readable by others */ S_IWOTH : 0); @@ -428,8 +394,7 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) { | MDBX_EXCLUSIVE #endif /* !Windows */ ))) { - rc = osal_openfile(MDBX_OPEN_DXB_DSYNC, env, env->pathname.dxb, - &env->dsync_fd, 0); + rc = osal_openfile(MDBX_OPEN_DXB_DSYNC, env, env->pathname.dxb, &env->dsync_fd, 0); if (unlikely(MDBX_IS_ERROR(rc))) return rc; if (env->dsync_fd != INVALID_HANDLE_VALUE) { @@ -439,19 +404,14 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) { } } - const MDBX_env_flags_t lazy_flags = - MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC | MDBX_NOMETASYNC; - const MDBX_env_flags_t mode_flags = lazy_flags | MDBX_LIFORECLAIM | - MDBX_NORDAHEAD | MDBX_RDONLY | - MDBX_WRITEMAP; + const MDBX_env_flags_t lazy_flags = MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC | MDBX_NOMETASYNC; + const MDBX_env_flags_t mode_flags = lazy_flags | MDBX_LIFORECLAIM | MDBX_NORDAHEAD | MDBX_RDONLY | MDBX_WRITEMAP; lck_t *const lck = env->lck_mmap.lck; if (lck && lck_rc != MDBX_RESULT_TRUE && (env->flags & MDBX_RDONLY) == 0) { MDBX_env_flags_t snap_flags; - while ((snap_flags = atomic_load32(&lck->envmode, mo_AcquireRelease)) == - MDBX_RDONLY) { - if (atomic_cas32(&lck->envmode, MDBX_RDONLY, - (snap_flags = (env->flags & mode_flags)))) { + while ((snap_flags = atomic_load32(&lck->envmode, mo_AcquireRelease)) == MDBX_RDONLY) { + if (atomic_cas32(&lck->envmode, MDBX_RDONLY, (snap_flags = (env->flags & mode_flags)))) { /* The case: * - let's assume that for some reason the DB file is smaller * than it should be according to the geometry, @@ -471,12 +431,9 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) { if (env->flags & MDBX_ACCEDE) { /* Pickup current mode-flags (MDBX_LIFORECLAIM, MDBX_NORDAHEAD, etc). */ const MDBX_env_flags_t diff = - (snap_flags ^ env->flags) & - ((snap_flags & lazy_flags) ? mode_flags - : mode_flags & ~MDBX_WRITEMAP); + (snap_flags ^ env->flags) & ((snap_flags & lazy_flags) ? mode_flags : mode_flags & ~MDBX_WRITEMAP); env->flags ^= diff; - NOTICE("accede mode-flags: 0x%X, 0x%X -> 0x%X", diff, env->flags ^ diff, - env->flags); + NOTICE("accede mode-flags: 0x%X, 0x%X -> 0x%X", diff, env->flags ^ diff, env->flags); } /* Ранее упущенный не очевидный момент: При работе БД в режимах @@ -498,12 +455,10 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) { * В результате, требуется либо запретить совместную работу процессам с * разным MDBX_WRITEMAP в режиме отложенной записи, либо отслеживать такое * смешивание и блокировать steady-пометки - что контрпродуктивно. */ - const MDBX_env_flags_t rigorous_flags = - (snap_flags & lazy_flags) - ? MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC | MDBX_WRITEMAP - : MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC; - const MDBX_env_flags_t rigorous_diff = - (snap_flags ^ env->flags) & rigorous_flags; + const MDBX_env_flags_t rigorous_flags = (snap_flags & lazy_flags) + ? MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC | MDBX_WRITEMAP + : MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC; + const MDBX_env_flags_t rigorous_diff = (snap_flags ^ env->flags) & rigorous_flags; if (rigorous_diff) { ERROR("current mode/flags 0x%X incompatible with requested 0x%X, " "rigorous diff 0x%X", @@ -529,8 +484,7 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) { } if (unlikely(/* recovery mode */ env->stuck_meta >= 0) && - (lck_rc != /* exclusive */ MDBX_RESULT_TRUE || - (env->flags & MDBX_EXCLUSIVE) == 0)) { + (lck_rc != /* exclusive */ MDBX_RESULT_TRUE || (env->flags & MDBX_EXCLUSIVE) == 0)) { ERROR("%s", "recovery requires exclusive mode"); return MDBX_BUSY; } @@ -545,8 +499,7 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) { if (lck) { if (lck_rc == MDBX_RESULT_TRUE) { rc = lck_downgrade(env); - DEBUG("lck-downgrade-%s: rc %i", - (env->flags & MDBX_EXCLUSIVE) ? "partial" : "full", rc); + DEBUG("lck-downgrade-%s: rc %i", (env->flags & MDBX_EXCLUSIVE) ? "partial" : "full", rc); if (rc != MDBX_SUCCESS) return rc; } else { @@ -556,14 +509,13 @@ __cold int env_open(MDBX_env *env, mdbx_mode_t mode) { } } - rc = (env->flags & MDBX_RDONLY) - ? MDBX_SUCCESS - : osal_ioring_create(&env->ioring + rc = (env->flags & MDBX_RDONLY) ? MDBX_SUCCESS + : osal_ioring_create(&env->ioring #if defined(_WIN32) || defined(_WIN64) - , - ior_direct, env->ioring.overlapped_fd + , + ior_direct, env->ioring.overlapped_fd #endif /* Windows */ - ); + ); return rc; } @@ -606,8 +558,7 @@ __cold int env_close(MDBX_env *env, bool resurrect_after_fork) { } #if defined(_WIN32) || defined(_WIN64) - eASSERT(env, !env->ioring.overlapped_fd || - env->ioring.overlapped_fd == INVALID_HANDLE_VALUE); + eASSERT(env, !env->ioring.overlapped_fd || env->ioring.overlapped_fd == INVALID_HANDLE_VALUE); if (env->dxb_lock_event != INVALID_HANDLE_VALUE) { CloseHandle(env->dxb_lock_event); env->dxb_lock_event = INVALID_HANDLE_VALUE; diff --git a/src/essentials.h b/src/essentials.h index e6f42305..3331e524 100644 --- a/src/essentials.h +++ b/src/essentials.h @@ -110,27 +110,22 @@ extern struct libmdbx_imports imports; extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif -#define MDBX_IS_ERROR(rc) \ - ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) +#define MDBX_IS_ERROR(rc) ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) /*----------------------------------------------------------------------------*/ -MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t -int64pgno(int64_t i64) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t int64pgno(int64_t i64) { if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) return (pgno_t)i64; return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; } -MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t -pgno_add(size_t base, size_t augend) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_add(size_t base, size_t augend) { assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); return int64pgno((int64_t)base + (int64_t)augend); } -MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t -pgno_sub(size_t base, size_t subtrahend) { - assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && - subtrahend < MAX_PAGENO); +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_sub(size_t base, size_t subtrahend) { + assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && subtrahend < MAX_PAGENO); return int64pgno((int64_t)base - (int64_t)subtrahend); } diff --git a/src/gc-get.c b/src/gc-get.c index 9b9c0826..196001f9 100644 --- a/src/gc-get.c +++ b/src/gc-get.c @@ -52,8 +52,7 @@ static bool mincore_fetch(MDBX_env *const env, const size_t unit_begin) { env->lck->pgops.mincore.weak += 1; #endif /* MDBX_ENABLE_PGOP_STAT */ uint8_t *const vector = alloca(pages); - if (unlikely(mincore(ptr_disp(env->dxb_mmap.base, offset), length, - (void *)vector))) { + if (unlikely(mincore(ptr_disp(env->dxb_mmap.base, offset), length, (void *)vector))) { NOTICE("mincore(+%zu, %zu), err %d", offset, length, errno); return false; } @@ -79,14 +78,10 @@ static bool mincore_fetch(MDBX_env *const env, const size_t unit_begin) { } #endif /* MDBX_USE_MINCORE */ -MDBX_MAYBE_UNUSED static inline bool mincore_probe(MDBX_env *const env, - const pgno_t pgno) { +MDBX_MAYBE_UNUSED static inline bool mincore_probe(MDBX_env *const env, const pgno_t pgno) { #if MDBX_USE_MINCORE - const size_t offset_aligned = - floor_powerof2(pgno2bytes(env, pgno), globals.sys_pagesize); - const unsigned unit_log2 = (env->ps2ln > globals.sys_pagesize_ln2) - ? env->ps2ln - : globals.sys_pagesize_ln2; + const size_t offset_aligned = floor_powerof2(pgno2bytes(env, pgno), globals.sys_pagesize); + const unsigned unit_log2 = (env->ps2ln > globals.sys_pagesize_ln2) ? env->ps2ln : globals.sys_pagesize_ln2; const size_t unit_begin = offset_aligned >> unit_log2; eASSERT(env, (unit_begin << unit_log2) == offset_aligned); const ptrdiff_t dist = unit_begin - env->lck->mincore_cache.begin[0]; @@ -102,8 +97,7 @@ MDBX_MAYBE_UNUSED static inline bool mincore_probe(MDBX_env *const env, /*----------------------------------------------------------------------------*/ -MDBX_MAYBE_UNUSED __hot static pgno_t * -scan4seq_fallback(pgno_t *range, const size_t len, const size_t seq) { +MDBX_MAYBE_UNUSED __hot static pgno_t *scan4seq_fallback(pgno_t *range, const size_t len, const size_t seq) { assert(seq > 0 && len > seq); #if MDBX_PNL_ASCENDING assert(range[-1] == len); @@ -167,8 +161,7 @@ scan4seq_fallback(pgno_t *range, const size_t len, const size_t seq) { return nullptr; } -MDBX_MAYBE_UNUSED static const pgno_t *scan4range_checker(const pnl_t pnl, - const size_t seq) { +MDBX_MAYBE_UNUSED static const pgno_t *scan4range_checker(const pnl_t pnl, const size_t seq) { size_t begin = MDBX_PNL_ASCENDING ? 1 : MDBX_PNL_GETSIZE(pnl); #if MDBX_PNL_ASCENDING while (seq <= MDBX_PNL_GETSIZE(pnl) - begin) { @@ -186,8 +179,7 @@ MDBX_MAYBE_UNUSED static const pgno_t *scan4range_checker(const pnl_t pnl, return nullptr; } -#if defined(_MSC_VER) && !defined(__builtin_clz) && \ - !__has_builtin(__builtin_clz) +#if defined(_MSC_VER) && !defined(__builtin_clz) && !__has_builtin(__builtin_clz) MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clz(uint32_t value) { unsigned long index; _BitScanReverse(&index, value); @@ -195,8 +187,7 @@ MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clz(uint32_t value) { } #endif /* _MSC_VER */ -#if defined(_MSC_VER) && !defined(__builtin_clzl) && \ - !__has_builtin(__builtin_clzl) +#if defined(_MSC_VER) && !defined(__builtin_clzl) && !__has_builtin(__builtin_clzl) MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clzl(size_t value) { unsigned long index; #ifdef _WIN64 @@ -213,8 +204,7 @@ MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clzl(size_t value) { #if !MDBX_PNL_ASCENDING -#if !defined(MDBX_ATTRIBUTE_TARGET) && \ - (__has_attribute(__target__) || __GNUC_PREREQ(5, 0)) +#if !defined(MDBX_ATTRIBUTE_TARGET) && (__has_attribute(__target__) || __GNUC_PREREQ(5, 0)) #define MDBX_ATTRIBUTE_TARGET(target) __attribute__((__target__(target))) #endif /* MDBX_ATTRIBUTE_TARGET */ @@ -223,9 +213,8 @@ MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clzl(size_t value) { * gcc/i686-buildroot-linux-gnu/12.2.0/include/xmmintrin.h:814:1: * error: inlining failed in call to 'always_inline' '_mm_movemask_ps': * target specific option mismatch */ -#if !defined(__FAST_MATH__) || !__FAST_MATH__ || !defined(__GNUC__) || \ - defined(__e2k__) || defined(__clang__) || defined(__amd64__) || \ - defined(__SSE2__) +#if !defined(__FAST_MATH__) || !__FAST_MATH__ || !defined(__GNUC__) || defined(__e2k__) || defined(__clang__) || \ + defined(__amd64__) || defined(__SSE2__) #define MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND 0 #else #define MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND 1 @@ -237,41 +226,36 @@ MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clzl(size_t value) { #elif (defined(_M_IX86_FP) && _M_IX86_FP >= 2) || defined(__amd64__) #define __SSE2__ #define MDBX_ATTRIBUTE_TARGET_SSE2 /* nope */ -#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && \ - !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND +#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND #define MDBX_ATTRIBUTE_TARGET_SSE2 MDBX_ATTRIBUTE_TARGET("sse,sse2") #endif /* __SSE2__ */ #if defined(__AVX2__) #define MDBX_ATTRIBUTE_TARGET_AVX2 /* nope */ -#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && \ - !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND +#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND #define MDBX_ATTRIBUTE_TARGET_AVX2 MDBX_ATTRIBUTE_TARGET("sse,sse2,avx,avx2") #endif /* __AVX2__ */ #if defined(MDBX_ATTRIBUTE_TARGET_AVX2) #if defined(__AVX512BW__) #define MDBX_ATTRIBUTE_TARGET_AVX512BW /* nope */ -#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && \ - !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND && \ +#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND && \ (__GNUC_PREREQ(6, 0) || __CLANG_PREREQ(5, 0)) -#define MDBX_ATTRIBUTE_TARGET_AVX512BW \ - MDBX_ATTRIBUTE_TARGET("sse,sse2,avx,avx2,avx512bw") +#define MDBX_ATTRIBUTE_TARGET_AVX512BW MDBX_ATTRIBUTE_TARGET("sse,sse2,avx,avx2,avx512bw") #endif /* __AVX512BW__ */ #endif /* MDBX_ATTRIBUTE_TARGET_AVX2 for MDBX_ATTRIBUTE_TARGET_AVX512BW */ #ifdef MDBX_ATTRIBUTE_TARGET_SSE2 MDBX_ATTRIBUTE_TARGET_SSE2 static __always_inline unsigned -diffcmp2mask_sse2(const pgno_t *const ptr, const ptrdiff_t offset, - const __m128i pattern) { +diffcmp2mask_sse2(const pgno_t *const ptr, const ptrdiff_t offset, const __m128i pattern) { const __m128i f = _mm_loadu_si128((const __m128i *)ptr); const __m128i l = _mm_loadu_si128((const __m128i *)(ptr + offset)); const __m128i cmp = _mm_cmpeq_epi32(_mm_sub_epi32(f, l), pattern); return _mm_movemask_ps(*(const __m128 *)&cmp); } -MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_SSE2 static pgno_t * -scan4seq_sse2(pgno_t *range, const size_t len, const size_t seq) { +MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_SSE2 static pgno_t *scan4seq_sse2(pgno_t *range, const size_t len, + const size_t seq) { assert(seq > 0 && len > seq); #if MDBX_PNL_ASCENDING #error "FIXME: Not implemented" @@ -303,8 +287,7 @@ scan4seq_sse2(pgno_t *range, const size_t len, const size_t seq) { * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */ #if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__) const unsigned on_page_safe_mask = 0xff0 /* enough for '-15' bytes offset */; - if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && - !RUNNING_ON_VALGRIND) { + if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) { const unsigned extra = (unsigned)(detent + 4 - range); assert(extra > 0 && extra < 4); mask = 0xF << extra; @@ -324,8 +307,7 @@ scan4seq_sse2(pgno_t *range, const size_t len, const size_t seq) { #ifdef MDBX_ATTRIBUTE_TARGET_AVX2 MDBX_ATTRIBUTE_TARGET_AVX2 static __always_inline unsigned -diffcmp2mask_avx2(const pgno_t *const ptr, const ptrdiff_t offset, - const __m256i pattern) { +diffcmp2mask_avx2(const pgno_t *const ptr, const ptrdiff_t offset, const __m256i pattern) { const __m256i f = _mm256_loadu_si256((const __m256i *)ptr); const __m256i l = _mm256_loadu_si256((const __m256i *)(ptr + offset)); const __m256i cmp = _mm256_cmpeq_epi32(_mm256_sub_epi32(f, l), pattern); @@ -333,16 +315,15 @@ diffcmp2mask_avx2(const pgno_t *const ptr, const ptrdiff_t offset, } MDBX_ATTRIBUTE_TARGET_AVX2 static __always_inline unsigned -diffcmp2mask_sse2avx(const pgno_t *const ptr, const ptrdiff_t offset, - const __m128i pattern) { +diffcmp2mask_sse2avx(const pgno_t *const ptr, const ptrdiff_t offset, const __m128i pattern) { const __m128i f = _mm_loadu_si128((const __m128i *)ptr); const __m128i l = _mm_loadu_si128((const __m128i *)(ptr + offset)); const __m128i cmp = _mm_cmpeq_epi32(_mm_sub_epi32(f, l), pattern); return _mm_movemask_ps(*(const __m128 *)&cmp); } -MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_AVX2 static pgno_t * -scan4seq_avx2(pgno_t *range, const size_t len, const size_t seq) { +MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_AVX2 static pgno_t *scan4seq_avx2(pgno_t *range, const size_t len, + const size_t seq) { assert(seq > 0 && len > seq); #if MDBX_PNL_ASCENDING #error "FIXME: Not implemented" @@ -374,8 +355,7 @@ scan4seq_avx2(pgno_t *range, const size_t len, const size_t seq) { * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */ #if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__) const unsigned on_page_safe_mask = 0xfe0 /* enough for '-31' bytes offset */; - if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && - !RUNNING_ON_VALGRIND) { + if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) { const unsigned extra = (unsigned)(detent + 8 - range); assert(extra > 0 && extra < 8); mask = 0xFF << extra; @@ -402,15 +382,14 @@ scan4seq_avx2(pgno_t *range, const size_t len, const size_t seq) { #ifdef MDBX_ATTRIBUTE_TARGET_AVX512BW MDBX_ATTRIBUTE_TARGET_AVX512BW static __always_inline unsigned -diffcmp2mask_avx512bw(const pgno_t *const ptr, const ptrdiff_t offset, - const __m512i pattern) { +diffcmp2mask_avx512bw(const pgno_t *const ptr, const ptrdiff_t offset, const __m512i pattern) { const __m512i f = _mm512_loadu_si512((const __m512i *)ptr); const __m512i l = _mm512_loadu_si512((const __m512i *)(ptr + offset)); return _mm512_cmpeq_epi32_mask(_mm512_sub_epi32(f, l), pattern); } -MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_AVX512BW static pgno_t * -scan4seq_avx512bw(pgno_t *range, const size_t len, const size_t seq) { +MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_AVX512BW static pgno_t *scan4seq_avx512bw(pgno_t *range, const size_t len, + const size_t seq) { assert(seq > 0 && len > seq); #if MDBX_PNL_ASCENDING #error "FIXME: Not implemented" @@ -442,8 +421,7 @@ scan4seq_avx512bw(pgno_t *range, const size_t len, const size_t seq) { * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */ #if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__) const unsigned on_page_safe_mask = 0xfc0 /* enough for '-63' bytes offset */; - if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && - !RUNNING_ON_VALGRIND) { + if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) { const unsigned extra = (unsigned)(detent + 16 - range); assert(extra > 0 && extra < 16); mask = 0xFFFF << extra; @@ -474,10 +452,8 @@ scan4seq_avx512bw(pgno_t *range, const size_t len, const size_t seq) { } #endif /* MDBX_ATTRIBUTE_TARGET_AVX512BW */ -#if (defined(__ARM_NEON) || defined(__ARM_NEON__)) && \ - (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) -static __always_inline size_t diffcmp2mask_neon(const pgno_t *const ptr, - const ptrdiff_t offset, +#if (defined(__ARM_NEON) || defined(__ARM_NEON__)) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +static __always_inline size_t diffcmp2mask_neon(const pgno_t *const ptr, const ptrdiff_t offset, const uint32x4_t pattern) { const uint32x4_t f = vld1q_u32(ptr); const uint32x4_t l = vld1q_u32(ptr + offset); @@ -485,12 +461,10 @@ static __always_inline size_t diffcmp2mask_neon(const pgno_t *const ptr, if (sizeof(size_t) > 7) return vget_lane_u64(vreinterpret_u64_u16(cmp), 0); else - return vget_lane_u32(vreinterpret_u32_u8(vmovn_u16(vcombine_u16(cmp, cmp))), - 0); + return vget_lane_u32(vreinterpret_u32_u8(vmovn_u16(vcombine_u16(cmp, cmp))), 0); } -__hot static pgno_t *scan4seq_neon(pgno_t *range, const size_t len, - const size_t seq) { +__hot static pgno_t *scan4seq_neon(pgno_t *range, const size_t len, const size_t seq) { assert(seq > 0 && len > seq); #if MDBX_PNL_ASCENDING #error "FIXME: Not implemented" @@ -522,8 +496,7 @@ __hot static pgno_t *scan4seq_neon(pgno_t *range, const size_t len, * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */ #if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__) const unsigned on_page_safe_mask = 0xff0 /* enough for '-15' bytes offset */; - if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && - !RUNNING_ON_VALGRIND) { + if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) { const unsigned extra = (unsigned)(detent + 4 - range); assert(extra > 0 && extra < 4); mask = (~(size_t)0) << (extra * sizeof(size_t) * 2); @@ -548,8 +521,7 @@ __hot static pgno_t *scan4seq_neon(pgno_t *range, const size_t len, #define scan4seq_default scan4seq_avx2 #elif defined(__SSE2__) && defined(MDBX_ATTRIBUTE_TARGET_SSE2) #define scan4seq_default scan4seq_sse2 -#elif (defined(__ARM_NEON) || defined(__ARM_NEON__)) && \ - (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +#elif (defined(__ARM_NEON) || defined(__ARM_NEON__)) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) #define scan4seq_default scan4seq_neon /* Choosing of another variants should be added here. */ #endif /* scan4seq_default */ @@ -570,17 +542,12 @@ __hot static pgno_t *scan4seq_neon(pgno_t *range, const size_t len, #else /* Selecting the most appropriate implementation at runtime, * depending on the available CPU features. */ -static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len, - const size_t seq); -static pgno_t *(*scan4seq_impl)(pgno_t *range, const size_t len, - const size_t seq) = scan4seq_resolver; +static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len, const size_t seq); +static pgno_t *(*scan4seq_impl)(pgno_t *range, const size_t len, const size_t seq) = scan4seq_resolver; -static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len, - const size_t seq) { - pgno_t *(*choice)(pgno_t *range, const size_t len, const size_t seq) = - nullptr; -#if __has_builtin(__builtin_cpu_init) || defined(__BUILTIN_CPU_INIT__) || \ - __GNUC_PREREQ(4, 8) +static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len, const size_t seq) { + pgno_t *(*choice)(pgno_t *range, const size_t len, const size_t seq) = nullptr; +#if __has_builtin(__builtin_cpu_init) || defined(__BUILTIN_CPU_INIT__) || __GNUC_PREREQ(4, 8) __builtin_cpu_init(); #endif /* __builtin_cpu_init() */ #ifdef MDBX_ATTRIBUTE_TARGET_SSE2 @@ -607,12 +574,10 @@ static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len, #define ALLOC_SHOULD_SCAN 8 /* внутреннее состояние */ #define ALLOC_LIFO 16 /* внутреннее состояние */ -static inline bool is_gc_usable(MDBX_txn *txn, const MDBX_cursor *mc, - const uint8_t flags) { +static inline bool is_gc_usable(MDBX_txn *txn, const MDBX_cursor *mc, const uint8_t flags) { /* If txn is updating the GC, then the retired-list cannot play catch-up with * itself by growing while trying to save it. */ - if (mc->tree == &txn->dbs[FREE_DBI] && !(flags & ALLOC_RESERVE) && - !(mc->flags & z_gcu_preparation)) + if (mc->tree == &txn->dbs[FREE_DBI] && !(flags & ALLOC_RESERVE) && !(mc->flags & z_gcu_preparation)) return false; /* avoid search inside empty tree and while tree is updating, @@ -690,8 +655,7 @@ __hot static pgno_t relist_get_single(MDBX_txn *txn) { #ifndef MDBX_ENABLE_SAVING_SEQUENCES #define MDBX_ENABLE_SAVING_SEQUENCES 0 #endif - if (MDBX_ENABLE_SAVING_SEQUENCES && unlikely(target[dir] == *target + 1) && - len > 2) { + if (MDBX_ENABLE_SAVING_SEQUENCES && unlikely(target[dir] == *target + 1) && len > 2) { /* Пытаемся пропускать последовательности при наличии одиночных элементов. * TODO: необходимо кэшировать пропускаемые последовательности * чтобы не сканировать список сначала при каждом выделении. */ @@ -719,8 +683,7 @@ __hot static pgno_t relist_get_single(MDBX_txn *txn) { #if MDBX_PNL_ASCENDING /* вырезаем элемент с перемещением хвоста */ MDBX_PNL_SETSIZE(txn->tw.relist, len - 1); - for (const pgno_t *const end = txn->tw.relist + len - 1; target <= end; - ++target) + for (const pgno_t *const end = txn->tw.relist + len - 1; target <= end; ++target) *target = target[1]; #else /* перемещать хвост не нужно, просто усекам список */ @@ -729,8 +692,7 @@ __hot static pgno_t relist_get_single(MDBX_txn *txn) { return pgno; } -__hot static pgno_t relist_get_sequence(MDBX_txn *txn, const size_t num, - uint8_t flags) { +__hot static pgno_t relist_get_sequence(MDBX_txn *txn, const size_t num, uint8_t flags) { const size_t len = MDBX_PNL_GETSIZE(txn->tw.relist); pgno_t *edge = MDBX_PNL_EDGE(txn->tw.relist); assert(len >= num && num > 1); @@ -754,8 +716,7 @@ __hot static pgno_t relist_get_sequence(MDBX_txn *txn, const size_t num, /* вырезаем найденную последовательность с перемещением хвоста */ MDBX_PNL_SETSIZE(txn->tw.relist, len - num); #if MDBX_PNL_ASCENDING - for (const pgno_t *const end = txn->tw.relist + len - num; target <= end; - ++target) + for (const pgno_t *const end = txn->tw.relist + len - num; target <= end; ++target) *target = target[num]; #else for (const pgno_t *const end = txn->tw.relist + len; ++target <= end;) @@ -766,16 +727,13 @@ __hot static pgno_t relist_get_sequence(MDBX_txn *txn, const size_t num, return 0; } -static inline pgr_t page_alloc_finalize(MDBX_env *const env, - MDBX_txn *const txn, - const MDBX_cursor *const mc, +static inline pgr_t page_alloc_finalize(MDBX_env *const env, MDBX_txn *const txn, const MDBX_cursor *const mc, const pgno_t pgno, const size_t num) { #if MDBX_ENABLE_PROFGC size_t majflt_before; const uint64_t cputime_before = osal_cputime(&majflt_before); - gc_prof_stat_t *const prof = (cursor_dbi(mc) == FREE_DBI) - ? &env->lck->pgops.gc_prof.self - : &env->lck->pgops.gc_prof.work; + gc_prof_stat_t *const prof = + (cursor_dbi(mc) == FREE_DBI) ? &env->lck->pgops.gc_prof.self : &env->lck->pgops.gc_prof.work; #else (void)mc; #endif /* MDBX_ENABLE_PROFGC */ @@ -811,8 +769,7 @@ static inline pgr_t page_alloc_finalize(MDBX_env *const env, * грязной I/O очереди. Из-за этого штраф за лишнюю запись может быть * сравним с избегаемым ненужным чтением. */ if (env->prefault_write_activated) { - void *const pattern = - ptr_disp(env->page_auxbuf, need_clean ? env->ps : env->ps * 2); + void *const pattern = ptr_disp(env->page_auxbuf, need_clean ? env->ps : env->ps * 2); size_t file_offset = pgno2bytes(env, pgno); if (likely(num == 1)) { if (!mincore_probe(env, pgno)) { @@ -831,8 +788,7 @@ static inline pgr_t page_alloc_finalize(MDBX_env *const env, iov[n].iov_len = env->ps; iov[n].iov_base = pattern; if (unlikely(++n == MDBX_AUXILARY_IOV_MAX)) { - osal_pwritev(env->lazy_fd, iov, MDBX_AUXILARY_IOV_MAX, - file_offset); + osal_pwritev(env->lazy_fd, iov, MDBX_AUXILARY_IOV_MAX, file_offset); #if MDBX_ENABLE_PGOP_STAT env->lck->pgops.prefault.weak += 1; #endif /* MDBX_ENABLE_PGOP_STAT */ @@ -873,8 +829,7 @@ static inline pgr_t page_alloc_finalize(MDBX_env *const env, ret.err = page_dirty(txn, ret.page, (pgno_t)num); bailout: - tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); #if MDBX_ENABLE_PROFGC size_t majflt_after; prof->xtime_cpu += osal_cputime(&majflt_after) - cputime_before; @@ -883,32 +838,25 @@ bailout: return ret; } -pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, - uint8_t flags) { +pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags) { pgr_t ret; MDBX_txn *const txn = mc->txn; MDBX_env *const env = txn->env; #if MDBX_ENABLE_PROFGC - gc_prof_stat_t *const prof = (cursor_dbi(mc) == FREE_DBI) - ? &env->lck->pgops.gc_prof.self - : &env->lck->pgops.gc_prof.work; + gc_prof_stat_t *const prof = + (cursor_dbi(mc) == FREE_DBI) ? &env->lck->pgops.gc_prof.self : &env->lck->pgops.gc_prof.work; prof->spe_counter += 1; #endif /* MDBX_ENABLE_PROFGC */ eASSERT(env, num > 0 || (flags & ALLOC_RESERVE)); - eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - - MDBX_ENABLE_REFUND)); + eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); size_t newnext; - const uint64_t monotime_begin = - (MDBX_ENABLE_PROFGC || (num > 1 && env->options.gc_time_limit)) - ? osal_monotime() - : 0; + const uint64_t monotime_begin = (MDBX_ENABLE_PROFGC || (num > 1 && env->options.gc_time_limit)) ? osal_monotime() : 0; struct monotime_cache now_cache; - now_cache.expire_countdown = - 1 /* старт с 1 позволяет избавиться как от лишних системных вызовов когда - лимит времени задан нулевой или уже исчерпан, так и от подсчета - времени при не-достижении rp_augment_limit */ + now_cache.expire_countdown = 1 /* старт с 1 позволяет избавиться как от лишних системных вызовов когда + лимит времени задан нулевой или уже исчерпан, так и от подсчета + времени при не-достижении rp_augment_limit */ ; now_cache.value = monotime_begin; pgno_t pgno = 0; @@ -917,9 +865,8 @@ pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, prof->xpages += 1; #endif /* MDBX_ENABLE_PROFGC */ if (MDBX_PNL_GETSIZE(txn->tw.relist) >= num) { - eASSERT(env, - MDBX_PNL_LAST(txn->tw.relist) < txn->geo.first_unallocated && - MDBX_PNL_FIRST(txn->tw.relist) < txn->geo.first_unallocated); + eASSERT(env, MDBX_PNL_LAST(txn->tw.relist) < txn->geo.first_unallocated && + MDBX_PNL_FIRST(txn->tw.relist) < txn->geo.first_unallocated); pgno = relist_get_sequence(txn, num, flags); if (likely(pgno)) goto done; @@ -936,16 +883,14 @@ pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, goto no_gc; } - eASSERT(env, - (flags & (ALLOC_COALESCE | ALLOC_LIFO | ALLOC_SHOULD_SCAN)) == 0); + eASSERT(env, (flags & (ALLOC_COALESCE | ALLOC_LIFO | ALLOC_SHOULD_SCAN)) == 0); flags += (env->flags & MDBX_LIFORECLAIM) ? ALLOC_LIFO : 0; if (/* Не коагулируем записи при подготовке резерва для обновления GC. * Иначе попытка увеличить резерв может приводить к необходимости ещё * большего резерва из-за увеличения списка переработанных страниц. */ (flags & ALLOC_RESERVE) == 0) { - if (txn->dbs[FREE_DBI].branch_pages && - MDBX_PNL_GETSIZE(txn->tw.relist) < env->maxgc_large1page / 2) + if (txn->dbs[FREE_DBI].branch_pages && MDBX_PNL_GETSIZE(txn->tw.relist) < env->maxgc_large1page / 2) flags += ALLOC_COALESCE; } @@ -976,9 +921,7 @@ retry_gc_refresh_oldest:; txnid_t oldest = txn_snapshot_oldest(txn); retry_gc_have_oldest: if (unlikely(oldest >= txn->txnid)) { - ERROR("unexpected/invalid oldest-readed txnid %" PRIaTXN - " for current-txnid %" PRIaTXN, - oldest, txn->txnid); + ERROR("unexpected/invalid oldest-readed txnid %" PRIaTXN " for current-txnid %" PRIaTXN, oldest, txn->txnid); ret.err = MDBX_PROBLEM; goto fail; } @@ -1026,8 +969,7 @@ next_gc:; goto depleted_gc; } if (unlikely(key.iov_len != sizeof(txnid_t))) { - ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid GC key-length"); + ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC key-length"); ret.err = MDBX_CORRUPTED; goto fail; } @@ -1046,26 +988,21 @@ next_gc:; /* Reading next GC record */ MDBX_val data; page_t *const mp = gc->pg[gc->top]; - if (unlikely((ret.err = node_read(gc, page_node(mp, gc->ki[gc->top]), &data, - mp)) != MDBX_SUCCESS)) + if (unlikely((ret.err = node_read(gc, page_node(mp, gc->ki[gc->top]), &data, mp)) != MDBX_SUCCESS)) goto fail; pgno_t *gc_pnl = (pgno_t *)data.iov_base; - if (unlikely(data.iov_len % sizeof(pgno_t) || - data.iov_len < MDBX_PNL_SIZEOF(gc_pnl) || + if (unlikely(data.iov_len % sizeof(pgno_t) || data.iov_len < MDBX_PNL_SIZEOF(gc_pnl) || !pnl_check(gc_pnl, txn->geo.first_unallocated))) { - ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid GC value-length"); + ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC value-length"); ret.err = MDBX_CORRUPTED; goto fail; } const size_t gc_len = MDBX_PNL_GETSIZE(gc_pnl); - TRACE("gc-read: id #%" PRIaTXN " len %zu, re-list will %zu ", id, gc_len, - gc_len + MDBX_PNL_GETSIZE(txn->tw.relist)); + TRACE("gc-read: id #%" PRIaTXN " len %zu, re-list will %zu ", id, gc_len, gc_len + MDBX_PNL_GETSIZE(txn->tw.relist)); - if (unlikely(gc_len + MDBX_PNL_GETSIZE(txn->tw.relist) >= - env->maxgc_large1page)) { + if (unlikely(gc_len + MDBX_PNL_GETSIZE(txn->tw.relist) >= env->maxgc_large1page)) { /* Don't try to coalesce too much. */ if (flags & ALLOC_SHOULD_SCAN) { eASSERT(env, flags & ALLOC_COALESCE); @@ -1076,10 +1013,8 @@ next_gc:; #endif /* MDBX_ENABLE_PROFGC */ TRACE("clear %s %s", "ALLOC_COALESCE", "since got threshold"); if (MDBX_PNL_GETSIZE(txn->tw.relist) >= num) { - eASSERT(env, - MDBX_PNL_LAST(txn->tw.relist) < txn->geo.first_unallocated && - MDBX_PNL_FIRST(txn->tw.relist) < - txn->geo.first_unallocated); + eASSERT(env, MDBX_PNL_LAST(txn->tw.relist) < txn->geo.first_unallocated && + MDBX_PNL_FIRST(txn->tw.relist) < txn->geo.first_unallocated); if (likely(num == 1)) { pgno = relist_get_single(txn); goto done; @@ -1090,25 +1025,19 @@ next_gc:; } flags -= ALLOC_COALESCE | ALLOC_SHOULD_SCAN; } - if (unlikely(/* list is too long already */ MDBX_PNL_GETSIZE( - txn->tw.relist) >= env->options.rp_augment_limit) && + if (unlikely(/* list is too long already */ MDBX_PNL_GETSIZE(txn->tw.relist) >= env->options.rp_augment_limit) && ((/* not a slot-request from gc-update */ num && - /* have enough unallocated space */ txn->geo.upper >= - txn->geo.first_unallocated + num && - monotime_since_cached(monotime_begin, &now_cache) + - txn->tw.gc.time_acc >= - env->options.gc_time_limit) || + /* have enough unallocated space */ txn->geo.upper >= txn->geo.first_unallocated + num && + monotime_since_cached(monotime_begin, &now_cache) + txn->tw.gc.time_acc >= env->options.gc_time_limit) || gc_len + MDBX_PNL_GETSIZE(txn->tw.relist) >= PAGELIST_LIMIT)) { /* Stop reclaiming to avoid large/overflow the page list. This is a rare * case while search for a continuously multi-page region in a * large database, see https://libmdbx.dqdkfa.ru/dead-github/issues/123 */ NOTICE("stop reclaiming %s: %zu (current) + %zu " "(chunk) -> %zu, rp_augment_limit %u", - likely(gc_len + MDBX_PNL_GETSIZE(txn->tw.relist) < PAGELIST_LIMIT) - ? "since rp_augment_limit was reached" - : "to avoid PNL overflow", - MDBX_PNL_GETSIZE(txn->tw.relist), gc_len, - gc_len + MDBX_PNL_GETSIZE(txn->tw.relist), + likely(gc_len + MDBX_PNL_GETSIZE(txn->tw.relist) < PAGELIST_LIMIT) ? "since rp_augment_limit was reached" + : "to avoid PNL overflow", + MDBX_PNL_GETSIZE(txn->tw.relist), gc_len, gc_len + MDBX_PNL_GETSIZE(txn->tw.relist), env->options.rp_augment_limit); goto depleted_gc; } @@ -1128,9 +1057,7 @@ next_gc:; goto fail; if (LOG_ENABLED(MDBX_LOG_EXTRA)) { - DEBUG_EXTRA("readed GC-pnl txn %" PRIaTXN " root %" PRIaPGNO - " len %zu, PNL", - id, txn->dbs[FREE_DBI].root, gc_len); + DEBUG_EXTRA("readed GC-pnl txn %" PRIaTXN " root %" PRIaPGNO " len %zu, PNL", id, txn->dbs[FREE_DBI].root, gc_len); for (size_t i = gc_len; i; i--) DEBUG_EXTRA_PRINT(" %" PRIaPGNO, gc_pnl[i]); DEBUG_EXTRA_PRINT(", first_unallocated %u\n", txn->geo.first_unallocated); @@ -1141,33 +1068,27 @@ next_gc:; flags |= ALLOC_SHOULD_SCAN; if (AUDIT_ENABLED()) { if (unlikely(!pnl_check(txn->tw.relist, txn->geo.first_unallocated))) { - ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid txn retired-list"); + ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid txn retired-list"); ret.err = MDBX_CORRUPTED; goto fail; } } else { - eASSERT(env, - pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated)); + eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated)); } eASSERT(env, dpl_check(txn)); - eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.relist) == 0 || - MDBX_PNL_MOST(txn->tw.relist) < txn->geo.first_unallocated); + eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.relist) == 0 || MDBX_PNL_MOST(txn->tw.relist) < txn->geo.first_unallocated); if (MDBX_ENABLE_REFUND && MDBX_PNL_GETSIZE(txn->tw.relist) && - unlikely(MDBX_PNL_MOST(txn->tw.relist) == - txn->geo.first_unallocated - 1)) { + unlikely(MDBX_PNL_MOST(txn->tw.relist) == txn->geo.first_unallocated - 1)) { /* Refund suitable pages into "unallocated" space */ txn_refund(txn); } - eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - - MDBX_ENABLE_REFUND)); + eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); /* Done for a kick-reclaim mode, actually no page needed */ if (unlikely(num == 0)) { eASSERT(env, ret.err == MDBX_SUCCESS); - TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "early-exit for slot", id, - MDBX_PNL_GETSIZE(txn->tw.relist)); + TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "early-exit for slot", id, MDBX_PNL_GETSIZE(txn->tw.relist)); goto early_exit; } @@ -1175,8 +1096,7 @@ next_gc:; eASSERT(env, op == MDBX_PREV || op == MDBX_NEXT); if (flags & ALLOC_COALESCE) { - TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "coalesce-continue", id, - MDBX_PNL_GETSIZE(txn->tw.relist)); + TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "coalesce-continue", id, MDBX_PNL_GETSIZE(txn->tw.relist)); goto next_gc; } @@ -1184,9 +1104,8 @@ scan: eASSERT(env, flags & ALLOC_SHOULD_SCAN); eASSERT(env, num > 0); if (MDBX_PNL_GETSIZE(txn->tw.relist) >= num) { - eASSERT(env, - MDBX_PNL_LAST(txn->tw.relist) < txn->geo.first_unallocated && - MDBX_PNL_FIRST(txn->tw.relist) < txn->geo.first_unallocated); + eASSERT(env, MDBX_PNL_LAST(txn->tw.relist) < txn->geo.first_unallocated && + MDBX_PNL_FIRST(txn->tw.relist) < txn->geo.first_unallocated); if (likely(num == 1)) { eASSERT(env, !(flags & ALLOC_RESERVE)); pgno = relist_get_single(txn); @@ -1198,14 +1117,12 @@ scan: } flags -= ALLOC_SHOULD_SCAN; if (ret.err == MDBX_SUCCESS) { - TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "continue-search", id, - MDBX_PNL_GETSIZE(txn->tw.relist)); + TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "continue-search", id, MDBX_PNL_GETSIZE(txn->tw.relist)); goto next_gc; } depleted_gc: - TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "gc-depleted", id, - MDBX_PNL_GETSIZE(txn->tw.relist)); + TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "gc-depleted", id, MDBX_PNL_GETSIZE(txn->tw.relist)); ret.err = MDBX_NOTFOUND; if (flags & ALLOC_SHOULD_SCAN) goto scan; @@ -1226,16 +1143,11 @@ depleted_gc: /* Does reclaiming stopped at the last steady point? */ const meta_ptr_t recent = meta_recent(env, &txn->tw.troika); const meta_ptr_t prefer_steady = meta_prefer_steady(env, &txn->tw.troika); - if (recent.ptr_c != prefer_steady.ptr_c && prefer_steady.is_steady && - detent == prefer_steady.txnid + 1) { - DEBUG("gc-kick-steady: recent %" PRIaTXN "-%s, steady %" PRIaTXN - "-%s, detent %" PRIaTXN, - recent.txnid, durable_caption(recent.ptr_c), prefer_steady.txnid, - durable_caption(prefer_steady.ptr_c), detent); - const pgno_t autosync_threshold = - atomic_load32(&env->lck->autosync_threshold, mo_Relaxed); - const uint64_t autosync_period = - atomic_load64(&env->lck->autosync_period, mo_Relaxed); + if (recent.ptr_c != prefer_steady.ptr_c && prefer_steady.is_steady && detent == prefer_steady.txnid + 1) { + DEBUG("gc-kick-steady: recent %" PRIaTXN "-%s, steady %" PRIaTXN "-%s, detent %" PRIaTXN, recent.txnid, + durable_caption(recent.ptr_c), prefer_steady.txnid, durable_caption(prefer_steady.ptr_c), detent); + const pgno_t autosync_threshold = atomic_load32(&env->lck->autosync_threshold, mo_Relaxed); + const uint64_t autosync_period = atomic_load64(&env->lck->autosync_period, mo_Relaxed); uint64_t eoos_timestamp; /* wipe the last steady-point if one of: * - UTTERLY_NOSYNC mode AND auto-sync threshold is NOT specified @@ -1246,8 +1158,7 @@ depleted_gc: * - database is full (with the current file size) * AND auto-sync threshold it NOT specified */ if (F_ISSET(env->flags, MDBX_UTTERLY_NOSYNC) && - ((autosync_threshold | autosync_period) == 0 || - newnext >= prefer_steady.ptr_c->geometry.now)) { + ((autosync_threshold | autosync_period) == 0 || newnext >= prefer_steady.ptr_c->geometry.now)) { /* wipe steady checkpoint in MDBX_UTTERLY_NOSYNC mode * without any auto-sync threshold(s). */ #if MDBX_ENABLE_PROFGC @@ -1257,39 +1168,30 @@ depleted_gc: DEBUG("gc-wipe-steady, rc %d", ret.err); if (unlikely(ret.err != MDBX_SUCCESS)) goto fail; - eASSERT(env, prefer_steady.ptr_c != - meta_prefer_steady(env, &txn->tw.troika).ptr_c); + eASSERT(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->tw.troika).ptr_c); goto retry_gc_refresh_oldest; } - if ((autosync_threshold && - atomic_load64(&env->lck->unsynced_pages, mo_Relaxed) >= - autosync_threshold) || - (autosync_period && - (eoos_timestamp = - atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) && + if ((autosync_threshold && atomic_load64(&env->lck->unsynced_pages, mo_Relaxed) >= autosync_threshold) || + (autosync_period && (eoos_timestamp = atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) && osal_monotime() - eoos_timestamp >= autosync_period) || newnext >= txn->geo.upper || - ((num == 0 || newnext >= txn->geo.end_pgno) && - (autosync_threshold | autosync_period) == 0)) { + ((num == 0 || newnext >= txn->geo.end_pgno) && (autosync_threshold | autosync_period) == 0)) { /* make steady checkpoint. */ #if MDBX_ENABLE_PROFGC env->lck->pgops.gc_prof.flushes += 1; #endif /* MDBX_ENABLE_PROFGC */ meta_t meta = *recent.ptr_c; - ret.err = dxb_sync_locked(env, env->flags & MDBX_WRITEMAP, &meta, - &txn->tw.troika); + ret.err = dxb_sync_locked(env, env->flags & MDBX_WRITEMAP, &meta, &txn->tw.troika); DEBUG("gc-make-steady, rc %d", ret.err); eASSERT(env, ret.err != MDBX_RESULT_TRUE); if (unlikely(ret.err != MDBX_SUCCESS)) goto fail; - eASSERT(env, prefer_steady.ptr_c != - meta_prefer_steady(env, &txn->tw.troika).ptr_c); + eASSERT(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->tw.troika).ptr_c); goto retry_gc_refresh_oldest; } } - if (unlikely(true == - atomic_load32(&env->lck->rdt_refresh_flag, mo_AcquireRelease))) { + if (unlikely(true == atomic_load32(&env->lck->rdt_refresh_flag, mo_AcquireRelease))) { oldest = txn_snapshot_oldest(txn); if (oldest >= detent) goto retry_gc_have_oldest; @@ -1315,8 +1217,7 @@ no_gc: #ifndef MDBX_ENABLE_BACKLOG_DEPLETED #define MDBX_ENABLE_BACKLOG_DEPLETED 0 #endif /* MDBX_ENABLE_BACKLOG_DEPLETED*/ - if (MDBX_ENABLE_BACKLOG_DEPLETED && - unlikely(!(txn->flags & txn_gc_drained))) { + if (MDBX_ENABLE_BACKLOG_DEPLETED && unlikely(!(txn->flags & txn_gc_drained))) { ret.err = MDBX_BACKLOG_DEPLETED; goto fail; } @@ -1338,20 +1239,16 @@ no_gc: eASSERT(env, newnext > txn->geo.end_pgno); const size_t grow_step = pv2pages(txn->geo.grow_pv); - size_t aligned = pgno_align2os_pgno( - env, (pgno_t)(newnext + grow_step - newnext % grow_step)); + size_t aligned = pgno_align2os_pgno(env, (pgno_t)(newnext + grow_step - newnext % grow_step)); if (aligned > txn->geo.upper) aligned = txn->geo.upper; eASSERT(env, aligned >= newnext); - VERBOSE("try growth datafile to %zu pages (+%zu)", aligned, - aligned - txn->geo.end_pgno); - ret.err = dxb_resize(env, txn->geo.first_unallocated, (pgno_t)aligned, - txn->geo.upper, implicit_grow); + VERBOSE("try growth datafile to %zu pages (+%zu)", aligned, aligned - txn->geo.end_pgno); + ret.err = dxb_resize(env, txn->geo.first_unallocated, (pgno_t)aligned, txn->geo.upper, implicit_grow); if (ret.err != MDBX_SUCCESS) { - ERROR("unable growth datafile to %zu pages (+%zu), errcode %d", aligned, - aligned - txn->geo.end_pgno, ret.err); + ERROR("unable growth datafile to %zu pages (+%zu), errcode %d", aligned, aligned - txn->geo.end_pgno, ret.err); goto fail; } env->txn->geo.end_pgno = (pgno_t)aligned; @@ -1363,26 +1260,20 @@ done: ret.err = MDBX_SUCCESS; if (likely((flags & ALLOC_RESERVE) == 0)) { if (pgno) { - eASSERT(env, - pgno + num <= txn->geo.first_unallocated && pgno >= NUM_METAS); - eASSERT(env, - pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - - MDBX_ENABLE_REFUND)); + eASSERT(env, pgno + num <= txn->geo.first_unallocated && pgno >= NUM_METAS); + eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); } else { pgno = txn->geo.first_unallocated; txn->geo.first_unallocated += (pgno_t)num; eASSERT(env, txn->geo.first_unallocated <= txn->geo.end_pgno); - eASSERT(env, - pgno >= NUM_METAS && pgno + num <= txn->geo.first_unallocated); + eASSERT(env, pgno >= NUM_METAS && pgno + num <= txn->geo.first_unallocated); } ret = page_alloc_finalize(env, txn, mc, pgno, num); if (unlikely(ret.err != MDBX_SUCCESS)) { fail: eASSERT(env, ret.err != MDBX_SUCCESS); - eASSERT(env, - pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - - MDBX_ENABLE_REFUND)); + eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); int level; const char *what; if (flags & ALLOC_RESERVE) { @@ -1398,12 +1289,9 @@ done: "unable alloc %zu %s, alloc-flags 0x%x, err %d, txn-flags " "0x%x, re-list-len %zu, loose-count %zu, gc: height %u, " "branch %zu, leaf %zu, large %zu, entries %zu\n", - num, what, flags, ret.err, txn->flags, - MDBX_PNL_GETSIZE(txn->tw.relist), txn->tw.loose_count, - txn->dbs[FREE_DBI].height, - (size_t)txn->dbs[FREE_DBI].branch_pages, - (size_t)txn->dbs[FREE_DBI].leaf_pages, - (size_t)txn->dbs[FREE_DBI].large_pages, + num, what, flags, ret.err, txn->flags, MDBX_PNL_GETSIZE(txn->tw.relist), txn->tw.loose_count, + txn->dbs[FREE_DBI].height, (size_t)txn->dbs[FREE_DBI].branch_pages, + (size_t)txn->dbs[FREE_DBI].leaf_pages, (size_t)txn->dbs[FREE_DBI].large_pages, (size_t)txn->dbs[FREE_DBI].items); ret.page = nullptr; } @@ -1411,8 +1299,7 @@ done: txn->tw.gc.time_acc += monotime_since_cached(monotime_begin, &now_cache); } else { early_exit: - DEBUG("return nullptr for %zu pages for ALLOC_%s, rc %d", num, - num ? "RESERVE" : "SLOT", ret.err); + DEBUG("return nullptr for %zu pages for ALLOC_%s, rc %d", num, num ? "RESERVE" : "SLOT", ret.err); ret.page = nullptr; } @@ -1425,8 +1312,7 @@ done: __hot pgr_t gc_alloc_single(const MDBX_cursor *const mc) { MDBX_txn *const txn = mc->txn; tASSERT(txn, mc->txn->flags & MDBX_TXN_DIRTY); - tASSERT(txn, - F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY)); + tASSERT(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY)); /* If there are any loose pages, just use them */ while (likely(txn->tw.loose_pages)) { @@ -1443,8 +1329,7 @@ __hot pgr_t gc_alloc_single(const MDBX_cursor *const mc) { VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); txn->tw.loose_pages = page_next(lp); txn->tw.loose_count--; - DEBUG_EXTRA("db %d use loose page %" PRIaPGNO, cursor_dbi_dbg(mc), - lp->pgno); + DEBUG_EXTRA("db %d use loose page %" PRIaPGNO, cursor_dbi_dbg(mc), lp->pgno); tASSERT(txn, lp->pgno < txn->geo.first_unallocated); tASSERT(txn, lp->pgno >= NUM_METAS); VALGRIND_MAKE_MEM_UNDEFINED(page_data(lp), page_space(txn->env)); diff --git a/src/gc-put.c b/src/gc-put.c index c8648830..04999c6a 100644 --- a/src/gc-put.c +++ b/src/gc-put.c @@ -11,9 +11,7 @@ MDBX_MAYBE_UNUSED static inline const char *dbg_prefix(const gcu_t *ctx) { return is_lifo(ctx->cursor.txn) ? " lifo" : " fifo"; } -static inline size_t backlog_size(MDBX_txn *txn) { - return MDBX_PNL_GETSIZE(txn->tw.relist) + txn->tw.loose_count; -} +static inline size_t backlog_size(MDBX_txn *txn) { return MDBX_PNL_GETSIZE(txn->tw.relist) + txn->tw.loose_count; } static int clean_stored_retired(MDBX_txn *txn, gcu_t *ctx) { int err = MDBX_SUCCESS; @@ -53,8 +51,7 @@ static int clean_stored_retired(MDBX_txn *txn, gcu_t *ctx) { } static int touch_gc(gcu_t *ctx) { - tASSERT(ctx->cursor.txn, is_pointed(&ctx->cursor) || - ctx->cursor.txn->dbs[FREE_DBI].leaf_pages == 0); + tASSERT(ctx->cursor.txn, is_pointed(&ctx->cursor) || ctx->cursor.txn->dbs[FREE_DBI].leaf_pages == 0); MDBX_val key, val; key.iov_base = val.iov_base = nullptr; key.iov_len = sizeof(txnid_t); @@ -70,24 +67,19 @@ static int touch_gc(gcu_t *ctx) { * during a deleting, when GC tree is unbalanced. */ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) { const size_t for_cow = txn->dbs[FREE_DBI].height; - const size_t for_rebalance = - for_cow + 1 + - (txn->dbs[FREE_DBI].height + 1ul >= txn->dbs[FREE_DBI].branch_pages); + const size_t for_rebalance = for_cow + 1 + (txn->dbs[FREE_DBI].height + 1ul >= txn->dbs[FREE_DBI].branch_pages); size_t for_split = ctx->retired_stored == 0; tASSERT(txn, is_pointed(&ctx->cursor) || txn->dbs[FREE_DBI].leaf_pages == 0); - const intptr_t retired_left = - MDBX_PNL_SIZEOF(txn->tw.retired_pages) - ctx->retired_stored; + const intptr_t retired_left = MDBX_PNL_SIZEOF(txn->tw.retired_pages) - ctx->retired_stored; size_t for_relist = 0; if (MDBX_ENABLE_BIGFOOT && retired_left > 0) { - for_relist = (retired_left + txn->env->maxgc_large1page - 1) / - txn->env->maxgc_large1page; + for_relist = (retired_left + txn->env->maxgc_large1page - 1) / txn->env->maxgc_large1page; const size_t per_branch_page = txn->env->maxgc_per_branch; for (size_t entries = for_relist; entries > 1; for_split += entries) entries = (entries + per_branch_page - 1) / per_branch_page; } else if (!MDBX_ENABLE_BIGFOOT && retired_left != 0) { - for_relist = - largechunk_npages(txn->env, MDBX_PNL_SIZEOF(txn->tw.retired_pages)); + for_relist = largechunk_npages(txn->env, MDBX_PNL_SIZEOF(txn->tw.retired_pages)); } const size_t for_tree_before_touch = for_cow + for_rebalance + for_split; @@ -96,23 +88,20 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) { const size_t for_all_after_touch = for_relist + for_tree_after_touch; if (likely(for_relist < 2 && backlog_size(txn) > for_all_before_touch) && - (ctx->cursor.top < 0 || - is_modifable(txn, ctx->cursor.pg[ctx->cursor.top]))) + (ctx->cursor.top < 0 || is_modifable(txn, ctx->cursor.pg[ctx->cursor.top]))) return MDBX_SUCCESS; TRACE(">> retired-stored %zu, left %zi, backlog %zu, need %zu (4list %zu, " "4split %zu, " "4cow %zu, 4tree %zu)", - ctx->retired_stored, retired_left, backlog_size(txn), - for_all_before_touch, for_relist, for_split, for_cow, + ctx->retired_stored, retired_left, backlog_size(txn), for_all_before_touch, for_relist, for_split, for_cow, for_tree_before_touch); int err = touch_gc(ctx); TRACE("== after-touch, backlog %zu, err %d", backlog_size(txn), err); if (!MDBX_ENABLE_BIGFOOT && unlikely(for_relist > 1) && - MDBX_PNL_GETSIZE(txn->tw.retired_pages) != ctx->retired_stored && - err == MDBX_SUCCESS) { + MDBX_PNL_GETSIZE(txn->tw.retired_pages) != ctx->retired_stored && err == MDBX_SUCCESS) { if (unlikely(ctx->retired_stored)) { err = clean_stored_retired(txn, ctx); if (unlikely(err != MDBX_SUCCESS)) @@ -122,8 +111,7 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) { } err = gc_alloc_ex(&ctx->cursor, for_relist, ALLOC_RESERVE).err; TRACE("== after-4linear, backlog %zu, err %d", backlog_size(txn), err); - cASSERT(&ctx->cursor, - backlog_size(txn) >= for_relist || err != MDBX_SUCCESS); + cASSERT(&ctx->cursor, backlog_size(txn) >= for_relist || err != MDBX_SUCCESS); } while (backlog_size(txn) < for_all_after_touch && err == MDBX_SUCCESS) @@ -131,10 +119,8 @@ static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) { TRACE("<< backlog %zu, err %d, gc: height %u, branch %zu, leaf %zu, large " "%zu, entries %zu", - backlog_size(txn), err, txn->dbs[FREE_DBI].height, - (size_t)txn->dbs[FREE_DBI].branch_pages, - (size_t)txn->dbs[FREE_DBI].leaf_pages, - (size_t)txn->dbs[FREE_DBI].large_pages, + backlog_size(txn), err, txn->dbs[FREE_DBI].height, (size_t)txn->dbs[FREE_DBI].branch_pages, + (size_t)txn->dbs[FREE_DBI].leaf_pages, (size_t)txn->dbs[FREE_DBI].large_pages, (size_t)txn->dbs[FREE_DBI].items); tASSERT(txn, err != MDBX_NOTFOUND || (txn->flags & txn_gc_drained) != 0); return (err != MDBX_NOTFOUND) ? err : MDBX_SUCCESS; @@ -164,12 +150,10 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) { * though usually none are left at this point. * The pages themselves remain in dirtylist. */ if (unlikely(!txn->tw.gc.reclaimed && txn->tw.gc.last_reclaimed < 1)) { - TRACE("%s: try allocate gc-slot for %zu loose-pages", dbg_prefix(ctx), - txn->tw.loose_count); + TRACE("%s: try allocate gc-slot for %zu loose-pages", dbg_prefix(ctx), txn->tw.loose_count); int err = gc_alloc_ex(&ctx->cursor, 0, ALLOC_RESERVE).err; if (err == MDBX_SUCCESS) { - TRACE("%s: retry since gc-slot for %zu loose-pages available", - dbg_prefix(ctx), txn->tw.loose_count); + TRACE("%s: retry since gc-slot for %zu loose-pages available", dbg_prefix(ctx), txn->tw.loose_count); return MDBX_RESULT_TRUE; } @@ -183,15 +167,13 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) { MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *)); VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); } - TRACE("%s: append %zu loose-pages to retired-pages", dbg_prefix(ctx), - txn->tw.loose_count); + TRACE("%s: append %zu loose-pages to retired-pages", dbg_prefix(ctx), txn->tw.loose_count); } else { /* Room for loose pages + temp PNL with same */ int err = pnl_need(&txn->tw.relist, 2 * txn->tw.loose_count + 2); if (unlikely(err != MDBX_SUCCESS)) return err; - pnl_t loose = txn->tw.relist + MDBX_PNL_ALLOCLEN(txn->tw.relist) - - txn->tw.loose_count - 1; + pnl_t loose = txn->tw.relist + MDBX_PNL_ALLOCLEN(txn->tw.relist) - txn->tw.loose_count - 1; size_t count = 0; for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) { tASSERT(txn, lp->flags == P_LOOSE); @@ -203,8 +185,7 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) { MDBX_PNL_SETSIZE(loose, count); pnl_sort(loose, txn->geo.first_unallocated); pnl_merge(txn->tw.relist, loose); - TRACE("%s: append %zu loose-pages to reclaimed-pages", dbg_prefix(ctx), - txn->tw.loose_count); + TRACE("%s: append %zu loose-pages to reclaimed-pages", dbg_prefix(ctx), txn->tw.loose_count); } /* filter-out list of dirty-pages from loose-pages */ @@ -227,8 +208,7 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) { page_shadow_release(txn->env, dp, 1); } } - TRACE("%s: filtered-out loose-pages from %zu -> %zu dirty-pages", - dbg_prefix(ctx), dl->length, w); + TRACE("%s: filtered-out loose-pages from %zu -> %zu dirty-pages", dbg_prefix(ctx), dl->length, w); tASSERT(txn, txn->tw.loose_count == dl->length - w); dl->sorted -= sorted_out; tASSERT(txn, dl->sorted <= w); @@ -236,8 +216,7 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) { dl->pages_including_loose -= txn->tw.loose_count; txn->tw.dirtyroom += txn->tw.loose_count; tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom - : txn->env->options.dp_limit)); + (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); } else { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); } @@ -276,8 +255,8 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) { if (unlikely(err != MDBX_SUCCESS)) return err; if (retired_pages_before != MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { - TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx), - retired_pages_before, MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx), retired_pages_before, + MDBX_PNL_GETSIZE(txn->tw.retired_pages)); break; } @@ -290,19 +269,16 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) { if (unlikely(err != MDBX_SUCCESS)) return err; if (ctx->retired_stored >= MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { - TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx), - retired_pages_before, MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx), retired_pages_before, + MDBX_PNL_GETSIZE(txn->tw.retired_pages)); break; } } key.iov_len = sizeof(txnid_t); key.iov_base = &ctx->bigfoot; - const size_t left = - MDBX_PNL_GETSIZE(txn->tw.retired_pages) - ctx->retired_stored; + const size_t left = MDBX_PNL_GETSIZE(txn->tw.retired_pages) - ctx->retired_stored; const size_t chunk = - (left > txn->env->maxgc_large1page && ctx->bigfoot < MAX_TXNID) - ? txn->env->maxgc_large1page - : left; + (left > txn->env->maxgc_large1page && ctx->bigfoot < MAX_TXNID) ? txn->env->maxgc_large1page : left; data.iov_len = (chunk + 1) * sizeof(pgno_t); err = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE); if (unlikely(err != MDBX_SUCCESS)) @@ -318,9 +294,7 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) { #endif /* MDBX_DEBUG && (ENABLE_MEMCHECK || __SANITIZE_ADDRESS__) */ if (retired_pages_before == MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { - const size_t at = (is_lifo(txn) == MDBX_PNL_ASCENDING) - ? left - chunk - : ctx->retired_stored; + const size_t at = (is_lifo(txn) == MDBX_PNL_ASCENDING) ? left - chunk : ctx->retired_stored; pgno_t *const begin = txn->tw.retired_pages + at; /* MDBX_PNL_ASCENDING == false && LIFO == false: * - the larger pgno is at the beginning of retired list @@ -332,15 +306,11 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) { *begin = (pgno_t)chunk; memcpy(data.iov_base, begin, data.iov_len); *begin = save; - TRACE("%s: put-retired/bigfoot @ %" PRIaTXN - " (slice #%u) #%zu [%zu..%zu] of %zu", - dbg_prefix(ctx), ctx->bigfoot, - (unsigned)(ctx->bigfoot - txn->txnid), chunk, at, at + chunk, - retired_pages_before); + TRACE("%s: put-retired/bigfoot @ %" PRIaTXN " (slice #%u) #%zu [%zu..%zu] of %zu", dbg_prefix(ctx), + ctx->bigfoot, (unsigned)(ctx->bigfoot - txn->txnid), chunk, at, at + chunk, retired_pages_before); } ctx->retired_stored += chunk; - } while (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages) && - (++ctx->bigfoot, true)); + } while (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages) && (++ctx->bigfoot, true)); } while (retired_pages_before != MDBX_PNL_GETSIZE(txn->tw.retired_pages)); #else /* Write to last page of GC */ @@ -369,13 +339,11 @@ static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) { tASSERT(txn, data.iov_len == MDBX_PNL_SIZEOF(txn->tw.retired_pages)); memcpy(data.iov_base, txn->tw.retired_pages, data.iov_len); - TRACE("%s: put-retired #%zu @ %" PRIaTXN, dbg_prefix(ctx), - ctx->retired_stored, txn->txnid); + TRACE("%s: put-retired #%zu @ %" PRIaTXN, dbg_prefix(ctx), ctx->retired_stored, txn->txnid); #endif /* MDBX_ENABLE_BIGFOOT */ if (LOG_ENABLED(MDBX_LOG_EXTRA)) { size_t i = ctx->retired_stored; - DEBUG_EXTRA("txn %" PRIaTXN " root %" PRIaPGNO " num %zu, retired-PNL", - txn->txnid, txn->dbs[FREE_DBI].root, i); + DEBUG_EXTRA("txn %" PRIaTXN " root %" PRIaPGNO " num %zu, retired-PNL", txn->txnid, txn->dbs[FREE_DBI].root, i); for (; i; i--) DEBUG_EXTRA_PRINT(" %" PRIaPGNO, txn->tw.retired_pages[i]); DEBUG_EXTRA_PRINT("%s\n", "."); @@ -388,8 +356,7 @@ typedef struct gcu_rid_result { txnid_t rid; } rid_t; -static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, - const size_t left) { +static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left) { rid_t r; if (is_lifo(txn)) { if (txn->tw.gc.reclaimed == nullptr) { @@ -400,8 +367,7 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, } } if (MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) < txl_max && - left > (MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot) * - txn->env->maxgc_large1page && + left > (MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot) * txn->env->maxgc_large1page && !ctx->dense) { /* Hужен свободный для для сохранения списка страниц. */ bool need_cleanup = false; @@ -411,15 +377,11 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, r.err = gc_alloc_ex(&ctx->cursor, 0, ALLOC_RESERVE).err; snap_oldest = txn->env->lck->cached_oldest.weak; if (likely(r.err == MDBX_SUCCESS)) { - TRACE("%s: took @%" PRIaTXN " from GC", dbg_prefix(ctx), - MDBX_PNL_LAST(txn->tw.gc.reclaimed)); + TRACE("%s: took @%" PRIaTXN " from GC", dbg_prefix(ctx), MDBX_PNL_LAST(txn->tw.gc.reclaimed)); need_cleanup = true; } - } while (r.err == MDBX_SUCCESS && - MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) < txl_max && - left > - (MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot) * - txn->env->maxgc_large1page); + } while (r.err == MDBX_SUCCESS && MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) < txl_max && + left > (MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot) * txn->env->maxgc_large1page); if (likely(r.err == MDBX_SUCCESS)) { TRACE("%s: got enough from GC.", dbg_prefix(ctx)); @@ -443,20 +405,16 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, /* no reclaimable GC entries, * therefore no entries with ID < mdbx_find_oldest(txn) */ txn->tw.gc.last_reclaimed = ctx->rid = snap_oldest; - TRACE("%s: none recycled yet, set rid to @%" PRIaTXN, dbg_prefix(ctx), - ctx->rid); + TRACE("%s: none recycled yet, set rid to @%" PRIaTXN, dbg_prefix(ctx), ctx->rid); } /* В GC нет годных к переработке записей, * будем использовать свободные id в обратном порядке. */ while (MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) < txl_max && - left > - (MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot) * - txn->env->maxgc_large1page) { + left > (MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot) * txn->env->maxgc_large1page) { if (unlikely(ctx->rid <= MIN_TXNID)) { ctx->dense = true; - if (unlikely(MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) <= - ctx->reused_slot)) { + if (unlikely(MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) <= ctx->reused_slot)) { NOTICE("** restart: reserve depleted (reused_gc_slot %zu >= " "gc.reclaimed %zu)", ctx->reused_slot, MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed)); @@ -470,21 +428,16 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, MDBX_val key = {&ctx->rid, sizeof(ctx->rid)}, data; r.err = cursor_seek(&ctx->cursor, &key, &data, MDBX_SET_KEY).err; if (unlikely(r.err == MDBX_SUCCESS)) { - DEBUG("%s: GC's id %" PRIaTXN " is present, going to first", - dbg_prefix(ctx), ctx->rid); + DEBUG("%s: GC's id %" PRIaTXN " is present, going to first", dbg_prefix(ctx), ctx->rid); r.err = outer_first(&ctx->cursor, &key, nullptr); - if (unlikely(r.err != MDBX_SUCCESS || - key.iov_len != sizeof(txnid_t))) { - ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid GC-key size", (unsigned)key.iov_len); + if (unlikely(r.err != MDBX_SUCCESS || key.iov_len != sizeof(txnid_t))) { + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-key size", (unsigned)key.iov_len); r.err = MDBX_CORRUPTED; goto return_error; } const txnid_t gc_first = unaligned_peek_u64(4, key.iov_base); if (unlikely(gc_first <= INITIAL_TXNID)) { - NOTICE("%s: no free GC's id(s) less than %" PRIaTXN - " (going dense-mode)", - dbg_prefix(ctx), ctx->rid); + NOTICE("%s: no free GC's id(s) less than %" PRIaTXN " (going dense-mode)", dbg_prefix(ctx), ctx->rid); ctx->dense = true; goto return_restart; } @@ -501,18 +454,15 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, * with less fragmentation. */ need_cleanup = true; else - ctx->cleaned_slot += - 1 /* mark cleanup is not needed for added slot. */; + ctx->cleaned_slot += 1 /* mark cleanup is not needed for added slot. */; - TRACE("%s: append @%" PRIaTXN - " to lifo-reclaimed, cleaned-gc-slot = %zu", - dbg_prefix(ctx), ctx->rid, ctx->cleaned_slot); + TRACE("%s: append @%" PRIaTXN " to lifo-reclaimed, cleaned-gc-slot = %zu", dbg_prefix(ctx), ctx->rid, + ctx->cleaned_slot); } if (need_cleanup) { if (ctx->cleaned_slot) { - TRACE("%s: restart to clear and re-create GC entries", - dbg_prefix(ctx)); + TRACE("%s: restart to clear and re-create GC entries", dbg_prefix(ctx)); goto return_restart; } goto return_continue; @@ -522,8 +472,7 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t i = MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot; tASSERT(txn, i > 0 && i <= MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed)); r.rid = txn->tw.gc.reclaimed[i]; - TRACE("%s: take @%" PRIaTXN " from lifo-reclaimed[%zu]", dbg_prefix(ctx), - r.rid, i); + TRACE("%s: take @%" PRIaTXN " from lifo-reclaimed[%zu]", dbg_prefix(ctx), r.rid, i); } else { tASSERT(txn, txn->tw.gc.reclaimed == nullptr); if (unlikely(ctx->rid == 0)) { @@ -532,8 +481,7 @@ static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, r.err = outer_first(&ctx->cursor, &key, nullptr); if (likely(r.err == MDBX_SUCCESS)) { if (unlikely(key.iov_len != sizeof(txnid_t))) { - ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid GC-key size", (unsigned)key.iov_len); + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-key size", (unsigned)key.iov_len); r.err = MDBX_CORRUPTED; goto return_error; } @@ -600,18 +548,15 @@ retry: ctx->loop += !(ctx->prev_first_unallocated > txn->geo.first_unallocated); TRACE(">> restart, loop %u", ctx->loop); - tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); tASSERT(txn, dpl_check(txn)); if (unlikely(/* paranoia */ ctx->loop > ((MDBX_DEBUG > 0) ? 12 : 42))) { - ERROR("txn #%" PRIaTXN " too more loops %u, bailout", txn->txnid, - ctx->loop); + ERROR("txn #%" PRIaTXN " too more loops %u, bailout", txn->txnid, ctx->loop); rc = MDBX_PROBLEM; goto bailout; } - if (unlikely(ctx->dense || - ctx->prev_first_unallocated > txn->geo.first_unallocated)) { + if (unlikely(ctx->dense || ctx->prev_first_unallocated > txn->geo.first_unallocated)) { rc = clean_stored_retired(txn, ctx); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; @@ -630,13 +575,10 @@ retry: /* Come back here after each Put() in case retired-list changed */ TRACE("%s", " >> continue"); - tASSERT(txn, - pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); MDBX_val key, data; if (is_lifo(txn)) { - if (ctx->cleaned_slot < - (txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) : 0)) { + if (ctx->cleaned_slot < (txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) : 0)) { ctx->reserved = 0; ctx->cleaned_slot = 0; ctx->reused_slot = 0; @@ -644,8 +586,7 @@ retry: /* LY: cleanup reclaimed records. */ do { ctx->cleaned_id = txn->tw.gc.reclaimed[++ctx->cleaned_slot]; - tASSERT(txn, ctx->cleaned_slot > 0 && - ctx->cleaned_id <= env->lck->cached_oldest.weak); + tASSERT(txn, ctx->cleaned_slot > 0 && ctx->cleaned_id <= env->lck->cached_oldest.weak); key.iov_base = &ctx->cleaned_id; key.iov_len = sizeof(ctx->cleaned_id); rc = cursor_seek(&ctx->cursor, &key, nullptr, MDBX_SET).err; @@ -657,8 +598,7 @@ retry: if (unlikely(rc != MDBX_SUCCESS)) goto bailout; tASSERT(txn, ctx->cleaned_id <= env->lck->cached_oldest.weak); - TRACE("%s: cleanup-reclaimed-id [%zu]%" PRIaTXN, dbg_prefix(ctx), - ctx->cleaned_slot, ctx->cleaned_id); + TRACE("%s: cleanup-reclaimed-id [%zu]%" PRIaTXN, dbg_prefix(ctx), ctx->cleaned_slot, ctx->cleaned_id); tASSERT(txn, *txn->cursors == &ctx->cursor); rc = cursor_del(&ctx->cursor, 0); if (unlikely(rc != MDBX_SUCCESS)) @@ -668,8 +608,7 @@ retry: } } else { /* Удаляем оставшиеся вынутые из GC записи. */ - while (txn->tw.gc.last_reclaimed && - ctx->cleaned_id <= txn->tw.gc.last_reclaimed) { + while (txn->tw.gc.last_reclaimed && ctx->cleaned_id <= txn->tw.gc.last_reclaimed) { rc = outer_first(&ctx->cursor, &key, nullptr); if (rc == MDBX_NOTFOUND) { ctx->cleaned_id = txn->tw.gc.last_reclaimed + 1; @@ -680,10 +619,8 @@ retry: } if (unlikely(rc != MDBX_SUCCESS)) goto bailout; - if (!MDBX_DISABLE_VALIDATION && - unlikely(key.iov_len != sizeof(txnid_t))) { - ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid GC-key size", (unsigned)key.iov_len); + if (!MDBX_DISABLE_VALIDATION && unlikely(key.iov_len != sizeof(txnid_t))) { + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-key size", (unsigned)key.iov_len); rc = MDBX_CORRUPTED; goto bailout; } @@ -700,8 +637,7 @@ retry: goto bailout; tASSERT(txn, ctx->cleaned_id <= txn->tw.gc.last_reclaimed); tASSERT(txn, ctx->cleaned_id <= env->lck->cached_oldest.weak); - TRACE("%s: cleanup-reclaimed-id %" PRIaTXN, dbg_prefix(ctx), - ctx->cleaned_id); + TRACE("%s: cleanup-reclaimed-id %" PRIaTXN, dbg_prefix(ctx), ctx->cleaned_id); tASSERT(txn, *txn->cursors == &ctx->cursor); rc = cursor_del(&ctx->cursor, 0); if (unlikely(rc != MDBX_SUCCESS)) @@ -709,9 +645,7 @@ retry: } } - tASSERT(txn, - pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); tASSERT(txn, dpl_check(txn)); if (AUDIT_ENABLED()) { rc = audit_ex(txn, ctx->retired_stored, false); @@ -721,9 +655,7 @@ retry: /* return suitable into unallocated space */ if (txn_refund(txn)) { - tASSERT(txn, - pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); if (AUDIT_ENABLED()) { rc = audit_ex(txn, ctx->retired_stored, false); if (unlikely(rc != MDBX_SUCCESS)) @@ -743,10 +675,9 @@ retry: } if (unlikely(ctx->reserved > MDBX_PNL_GETSIZE(txn->tw.relist)) && - (ctx->loop < 5 || ctx->reserved - MDBX_PNL_GETSIZE(txn->tw.relist) > - env->maxgc_large1page / 2)) { - TRACE("%s: reclaimed-list changed %zu -> %zu, retry", dbg_prefix(ctx), - ctx->amount, MDBX_PNL_GETSIZE(txn->tw.relist)); + (ctx->loop < 5 || ctx->reserved - MDBX_PNL_GETSIZE(txn->tw.relist) > env->maxgc_large1page / 2)) { + TRACE("%s: reclaimed-list changed %zu -> %zu, retry", dbg_prefix(ctx), ctx->amount, + MDBX_PNL_GETSIZE(txn->tw.relist)); ctx->reserve_adj += ctx->reserved - MDBX_PNL_GETSIZE(txn->tw.relist); goto retry; } @@ -760,9 +691,7 @@ retry: continue; } - tASSERT(txn, - pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); tASSERT(txn, txn->tw.loose_count == 0); TRACE("%s", " >> reserving"); @@ -776,8 +705,7 @@ retry: "lifo-reclaimed-slots %zu, " "reused-gc-slots %zu", dbg_prefix(ctx), ctx->amount, ctx->reserved, ctx->reserve_adj, left, - txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) : 0, - ctx->reused_slot); + txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) : 0, ctx->reused_slot); if (0 >= (intptr_t)left) break; @@ -795,59 +723,46 @@ retry: size_t chunk = left; if (unlikely(left > env->maxgc_large1page)) { - const size_t avail_gc_slots = - txn->tw.gc.reclaimed - ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot + 1 - : (ctx->rid < INT16_MAX) ? (size_t)ctx->rid - : INT16_MAX; + const size_t avail_gc_slots = txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot + 1 + : (ctx->rid < INT16_MAX) ? (size_t)ctx->rid + : INT16_MAX; if (likely(avail_gc_slots > 1)) { #if MDBX_ENABLE_BIGFOOT chunk = env->maxgc_large1page; - if (avail_gc_slots < INT16_MAX && - unlikely(left > env->maxgc_large1page * avail_gc_slots)) + if (avail_gc_slots < INT16_MAX && unlikely(left > env->maxgc_large1page * avail_gc_slots)) /* TODO: Можно смотреть последовательности какой длины есть в relist * и пробовать нарезать куски соответствующего размера. * Смысл в том, чтобы не дробить последовательности страниц, * а использовать целиком. */ - chunk = env->maxgc_large1page + - left / (env->maxgc_large1page * avail_gc_slots) * - env->maxgc_large1page; + chunk = env->maxgc_large1page + left / (env->maxgc_large1page * avail_gc_slots) * env->maxgc_large1page; #else if (chunk < env->maxgc_large1page * 2) chunk /= 2; else { const size_t prefer_max_scatter = 257; const size_t threshold = - env->maxgc_large1page * ((avail_gc_slots < prefer_max_scatter) - ? avail_gc_slots - : prefer_max_scatter); + env->maxgc_large1page * ((avail_gc_slots < prefer_max_scatter) ? avail_gc_slots : prefer_max_scatter); if (left < threshold) chunk = env->maxgc_large1page; else { const size_t tail = left - threshold + env->maxgc_large1page + 1; size_t span = 1; - size_t avail = ((pgno2bytes(env, span) - PAGEHDRSZ) / - sizeof(pgno_t)) /* - 1 + span */; + size_t avail = ((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) /* - 1 + span */; if (tail > avail) { for (size_t i = ctx->amount - span; i > 0; --i) { if (MDBX_PNL_ASCENDING ? (txn->tw.relist[i] + span) - : (txn->tw.relist[i] - span) == - txn->tw.relist[i + span]) { + : (txn->tw.relist[i] - span) == txn->tw.relist[i + span]) { span += 1; - avail = - ((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) - - 1 + span; + avail = ((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) - 1 + span; if (avail >= tail) break; } } } - chunk = (avail >= tail) ? tail - span - : (avail_gc_slots > 3 && - ctx->reused_slot < prefer_max_scatter - 3) - ? avail - span - : tail; + chunk = (avail >= tail) ? tail - span + : (avail_gc_slots > 3 && ctx->reused_slot < prefer_max_scatter - 3) ? avail - span + : tail; } } #endif /* MDBX_ENABLE_BIGFOOT */ @@ -859,57 +774,43 @@ retry: "%" PRIaTXN, dbg_prefix(ctx), ctx->rid, ctx->reused_slot, reservation_gc_id); - TRACE("%s: chunk %zu, gc-per-ovpage %u", dbg_prefix(ctx), chunk, - env->maxgc_large1page); + TRACE("%s: chunk %zu, gc-per-ovpage %u", dbg_prefix(ctx), chunk, env->maxgc_large1page); tASSERT(txn, reservation_gc_id <= env->lck->cached_oldest.weak); if (unlikely(reservation_gc_id < MIN_TXNID || - reservation_gc_id > - atomic_load64(&env->lck->cached_oldest, mo_Relaxed))) { - ERROR("** internal error (reservation_gc_id %" PRIaTXN ")", - reservation_gc_id); + reservation_gc_id > atomic_load64(&env->lck->cached_oldest, mo_Relaxed))) { + ERROR("** internal error (reservation_gc_id %" PRIaTXN ")", reservation_gc_id); rc = MDBX_PROBLEM; goto bailout; } - tASSERT(txn, - reservation_gc_id >= MIN_TXNID && reservation_gc_id <= MAX_TXNID); + tASSERT(txn, reservation_gc_id >= MIN_TXNID && reservation_gc_id <= MAX_TXNID); key.iov_len = sizeof(reservation_gc_id); key.iov_base = (void *)&reservation_gc_id; data.iov_len = (chunk + 1) * sizeof(pgno_t); - TRACE("%s: reserve %zu [%zu...%zu) @%" PRIaTXN, dbg_prefix(ctx), chunk, - ctx->reserved + 1, ctx->reserved + chunk + 1, reservation_gc_id); + TRACE("%s: reserve %zu [%zu...%zu) @%" PRIaTXN, dbg_prefix(ctx), chunk, ctx->reserved + 1, + ctx->reserved + chunk + 1, reservation_gc_id); prepare_backlog(txn, ctx); rc = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE | MDBX_NOOVERWRITE); - tASSERT(txn, - pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; zeroize_reserved(env, data); ctx->reserved += chunk; - TRACE("%s: reserved %zu (+%zu), continue", dbg_prefix(ctx), ctx->reserved, - chunk); + TRACE("%s: reserved %zu (+%zu), continue", dbg_prefix(ctx), ctx->reserved, chunk); continue; } - tASSERT( - txn, - ctx->cleaned_slot == - (txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) : 0)); + tASSERT(txn, ctx->cleaned_slot == (txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) : 0)); TRACE("%s", " >> filling"); /* Fill in the reserved records */ size_t excess_slots = 0; - ctx->fill_idx = - txn->tw.gc.reclaimed - ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot - : ctx->reused_slot; + ctx->fill_idx = txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - ctx->reused_slot : ctx->reused_slot; rc = MDBX_SUCCESS; - tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); tASSERT(txn, dpl_check(txn)); if (ctx->amount) { MDBX_val key, data; @@ -930,20 +831,17 @@ retry: while (true) { txnid_t fill_gc_id; - TRACE("%s: left %zu of %zu", dbg_prefix(ctx), left, - MDBX_PNL_GETSIZE(txn->tw.relist)); + TRACE("%s: left %zu of %zu", dbg_prefix(ctx), left, MDBX_PNL_GETSIZE(txn->tw.relist)); if (txn->tw.gc.reclaimed == nullptr) { tASSERT(txn, is_lifo(txn) == 0); - fill_gc_id = - key.iov_base ? unaligned_peek_u64(4, key.iov_base) : MIN_TXNID; + fill_gc_id = key.iov_base ? unaligned_peek_u64(4, key.iov_base) : MIN_TXNID; if (ctx->fill_idx == 0 || fill_gc_id > txn->tw.gc.last_reclaimed) { if (!left) break; - NOTICE("** restart: reserve depleted (fill_idx %zu, fill_id %" PRIaTXN - " > last_reclaimed %" PRIaTXN ", left %zu", + NOTICE("** restart: reserve depleted (fill_idx %zu, fill_id %" PRIaTXN " > last_reclaimed %" PRIaTXN + ", left %zu", ctx->fill_idx, fill_gc_id, txn->tw.gc.last_reclaimed, left); - ctx->reserve_adj = - (ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0; + ctx->reserve_adj = (ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0; goto retry; } ctx->fill_idx -= 1; @@ -955,26 +853,20 @@ retry: NOTICE("** restart: reserve depleted (fill_idx %zu >= " "gc.reclaimed %zu, left %zu", ctx->fill_idx, MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed), left); - ctx->reserve_adj = - (ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0; + ctx->reserve_adj = (ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0; goto retry; } ctx->fill_idx += 1; fill_gc_id = txn->tw.gc.reclaimed[ctx->fill_idx]; - TRACE("%s: seek-reservation @%" PRIaTXN " at gc.reclaimed[%zu]", - dbg_prefix(ctx), fill_gc_id, ctx->fill_idx); + TRACE("%s: seek-reservation @%" PRIaTXN " at gc.reclaimed[%zu]", dbg_prefix(ctx), fill_gc_id, ctx->fill_idx); key.iov_base = &fill_gc_id; key.iov_len = sizeof(fill_gc_id); rc = cursor_seek(&ctx->cursor, &key, &data, MDBX_SET_KEY).err; if (unlikely(rc != MDBX_SUCCESS)) goto bailout; } - tASSERT(txn, - ctx->cleaned_slot == (txn->tw.gc.reclaimed - ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - : 0)); - tASSERT(txn, - fill_gc_id > 0 && fill_gc_id <= env->lck->cached_oldest.weak); + tASSERT(txn, ctx->cleaned_slot == (txn->tw.gc.reclaimed ? MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) : 0)); + tASSERT(txn, fill_gc_id > 0 && fill_gc_id <= env->lck->cached_oldest.weak); key.iov_base = &fill_gc_id; key.iov_len = sizeof(fill_gc_id); @@ -983,14 +875,12 @@ retry: if (unlikely(chunk > left)) { const size_t delta = chunk - left; excess += delta; - TRACE("%s: chunk %zu > left %zu, @%" PRIaTXN, dbg_prefix(ctx), chunk, - left, fill_gc_id); + TRACE("%s: chunk %zu > left %zu, @%" PRIaTXN, dbg_prefix(ctx), chunk, left, fill_gc_id); if (!left) { excess_slots += 1; goto next; } - if ((ctx->loop < 5 && delta > (ctx->loop / 2)) || - delta > env->maxgc_large1page) + if ((ctx->loop < 5 && delta > (ctx->loop / 2)) || delta > env->maxgc_large1page) data.iov_len = (left + 1) * sizeof(pgno_t); chunk = left; } @@ -999,29 +889,23 @@ retry: goto bailout; zeroize_reserved(env, data); - if (unlikely(txn->tw.loose_count || - ctx->amount != MDBX_PNL_GETSIZE(txn->tw.relist))) { - NOTICE("** restart: reclaimed-list changed (%zu -> %zu, loose +%zu)", - ctx->amount, MDBX_PNL_GETSIZE(txn->tw.relist), - txn->tw.loose_count); + if (unlikely(txn->tw.loose_count || ctx->amount != MDBX_PNL_GETSIZE(txn->tw.relist))) { + NOTICE("** restart: reclaimed-list changed (%zu -> %zu, loose +%zu)", ctx->amount, + MDBX_PNL_GETSIZE(txn->tw.relist), txn->tw.loose_count); if (ctx->loop < 5 || (ctx->loop > 10 && (ctx->loop & 1))) goto retry_clean_adj; goto retry; } - if (unlikely(txn->tw.gc.reclaimed - ? ctx->cleaned_slot < - MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) - : ctx->cleaned_id < txn->tw.gc.last_reclaimed)) { + if (unlikely(txn->tw.gc.reclaimed ? ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) + : ctx->cleaned_id < txn->tw.gc.last_reclaimed)) { NOTICE("%s", "** restart: reclaimed-slots changed"); goto retry; } - if (unlikely(ctx->retired_stored != - MDBX_PNL_GETSIZE(txn->tw.retired_pages))) { - tASSERT(txn, - ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages)); - NOTICE("** restart: retired-list growth (%zu -> %zu)", - ctx->retired_stored, MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + if (unlikely(ctx->retired_stored != MDBX_PNL_GETSIZE(txn->tw.retired_pages))) { + tASSERT(txn, ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + NOTICE("** restart: retired-list growth (%zu -> %zu)", ctx->retired_stored, + MDBX_PNL_GETSIZE(txn->tw.retired_pages)); goto retry; } @@ -1030,9 +914,8 @@ retry: pgno_t *src = MDBX_PNL_BEGIN(txn->tw.relist) + left - chunk; memcpy(dst, src, chunk * sizeof(pgno_t)); pgno_t *from = src, *to = src + chunk; - TRACE("%s: fill %zu [ %zu:%" PRIaPGNO "...%zu:%" PRIaPGNO "] @%" PRIaTXN, - dbg_prefix(ctx), chunk, from - txn->tw.relist, from[0], - to - txn->tw.relist, to[-1], fill_gc_id); + TRACE("%s: fill %zu [ %zu:%" PRIaPGNO "...%zu:%" PRIaPGNO "] @%" PRIaTXN, dbg_prefix(ctx), chunk, + from - txn->tw.relist, from[0], to - txn->tw.relist, to[-1], fill_gc_id); left -= chunk; if (AUDIT_ENABLED()) { @@ -1063,16 +946,14 @@ retry: while (n >= env->maxgc_large1page) adj -= n /= env->maxgc_large1page; ctx->reserve_adj += adj; - TRACE("%s: extra %zu reserved space, adj +%zu (%zu)", dbg_prefix(ctx), - excess, adj, ctx->reserve_adj); + TRACE("%s: extra %zu reserved space, adj +%zu (%zu)", dbg_prefix(ctx), excess, adj, ctx->reserve_adj); } } tASSERT(txn, rc == MDBX_SUCCESS); - if (unlikely(txn->tw.loose_count != 0 || - ctx->amount != MDBX_PNL_GETSIZE(txn->tw.relist))) { - NOTICE("** restart: got %zu loose pages (reclaimed-list %zu -> %zu)", - txn->tw.loose_count, ctx->amount, MDBX_PNL_GETSIZE(txn->tw.relist)); + if (unlikely(txn->tw.loose_count != 0 || ctx->amount != MDBX_PNL_GETSIZE(txn->tw.relist))) { + NOTICE("** restart: got %zu loose pages (reclaimed-list %zu -> %zu)", txn->tw.loose_count, ctx->amount, + MDBX_PNL_GETSIZE(txn->tw.relist)); goto retry; } @@ -1080,14 +961,12 @@ retry: const bool will_retry = ctx->loop < 5 || excess_slots > 1; NOTICE("** %s: reserve excess (excess-slots %zu, filled-slot %zu, adj %zu, " "loop %u)", - will_retry ? "restart" : "ignore", excess_slots, ctx->fill_idx, - ctx->reserve_adj, ctx->loop); + will_retry ? "restart" : "ignore", excess_slots, ctx->fill_idx, ctx->reserve_adj, ctx->loop); if (will_retry) goto retry; } - tASSERT(txn, txn->tw.gc.reclaimed == nullptr || - ctx->cleaned_slot == MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed)); + tASSERT(txn, txn->tw.gc.reclaimed == nullptr || ctx->cleaned_slot == MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed)); bailout: txn->cursors[FREE_DBI] = ctx->cursor.next; diff --git a/src/gc.h b/src/gc.h index a8a68a24..b3ec23cf 100644 --- a/src/gc.h +++ b/src/gc.h @@ -34,8 +34,7 @@ static inline int gc_update_init(MDBX_txn *txn, gcu_t *ctx) { #define ALLOC_DEFAULT 0 #define ALLOC_RESERVE 1 #define ALLOC_UNIMPORTANT 2 -MDBX_INTERNAL pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, - uint8_t flags); +MDBX_INTERNAL pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags); MDBX_INTERNAL pgr_t gc_alloc_single(const MDBX_cursor *const mc); MDBX_INTERNAL int gc_update(MDBX_txn *txn, gcu_t *ctx); diff --git a/src/global.c b/src/global.c index 88432580..69340319 100644 --- a/src/global.c +++ b/src/global.c @@ -122,8 +122,7 @@ extern void __gmon_start__(void) __attribute__((__weak__)); #endif /* ENABLE_GPROF */ MDBX_EXCLUDE_FOR_GPROF -__cold static __attribute__((__constructor__)) void -mdbx_global_constructor(void) { +__cold static __attribute__((__constructor__)) void mdbx_global_constructor(void) { #ifdef ENABLE_GPROF if (!&__gmon_start__) monstartup((uintptr_t)&_init, (uintptr_t)&_fini); @@ -154,9 +153,8 @@ mdbx_global_constructor(void) { * So, the REQUIREMENTS for this code: * 1. MUST detect WSL1 without false-negatives. * 2. DESIRABLE detect WSL2 but without the risk of violating the first. */ - globals.running_on_WSL1 = probe_for_WSL(buffer.version) == 1 || - probe_for_WSL(buffer.sysname) == 1 || - probe_for_WSL(buffer.release) == 1; + globals.running_on_WSL1 = + probe_for_WSL(buffer.version) == 1 || probe_for_WSL(buffer.sysname) == 1 || probe_for_WSL(buffer.release) == 1; } #endif /* Linux */ @@ -164,8 +162,7 @@ mdbx_global_constructor(void) { } MDBX_EXCLUDE_FOR_GPROF -__cold static __attribute__((__destructor__)) void -mdbx_global_destructor(void) { +__cold static __attribute__((__destructor__)) void mdbx_global_destructor(void) { mdbx_fini(); #ifdef ENABLE_GPROF if (!&__gmon_start__) @@ -180,13 +177,11 @@ mdbx_global_destructor(void) { struct libmdbx_globals globals; __cold static void mdbx_init(void) { - globals.runtime_flags = ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + - ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT; + globals.runtime_flags = ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT; globals.loglevel = MDBX_LOG_FATAL; ENSURE(nullptr, osal_fastmutex_init(&globals.debug_lock) == 0); osal_ctor(); - assert(globals.sys_pagesize > 0 && - (globals.sys_pagesize & (globals.sys_pagesize - 1)) == 0); + assert(globals.sys_pagesize > 0 && (globals.sys_pagesize & (globals.sys_pagesize - 1)) == 0); rthc_ctor(); #if MDBX_DEBUG ENSURE(nullptr, troika_verify_fsm()); diff --git a/src/internals.h b/src/internals.h index 9d45fda6..2dca2d23 100644 --- a/src/internals.h +++ b/src/internals.h @@ -156,9 +156,8 @@ enum txn_flags { txn_shrink_allowed = UINT32_C(0x40000000), txn_parked = MDBX_TXN_PARKED, txn_gc_drained = 0x40 /* GC was depleted up to oldest reader */, - txn_state_flags = MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | - MDBX_TXN_SPILLS | MDBX_TXN_HAS_CHILD | MDBX_TXN_INVALID | - txn_gc_drained + txn_state_flags = MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | MDBX_TXN_HAS_CHILD | + MDBX_TXN_INVALID | txn_gc_drained }; /* A database transaction. @@ -336,12 +335,9 @@ enum env_flags { /* Only a subset of the mdbx_env flags can be changed * at runtime. Changing other flags requires closing the * environment and re-opening it with the new flags. */ - ENV_CHANGEABLE_FLAGS = MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | - DEPRECATED_MAPASYNC | MDBX_NOMEMINIT | - DEPRECATED_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE | - MDBX_VALIDATION, - ENV_CHANGELESS_FLAGS = MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | - MDBX_NOSTICKYTHREADS | MDBX_NORDAHEAD | + ENV_CHANGEABLE_FLAGS = MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | DEPRECATED_MAPASYNC | MDBX_NOMEMINIT | + DEPRECATED_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE | MDBX_VALIDATION, + ENV_CHANGELESS_FLAGS = MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOSTICKYTHREADS | MDBX_NORDAHEAD | MDBX_LIFORECLAIM | MDBX_EXCLUSIVE, ENV_USABLE_FLAGS = ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS }; @@ -368,8 +364,8 @@ struct MDBX_env { uint16_t subpage_reserve_prereq; uint16_t subpage_reserve_limit; atomic_pgno_t mlocked_pgno; - uint8_t ps2ln; /* log2 of DB page size */ - int8_t stuck_meta; /* recovery-only: target meta page or less that zero */ + uint8_t ps2ln; /* log2 of DB page size */ + int8_t stuck_meta; /* recovery-only: target meta page or less that zero */ uint16_t merge_threshold, merge_threshold_gc; /* pages emptier than this are candidates for merging */ unsigned max_readers; /* size of the reader table */ @@ -385,7 +381,7 @@ struct MDBX_env { kvx_t *kvs; /* array of auxiliary key-value properties */ uint8_t *__restrict dbs_flags; /* array of flags from tree_t.flags */ mdbx_atomic_uint32_t *dbi_seqs; /* array of dbi sequence numbers */ - unsigned maxgc_large1page; /* Number of pgno_t fit in a single large page */ + unsigned maxgc_large1page; /* Number of pgno_t fit in a single large page */ unsigned maxgc_per_branch; uint32_t registered_reader_pid; /* have liveness lock in reader table */ void *userctx; /* User-settable context */ @@ -492,9 +488,7 @@ struct MDBX_env { #endif /* ------------------------------------------------- stub for lck-less mode */ - mdbx_atomic_uint64_t - lckless_placeholder[(sizeof(lck_t) + MDBX_CACHELINE_SIZE - 1) / - sizeof(mdbx_atomic_uint64_t)]; + mdbx_atomic_uint64_t lckless_placeholder[(sizeof(lck_t) + MDBX_CACHELINE_SIZE - 1) / sizeof(mdbx_atomic_uint64_t)]; }; /*----------------------------------------------------------------------------*/ @@ -509,8 +503,8 @@ struct MDBX_env { #define DEFAULT_READERS 61 enum db_flags { - DB_PERSISTENT_FLAGS = MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | - MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP, + DB_PERSISTENT_FLAGS = + MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP, /* mdbx_dbi_open() flags */ DB_USABLE_FLAGS = DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE, @@ -524,27 +518,19 @@ enum db_flags { MDBX_MAYBE_UNUSED static void static_checks(void) { STATIC_ASSERT(MDBX_WORDBITS == sizeof(void *) * CHAR_BIT); STATIC_ASSERT(UINT64_C(0x80000000) == (uint32_t)ENV_FATAL_ERROR); - STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, - "Oops, MDBX_MAX_DBI or CORE_DBS?"); + STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, "Oops, MDBX_MAX_DBI or CORE_DBS?"); STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == - ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & - (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), - "Oops, some flags overlapped or wrong"); - STATIC_ASSERT_MSG((DB_INTERNAL_FLAGS & DB_USABLE_FLAGS) == 0, - "Oops, some flags overlapped or wrong"); - STATIC_ASSERT_MSG((DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS) == 0, + ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), "Oops, some flags overlapped or wrong"); + STATIC_ASSERT_MSG((DB_INTERNAL_FLAGS & DB_USABLE_FLAGS) == 0, "Oops, some flags overlapped or wrong"); + STATIC_ASSERT_MSG((DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS) == 0, "Oops, some flags overlapped or wrong"); STATIC_ASSERT(DB_PERSISTENT_FLAGS <= UINT8_MAX); - STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, - "Oops, some flags overlapped or wrong"); + STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, "Oops, some flags overlapped or wrong"); - STATIC_ASSERT_MSG( - (txn_state_flags & (txn_rw_begin_flags | txn_ro_begin_flags)) == 0, - "Oops, some txn flags overlapped or wrong"); - STATIC_ASSERT_MSG( - ((txn_rw_begin_flags | txn_ro_begin_flags | txn_state_flags) & - txn_shrink_allowed) == 0, - "Oops, some txn flags overlapped or wrong"); + STATIC_ASSERT_MSG((txn_state_flags & (txn_rw_begin_flags | txn_ro_begin_flags)) == 0, + "Oops, some txn flags overlapped or wrong"); + STATIC_ASSERT_MSG(((txn_rw_begin_flags | txn_ro_begin_flags | txn_state_flags) & txn_shrink_allowed) == 0, + "Oops, some txn flags overlapped or wrong"); STATIC_ASSERT(sizeof(reader_slot_t) == 32); #if MDBX_LOCKING > 0 diff --git a/src/layout-dxb.h b/src/layout-dxb.h index 74d09166..ed2f261d 100644 --- a/src/layout-dxb.h +++ b/src/layout-dxb.h @@ -17,10 +17,8 @@ /* FROZEN: The version number for a database's datafile format. */ #define MDBX_DATA_VERSION 3 -#define MDBX_DATA_MAGIC \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) -#define MDBX_DATA_MAGIC_LEGACY_COMPAT \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_LEGACY_COMPAT ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) #define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) /* handle for the DB used to track free pages. */ @@ -261,40 +259,30 @@ typedef enum node_flags { #pragma pack(pop) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t -page_type(const page_t *mp) { - return mp->flags; -} +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type(const page_t *mp) { return mp->flags; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t -page_type_compat(const page_t *mp) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type_compat(const page_t *mp) { /* Drop legacy P_DIRTY flag for sub-pages for compatilibity, * for assertions only. */ - return unlikely(mp->flags & P_SUBP) ? mp->flags & ~(P_SUBP | P_LEGACY_DIRTY) - : mp->flags; + return unlikely(mp->flags & P_SUBP) ? mp->flags & ~(P_SUBP | P_LEGACY_DIRTY) : mp->flags; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -is_leaf(const page_t *mp) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_leaf(const page_t *mp) { return (mp->flags & P_LEAF) != 0; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -is_dupfix_leaf(const page_t *mp) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_dupfix_leaf(const page_t *mp) { return (mp->flags & P_DUPFIX) != 0; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -is_branch(const page_t *mp) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_branch(const page_t *mp) { return (mp->flags & P_BRANCH) != 0; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -is_largepage(const page_t *mp) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_largepage(const page_t *mp) { return (mp->flags & P_LARGE) != 0; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -is_subpage(const page_t *mp) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_subpage(const page_t *mp) { return (mp->flags & P_SUBP) != 0; } diff --git a/src/layout-lck.h b/src/layout-lck.h index b24b20c6..f4a2a368 100644 --- a/src/layout-lck.h +++ b/src/layout-lck.h @@ -19,8 +19,7 @@ typedef void osal_ipclock_t; #define MDBX_LCK_SIGN UINT32_C(0xF18D) typedef mdbx_pid_t osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 #define MDBX_LCK_SIGN UINT32_C(0x8017) typedef pthread_mutex_t osal_ipclock_t; @@ -64,19 +63,15 @@ typedef struct pgops { mdbx_atomic_uint64_t merge; /* Page merges */ mdbx_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ mdbx_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ - mdbx_atomic_uint64_t - wops; /* Number of explicit write operations (not a pages) to a disk */ - mdbx_atomic_uint64_t - msync; /* Number of explicit msync/flush-to-disk operations */ - mdbx_atomic_uint64_t - fsync; /* Number of explicit fsync/flush-to-disk operations */ + mdbx_atomic_uint64_t wops; /* Number of explicit write operations (not a pages) to a disk */ + mdbx_atomic_uint64_t msync; /* Number of explicit msync/flush-to-disk operations */ + mdbx_atomic_uint64_t fsync; /* Number of explicit fsync/flush-to-disk operations */ mdbx_atomic_uint64_t prefault; /* Number of prefault write operations */ mdbx_atomic_uint64_t mincore; /* Number of mincore() calls */ - mdbx_atomic_uint32_t - incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 - caught */ + mdbx_atomic_uint32_t incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 + caught */ mdbx_atomic_uint32_t reserved; /* Статистика для профилирования GC. @@ -202,8 +197,7 @@ typedef struct shared_lck { * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ #define MDBX_NOMETASYNC_LAZY_UNK (UINT32_MAX / 3) #define MDBX_NOMETASYNC_LAZY_FD (MDBX_NOMETASYNC_LAZY_UNK + UINT32_MAX / 8) -#define MDBX_NOMETASYNC_LAZY_WRITEMAP \ - (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) +#define MDBX_NOMETASYNC_LAZY_WRITEMAP (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) mdbx_atomic_uint32_t meta_sync_txnid; /* Period for timed auto-sync feature, i.e. at the every steady checkpoint @@ -277,12 +271,10 @@ typedef struct shared_lck { reader_slot_t rdt[] /* dynamic size */; /* Lockfile format signature: version, features and field layout */ -#define MDBX_LOCK_FORMAT \ - (MDBX_LCK_SIGN * 27733 + (unsigned)sizeof(reader_slot_t) * 13 + \ - (unsigned)offsetof(reader_slot_t, snapshot_pages_used) * 251 + \ - (unsigned)offsetof(lck_t, cached_oldest) * 83 + \ - (unsigned)offsetof(lck_t, rdt_length) * 37 + \ - (unsigned)offsetof(lck_t, rdt) * 29) +#define MDBX_LOCK_FORMAT \ + (MDBX_LCK_SIGN * 27733 + (unsigned)sizeof(reader_slot_t) * 13 + \ + (unsigned)offsetof(reader_slot_t, snapshot_pages_used) * 251 + (unsigned)offsetof(lck_t, cached_oldest) * 83 + \ + (unsigned)offsetof(lck_t, rdt_length) * 37 + (unsigned)offsetof(lck_t, rdt) * 29) #endif /* FLEXIBLE_ARRAY_MEMBERS */ } lck_t; diff --git a/src/lck-posix.c b/src/lck-posix.c index 083817de..4cf83c46 100644 --- a/src/lck-posix.c +++ b/src/lck-posix.c @@ -71,11 +71,10 @@ __cold static void choice_fcntl(void) { assert(!op_setlk && !op_setlkw && !op_getlk); if ((globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 #if defined(__linux__) || defined(__gnu_linux__) - && globals.linux_kernel_version > - 0x030f0000 /* OFD locks are available since 3.15, but engages here - only for 3.16 and later kernels (i.e. LTS) because - of reliability reasons */ -#endif /* linux */ + && globals.linux_kernel_version > 0x030f0000 /* OFD locks are available since 3.15, but engages here + only for 3.16 and later kernels (i.e. LTS) because + of reliability reasons */ +#endif /* linux */ ) { op_setlk = MDBX_F_OFD_SETLK; op_setlkw = MDBX_F_OFD_SETLKW; @@ -92,32 +91,25 @@ __cold static void choice_fcntl(void) { #define op_getlk MDBX_F_GETLK #endif /* MDBX_USE_OFDLOCKS */ -static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck, - const off_t offset, off_t len) { - STATIC_ASSERT(sizeof(off_t) >= sizeof(void *) && - sizeof(off_t) >= sizeof(size_t)); +static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck, const off_t offset, off_t len) { + STATIC_ASSERT(sizeof(off_t) >= sizeof(void *) && sizeof(off_t) >= sizeof(size_t)); #ifdef __ANDROID_API__ - STATIC_ASSERT_MSG((sizeof(off_t) * 8 == MDBX_WORDBITS), - "The bitness of system `off_t` type is mismatch. Please " - "fix build and/or NDK configuration."); + STATIC_ASSERT_MSG((sizeof(off_t) * 8 == MDBX_WORDBITS), "The bitness of system `off_t` type is mismatch. Please " + "fix build and/or NDK configuration."); #endif /* Android */ assert(offset >= 0 && len > 0); - assert((uint64_t)offset < (uint64_t)INT64_MAX && - (uint64_t)len < (uint64_t)INT64_MAX && + assert((uint64_t)offset < (uint64_t)INT64_MAX && (uint64_t)len < (uint64_t)INT64_MAX && (uint64_t)(offset + len) > (uint64_t)offset); - assert((uint64_t)offset < (uint64_t)OFF_T_MAX && - (uint64_t)len <= (uint64_t)OFF_T_MAX && + assert((uint64_t)offset < (uint64_t)OFF_T_MAX && (uint64_t)len <= (uint64_t)OFF_T_MAX && (uint64_t)(offset + len) <= (uint64_t)OFF_T_MAX); - assert((uint64_t)((off_t)((uint64_t)offset + (uint64_t)len)) == - ((uint64_t)offset + (uint64_t)len)); + assert((uint64_t)((off_t)((uint64_t)offset + (uint64_t)len)) == ((uint64_t)offset + (uint64_t)len)); jitter4testing(true); for (;;) { MDBX_STRUCT_FLOCK lock_op; - STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(lock_op.l_start) && - sizeof(off_t) <= sizeof(lock_op.l_len) && + STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(lock_op.l_start) && sizeof(off_t) <= sizeof(lock_op.l_len) && OFF_T_MAX == (off_t)OFF_T_MAX, "Support for large/64-bit-sized files is misconfigured " "for the target system and/or toolchain. " @@ -134,15 +126,13 @@ static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck, /* Checks reader by pid. Returns: * MDBX_RESULT_TRUE - if pid is live (reader holds a lock). * MDBX_RESULT_FALSE - if pid is dead (a lock could be placed). */ - return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE - : MDBX_RESULT_TRUE; + return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE; } return MDBX_SUCCESS; } rc = errno; #if MDBX_USE_OFDLOCKS - if (rc == EINVAL && (cmd == MDBX_F_OFD_SETLK || cmd == MDBX_F_OFD_SETLKW || - cmd == MDBX_F_OFD_GETLK)) { + if (rc == EINVAL && (cmd == MDBX_F_OFD_SETLK || cmd == MDBX_F_OFD_SETLKW || cmd == MDBX_F_OFD_GETLK)) { /* fallback to non-OFD locks */ if (cmd == MDBX_F_OFD_SETLK) cmd = MDBX_F_SETLK; @@ -197,8 +187,7 @@ MDBX_INTERNAL int lck_rpid_check(MDBX_env *env, uint32_t pid) { MDBX_INTERNAL int lck_ipclock_stubinit(osal_ipclock_t *ipc) { #if MDBX_LOCKING == MDBX_LOCKING_POSIX1988 return sem_init(ipc, false, 1) ? errno : 0; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 return pthread_mutex_init(ipc, nullptr); #else #error "FIXME" @@ -208,8 +197,7 @@ MDBX_INTERNAL int lck_ipclock_stubinit(osal_ipclock_t *ipc) { MDBX_INTERNAL int lck_ipclock_destroy(osal_ipclock_t *ipc) { #if MDBX_LOCKING == MDBX_LOCKING_POSIX1988 return sem_destroy(ipc) ? errno : 0; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 return pthread_mutex_destroy(ipc); #else #error "FIXME" @@ -233,14 +221,12 @@ static int check_fstat(MDBX_env *env) { #else rc = EPERM; #endif - ERROR("%s %s, err %d", "DXB", - (st.st_nlink < 1) ? "file was removed" : "not a regular file", rc); + ERROR("%s %s, err %d", "DXB", (st.st_nlink < 1) ? "file was removed" : "not a regular file", rc); return rc; } if (st.st_size < (off_t)(MDBX_MIN_PAGESIZE * NUM_METAS)) { - VERBOSE("dxb-file is too short (%u), exclusive-lock needed", - (unsigned)st.st_size); + VERBOSE("dxb-file is too short (%u), exclusive-lock needed", (unsigned)st.st_size); rc = MDBX_RESULT_TRUE; } @@ -258,16 +244,14 @@ static int check_fstat(MDBX_env *env) { #else rc = EPERM; #endif - ERROR("%s %s, err %d", "LCK", - (st.st_nlink < 1) ? "file was removed" : "not a regular file", rc); + ERROR("%s %s, err %d", "LCK", (st.st_nlink < 1) ? "file was removed" : "not a regular file", rc); return rc; } /* Checking file size for detect the situation when we got the shared lock * immediately after lck_destroy(). */ if (st.st_size < (off_t)(sizeof(lck_t) + sizeof(reader_slot_t))) { - VERBOSE("lck-file is too short (%u), exclusive-lock needed", - (unsigned)st.st_size); + VERBOSE("lck-file is too short (%u), exclusive-lock needed", (unsigned)st.st_size); rc = MDBX_RESULT_TRUE; } @@ -298,8 +282,7 @@ __cold MDBX_INTERNAL int lck_seize(MDBX_env *env) { if (env->lck_mmap.fd == INVALID_HANDLE_VALUE) { /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */ - rc = lck_op(env->lazy_fd, op_setlk, - (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX); + rc = lck_op(env->lazy_fd, op_setlk, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX); if (rc != MDBX_SUCCESS) { ERROR("%s, err %u", "without-lck", rc); eASSERT(env, MDBX_IS_ERROR(rc)); @@ -329,8 +312,7 @@ retry: return rc; continue_dxb_exclusive: - rc = lck_op(env->lazy_fd, op_setlk, - (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX); + rc = lck_op(env->lazy_fd, op_setlk, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX); if (rc == MDBX_SUCCESS) return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */; @@ -339,16 +321,14 @@ retry: return err; /* the cause may be a collision with POSIX's file-lock recovery. */ - if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || - rc == EDEADLK)) { + if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || rc == EDEADLK)) { ERROR("%s, err %u", "dxb-exclusive", rc); eASSERT(env, MDBX_IS_ERROR(rc)); return rc; } /* Fallback to lck-shared */ - } else if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || - rc == EWOULDBLOCK || rc == EDEADLK)) { + } else if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || rc == EDEADLK)) { ERROR("%s, err %u", "try-exclusive", rc); eASSERT(env, MDBX_IS_ERROR(rc)); return rc; @@ -384,16 +364,14 @@ retry: if (rc == MDBX_SUCCESS) goto continue_dxb_exclusive; - if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || - rc == EDEADLK)) { + if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || rc == EDEADLK)) { ERROR("%s, err %u", "try-exclusive", rc); eASSERT(env, MDBX_IS_ERROR(rc)); return rc; } /* Lock against another process operating in without-lck or exclusive mode. */ - rc = lck_op(env->lazy_fd, op_setlk, - (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, env->pid, 1); + rc = lck_op(env->lazy_fd, op_setlk, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, env->pid, 1); if (rc != MDBX_SUCCESS) { ERROR("%s, err %u", "lock-against-without-lck", rc); eASSERT(env, MDBX_IS_ERROR(rc)); @@ -413,8 +391,7 @@ MDBX_INTERNAL int lck_downgrade(MDBX_env *env) { if ((env->flags & MDBX_EXCLUSIVE) == 0) { rc = lck_op(env->lazy_fd, op_setlk, F_UNLCK, 0, env->pid); if (rc == MDBX_SUCCESS) - rc = lck_op(env->lazy_fd, op_setlk, F_UNLCK, env->pid + 1, - OFF_T_MAX - env->pid - 1); + rc = lck_op(env->lazy_fd, op_setlk, F_UNLCK, env->pid + 1, OFF_T_MAX - env->pid - 1); } if (rc == MDBX_SUCCESS) rc = lck_op(env->lck_mmap.fd, op_setlk, F_RDLCK, 0, 1); @@ -433,13 +410,10 @@ MDBX_INTERNAL int lck_upgrade(MDBX_env *env, bool dont_wait) { const int cmd = dont_wait ? op_setlk : op_setlkw; int rc = lck_op(env->lck_mmap.fd, cmd, F_WRLCK, 0, 1); if (rc == MDBX_SUCCESS && (env->flags & MDBX_EXCLUSIVE) == 0) { - rc = (env->pid > 1) ? lck_op(env->lazy_fd, cmd, F_WRLCK, 0, env->pid - 1) - : MDBX_SUCCESS; + rc = (env->pid > 1) ? lck_op(env->lazy_fd, cmd, F_WRLCK, 0, env->pid - 1) : MDBX_SUCCESS; if (rc == MDBX_SUCCESS) { - rc = lck_op(env->lazy_fd, cmd, F_WRLCK, env->pid + 1, - OFF_T_MAX - env->pid - 1); - if (rc != MDBX_SUCCESS && env->pid > 1 && - lck_op(env->lazy_fd, op_setlk, F_UNLCK, 0, env->pid - 1)) + rc = lck_op(env->lazy_fd, cmd, F_WRLCK, env->pid + 1, OFF_T_MAX - env->pid - 1); + if (rc != MDBX_SUCCESS && env->pid > 1 && lck_op(env->lazy_fd, op_setlk, F_UNLCK, 0, env->pid - 1)) rc = MDBX_PANIC; } if (rc != MDBX_SUCCESS && lck_op(env->lck_mmap.fd, op_setlk, F_RDLCK, 0, 1)) @@ -452,9 +426,7 @@ MDBX_INTERNAL int lck_upgrade(MDBX_env *env, bool dont_wait) { return rc; } -__cold MDBX_INTERNAL int lck_destroy(MDBX_env *env, - MDBX_env *inprocess_neighbor, - const uint32_t current_pid) { +__cold MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, const uint32_t current_pid) { eASSERT(env, osal_getpid() == current_pid); int rc = MDBX_SUCCESS; struct stat lck_info; @@ -464,9 +436,7 @@ __cold MDBX_INTERNAL int lck_destroy(MDBX_env *env, lck_op(env->lck_mmap.fd, op_setlk, F_WRLCK, 0, OFF_T_MAX) == 0 && /* if LCK was not removed */ fstat(env->lck_mmap.fd, &lck_info) == 0 && lck_info.st_nlink > 0 && - lck_op(env->lazy_fd, op_setlk, - (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, - OFF_T_MAX) == 0) { + lck_op(env->lazy_fd, op_setlk, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX) == 0) { VERBOSE("%p got exclusive, drown ipc-locks", (void *)env); eASSERT(env, current_pid == env->pid); @@ -492,8 +462,7 @@ __cold MDBX_INTERNAL int lck_destroy(MDBX_env *env, if (current_pid != env->pid) { eASSERT(env, !inprocess_neighbor); - NOTICE("drown env %p after-fork pid %d -> %d", - __Wpedantic_format_voidptr(env), env->pid, current_pid); + NOTICE("drown env %p after-fork pid %d -> %d", __Wpedantic_format_voidptr(env), env->pid, current_pid); inprocess_neighbor = nullptr; } @@ -516,11 +485,8 @@ __cold MDBX_INTERNAL int lck_destroy(MDBX_env *env, env->lazy_fd = INVALID_HANDLE_VALUE; if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) { /* restore file-lock */ - rc = lck_op(inprocess_neighbor->lazy_fd, F_SETLKW, - (inprocess_neighbor->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, - (inprocess_neighbor->flags & MDBX_EXCLUSIVE) - ? 0 - : inprocess_neighbor->pid, + rc = lck_op(inprocess_neighbor->lazy_fd, F_SETLKW, (inprocess_neighbor->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, + (inprocess_neighbor->flags & MDBX_EXCLUSIVE) ? 0 : inprocess_neighbor->pid, (inprocess_neighbor->flags & MDBX_EXCLUSIVE) ? OFF_T_MAX : 1); } } @@ -545,8 +511,7 @@ __cold MDBX_INTERNAL int lck_destroy(MDBX_env *env, /*---------------------------------------------------------------------------*/ -__cold MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, - int global_uniqueness_flag) { +__cold MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, int global_uniqueness_flag) { #if MDBX_LOCKING == MDBX_LOCKING_SYSV int semid = -1; /* don't initialize semaphores twice */ @@ -556,9 +521,7 @@ __cold MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, if (fstat(env->lazy_fd, &st)) return errno; sysv_retry_create: - semid = semget(env->me_sysv_ipc.key, 2, - IPC_CREAT | IPC_EXCL | - (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO))); + semid = semget(env->me_sysv_ipc.key, 2, IPC_CREAT | IPC_EXCL | (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO))); if (unlikely(semid == -1)) { int err = errno; if (err != EEXIST) @@ -614,8 +577,7 @@ __cold MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, } return MDBX_SUCCESS; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 if (inprocess_neighbor) return MDBX_SUCCESS /* don't need any initialization for mutexes if LCK already opened/used inside current process */ @@ -653,8 +615,7 @@ __cold MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, #if MDBX_LOCKING == MDBX_LOCKING_POSIX2008 #if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutexattr_setrobust) rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST); -#elif defined(PTHREAD_MUTEX_ROBUST_NP) || \ - defined(pthread_mutexattr_setrobust_np) +#elif defined(PTHREAD_MUTEX_ROBUST_NP) || defined(pthread_mutexattr_setrobust_np) rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP); #elif _POSIX_THREAD_PROCESS_SHARED < 200809L rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP); @@ -665,8 +626,7 @@ __cold MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, goto bailout; #endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */ -#if defined(_POSIX_THREAD_PRIO_INHERIT) && _POSIX_THREAD_PRIO_INHERIT >= 0 && \ - !defined(MDBX_SAFE4QEMU) +#if defined(_POSIX_THREAD_PRIO_INHERIT) && _POSIX_THREAD_PRIO_INHERIT >= 0 && !defined(MDBX_SAFE4QEMU) rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT); if (rc == ENOTSUP) rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE); @@ -691,8 +651,7 @@ bailout: #endif /* MDBX_LOCKING > 0 */ } -__cold static int osal_ipclock_failed(MDBX_env *env, osal_ipclock_t *ipc, - const int err) { +__cold static int osal_ipclock_failed(MDBX_env *env, osal_ipclock_t *ipc, const int err) { int rc = err; #if MDBX_LOCKING == MDBX_LOCKING_POSIX2008 || MDBX_LOCKING == MDBX_LOCKING_SYSV @@ -712,8 +671,7 @@ __cold static int osal_ipclock_failed(MDBX_env *env, osal_ipclock_t *ipc, rc = MDBX_PANIC; } } - WARNING("%clock owner died, %s", (rlocked ? 'r' : 'w'), - (rc ? "this process' env is hosed" : "recovering")); + WARNING("%clock owner died, %s", (rlocked ? 'r' : 'w'), (rc ? "this process' env is hosed" : "recovering")); int check_rc = mvcc_cleanup_dead(env, rlocked, nullptr); check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc; @@ -781,10 +739,8 @@ MDBX_INTERNAL int osal_check_tid4bionic(void) { } #endif /* __ANDROID_API__ || ANDROID) || BIONIC */ -static int osal_ipclock_lock(MDBX_env *env, osal_ipclock_t *ipc, - const bool dont_wait) { -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +static int osal_ipclock_lock(MDBX_env *env, osal_ipclock_t *ipc, const bool dont_wait) { +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 int rc = osal_check_tid4bionic(); if (likely(rc == 0)) rc = dont_wait ? pthread_mutex_trylock(ipc) : pthread_mutex_lock(ipc); @@ -800,9 +756,8 @@ static int osal_ipclock_lock(MDBX_env *env, osal_ipclock_t *ipc, } else if (sem_wait(ipc)) rc = errno; #elif MDBX_LOCKING == MDBX_LOCKING_SYSV - struct sembuf op = {.sem_num = (ipc != &env->lck->wrt_lock), - .sem_op = -1, - .sem_flg = dont_wait ? IPC_NOWAIT | SEM_UNDO : SEM_UNDO}; + struct sembuf op = { + .sem_num = (ipc != &env->lck->wrt_lock), .sem_op = -1, .sem_flg = dont_wait ? IPC_NOWAIT | SEM_UNDO : SEM_UNDO}; int rc; if (semop(env->me_sysv_ipc.semid, &op, 1)) { rc = errno; @@ -823,8 +778,7 @@ static int osal_ipclock_lock(MDBX_env *env, osal_ipclock_t *ipc, int osal_ipclock_unlock(MDBX_env *env, osal_ipclock_t *ipc) { int err = MDBX_ENOSYS; -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 err = pthread_mutex_unlock(ipc); #elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 err = sem_post(ipc) ? errno : MDBX_SUCCESS; @@ -833,9 +787,7 @@ int osal_ipclock_unlock(MDBX_env *env, osal_ipclock_t *ipc) { err = EPERM; else { *ipc = 0; - struct sembuf op = {.sem_num = (ipc != &env->lck->wrt_lock), - .sem_op = 1, - .sem_flg = SEM_UNDO}; + struct sembuf op = {.sem_num = (ipc != &env->lck->wrt_lock), .sem_op = 1, .sem_flg = SEM_UNDO}; err = semop(env->me_sysv_ipc.semid, &op, 1) ? errno : MDBX_SUCCESS; } #else @@ -845,13 +797,9 @@ int osal_ipclock_unlock(MDBX_env *env, osal_ipclock_t *ipc) { if (unlikely(rc != MDBX_SUCCESS)) { const uint32_t current_pid = osal_getpid(); if (current_pid == env->pid || LOG_ENABLED(MDBX_LOG_NOTICE)) - debug_log((current_pid == env->pid) - ? MDBX_LOG_FATAL - : (rc = MDBX_SUCCESS, MDBX_LOG_NOTICE), - "ipc-unlock()", __LINE__, "failed: env %p, lck-%s %p, err %d\n", - __Wpedantic_format_voidptr(env), - (env->lck == env->lck_mmap.lck) ? "mmap" : "stub", - __Wpedantic_format_voidptr(env->lck), err); + debug_log((current_pid == env->pid) ? MDBX_LOG_FATAL : (rc = MDBX_SUCCESS, MDBX_LOG_NOTICE), "ipc-unlock()", + __LINE__, "failed: env %p, lck-%s %p, err %d\n", __Wpedantic_format_voidptr(env), + (env->lck == env->lck_mmap.lck) ? "mmap" : "stub", __Wpedantic_format_voidptr(env->lck), err); } return rc; } @@ -879,10 +827,9 @@ int lck_txn_lock(MDBX_env *env, bool dont_wait) { const int err = osal_ipclock_lock(env, &env->lck->wrt_lock, dont_wait); int rc = err; if (likely(!MDBX_IS_ERROR(err))) { - eASSERT(env, !env->basal_txn->owner || - err == /* если другой поток в этом-же процессе завершился - не освободив блокировку */ - MDBX_RESULT_TRUE); + eASSERT(env, !env->basal_txn->owner || err == /* если другой поток в этом-же процессе завершился + не освободив блокировку */ + MDBX_RESULT_TRUE); env->basal_txn->owner = osal_thread_self(); rc = MDBX_SUCCESS; } diff --git a/src/lck-windows.c b/src/lck-windows.c index fb8aa78e..10c362a6 100644 --- a/src/lck-windows.c +++ b/src/lck-windows.c @@ -16,10 +16,8 @@ #define LCK_WAITFOR 0 #define LCK_DONTWAIT LOCKFILE_FAIL_IMMEDIATELY -static int flock_with_event(HANDLE fd, HANDLE event, unsigned flags, - size_t offset, size_t bytes) { - TRACE("lock>>: fd %p, event %p, flags 0x%x offset %zu, bytes %zu >>", fd, - event, flags, offset, bytes); +static int flock_with_event(HANDLE fd, HANDLE event, unsigned flags, size_t offset, size_t bytes) { + TRACE("lock>>: fd %p, event %p, flags 0x%x offset %zu, bytes %zu >>", fd, event, flags, offset, bytes); OVERLAPPED ov; ov.Internal = 0; ov.InternalHigh = 0; @@ -27,8 +25,7 @@ static int flock_with_event(HANDLE fd, HANDLE event, unsigned flags, ov.Offset = (DWORD)offset; ov.OffsetHigh = HIGH_DWORD(offset); if (LockFileEx(fd, flags, 0, (DWORD)bytes, HIGH_DWORD(bytes), &ov)) { - TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << %s", fd, - event, flags, offset, bytes, "done"); + TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << %s", fd, event, flags, offset, bytes, "done"); return MDBX_SUCCESS; } @@ -36,37 +33,32 @@ static int flock_with_event(HANDLE fd, HANDLE event, unsigned flags, if (rc == ERROR_IO_PENDING) { if (event) { if (GetOverlappedResult(fd, &ov, &rc, true)) { - TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << %s", - fd, event, flags, offset, bytes, "overlapped-done"); + TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << %s", fd, event, flags, offset, bytes, + "overlapped-done"); return MDBX_SUCCESS; } rc = GetLastError(); } else CancelIo(fd); } - TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << err %d", - fd, event, flags, offset, bytes, (int)rc); + TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << err %d", fd, event, flags, offset, bytes, + (int)rc); return (int)rc; } -static inline int flock(HANDLE fd, unsigned flags, size_t offset, - size_t bytes) { +static inline int flock(HANDLE fd, unsigned flags, size_t offset, size_t bytes) { return flock_with_event(fd, 0, flags, offset, bytes); } -static inline int flock_data(const MDBX_env *env, unsigned flags, size_t offset, - size_t bytes) { - const HANDLE fd4data = - env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; +static inline int flock_data(const MDBX_env *env, unsigned flags, size_t offset, size_t bytes) { + const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; return flock_with_event(fd4data, env->dxb_lock_event, flags, offset, bytes); } static int funlock(mdbx_filehandle_t fd, size_t offset, size_t bytes) { TRACE("unlock: fd %p, offset %zu, bytes %zu", fd, offset, bytes); - return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes, - HIGH_DWORD(bytes)) - ? MDBX_SUCCESS - : (int)GetLastError(); + return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes, HIGH_DWORD(bytes)) ? MDBX_SUCCESS + : (int)GetLastError(); } /*----------------------------------------------------------------------------*/ @@ -88,9 +80,7 @@ int lck_txn_lock(MDBX_env *env, bool dontwait) { } else { __try { EnterCriticalSection(&env->windowsbug_lock); - } - __except ((GetExceptionCode() == - 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */) + } __except ((GetExceptionCode() == 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */) ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) { return MDBX_EDEADLK; @@ -101,20 +91,15 @@ int lck_txn_lock(MDBX_env *env, bool dontwait) { if (env->flags & MDBX_EXCLUSIVE) goto done; - const HANDLE fd4data = - env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; + const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; int rc = flock_with_event(fd4data, env->dxb_lock_event, - dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT) - : (LCK_EXCLUSIVE | LCK_WAITFOR), - DXB_BODY); + dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT) : (LCK_EXCLUSIVE | LCK_WAITFOR), DXB_BODY); if (rc == ERROR_LOCK_VIOLATION && dontwait) { SleepEx(0, true); - rc = flock_with_event(fd4data, env->dxb_lock_event, - LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_BODY); + rc = flock_with_event(fd4data, env->dxb_lock_event, LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_BODY); if (rc == ERROR_LOCK_VIOLATION) { SleepEx(0, true); - rc = flock_with_event(fd4data, env->dxb_lock_event, - LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_BODY); + rc = flock_with_event(fd4data, env->dxb_lock_event, LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_BODY); } } if (rc == MDBX_SUCCESS) { @@ -133,8 +118,7 @@ int lck_txn_lock(MDBX_env *env, bool dontwait) { void lck_txn_unlock(MDBX_env *env) { eASSERT(env, env->basal_txn->owner == osal_thread_self()); if ((env->flags & MDBX_EXCLUSIVE) == 0) { - const HANDLE fd4data = - env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; + const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; int err = funlock(fd4data, DXB_BODY); if (err != MDBX_SUCCESS) mdbx_panic("%s failed: err %u", __func__, err); @@ -173,8 +157,7 @@ MDBX_INTERNAL int lck_rdt_lock(MDBX_env *env) { } MDBX_INTERNAL void lck_rdt_unlock(MDBX_env *env) { - if (env->lck_mmap.fd != INVALID_HANDLE_VALUE && - (env->flags & MDBX_EXCLUSIVE) == 0) { + if (env->lck_mmap.fd != INVALID_HANDLE_VALUE && (env->flags & MDBX_EXCLUSIVE) == 0) { /* transition from S-E (locked) to S-? (used), e.g. unlock upper-part */ int err = funlock(env->lck_mmap.fd, LCK_UPPER); if (err != MDBX_SUCCESS) @@ -184,22 +167,15 @@ MDBX_INTERNAL void lck_rdt_unlock(MDBX_env *env) { } MDBX_INTERNAL int osal_lockfile(mdbx_filehandle_t fd, bool wait) { - return flock( - fd, wait ? LCK_EXCLUSIVE | LCK_WAITFOR : LCK_EXCLUSIVE | LCK_DONTWAIT, 0, - DXB_MAXLEN); + return flock(fd, wait ? LCK_EXCLUSIVE | LCK_WAITFOR : LCK_EXCLUSIVE | LCK_DONTWAIT, 0, DXB_MAXLEN); } -static int suspend_and_append(mdbx_handle_array_t **array, - const DWORD ThreadId) { +static int suspend_and_append(mdbx_handle_array_t **array, const DWORD ThreadId) { const unsigned limit = (*array)->limit; if ((*array)->count == limit) { - mdbx_handle_array_t *const ptr = - osal_realloc((limit > ARRAY_LENGTH((*array)->handles)) - ? *array - : /* don't free initial array on the stack */ nullptr, - sizeof(mdbx_handle_array_t) + - sizeof(HANDLE) * (limit * (size_t)2 - - ARRAY_LENGTH((*array)->handles))); + mdbx_handle_array_t *const ptr = osal_realloc( + (limit > ARRAY_LENGTH((*array)->handles)) ? *array : /* don't free initial array on the stack */ nullptr, + sizeof(mdbx_handle_array_t) + sizeof(HANDLE) * (limit * (size_t)2 - ARRAY_LENGTH((*array)->handles))); if (!ptr) return MDBX_ENOMEM; if (limit == ARRAY_LENGTH((*array)->handles)) @@ -208,16 +184,15 @@ static int suspend_and_append(mdbx_handle_array_t **array, (*array)->limit = limit * 2; } - HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION, - FALSE, ThreadId); + HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION, FALSE, ThreadId); if (hThread == nullptr) return (int)GetLastError(); if (SuspendThread(hThread) == (DWORD)-1) { int err = (int)GetLastError(); DWORD ExitCode; - if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED || - !GetExitCodeThread(hThread, &ExitCode) || ExitCode != STILL_ACTIVE) + if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED || !GetExitCodeThread(hThread, &ExitCode) || + ExitCode != STILL_ACTIVE) err = MDBX_SUCCESS; CloseHandle(hThread); return err; @@ -227,21 +202,17 @@ static int suspend_and_append(mdbx_handle_array_t **array, return MDBX_SUCCESS; } -MDBX_INTERNAL int -osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) { +MDBX_INTERNAL int osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) { eASSERT(env, (env->flags & MDBX_NOSTICKYTHREADS) == 0); const uintptr_t CurrentTid = GetCurrentThreadId(); int rc; if (env->lck_mmap.lck) { /* Scan LCK for threads of the current process */ const reader_slot_t *const begin = env->lck_mmap.lck->rdt; - const reader_slot_t *const end = - begin + - atomic_load32(&env->lck_mmap.lck->rdt_length, mo_AcquireRelease); + const reader_slot_t *const end = begin + atomic_load32(&env->lck_mmap.lck->rdt_length, mo_AcquireRelease); const uintptr_t WriteTxnOwner = env->basal_txn ? env->basal_txn->owner : 0; for (const reader_slot_t *reader = begin; reader < end; ++reader) { - if (reader->pid.weak != env->pid || !reader->tid.weak || - reader->tid.weak >= MDBX_TID_TXN_OUSTED) { + if (reader->pid.weak != env->pid || !reader->tid.weak || reader->tid.weak >= MDBX_TID_TXN_OUSTED) { skip_lck: continue; } @@ -280,8 +251,7 @@ osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) { } do { - if (entry.th32OwnerProcessID != env->pid || - entry.th32ThreadID == CurrentTid) + if (entry.th32OwnerProcessID != env->pid || entry.th32ThreadID == CurrentTid) continue; rc = suspend_and_append(array, entry.th32ThreadID); @@ -306,8 +276,8 @@ MDBX_INTERNAL int osal_resume_threads_after_remap(mdbx_handle_array_t *array) { if (ResumeThread(hThread) == (DWORD)-1) { const int err = (int)GetLastError(); DWORD ExitCode; - if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED && - GetExitCodeThread(hThread, &ExitCode) && ExitCode == STILL_ACTIVE) + if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED && GetExitCodeThread(hThread, &ExitCode) && + ExitCode == STILL_ACTIVE) rc = err; } CloseHandle(hThread); @@ -358,35 +328,30 @@ static void lck_unlock(MDBX_env *env) { do err = funlock(env->lck_mmap.fd, LCK_LOWER); while (err == MDBX_SUCCESS); - assert(err == ERROR_NOT_LOCKED || - (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION)); + assert(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION)); SetLastError(ERROR_SUCCESS); do err = funlock(env->lck_mmap.fd, LCK_UPPER); while (err == MDBX_SUCCESS); - assert(err == ERROR_NOT_LOCKED || - (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION)); + assert(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION)); SetLastError(ERROR_SUCCESS); } - const HANDLE fd4data = - env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; + const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; if (fd4data != INVALID_HANDLE_VALUE) { /* explicitly unlock to avoid latency for other processes (windows kernel * releases such locks via deferred queues) */ do err = funlock(fd4data, DXB_BODY); while (err == MDBX_SUCCESS); - assert(err == ERROR_NOT_LOCKED || - (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION)); + assert(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION)); SetLastError(ERROR_SUCCESS); do err = funlock(fd4data, DXB_WHOLE); while (err == MDBX_SUCCESS); - assert(err == ERROR_NOT_LOCKED || - (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION)); + assert(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION)); SetLastError(ERROR_SUCCESS); } } @@ -418,8 +383,7 @@ static int internal_seize_lck(HANDLE lfd) { /* 6) something went wrong, give up */ rc = funlock(lfd, LCK_UPPER); if (rc != MDBX_SUCCESS) - mdbx_panic("%s(%s) failed: err %u", __func__, "?-E(middle) >> ?-?(free)", - rc); + mdbx_panic("%s(%s) failed: err %u", __func__, "?-E(middle) >> ?-?(free)", rc); return rc; } @@ -435,16 +399,14 @@ static int internal_seize_lck(HANDLE lfd) { * transition to S-? (used) or ?-? (free) */ int err = funlock(lfd, LCK_UPPER); if (err != MDBX_SUCCESS) - mdbx_panic("%s(%s) failed: err %u", __func__, - "X-E(locked/middle) >> X-?(used/free)", err); + mdbx_panic("%s(%s) failed: err %u", __func__, "X-E(locked/middle) >> X-?(used/free)", err); /* 9) now on S-? (used, DONE) or ?-? (free, FAILURE) */ return rc; } MDBX_INTERNAL int lck_seize(MDBX_env *env) { - const HANDLE fd4data = - env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; + const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; assert(fd4data != INVALID_HANDLE_VALUE); if (env->flags & MDBX_EXCLUSIVE) return MDBX_RESULT_TRUE /* nope since files were must be opened @@ -479,16 +441,14 @@ MDBX_INTERNAL int lck_seize(MDBX_env *env) { jitter4testing(false); err = funlock(fd4data, DXB_WHOLE); if (err != MDBX_SUCCESS) - mdbx_panic("%s(%s) failed: err %u", __func__, - "unlock-against-without-lck", err); + mdbx_panic("%s(%s) failed: err %u", __func__, "unlock-against-without-lck", err); } return rc; } MDBX_INTERNAL int lck_downgrade(MDBX_env *env) { - const HANDLE fd4data = - env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; + const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; /* Transite from exclusive-write state (E-E) to used (S-?) */ assert(fd4data != INVALID_HANDLE_VALUE); assert(env->lck_mmap.fd != INVALID_HANDLE_VALUE); @@ -499,8 +459,7 @@ MDBX_INTERNAL int lck_downgrade(MDBX_env *env) { /* 1) now at E-E (exclusive-write), transition to ?_E (middle) */ int rc = funlock(env->lck_mmap.fd, LCK_LOWER); if (rc != MDBX_SUCCESS) - mdbx_panic("%s(%s) failed: err %u", __func__, - "E-E(exclusive-write) >> ?-E(middle)", rc); + mdbx_panic("%s(%s) failed: err %u", __func__, "E-E(exclusive-write) >> ?-E(middle)", rc); /* 2) now at ?-E (middle), transition to S-E (locked) */ rc = flock(env->lck_mmap.fd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER); @@ -513,8 +472,7 @@ MDBX_INTERNAL int lck_downgrade(MDBX_env *env) { /* 4) got S-E (locked), continue transition to S-? (used) */ rc = funlock(env->lck_mmap.fd, LCK_UPPER); if (rc != MDBX_SUCCESS) - mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> S-?(used)", - rc); + mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> S-?(used)", rc); return MDBX_SUCCESS /* 5) now at S-? (used), done */; } @@ -529,9 +487,7 @@ MDBX_INTERNAL int lck_upgrade(MDBX_env *env, bool dont_wait) { /* 1) now on S-? (used), try S-E (locked) */ jitter4testing(false); - int rc = flock(env->lck_mmap.fd, - dont_wait ? LCK_EXCLUSIVE | LCK_DONTWAIT : LCK_EXCLUSIVE, - LCK_UPPER); + int rc = flock(env->lck_mmap.fd, dont_wait ? LCK_EXCLUSIVE | LCK_DONTWAIT : LCK_EXCLUSIVE, LCK_UPPER); if (rc != MDBX_SUCCESS) { /* 2) something went wrong, give up */; VERBOSE("%s, err %u", "S-?(used) >> S-E(locked)", rc); @@ -541,14 +497,11 @@ MDBX_INTERNAL int lck_upgrade(MDBX_env *env, bool dont_wait) { /* 3) now on S-E (locked), transition to ?-E (middle) */ rc = funlock(env->lck_mmap.fd, LCK_LOWER); if (rc != MDBX_SUCCESS) - mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> ?-E(middle)", - rc); + mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> ?-E(middle)", rc); /* 4) now on ?-E (middle), try E-E (exclusive-write) */ jitter4testing(false); - rc = flock(env->lck_mmap.fd, - dont_wait ? LCK_EXCLUSIVE | LCK_DONTWAIT : LCK_EXCLUSIVE, - LCK_LOWER); + rc = flock(env->lck_mmap.fd, dont_wait ? LCK_EXCLUSIVE | LCK_DONTWAIT : LCK_EXCLUSIVE, LCK_LOWER); if (rc != MDBX_SUCCESS) { /* 5) something went wrong, give up */; VERBOSE("%s, err %u", "?-E(middle) >> E-E(exclusive-write)", rc); @@ -558,8 +511,7 @@ MDBX_INTERNAL int lck_upgrade(MDBX_env *env, bool dont_wait) { return MDBX_SUCCESS /* 6) now at E-E (exclusive-write), done */; } -MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, - int global_uniqueness_flag) { +MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, int global_uniqueness_flag) { (void)env; (void)inprocess_neighbor; (void)global_uniqueness_flag; @@ -568,12 +520,9 @@ MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, TOKEN_PRIVILEGES privileges; privileges.PrivilegeCount = 1; privileges.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; - if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, - &token) || - !LookupPrivilegeValue(nullptr, SE_LOCK_MEMORY_NAME, - &privileges.Privileges[0].Luid) || - !AdjustTokenPrivileges(token, FALSE, &privileges, sizeof(privileges), - nullptr, nullptr) || + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &token) || + !LookupPrivilegeValue(nullptr, SE_LOCK_MEMORY_NAME, &privileges.Privileges[0].Luid) || + !AdjustTokenPrivileges(token, FALSE, &privileges, sizeof(privileges), nullptr, nullptr) || GetLastError() != ERROR_SUCCESS) imports.SetFileIoOverlappedRange = nullptr; @@ -583,8 +532,7 @@ MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, return MDBX_SUCCESS; } -MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, - const uint32_t current_pid) { +MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, const uint32_t current_pid) { (void)current_pid; /* LY: should unmap before releasing the locks to avoid race condition and * STATUS_USER_MAPPED_FILE/ERROR_USER_MAPPED_FILE */ @@ -593,8 +541,7 @@ MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, if (env->lck_mmap.lck) { const bool synced = env->lck_mmap.lck->unsynced_pages.weak == 0; osal_munmap(&env->lck_mmap); - if (synced && !inprocess_neighbor && - env->lck_mmap.fd != INVALID_HANDLE_VALUE && + if (synced && !inprocess_neighbor && env->lck_mmap.fd != INVALID_HANDLE_VALUE && lck_upgrade(env, true) == MDBX_SUCCESS) /* this will fail if LCK is used/mmapped by other process(es) */ osal_ftruncate(env->lck_mmap.fd, 0); diff --git a/src/lck.c b/src/lck.c index 291a257d..1c7c3811 100644 --- a/src/lck.c +++ b/src/lck.c @@ -15,14 +15,12 @@ __cold static int lck_setup_locked(MDBX_env *env) { if (env->lck_mmap.fd == INVALID_HANDLE_VALUE) { env->lck = lckless_stub(env); env->max_readers = UINT_MAX; - DEBUG("lck-setup:%s%s%s", " lck-less", - (env->flags & MDBX_RDONLY) ? " readonly" : "", + DEBUG("lck-setup:%s%s%s", " lck-less", (env->flags & MDBX_RDONLY) ? " readonly" : "", (lck_seize_rc == MDBX_RESULT_TRUE) ? " exclusive" : " cooperative"); return lck_seize_rc; } - DEBUG("lck-setup:%s%s%s", " with-lck", - (env->flags & MDBX_RDONLY) ? " readonly" : "", + DEBUG("lck-setup:%s%s%s", " with-lck", (env->flags & MDBX_RDONLY) ? " readonly" : "", (lck_seize_rc == MDBX_RESULT_TRUE) ? " exclusive" : " cooperative"); MDBX_env *inprocess_neighbor = nullptr; @@ -30,8 +28,7 @@ __cold static int lck_setup_locked(MDBX_env *env) { if (unlikely(MDBX_IS_ERROR(err))) return err; if (inprocess_neighbor) { - if ((globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 || - (inprocess_neighbor->flags & MDBX_EXCLUSIVE) != 0) + if ((globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 || (inprocess_neighbor->flags & MDBX_EXCLUSIVE) != 0) return MDBX_BUSY; if (lck_seize_rc == MDBX_RESULT_TRUE) { err = lck_downgrade(env); @@ -47,52 +44,41 @@ __cold static int lck_setup_locked(MDBX_env *env) { return err; if (lck_seize_rc == MDBX_RESULT_TRUE) { - size = - ceil_powerof2(env->max_readers * sizeof(reader_slot_t) + sizeof(lck_t), - globals.sys_pagesize); + size = ceil_powerof2(env->max_readers * sizeof(reader_slot_t) + sizeof(lck_t), globals.sys_pagesize); jitter4testing(false); } else { if (env->flags & MDBX_EXCLUSIVE) return MDBX_BUSY; - if (size > INT_MAX || (size & (globals.sys_pagesize - 1)) != 0 || - size < globals.sys_pagesize) { + if (size > INT_MAX || (size & (globals.sys_pagesize - 1)) != 0 || size < globals.sys_pagesize) { ERROR("lck-file has invalid size %" PRIu64 " bytes", size); return MDBX_PROBLEM; } } - const size_t maxreaders = - ((size_t)size - sizeof(lck_t)) / sizeof(reader_slot_t); + const size_t maxreaders = ((size_t)size - sizeof(lck_t)) / sizeof(reader_slot_t); if (maxreaders < 4) { ERROR("lck-size too small (up to %" PRIuPTR " readers)", maxreaders); return MDBX_PROBLEM; } - env->max_readers = (maxreaders <= MDBX_READERS_LIMIT) - ? (unsigned)maxreaders - : (unsigned)MDBX_READERS_LIMIT; + env->max_readers = (maxreaders <= MDBX_READERS_LIMIT) ? (unsigned)maxreaders : (unsigned)MDBX_READERS_LIMIT; - err = osal_mmap((env->flags & MDBX_EXCLUSIVE) | MDBX_WRITEMAP, &env->lck_mmap, - (size_t)size, (size_t)size, - lck_seize_rc ? MMAP_OPTION_TRUNCATE | MMAP_OPTION_SEMAPHORE - : MMAP_OPTION_SEMAPHORE); + err = osal_mmap((env->flags & MDBX_EXCLUSIVE) | MDBX_WRITEMAP, &env->lck_mmap, (size_t)size, (size_t)size, + lck_seize_rc ? MMAP_OPTION_TRUNCATE | MMAP_OPTION_SEMAPHORE : MMAP_OPTION_SEMAPHORE); if (unlikely(err != MDBX_SUCCESS)) return err; #ifdef MADV_DODUMP - err = madvise(env->lck_mmap.lck, size, MADV_DODUMP) ? ignore_enosys(errno) - : MDBX_SUCCESS; + err = madvise(env->lck_mmap.lck, size, MADV_DODUMP) ? ignore_enosys(errno) : MDBX_SUCCESS; if (unlikely(MDBX_IS_ERROR(err))) return err; #endif /* MADV_DODUMP */ #ifdef MADV_WILLNEED - err = madvise(env->lck_mmap.lck, size, MADV_WILLNEED) ? ignore_enosys(errno) - : MDBX_SUCCESS; + err = madvise(env->lck_mmap.lck, size, MADV_WILLNEED) ? ignore_enosys(errno) : MDBX_SUCCESS; if (unlikely(MDBX_IS_ERROR(err))) return err; #elif defined(POSIX_MADV_WILLNEED) - err = ignore_enosys( - posix_madvise(env->lck_mmap.lck, size, POSIX_MADV_WILLNEED)); + err = ignore_enosys(posix_madvise(env->lck_mmap.lck, size, POSIX_MADV_WILLNEED)); if (unlikely(MDBX_IS_ERROR(err))) return err; #endif /* MADV_WILLNEED */ @@ -108,8 +94,7 @@ __cold static int lck_setup_locked(MDBX_env *env) { #if MDBX_ENABLE_PGOP_STAT lck->pgops.wops.weak = 1; #endif /* MDBX_ENABLE_PGOP_STAT */ - err = osal_msync(&env->lck_mmap, 0, (size_t)size, - MDBX_SYNC_DATA | MDBX_SYNC_SIZE); + err = osal_msync(&env->lck_mmap, 0, (size_t)size, MDBX_SYNC_DATA | MDBX_SYNC_SIZE); if (unlikely(err != MDBX_SUCCESS)) { ERROR("initial-%s for lck-file failed, err %d", "msync/fsync", err); eASSERT(env, MDBX_IS_ERROR(err)); @@ -118,17 +103,14 @@ __cold static int lck_setup_locked(MDBX_env *env) { } else { if (lck->magic_and_version != MDBX_LOCK_MAGIC) { const bool invalid = (lck->magic_and_version >> 8) != MDBX_MAGIC; - ERROR("lock region has %s", - invalid - ? "invalid magic" - : "incompatible version (only applications with nearly or the " - "same versions of libmdbx can share the same database)"); + ERROR("lock region has %s", invalid ? "invalid magic" + : "incompatible version (only applications with nearly or the " + "same versions of libmdbx can share the same database)"); return invalid ? MDBX_INVALID : MDBX_VERSION_MISMATCH; } if (lck->os_and_format != MDBX_LOCK_FORMAT) { - ERROR("lock region has os/format signature 0x%" PRIx32 - ", expected 0x%" PRIx32, - lck->os_and_format, MDBX_LOCK_FORMAT); + ERROR("lock region has os/format signature 0x%" PRIx32 ", expected 0x%" PRIx32, lck->os_and_format, + MDBX_LOCK_FORMAT); return MDBX_VERSION_MISMATCH; } } @@ -148,8 +130,7 @@ __cold int lck_setup(MDBX_env *env, mdbx_mode_t mode) { eASSERT(env, env->lazy_fd != INVALID_HANDLE_VALUE); eASSERT(env, env->lck_mmap.fd == INVALID_HANDLE_VALUE); - int err = osal_openfile(MDBX_OPEN_LCK, env, env->pathname.lck, - &env->lck_mmap.fd, mode); + int err = osal_openfile(MDBX_OPEN_LCK, env, env->pathname.lck, &env->lck_mmap.fd, mode); if (err != MDBX_SUCCESS) { switch (err) { default: @@ -186,6 +167,5 @@ __cold int lck_setup(MDBX_env *env, mdbx_mode_t mode) { } void mincore_clean_cache(const MDBX_env *const env) { - memset(env->lck->mincore_cache.begin, -1, - sizeof(env->lck->mincore_cache.begin)); + memset(env->lck->mincore_cache.begin, -1, sizeof(env->lck->mincore_cache.begin)); } diff --git a/src/lck.h b/src/lck.h index 00ef6189..eee38e15 100644 --- a/src/lck.h +++ b/src/lck.h @@ -23,8 +23,7 @@ MDBX_INTERNAL int lck_ipclock_destroy(osal_ipclock_t *ipc); /// MUST NOT initialize shared synchronization objects in memory-mapped /// LCK-file that are already in use. /// \return Error code or zero on success. -MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, - int global_uniqueness_flag); +MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, int global_uniqueness_flag); /// \brief Disconnects from shared interprocess objects and destructs /// synchronization objects linked with MDBX_env instance @@ -43,8 +42,7 @@ MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, /// of other instances of MDBX_env within the current process, e.g. /// restore POSIX-fcntl locks after the closing of file descriptors. /// \return Error code (MDBX_PANIC) or zero on success. -MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, - const uint32_t current_pid); +MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, const uint32_t current_pid); /// \brief Connects to shared interprocess locking objects and tries to acquire /// the maximum lock level (shared if exclusive is not available) diff --git a/src/logging_and_debug.c b/src/logging_and_debug.c index 20d8419e..e09f4dd6 100644 --- a/src/logging_and_debug.c +++ b/src/logging_and_debug.c @@ -3,15 +3,13 @@ #include "internals.h" -__cold void debug_log_va(int level, const char *function, int line, - const char *fmt, va_list args) { +__cold void debug_log_va(int level, const char *function, int line, const char *fmt, va_list args) { ENSURE(nullptr, osal_fastmutex_acquire(&globals.debug_lock) == 0); if (globals.logger.ptr) { if (globals.logger_buffer == nullptr) globals.logger.fmt(level, function, line, fmt, args); else { - const int len = vsnprintf(globals.logger_buffer, - globals.logger_buffer_size, fmt, args); + const int len = vsnprintf(globals.logger_buffer, globals.logger_buffer_size, fmt, args); if (len > 0) globals.logger.nofmt(level, function, line, globals.logger_buffer, len); } @@ -51,8 +49,7 @@ __cold void debug_log_va(int level, const char *function, int line, ENSURE(nullptr, osal_fastmutex_release(&globals.debug_lock) == 0); } -__cold void debug_log(int level, const char *function, int line, - const char *fmt, ...) { +__cold void debug_log(int level, const char *function, int line, const char *fmt, ...) { va_list args; va_start(args, fmt); debug_log_va(level, function, line, fmt, args); @@ -62,18 +59,15 @@ __cold void debug_log(int level, const char *function, int line, __cold int log_error(const int err, const char *func, unsigned line) { assert(err != MDBX_SUCCESS); if (unlikely(globals.loglevel >= MDBX_LOG_DEBUG) && - (globals.loglevel >= MDBX_LOG_TRACE || - !(err == MDBX_RESULT_TRUE || err == MDBX_NOTFOUND))) { + (globals.loglevel >= MDBX_LOG_TRACE || !(err == MDBX_RESULT_TRUE || err == MDBX_NOTFOUND))) { char buf[256]; - debug_log(MDBX_LOG_ERROR, func, line, "error %d (%s)\n", err, - mdbx_strerror_r(err, buf, sizeof(buf))); + debug_log(MDBX_LOG_ERROR, func, line, "error %d (%s)\n", err, mdbx_strerror_r(err, buf, sizeof(buf))); } return err; } /* Dump a val in ascii or hexadecimal. */ -__cold const char *mdbx_dump_val(const MDBX_val *val, char *const buf, - const size_t bufsize) { +__cold const char *mdbx_dump_val(const MDBX_val *val, char *const buf, const size_t bufsize) { if (!val) return ""; if (!val->iov_len) @@ -97,9 +91,7 @@ __cold const char *mdbx_dump_val(const MDBX_val *val, char *const buf, } if (is_ascii) { - int len = - snprintf(buf, bufsize, "%.*s", - (val->iov_len > INT_MAX) ? INT_MAX : (int)val->iov_len, data); + int len = snprintf(buf, bufsize, "%.*s", (val->iov_len > INT_MAX) ? INT_MAX : (int)val->iov_len, data); assert(len > 0 && (size_t)len < bufsize); (void)len; } else { @@ -107,8 +99,7 @@ __cold const char *mdbx_dump_val(const MDBX_val *val, char *const buf, char *ptr = buf; *ptr++ = '<'; for (size_t i = 0; i < val->iov_len && ptr < detent; i++) { - const char hex[16] = {'0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + const char hex[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; *ptr++ = hex[data[i] >> 4]; *ptr++ = hex[data[i] & 15]; } @@ -145,11 +136,8 @@ __cold const char *pagetype_caption(const uint8_t type, char buf4unknown[16]) { } __cold static const char *leafnode_type(node_t *n) { - static const char *const tp[2][2] = {{"", ": DB"}, - {": sub-page", ": sub-DB"}}; - return (node_flags(n) & N_BIG) - ? ": large page" - : tp[!!(node_flags(n) & N_DUP)][!!(node_flags(n) & N_TREE)]; + static const char *const tp[2][2] = {{"", ": DB"}, {": sub-page", ": sub-DB"}}; + return (node_flags(n) & N_BIG) ? ": large page" : tp[!!(node_flags(n) & N_DUP)][!!(node_flags(n) & N_TREE)]; } /* Display all the keys in the page. */ @@ -181,8 +169,7 @@ __cold void page_list(page_t *mp) { VERBOSE("Overflow page %" PRIaPGNO " pages %u\n", pgno, mp->pages); return; case P_META: - VERBOSE("Meta-page %" PRIaPGNO " txnid %" PRIu64 "\n", pgno, - unaligned_peek_u64(4, page_meta(mp)->txnid_a)); + VERBOSE("Meta-page %" PRIaPGNO " txnid %" PRIu64 "\n", pgno, unaligned_peek_u64(4, page_meta(mp)->txnid_a)); return; default: VERBOSE("Bad page %" PRIaPGNO " flags 0x%X\n", pgno, mp->flags); @@ -193,8 +180,7 @@ __cold void page_list(page_t *mp) { VERBOSE("%s %" PRIaPGNO " numkeys %zu\n", type, pgno, nkeys); for (i = 0; i < nkeys; i++) { - if (is_dupfix_leaf( - mp)) { /* DUPFIX pages have no entries[] or node headers */ + if (is_dupfix_leaf(mp)) { /* DUPFIX pages have no entries[] or node headers */ key = page_dupfix_key(mp, i, nsize = mp->dupfix_ksize); total += nsize; VERBOSE("key %zu: nsize %zu, %s\n", i, nsize, DKEY(&key)); @@ -205,8 +191,7 @@ __cold void page_list(page_t *mp) { key.iov_base = node->payload; nsize = NODESIZE + key.iov_len; if (is_branch(mp)) { - VERBOSE("key %zu: page %" PRIaPGNO ", %s\n", i, node_pgno(node), - DKEY(&key)); + VERBOSE("key %zu: page %" PRIaPGNO ", %s\n", i, node_pgno(node), DKEY(&key)); total += nsize; } else { if (node_flags(node) & N_BIG) @@ -215,18 +200,15 @@ __cold void page_list(page_t *mp) { nsize += node_ds(node); total += nsize; nsize += sizeof(indx_t); - VERBOSE("key %zu: nsize %zu, %s%s\n", i, nsize, DKEY(&key), - leafnode_type(node)); + VERBOSE("key %zu: nsize %zu, %s%s\n", i, nsize, DKEY(&key), leafnode_type(node)); } total = EVEN_CEIL(total); } - VERBOSE("Total: header %u + contents %zu + unused %zu\n", - is_dupfix_leaf(mp) ? PAGEHDRSZ : PAGEHDRSZ + mp->lower, total, - page_room(mp)); + VERBOSE("Total: header %u + contents %zu + unused %zu\n", is_dupfix_leaf(mp) ? PAGEHDRSZ : PAGEHDRSZ + mp->lower, + total, page_room(mp)); } -__cold static int setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags, - union logger_union logger, char *buffer, +__cold static int setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags, union logger_union logger, char *buffer, size_t buffer_size) { ENSURE(nullptr, osal_fastmutex_acquire(&globals.debug_lock) == 0); @@ -239,8 +221,7 @@ __cold static int setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags, #if MDBX_DEBUG MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER | #endif - MDBX_DBG_DUMP | MDBX_DBG_LEGACY_MULTIOPEN | MDBX_DBG_LEGACY_OVERLAP | - MDBX_DBG_DONT_UPGRADE; + MDBX_DBG_DUMP | MDBX_DBG_LEGACY_MULTIOPEN | MDBX_DBG_LEGACY_OVERLAP | MDBX_DBG_DONT_UPGRADE; globals.runtime_flags = (uint8_t)flags; } @@ -255,18 +236,14 @@ __cold static int setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags, return rc; } -__cold int mdbx_setup_debug_nofmt(MDBX_log_level_t level, - MDBX_debug_flags_t flags, - MDBX_debug_func_nofmt *logger, char *buffer, - size_t buffer_size) { +__cold int mdbx_setup_debug_nofmt(MDBX_log_level_t level, MDBX_debug_flags_t flags, MDBX_debug_func_nofmt *logger, + char *buffer, size_t buffer_size) { union logger_union thunk; - thunk.nofmt = - (logger && buffer && buffer_size) ? logger : MDBX_LOGGER_NOFMT_DONTCHANGE; + thunk.nofmt = (logger && buffer && buffer_size) ? logger : MDBX_LOGGER_NOFMT_DONTCHANGE; return setup_debug(level, flags, thunk, buffer, buffer_size); } -__cold int mdbx_setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags, - MDBX_debug_func *logger) { +__cold int mdbx_setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags, MDBX_debug_func *logger) { union logger_union thunk; thunk.fmt = logger; return setup_debug(level, flags, thunk, nullptr, 0); diff --git a/src/logging_and_debug.h b/src/logging_and_debug.h index 4feac0b8..9382eafc 100644 --- a/src/logging_and_debug.h +++ b/src/logging_and_debug.h @@ -6,23 +6,17 @@ #include "essentials.h" #ifndef __Wpedantic_format_voidptr -MDBX_MAYBE_UNUSED static inline const void * -__Wpedantic_format_voidptr(const void *ptr) { - return ptr; -} +MDBX_MAYBE_UNUSED static inline const void *__Wpedantic_format_voidptr(const void *ptr) { return ptr; } #define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) #endif /* __Wpedantic_format_voidptr */ -MDBX_INTERNAL void MDBX_PRINTF_ARGS(4, 5) - debug_log(int level, const char *function, int line, const char *fmt, ...) - MDBX_PRINTF_ARGS(4, 5); -MDBX_INTERNAL void debug_log_va(int level, const char *function, int line, - const char *fmt, va_list args); +MDBX_INTERNAL void MDBX_PRINTF_ARGS(4, 5) debug_log(int level, const char *function, int line, const char *fmt, ...) + MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL void debug_log_va(int level, const char *function, int line, const char *fmt, va_list args); #if MDBX_DEBUG #define LOG_ENABLED(LVL) unlikely(LVL <= globals.loglevel) -#define AUDIT_ENABLED() \ - unlikely((globals.runtime_flags & (unsigned)MDBX_DBG_AUDIT)) +#define AUDIT_ENABLED() unlikely((globals.runtime_flags & (unsigned)MDBX_DBG_AUDIT)) #else /* MDBX_DEBUG */ #define LOG_ENABLED(LVL) (LVL < MDBX_LOG_VERBOSE && LVL <= globals.loglevel) #define AUDIT_ENABLED() (0) @@ -31,91 +25,88 @@ MDBX_INTERNAL void debug_log_va(int level, const char *function, int line, #if MDBX_FORCE_ASSERTIONS #define ASSERT_ENABLED() (1) #elif MDBX_DEBUG -#define ASSERT_ENABLED() \ - likely((globals.runtime_flags & (unsigned)MDBX_DBG_ASSERT)) +#define ASSERT_ENABLED() likely((globals.runtime_flags & (unsigned)MDBX_DBG_ASSERT)) #else #define ASSERT_ENABLED() (0) #endif /* ASSERT_ENABLED() */ -#define DEBUG_EXTRA(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ +#define DEBUG_EXTRA(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ } while (0) -#define DEBUG_EXTRA_PRINT(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, nullptr, 0, fmt, __VA_ARGS__); \ +#define DEBUG_EXTRA_PRINT(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, nullptr, 0, fmt, __VA_ARGS__); \ } while (0) -#define TRACE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_TRACE)) \ - debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ +#define TRACE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_TRACE)) \ + debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ } while (0) -#define DEBUG(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ - debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ +#define DEBUG(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ + debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ } while (0) -#define VERBOSE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ - debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ +#define VERBOSE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ + debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ } while (0) -#define NOTICE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ - debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ +#define NOTICE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ + debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ } while (0) -#define WARNING(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_WARN)) \ - debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ +#define WARNING(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_WARN)) \ + debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ } while (0) -#undef ERROR /* wingdi.h \ +#undef ERROR /* wingdi.h \ Yeah, morons from M$ put such definition to the public header. */ -#define ERROR(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_ERROR)) \ - debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ +#define ERROR(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_ERROR)) \ + debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ } while (0) -#define FATAL(fmt, ...) \ - debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); +#define FATAL(fmt, ...) debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); #if MDBX_DEBUG #define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) #else /* MDBX_DEBUG */ -MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, - unsigned line); -#define ASSERT_FAIL(env, msg, func, line) \ - do { \ - (void)(env); \ - assert_fail(msg, func, line); \ +MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, unsigned line); +#define ASSERT_FAIL(env, msg, func, line) \ + do { \ + (void)(env); \ + assert_fail(msg, func, line); \ } while (0) #endif /* MDBX_DEBUG */ -#define ENSURE_MSG(env, expr, msg) \ - do { \ - if (unlikely(!(expr))) \ - ASSERT_FAIL(env, msg, __func__, __LINE__); \ +#define ENSURE_MSG(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + ASSERT_FAIL(env, msg, __func__, __LINE__); \ } while (0) #define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) /* assert(3) variant in environment context */ -#define eASSERT(env, expr) \ - do { \ - if (ASSERT_ENABLED()) \ - ENSURE(env, expr); \ +#define eASSERT(env, expr) \ + do { \ + if (ASSERT_ENABLED()) \ + ENSURE(env, expr); \ } while (0) /* assert(3) variant in cursor context */ @@ -140,14 +131,12 @@ MDBX_MAYBE_UNUSED static inline void jitter4testing(bool tiny) { MDBX_MAYBE_UNUSED MDBX_INTERNAL void page_list(page_t *mp); -MDBX_INTERNAL const char *pagetype_caption(const uint8_t type, - char buf4unknown[16]); +MDBX_INTERNAL const char *pagetype_caption(const uint8_t type, char buf4unknown[16]); /* Key size which fits in a DKBUF (debug key buffer). */ #define DKBUF_MAX 127 #define DKBUF char dbg_kbuf[DKBUF_MAX * 4 + 2] #define DKEY(x) mdbx_dump_val(x, dbg_kbuf, DKBUF_MAX * 2 + 1) -#define DVAL(x) \ - mdbx_dump_val(x, dbg_kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) +#define DVAL(x) mdbx_dump_val(x, dbg_kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) #if MDBX_DEBUG #define DKBUF_DEBUG DKBUF @@ -161,8 +150,7 @@ MDBX_INTERNAL const char *pagetype_caption(const uint8_t type, MDBX_INTERNAL int log_error(const int err, const char *func, unsigned line); -MDBX_MAYBE_UNUSED static inline int -log_if_error(const int err, const char *func, unsigned line) { +MDBX_MAYBE_UNUSED static inline int log_if_error(const int err, const char *func, unsigned line) { if (likely(err == MDBX_SUCCESS)) return err; int rc = log_error(err, func, line); diff --git a/src/mdbx.c++ b/src/mdbx.c++ index 7c83aa8b..17ca4718 100644 --- a/src/mdbx.c++ +++ b/src/mdbx.c++ @@ -187,21 +187,19 @@ __cold bug::~bug() noexcept {} throw bug(what_and_where); } -#define RAISE_BUG(line, condition, function, file) \ - do { \ - static MDBX_CXX11_CONSTEXPR_VAR trouble_location bug(line, condition, \ - function, file); \ - raise_bug(bug); \ +#define RAISE_BUG(line, condition, function, file) \ + do { \ + static MDBX_CXX11_CONSTEXPR_VAR trouble_location bug(line, condition, function, file); \ + raise_bug(bug); \ } while (0) -#define ENSURE(condition) \ - do \ - if (MDBX_UNLIKELY(!(condition))) \ - MDBX_CXX20_UNLIKELY RAISE_BUG(__LINE__, #condition, __func__, __FILE__); \ +#define ENSURE(condition) \ + do \ + if (MDBX_UNLIKELY(!(condition))) \ + MDBX_CXX20_UNLIKELY RAISE_BUG(__LINE__, #condition, __func__, __FILE__); \ while (0) -#define NOT_IMPLEMENTED() \ - RAISE_BUG(__LINE__, "not_implemented", __func__, __FILE__); +#define NOT_IMPLEMENTED() RAISE_BUG(__LINE__, "not_implemented", __func__, __FILE__); #endif /* Unused*/ @@ -226,14 +224,12 @@ struct line_wrapper { } }; -template -struct temp_buffer { +template struct temp_buffer { TYPE inplace[(INPLACE_BYTES + sizeof(TYPE) - 1) / sizeof(TYPE)]; const size_t size; TYPE *const area; temp_buffer(size_t bytes) - : size((bytes + sizeof(TYPE) - 1) / sizeof(TYPE)), - area((bytes > sizeof(inplace)) ? new TYPE[size] : inplace) { + : size((bytes + sizeof(TYPE) - 1) / sizeof(TYPE)), area((bytes > sizeof(inplace)) ? new TYPE[size] : inplace) { memset(area, 0, sizeof(TYPE) * size); } ~temp_buffer() { @@ -265,8 +261,7 @@ struct temp_buffer { namespace mdbx { [[noreturn]] __cold void throw_max_length_exceeded() { - throw std::length_error( - "mdbx:: Exceeded the maximal length of data/slice/buffer."); + throw std::length_error("mdbx:: Exceeded the maximal length of data/slice/buffer."); } [[noreturn]] __cold void throw_too_small_target_buffer() { @@ -279,38 +274,31 @@ namespace mdbx { } [[noreturn]] __cold void throw_allocators_mismatch() { - throw std::logic_error( - "mdbx:: An allocators mismatch, so an object could not be transferred " - "into an incompatible memory allocation scheme."); + throw std::logic_error("mdbx:: An allocators mismatch, so an object could not be transferred " + "into an incompatible memory allocation scheme."); } [[noreturn]] __cold void throw_incomparable_cursors() { - throw std::logic_error( - "mdbx:: incomparable and/or invalid cursors to compare positions."); + throw std::logic_error("mdbx:: incomparable and/or invalid cursors to compare positions."); } -[[noreturn]] __cold void throw_bad_value_size() { - throw bad_value_size(MDBX_BAD_VALSIZE); -} +[[noreturn]] __cold void throw_bad_value_size() { throw bad_value_size(MDBX_BAD_VALSIZE); } -__cold exception::exception(const ::mdbx::error &error) noexcept - : base(error.what()), error_(error) {} +__cold exception::exception(const ::mdbx::error &error) noexcept : base(error.what()), error_(error) {} __cold exception::~exception() noexcept {} static std::atomic_int fatal_countdown; -__cold fatal::fatal(const ::mdbx::error &error) noexcept : base(error) { - ++fatal_countdown; -} +__cold fatal::fatal(const ::mdbx::error &error) noexcept : base(error) { ++fatal_countdown; } __cold fatal::~fatal() noexcept { if (--fatal_countdown == 0) std::terminate(); } -#define DEFINE_EXCEPTION(NAME) \ - __cold NAME::NAME(const ::mdbx::error &rc) : exception(rc) {} \ +#define DEFINE_EXCEPTION(NAME) \ + __cold NAME::NAME(const ::mdbx::error &rc) : exception(rc) {} \ __cold NAME::~NAME() noexcept {} DEFINE_EXCEPTION(bad_map_id) @@ -352,8 +340,8 @@ __cold const char *error::what() const noexcept { return mdbx_liberr2str(code()); switch (code()) { -#define ERROR_CASE(CODE) \ - case CODE: \ +#define ERROR_CASE(CODE) \ + case CODE: \ return MDBX_STRINGIFY(CODE) ERROR_CASE(MDBX_ENODATA); ERROR_CASE(MDBX_EINVAL); @@ -379,8 +367,7 @@ __cold std::string error::message() const { return std::string(msg ? msg : "unknown"); } -[[noreturn]] __cold void error::panic(const char *context, - const char *func) const noexcept { +[[noreturn]] __cold void error::panic(const char *context, const char *func) const noexcept { assert(code() != MDBX_SUCCESS); ::mdbx_panic("mdbx::%s.%s(): \"%s\" (%d)", context, func, what(), code()); std::terminate(); @@ -397,8 +384,8 @@ __cold void error::throw_exception() const { throw std::logic_error("MDBX_SUCCESS (MDBX_RESULT_FALSE)"); case MDBX_RESULT_TRUE: throw std::logic_error("MDBX_RESULT_TRUE"); -#define CASE_EXCEPTION(NAME, CODE) \ - case CODE: \ +#define CASE_EXCEPTION(NAME, CODE) \ + case CODE: \ throw NAME(code()) CASE_EXCEPTION(bad_map_id, MDBX_BAD_DBI); CASE_EXCEPTION(bad_transaction, MDBX_BAD_TXN); @@ -702,27 +689,23 @@ char *to_hex::write_bytes(char *__restrict const dest, size_t dest_size) const { return out; } -char *from_hex::write_bytes(char *__restrict const dest, - size_t dest_size) const { +char *from_hex::write_bytes(char *__restrict const dest, size_t dest_size) const { if (MDBX_UNLIKELY(source.length() % 2 && !ignore_spaces)) - MDBX_CXX20_UNLIKELY throw std::domain_error( - "mdbx::from_hex:: odd length of hexadecimal string"); + MDBX_CXX20_UNLIKELY throw std::domain_error("mdbx::from_hex:: odd length of hexadecimal string"); if (MDBX_UNLIKELY(envisage_result_length() > dest_size)) MDBX_CXX20_UNLIKELY throw_too_small_target_buffer(); auto ptr = dest; auto src = source.byte_ptr(); for (auto left = source.length(); left > 0;) { - if (MDBX_UNLIKELY(*src <= ' ') && - MDBX_LIKELY(ignore_spaces && isspace(*src))) { + if (MDBX_UNLIKELY(*src <= ' ') && MDBX_LIKELY(ignore_spaces && isspace(*src))) { ++src; --left; continue; } if (MDBX_UNLIKELY(left < 1 || !isxdigit(src[0]) || !isxdigit(src[1]))) - MDBX_CXX20_UNLIKELY throw std::domain_error( - "mdbx::from_hex:: invalid hexadecimal string"); + MDBX_CXX20_UNLIKELY throw std::domain_error("mdbx::from_hex:: invalid hexadecimal string"); int8_t hi = src[0]; hi = (hi | 0x20) - 'a'; @@ -747,8 +730,7 @@ bool from_hex::is_erroneous() const noexcept { bool got = false; auto src = source.byte_ptr(); for (auto left = source.length(); left > 0;) { - if (MDBX_UNLIKELY(*src <= ' ') && - MDBX_LIKELY(ignore_spaces && isspace(*src))) { + if (MDBX_UNLIKELY(*src <= ' ') && MDBX_LIKELY(ignore_spaces && isspace(*src))) { ++src; --left; continue; @@ -780,25 +762,21 @@ using b58_uint = uint_fast32_t; #endif struct b58_buffer : public temp_buffer { - b58_buffer(size_t bytes, size_t estimation_ratio_numerator, - size_t estimation_ratio_denominator, size_t extra = 0) - : temp_buffer((/* пересчитываем по указанной пропорции */ - bytes = (bytes * estimation_ratio_numerator + - estimation_ratio_denominator - 1) / - estimation_ratio_denominator, - /* учитываем резервный старший байт в каждом слове */ - ((bytes + sizeof(b58_uint) - 2) / (sizeof(b58_uint) - 1) * - sizeof(b58_uint) + - extra) * - sizeof(b58_uint))) {} + b58_buffer(size_t bytes, size_t estimation_ratio_numerator, size_t estimation_ratio_denominator, size_t extra = 0) + : temp_buffer( + (/* пересчитываем по указанной пропорции */ + bytes = + (bytes * estimation_ratio_numerator + estimation_ratio_denominator - 1) / estimation_ratio_denominator, + /* учитываем резервный старший байт в каждом слове */ + ((bytes + sizeof(b58_uint) - 2) / (sizeof(b58_uint) - 1) * sizeof(b58_uint) + extra) * sizeof(b58_uint))) { + } }; static byte b58_8to11(b58_uint &v) noexcept { - static const char b58_alphabet[58] = { - '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', - 'G', 'H', 'J', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', - 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'm', - 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; + static const char b58_alphabet[58] = {'1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', + 'G', 'H', 'J', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', + 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; const auto i = size_t(v % 58); v /= 58; @@ -807,9 +785,8 @@ static byte b58_8to11(b58_uint &v) noexcept { static slice b58_encode(b58_buffer &buf, const byte *begin, const byte *end) { auto high = buf.end(); - const auto modulo = - b58_uint((sizeof(b58_uint) > 4) ? UINT64_C(0x1A636A90B07A00) /* 58^9 */ - : UINT32_C(0xACAD10) /* 58^4 */); + const auto modulo = b58_uint((sizeof(b58_uint) > 4) ? UINT64_C(0x1A636A90B07A00) /* 58^9 */ + : UINT32_C(0xACAD10) /* 58^4 */); static_assert(sizeof(modulo) == 4 || sizeof(modulo) == 8, "WTF?"); while (begin < end) { b58_uint carry = *begin++; @@ -855,8 +832,7 @@ static slice b58_encode(b58_buffer &buf, const byte *begin, const byte *end) { return slice(output, ptr); } -char *to_base58::write_bytes(char *__restrict const dest, - size_t dest_size) const { +char *to_base58::write_bytes(char *__restrict const dest, size_t dest_size) const { if (MDBX_UNLIKELY(envisage_result_length() > dest_size)) MDBX_CXX20_UNLIKELY throw_too_small_target_buffer(); @@ -927,8 +903,7 @@ const signed char b58_map[256] = { IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL // f0 }; -static slice b58_decode(b58_buffer &buf, const byte *begin, const byte *end, - bool ignore_spaces) { +static slice b58_decode(b58_buffer &buf, const byte *begin, const byte *end, bool ignore_spaces) { auto high = buf.end(); while (begin < end) { const auto c = b58_map[*begin++]; @@ -969,8 +944,7 @@ static slice b58_decode(b58_buffer &buf, const byte *begin, const byte *end, return slice(output, ptr); } -char *from_base58::write_bytes(char *__restrict const dest, - size_t dest_size) const { +char *from_base58::write_bytes(char *__restrict const dest, size_t dest_size) const { if (MDBX_UNLIKELY(envisage_result_length() > dest_size)) MDBX_CXX20_UNLIKELY throw_too_small_target_buffer(); @@ -996,8 +970,7 @@ bool from_base58::is_erroneous() const noexcept { auto begin = source.byte_ptr(); auto const end = source.end_byte_ptr(); while (begin < end) { - if (MDBX_UNLIKELY(b58_map[*begin] < 0 && - !(ignore_spaces && isspace(*begin)))) + if (MDBX_UNLIKELY(b58_map[*begin] < 0 && !(ignore_spaces && isspace(*begin)))) return true; ++begin; } @@ -1006,22 +979,18 @@ bool from_base58::is_erroneous() const noexcept { //------------------------------------------------------------------------------ -static inline void b64_3to4(const byte x, const byte y, const byte z, - char *__restrict dest) noexcept { - static const byte alphabet[64] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', - 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', - 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; +static inline void b64_3to4(const byte x, const byte y, const byte z, char *__restrict dest) noexcept { + static const byte alphabet[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; dest[0] = alphabet[(x & 0xfc) >> 2]; dest[1] = alphabet[((x & 0x03) << 4) + ((y & 0xf0) >> 4)]; dest[2] = alphabet[((y & 0x0f) << 2) + ((z & 0xc0) >> 6)]; dest[3] = alphabet[z & 0x3f]; } -char *to_base64::write_bytes(char *__restrict const dest, - size_t dest_size) const { +char *to_base64::write_bytes(char *__restrict const dest, size_t dest_size) const { if (MDBX_UNLIKELY(envisage_result_length() > dest_size)) MDBX_CXX20_UNLIKELY throw_too_small_target_buffer(); @@ -1115,8 +1084,7 @@ static const signed char b64_map[256] = { IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL // f0 }; -static inline signed char b64_4to3(signed char a, signed char b, signed char c, - signed char d, +static inline signed char b64_4to3(signed char a, signed char b, signed char c, signed char d, char *__restrict dest) noexcept { dest[0] = byte((a << 2) + ((b & 0x30) >> 4)); dest[1] = byte(((b & 0xf) << 4) + ((c & 0x3c) >> 2)); @@ -1124,19 +1092,16 @@ static inline signed char b64_4to3(signed char a, signed char b, signed char c, return a | b | c | d; } -char *from_base64::write_bytes(char *__restrict const dest, - size_t dest_size) const { +char *from_base64::write_bytes(char *__restrict const dest, size_t dest_size) const { if (MDBX_UNLIKELY(source.length() % 4 && !ignore_spaces)) - MDBX_CXX20_UNLIKELY throw std::domain_error( - "mdbx::from_base64:: odd length of base64 string"); + MDBX_CXX20_UNLIKELY throw std::domain_error("mdbx::from_base64:: odd length of base64 string"); if (MDBX_UNLIKELY(envisage_result_length() > dest_size)) MDBX_CXX20_UNLIKELY throw_too_small_target_buffer(); auto ptr = dest; auto src = source.byte_ptr(); for (auto left = source.length(); left > 0;) { - if (MDBX_UNLIKELY(*src <= ' ') && - MDBX_LIKELY(ignore_spaces && isspace(*src))) { + if (MDBX_UNLIKELY(*src <= ' ') && MDBX_LIKELY(ignore_spaces && isspace(*src))) { ++src; --left; continue; @@ -1147,8 +1112,7 @@ char *from_base64::write_bytes(char *__restrict const dest, bailout: throw std::domain_error("mdbx::from_base64:: invalid base64 string"); } - const signed char a = b64_map[src[0]], b = b64_map[src[1]], - c = b64_map[src[2]], d = b64_map[src[3]]; + const signed char a = b64_map[src[0]], b = b64_map[src[1]], c = b64_map[src[2]], d = b64_map[src[3]]; if (MDBX_UNLIKELY(b64_4to3(a, b, c, d, ptr) < 0)) { if (left == 4 && (a | b) >= 0 && d == EQ) { if (c >= 0) { @@ -1177,8 +1141,7 @@ bool from_base64::is_erroneous() const noexcept { bool got = false; auto src = source.byte_ptr(); for (auto left = source.length(); left > 0;) { - if (MDBX_UNLIKELY(*src <= ' ') && - MDBX_LIKELY(ignore_spaces && isspace(*src))) { + if (MDBX_UNLIKELY(*src <= ' ') && MDBX_LIKELY(ignore_spaces && isspace(*src))) { ++src; --left; continue; @@ -1186,8 +1149,7 @@ bool from_base64::is_erroneous() const noexcept { if (MDBX_UNLIKELY(left < 3)) MDBX_CXX20_UNLIKELY return false; - const signed char a = b64_map[src[0]], b = b64_map[src[1]], - c = b64_map[src[2]], d = b64_map[src[3]]; + const signed char a = b64_map[src[0]], b = b64_map[src[1]], c = b64_map[src[2]], d = b64_map[src[3]]; if (MDBX_UNLIKELY((a | b | c | d) < 0)) MDBX_CXX20_UNLIKELY { if (left == 4 && (a | b) >= 0 && d == EQ && (c >= 0 || c == d)) @@ -1205,8 +1167,7 @@ bool from_base64::is_erroneous() const noexcept { template class LIBMDBX_API_TYPE buffer; -#if defined(__cpp_lib_memory_resource) && \ - __cpp_lib_memory_resource >= 201603L && _GLIBCXX_USE_CXX11_ABI +#if defined(__cpp_lib_memory_resource) && __cpp_lib_memory_resource >= 201603L && _GLIBCXX_USE_CXX11_ABI template class LIBMDBX_API_TYPE buffer; #endif /* __cpp_lib_memory_resource >= 201603L */ @@ -1225,8 +1186,7 @@ static inline MDBX_env_flags_t mode2flags(env::mode mode) { } } -__cold MDBX_env_flags_t -env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const { +__cold MDBX_env_flags_t env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const { MDBX_env_flags_t flags = mode2flags(mode); if (accede) flags |= MDBX_ACCEDE; @@ -1252,8 +1212,7 @@ env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const { flags |= MDBX_LIFORECLAIM; switch (durability) { default: - MDBX_CXX20_UNLIKELY throw std::invalid_argument( - "db::durability is invalid"); + MDBX_CXX20_UNLIKELY throw std::invalid_argument("db::durability is invalid"); case env::durability::robust_synchronous: break; case env::durability::half_synchronous_weak_last: @@ -1271,16 +1230,13 @@ env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const { return flags; } -env::mode -env::operate_parameters::mode_from_flags(MDBX_env_flags_t flags) noexcept { +env::mode env::operate_parameters::mode_from_flags(MDBX_env_flags_t flags) noexcept { if (flags & MDBX_RDONLY) return env::mode::readonly; - return (flags & MDBX_WRITEMAP) ? env::mode::write_mapped_io - : env::mode::write_file_io; + return (flags & MDBX_WRITEMAP) ? env::mode::write_mapped_io : env::mode::write_file_io; } -env::durability env::operate_parameters::durability_from_flags( - MDBX_env_flags_t flags) noexcept { +env::durability env::operate_parameters::durability_from_flags(MDBX_env_flags_t flags) noexcept { if ((flags & MDBX_UTTERLY_NOSYNC) == MDBX_UTTERLY_NOSYNC) return env::durability::whole_fragile; if (flags & MDBX_SAFE_NOSYNC) @@ -1291,71 +1247,51 @@ env::durability env::operate_parameters::durability_from_flags( } env::reclaiming_options::reclaiming_options(MDBX_env_flags_t flags) noexcept - : lifo((flags & MDBX_LIFORECLAIM) ? true : false), - coalesce((flags & MDBX_COALESCE) ? true : false) {} + : lifo((flags & MDBX_LIFORECLAIM) ? true : false), coalesce((flags & MDBX_COALESCE) ? true : false) {} env::operate_options::operate_options(MDBX_env_flags_t flags) noexcept - : no_sticky_threads(((flags & (MDBX_NOSTICKYTHREADS | MDBX_EXCLUSIVE)) == - MDBX_NOSTICKYTHREADS) - ? true - : false), - nested_write_transactions((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) ? false - : true), - exclusive((flags & MDBX_EXCLUSIVE) ? true : false), - disable_readahead((flags & MDBX_NORDAHEAD) ? true : false), + : no_sticky_threads(((flags & (MDBX_NOSTICKYTHREADS | MDBX_EXCLUSIVE)) == MDBX_NOSTICKYTHREADS) ? true : false), + nested_write_transactions((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) ? false : true), + exclusive((flags & MDBX_EXCLUSIVE) ? true : false), disable_readahead((flags & MDBX_NORDAHEAD) ? true : false), disable_clear_memory((flags & MDBX_NOMEMINIT) ? true : false) {} -bool env::is_pristine() const { - return get_stat().ms_mod_txnid == 0 && - get_info().mi_recent_txnid == INITIAL_TXNID; -} +bool env::is_pristine() const { return get_stat().ms_mod_txnid == 0 && get_info().mi_recent_txnid == INITIAL_TXNID; } bool env::is_empty() const { return get_stat().ms_leaf_pages == 0; } __cold env &env::copy(filehandle fd, bool compactify, bool force_dynamic_size) { - error::success_or_throw( - ::mdbx_env_copy2fd(handle_, fd, - (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | - (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE - : MDBX_CP_DEFAULTS))); + error::success_or_throw(::mdbx_env_copy2fd(handle_, fd, + (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | + (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE : MDBX_CP_DEFAULTS))); return *this; } -__cold env &env::copy(const char *destination, bool compactify, - bool force_dynamic_size) { - error::success_or_throw( - ::mdbx_env_copy(handle_, destination, - (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | - (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE - : MDBX_CP_DEFAULTS))); +__cold env &env::copy(const char *destination, bool compactify, bool force_dynamic_size) { + error::success_or_throw(::mdbx_env_copy(handle_, destination, + (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | + (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE : MDBX_CP_DEFAULTS))); return *this; } -__cold env &env::copy(const ::std::string &destination, bool compactify, - bool force_dynamic_size) { +__cold env &env::copy(const ::std::string &destination, bool compactify, bool force_dynamic_size) { return copy(destination.c_str(), compactify, force_dynamic_size); } #if defined(_WIN32) || defined(_WIN64) -__cold env &env::copy(const wchar_t *destination, bool compactify, - bool force_dynamic_size) { - error::success_or_throw( - ::mdbx_env_copyW(handle_, destination, - (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | - (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE - : MDBX_CP_DEFAULTS))); +__cold env &env::copy(const wchar_t *destination, bool compactify, bool force_dynamic_size) { + error::success_or_throw(::mdbx_env_copyW(handle_, destination, + (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | + (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE : MDBX_CP_DEFAULTS))); return *this; } -env &env::copy(const ::std::wstring &destination, bool compactify, - bool force_dynamic_size) { +env &env::copy(const ::std::wstring &destination, bool compactify, bool force_dynamic_size) { return copy(destination.c_str(), compactify, force_dynamic_size); } #endif /* Windows */ #ifdef MDBX_STD_FILESYSTEM_PATH -__cold env &env::copy(const MDBX_STD_FILESYSTEM_PATH &destination, - bool compactify, bool force_dynamic_size) { +__cold env &env::copy(const MDBX_STD_FILESYSTEM_PATH &destination, bool compactify, bool force_dynamic_size) { return copy(destination.native(), compactify, force_dynamic_size); } #endif /* MDBX_STD_FILESYSTEM_PATH */ @@ -1375,8 +1311,7 @@ __cold path env::get_path() const { } __cold bool env::remove(const char *pathname, const remove_mode mode) { - return !error::boolean_or_throw( - ::mdbx_env_delete(pathname, MDBX_env_delete_mode_t(mode))); + return !error::boolean_or_throw(::mdbx_env_delete(pathname, MDBX_env_delete_mode_t(mode))); } __cold bool env::remove(const ::std::string &pathname, const remove_mode mode) { @@ -1385,19 +1320,16 @@ __cold bool env::remove(const ::std::string &pathname, const remove_mode mode) { #if defined(_WIN32) || defined(_WIN64) __cold bool env::remove(const wchar_t *pathname, const remove_mode mode) { - return !error::boolean_or_throw( - ::mdbx_env_deleteW(pathname, MDBX_env_delete_mode_t(mode))); + return !error::boolean_or_throw(::mdbx_env_deleteW(pathname, MDBX_env_delete_mode_t(mode))); } -__cold bool env::remove(const ::std::wstring &pathname, - const remove_mode mode) { +__cold bool env::remove(const ::std::wstring &pathname, const remove_mode mode) { return remove(pathname.c_str(), mode); } #endif /* Windows */ #ifdef MDBX_STD_FILESYSTEM_PATH -__cold bool env::remove(const MDBX_STD_FILESYSTEM_PATH &pathname, - const remove_mode mode) { +__cold bool env::remove(const MDBX_STD_FILESYSTEM_PATH &pathname, const remove_mode mode) { return remove(pathname.native(), mode); } #endif /* MDBX_STD_FILESYSTEM_PATH */ @@ -1413,13 +1345,11 @@ static inline MDBX_env *create_env() { __cold env_managed::~env_managed() noexcept { if (MDBX_UNLIKELY(handle_)) - MDBX_CXX20_UNLIKELY error::success_or_panic( - ::mdbx_env_close(handle_), "mdbx::~env()", "mdbx_env_close"); + MDBX_CXX20_UNLIKELY error::success_or_panic(::mdbx_env_close(handle_), "mdbx::~env()", "mdbx_env_close"); } __cold void env_managed::close(bool dont_sync) { - const error rc = - static_cast(::mdbx_env_close_ex(handle_, dont_sync)); + const error rc = static_cast(::mdbx_env_close_ex(handle_, dont_sync)); switch (rc.code()) { case MDBX_EBADSIGN: MDBX_CXX20_UNLIKELY handle_ = nullptr; @@ -1438,87 +1368,69 @@ __cold void env_managed::setup(unsigned max_maps, unsigned max_readers) { error::success_or_throw(::mdbx_env_set_maxdbs(handle_, max_maps)); } -__cold env_managed::env_managed(const char *pathname, - const operate_parameters &op, bool accede) +__cold env_managed::env_managed(const char *pathname, const operate_parameters &op, bool accede) : env_managed(create_env()) { setup(op.max_maps, op.max_readers); - error::success_or_throw( - ::mdbx_env_open(handle_, pathname, op.make_flags(accede), 0)); + error::success_or_throw(::mdbx_env_open(handle_, pathname, op.make_flags(accede), 0)); - if (op.options.nested_write_transactions && - !get_options().nested_write_transactions) + if (op.options.nested_write_transactions && !get_options().nested_write_transactions) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE); } -__cold env_managed::env_managed(const char *pathname, - const env_managed::create_parameters &cp, +__cold env_managed::env_managed(const char *pathname, const env_managed::create_parameters &cp, const env::operate_parameters &op, bool accede) : env_managed(create_env()) { setup(op.max_maps, op.max_readers); set_geometry(cp.geometry); - error::success_or_throw(::mdbx_env_open( - handle_, pathname, op.make_flags(accede, cp.use_subdirectory), - cp.file_mode_bits)); + error::success_or_throw( + ::mdbx_env_open(handle_, pathname, op.make_flags(accede, cp.use_subdirectory), cp.file_mode_bits)); - if (op.options.nested_write_transactions && - !get_options().nested_write_transactions) + if (op.options.nested_write_transactions && !get_options().nested_write_transactions) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE); } -__cold env_managed::env_managed(const ::std::string &pathname, - const operate_parameters &op, bool accede) +__cold env_managed::env_managed(const ::std::string &pathname, const operate_parameters &op, bool accede) : env_managed(pathname.c_str(), op, accede) {} -__cold env_managed::env_managed(const ::std::string &pathname, - const env_managed::create_parameters &cp, +__cold env_managed::env_managed(const ::std::string &pathname, const env_managed::create_parameters &cp, const env::operate_parameters &op, bool accede) : env_managed(pathname.c_str(), cp, op, accede) {} #if defined(_WIN32) || defined(_WIN64) -__cold env_managed::env_managed(const wchar_t *pathname, - const operate_parameters &op, bool accede) +__cold env_managed::env_managed(const wchar_t *pathname, const operate_parameters &op, bool accede) : env_managed(create_env()) { setup(op.max_maps, op.max_readers); - error::success_or_throw( - ::mdbx_env_openW(handle_, pathname, op.make_flags(accede), 0)); + error::success_or_throw(::mdbx_env_openW(handle_, pathname, op.make_flags(accede), 0)); - if (op.options.nested_write_transactions && - !get_options().nested_write_transactions) + if (op.options.nested_write_transactions && !get_options().nested_write_transactions) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE); } -__cold env_managed::env_managed(const wchar_t *pathname, - const env_managed::create_parameters &cp, +__cold env_managed::env_managed(const wchar_t *pathname, const env_managed::create_parameters &cp, const env::operate_parameters &op, bool accede) : env_managed(create_env()) { setup(op.max_maps, op.max_readers); set_geometry(cp.geometry); - error::success_or_throw(::mdbx_env_openW( - handle_, pathname, op.make_flags(accede, cp.use_subdirectory), - cp.file_mode_bits)); + error::success_or_throw( + ::mdbx_env_openW(handle_, pathname, op.make_flags(accede, cp.use_subdirectory), cp.file_mode_bits)); - if (op.options.nested_write_transactions && - !get_options().nested_write_transactions) + if (op.options.nested_write_transactions && !get_options().nested_write_transactions) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE); } -__cold env_managed::env_managed(const ::std::wstring &pathname, - const operate_parameters &op, bool accede) +__cold env_managed::env_managed(const ::std::wstring &pathname, const operate_parameters &op, bool accede) : env_managed(pathname.c_str(), op, accede) {} -__cold env_managed::env_managed(const ::std::wstring &pathname, - const env_managed::create_parameters &cp, +__cold env_managed::env_managed(const ::std::wstring &pathname, const env_managed::create_parameters &cp, const env::operate_parameters &op, bool accede) : env_managed(pathname.c_str(), cp, op, accede) {} #endif /* Windows */ #ifdef MDBX_STD_FILESYSTEM_PATH -__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, - const operate_parameters &op, bool accede) +__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, const operate_parameters &op, bool accede) : env_managed(pathname.native(), op, accede) {} -__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, - const env_managed::create_parameters &cp, +__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, const env_managed::create_parameters &cp, const env::operate_parameters &op, bool accede) : env_managed(pathname.native(), cp, op, accede) {} #endif /* MDBX_STD_FILESYSTEM_PATH */ @@ -1528,16 +1440,14 @@ __cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, txn_managed txn::start_nested() { MDBX_txn *nested; error::throw_on_nullptr(handle_, MDBX_BAD_TXN); - error::success_or_throw(::mdbx_txn_begin(mdbx_txn_env(handle_), handle_, - MDBX_TXN_READWRITE, &nested)); + error::success_or_throw(::mdbx_txn_begin(mdbx_txn_env(handle_), handle_, MDBX_TXN_READWRITE, &nested)); assert(nested != nullptr); return txn_managed(nested); } txn_managed::~txn_managed() noexcept { if (MDBX_UNLIKELY(handle_)) - MDBX_CXX20_UNLIKELY error::success_or_panic(::mdbx_txn_abort(handle_), - "mdbx::~txn", "mdbx_txn_abort"); + MDBX_CXX20_UNLIKELY error::success_or_panic(::mdbx_txn_abort(handle_), "mdbx::~txn", "mdbx_txn_abort"); } void txn_managed::abort() { @@ -1557,8 +1467,7 @@ void txn_managed::commit() { } void txn_managed::commit(commit_latency *latency) { - const error err = - static_cast(::mdbx_txn_commit_ex(handle_, latency)); + const error err = static_cast(::mdbx_txn_commit_ex(handle_, latency)); if (MDBX_LIKELY(err.code() != MDBX_THREAD_MISMATCH)) MDBX_CXX20_LIKELY handle_ = nullptr; if (MDBX_UNLIKELY(err.code() != MDBX_SUCCESS)) @@ -1568,8 +1477,7 @@ void txn_managed::commit(commit_latency *latency) { void txn_managed::commit_embark_read() { auto env = this->env(); commit(); - error::success_or_throw( - ::mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &handle_)); + error::success_or_throw(::mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &handle_)); } //------------------------------------------------------------------------------ @@ -1608,8 +1516,7 @@ __cold bool txn::clear_map(const char *name, bool throw_if_absent) { } } -__cold bool txn::rename_map(const char *old_name, const char *new_name, - bool throw_if_absent) { +__cold bool txn::rename_map(const char *old_name, const char *new_name, bool throw_if_absent) { map_handle map; const int err = ::mdbx_dbi_open(handle_, old_name, MDBX_DB_ACCEDE, &map.dbi); switch (err) { @@ -1660,9 +1567,7 @@ __cold bool txn::clear_map(const ::mdbx::slice &name, bool throw_if_absent) { } } -__cold bool txn::rename_map(const ::mdbx::slice &old_name, - const ::mdbx::slice &new_name, - bool throw_if_absent) { +__cold bool txn::rename_map(const ::mdbx::slice &old_name, const ::mdbx::slice &new_name, bool throw_if_absent) { map_handle map; const int err = ::mdbx_dbi_open2(handle_, old_name, MDBX_DB_ACCEDE, &map.dbi); switch (err) { @@ -1679,11 +1584,8 @@ __cold bool txn::rename_map(const ::mdbx::slice &old_name, } } -__cold bool txn::rename_map(const ::std::string &old_name, - const ::std::string &new_name, - bool throw_if_absent) { - return rename_map(::mdbx::slice(old_name), ::mdbx::slice(new_name), - throw_if_absent); +__cold bool txn::rename_map(const ::std::string &old_name, const ::std::string &new_name, bool throw_if_absent) { + return rename_map(::mdbx::slice(old_name), ::mdbx::slice(new_name), throw_if_absent); } //------------------------------------------------------------------------------ @@ -1723,12 +1625,10 @@ __cold ::std::ostream &operator<<(::std::ostream &out, const pair &it) { } __cold ::std::ostream &operator<<(::std::ostream &out, const pair_result &it) { - return out << "{" << (it.done ? "done: " : "non-done: ") << it.key << " => " - << it.value << "}"; + return out << "{" << (it.done ? "done: " : "non-done: ") << it.key << " => " << it.value << "}"; } -__cold ::std::ostream &operator<<(::std::ostream &out, - const ::mdbx::env::geometry::size &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const ::mdbx::env::geometry::size &it) { switch (it.bytes) { case ::mdbx::env::geometry::default_value: return out << "default"; @@ -1738,8 +1638,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, return out << "maximal"; } - const auto bytes = (it.bytes < 0) ? out << "-", - size_t(-it.bytes) : size_t(it.bytes); + const auto bytes = (it.bytes < 0) ? out << "-", size_t(-it.bytes) : size_t(it.bytes); struct { size_t one; const char *suffix; @@ -1769,8 +1668,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, return out; } -__cold ::std::ostream &operator<<(::std::ostream &out, - const env::geometry &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const env::geometry &it) { return // out << "\tlower " << env::geometry::size(it.size_lower) // << ",\n\tnow " << env::geometry::size(it.size_now) // @@ -1780,8 +1678,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, << ",\n\tpagesize " << env::geometry::size(it.pagesize) << "\n"; } -__cold ::std::ostream &operator<<(::std::ostream &out, - const env::operate_parameters &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const env::operate_parameters &it) { return out << "{\n" // << "\tmax_maps " << it.max_maps // << ",\n\tmax_readers " << it.max_readers // @@ -1805,8 +1702,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, const env::mode &it) { } } -__cold ::std::ostream &operator<<(::std::ostream &out, - const env::durability &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const env::durability &it) { switch (it) { case env::durability::robust_synchronous: return out << "robust_synchronous"; @@ -1821,16 +1717,14 @@ __cold ::std::ostream &operator<<(::std::ostream &out, } } -__cold ::std::ostream &operator<<(::std::ostream &out, - const env::reclaiming_options &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const env::reclaiming_options &it) { return out << "{" // << "lifo: " << (it.lifo ? "yes" : "no") // << ", coalesce: " << (it.coalesce ? "yes" : "no") // << "}"; } -__cold ::std::ostream &operator<<(::std::ostream &out, - const env::operate_options &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const env::operate_options &it) { static const char comma[] = ", "; const char *delimiter = ""; out << "{"; @@ -1859,8 +1753,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, return out << "}"; } -__cold ::std::ostream &operator<<(::std::ostream &out, - const env_managed::create_parameters &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const env_managed::create_parameters &it) { return out << "{\n" // << "\tfile_mode " << std::oct << it.file_mode_bits << std::dec // << ",\n\tsubdirectory " << (it.use_subdirectory ? "yes" : "no") // @@ -1868,8 +1761,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, << it.geometry << "}"; } -__cold ::std::ostream &operator<<(::std::ostream &out, - const MDBX_log_level_t &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const MDBX_log_level_t &it) { switch (it) { case MDBX_LOG_FATAL: return out << "LOG_FATAL"; @@ -1894,8 +1786,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, } } -__cold ::std::ostream &operator<<(::std::ostream &out, - const MDBX_debug_flags_t &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const MDBX_debug_flags_t &it) { if (it == MDBX_DBG_DONTCHANGE) return out << "DBG_DONTCHANGE"; @@ -1931,8 +1822,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, return out << "}"; } -__cold ::std::ostream &operator<<(::std::ostream &out, - const ::mdbx::error &err) { +__cold ::std::ostream &operator<<(::std::ostream &out, const ::mdbx::error &err) { return out << err.what() << " (" << long(err.code()) << ")"; } diff --git a/src/meta.c b/src/meta.c index b45d71c1..cbdea2d0 100644 --- a/src/meta.c +++ b/src/meta.c @@ -9,15 +9,11 @@ typedef struct meta_snap { } meta_snap_t; static inline txnid_t fetch_txnid(const volatile mdbx_atomic_uint32_t *ptr) { -#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && \ - MDBX_UNALIGNED_OK >= 8 - return atomic_load64((const volatile mdbx_atomic_uint64_t *)ptr, - mo_AcquireRelease); +#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && MDBX_UNALIGNED_OK >= 8 + return atomic_load64((const volatile mdbx_atomic_uint64_t *)ptr, mo_AcquireRelease); #else - const uint32_t l = atomic_load32( - &ptr[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], mo_AcquireRelease); - const uint32_t h = atomic_load32( - &ptr[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], mo_AcquireRelease); + const uint32_t l = atomic_load32(&ptr[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], mo_AcquireRelease); + const uint32_t h = atomic_load32(&ptr[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], mo_AcquireRelease); return (uint64_t)h << 32 | l; #endif } @@ -33,9 +29,7 @@ static inline meta_snap_t meta_snap(const volatile meta_t *meta) { return r; } -txnid_t meta_txnid(const volatile meta_t *meta) { - return meta_snap(meta).txnid; -} +txnid_t meta_txnid(const volatile meta_t *meta) { return meta_snap(meta).txnid; } meta_ptr_t meta_ptr(const MDBX_env *env, unsigned n) { eASSERT(env, n < NUM_METAS); @@ -46,16 +40,13 @@ meta_ptr_t meta_ptr(const MDBX_env *env, unsigned n) { return r; } -static uint8_t meta_cmp2pack(uint8_t c01, uint8_t c02, uint8_t c12, bool s0, - bool s1, bool s2) { +static uint8_t meta_cmp2pack(uint8_t c01, uint8_t c02, uint8_t c12, bool s0, bool s1, bool s2) { assert(c01 < 3 && c02 < 3 && c12 < 3); /* assert(s0 < 2 && s1 < 2 && s2 < 2); */ - const uint8_t recent = meta_cmp2recent(c01, s0, s1) - ? (meta_cmp2recent(c02, s0, s2) ? 0 : 2) - : (meta_cmp2recent(c12, s1, s2) ? 1 : 2); - const uint8_t prefer_steady = meta_cmp2steady(c01, s0, s1) - ? (meta_cmp2steady(c02, s0, s2) ? 0 : 2) - : (meta_cmp2steady(c12, s1, s2) ? 1 : 2); + const uint8_t recent = + meta_cmp2recent(c01, s0, s1) ? (meta_cmp2recent(c02, s0, s2) ? 0 : 2) : (meta_cmp2recent(c12, s1, s2) ? 1 : 2); + const uint8_t prefer_steady = + meta_cmp2steady(c01, s0, s1) ? (meta_cmp2steady(c02, s0, s2) ? 0 : 2) : (meta_cmp2steady(c12, s1, s2) ? 1 : 2); uint8_t tail; if (recent == 0) @@ -65,10 +56,8 @@ static uint8_t meta_cmp2pack(uint8_t c01, uint8_t c02, uint8_t c12, bool s0, else tail = meta_cmp2steady(c01, s0, s1) ? 1 : 0; - const bool valid = - c01 != 1 || s0 != s1 || c02 != 1 || s0 != s2 || c12 != 1 || s1 != s2; - const bool strict = (c01 != 1 || s0 != s1) && (c02 != 1 || s0 != s2) && - (c12 != 1 || s1 != s2); + const bool valid = c01 != 1 || s0 != s1 || c02 != 1 || s0 != s2 || c12 != 1 || s1 != s2; + const bool strict = (c01 != 1 || s0 != s1) && (c02 != 1 || s0 != s2) && (c12 != 1 || s1 != s2); return tail | recent << 2 | prefer_steady << 4 | strict << 6 | valid << 7; } @@ -82,21 +71,16 @@ static inline void meta_troika_unpack(troika_t *troika, const uint8_t packed) { } static const uint8_t troika_fsm_map[2 * 2 * 2 * 3 * 3 * 3] = { - 232, 201, 216, 216, 232, 233, 232, 232, 168, 201, 216, 152, 168, 233, 232, - 168, 233, 201, 216, 201, 233, 233, 232, 233, 168, 201, 152, 216, 232, 169, - 232, 168, 168, 193, 152, 152, 168, 169, 232, 168, 169, 193, 152, 194, 233, - 169, 232, 169, 232, 201, 216, 216, 232, 201, 232, 232, 168, 193, 216, 152, - 168, 193, 232, 168, 193, 193, 210, 194, 225, 193, 225, 193, 168, 137, 212, - 214, 232, 233, 168, 168, 168, 137, 212, 150, 168, 233, 168, 168, 169, 137, - 216, 201, 233, 233, 168, 169, 168, 137, 148, 214, 232, 169, 168, 168, 40, - 129, 148, 150, 168, 169, 168, 40, 169, 129, 152, 194, 233, 169, 168, 169, - 168, 137, 214, 214, 232, 201, 168, 168, 168, 129, 214, 150, 168, 193, 168, - 168, 129, 129, 210, 194, 225, 193, 161, 129, 212, 198, 212, 214, 228, 228, - 212, 212, 148, 201, 212, 150, 164, 233, 212, 148, 233, 201, 216, 201, 233, - 233, 216, 233, 148, 198, 148, 214, 228, 164, 212, 148, 148, 194, 148, 150, - 164, 169, 212, 148, 169, 194, 152, 194, 233, 169, 216, 169, 214, 198, 214, - 214, 228, 198, 212, 214, 150, 194, 214, 150, 164, 193, 212, 150, 194, 194, - 210, 194, 225, 193, 210, 194}; + 232, 201, 216, 216, 232, 233, 232, 232, 168, 201, 216, 152, 168, 233, 232, 168, 233, 201, 216, 201, 233, 233, + 232, 233, 168, 201, 152, 216, 232, 169, 232, 168, 168, 193, 152, 152, 168, 169, 232, 168, 169, 193, 152, 194, + 233, 169, 232, 169, 232, 201, 216, 216, 232, 201, 232, 232, 168, 193, 216, 152, 168, 193, 232, 168, 193, 193, + 210, 194, 225, 193, 225, 193, 168, 137, 212, 214, 232, 233, 168, 168, 168, 137, 212, 150, 168, 233, 168, 168, + 169, 137, 216, 201, 233, 233, 168, 169, 168, 137, 148, 214, 232, 169, 168, 168, 40, 129, 148, 150, 168, 169, + 168, 40, 169, 129, 152, 194, 233, 169, 168, 169, 168, 137, 214, 214, 232, 201, 168, 168, 168, 129, 214, 150, + 168, 193, 168, 168, 129, 129, 210, 194, 225, 193, 161, 129, 212, 198, 212, 214, 228, 228, 212, 212, 148, 201, + 212, 150, 164, 233, 212, 148, 233, 201, 216, 201, 233, 233, 216, 233, 148, 198, 148, 214, 228, 164, 212, 148, + 148, 194, 148, 150, 164, 169, 212, 148, 169, 194, 152, 194, 233, 169, 216, 169, 214, 198, 214, 214, 228, 198, + 212, 214, 150, 194, 214, 150, 164, 193, 212, 150, 194, 194, 210, 194, 225, 193, 210, 194}; __cold bool troika_verify_fsm(void) { bool ok = true; @@ -117,12 +101,10 @@ __cold bool troika_verify_fsm(void) { const bool strict = TROIKA_STRICT_VALID(&troika); const bool valid = TROIKA_VALID(&troika); - const uint8_t recent_chk = meta_cmp2recent(c01, s0, s1) - ? (meta_cmp2recent(c02, s0, s2) ? 0 : 2) - : (meta_cmp2recent(c12, s1, s2) ? 1 : 2); + const uint8_t recent_chk = + meta_cmp2recent(c01, s0, s1) ? (meta_cmp2recent(c02, s0, s2) ? 0 : 2) : (meta_cmp2recent(c12, s1, s2) ? 1 : 2); const uint8_t prefer_steady_chk = - meta_cmp2steady(c01, s0, s1) ? (meta_cmp2steady(c02, s0, s2) ? 0 : 2) - : (meta_cmp2steady(c12, s1, s2) ? 1 : 2); + meta_cmp2steady(c01, s0, s1) ? (meta_cmp2steady(c02, s0, s2) ? 0 : 2) : (meta_cmp2steady(c12, s1, s2) ? 1 : 2); uint8_t tail_chk; if (recent_chk == 0) @@ -132,20 +114,16 @@ __cold bool troika_verify_fsm(void) { else tail_chk = meta_cmp2steady(c01, s0, s1) ? 1 : 0; - const bool valid_chk = - c01 != 1 || s0 != s1 || c02 != 1 || s0 != s2 || c12 != 1 || s1 != s2; - const bool strict_chk = (c01 != 1 || s0 != s1) && (c02 != 1 || s0 != s2) && - (c12 != 1 || s1 != s2); + const bool valid_chk = c01 != 1 || s0 != s1 || c02 != 1 || s0 != s2 || c12 != 1 || s1 != s2; + const bool strict_chk = (c01 != 1 || s0 != s1) && (c02 != 1 || s0 != s2) && (c12 != 1 || s1 != s2); assert(troika.recent == recent_chk); assert(troika.prefer_steady == prefer_steady_chk); assert(tail == tail_chk); assert(valid == valid_chk); assert(strict == strict_chk); assert(troika_fsm_map[troika.fsm] == packed); - if (troika.recent != recent_chk || - troika.prefer_steady != prefer_steady_chk || tail != tail_chk || - valid != valid_chk || strict != strict_chk || - troika_fsm_map[troika.fsm] != packed) { + if (troika.recent != recent_chk || troika.prefer_steady != prefer_steady_chk || tail != tail_chk || + valid != valid_chk || strict != strict_chk || troika_fsm_map[troika.fsm] != packed) { ok = false; } } @@ -181,27 +159,24 @@ txnid_t recent_committed_txnid(const MDBX_env *env) { static inline bool meta_eq(const troika_t *troika, size_t a, size_t b) { assert(a < NUM_METAS && b < NUM_METAS); - return troika->txnid[a] == troika->txnid[b] && - (((troika->fsm >> a) ^ (troika->fsm >> b)) & 1) == 0 && + return troika->txnid[a] == troika->txnid[b] && (((troika->fsm >> a) ^ (troika->fsm >> b)) & 1) == 0 && troika->txnid[a]; } unsigned meta_eq_mask(const troika_t *troika) { - return meta_eq(troika, 0, 1) | meta_eq(troika, 1, 2) << 1 | - meta_eq(troika, 2, 0) << 2; + return meta_eq(troika, 0, 1) | meta_eq(troika, 1, 2) << 1 | meta_eq(troika, 2, 0) << 2; } __hot bool meta_should_retry(const MDBX_env *env, troika_t *troika) { const troika_t prev = *troika; *troika = meta_tap(env); - return prev.fsm != troika->fsm || prev.txnid[0] != troika->txnid[0] || - prev.txnid[1] != troika->txnid[1] || prev.txnid[2] != troika->txnid[2]; + return prev.fsm != troika->fsm || prev.txnid[0] != troika->txnid[0] || prev.txnid[1] != troika->txnid[1] || + prev.txnid[2] != troika->txnid[2]; } const char *durable_caption(const meta_t *const meta) { if (meta_is_steady(meta)) - return (meta_sign_get(meta) == meta_sign_calculate(meta)) ? "Steady" - : "Tainted"; + return (meta_sign_get(meta) == meta_sign_calculate(meta)) ? "Steady" : "Tainted"; return "Weak"; } @@ -214,20 +189,16 @@ __cold void meta_troika_dump(const MDBX_env *env, const troika_t *troika) { "base=%d-%" PRIaTXN ".%c, " "tail=%d-%" PRIaTXN ".%c, " "valid %c, strict %c", - troika->txnid[0], (troika->fsm & 1) ? 's' : 'w', troika->txnid[1], - (troika->fsm & 2) ? 's' : 'w', troika->txnid[2], - (troika->fsm & 4) ? 's' : 'w', troika->fsm, troika->recent, - recent.txnid, recent.is_steady ? 's' : 'w', troika->prefer_steady, - prefer_steady.txnid, prefer_steady.is_steady ? 's' : 'w', - troika->tail_and_flags % NUM_METAS, tail.txnid, - tail.is_steady ? 's' : 'w', TROIKA_VALID(troika) ? 'Y' : 'N', + troika->txnid[0], (troika->fsm & 1) ? 's' : 'w', troika->txnid[1], (troika->fsm & 2) ? 's' : 'w', + troika->txnid[2], (troika->fsm & 4) ? 's' : 'w', troika->fsm, troika->recent, recent.txnid, + recent.is_steady ? 's' : 'w', troika->prefer_steady, prefer_steady.txnid, prefer_steady.is_steady ? 's' : 'w', + troika->tail_and_flags % NUM_METAS, tail.txnid, tail.is_steady ? 's' : 'w', TROIKA_VALID(troika) ? 'Y' : 'N', TROIKA_STRICT_VALID(troika) ? 'Y' : 'N'); } /*----------------------------------------------------------------------------*/ -static int meta_unsteady(MDBX_env *env, const txnid_t inclusive_upto, - const pgno_t pgno) { +static int meta_unsteady(MDBX_env *env, const txnid_t inclusive_upto, const pgno_t pgno) { meta_t *const meta = METAPAGE(env, pgno); const txnid_t txnid = constmeta_txnid(meta); if (!meta_is_steady(meta) || txnid > inclusive_upto) @@ -236,8 +207,7 @@ static int meta_unsteady(MDBX_env *env, const txnid_t inclusive_upto, WARNING("wipe txn #%" PRIaTXN ", meta %" PRIaPGNO, txnid, pgno); const uint64_t wipe = DATASIGN_NONE; const void *ptr = &wipe; - size_t bytes = sizeof(meta->sign), - offset = ptr_dist(&meta->sign, env->dxb_mmap.base); + size_t bytes = sizeof(meta->sign), offset = ptr_dist(&meta->sign, env->dxb_mmap.base); if (env->flags & MDBX_WRITEMAP) { unaligned_poke_u64(4, meta->sign, wipe); osal_flush_incoherent_cpu_writeback(); @@ -265,8 +235,7 @@ __cold int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto) { if (err == MDBX_RESULT_TRUE) { err = MDBX_SUCCESS; if (!MDBX_AVOID_MSYNC && (env->flags & MDBX_WRITEMAP)) { - err = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), - MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + err = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), MDBX_SYNC_DATA | MDBX_SYNC_IODQ); #if MDBX_ENABLE_PGOP_STAT env->lck->pgops.msync.weak += 1; #endif /* MDBX_ENABLE_PGOP_STAT */ @@ -278,8 +247,7 @@ __cold int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto) { } } - osal_flush_incoherent_mmap(env->dxb_mmap.base, pgno2bytes(env, NUM_METAS), - globals.sys_pagesize); + osal_flush_incoherent_mmap(env->dxb_mmap.base, pgno2bytes(env, NUM_METAS), globals.sys_pagesize); /* force oldest refresh */ atomic_store32(&env->lck->rdt_refresh_flag, true, mo_Relaxed); @@ -291,8 +259,7 @@ __cold int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto) { } int meta_sync(const MDBX_env *env, const meta_ptr_t head) { - eASSERT(env, atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed) != - (uint32_t)head.txnid); + eASSERT(env, atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed) != (uint32_t)head.txnid); /* Функция может вызываться (в том числе) при (env->flags & * MDBX_NOMETASYNC) == 0 и env->fd4meta == env->dsync_fd, например если * предыдущая транзакция была выполненна с флагом MDBX_NOMETASYNC. */ @@ -300,8 +267,7 @@ int meta_sync(const MDBX_env *env, const meta_ptr_t head) { int rc = MDBX_RESULT_TRUE; if (env->flags & MDBX_WRITEMAP) { if (!MDBX_AVOID_MSYNC) { - rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), - MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), MDBX_SYNC_DATA | MDBX_SYNC_IODQ); #if MDBX_ENABLE_PGOP_STAT env->lck->pgops.msync.weak += 1; #endif /* MDBX_ENABLE_PGOP_STAT */ @@ -310,8 +276,7 @@ int meta_sync(const MDBX_env *env, const meta_ptr_t head) { env->lck->pgops.wops.weak += 1; #endif /* MDBX_ENABLE_PGOP_STAT */ const page_t *page = data_page(head.ptr_c); - rc = osal_pwrite(env->fd4meta, page, env->ps, - ptr_dist(page, env->dxb_mmap.base)); + rc = osal_pwrite(env->fd4meta, page, env->ps, ptr_dist(page, env->dxb_mmap.base)); if (likely(rc == MDBX_SUCCESS) && env->fd4meta == env->lazy_fd) { rc = osal_fsync(env->lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); @@ -332,8 +297,7 @@ int meta_sync(const MDBX_env *env, const meta_ptr_t head) { return rc; } -__cold static page_t *meta_model(const MDBX_env *env, page_t *model, size_t num, - const bin128_t *guid) { +__cold static page_t *meta_model(const MDBX_env *env, page_t *model, size_t num, const bin128_t *guid) { ENSURE(env, is_powerof2(env->ps)); ENSURE(env, env->ps >= MDBX_MIN_PAGESIZE); ENSURE(env, env->ps <= MDBX_MAX_PAGESIZE); @@ -350,10 +314,8 @@ __cold static page_t *meta_model(const MDBX_env *env, page_t *model, size_t num, model_meta->geometry.lower = bytes2pgno(env, env->geo_in_bytes.lower); model_meta->geometry.upper = bytes2pgno(env, env->geo_in_bytes.upper); - model_meta->geometry.grow_pv = - pages2pv(bytes2pgno(env, env->geo_in_bytes.grow)); - model_meta->geometry.shrink_pv = - pages2pv(bytes2pgno(env, env->geo_in_bytes.shrink)); + model_meta->geometry.grow_pv = pages2pv(bytes2pgno(env, env->geo_in_bytes.grow)); + model_meta->geometry.shrink_pv = pages2pv(bytes2pgno(env, env->geo_in_bytes.shrink)); model_meta->geometry.now = bytes2pgno(env, env->geo_in_bytes.now); model_meta->geometry.first_unallocated = NUM_METAS; @@ -362,12 +324,9 @@ __cold static page_t *meta_model(const MDBX_env *env, page_t *model, size_t num, ENSURE(env, model_meta->geometry.now >= model_meta->geometry.lower); ENSURE(env, model_meta->geometry.now <= model_meta->geometry.upper); ENSURE(env, model_meta->geometry.first_unallocated >= MIN_PAGENO); - ENSURE(env, - model_meta->geometry.first_unallocated <= model_meta->geometry.now); - ENSURE(env, model_meta->geometry.grow_pv == - pages2pv(pv2pages(model_meta->geometry.grow_pv))); - ENSURE(env, model_meta->geometry.shrink_pv == - pages2pv(pv2pages(model_meta->geometry.shrink_pv))); + ENSURE(env, model_meta->geometry.first_unallocated <= model_meta->geometry.now); + ENSURE(env, model_meta->geometry.grow_pv == pages2pv(pv2pages(model_meta->geometry.grow_pv))); + ENSURE(env, model_meta->geometry.shrink_pv == pages2pv(pv2pages(model_meta->geometry.shrink_pv))); model_meta->pagesize = env->ps; model_meta->trees.gc.flags = MDBX_INTEGERKEY; @@ -389,12 +348,9 @@ __cold meta_t *meta_init_triplet(const MDBX_env *env, void *buffer) { return page_meta(page2); } -__cold int __must_check_result meta_override(MDBX_env *env, size_t target, - txnid_t txnid, - const meta_t *shape) { +__cold int __must_check_result meta_override(MDBX_env *env, size_t target, txnid_t txnid, const meta_t *shape) { page_t *const page = env->page_auxbuf; - meta_model(env, page, target, - &((target == 0 && shape) ? shape : METAPAGE(env, 0))->dxbid); + meta_model(env, page, target, &((target == 0 && shape) ? shape : METAPAGE(env, 0))->dxbid); meta_t *const model = page_meta(page); meta_set_txnid(env, model, txnid); if (txnid) @@ -407,21 +363,18 @@ __cold int __must_check_result meta_override(MDBX_env *env, size_t target, return MDBX_PROBLEM; } if (globals.runtime_flags & MDBX_DBG_DONT_UPGRADE) - memcpy(&model->magic_and_version, &shape->magic_and_version, - sizeof(model->magic_and_version)); + memcpy(&model->magic_and_version, &shape->magic_and_version, sizeof(model->magic_and_version)); model->reserve16 = shape->reserve16; model->validator_id = shape->validator_id; model->extra_pagehdr = shape->extra_pagehdr; memcpy(&model->geometry, &shape->geometry, sizeof(model->geometry)); memcpy(&model->trees, &shape->trees, sizeof(model->trees)); memcpy(&model->canary, &shape->canary, sizeof(model->canary)); - memcpy(&model->pages_retired, &shape->pages_retired, - sizeof(model->pages_retired)); + memcpy(&model->pages_retired, &shape->pages_retired, sizeof(model->pages_retired)); if (txnid) { if ((!model->trees.gc.mod_txnid && model->trees.gc.root != P_INVALID) || (!model->trees.main.mod_txnid && model->trees.main.root != P_INVALID)) - memcpy(&model->magic_and_version, &shape->magic_and_version, - sizeof(model->magic_and_version)); + memcpy(&model->magic_and_version, &shape->magic_and_version, sizeof(model->magic_and_version)); if (unlikely(!coherency_check_meta(env, model, false))) { ERROR("bailout overriding meta-%zu since model failed " "FreeDB/MainDB %s-check for txnid #%" PRIaTXN, @@ -452,8 +405,7 @@ __cold int __must_check_result meta_override(MDBX_env *env, size_t target, #if MDBX_ENABLE_PGOP_STAT env->lck->pgops.msync.weak += 1; #endif /* MDBX_ENABLE_PGOP_STAT */ - rc = osal_msync(&env->dxb_mmap, 0, - pgno_align2os_bytes(env, model->geometry.first_unallocated), + rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, model->geometry.first_unallocated), MDBX_SYNC_DATA | MDBX_SYNC_IODQ); if (unlikely(rc != MDBX_SUCCESS)) return rc; @@ -465,8 +417,7 @@ __cold int __must_check_result meta_override(MDBX_env *env, size_t target, #if MDBX_ENABLE_PGOP_STAT env->lck->pgops.msync.weak += 1; #endif /* MDBX_ENABLE_PGOP_STAT */ - rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, target + 1), - MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, target + 1), MDBX_SYNC_DATA | MDBX_SYNC_IODQ); } else { #if MDBX_ENABLE_PGOP_STAT env->lck->pgops.wops.weak += 1; @@ -478,28 +429,20 @@ __cold int __must_check_result meta_override(MDBX_env *env, size_t target, #endif /* MDBX_ENABLE_PGOP_STAT */ rc = osal_fsync(env->lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); } - osal_flush_incoherent_mmap(env->dxb_mmap.base, pgno2bytes(env, NUM_METAS), - globals.sys_pagesize); + osal_flush_incoherent_mmap(env->dxb_mmap.base, pgno2bytes(env, NUM_METAS), globals.sys_pagesize); } eASSERT(env, (!env->txn && (env->flags & ENV_ACTIVE) == 0) || - (env->stuck_meta == (int)target && - (env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == - MDBX_EXCLUSIVE)); + (env->stuck_meta == (int)target && (env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == MDBX_EXCLUSIVE)); return rc; } -__cold int meta_validate(MDBX_env *env, meta_t *const meta, - const page_t *const page, const unsigned meta_number, +__cold int meta_validate(MDBX_env *env, meta_t *const meta, const page_t *const page, const unsigned meta_number, unsigned *guess_pagesize) { - const uint64_t magic_and_version = - unaligned_peek_u64(4, &meta->magic_and_version); - if (unlikely(magic_and_version != MDBX_DATA_MAGIC && - magic_and_version != MDBX_DATA_MAGIC_LEGACY_COMPAT && + const uint64_t magic_and_version = unaligned_peek_u64(4, &meta->magic_and_version); + if (unlikely(magic_and_version != MDBX_DATA_MAGIC && magic_and_version != MDBX_DATA_MAGIC_LEGACY_COMPAT && magic_and_version != MDBX_DATA_MAGIC_LEGACY_DEVEL)) { - ERROR("meta[%u] has invalid magic/version %" PRIx64, meta_number, - magic_and_version); - return ((magic_and_version >> 8) != MDBX_MAGIC) ? MDBX_INVALID - : MDBX_VERSION_MISMATCH; + ERROR("meta[%u] has invalid magic/version %" PRIx64, meta_number, magic_and_version); + return ((magic_and_version >> 8) != MDBX_MAGIC) ? MDBX_INVALID : MDBX_VERSION_MISMATCH; } if (unlikely(page->pgno != meta_number)) { @@ -512,11 +455,9 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta, return MDBX_INVALID; } - if (unlikely(!is_powerof2(meta->pagesize) || - meta->pagesize < MDBX_MIN_PAGESIZE || + if (unlikely(!is_powerof2(meta->pagesize) || meta->pagesize < MDBX_MIN_PAGESIZE || meta->pagesize > MDBX_MAX_PAGESIZE)) { - WARNING("meta[%u] has invalid pagesize (%u), skip it", meta_number, - meta->pagesize); + WARNING("meta[%u] has invalid pagesize (%u), skip it", meta_number, meta->pagesize); return is_powerof2(meta->pagesize) ? MDBX_VERSION_MISMATCH : MDBX_INVALID; } @@ -535,81 +476,63 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta, const uint64_t sign = meta_sign_get(meta); const uint64_t sign_stready = meta_sign_calculate(meta); if (SIGN_IS_STEADY(sign) && unlikely(sign != sign_stready)) { - WARNING("meta[%u] has invalid steady-checksum (0x%" PRIx64 " != 0x%" PRIx64 - "), skip it", - meta_number, sign, sign_stready); + WARNING("meta[%u] has invalid steady-checksum (0x%" PRIx64 " != 0x%" PRIx64 "), skip it", meta_number, sign, + sign_stready); return MDBX_RESULT_TRUE; } if (unlikely(meta->trees.gc.flags != MDBX_INTEGERKEY) && - ((meta->trees.gc.flags & DB_PERSISTENT_FLAGS) != MDBX_INTEGERKEY || - magic_and_version == MDBX_DATA_MAGIC)) { - WARNING("meta[%u] has invalid %s flags 0x%x, skip it", meta_number, - "GC/FreeDB", meta->trees.gc.flags); + ((meta->trees.gc.flags & DB_PERSISTENT_FLAGS) != MDBX_INTEGERKEY || magic_and_version == MDBX_DATA_MAGIC)) { + WARNING("meta[%u] has invalid %s flags 0x%x, skip it", meta_number, "GC/FreeDB", meta->trees.gc.flags); return MDBX_INCOMPATIBLE; } if (unlikely(!check_table_flags(meta->trees.main.flags))) { - WARNING("meta[%u] has invalid %s flags 0x%x, skip it", meta_number, - "MainDB", meta->trees.main.flags); + WARNING("meta[%u] has invalid %s flags 0x%x, skip it", meta_number, "MainDB", meta->trees.main.flags); return MDBX_INCOMPATIBLE; } - DEBUG("checking meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO - ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO - " +%u -%u, txn_id %" PRIaTXN ", %s", - page->pgno, meta->trees.main.root, meta->trees.gc.root, - meta->geometry.lower, meta->geometry.first_unallocated, - meta->geometry.now, meta->geometry.upper, - pv2pages(meta->geometry.grow_pv), pv2pages(meta->geometry.shrink_pv), + DEBUG("checking meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO + "/%" PRIaPGNO " +%u -%u, txn_id %" PRIaTXN ", %s", + page->pgno, meta->trees.main.root, meta->trees.gc.root, meta->geometry.lower, meta->geometry.first_unallocated, + meta->geometry.now, meta->geometry.upper, pv2pages(meta->geometry.grow_pv), pv2pages(meta->geometry.shrink_pv), txnid, durable_caption(meta)); if (unlikely(txnid < MIN_TXNID || txnid > MAX_TXNID)) { - WARNING("meta[%u] has invalid txnid %" PRIaTXN ", skip it", meta_number, - txnid); + WARNING("meta[%u] has invalid txnid %" PRIaTXN ", skip it", meta_number, txnid); return MDBX_RESULT_TRUE; } - if (unlikely(meta->geometry.lower < MIN_PAGENO || - meta->geometry.lower > MAX_PAGENO + 1)) { - WARNING("meta[%u] has invalid min-pages (%" PRIaPGNO "), skip it", - meta_number, meta->geometry.lower); + if (unlikely(meta->geometry.lower < MIN_PAGENO || meta->geometry.lower > MAX_PAGENO + 1)) { + WARNING("meta[%u] has invalid min-pages (%" PRIaPGNO "), skip it", meta_number, meta->geometry.lower); return MDBX_INVALID; } - if (unlikely(meta->geometry.upper < MIN_PAGENO || - meta->geometry.upper > MAX_PAGENO + 1 || + if (unlikely(meta->geometry.upper < MIN_PAGENO || meta->geometry.upper > MAX_PAGENO + 1 || meta->geometry.upper < meta->geometry.lower)) { - WARNING("meta[%u] has invalid max-pages (%" PRIaPGNO "), skip it", - meta_number, meta->geometry.upper); + WARNING("meta[%u] has invalid max-pages (%" PRIaPGNO "), skip it", meta_number, meta->geometry.upper); return MDBX_INVALID; } - if (unlikely(meta->geometry.first_unallocated < MIN_PAGENO || - meta->geometry.first_unallocated - 1 > MAX_PAGENO)) { - WARNING("meta[%u] has invalid next-pageno (%" PRIaPGNO "), skip it", - meta_number, meta->geometry.first_unallocated); + if (unlikely(meta->geometry.first_unallocated < MIN_PAGENO || meta->geometry.first_unallocated - 1 > MAX_PAGENO)) { + WARNING("meta[%u] has invalid next-pageno (%" PRIaPGNO "), skip it", meta_number, meta->geometry.first_unallocated); return MDBX_CORRUPTED; } - const uint64_t used_bytes = - meta->geometry.first_unallocated * (uint64_t)meta->pagesize; + const uint64_t used_bytes = meta->geometry.first_unallocated * (uint64_t)meta->pagesize; if (unlikely(used_bytes > env->dxb_mmap.filesize)) { /* Here could be a race with DB-shrinking performed by other process */ int err = osal_filesize(env->lazy_fd, &env->dxb_mmap.filesize); if (unlikely(err != MDBX_SUCCESS)) return err; if (unlikely(used_bytes > env->dxb_mmap.filesize)) { - WARNING("meta[%u] used-bytes (%" PRIu64 ") beyond filesize (%" PRIu64 - "), skip it", - meta_number, used_bytes, env->dxb_mmap.filesize); + WARNING("meta[%u] used-bytes (%" PRIu64 ") beyond filesize (%" PRIu64 "), skip it", meta_number, used_bytes, + env->dxb_mmap.filesize); return MDBX_CORRUPTED; } } - if (unlikely(meta->geometry.first_unallocated - 1 > MAX_PAGENO || - used_bytes > MAX_MAPSIZE)) { - WARNING("meta[%u] has too large used-space (%" PRIu64 "), skip it", - meta_number, used_bytes); + if (unlikely(meta->geometry.first_unallocated - 1 > MAX_PAGENO || used_bytes > MAX_MAPSIZE)) { + WARNING("meta[%u] has too large used-space (%" PRIu64 "), skip it", meta_number, used_bytes); return MDBX_TOO_LARGE; } @@ -617,13 +540,10 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta, uint64_t mapsize_min = geo_lower * (uint64_t)meta->pagesize; STATIC_ASSERT(MAX_MAPSIZE < PTRDIFF_MAX - MDBX_MAX_PAGESIZE); STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE); - STATIC_ASSERT((uint64_t)(MAX_PAGENO + 1) * MDBX_MIN_PAGESIZE % (4ul << 20) == - 0); + STATIC_ASSERT((uint64_t)(MAX_PAGENO + 1) * MDBX_MIN_PAGESIZE % (4ul << 20) == 0); if (unlikely(mapsize_min < MIN_MAPSIZE || mapsize_min > MAX_MAPSIZE)) { - if (MAX_MAPSIZE != MAX_MAPSIZE64 && mapsize_min > MAX_MAPSIZE && - mapsize_min <= MAX_MAPSIZE64) { - eASSERT(env, meta->geometry.first_unallocated - 1 <= MAX_PAGENO && - used_bytes <= MAX_MAPSIZE); + if (MAX_MAPSIZE != MAX_MAPSIZE64 && mapsize_min > MAX_MAPSIZE && mapsize_min <= MAX_MAPSIZE64) { + eASSERT(env, meta->geometry.first_unallocated - 1 <= MAX_PAGENO && used_bytes <= MAX_MAPSIZE); WARNING("meta[%u] has too large min-mapsize (%" PRIu64 "), " "but size of used space still acceptable (%" PRIu64 ")", meta_number, mapsize_min, used_bytes); @@ -632,14 +552,12 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta, geo_lower = MAX_PAGENO + 1; mapsize_min = geo_lower * (uint64_t)meta->pagesize; } - WARNING("meta[%u] consider get-%s pageno is %" PRIaPGNO - " instead of wrong %" PRIaPGNO + WARNING("meta[%u] consider get-%s pageno is %" PRIaPGNO " instead of wrong %" PRIaPGNO ", will be corrected on next commit(s)", meta_number, "lower", geo_lower, meta->geometry.lower); meta->geometry.lower = geo_lower; } else { - WARNING("meta[%u] has invalid min-mapsize (%" PRIu64 "), skip it", - meta_number, mapsize_min); + WARNING("meta[%u] has invalid min-mapsize (%" PRIu64 "), skip it", meta_number, mapsize_min); return MDBX_VERSION_MISMATCH; } } @@ -648,17 +566,13 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta, uint64_t mapsize_max = geo_upper * (uint64_t)meta->pagesize; STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE); if (unlikely(mapsize_max > MAX_MAPSIZE || - (MAX_PAGENO + 1) < - ceil_powerof2((size_t)mapsize_max, globals.sys_pagesize) / - (size_t)meta->pagesize)) { + (MAX_PAGENO + 1) < ceil_powerof2((size_t)mapsize_max, globals.sys_pagesize) / (size_t)meta->pagesize)) { if (mapsize_max > MAX_MAPSIZE64) { - WARNING("meta[%u] has invalid max-mapsize (%" PRIu64 "), skip it", - meta_number, mapsize_max); + WARNING("meta[%u] has invalid max-mapsize (%" PRIu64 "), skip it", meta_number, mapsize_max); return MDBX_VERSION_MISMATCH; } /* allow to open large DB from a 32-bit environment */ - eASSERT(env, meta->geometry.first_unallocated - 1 <= MAX_PAGENO && - used_bytes <= MAX_MAPSIZE); + eASSERT(env, meta->geometry.first_unallocated - 1 <= MAX_PAGENO && used_bytes <= MAX_MAPSIZE); WARNING("meta[%u] has too large max-mapsize (%" PRIu64 "), " "but size of used space still acceptable (%" PRIu64 ")", meta_number, mapsize_max, used_bytes); @@ -667,8 +581,7 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta, geo_upper = MAX_PAGENO + 1; mapsize_max = geo_upper * (uint64_t)meta->pagesize; } - WARNING("meta[%u] consider get-%s pageno is %" PRIaPGNO - " instead of wrong %" PRIaPGNO + WARNING("meta[%u] consider get-%s pageno is %" PRIaPGNO " instead of wrong %" PRIaPGNO ", will be corrected on next commit(s)", meta_number, "upper", geo_upper, meta->geometry.upper); meta->geometry.upper = geo_upper; @@ -688,14 +601,12 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta, geo_now = geo_upper; if (unlikely(meta->geometry.first_unallocated > geo_now)) { - WARNING("meta[%u] next-pageno (%" PRIaPGNO - ") is beyond end-pgno (%" PRIaPGNO "), skip it", - meta_number, meta->geometry.first_unallocated, geo_now); + WARNING("meta[%u] next-pageno (%" PRIaPGNO ") is beyond end-pgno (%" PRIaPGNO "), skip it", meta_number, + meta->geometry.first_unallocated, geo_now); return MDBX_CORRUPTED; } if (meta->geometry.now != geo_now) { - WARNING("meta[%u] consider geo-%s pageno is %" PRIaPGNO - " instead of wrong %" PRIaPGNO + WARNING("meta[%u] consider geo-%s pageno is %" PRIaPGNO " instead of wrong %" PRIaPGNO ", will be corrected on next commit(s)", meta_number, "now", geo_now, meta->geometry.now); meta->geometry.now = geo_now; @@ -703,43 +614,36 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta, /* GC */ if (meta->trees.gc.root == P_INVALID) { - if (unlikely(meta->trees.gc.branch_pages || meta->trees.gc.height || - meta->trees.gc.items || meta->trees.gc.leaf_pages || - meta->trees.gc.large_pages)) { + if (unlikely(meta->trees.gc.branch_pages || meta->trees.gc.height || meta->trees.gc.items || + meta->trees.gc.leaf_pages || meta->trees.gc.large_pages)) { WARNING("meta[%u] has false-empty %s, skip it", meta_number, "GC"); return MDBX_CORRUPTED; } - } else if (unlikely(meta->trees.gc.root >= - meta->geometry.first_unallocated)) { - WARNING("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it", meta_number, - "GC", meta->trees.gc.root); + } else if (unlikely(meta->trees.gc.root >= meta->geometry.first_unallocated)) { + WARNING("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it", meta_number, "GC", meta->trees.gc.root); return MDBX_CORRUPTED; } /* MainDB */ if (meta->trees.main.root == P_INVALID) { - if (unlikely(meta->trees.main.branch_pages || meta->trees.main.height || - meta->trees.main.items || meta->trees.main.leaf_pages || - meta->trees.main.large_pages)) { + if (unlikely(meta->trees.main.branch_pages || meta->trees.main.height || meta->trees.main.items || + meta->trees.main.leaf_pages || meta->trees.main.large_pages)) { WARNING("meta[%u] has false-empty %s", meta_number, "MainDB"); return MDBX_CORRUPTED; } - } else if (unlikely(meta->trees.main.root >= - meta->geometry.first_unallocated)) { - WARNING("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it", meta_number, - "MainDB", meta->trees.main.root); + } else if (unlikely(meta->trees.main.root >= meta->geometry.first_unallocated)) { + WARNING("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it", meta_number, "MainDB", meta->trees.main.root); return MDBX_CORRUPTED; } if (unlikely(meta->trees.gc.mod_txnid > txnid)) { - WARNING("meta[%u] has wrong mod_txnid %" PRIaTXN " for %s, skip it", - meta_number, meta->trees.gc.mod_txnid, "GC"); + WARNING("meta[%u] has wrong mod_txnid %" PRIaTXN " for %s, skip it", meta_number, meta->trees.gc.mod_txnid, "GC"); return MDBX_CORRUPTED; } if (unlikely(meta->trees.main.mod_txnid > txnid)) { - WARNING("meta[%u] has wrong mod_txnid %" PRIaTXN " for %s, skip it", - meta_number, meta->trees.main.mod_txnid, "MainDB"); + WARNING("meta[%u] has wrong mod_txnid %" PRIaTXN " for %s, skip it", meta_number, meta->trees.main.mod_txnid, + "MainDB"); return MDBX_CORRUPTED; } @@ -748,7 +652,5 @@ __cold int meta_validate(MDBX_env *env, meta_t *const meta, __cold int meta_validate_copy(MDBX_env *env, const meta_t *meta, meta_t *dest) { *dest = *meta; - return meta_validate(env, dest, data_page(meta), - bytes2pgno(env, ptr_dist(meta, env->dxb_mmap.base)), - nullptr); + return meta_validate(env, dest, data_page(meta), bytes2pgno(env, ptr_dist(meta, env->dxb_mmap.base)), nullptr); } diff --git a/src/meta.h b/src/meta.h index 706061c1..899f1d5a 100644 --- a/src/meta.h +++ b/src/meta.h @@ -16,17 +16,11 @@ static inline uint64_t meta_sign_calculate(const meta_t *meta) { return (sign > DATASIGN_WEAK) ? sign : ~sign; } -static inline uint64_t meta_sign_get(const volatile meta_t *meta) { - return unaligned_peek_u64_volatile(4, meta->sign); -} +static inline uint64_t meta_sign_get(const volatile meta_t *meta) { return unaligned_peek_u64_volatile(4, meta->sign); } -static inline void meta_sign_as_steady(meta_t *meta) { - unaligned_poke_u64(4, meta->sign, meta_sign_calculate(meta)); -} +static inline void meta_sign_as_steady(meta_t *meta) { unaligned_poke_u64(4, meta->sign, meta_sign_calculate(meta)); } -static inline bool meta_is_steady(const volatile meta_t *meta) { - return SIGN_IS_STEADY(meta_sign_get(meta)); -} +static inline bool meta_is_steady(const volatile meta_t *meta) { return SIGN_IS_STEADY(meta_sign_get(meta)); } MDBX_INTERNAL troika_t meta_tap(const MDBX_env *env); MDBX_INTERNAL unsigned meta_eq_mask(const troika_t *troika); @@ -48,14 +42,12 @@ MDBX_INTERNAL txnid_t recent_committed_txnid(const MDBX_env *env); MDBX_INTERNAL int meta_sync(const MDBX_env *env, const meta_ptr_t head); MDBX_INTERNAL const char *durable_caption(const meta_t *const meta); -MDBX_INTERNAL void meta_troika_dump(const MDBX_env *env, - const troika_t *troika); +MDBX_INTERNAL void meta_troika_dump(const MDBX_env *env, const troika_t *troika); #define METAPAGE(env, n) page_meta(pgno2page(env, n)) #define METAPAGE_END(env) METAPAGE(env, NUM_METAS) -static inline meta_ptr_t meta_recent(const MDBX_env *env, - const troika_t *troika) { +static inline meta_ptr_t meta_recent(const MDBX_env *env, const troika_t *troika) { meta_ptr_t r; r.txnid = troika->txnid[troika->recent]; r.ptr_v = METAPAGE(env, troika->recent); @@ -63,8 +55,7 @@ static inline meta_ptr_t meta_recent(const MDBX_env *env, return r; } -static inline meta_ptr_t meta_prefer_steady(const MDBX_env *env, - const troika_t *troika) { +static inline meta_ptr_t meta_prefer_steady(const MDBX_env *env, const troika_t *troika) { meta_ptr_t r; r.txnid = troika->txnid[troika->prefer_steady]; r.ptr_v = METAPAGE(env, troika->prefer_steady); @@ -72,8 +63,7 @@ static inline meta_ptr_t meta_prefer_steady(const MDBX_env *env, return r; } -static inline meta_ptr_t meta_tail(const MDBX_env *env, - const troika_t *troika) { +static inline meta_ptr_t meta_tail(const MDBX_env *env, const troika_t *troika) { const uint8_t tail = troika->tail_and_flags & 3; MDBX_ANALYSIS_ASSUME(tail < NUM_METAS); meta_ptr_t r; @@ -89,72 +79,53 @@ static inline bool meta_is_used(const troika_t *troika, unsigned n) { static inline bool meta_bootid_match(const meta_t *meta) { - return memcmp(&meta->bootid, &globals.bootid, 16) == 0 && - (globals.bootid.x | globals.bootid.y) != 0; + return memcmp(&meta->bootid, &globals.bootid, 16) == 0 && (globals.bootid.x | globals.bootid.y) != 0; } -static inline bool meta_weak_acceptable(const MDBX_env *env, const meta_t *meta, - const int lck_exclusive) { +static inline bool meta_weak_acceptable(const MDBX_env *env, const meta_t *meta, const int lck_exclusive) { return lck_exclusive ? /* exclusive lock */ meta_bootid_match(meta) - : /* db already opened */ env->lck_mmap.lck && - (env->lck_mmap.lck->envmode.weak & MDBX_RDONLY) == 0; + : /* db already opened */ env->lck_mmap.lck && (env->lck_mmap.lck->envmode.weak & MDBX_RDONLY) == 0; } -MDBX_NOTHROW_PURE_FUNCTION static inline txnid_t -constmeta_txnid(const meta_t *meta) { +MDBX_NOTHROW_PURE_FUNCTION static inline txnid_t constmeta_txnid(const meta_t *meta) { const txnid_t a = unaligned_peek_u64(4, &meta->txnid_a); const txnid_t b = unaligned_peek_u64(4, &meta->txnid_b); return likely(a == b) ? a : 0; } -static inline void meta_update_begin(const MDBX_env *env, meta_t *meta, - txnid_t txnid) { +static inline void meta_update_begin(const MDBX_env *env, meta_t *meta, txnid_t txnid) { eASSERT(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env)); - eASSERT(env, unaligned_peek_u64(4, meta->txnid_a) < txnid && - unaligned_peek_u64(4, meta->txnid_b) < txnid); + eASSERT(env, unaligned_peek_u64(4, meta->txnid_a) < txnid && unaligned_peek_u64(4, meta->txnid_b) < txnid); (void)env; -#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && \ - MDBX_UNALIGNED_OK >= 8 +#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && MDBX_UNALIGNED_OK >= 8 atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_b, 0, mo_AcquireRelease); - atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_a, txnid, - mo_AcquireRelease); + atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_a, txnid, mo_AcquireRelease); #else - atomic_store32(&meta->txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], 0, - mo_AcquireRelease); - atomic_store32(&meta->txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], 0, - mo_AcquireRelease); - atomic_store32(&meta->txnid_a[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], - (uint32_t)txnid, mo_AcquireRelease); - atomic_store32(&meta->txnid_a[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], - (uint32_t)(txnid >> 32), mo_AcquireRelease); + atomic_store32(&meta->txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], 0, mo_AcquireRelease); + atomic_store32(&meta->txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], 0, mo_AcquireRelease); + atomic_store32(&meta->txnid_a[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], (uint32_t)txnid, mo_AcquireRelease); + atomic_store32(&meta->txnid_a[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], (uint32_t)(txnid >> 32), mo_AcquireRelease); #endif } -static inline void meta_update_end(const MDBX_env *env, meta_t *meta, - txnid_t txnid) { +static inline void meta_update_end(const MDBX_env *env, meta_t *meta, txnid_t txnid) { eASSERT(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env)); eASSERT(env, unaligned_peek_u64(4, meta->txnid_a) == txnid); eASSERT(env, unaligned_peek_u64(4, meta->txnid_b) < txnid); (void)env; jitter4testing(true); memcpy(&meta->bootid, &globals.bootid, 16); -#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && \ - MDBX_UNALIGNED_OK >= 8 - atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_b, txnid, - mo_AcquireRelease); +#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && MDBX_UNALIGNED_OK >= 8 + atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_b, txnid, mo_AcquireRelease); #else - atomic_store32(&meta->txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], - (uint32_t)txnid, mo_AcquireRelease); - atomic_store32(&meta->txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], - (uint32_t)(txnid >> 32), mo_AcquireRelease); + atomic_store32(&meta->txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], (uint32_t)txnid, mo_AcquireRelease); + atomic_store32(&meta->txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], (uint32_t)(txnid >> 32), mo_AcquireRelease); #endif } -static inline void meta_set_txnid(const MDBX_env *env, meta_t *meta, - const txnid_t txnid) { - eASSERT(env, !env->dxb_mmap.base || meta < METAPAGE(env, 0) || - meta >= METAPAGE_END(env)); +static inline void meta_set_txnid(const MDBX_env *env, meta_t *meta, const txnid_t txnid) { + eASSERT(env, !env->dxb_mmap.base || meta < METAPAGE(env, 0) || meta >= METAPAGE_END(env)); (void)env; /* update inconsistently since this function used ONLY for filling meta-image * for writing, but not the actual meta-page */ @@ -167,42 +138,31 @@ static inline uint8_t meta_cmp2int(txnid_t a, txnid_t b, uint8_t s) { return unlikely(a == b) ? 1 * s : (a > b) ? 2 * s : 0 * s; } -static inline uint8_t meta_cmp2recent(uint8_t ab_cmp2int, bool a_steady, - bool b_steady) { +static inline uint8_t meta_cmp2recent(uint8_t ab_cmp2int, bool a_steady, bool b_steady) { assert(ab_cmp2int < 3 /* && a_steady< 2 && b_steady < 2 */); return ab_cmp2int > 1 || (ab_cmp2int == 1 && a_steady > b_steady); } -static inline uint8_t meta_cmp2steady(uint8_t ab_cmp2int, bool a_steady, - bool b_steady) { +static inline uint8_t meta_cmp2steady(uint8_t ab_cmp2int, bool a_steady, bool b_steady) { assert(ab_cmp2int < 3 /* && a_steady< 2 && b_steady < 2 */); return a_steady > b_steady || (a_steady == b_steady && ab_cmp2int > 1); } -static inline bool meta_choice_recent(txnid_t a_txnid, bool a_steady, - txnid_t b_txnid, bool b_steady) { +static inline bool meta_choice_recent(txnid_t a_txnid, bool a_steady, txnid_t b_txnid, bool b_steady) { return meta_cmp2recent(meta_cmp2int(a_txnid, b_txnid, 1), a_steady, b_steady); } -static inline bool meta_choice_steady(txnid_t a_txnid, bool a_steady, - txnid_t b_txnid, bool b_steady) { +static inline bool meta_choice_steady(txnid_t a_txnid, bool a_steady, txnid_t b_txnid, bool b_steady) { return meta_cmp2steady(meta_cmp2int(a_txnid, b_txnid, 1), a_steady, b_steady); } MDBX_INTERNAL meta_t *meta_init_triplet(const MDBX_env *env, void *buffer); -MDBX_INTERNAL int meta_validate(MDBX_env *env, meta_t *const meta, - const page_t *const page, - const unsigned meta_number, +MDBX_INTERNAL int meta_validate(MDBX_env *env, meta_t *const meta, const page_t *const page, const unsigned meta_number, unsigned *guess_pagesize); -MDBX_INTERNAL int __must_check_result meta_validate_copy(MDBX_env *env, - const meta_t *meta, - meta_t *dest); +MDBX_INTERNAL int __must_check_result meta_validate_copy(MDBX_env *env, const meta_t *meta, meta_t *dest); -MDBX_INTERNAL int __must_check_result meta_override(MDBX_env *env, - size_t target, - txnid_t txnid, - const meta_t *shape); +MDBX_INTERNAL int __must_check_result meta_override(MDBX_env *env, size_t target, txnid_t txnid, const meta_t *shape); MDBX_INTERNAL int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto); diff --git a/src/misc.c b/src/misc.c index f1a58014..ef3a172f 100644 --- a/src/misc.c +++ b/src/misc.c @@ -14,11 +14,9 @@ __cold int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy) { const int log2page = log2n_powerof2(pagesize); const intptr_t volume_pages = (volume + pagesize - 1) >> log2page; - const intptr_t redundancy_pages = - (redundancy < 0) ? -(intptr_t)((-redundancy + pagesize - 1) >> log2page) - : (intptr_t)(redundancy + pagesize - 1) >> log2page; - if (volume_pages >= total_ram_pages || - volume_pages + redundancy_pages >= total_ram_pages) + const intptr_t redundancy_pages = (redundancy < 0) ? -(intptr_t)((-redundancy + pagesize - 1) >> log2page) + : (intptr_t)(redundancy + pagesize - 1) >> log2page; + if (volume_pages >= total_ram_pages || volume_pages + redundancy_pages >= total_ram_pages) return MDBX_RESULT_FALSE; intptr_t avail_ram_pages; @@ -26,13 +24,10 @@ __cold int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy) { if (unlikely(err != MDBX_SUCCESS)) return LOG_IFERR(err); - return (volume_pages + redundancy_pages >= avail_ram_pages) - ? MDBX_RESULT_FALSE - : MDBX_RESULT_TRUE; + return (volume_pages + redundancy_pages >= avail_ram_pages) ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE; } -int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, - uint64_t increment) { +int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, uint64_t increment) { int rc = check_txn(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) { bailout: @@ -111,30 +106,23 @@ int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, return MDBX_SUCCESS; } -int mdbx_cmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, - const MDBX_val *b) { +int mdbx_cmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, const MDBX_val *b) { eASSERT(nullptr, txn->signature == txn_signature); tASSERT(txn, (dbi_state(txn, dbi) & DBI_VALID) && !dbi_changed(txn, dbi)); - tASSERT(txn, - dbi < txn->env->n_dbi && (txn->env->dbs_flags[dbi] & DB_VALID) != 0); + tASSERT(txn, dbi < txn->env->n_dbi && (txn->env->dbs_flags[dbi] & DB_VALID) != 0); return txn->env->kvs[dbi].clc.k.cmp(a, b); } -int mdbx_dcmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, - const MDBX_val *b) { +int mdbx_dcmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, const MDBX_val *b) { eASSERT(nullptr, txn->signature == txn_signature); tASSERT(txn, (dbi_state(txn, dbi) & DBI_VALID) && !dbi_changed(txn, dbi)); tASSERT(txn, dbi < txn->env->n_dbi && (txn->env->dbs_flags[dbi] & DB_VALID)); return txn->env->kvs[dbi].clc.v.cmp(a, b); } -__cold MDBX_cmp_func *mdbx_get_keycmp(MDBX_db_flags_t flags) { - return builtin_keycmp(flags); -} +__cold MDBX_cmp_func *mdbx_get_keycmp(MDBX_db_flags_t flags) { return builtin_keycmp(flags); } -__cold MDBX_cmp_func *mdbx_get_datacmp(MDBX_db_flags_t flags) { - return builtin_datacmp(flags); -} +__cold MDBX_cmp_func *mdbx_get_datacmp(MDBX_db_flags_t flags) { return builtin_datacmp(flags); } /*----------------------------------------------------------------------------*/ @@ -227,10 +215,8 @@ __cold const char *mdbx_strerror_r(int errnum, char *buf, size_t buflen) { const char *msg = mdbx_liberr2str(errnum); if (!msg && buflen > 0 && buflen < INT_MAX) { #if defined(_WIN32) || defined(_WIN64) - DWORD size = FormatMessageA( - FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, - errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, - nullptr); + DWORD size = FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, errnum, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, nullptr); while (size && buf[size - 1] <= ' ') --size; buf[size] = 0; @@ -284,10 +270,8 @@ __cold const char *mdbx_strerror(int errnum) { const char *mdbx_strerror_r_ANSI2OEM(int errnum, char *buf, size_t buflen) { const char *msg = mdbx_liberr2str(errnum); if (!msg && buflen > 0 && buflen < INT_MAX) { - DWORD size = FormatMessageA( - FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, - errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, - nullptr); + DWORD size = FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, errnum, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, nullptr); while (size && buf[size - 1] <= ' ') --size; buf[size] = 0; diff --git a/src/mvcc-readers.c b/src/mvcc-readers.c index f342599f..d51576da 100644 --- a/src/mvcc-readers.c +++ b/src/mvcc-readers.c @@ -45,8 +45,7 @@ bsr_t mvcc_bind_slot(MDBX_env *env) { result.err = mvcc_cleanup_dead(env, true, nullptr); if (result.err != MDBX_RESULT_TRUE) { lck_rdt_unlock(env); - result.err = - (result.err == MDBX_SUCCESS) ? MDBX_READERS_FULL : result.err; + result.err = (result.err == MDBX_SUCCESS) ? MDBX_READERS_FULL : result.err; return result; } } @@ -61,8 +60,7 @@ bsr_t mvcc_bind_slot(MDBX_env *env) { safe64_reset(&result.rslot->txnid, true); if (slot == nreaders) env->lck->rdt_length.weak = (uint32_t)++nreaders; - result.rslot->tid.weak = - (env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self(); + result.rslot->tid.weak = (env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self(); atomic_store32(&result.rslot->pid, env->pid, mo_AcquireRelease); lck_rdt_unlock(env); @@ -84,17 +82,14 @@ __hot txnid_t mvcc_shapshot_oldest(MDBX_env *const env, const txnid_t steady) { return env->lck->cached_oldest.weak = steady; } - const txnid_t prev_oldest = - atomic_load64(&lck->cached_oldest, mo_AcquireRelease); + const txnid_t prev_oldest = atomic_load64(&lck->cached_oldest, mo_AcquireRelease); eASSERT(env, steady >= prev_oldest); txnid_t new_oldest = prev_oldest; - while (nothing_changed != - atomic_load32(&lck->rdt_refresh_flag, mo_AcquireRelease)) { + while (nothing_changed != atomic_load32(&lck->rdt_refresh_flag, mo_AcquireRelease)) { lck->rdt_refresh_flag.weak = nothing_changed; jitter4testing(false); - const size_t snap_nreaders = - atomic_load32(&lck->rdt_length, mo_AcquireRelease); + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); new_oldest = steady; for (size_t i = 0; i < snap_nreaders; ++i) { @@ -105,11 +100,9 @@ __hot txnid_t mvcc_shapshot_oldest(MDBX_env *const env, const txnid_t steady) { const txnid_t rtxn = safe64_read(&lck->rdt[i].txnid); if (unlikely(rtxn < prev_oldest)) { - if (unlikely(nothing_changed == atomic_load32(&lck->rdt_refresh_flag, - mo_AcquireRelease)) && + if (unlikely(nothing_changed == atomic_load32(&lck->rdt_refresh_flag, mo_AcquireRelease)) && safe64_reset_compare(&lck->rdt[i].txnid, rtxn)) { - NOTICE("kick stuck reader[%zu of %zu].pid_%u %" PRIaTXN - " < prev-oldest %" PRIaTXN ", steady-txn %" PRIaTXN, + NOTICE("kick stuck reader[%zu of %zu].pid_%u %" PRIaTXN " < prev-oldest %" PRIaTXN ", steady-txn %" PRIaTXN, i, snap_nreaders, pid, rtxn, prev_oldest, steady); } continue; @@ -135,17 +128,13 @@ pgno_t mvcc_snapshot_largest(const MDBX_env *env, pgno_t last_used_page) { lck_t *const lck = env->lck_mmap.lck; if (likely(lck != nullptr /* check for exclusive without-lck mode */)) { retry:; - const size_t snap_nreaders = - atomic_load32(&lck->rdt_length, mo_AcquireRelease); + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); for (size_t i = 0; i < snap_nreaders; ++i) { if (atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease)) { /* jitter4testing(true); */ - const pgno_t snap_pages = - atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_Relaxed); + const pgno_t snap_pages = atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_Relaxed); const txnid_t snap_txnid = safe64_read(&lck->rdt[i].txnid); - if (unlikely(snap_pages != - atomic_load32(&lck->rdt[i].snapshot_pages_used, - mo_AcquireRelease) || + if (unlikely(snap_pages != atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_AcquireRelease) || snap_txnid != safe64_read(&lck->rdt[i].txnid))) goto retry; if (last_used_page < snap_pages && snap_txnid <= env->basal_txn->txnid) @@ -161,18 +150,14 @@ pgno_t mvcc_snapshot_largest(const MDBX_env *env, pgno_t last_used_page) { pgno_t mvcc_largest_this(MDBX_env *env, pgno_t largest) { lck_t *const lck = env->lck_mmap.lck; if (likely(lck != nullptr /* exclusive mode */)) { - const size_t snap_nreaders = - atomic_load32(&lck->rdt_length, mo_AcquireRelease); + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); for (size_t i = 0; i < snap_nreaders; ++i) { retry: if (atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease) == env->pid) { /* jitter4testing(true); */ - const pgno_t snap_pages = - atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_Relaxed); + const pgno_t snap_pages = atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_Relaxed); const txnid_t snap_txnid = safe64_read(&lck->rdt[i].txnid); - if (unlikely(snap_pages != - atomic_load32(&lck->rdt[i].snapshot_pages_used, - mo_AcquireRelease) || + if (unlikely(snap_pages != atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_AcquireRelease) || snap_txnid != safe64_read(&lck->rdt[i].txnid))) goto retry; if (largest < snap_pages && @@ -219,8 +204,7 @@ static bool pid_insert(uint32_t *list, uint32_t pid) { return true; } -__cold MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rdt_locked, - int *dead) { +__cold MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rdt_locked, int *dead) { int rc = check_env(env, true); if (unlikely(rc != MDBX_SUCCESS)) return rc; @@ -234,13 +218,11 @@ __cold MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rdt_locked, return MDBX_SUCCESS; } - const size_t snap_nreaders = - atomic_load32(&lck->rdt_length, mo_AcquireRelease); + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); uint32_t pidsbuf_onstask[142]; - uint32_t *const pids = - (snap_nreaders < ARRAY_LENGTH(pidsbuf_onstask)) - ? pidsbuf_onstask - : osal_malloc((snap_nreaders + 1) * sizeof(uint32_t)); + uint32_t *const pids = (snap_nreaders < ARRAY_LENGTH(pidsbuf_onstask)) + ? pidsbuf_onstask + : osal_malloc((snap_nreaders + 1) * sizeof(uint32_t)); if (unlikely(!pids)) return MDBX_ENOMEM; @@ -296,8 +278,7 @@ __cold MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rdt_locked, /* clean it */ for (size_t ii = i; ii < snap_nreaders; ii++) { if (lck->rdt[ii].pid.weak == pid) { - DEBUG("clear stale reader pid %" PRIuPTR " txn %" PRIaTXN, (size_t)pid, - lck->rdt[ii].txnid.weak); + DEBUG("clear stale reader pid %" PRIuPTR " txn %" PRIaTXN, (size_t)pid, lck->rdt[ii].txnid.weak); atomic_store32(&lck->rdt[ii].pid, 0, mo_Relaxed); atomic_store32(&lck->rdt_refresh_flag, true, mo_AcquireRelease); count++; @@ -321,11 +302,9 @@ __cold MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rdt_locked, int txn_park(MDBX_txn *txn, bool autounpark) { reader_slot_t *const rslot = txn->to.reader; - tASSERT(txn, (txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY | - MDBX_TXN_PARKED)) == MDBX_TXN_RDONLY); + tASSERT(txn, (txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) == MDBX_TXN_RDONLY); tASSERT(txn, txn->to.reader->tid.weak < MDBX_TID_TXN_OUSTED); - if (unlikely((txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY | - MDBX_TXN_PARKED)) != MDBX_TXN_RDONLY)) + if (unlikely((txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) != MDBX_TXN_RDONLY)) return MDBX_BAD_TXN; const uint32_t pid = atomic_load32(&rslot->pid, mo_Relaxed); @@ -344,14 +323,12 @@ int txn_park(MDBX_txn *txn, bool autounpark) { atomic_store64(&rslot->tid, MDBX_TID_TXN_PARKED, mo_AcquireRelease); atomic_store32(&txn->env->lck->rdt_refresh_flag, true, mo_Relaxed); - txn->flags += - autounpark ? MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK : MDBX_TXN_PARKED; + txn->flags += autounpark ? MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK : MDBX_TXN_PARKED; return MDBX_SUCCESS; } int txn_unpark(MDBX_txn *txn) { - if (unlikely((txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD | - MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) != + if (unlikely((txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD | MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) != (MDBX_TXN_RDONLY | MDBX_TXN_PARKED))) return MDBX_BAD_TXN; @@ -363,14 +340,11 @@ int txn_unpark(MDBX_txn *txn) { ERROR("unexpected pid %u%s%u", pid, " != expected ", txn->env->pid); return MDBX_PROBLEM; } - if (unlikely(tid == MDBX_TID_TXN_OUSTED || - txnid >= SAFE64_INVALID_THRESHOLD)) + if (unlikely(tid == MDBX_TID_TXN_OUSTED || txnid >= SAFE64_INVALID_THRESHOLD)) break; if (unlikely(tid != MDBX_TID_TXN_PARKED || txnid != txn->txnid)) { - ERROR("unexpected thread-id 0x%" PRIx64 "%s0x%" PRIx64 - " and/or txn-id %" PRIaTXN "%s%" PRIaTXN, - tid, " != must ", MDBX_TID_TXN_OUSTED, txnid, " != must ", - txn->txnid); + ERROR("unexpected thread-id 0x%" PRIx64 "%s0x%" PRIx64 " and/or txn-id %" PRIaTXN "%s%" PRIaTXN, tid, " != must ", + MDBX_TID_TXN_OUSTED, txnid, " != must ", txn->txnid); break; } if (unlikely((txn->flags & MDBX_TXN_ERROR))) @@ -380,12 +354,9 @@ int txn_unpark(MDBX_txn *txn) { if (unlikely(!atomic_cas64(&rslot->tid, MDBX_TID_TXN_PARKED, txn->owner))) continue; #else - atomic_store32(&rslot->tid.high, (uint32_t)((uint64_t)txn->owner >> 32), - mo_Relaxed); - if (unlikely(!atomic_cas32(&rslot->tid.low, (uint32_t)MDBX_TID_TXN_PARKED, - (uint32_t)txn->owner))) { - atomic_store32(&rslot->tid.high, (uint32_t)(MDBX_TID_TXN_PARKED >> 32), - mo_AcquireRelease); + atomic_store32(&rslot->tid.high, (uint32_t)((uint64_t)txn->owner >> 32), mo_Relaxed); + if (unlikely(!atomic_cas32(&rslot->tid.low, (uint32_t)MDBX_TID_TXN_PARKED, (uint32_t)txn->owner))) { + atomic_store32(&rslot->tid.high, (uint32_t)(MDBX_TID_TXN_PARKED >> 32), mo_AcquireRelease); continue; } #endif @@ -413,8 +384,7 @@ __cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) { bool notify_eof_of_loop = false; int retry = 0; do { - const txnid_t steady = - env->txn->tw.troika.txnid[env->txn->tw.troika.prefer_steady]; + const txnid_t steady = env->txn->tw.troika.txnid[env->txn->tw.troika.prefer_steady]; env->lck->rdt_refresh_flag.weak = /* force refresh */ true; oldest = mvcc_shapshot_oldest(env, steady); eASSERT(env, oldest < env->basal_txn->txnid); @@ -435,8 +405,7 @@ __cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) { reader_slot_t *const rslot = &lck->rdt[i]; txnid_t rtxn = safe64_read(&rslot->txnid); retry: - if (rtxn == straggler && - (pid = atomic_load32(&rslot->pid, mo_AcquireRelease)) != 0) { + if (rtxn == straggler && (pid = atomic_load32(&rslot->pid, mo_AcquireRelease)) != 0) { const uint64_t tid = safe64_read(&rslot->tid); if (tid == MDBX_TID_TXN_PARKED) { /* Читающая транзакция была помечена владельцем как "припаркованная", @@ -454,25 +423,21 @@ __cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) { */ bool ousted = #if MDBX_64BIT_CAS - atomic_cas64(&rslot->tid, MDBX_TID_TXN_PARKED, - MDBX_TID_TXN_OUSTED); + atomic_cas64(&rslot->tid, MDBX_TID_TXN_PARKED, MDBX_TID_TXN_OUSTED); #else - atomic_cas32(&rslot->tid.low, (uint32_t)MDBX_TID_TXN_PARKED, - (uint32_t)MDBX_TID_TXN_OUSTED); + atomic_cas32(&rslot->tid.low, (uint32_t)MDBX_TID_TXN_PARKED, (uint32_t)MDBX_TID_TXN_OUSTED); #endif if (likely(ousted)) { ousted = safe64_reset_compare(&rslot->txnid, rtxn); - NOTICE("ousted-%s parked read-txn %" PRIaTXN - ", pid %u, tid 0x%" PRIx64, - ousted ? "complete" : "half", rtxn, pid, tid); + NOTICE("ousted-%s parked read-txn %" PRIaTXN ", pid %u, tid 0x%" PRIx64, ousted ? "complete" : "half", rtxn, + pid, tid); eASSERT(env, ousted || safe64_read(&rslot->txnid) > straggler); continue; } rtxn = safe64_read(&rslot->txnid); goto retry; } - hold_retired = - atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_Relaxed); + hold_retired = atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_Relaxed); stucked = rslot; } } @@ -487,15 +452,10 @@ __cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) { const meta_ptr_t head = meta_recent(env, &env->txn->tw.troika); const txnid_t gap = (head.txnid - straggler) / xMDBX_TXNID_STEP; - const uint64_t head_retired = - unaligned_peek_u64(4, head.ptr_c->pages_retired); - const size_t space = - (head_retired > hold_retired) - ? pgno2bytes(env, (pgno_t)(head_retired - hold_retired)) - : 0; - int rc = - callback(env, env->txn, pid, (mdbx_tid_t)((intptr_t)tid), straggler, - (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, space, retry); + const uint64_t head_retired = unaligned_peek_u64(4, head.ptr_c->pages_retired); + const size_t space = (head_retired > hold_retired) ? pgno2bytes(env, (pgno_t)(head_retired - hold_retired)) : 0; + int rc = callback(env, env->txn, pid, (mdbx_tid_t)((intptr_t)tid), straggler, + (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, space, retry); if (rc < 0) /* hsr returned error and/or agree MDBX_MAP_FULL error */ break; @@ -523,10 +483,8 @@ __cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) { /* notify end of hsr-loop */ const txnid_t turn = oldest - straggler; if (turn) - NOTICE("hsr-kick: done turn %" PRIaTXN " -> %" PRIaTXN " +%" PRIaTXN, - straggler, oldest, turn); - callback(env, env->txn, 0, 0, straggler, - (turn < UINT_MAX) ? (unsigned)turn : UINT_MAX, 0, -retry); + NOTICE("hsr-kick: done turn %" PRIaTXN " -> %" PRIaTXN " +%" PRIaTXN, straggler, oldest, turn); + callback(env, env->txn, 0, 0, straggler, (turn < UINT_MAX) ? (unsigned)turn : UINT_MAX, 0, -retry); } return oldest; } diff --git a/src/node.c b/src/node.c index c8588fd0..934bad65 100644 --- a/src/node.c +++ b/src/node.c @@ -5,15 +5,13 @@ #include "internals.h" -__hot int __must_check_result node_add_dupfix(MDBX_cursor *mc, size_t indx, - const MDBX_val *key) { +__hot int __must_check_result node_add_dupfix(MDBX_cursor *mc, size_t indx, const MDBX_val *key) { page_t *mp = mc->pg[mc->top]; MDBX_ANALYSIS_ASSUME(key != nullptr); DKBUF_DEBUG; DEBUG("add to leaf2-%spage %" PRIaPGNO " index %zi, " " key size %" PRIuPTR " [%s]", - is_subpage(mp) ? "sub-" : "", mp->pgno, indx, key ? key->iov_len : 0, - DKEY_DEBUG(key)); + is_subpage(mp) ? "sub-" : "", mp->pgno, indx, key ? key->iov_len : 0, DKEY_DEBUG(key)); cASSERT(mc, key); cASSERT(mc, page_type_compat(mp) == (P_LEAF | P_DUPFIX)); @@ -45,14 +43,11 @@ __hot int __must_check_result node_add_dupfix(MDBX_cursor *mc, size_t indx, return MDBX_SUCCESS; } -int __must_check_result node_add_branch(MDBX_cursor *mc, size_t indx, - const MDBX_val *key, pgno_t pgno) { +int __must_check_result node_add_branch(MDBX_cursor *mc, size_t indx, const MDBX_val *key, pgno_t pgno) { page_t *mp = mc->pg[mc->top]; DKBUF_DEBUG; - DEBUG("add to branch-%spage %" PRIaPGNO " index %zi, node-pgno %" PRIaPGNO - " key size %" PRIuPTR " [%s]", - is_subpage(mp) ? "sub-" : "", mp->pgno, indx, pgno, - key ? key->iov_len : 0, DKEY_DEBUG(key)); + DEBUG("add to branch-%spage %" PRIaPGNO " index %zi, node-pgno %" PRIaPGNO " key size %" PRIuPTR " [%s]", + is_subpage(mp) ? "sub-" : "", mp->pgno, indx, pgno, key ? key->iov_len : 0, DKEY_DEBUG(key)); cASSERT(mc, page_type(mp) == P_BRANCH); STATIC_ASSERT(NODESIZE % 2 == 0); @@ -87,17 +82,15 @@ int __must_check_result node_add_branch(MDBX_cursor *mc, size_t indx, return MDBX_SUCCESS; } -__hot int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx, - const MDBX_val *key, MDBX_val *data, +__hot int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx, const MDBX_val *key, MDBX_val *data, unsigned flags) { MDBX_ANALYSIS_ASSUME(key != nullptr); MDBX_ANALYSIS_ASSUME(data != nullptr); page_t *mp = mc->pg[mc->top]; DKBUF_DEBUG; - DEBUG("add to leaf-%spage %" PRIaPGNO " index %zi, data size %" PRIuPTR - " key size %" PRIuPTR " [%s]", - is_subpage(mp) ? "sub-" : "", mp->pgno, indx, data ? data->iov_len : 0, - key ? key->iov_len : 0, DKEY_DEBUG(key)); + DEBUG("add to leaf-%spage %" PRIaPGNO " index %zi, data size %" PRIuPTR " key size %" PRIuPTR " [%s]", + is_subpage(mp) ? "sub-" : "", mp->pgno, indx, data ? data->iov_len : 0, key ? key->iov_len : 0, + DKEY_DEBUG(key)); cASSERT(mc, key != nullptr && data != nullptr); cASSERT(mc, page_type_compat(mp) == P_LEAF); page_t *largepage = nullptr; @@ -106,19 +99,16 @@ __hot int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx, if (unlikely(flags & N_BIG)) { /* Data already on large/overflow page. */ STATIC_ASSERT(sizeof(pgno_t) % 2 == 0); - node_bytes = - node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t); + node_bytes = node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t); cASSERT(mc, page_room(mp) >= node_bytes); } else if (unlikely(node_size(key, data) > mc->txn->env->leaf_nodemax)) { /* Put data on large/overflow page. */ if (unlikely(mc->tree->flags & MDBX_DUPSORT)) { - ERROR("Unexpected target %s flags 0x%x for large data-item", "dupsort-db", - mc->tree->flags); + ERROR("Unexpected target %s flags 0x%x for large data-item", "dupsort-db", mc->tree->flags); return MDBX_PROBLEM; } if (unlikely(flags & (N_DUP | N_TREE))) { - ERROR("Unexpected target %s flags 0x%x for large data-item", "node", - flags); + ERROR("Unexpected target %s flags 0x%x for large data-item", "node", flags); return MDBX_PROBLEM; } cASSERT(mc, page_room(mp) >= leaf_size(mc->txn->env, key, data)); @@ -127,12 +117,10 @@ __hot int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx, if (unlikely(npr.err != MDBX_SUCCESS)) return npr.err; largepage = npr.page; - DEBUG("allocated %u large/overflow page(s) %" PRIaPGNO "for %" PRIuPTR - " data bytes", - largepage->pages, largepage->pgno, data->iov_len); + DEBUG("allocated %u large/overflow page(s) %" PRIaPGNO "for %" PRIuPTR " data bytes", largepage->pages, + largepage->pgno, data->iov_len); flags |= N_BIG; - node_bytes = - node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t); + node_bytes = node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t); cASSERT(mc, node_bytes == leaf_size(mc->txn->env, key, data)); } else { cASSERT(mc, page_room(mp) >= leaf_size(mc->txn->env, key, data)); @@ -186,8 +174,7 @@ __hot void node_del(MDBX_cursor *mc, size_t ksize) { const size_t hole = mc->ki[mc->top]; const size_t nkeys = page_numkeys(mp); - DEBUG("delete node %zu on %s page %" PRIaPGNO, hole, - is_leaf(mp) ? "leaf" : "branch", mp->pgno); + DEBUG("delete node %zu on %s page %" PRIaPGNO, hole, is_leaf(mp) ? "leaf" : "branch", mp->pgno); cASSERT(mc, hole < nkeys); if (is_dupfix_leaf(mp)) { @@ -215,9 +202,7 @@ __hot void node_del(MDBX_cursor *mc, size_t ksize) { size_t r, w; for (r = w = 0; r < nkeys; r++) if (r != hole) - mp->entries[w++] = (mp->entries[r] < hole_offset) - ? mp->entries[r] + (indx_t)hole_size - : mp->entries[r]; + mp->entries[w++] = (mp->entries[r] < hole_offset) ? mp->entries[r] + (indx_t)hole_size : mp->entries[r]; void *const base = ptr_disp(mp, mp->upper + PAGEHDRSZ); memmove(ptr_disp(base, hole_size), base, hole_offset - mp->upper); @@ -236,14 +221,12 @@ __hot void node_del(MDBX_cursor *mc, size_t ksize) { } } -__noinline int node_read_bigdata(MDBX_cursor *mc, const node_t *node, - MDBX_val *data, const page_t *mp) { +__noinline int node_read_bigdata(MDBX_cursor *mc, const node_t *node, MDBX_val *data, const page_t *mp) { cASSERT(mc, node_flags(node) == N_BIG && data->iov_len == node_ds(node)); pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->txnid); if (unlikely((lp.err != MDBX_SUCCESS))) { - DEBUG("read large/overflow page %" PRIaPGNO " failed", - node_largedata_pgno(node)); + DEBUG("read large/overflow page %" PRIaPGNO " failed", node_largedata_pgno(node)); return lp.err; } @@ -254,9 +237,7 @@ __noinline int node_read_bigdata(MDBX_cursor *mc, const node_t *node, const size_t dsize = data->iov_len; const unsigned npages = largechunk_npages(env, dsize); if (unlikely(lp.page->pages < npages)) - return bad_page(lp.page, - "too less n-pages %u for bigdata-node (%zu bytes)", - lp.page->pages, dsize); + return bad_page(lp.page, "too less n-pages %u for bigdata-node (%zu bytes)", lp.page->pages, dsize); } return MDBX_SUCCESS; } @@ -265,8 +246,7 @@ node_t *node_shrink(page_t *mp, size_t indx, node_t *node) { assert(node == page_node(mp, indx)); page_t *sp = (page_t *)node_data(node); assert(is_subpage(sp) && page_numkeys(sp) > 0); - const size_t delta = - EVEN_FLOOR(page_room(sp) /* avoid the node uneven-sized */); + const size_t delta = EVEN_FLOOR(page_room(sp) /* avoid the node uneven-sized */); if (unlikely(delta) == 0) return node; @@ -303,15 +283,13 @@ node_t *node_shrink(page_t *mp, size_t indx, node_t *node) { return ptr_disp(node, delta); } -__hot struct node_search_result node_search(MDBX_cursor *mc, - const MDBX_val *key) { +__hot struct node_search_result node_search(MDBX_cursor *mc, const MDBX_val *key) { page_t *mp = mc->pg[mc->top]; const intptr_t nkeys = page_numkeys(mp); DKBUF_DEBUG; - DEBUG("searching %zu keys in %s %spage %" PRIaPGNO, nkeys, - is_leaf(mp) ? "leaf" : "branch", is_subpage(mp) ? "sub-" : "", - mp->pgno); + DEBUG("searching %zu keys in %s %spage %" PRIaPGNO, nkeys, is_leaf(mp) ? "leaf" : "branch", + is_subpage(mp) ? "sub-" : "", mp->pgno); struct node_search_result ret; ret.exact = false; @@ -333,8 +311,7 @@ __hot struct node_search_result node_search(MDBX_cursor *mc, do { i = (low + high) >> 1; nodekey.iov_base = page_dupfix_ptr(mp, i, nodekey.iov_len); - cASSERT(mc, ptr_disp(mp, mc->txn->env->ps) >= - ptr_disp(nodekey.iov_base, nodekey.iov_len)); + cASSERT(mc, ptr_disp(mp, mc->txn->env->ps) >= ptr_disp(nodekey.iov_base, nodekey.iov_len)); int cr = cmp(key, &nodekey); DEBUG("found leaf index %zu [%s], rc = %i", i, DKEY_DEBUG(&nodekey), cr); if (cr > 0) @@ -349,10 +326,8 @@ __hot struct node_search_result node_search(MDBX_cursor *mc, /* store the key index */ mc->ki[mc->top] = (indx_t)i; - ret.node = - (i < nkeys) - ? /* fake for DUPFIX */ (node_t *)(intptr_t)-1 - : /* There is no entry larger or equal to the key. */ nullptr; + ret.node = (i < nkeys) ? /* fake for DUPFIX */ (node_t *)(intptr_t)-1 + : /* There is no entry larger or equal to the key. */ nullptr; return ret; } @@ -367,14 +342,12 @@ __hot struct node_search_result node_search(MDBX_cursor *mc, node = page_node(mp, i); nodekey.iov_len = node_ks(node); nodekey.iov_base = node_key(node); - cASSERT(mc, ptr_disp(mp, mc->txn->env->ps) >= - ptr_disp(nodekey.iov_base, nodekey.iov_len)); + cASSERT(mc, ptr_disp(mp, mc->txn->env->ps) >= ptr_disp(nodekey.iov_base, nodekey.iov_len)); int cr = cmp(key, &nodekey); if (is_leaf(mp)) DEBUG("found leaf index %zu [%s], rc = %i", i, DKEY_DEBUG(&nodekey), cr); else - DEBUG("found branch index %zu [%s -> %" PRIaPGNO "], rc = %i", i, - DKEY_DEBUG(&nodekey), node_pgno(node), cr); + DEBUG("found branch index %zu [%s -> %" PRIaPGNO "], rc = %i", i, DKEY_DEBUG(&nodekey), node_pgno(node), cr); if (cr > 0) low = ++i; else if (cr < 0) @@ -387,8 +360,6 @@ __hot struct node_search_result node_search(MDBX_cursor *mc, /* store the key index */ mc->ki[mc->top] = (indx_t)i; - ret.node = (i < nkeys) - ? page_node(mp, i) - : /* There is no entry larger or equal to the key. */ nullptr; + ret.node = (i < nkeys) ? page_node(mp, i) : /* There is no entry larger or equal to the key. */ nullptr; return ret; } diff --git a/src/node.h b/src/node.h index 5411aeed..50de16e1 100644 --- a/src/node.h +++ b/src/node.h @@ -9,8 +9,7 @@ #define NODE_ADD_FLAGS (N_DUP | N_TREE | MDBX_RESERVE | MDBX_APPEND) /* Get the page number pointed to by a branch node */ -MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t -node_pgno(const node_t *const __restrict node) { +MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t node_pgno(const node_t *const __restrict node) { pgno_t pgno = UNALIGNED_PEEK_32(node, node_t, child_pgno); return pgno; } @@ -23,8 +22,7 @@ static inline void node_set_pgno(node_t *const __restrict node, pgno_t pgno) { } /* Get the size of the data in a leaf node */ -MDBX_NOTHROW_PURE_FUNCTION static inline size_t -node_ds(const node_t *const __restrict node) { +MDBX_NOTHROW_PURE_FUNCTION static inline size_t node_ds(const node_t *const __restrict node) { return UNALIGNED_PEEK_32(node, node_t, dsize); } @@ -35,8 +33,7 @@ static inline void node_set_ds(node_t *const __restrict node, size_t size) { } /* The size of a key in a node */ -MDBX_NOTHROW_PURE_FUNCTION static inline size_t -node_ks(const node_t *const __restrict node) { +MDBX_NOTHROW_PURE_FUNCTION static inline size_t node_ks(const node_t *const __restrict node) { return UNALIGNED_PEEK_16(node, node_t, ksize); } @@ -46,54 +43,42 @@ static inline void node_set_ks(node_t *const __restrict node, size_t size) { UNALIGNED_POKE_16(node, node_t, ksize, (uint16_t)size); } -MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t -node_flags(const node_t *const __restrict node) { +MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t node_flags(const node_t *const __restrict node) { return UNALIGNED_PEEK_8(node, node_t, flags); } -static inline void node_set_flags(node_t *const __restrict node, - uint8_t flags) { +static inline void node_set_flags(node_t *const __restrict node, uint8_t flags) { UNALIGNED_POKE_8(node, node_t, flags, flags); } /* Address of the key for the node */ -MDBX_NOTHROW_PURE_FUNCTION static inline void * -node_key(const node_t *const __restrict node) { +MDBX_NOTHROW_PURE_FUNCTION static inline void *node_key(const node_t *const __restrict node) { return ptr_disp(node, NODESIZE); } /* Address of the data for a node */ -MDBX_NOTHROW_PURE_FUNCTION static inline void * -node_data(const node_t *const __restrict node) { +MDBX_NOTHROW_PURE_FUNCTION static inline void *node_data(const node_t *const __restrict node) { return ptr_disp(node_key(node), node_ks(node)); } /* Size of a node in a leaf page with a given key and data. * This is node header plus key plus data size. */ -MDBX_NOTHROW_CONST_FUNCTION static inline size_t -node_size_len(const size_t key_len, const size_t value_len) { +MDBX_NOTHROW_CONST_FUNCTION static inline size_t node_size_len(const size_t key_len, const size_t value_len) { return NODESIZE + EVEN_CEIL(key_len + value_len); } -MDBX_NOTHROW_PURE_FUNCTION static inline size_t -node_size(const MDBX_val *key, const MDBX_val *value) { +MDBX_NOTHROW_PURE_FUNCTION static inline size_t node_size(const MDBX_val *key, const MDBX_val *value) { return node_size_len(key ? key->iov_len : 0, value ? value->iov_len : 0); } -MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t -node_largedata_pgno(const node_t *const __restrict node) { +MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t node_largedata_pgno(const node_t *const __restrict node) { assert(node_flags(node) & N_BIG); return peek_pgno(node_data(node)); } -MDBX_INTERNAL int __must_check_result node_read_bigdata(MDBX_cursor *mc, - const node_t *node, - MDBX_val *data, +MDBX_INTERNAL int __must_check_result node_read_bigdata(MDBX_cursor *mc, const node_t *node, MDBX_val *data, const page_t *mp); -static inline int __must_check_result node_read(MDBX_cursor *mc, - const node_t *node, - MDBX_val *data, - const page_t *mp) { +static inline int __must_check_result node_read(MDBX_cursor *mc, const node_t *node, MDBX_val *data, const page_t *mp) { data->iov_len = node_ds(node); data->iov_base = node_data(node); if (likely(node_flags(node) != N_BIG)) @@ -105,20 +90,12 @@ static inline int __must_check_result node_read(MDBX_cursor *mc, MDBX_INTERNAL nsr_t node_search(MDBX_cursor *mc, const MDBX_val *key); -MDBX_INTERNAL int __must_check_result node_add_branch(MDBX_cursor *mc, - size_t indx, - const MDBX_val *key, - pgno_t pgno); +MDBX_INTERNAL int __must_check_result node_add_branch(MDBX_cursor *mc, size_t indx, const MDBX_val *key, pgno_t pgno); -MDBX_INTERNAL int __must_check_result node_add_leaf(MDBX_cursor *mc, - size_t indx, - const MDBX_val *key, - MDBX_val *data, +MDBX_INTERNAL int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx, const MDBX_val *key, MDBX_val *data, unsigned flags); -MDBX_INTERNAL int __must_check_result node_add_dupfix(MDBX_cursor *mc, - size_t indx, - const MDBX_val *key); +MDBX_INTERNAL int __must_check_result node_add_dupfix(MDBX_cursor *mc, size_t indx, const MDBX_val *key); MDBX_INTERNAL void node_del(MDBX_cursor *mc, size_t ksize); diff --git a/src/options.h b/src/options.h index acfccfb1..1a80cd20 100644 --- a/src/options.h +++ b/src/options.h @@ -66,8 +66,7 @@ /** Does a system have battery-backed Real-Time Clock or just a fake. */ #ifndef MDBX_TRUST_RTC -#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ - defined(__OpenBSD__) +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || defined(__OpenBSD__) #define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ #else #define MDBX_TRUST_RTC 1 @@ -131,15 +130,13 @@ #ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT #define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ #ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT #define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ @@ -204,8 +201,7 @@ /** Size of buffer used during copying a environment/database file. */ #ifndef MDBX_ENVCOPY_WRITEBUF #define MDBX_ENVCOPY_WRITEBUF 1048576u -#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \ - MDBX_ENVCOPY_WRITEBUF % 65536u +#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || MDBX_ENVCOPY_WRITEBUF % 65536u #error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536 #endif /* MDBX_ENVCOPY_WRITEBUF */ @@ -224,8 +220,7 @@ #else #define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) #endif -#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \ - MDBX_ASSUME_MALLOC_OVERHEAD % 4 +#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || MDBX_ASSUME_MALLOC_OVERHEAD % 4 #error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4 #endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ @@ -252,15 +247,13 @@ #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #elif defined(__e2k__) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 -#elif __has_builtin(__builtin_cpu_supports) || \ - defined(__BUILTIN_CPU_SUPPORTS__) || \ +#elif __has_builtin(__builtin_cpu_supports) || defined(__BUILTIN_CPU_SUPPORTS__) || \ (defined(__ia32__) && __GNUC_PREREQ(4, 8) && __GLIBC_PREREQ(2, 23)) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 1 #else #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #endif -#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || \ - MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) +#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) #error MDBX_HAVE_BUILTIN_CPU_SUPPORTS must be defined as 0 or 1 #endif /* MDBX_HAVE_BUILTIN_CPU_SUPPORTS */ @@ -286,19 +279,15 @@ #define MDBX_LOCKING MDBX_LOCKING_WIN32FILES #else #ifndef MDBX_LOCKING -#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ - _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) +#if defined(_POSIX_THREAD_PROCESS_SHARED) && _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) /* Some platforms define the EOWNERDEAD error code even though they * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ -#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ - ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ - _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ - (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ - _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ - defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ - (!defined(__GLIBC__) || \ - __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) #define MDBX_LOCKING MDBX_LOCKING_POSIX2008 #else #define MDBX_LOCKING MDBX_LOCKING_POSIX2001 @@ -316,12 +305,9 @@ /** Advanced: Using POSIX OFD-locks (autodetection by default). */ #ifndef MDBX_USE_OFDLOCKS -#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && \ - defined(F_OFD_GETLK)) || \ - (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64))) && \ - !defined(MDBX_SAFE4QEMU) && \ - !defined(__sun) /* OFD-lock are broken on Solaris */ +#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK)) || \ + (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64))) && \ + !defined(MDBX_SAFE4QEMU) && !defined(__sun) /* OFD-lock are broken on Solaris */ #define MDBX_USE_OFDLOCKS 1 #else #define MDBX_USE_OFDLOCKS 0 @@ -335,8 +321,7 @@ /** Advanced: Using sendfile() syscall (autodetection by default). */ #ifndef MDBX_USE_SENDFILE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - !defined(__ANDROID_API__)) || \ +#if ((defined(__linux__) || defined(__gnu_linux__)) && !defined(__ANDROID_API__)) || \ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) #define MDBX_USE_SENDFILE 1 #else @@ -360,14 +345,12 @@ //------------------------------------------------------------------------------ #ifndef MDBX_CPU_WRITEBACK_INCOHERENT -#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ - defined(__hppa__) || defined(DOXYGEN) +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || defined(__hppa__) || defined(DOXYGEN) #define MDBX_CPU_WRITEBACK_INCOHERENT 0 #else #define MDBX_CPU_WRITEBACK_INCOHERENT 1 #endif -#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || \ - MDBX_CPU_WRITEBACK_INCOHERENT == 1) +#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || MDBX_CPU_WRITEBACK_INCOHERENT == 1) #error MDBX_CPU_WRITEBACK_INCOHERENT must be defined as 0 or 1 #endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ @@ -377,31 +360,27 @@ #else #define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || \ - MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) +#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) #error MDBX_MMAP_INCOHERENT_FILE_WRITE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ #ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE -#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#if defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS has cache coherency issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 #else /* LY: assume no relevant mmap/dcache issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || \ - MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) +#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) #error MDBX_MMAP_INCOHERENT_CPU_CACHE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ /** Assume system needs explicit syscall to sync/flush/write modified mapped * memory. */ #ifndef MDBX_MMAP_NEEDS_JOLT -#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE || \ - !(defined(__linux__) || defined(__gnu_linux__)) +#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE || !(defined(__linux__) || defined(__gnu_linux__)) #define MDBX_MMAP_NEEDS_JOLT 1 #else #define MDBX_MMAP_NEEDS_JOLT 0 @@ -456,8 +435,7 @@ #endif /* MDBX_64BIT_CAS */ #ifndef MDBX_UNALIGNED_OK -#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || \ - defined(ENABLE_UBSAN) +#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || defined(ENABLE_UBSAN) #define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */ #elif defined(__ARM_FEATURE_UNALIGNED) #define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */ diff --git a/src/osal.c b/src/osal.c index caff1ad2..cdef5712 100644 --- a/src/osal.c +++ b/src/osal.c @@ -39,8 +39,7 @@ static int ntstatus2errcode(NTSTATUS status) { ov.Internal = status; /* Zap: '_Param_(1)' could be '0' */ MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6387); - return GetOverlappedResult(nullptr, &ov, &dummy, FALSE) ? MDBX_SUCCESS - : (int)GetLastError(); + return GetOverlappedResult(nullptr, &ov, &dummy, FALSE) ? MDBX_SUCCESS : (int)GetLastError(); } /* We use native NT APIs to setup the memory map, so that we can @@ -51,11 +50,10 @@ static int ntstatus2errcode(NTSTATUS status) { * declare them here. Using these APIs also means we must link to * ntdll.dll, which is not linked by default in user code. */ -extern NTSTATUS NTAPI NtCreateSection( - OUT PHANDLE SectionHandle, IN ACCESS_MASK DesiredAccess, - IN OPTIONAL POBJECT_ATTRIBUTES ObjectAttributes, - IN OPTIONAL PLARGE_INTEGER MaximumSize, IN ULONG SectionPageProtection, - IN ULONG AllocationAttributes, IN OPTIONAL HANDLE FileHandle); +extern NTSTATUS NTAPI NtCreateSection(OUT PHANDLE SectionHandle, IN ACCESS_MASK DesiredAccess, + IN OPTIONAL POBJECT_ATTRIBUTES ObjectAttributes, + IN OPTIONAL PLARGE_INTEGER MaximumSize, IN ULONG SectionPageProtection, + IN ULONG AllocationAttributes, IN OPTIONAL HANDLE FileHandle); typedef struct _SECTION_BASIC_INFORMATION { ULONG Unknown; @@ -63,27 +61,22 @@ typedef struct _SECTION_BASIC_INFORMATION { LARGE_INTEGER SectionSize; } SECTION_BASIC_INFORMATION, *PSECTION_BASIC_INFORMATION; -extern NTSTATUS NTAPI NtMapViewOfSection( - IN HANDLE SectionHandle, IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, - IN ULONG_PTR ZeroBits, IN SIZE_T CommitSize, - IN OUT OPTIONAL PLARGE_INTEGER SectionOffset, IN OUT PSIZE_T ViewSize, - IN SECTION_INHERIT InheritDisposition, IN ULONG AllocationType, - IN ULONG Win32Protect); +extern NTSTATUS NTAPI NtMapViewOfSection(IN HANDLE SectionHandle, IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, + IN ULONG_PTR ZeroBits, IN SIZE_T CommitSize, + IN OUT OPTIONAL PLARGE_INTEGER SectionOffset, IN OUT PSIZE_T ViewSize, + IN SECTION_INHERIT InheritDisposition, IN ULONG AllocationType, + IN ULONG Win32Protect); -extern NTSTATUS NTAPI NtUnmapViewOfSection(IN HANDLE ProcessHandle, - IN OPTIONAL PVOID BaseAddress); +extern NTSTATUS NTAPI NtUnmapViewOfSection(IN HANDLE ProcessHandle, IN OPTIONAL PVOID BaseAddress); /* Zap: Inconsistent annotation for 'NtClose'... */ MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(28251) extern NTSTATUS NTAPI NtClose(HANDLE Handle); -extern NTSTATUS NTAPI NtAllocateVirtualMemory( - IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, IN ULONG_PTR ZeroBits, - IN OUT PSIZE_T RegionSize, IN ULONG AllocationType, IN ULONG Protect); +extern NTSTATUS NTAPI NtAllocateVirtualMemory(IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, IN ULONG_PTR ZeroBits, + IN OUT PSIZE_T RegionSize, IN ULONG AllocationType, IN ULONG Protect); -extern NTSTATUS NTAPI NtFreeVirtualMemory(IN HANDLE ProcessHandle, - IN PVOID *BaseAddress, - IN OUT PSIZE_T RegionSize, +extern NTSTATUS NTAPI NtFreeVirtualMemory(IN HANDLE ProcessHandle, IN PVOID *BaseAddress, IN OUT PSIZE_T RegionSize, IN ULONG FreeType); #ifndef WOF_CURRENT_VERSION @@ -127,8 +120,7 @@ typedef struct _FILE_PROVIDER_EXTERNAL_INFO_V1 { #endif #ifndef FSCTL_GET_EXTERNAL_BACKING -#define FSCTL_GET_EXTERNAL_BACKING \ - CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 196, METHOD_BUFFERED, FILE_ANY_ACCESS) +#define FSCTL_GET_EXTERNAL_BACKING CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 196, METHOD_BUFFERED, FILE_ANY_ACCESS) #endif #ifndef ERROR_NOT_CAPABLE @@ -140,29 +132,23 @@ typedef struct _FILE_PROVIDER_EXTERNAL_INFO_V1 { /*----------------------------------------------------------------------------*/ #if defined(__ANDROID_API__) -__extern_C void __assert2(const char *file, int line, const char *function, - const char *msg) __noreturn; -#define __assert_fail(assertion, file, line, function) \ - __assert2(file, line, function, assertion) +__extern_C void __assert2(const char *file, int line, const char *function, const char *msg) __noreturn; +#define __assert_fail(assertion, file, line, function) __assert2(file, line, function, assertion) #elif defined(__UCLIBC__) -MDBX_NORETURN __extern_C void __assert(const char *, const char *, unsigned, - const char *) +MDBX_NORETURN __extern_C void __assert(const char *, const char *, unsigned, const char *) #ifdef __THROW __THROW #else __nothrow #endif /* __THROW */ ; -#define __assert_fail(assertion, file, line, function) \ - __assert(assertion, file, line, function) +#define __assert_fail(assertion, file, line, function) __assert(assertion, file, line, function) -#elif _POSIX_C_SOURCE > 200212 && \ - /* workaround for avoid musl libc wrong prototype */ ( \ - defined(__GLIBC__) || defined(__GNU_LIBRARY__)) +#elif _POSIX_C_SOURCE > 200212 && \ + /* workaround for avoid musl libc wrong prototype */ (defined(__GLIBC__) || defined(__GNU_LIBRARY__)) /* Prototype should match libc runtime. ISO POSIX (2003) & LSB 1.x-3.x */ -MDBX_NORETURN __extern_C void __assert_fail(const char *assertion, - const char *file, unsigned line, +MDBX_NORETURN __extern_C void __assert_fail(const char *assertion, const char *file, unsigned line, const char *function) #ifdef __THROW __THROW @@ -172,8 +158,7 @@ MDBX_NORETURN __extern_C void __assert_fail(const char *assertion, ; #elif defined(__APPLE__) || defined(__MACH__) -__extern_C void __assert_rtn(const char *function, const char *file, int line, - const char *assertion) /* __nothrow */ +__extern_C void __assert_rtn(const char *function, const char *file, int line, const char *assertion) /* __nothrow */ #ifdef __dead2 __dead2 #else @@ -184,30 +169,20 @@ __extern_C void __assert_rtn(const char *function, const char *file, int line, #endif /* __disable_tail_calls */ ; -#define __assert_fail(assertion, file, line, function) \ - __assert_rtn(function, file, line, assertion) +#define __assert_fail(assertion, file, line, function) __assert_rtn(function, file, line, assertion) #elif defined(__sun) || defined(__SVR4) || defined(__svr4__) -MDBX_NORETURN __extern_C void __assert_c99(const char *assection, - const char *file, int line, - const char *function); -#define __assert_fail(assertion, file, line, function) \ - __assert_c99(assertion, file, line, function) +MDBX_NORETURN __extern_C void __assert_c99(const char *assection, const char *file, int line, const char *function); +#define __assert_fail(assertion, file, line, function) __assert_c99(assertion, file, line, function) #elif defined(__OpenBSD__) -__extern_C __dead void __assert2(const char *file, int line, - const char *function, +__extern_C __dead void __assert2(const char *file, int line, const char *function, const char *assertion) /* __nothrow */; -#define __assert_fail(assertion, file, line, function) \ - __assert2(file, line, function, assertion) +#define __assert_fail(assertion, file, line, function) __assert2(file, line, function, assertion) #elif defined(__NetBSD__) -__extern_C __dead void __assert13(const char *file, int line, - const char *function, +__extern_C __dead void __assert13(const char *file, int line, const char *function, const char *assertion) /* __nothrow */; -#define __assert_fail(assertion, file, line, function) \ - __assert13(file, line, function, assertion) -#elif defined(__FreeBSD__) || defined(__BSD__) || defined(__bsdi__) || \ - defined(__DragonFly__) -__extern_C void __assert(const char *function, const char *file, int line, - const char *assertion) /* __nothrow */ +#define __assert_fail(assertion, file, line, function) __assert13(file, line, function, assertion) +#elif defined(__FreeBSD__) || defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) +__extern_C void __assert(const char *function, const char *file, int line, const char *assertion) /* __nothrow */ #ifdef __dead2 __dead2 #else @@ -217,13 +192,11 @@ __extern_C void __assert(const char *function, const char *file, int line, __disable_tail_calls #endif /* __disable_tail_calls */ ; -#define __assert_fail(assertion, file, line, function) \ - __assert(function, file, line, assertion) +#define __assert_fail(assertion, file, line, function) __assert(function, file, line, assertion) #endif /* __assert_fail */ -__cold void mdbx_assert_fail(const MDBX_env *env, const char *msg, - const char *func, unsigned line) { +__cold void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func, unsigned line) { #if MDBX_DEBUG if (env && env->assert_func) env->assert_func(env, msg, func, line); @@ -232,8 +205,7 @@ __cold void mdbx_assert_fail(const MDBX_env *env, const char *msg, assert_fail(msg, func, line); } -MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, - unsigned line) { +MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, unsigned line) { #endif /* MDBX_DEBUG */ if (globals.logger.ptr) @@ -241,8 +213,7 @@ MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, else { #if defined(_WIN32) || defined(_WIN64) char *message = nullptr; - const int num = osal_asprintf(&message, "\r\nMDBX-ASSERTION: %s, %s:%u", - msg, func ? func : "unknown", line); + const int num = osal_asprintf(&message, "\r\nMDBX-ASSERTION: %s, %s:%u", msg, func ? func : "unknown", line); if (num < 1 || !message) message = ""; OutputDebugStringA(message); @@ -254,8 +225,7 @@ MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, while (1) { #if defined(_WIN32) || defined(_WIN64) #if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG) - _CrtDbgReport(_CRT_ASSERT, func ? func : "unknown", line, "libmdbx", - "assertion failed: %s", msg); + _CrtDbgReport(_CRT_ASSERT, func ? func : "unknown", line, "libmdbx", "assertion failed: %s", msg); #else if (IsDebuggerPresent()) DebugBreak(); @@ -275,9 +245,7 @@ __cold void mdbx_panic(const char *fmt, ...) { const int num = osal_vasprintf(&message, fmt, ap); va_end(ap); const char *const const_message = - unlikely(num < 1 || !message) - ? "" - : message; + unlikely(num < 1 || !message) ? "" : message; if (globals.logger.ptr) debug_log(MDBX_LOG_FATAL, "panic", 0, "%s", const_message); @@ -285,8 +253,7 @@ __cold void mdbx_panic(const char *fmt, ...) { while (1) { #if defined(_WIN32) || defined(_WIN64) #if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG) - _CrtDbgReport(_CRT_ASSERT, "mdbx.c", 0, "libmdbx", "panic: %s", - const_message); + _CrtDbgReport(_CRT_ASSERT, "mdbx.c", 0, "libmdbx", "panic: %s", const_message); #else OutputDebugStringA("\r\nMDBX-PANIC: "); OutputDebugStringA(const_message); @@ -346,19 +313,16 @@ MDBX_INTERNAL int osal_asprintf(char **strp, const char *fmt, ...) { #endif /* osal_asprintf */ #ifndef osal_memalign_alloc -MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, - void **result) { +MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, void **result) { assert(is_powerof2(alignment) && alignment >= sizeof(void *)); #if defined(_WIN32) || defined(_WIN64) (void)alignment; - *result = - VirtualAlloc(nullptr, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + *result = VirtualAlloc(nullptr, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); return *result ? MDBX_SUCCESS : MDBX_ENOMEM /* ERROR_OUTOFMEMORY */; #elif defined(_ISOC11_SOURCE) *result = aligned_alloc(alignment, ceil_powerof2(bytes, alignment)); return *result ? MDBX_SUCCESS : errno; -#elif _POSIX_VERSION >= 200112L && \ - (!defined(__ANDROID_API__) || __ANDROID_API__ >= 17) +#elif _POSIX_VERSION >= 200112L && (!defined(__ANDROID_API__) || __ANDROID_API__ >= 17) *result = nullptr; return posix_memalign(result, alignment, bytes); #elif __GLIBC_PREREQ(2, 16) || __STDC_VERSION__ >= 201112L @@ -474,8 +438,7 @@ MDBX_INTERNAL int osal_condpair_signal(osal_condpair_t *condpair, bool part) { MDBX_INTERNAL int osal_condpair_wait(osal_condpair_t *condpair, bool part) { #if defined(_WIN32) || defined(_WIN64) - DWORD code = SignalObjectAndWait(condpair->mutex, condpair->event[part], - INFINITE, FALSE); + DWORD code = SignalObjectAndWait(condpair->mutex, condpair->event[part], INFINITE, FALSE); if (code == WAIT_OBJECT_0) { code = WaitForSingleObject(condpair->mutex, INFINITE); if (code == WAIT_OBJECT_0) @@ -521,11 +484,9 @@ MDBX_INTERNAL int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex) { #if defined(_WIN32) || defined(_WIN64) __try { EnterCriticalSection(fastmutex); - } __except ( - (GetExceptionCode() == - 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */) - ? EXCEPTION_EXECUTE_HANDLER - : EXCEPTION_CONTINUE_SEARCH) { + } __except ((GetExceptionCode() == 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */) + ? EXCEPTION_EXECUTE_HANDLER + : EXCEPTION_CONTINUE_SEARCH) { return MDBX_EDEADLK; } return MDBX_SUCCESS; @@ -548,8 +509,7 @@ MDBX_INTERNAL int osal_fastmutex_release(osal_fastmutex_t *fastmutex) { #if defined(_WIN32) || defined(_WIN64) MDBX_INTERNAL int osal_mb2w(const char *const src, wchar_t **const pdst) { - const size_t dst_wlen = MultiByteToWideChar( - CP_THREAD_ACP, MB_ERR_INVALID_CHARS, src, -1, nullptr, 0); + const size_t dst_wlen = MultiByteToWideChar(CP_THREAD_ACP, MB_ERR_INVALID_CHARS, src, -1, nullptr, 0); wchar_t *dst = *pdst; int rc = ERROR_INVALID_NAME; if (unlikely(dst_wlen < 2 || dst_wlen > /* MAX_PATH */ INT16_MAX)) @@ -561,9 +521,7 @@ MDBX_INTERNAL int osal_mb2w(const char *const src, wchar_t **const pdst) { goto bailout; *pdst = dst; - if (likely(dst_wlen == (size_t)MultiByteToWideChar(CP_THREAD_ACP, - MB_ERR_INVALID_CHARS, src, - -1, dst, (int)dst_wlen))) + if (likely(dst_wlen == (size_t)MultiByteToWideChar(CP_THREAD_ACP, MB_ERR_INVALID_CHARS, src, -1, dst, (int)dst_wlen))) return MDBX_SUCCESS; rc = ERROR_INVALID_NAME; @@ -623,8 +581,7 @@ static size_t osal_iov_max; MDBX_INTERNAL int osal_ioring_create(osal_ioring_t *ior #if defined(_WIN32) || defined(_WIN64) , - bool enable_direct, - mdbx_filehandle_t overlapped_fd + bool enable_direct, mdbx_filehandle_t overlapped_fd #endif /* Windows */ ) { memset(ior, 0, sizeof(osal_ioring_t)); @@ -649,9 +606,8 @@ MDBX_INTERNAL int osal_ioring_create(osal_ioring_t *ior static inline size_t ior_offset(const ior_item_t *item) { #if defined(_WIN32) || defined(_WIN64) - return item->ov.Offset | (size_t)((sizeof(size_t) > sizeof(item->ov.Offset)) - ? (uint64_t)item->ov.OffsetHigh << 32 - : 0); + return item->ov.Offset | + (size_t)((sizeof(size_t) > sizeof(item->ov.Offset)) ? (uint64_t)item->ov.OffsetHigh << 32 : 0); #else return item->offset; #endif /* !Windows */ @@ -660,9 +616,7 @@ static inline size_t ior_offset(const ior_item_t *item) { static inline ior_item_t *ior_next(ior_item_t *item, size_t sgvcnt) { #if defined(ior_sgv_element) assert(sgvcnt > 0); - return (ior_item_t *)ptr_disp(item, sizeof(ior_item_t) - - sizeof(ior_sgv_element) + - sizeof(ior_sgv_element) * sgvcnt); + return (ior_item_t *)ptr_disp(item, sizeof(ior_item_t) - sizeof(ior_sgv_element) + sizeof(ior_sgv_element) * sgvcnt); #else assert(sgvcnt == 1); (void)sgvcnt; @@ -670,17 +624,14 @@ static inline ior_item_t *ior_next(ior_item_t *item, size_t sgvcnt) { #endif } -MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ior, const size_t offset, - void *data, const size_t bytes) { +MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ior, const size_t offset, void *data, const size_t bytes) { assert(bytes && data); assert(bytes % MDBX_MIN_PAGESIZE == 0 && bytes <= MAX_WRITE); - assert(offset % MDBX_MIN_PAGESIZE == 0 && - offset + (uint64_t)bytes <= MAX_MAPSIZE); + assert(offset % MDBX_MIN_PAGESIZE == 0 && offset + (uint64_t)bytes <= MAX_MAPSIZE); #if defined(_WIN32) || defined(_WIN64) const unsigned segments = (unsigned)(bytes >> ior->pagesize_ln2); - const bool use_gather = - ior->direct && ior->overlapped_fd && ior->slots_left >= segments; + const bool use_gather = ior->direct && ior->overlapped_fd && ior->slots_left >= segments; #endif /* Windows */ ior_item_t *item = ior->pool; @@ -690,8 +641,7 @@ MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ior, const size_t offset, likely(ior_last_bytes(ior, item) + bytes <= MAX_WRITE)) { #if defined(_WIN32) || defined(_WIN64) if (use_gather && - ((bytes | (uintptr_t)data | ior->last_bytes | - (uintptr_t)(uint64_t)item->sgv[0].Buffer) & + ((bytes | (uintptr_t)data | ior->last_bytes | (uintptr_t)(uint64_t)item->sgv[0].Buffer) & ior_alignment_mask) == 0 && ior->last_sgvcnt + (size_t)segments < OSAL_IOV_MAX) { assert(ior->overlapped_fd); @@ -708,8 +658,7 @@ MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ior, const size_t offset, assert((item->single.iov_len & ior_WriteFile_flag) == 0); return MDBX_SUCCESS; } - const void *end = ptr_disp(item->single.iov_base, - item->single.iov_len - ior_WriteFile_flag); + const void *end = ptr_disp(item->single.iov_base, item->single.iov_len - ior_WriteFile_flag); if (unlikely(end == data)) { assert((item->single.iov_len & ior_WriteFile_flag) != 0); item->single.iov_len += bytes; @@ -717,8 +666,7 @@ MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ior, const size_t offset, } #elif MDBX_HAVE_PWRITEV assert((int)item->sgvcnt > 0); - const void *end = ptr_disp(item->sgv[item->sgvcnt - 1].iov_base, - item->sgv[item->sgvcnt - 1].iov_len); + const void *end = ptr_disp(item->sgv[item->sgvcnt - 1].iov_base, item->sgv[item->sgvcnt - 1].iov_len); if (unlikely(end == data)) { item->sgv[item->sgvcnt - 1].iov_len += bytes; ior->last_bytes += bytes; @@ -754,8 +702,7 @@ MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ior, const size_t offset, item->ov.Offset = (DWORD)offset; item->ov.OffsetHigh = HIGH_DWORD(offset); item->ov.hEvent = 0; - if (!use_gather || ((bytes | (uintptr_t)(data)) & ior_alignment_mask) != 0 || - segments > OSAL_IOV_MAX) { + if (!use_gather || ((bytes | (uintptr_t)(data)) & ior_alignment_mask) != 0 || segments > OSAL_IOV_MAX) { /* WriteFile() */ item->single.iov_base = data; item->single.iov_len = bytes + ior_WriteFile_flag; @@ -790,9 +737,7 @@ MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ior, const size_t offset, } MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx, - void (*callback)(iov_ctx_t *ctx, - size_t offset, void *data, - size_t bytes)) { + void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)) { for (ior_item_t *item = ior->pool; item <= ior->last;) { #if defined(_WIN32) || defined(_WIN64) size_t offset = ior_offset(item); @@ -833,14 +778,12 @@ MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx, } } -MDBX_INTERNAL osal_ioring_write_result_t -osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { +MDBX_INTERNAL osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { osal_ioring_write_result_t r = {MDBX_SUCCESS, 0}; #if defined(_WIN32) || defined(_WIN64) - HANDLE *const end_wait_for = - ior->event_pool + ior->allocated + - /* был выделен один дополнительный элемент для async_done */ 1; + HANDLE *const end_wait_for = ior->event_pool + ior->allocated + + /* был выделен один дополнительный элемент для async_done */ 1; HANDLE *wait_for = end_wait_for; LONG async_started = 0; for (ior_item_t *item = ior->pool; item <= ior->last;) { @@ -867,8 +810,7 @@ osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { return r; } if (WriteFileGather(fd, item->sgv, (DWORD)bytes, nullptr, &item->ov)) { - assert(item->ov.Internal == 0 && - WaitForSingleObject(item->ov.hEvent, 0) == WAIT_OBJECT_0); + assert(item->ov.Internal == 0 && WaitForSingleObject(item->ov.hEvent, 0) == WAIT_OBJECT_0); ior_put_event(ior, item->ov.hEvent); item->ov.hEvent = 0; } else { @@ -877,9 +819,8 @@ osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { void *data = Ptr64ToPtr(item->sgv[0].Buffer); ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu," " offset %" PRId64 ", err %d", - "WriteFileGather", fd, __Wpedantic_format_voidptr(item), - item - ior->pool, data, ((page_t *)data)->pgno, bytes, - item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), r.err); + "WriteFileGather", fd, __Wpedantic_format_voidptr(item), item - ior->pool, data, ((page_t *)data)->pgno, + bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), r.err); goto bailout_rc; } assert(wait_for > ior->event_pool + ior->event_stack); @@ -889,8 +830,7 @@ osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { assert(bytes < MAX_WRITE); retry: item->ov.hEvent = ior; - if (WriteFileEx(fd, item->single.iov_base, (DWORD)bytes, &item->ov, - ior_wocr)) { + if (WriteFileEx(fd, item->single.iov_base, (DWORD)bytes, &item->ov, ior_wocr)) { async_started += 1; } else { r.err = (int)GetLastError(); @@ -898,21 +838,18 @@ osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { default: ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu," " offset %" PRId64 ", err %d", - "WriteFileEx", fd, __Wpedantic_format_voidptr(item), - item - ior->pool, item->single.iov_base, - ((page_t *)item->single.iov_base)->pgno, bytes, - item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), r.err); + "WriteFileEx", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base, + ((page_t *)item->single.iov_base)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), + r.err); goto bailout_rc; case ERROR_NOT_FOUND: case ERROR_USER_MAPPED_FILE: case ERROR_LOCK_VIOLATION: WARNING("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu," " offset %" PRId64 ", err %d", - "WriteFileEx", fd, __Wpedantic_format_voidptr(item), - item - ior->pool, item->single.iov_base, + "WriteFileEx", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base, ((page_t *)item->single.iov_base)->pgno, bytes, - item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), - r.err); + item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), r.err); SleepEx(0, true); goto retry; case ERROR_INVALID_USER_BUFFER: @@ -927,15 +864,13 @@ osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { } else { assert(bytes < MAX_WRITE); DWORD written = 0; - if (!WriteFile(fd, item->single.iov_base, (DWORD)bytes, &written, - &item->ov)) { + if (!WriteFile(fd, item->single.iov_base, (DWORD)bytes, &written, &item->ov)) { r.err = (int)GetLastError(); ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu," " offset %" PRId64 ", err %d", - "WriteFile", fd, __Wpedantic_format_voidptr(item), - item - ior->pool, item->single.iov_base, - ((page_t *)item->single.iov_base)->pgno, bytes, - item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), r.err); + "WriteFile", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base, + ((page_t *)item->single.iov_base)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), + r.err); goto bailout_rc; } else if (unlikely(written != bytes)) { r.err = ERROR_WRITE_FAULT; @@ -945,8 +880,7 @@ osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { item = ior_next(item, i); } - assert(ior->async_waiting > ior->async_completed && - ior->async_waiting == INT_MAX); + assert(ior->async_waiting > ior->async_completed && ior->async_waiting == INT_MAX); ior->async_waiting = async_started; if (async_started > ior->async_completed && end_wait_for == wait_for) { assert(wait_for > ior->event_pool + ior->event_stack); @@ -963,18 +897,15 @@ osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { * WaitForMultipleObjectsEx(), но тогда это проблемы на стороне M$. */ DWORD madness; do - madness = WaitForMultipleObjectsEx((pending_count < MAXIMUM_WAIT_OBJECTS) - ? (DWORD)pending_count - : MAXIMUM_WAIT_OBJECTS, - wait_for, true, - /* сутки */ 86400000ul, true); + madness = WaitForMultipleObjectsEx( + (pending_count < MAXIMUM_WAIT_OBJECTS) ? (DWORD)pending_count : MAXIMUM_WAIT_OBJECTS, wait_for, true, + /* сутки */ 86400000ul, true); while (madness == WAIT_IO_COMPLETION); STATIC_ASSERT(WAIT_OBJECT_0 == 0); if (/* madness >= WAIT_OBJECT_0 && */ madness < WAIT_OBJECT_0 + MAXIMUM_WAIT_OBJECTS) r.err = MDBX_SUCCESS; - else if (madness >= WAIT_ABANDONED_0 && - madness < WAIT_ABANDONED_0 + MAXIMUM_WAIT_OBJECTS) { + else if (madness >= WAIT_ABANDONED_0 && madness < WAIT_ABANDONED_0 + MAXIMUM_WAIT_OBJECTS) { r.err = ERROR_ABANDONED_WAIT_0; goto bailout_rc; } else if (madness == WAIT_TIMEOUT) { @@ -1003,9 +934,8 @@ osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { if (unlikely(!GetOverlappedResult(fd, &item->ov, &written, true))) { ERROR("%s: item %p (%zu), addr %p pgno %u, bytes %zu," " offset %" PRId64 ", err %d", - "GetOverlappedResult", __Wpedantic_format_voidptr(item), - item - ior->pool, data, ((page_t *)data)->pgno, bytes, - item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), + "GetOverlappedResult", __Wpedantic_format_voidptr(item), item - ior->pool, data, + ((page_t *)data)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), (int)GetLastError()); goto bailout_geterr; } @@ -1019,15 +949,12 @@ osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { if (unlikely(item->ov.Internal != MDBX_SUCCESS)) { DWORD written = 0; r.err = (int)item->ov.Internal; - if ((r.err & 0x80000000) && - GetOverlappedResult(nullptr, &item->ov, &written, true)) + if ((r.err & 0x80000000) && GetOverlappedResult(nullptr, &item->ov, &written, true)) r.err = (int)GetLastError(); ERROR("%s: item %p (%zu), addr %p pgno %u, bytes %zu," " offset %" PRId64 ", err %d", - "Result", __Wpedantic_format_voidptr(item), item - ior->pool, - data, ((page_t *)data)->pgno, bytes, - item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), - (int)GetLastError()); + "Result", __Wpedantic_format_voidptr(item), item - ior->pool, data, ((page_t *)data)->pgno, bytes, + item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), (int)GetLastError()); goto bailout_rc; } if (unlikely(item->ov.InternalHigh != bytes)) { @@ -1043,14 +970,12 @@ osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { assert(ior->async_waiting == ior->async_completed); #else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); for (ior_item_t *item = ior->pool; item <= ior->last;) { #if MDBX_HAVE_PWRITEV assert(item->sgvcnt > 0); if (item->sgvcnt == 1) - r.err = osal_pwrite(fd, item->sgv[0].iov_base, item->sgv[0].iov_len, - item->offset); + r.err = osal_pwrite(fd, item->sgv[0].iov_base, item->sgv[0].iov_len, item->offset); else r.err = osal_pwritev(fd, item->sgv, item->sgvcnt, item->offset); @@ -1058,8 +983,7 @@ osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { item = ior_next(item, item->sgvcnt); #else - r.err = osal_pwrite(fd, item->single.iov_base, item->single.iov_len, - item->offset); + r.err = osal_pwrite(fd, item->single.iov_base, item->single.iov_len, item->offset); item = ior_next(item, 1); #endif r.wops += 1; @@ -1122,12 +1046,9 @@ MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *ior, size_t items) { #if defined(_WIN32) || defined(_WIN64) if (ior->state & IOR_STATE_LOCKED) return MDBX_SUCCESS; - const bool useSetFileIoOverlappedRange = - ior->overlapped_fd && imports.SetFileIoOverlappedRange && items > 42; + const bool useSetFileIoOverlappedRange = ior->overlapped_fd && imports.SetFileIoOverlappedRange && items > 42; const size_t ceiling = - useSetFileIoOverlappedRange - ? ((items < 65536 / 2 / sizeof(ior_item_t)) ? 65536 : 65536 * 4) - : 1024; + useSetFileIoOverlappedRange ? ((items < 65536 / 2 / sizeof(ior_item_t)) ? 65536 : 65536 * 4) : 1024; const size_t bytes = ceil_powerof2(sizeof(ior_item_t) * items, ceiling); items = bytes / sizeof(ior_item_t); #endif /* Windows */ @@ -1137,9 +1058,7 @@ MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *ior, size_t items) { if (items < ior->allocated) ior_cleanup(ior, items); #if defined(_WIN32) || defined(_WIN64) - void *ptr = osal_realloc( - ior->event_pool, - (items + /* extra for waiting the async_done */ 1) * sizeof(HANDLE)); + void *ptr = osal_realloc(ior->event_pool, (items + /* extra for waiting the async_done */ 1) * sizeof(HANDLE)); if (unlikely(!ptr)) return MDBX_ENOMEM; ior->event_pool = ptr; @@ -1159,14 +1078,12 @@ MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *ior, size_t items) { ior->pool = ptr; if (items > ior->allocated) - memset(ior->pool + ior->allocated, 0, - sizeof(ior_item_t) * (items - ior->allocated)); + memset(ior->pool + ior->allocated, 0, sizeof(ior_item_t) * (items - ior->allocated)); ior->allocated = (unsigned)items; ior->boundary = ptr_disp(ior->pool, ior->allocated); #if defined(_WIN32) || defined(_WIN64) if (useSetFileIoOverlappedRange) { - if (imports.SetFileIoOverlappedRange(ior->overlapped_fd, ptr, - (ULONG)bytes)) + if (imports.SetFileIoOverlappedRange(ior->overlapped_fd, ptr, (ULONG)bytes)) ior->state += IOR_STATE_LOCKED; else return GetLastError(); @@ -1218,9 +1135,7 @@ MDBX_INTERNAL int osal_fileexists(const pathchar_t *pathname) { if (GetFileAttributesW(pathname) != INVALID_FILE_ATTRIBUTES) return MDBX_RESULT_TRUE; int err = GetLastError(); - return (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) - ? MDBX_RESULT_FALSE - : err; + return (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) ? MDBX_RESULT_FALSE : err; #else if (access(pathname, F_OK) == 0) return MDBX_RESULT_TRUE; @@ -1239,8 +1154,7 @@ MDBX_INTERNAL pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len) { return (pathchar_t *)ext; } -MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, - size_t len) { +MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len) { #if defined(_WIN32) || defined(_WIN64) for (size_t i = 0; i < len; ++i) { pathchar_t a = l[i]; @@ -1256,19 +1170,15 @@ MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, #endif } -MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, - const MDBX_env *env, const pathchar_t *pathname, - mdbx_filehandle_t *fd, - mdbx_mode_t unix_mode_bits) { +MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env, + const pathchar_t *pathname, mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits) { *fd = INVALID_HANDLE_VALUE; #if defined(_WIN32) || defined(_WIN64) DWORD CreationDisposition = unix_mode_bits ? OPEN_ALWAYS : OPEN_EXISTING; - DWORD FlagsAndAttributes = - FILE_FLAG_POSIX_SEMANTICS | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED; + DWORD FlagsAndAttributes = FILE_FLAG_POSIX_SEMANTICS | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED; DWORD DesiredAccess = FILE_READ_ATTRIBUTES; - DWORD ShareMode = - (env->flags & MDBX_EXCLUSIVE) ? 0 : (FILE_SHARE_READ | FILE_SHARE_WRITE); + DWORD ShareMode = (env->flags & MDBX_EXCLUSIVE) ? 0 : (FILE_SHARE_READ | FILE_SHARE_WRITE); switch (purpose) { default: @@ -1309,18 +1219,15 @@ MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, case MDBX_OPEN_DELETE: CreationDisposition = OPEN_EXISTING; ShareMode |= FILE_SHARE_DELETE; - DesiredAccess = - FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES | DELETE | SYNCHRONIZE; + DesiredAccess = FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES | DELETE | SYNCHRONIZE; break; } - *fd = CreateFileW(pathname, DesiredAccess, ShareMode, nullptr, - CreationDisposition, FlagsAndAttributes, nullptr); + *fd = CreateFileW(pathname, DesiredAccess, ShareMode, nullptr, CreationDisposition, FlagsAndAttributes, nullptr); if (*fd == INVALID_HANDLE_VALUE) { int err = (int)GetLastError(); if (err == ERROR_ACCESS_DENIED && purpose == MDBX_OPEN_LCK) { - if (GetFileAttributesW(pathname) == INVALID_FILE_ATTRIBUTES && - GetLastError() == ERROR_FILE_NOT_FOUND) + if (GetFileAttributesW(pathname) == INVALID_FILE_ATTRIBUTES && GetLastError() == ERROR_FILE_NOT_FOUND) err = ERROR_FILE_NOT_FOUND; } return err; @@ -1334,9 +1241,8 @@ MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, return err; } const DWORD AttributesDiff = - (info.dwFileAttributes ^ FlagsAndAttributes) & - (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED | - FILE_ATTRIBUTE_TEMPORARY | FILE_ATTRIBUTE_COMPRESSED); + (info.dwFileAttributes ^ FlagsAndAttributes) & (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED | + FILE_ATTRIBUTE_TEMPORARY | FILE_ATTRIBUTE_COMPRESSED); if (AttributesDiff) (void)SetFileAttributesW(pathname, info.dwFileAttributes ^ AttributesDiff); @@ -1372,8 +1278,7 @@ MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, break; } - const bool direct_nocache_for_copy = - env->ps >= globals.sys_pagesize && purpose == MDBX_OPEN_COPY; + const bool direct_nocache_for_copy = env->ps >= globals.sys_pagesize && purpose == MDBX_OPEN_COPY; if (direct_nocache_for_copy) { #if defined(O_DIRECT) flags |= O_DIRECT; @@ -1392,18 +1297,15 @@ MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, int stub_fd0 = -1, stub_fd1 = -1, stub_fd2 = -1; static const char dev_null[] = "/dev/null"; if (!is_valid_fd(STDIN_FILENO)) { - WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "IN", - STDIN_FILENO, dev_null); + WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "IN", STDIN_FILENO, dev_null); stub_fd0 = open(dev_null, O_RDONLY | O_NOCTTY); } if (!is_valid_fd(STDOUT_FILENO)) { - WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "OUT", - STDOUT_FILENO, dev_null); + WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "OUT", STDOUT_FILENO, dev_null); stub_fd1 = open(dev_null, O_WRONLY | O_NOCTTY); } if (!is_valid_fd(STDERR_FILENO)) { - WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "ERR", - STDERR_FILENO, dev_null); + WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "ERR", STDERR_FILENO, dev_null); stub_fd2 = open(dev_null, O_WRONLY | O_NOCTTY); } #else @@ -1412,8 +1314,7 @@ MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, *fd = open(pathname, flags, unix_mode_bits); #if defined(O_DIRECT) - if (*fd < 0 && (flags & O_DIRECT) && - (errno == EINVAL || errno == EAFNOSUPPORT)) { + if (*fd < 0 && (flags & O_DIRECT) && (errno == EINVAL || errno == EAFNOSUPPORT)) { flags &= ~(O_DIRECT | O_EXCL); *fd = open(pathname, flags, unix_mode_bits); } @@ -1428,20 +1329,17 @@ MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, /* Safeguard for https://libmdbx.dqdkfa.ru/dead-github/issues/144 */ #if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2 if (*fd == STDIN_FILENO) { - WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "IN", - STDIN_FILENO); + WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "IN", STDIN_FILENO); assert(stub_fd0 == -1); *fd = dup(stub_fd0 = *fd); } if (*fd == STDOUT_FILENO) { - WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "OUT", - STDOUT_FILENO); + WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "OUT", STDOUT_FILENO); assert(stub_fd1 == -1); *fd = dup(stub_fd1 = *fd); } if (*fd == STDERR_FILENO) { - WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "ERR", - STDERR_FILENO); + WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "ERR", STDERR_FILENO); assert(stub_fd2 == -1); *fd = dup(stub_fd2 = *fd); } @@ -1490,8 +1388,7 @@ MDBX_INTERNAL int osal_closefile(mdbx_filehandle_t fd) { #endif } -MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t bytes, - uint64_t offset) { +MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t bytes, uint64_t offset) { if (bytes > MAX_WRITE) return MDBX_EINVAL; #if defined(_WIN32) || defined(_WIN64) @@ -1506,8 +1403,7 @@ MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t bytes, return (rc == MDBX_SUCCESS) ? /* paranoia */ ERROR_READ_FAULT : rc; } #else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); intptr_t read = pread(fd, buf, bytes, offset); if (read < 0) { int rc = errno; @@ -1517,8 +1413,7 @@ MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t bytes, return (bytes == (size_t)read) ? MDBX_SUCCESS : MDBX_ENODATA; } -MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, - size_t bytes, uint64_t offset) { +MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t bytes, uint64_t offset) { while (true) { #if defined(_WIN32) || defined(_WIN64) OVERLAPPED ov; @@ -1527,17 +1422,13 @@ MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, ov.OffsetHigh = HIGH_DWORD(offset); DWORD written; - if (unlikely(!WriteFile( - fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, - &written, &ov))) + if (unlikely(!WriteFile(fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, &written, &ov))) return (int)GetLastError(); if (likely(bytes == written)) return MDBX_SUCCESS; #else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - const intptr_t written = - pwrite(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE, offset); + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); + const intptr_t written = pwrite(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE, offset); if (likely(bytes == (size_t)written)) return MDBX_SUCCESS; if (written < 0) { @@ -1553,22 +1444,17 @@ MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, } } -MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, - size_t bytes) { +MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, size_t bytes) { while (true) { #if defined(_WIN32) || defined(_WIN64) DWORD written; - if (unlikely(!WriteFile( - fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, - &written, nullptr))) + if (unlikely(!WriteFile(fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, &written, nullptr))) return (int)GetLastError(); if (likely(bytes == written)) return MDBX_SUCCESS; #else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - const intptr_t written = - write(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE); + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); + const intptr_t written = write(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE); if (likely(bytes == (size_t)written)) return MDBX_SUCCESS; if (written < 0) { @@ -1583,8 +1469,7 @@ MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, } } -int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, - uint64_t offset) { +int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset) { size_t expected = 0; for (size_t i = 0; i < sgvcnt; ++i) expected += iov[i].iov_len; @@ -1597,14 +1482,12 @@ int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, written += iov[i].iov_len; offset += iov[i].iov_len; } - return (expected == written) ? MDBX_SUCCESS - : MDBX_EIO /* ERROR_WRITE_FAULT */; + return (expected == written) ? MDBX_SUCCESS : MDBX_EIO /* ERROR_WRITE_FAULT */; #else int rc; intptr_t written; do { - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); written = pwritev(fd, iov, sgvcnt, offset); if (likely(expected == (size_t)written)) return MDBX_SUCCESS; @@ -1614,16 +1497,14 @@ int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, #endif } -MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, - enum osal_syncmode_bits mode_bits) { +MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, enum osal_syncmode_bits mode_bits) { #if defined(_WIN32) || defined(_WIN64) if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && !FlushFileBuffers(fd)) return (int)GetLastError(); return MDBX_SUCCESS; #else -#if defined(__APPLE__) && \ - MDBX_APPLE_SPEED_INSTEADOF_DURABILITY == MDBX_OSX_WANNA_DURABILITY +#if defined(__APPLE__) && MDBX_APPLE_SPEED_INSTEADOF_DURABILITY == MDBX_OSX_WANNA_DURABILITY if (mode_bits & MDBX_SYNC_IODQ) return likely(fcntl(fd, F_FULLFSYNC) != -1) ? MDBX_SUCCESS : errno; #endif /* MacOS */ @@ -1670,8 +1551,7 @@ int osal_filesize(mdbx_filehandle_t fd, uint64_t *length) { #else struct stat st; - STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); + STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); if (fstat(fd, &st)) return errno; @@ -1716,21 +1596,16 @@ MDBX_INTERNAL int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length) { if (imports.SetFileInformationByHandle) { FILE_END_OF_FILE_INFO EndOfFileInfo; EndOfFileInfo.EndOfFile.QuadPart = length; - return imports.SetFileInformationByHandle(fd, FileEndOfFileInfo, - &EndOfFileInfo, - sizeof(FILE_END_OF_FILE_INFO)) + return imports.SetFileInformationByHandle(fd, FileEndOfFileInfo, &EndOfFileInfo, sizeof(FILE_END_OF_FILE_INFO)) ? MDBX_SUCCESS : (int)GetLastError(); } else { LARGE_INTEGER li; li.QuadPart = length; - return (SetFilePointerEx(fd, li, nullptr, FILE_BEGIN) && SetEndOfFile(fd)) - ? MDBX_SUCCESS - : (int)GetLastError(); + return (SetFilePointerEx(fd, li, nullptr, FILE_BEGIN) && SetEndOfFile(fd)) ? MDBX_SUCCESS : (int)GetLastError(); } #else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); return ftruncate(fd, length) == 0 ? MDBX_SUCCESS : errno; #endif } @@ -1739,21 +1614,17 @@ MDBX_INTERNAL int osal_fseek(mdbx_filehandle_t fd, uint64_t pos) { #if defined(_WIN32) || defined(_WIN64) LARGE_INTEGER li; li.QuadPart = pos; - return SetFilePointerEx(fd, li, nullptr, FILE_BEGIN) ? MDBX_SUCCESS - : (int)GetLastError(); + return SetFilePointerEx(fd, li, nullptr, FILE_BEGIN) ? MDBX_SUCCESS : (int)GetLastError(); #else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); return (lseek(fd, pos, SEEK_SET) < 0) ? errno : MDBX_SUCCESS; #endif } /*----------------------------------------------------------------------------*/ -MDBX_INTERNAL int -osal_thread_create(osal_thread_t *thread, - THREAD_RESULT(THREAD_CALL *start_routine)(void *), - void *arg) { +MDBX_INTERNAL int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg) { #if defined(_WIN32) || defined(_WIN64) *thread = CreateThread(nullptr, 0, start_routine, arg, 0, nullptr); return *thread ? MDBX_SUCCESS : (int)GetLastError(); @@ -1774,8 +1645,7 @@ MDBX_INTERNAL int osal_thread_join(osal_thread_t thread) { /*----------------------------------------------------------------------------*/ -MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, - size_t length, enum osal_syncmode_bits mode_bits) { +MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, size_t length, enum osal_syncmode_bits mode_bits) { if (!MDBX_MMAP_NEEDS_JOLT && mode_bits == MDBX_SYNC_NONE) return MDBX_SUCCESS; @@ -1783,8 +1653,7 @@ MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, #if defined(_WIN32) || defined(_WIN64) if (!FlushViewOfFile(ptr, length)) return (int)GetLastError(); - if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && - !FlushFileBuffers(map->fd)) + if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && !FlushFileBuffers(map->fd)) return (int)GetLastError(); #else #if defined(__linux__) || defined(__gnu_linux__) @@ -1807,16 +1676,14 @@ MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, return MDBX_SUCCESS; } -MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, - const pathchar_t *pathname, int err) { +MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err) { #if defined(_WIN32) || defined(_WIN64) (void)pathname; (void)err; if (!imports.GetVolumeInformationByHandleW) return MDBX_ENOSYS; DWORD unused, flags; - if (!imports.GetVolumeInformationByHandleW(handle, nullptr, 0, nullptr, - &unused, &flags, nullptr, 0)) + if (!imports.GetVolumeInformationByHandleW(handle, nullptr, 0, nullptr, &unused, &flags, nullptr, 0)) return (int)GetLastError(); if ((flags & FILE_READ_ONLY_VOLUME) == 0) return MDBX_EACCESS; @@ -1858,9 +1725,8 @@ MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle) { return MDBX_RESULT_TRUE; } -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \ +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \ defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN) const char *const name = statfs_info.f_fstypename; const size_t name_len = sizeof(statfs_info.f_fstypename); @@ -1869,9 +1735,7 @@ MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle) { const size_t name_len = 0; #endif if (name_len) { - if (strncasecmp("tmpfs", name, 6) == 0 || - strncasecmp("mfs", name, 4) == 0 || - strncasecmp("ramfs", name, 6) == 0 || + if (strncasecmp("tmpfs", name, 6) == 0 || strncasecmp("mfs", name, 4) == 0 || strncasecmp("ramfs", name, 6) == 0 || strncasecmp("romfs", name, 6) == 0) return MDBX_RESULT_TRUE; } @@ -1890,14 +1754,11 @@ static int osal_check_fs_local(mdbx_filehandle_t handle, int flags) { if (imports.GetFileInformationByHandleEx) { FILE_REMOTE_PROTOCOL_INFO RemoteProtocolInfo; - if (imports.GetFileInformationByHandleEx(handle, FileRemoteProtocolInfo, - &RemoteProtocolInfo, + if (imports.GetFileInformationByHandleEx(handle, FileRemoteProtocolInfo, &RemoteProtocolInfo, sizeof(RemoteProtocolInfo))) { - if ((RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_OFFLINE) && - !(flags & MDBX_RDONLY)) + if ((RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_OFFLINE) && !(flags & MDBX_RDONLY)) return ERROR_FILE_OFFLINE; - if (!(RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK) && - !(flags & MDBX_EXCLUSIVE)) + if (!(RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK) && !(flags & MDBX_EXCLUSIVE)) return ERROR_REMOTE_STORAGE_MEDIA_ERROR; } } @@ -1913,46 +1774,37 @@ static int osal_check_fs_local(mdbx_filehandle_t handle, int flags) { size_t reserved_for_microsoft_madness[42]; } GetExternalBacking_OutputBuffer; IO_STATUS_BLOCK StatusBlock; - rc = imports.NtFsControlFile(handle, nullptr, nullptr, nullptr, - &StatusBlock, FSCTL_GET_EXTERNAL_BACKING, - nullptr, 0, &GetExternalBacking_OutputBuffer, - sizeof(GetExternalBacking_OutputBuffer)); + rc = imports.NtFsControlFile(handle, nullptr, nullptr, nullptr, &StatusBlock, FSCTL_GET_EXTERNAL_BACKING, nullptr, + 0, &GetExternalBacking_OutputBuffer, sizeof(GetExternalBacking_OutputBuffer)); if (NT_SUCCESS(rc)) { if (!(flags & MDBX_EXCLUSIVE)) return ERROR_REMOTE_STORAGE_MEDIA_ERROR; - } else if (rc != STATUS_OBJECT_NOT_EXTERNALLY_BACKED && - rc != STATUS_INVALID_DEVICE_REQUEST && + } else if (rc != STATUS_OBJECT_NOT_EXTERNALLY_BACKED && rc != STATUS_INVALID_DEVICE_REQUEST && rc != STATUS_NOT_SUPPORTED) return ntstatus2errcode(rc); } - if (imports.GetVolumeInformationByHandleW && - imports.GetFinalPathNameByHandleW) { + if (imports.GetVolumeInformationByHandleW && imports.GetFinalPathNameByHandleW) { WCHAR *PathBuffer = osal_malloc(sizeof(WCHAR) * INT16_MAX); if (!PathBuffer) return MDBX_ENOMEM; int rc = MDBX_SUCCESS; DWORD VolumeSerialNumber, FileSystemFlags; - if (!imports.GetVolumeInformationByHandleW(handle, PathBuffer, INT16_MAX, - &VolumeSerialNumber, nullptr, + if (!imports.GetVolumeInformationByHandleW(handle, PathBuffer, INT16_MAX, &VolumeSerialNumber, nullptr, &FileSystemFlags, nullptr, 0)) { rc = (int)GetLastError(); goto bailout; } if ((flags & MDBX_RDONLY) == 0) { - if (FileSystemFlags & - (FILE_SEQUENTIAL_WRITE_ONCE | FILE_READ_ONLY_VOLUME | - FILE_VOLUME_IS_COMPRESSED)) { + if (FileSystemFlags & (FILE_SEQUENTIAL_WRITE_ONCE | FILE_READ_ONLY_VOLUME | FILE_VOLUME_IS_COMPRESSED)) { rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR; goto bailout; } } - if (imports.GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, - FILE_NAME_NORMALIZED | - VOLUME_NAME_NT)) { + if (imports.GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, FILE_NAME_NORMALIZED | VOLUME_NAME_NT)) { if (_wcsnicmp(PathBuffer, L"\\Device\\Mup\\", 12) == 0) { if (!(flags & MDBX_EXCLUSIVE)) { rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR; @@ -1961,18 +1813,14 @@ static int osal_check_fs_local(mdbx_filehandle_t handle, int flags) { } } - if (F_ISSET(flags, MDBX_RDONLY | MDBX_EXCLUSIVE) && - (FileSystemFlags & FILE_READ_ONLY_VOLUME)) { + if (F_ISSET(flags, MDBX_RDONLY | MDBX_EXCLUSIVE) && (FileSystemFlags & FILE_READ_ONLY_VOLUME)) { /* without-LCK (exclusive readonly) mode for DB on a read-only volume */ goto bailout; } - if (imports.GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, - FILE_NAME_NORMALIZED | - VOLUME_NAME_DOS)) { + if (imports.GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, FILE_NAME_NORMALIZED | VOLUME_NAME_DOS)) { UINT DriveType = GetDriveTypeW(PathBuffer); - if (DriveType == DRIVE_NO_ROOT_DIR && - _wcsnicmp(PathBuffer, L"\\\\?\\", 4) == 0 && + if (DriveType == DRIVE_NO_ROOT_DIR && _wcsnicmp(PathBuffer, L"\\\\?\\", 4) == 0 && _wcsnicmp(PathBuffer + 5, L":\\", 2) == 0) { PathBuffer[7] = 0; DriveType = GetDriveTypeW(PathBuffer + 4); @@ -2027,8 +1875,7 @@ static int osal_check_fs_local(mdbx_filehandle_t handle, int flags) { const unsigned type = 0; const char *const name = statvfs_info.f_basetype; const size_t name_len = sizeof(statvfs_info.f_basetype); -#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ - defined(ST_FSTYPSZ) || defined(_ST_FSTYPSZ) +#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(ST_FSTYPSZ) || defined(_ST_FSTYPSZ) const unsigned type = 0; struct stat st; if (fstat(handle, &st)) @@ -2047,9 +1894,8 @@ static int osal_check_fs_local(mdbx_filehandle_t handle, int flags) { #if defined(MNT_LOCAL) || defined(MNT_EXPORTED) const unsigned long mnt_flags = statfs_info.f_flags; #endif /* MNT_LOCAL || MNT_EXPORTED */ -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \ +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \ defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN) const char *const name = statfs_info.f_fstypename; const size_t name_len = sizeof(statfs_info.f_fstypename); @@ -2074,12 +1920,11 @@ static int osal_check_fs_local(mdbx_filehandle_t handle, int flags) { mounted = setmntent("/etc/mtab", "r"); if (mounted) { const struct mntent *ent; -#if defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(__BIONIC__) || \ +#if defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(__BIONIC__) || \ (defined(_DEFAULT_SOURCE) && __GLIBC_PREREQ(2, 19)) struct mntent entbuf; const bool should_copy = false; - while (nullptr != - (ent = getmntent_r(mounted, &entbuf, pathbuf, sizeof(pathbuf)))) + while (nullptr != (ent = getmntent_r(mounted, &entbuf, pathbuf, sizeof(pathbuf)))) #else const bool should_copy = true; while (nullptr != (ent = getmntent(mounted))) @@ -2088,8 +1933,7 @@ static int osal_check_fs_local(mdbx_filehandle_t handle, int flags) { struct stat mnt; if (!stat(ent->mnt_dir, &mnt) && mnt.st_dev == st.st_dev) { if (should_copy) { - name = - strncpy(pathbuf, ent->mnt_fsname, name_len = sizeof(pathbuf) - 1); + name = strncpy(pathbuf, ent->mnt_fsname, name_len = sizeof(pathbuf) - 1); pathbuf[name_len] = 0; } else { name = ent->mnt_fsname; @@ -2104,17 +1948,13 @@ static int osal_check_fs_local(mdbx_filehandle_t handle, int flags) { #endif if (name_len) { - if (((name_len > 2 && strncasecmp("nfs", name, 3) == 0) || - strncasecmp("cifs", name, name_len) == 0 || - strncasecmp("ncpfs", name, name_len) == 0 || - strncasecmp("smbfs", name, name_len) == 0 || + if (((name_len > 2 && strncasecmp("nfs", name, 3) == 0) || strncasecmp("cifs", name, name_len) == 0 || + strncasecmp("ncpfs", name, name_len) == 0 || strncasecmp("smbfs", name, name_len) == 0 || strcasecmp("9P" /* WSL2 */, name) == 0 || - ((name_len > 3 && strncasecmp("fuse", name, 4) == 0) && - strncasecmp("fuseblk", name, name_len) != 0)) && + ((name_len > 3 && strncasecmp("fuse", name, 4) == 0) && strncasecmp("fuseblk", name, name_len) != 0)) && !(flags & MDBX_EXCLUSIVE)) return MDBX_EREMOTE; - if (strcasecmp("ftp", name) == 0 || strcasecmp("http", name) == 0 || - strcasecmp("sshfs", name) == 0) + if (strcasecmp("ftp", name) == 0 || strcasecmp("http", name) == 0 || strcasecmp("sshfs", name) == 0) return MDBX_EREMOTE; } @@ -2166,18 +2006,15 @@ static int check_mmap_limit(const size_t limit) { if (should_check) { intptr_t pagesize, total_ram_pages, avail_ram_pages; - int err = - mdbx_get_sysraminfo(&pagesize, &total_ram_pages, &avail_ram_pages); + int err = mdbx_get_sysraminfo(&pagesize, &total_ram_pages, &avail_ram_pages); if (unlikely(err != MDBX_SUCCESS)) return err; const int log2page = log2n_powerof2(pagesize); - if ((limit >> (log2page + 7)) > (size_t)total_ram_pages || - (limit >> (log2page + 6)) > (size_t)avail_ram_pages) { + if ((limit >> (log2page + 7)) > (size_t)total_ram_pages || (limit >> (log2page + 6)) > (size_t)avail_ram_pages) { ERROR("%s (%zu pages) is too large for available (%zu pages) or total " "(%zu pages) system RAM", - "database upper size limit", limit >> log2page, avail_ram_pages, - total_ram_pages); + "database upper size limit", limit >> log2page, avail_ram_pages, total_ram_pages); return MDBX_TOO_LARGE; } } @@ -2185,8 +2022,8 @@ static int check_mmap_limit(const size_t limit) { return MDBX_SUCCESS; } -MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, - const size_t limit, const unsigned options) { +MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, + const unsigned options) { assert(size <= limit); map->limit = 0; map->current = 0; @@ -2220,8 +2057,7 @@ MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, return err; #if defined(_WIN32) || defined(_WIN64) if (map->filesize < size) { - WARNING("file size (%zu) less than requested for mapping (%zu)", - (size_t)map->filesize, size); + WARNING("file size (%zu) less than requested for mapping (%zu)", (size_t)map->filesize, size); size = (size_t)map->filesize; } #else @@ -2235,10 +2071,8 @@ MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, err = NtCreateSection(&map->section, /* DesiredAccess */ (flags & MDBX_WRITEMAP) - ? SECTION_QUERY | SECTION_MAP_READ | - SECTION_EXTEND_SIZE | SECTION_MAP_WRITE - : SECTION_QUERY | SECTION_MAP_READ | - SECTION_EXTEND_SIZE, + ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | SECTION_MAP_WRITE + : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE, /* ObjectAttributes */ nullptr, /* MaximumSize (InitialSize) */ &SectionSize, /* SectionPageProtection */ @@ -2247,18 +2081,15 @@ MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, if (!NT_SUCCESS(err)) return ntstatus2errcode(err); - SIZE_T ViewSize = (flags & MDBX_RDONLY) ? 0 - : globals.running_under_Wine ? size - : limit; - err = NtMapViewOfSection( - map->section, GetCurrentProcess(), &map->base, - /* ZeroBits */ 0, - /* CommitSize */ 0, - /* SectionOffset */ nullptr, &ViewSize, - /* InheritDisposition */ ViewUnmap, - /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE, - /* Win32Protect */ - (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY); + SIZE_T ViewSize = (flags & MDBX_RDONLY) ? 0 : globals.running_under_Wine ? size : limit; + err = NtMapViewOfSection(map->section, GetCurrentProcess(), &map->base, + /* ZeroBits */ 0, + /* CommitSize */ 0, + /* SectionOffset */ nullptr, &ViewSize, + /* InheritDisposition */ ViewUnmap, + /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE, + /* Win32Protect */ + (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY); if (!NT_SUCCESS(err)) { NtClose(map->section); map->section = 0; @@ -2296,13 +2127,9 @@ MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, #define MAP_NORESERVE 0 #endif - map->base = mmap(nullptr, limit, - (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ, - MAP_SHARED | MAP_FILE | MAP_NORESERVE | - (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0) | - ((options & MMAP_OPTION_SEMAPHORE) - ? MAP_HASSEMAPHORE | MAP_NOSYNC - : MAP_CONCEAL), + map->base = mmap(nullptr, limit, (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ, + MAP_SHARED | MAP_FILE | MAP_NORESERVE | (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0) | + ((options & MMAP_OPTION_SEMAPHORE) ? MAP_HASSEMAPHORE | MAP_NOSYNC : MAP_CONCEAL), map->fd, 0); if (unlikely(map->base == MAP_FAILED)) { @@ -2334,9 +2161,8 @@ MDBX_INTERNAL int osal_munmap(osal_mmap_t *map) { /* Unpoisoning is required for ASAN to avoid false-positive diagnostic * when this memory will re-used by malloc or another mmapping. * See https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 */ - MDBX_ASAN_UNPOISON_MEMORY_REGION( - map->base, (map->filesize && map->filesize < map->limit) ? map->filesize - : map->limit); + MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, + (map->filesize && map->filesize < map->limit) ? map->filesize : map->limit); #if defined(_WIN32) || defined(_WIN64) if (map->section) NtClose(map->section); @@ -2356,11 +2182,9 @@ MDBX_INTERNAL int osal_munmap(osal_mmap_t *map) { return MDBX_SUCCESS; } -MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, - size_t limit) { +MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit) { int rc = osal_filesize(map->fd, &map->filesize); - VERBOSE("flags 0x%x, size %zu, limit %zu, filesize %" PRIu64, flags, size, - limit, map->filesize); + VERBOSE("flags 0x%x, size %zu, limit %zu, filesize %" PRIu64, flags, size, limit, map->filesize); assert(size <= limit); if (rc != MDBX_SUCCESS) { map->filesize = 0; @@ -2400,15 +2224,13 @@ MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, /* check ability of address space for growth before unmap */ PVOID BaseAddress = (PBYTE)map->base + map->limit; SIZE_T RegionSize = limit - map->limit; - status = NtAllocateVirtualMemory(GetCurrentProcess(), &BaseAddress, 0, - &RegionSize, MEM_RESERVE, PAGE_NOACCESS); + status = NtAllocateVirtualMemory(GetCurrentProcess(), &BaseAddress, 0, &RegionSize, MEM_RESERVE, PAGE_NOACCESS); if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018) return MDBX_UNABLE_EXTEND_MAPSIZE; if (!NT_SUCCESS(status)) return ntstatus2errcode(status); - status = NtFreeVirtualMemory(GetCurrentProcess(), &BaseAddress, &RegionSize, - MEM_RELEASE); + status = NtFreeVirtualMemory(GetCurrentProcess(), &BaseAddress, &RegionSize, MEM_RELEASE); if (!NT_SUCCESS(status)) return ntstatus2errcode(status); } @@ -2443,8 +2265,7 @@ MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, map->current = map->limit = 0; if (ReservedAddress) { ReservedSize = 0; - status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, - &ReservedSize, MEM_RELEASE); + status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, &ReservedSize, MEM_RELEASE); assert(NT_SUCCESS(status)); (void)status; } @@ -2455,8 +2276,7 @@ retry_file_and_section: /* resizing of the file may take a while, * therefore we reserve address space to avoid occupy it by other threads */ ReservedAddress = map->base; - status = NtAllocateVirtualMemory(GetCurrentProcess(), &ReservedAddress, 0, - &ReservedSize, MEM_RESERVE, PAGE_NOACCESS); + status = NtAllocateVirtualMemory(GetCurrentProcess(), &ReservedAddress, 0, &ReservedSize, MEM_RESERVE, PAGE_NOACCESS); if (!NT_SUCCESS(status)) { ReservedAddress = nullptr; if (status != (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018) @@ -2476,18 +2296,16 @@ retry_file_and_section: } SectionSize.QuadPart = size; - status = NtCreateSection( - &map->section, - /* DesiredAccess */ - (flags & MDBX_WRITEMAP) - ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | - SECTION_MAP_WRITE - : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE, - /* ObjectAttributes */ nullptr, - /* MaximumSize (InitialSize) */ &SectionSize, - /* SectionPageProtection */ - (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE, - /* AllocationAttributes */ SEC_RESERVE, map->fd); + status = NtCreateSection(&map->section, + /* DesiredAccess */ + (flags & MDBX_WRITEMAP) + ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | SECTION_MAP_WRITE + : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE, + /* ObjectAttributes */ nullptr, + /* MaximumSize (InitialSize) */ &SectionSize, + /* SectionPageProtection */ + (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE, + /* AllocationAttributes */ SEC_RESERVE, map->fd); if (!NT_SUCCESS(status)) goto bailout_ntstatus; @@ -2495,8 +2313,7 @@ retry_file_and_section: if (ReservedAddress) { /* release reserved address space */ ReservedSize = 0; - status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, - &ReservedSize, MEM_RELEASE); + status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, &ReservedSize, MEM_RELEASE); ReservedAddress = nullptr; if (!NT_SUCCESS(status)) goto bailout_ntstatus; @@ -2504,19 +2321,18 @@ retry_file_and_section: retry_mapview:; SIZE_T ViewSize = (flags & MDBX_RDONLY) ? size : limit; - status = NtMapViewOfSection( - map->section, GetCurrentProcess(), &map->base, - /* ZeroBits */ 0, - /* CommitSize */ 0, - /* SectionOffset */ nullptr, &ViewSize, - /* InheritDisposition */ ViewUnmap, - /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE, - /* Win32Protect */ - (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY); + status = NtMapViewOfSection(map->section, GetCurrentProcess(), &map->base, + /* ZeroBits */ 0, + /* CommitSize */ 0, + /* SectionOffset */ nullptr, &ViewSize, + /* InheritDisposition */ ViewUnmap, + /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE, + /* Win32Protect */ + (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY); if (!NT_SUCCESS(status)) { - if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018 && - map->base && (flags & MDBX_MRESIZE_MAY_MOVE) != 0) { + if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018 && map->base && + (flags & MDBX_MRESIZE_MAY_MOVE) != 0) { /* try remap at another base address */ map->base = nullptr; goto retry_mapview; @@ -2550,8 +2366,7 @@ retry_mapview:; rc = MDBX_EPERM; map->current = (map->filesize > limit) ? limit : (size_t)map->filesize; } else { - if (size > map->filesize || - (size < map->filesize && (flags & txn_shrink_allowed))) { + if (size > map->filesize || (size < map->filesize && (flags & txn_shrink_allowed))) { rc = osal_ftruncate(map->fd, size); VERBOSE("ftruncate %zu, err %d", size, rc); if (rc != MDBX_SUCCESS) @@ -2566,9 +2381,8 @@ retry_mapview:; * this region and (therefore) do not need the help of ASAN. * - this allows us to clear the mask only within the file size * when closing the mapping. */ - MDBX_ASAN_UNPOISON_MEMORY_REGION( - ptr_disp(map->base, size), - ((map->current < map->limit) ? map->current : map->limit) - size); + MDBX_ASAN_UNPOISON_MEMORY_REGION(ptr_disp(map->base, size), + ((map->current < map->limit) ? map->current : map->limit) - size); } map->current = (size < map->limit) ? size : map->limit; } @@ -2617,15 +2431,13 @@ retry_mapview:; #endif /* Linux & _GNU_SOURCE */ const unsigned mmap_flags = - MAP_CONCEAL | MAP_SHARED | MAP_FILE | MAP_NORESERVE | - (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0); - const unsigned mmap_prot = - (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ; + MAP_CONCEAL | MAP_SHARED | MAP_FILE | MAP_NORESERVE | (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0); + const unsigned mmap_prot = (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ; if (ptr == MAP_FAILED) { /* Try to mmap additional space beyond the end of mapping. */ - ptr = mmap(ptr_disp(map->base, map->limit), limit - map->limit, mmap_prot, - mmap_flags | MAP_FIXED_NOREPLACE, map->fd, map->limit); + ptr = mmap(ptr_disp(map->base, map->limit), limit - map->limit, mmap_prot, mmap_flags | MAP_FIXED_NOREPLACE, + map->fd, map->limit); if (ptr == ptr_disp(map->base, map->limit)) /* успешно прилепили отображение в конец */ ptr = map->base; @@ -2668,33 +2480,25 @@ retry_mapview:; // coverity[pass_freed_arg : FALSE] ptr = mmap(map->base, limit, mmap_prot, - (flags & MDBX_MRESIZE_MAY_MOVE) - ? mmap_flags - : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE - : MAP_FIXED), + (flags & MDBX_MRESIZE_MAY_MOVE) ? mmap_flags + : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE : MAP_FIXED), map->fd, 0); - if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED && - unlikely(ptr == MAP_FAILED) && !(flags & MDBX_MRESIZE_MAY_MOVE) && - errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL) + if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED && unlikely(ptr == MAP_FAILED) && + !(flags & MDBX_MRESIZE_MAY_MOVE) && errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL) // coverity[pass_freed_arg : FALSE] - ptr = - mmap(map->base, limit, mmap_prot, mmap_flags | MAP_FIXED, map->fd, 0); + ptr = mmap(map->base, limit, mmap_prot, mmap_flags | MAP_FIXED, map->fd, 0); if (unlikely(ptr == MAP_FAILED)) { /* try to restore prev mapping */ // coverity[pass_freed_arg : FALSE] ptr = mmap(map->base, map->limit, mmap_prot, - (flags & MDBX_MRESIZE_MAY_MOVE) - ? mmap_flags - : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE - : MAP_FIXED), + (flags & MDBX_MRESIZE_MAY_MOVE) ? mmap_flags + : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE : MAP_FIXED), map->fd, 0); - if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED && - unlikely(ptr == MAP_FAILED) && !(flags & MDBX_MRESIZE_MAY_MOVE) && - errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL) + if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED && unlikely(ptr == MAP_FAILED) && + !(flags & MDBX_MRESIZE_MAY_MOVE) && errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL) // coverity[pass_freed_arg : FALSE] - ptr = mmap(map->base, map->limit, mmap_prot, mmap_flags | MAP_FIXED, - map->fd, 0); + ptr = mmap(map->base, map->limit, mmap_prot, mmap_flags | MAP_FIXED, map->fd, 0); if (unlikely(ptr == MAP_FAILED)) { VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current); /* Unpoisoning is required for ASAN to avoid false-positive diagnostic @@ -2702,8 +2506,7 @@ retry_mapview:; * See * https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 */ - MDBX_ASAN_UNPOISON_MEMORY_REGION( - map->base, (map->current < map->limit) ? map->current : map->limit); + MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, (map->current < map->limit) ? map->current : map->limit); map->limit = 0; map->current = 0; map->base = nullptr; @@ -2723,8 +2526,7 @@ retry_mapview:; * See * https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 */ - MDBX_ASAN_UNPOISON_MEMORY_REGION( - map->base, (map->current < map->limit) ? map->current : map->limit); + MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, (map->current < map->limit) ? map->current : map->limit); VALGRIND_MAKE_MEM_DEFINED(ptr, map->current); MDBX_ASAN_UNPOISON_MEMORY_REGION(ptr, map->current); @@ -2747,10 +2549,8 @@ retry_mapview:; /* Zap: Redundant code */ MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6287); - assert(rc != MDBX_SUCCESS || - (map->base != nullptr && map->base != MAP_FAILED && - map->current == size && map->limit == limit && - map->filesize >= size)); + assert(rc != MDBX_SUCCESS || (map->base != nullptr && map->base != MAP_FAILED && map->current == size && + map->limit == limit && map->filesize >= size)); return rc; } @@ -2758,8 +2558,7 @@ retry_mapview:; __cold MDBX_INTERNAL void osal_jitter(bool tiny) { for (;;) { -#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \ - defined(__x86_64__) +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) unsigned salt = 5296013u * (unsigned)__rdtsc(); salt ^= salt >> 11; salt *= 25810541u; @@ -2782,9 +2581,8 @@ __cold MDBX_INTERNAL void osal_jitter(bool tiny) { timer = CreateWaitableTimer(NULL, TRUE, NULL); LARGE_INTEGER ft; - ft.QuadPart = - coin * (int64_t)-10; // Convert to 100 nanosecond interval, - // negative value indicates relative time. + ft.QuadPart = coin * (int64_t)-10; // Convert to 100 nanosecond interval, + // negative value indicates relative time. SetWaitableTimer(timer, &ft, 0, NULL, NULL, 0); WaitForSingleObject(timer, INFINITE); // CloseHandle(timer); @@ -2874,10 +2672,7 @@ MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults) { #if defined(_WIN32) || defined(_WIN64) if (optional_page_faults) { PROCESS_MEMORY_COUNTERS pmc; - *optional_page_faults = - GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc)) - ? pmc.PageFaultCount - : 0; + *optional_page_faults = GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc)) ? pmc.PageFaultCount : 0; } FILETIME unused, usermode; if (GetThreadTimes(GetCurrentThread(), @@ -2905,8 +2700,7 @@ MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults) { if (getrusage(RUSAGE_THREAD, &usage) == 0) { if (optional_page_faults) *optional_page_faults = usage.ru_majflt; - return usage.ru_utime.tv_sec * UINT64_C(1000000000) + - usage.ru_utime.tv_usec * 1000u; + return usage.ru_utime.tv_sec * UINT64_C(1000000000) + usage.ru_utime.tv_usec * 1000u; } if (optional_page_faults) *optional_page_faults = 0; @@ -3016,41 +2810,33 @@ __cold static uint64_t windows_bootime(void) { return 0; } -__cold static LSTATUS mdbx_RegGetValue(HKEY hKey, LPCSTR lpSubKey, - LPCSTR lpValue, PVOID pvData, - LPDWORD pcbData) { +__cold static LSTATUS mdbx_RegGetValue(HKEY hKey, LPCSTR lpSubKey, LPCSTR lpValue, PVOID pvData, LPDWORD pcbData) { LSTATUS rc; if (!imports.RegGetValueA) { /* an old Windows 2000/XP */ HKEY hSubKey; rc = RegOpenKeyA(hKey, lpSubKey, &hSubKey); if (rc == ERROR_SUCCESS) { - rc = - RegQueryValueExA(hSubKey, lpValue, nullptr, nullptr, pvData, pcbData); + rc = RegQueryValueExA(hSubKey, lpValue, nullptr, nullptr, pvData, pcbData); RegCloseKey(hSubKey); } return rc; } - rc = imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY, nullptr, + rc = imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY, nullptr, pvData, pcbData); + if (rc != ERROR_FILE_NOT_FOUND) + return rc; + + rc = imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY | 0x00010000 /* RRF_SUBKEY_WOW6464KEY */, nullptr, pvData, pcbData); if (rc != ERROR_FILE_NOT_FOUND) return rc; - - rc = imports.RegGetValueA(hKey, lpSubKey, lpValue, - RRF_RT_ANY | 0x00010000 /* RRF_SUBKEY_WOW6464KEY */, - nullptr, pvData, pcbData); - if (rc != ERROR_FILE_NOT_FOUND) - return rc; - return imports.RegGetValueA(hKey, lpSubKey, lpValue, - RRF_RT_ANY | - 0x00020000 /* RRF_SUBKEY_WOW6432KEY */, - nullptr, pvData, pcbData); + return imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY | 0x00020000 /* RRF_SUBKEY_WOW6432KEY */, nullptr, + pvData, pcbData); } #endif -__cold MDBX_MAYBE_UNUSED static bool -bootid_parse_uuid(bin128_t *s, const void *p, const size_t n) { +__cold MDBX_MAYBE_UNUSED static bool bootid_parse_uuid(bin128_t *s, const void *p, const size_t n) { if (n > 31) { unsigned bits = 0; for (unsigned i = 0; i < n; ++i) /* try parse an UUID in text form */ { @@ -3096,10 +2882,7 @@ __cold static bool proc_read_uuid(const char *path, bin128_t *target) { if (fd != -1) { struct statfs fs; char buf[42]; - const ssize_t len = - (fstatfs(fd, &fs) == 0 && fs.f_type == /* procfs */ 0x9FA0) - ? read(fd, buf, sizeof(buf)) - : -1; + const ssize_t len = (fstatfs(fd, &fs) == 0 && fs.f_type == /* procfs */ 0x9FA0) ? read(fd, buf, sizeof(buf)) : -1; const int err = close(fd); assert(err == 0); (void)err; @@ -3123,12 +2906,10 @@ __cold static bin128_t osal_bootid(void) { { char buf[42]; size_t len = sizeof(buf); - if (!sysctlbyname("kern.bootsessionuuid", buf, &len, nullptr, 0) && - bootid_parse_uuid(&uuid, buf, len)) + if (!sysctlbyname("kern.bootsessionuuid", buf, &len, nullptr, 0) && bootid_parse_uuid(&uuid, buf, len)) return uuid; -#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && \ - __MAC_OS_X_VERSION_MIN_REQUIRED > 1050 +#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED > 1050 uuid_t hostuuid; struct timespec wait = {0, 1000000000u / 42}; if (!gethostuuid(hostuuid, &wait)) @@ -3137,8 +2918,7 @@ __cold static bin128_t osal_bootid(void) { struct timeval boottime; len = sizeof(boottime); - if (!sysctlbyname("kern.boottime", &boottime, &len, nullptr, 0) && - len == sizeof(boottime) && boottime.tv_sec) + if (!sysctlbyname("kern.boottime", &boottime, &len, nullptr, 0) && len == sizeof(boottime) && boottime.tv_sec) got_boottime = true; } #endif /* Apple/Darwin */ @@ -3162,83 +2942,70 @@ __cold static bin128_t osal_bootid(void) { char DigitalProductId[248]; } buf; - static const char HKLM_MicrosoftCryptography[] = - "SOFTWARE\\Microsoft\\Cryptography"; + static const char HKLM_MicrosoftCryptography[] = "SOFTWARE\\Microsoft\\Cryptography"; DWORD len = sizeof(buf); /* Windows is madness and must die */ - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_MicrosoftCryptography, - "MachineGuid", &buf.MachineGuid, - &len) == ERROR_SUCCESS && + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_MicrosoftCryptography, "MachineGuid", &buf.MachineGuid, &len) == + ERROR_SUCCESS && len < sizeof(buf)) got_machineid = bootid_parse_uuid(&uuid, &buf.MachineGuid, len); if (!got_machineid) { /* again, Windows is madness */ - static const char HKLM_WindowsNT[] = - "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion"; - static const char HKLM_WindowsNT_DPK[] = - "SOFTWARE\\Microsoft\\Windows " - "NT\\CurrentVersion\\DefaultProductKey"; - static const char HKLM_WindowsNT_DPK2[] = - "SOFTWARE\\Microsoft\\Windows " - "NT\\CurrentVersion\\DefaultProductKey2"; + static const char HKLM_WindowsNT[] = "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion"; + static const char HKLM_WindowsNT_DPK[] = "SOFTWARE\\Microsoft\\Windows " + "NT\\CurrentVersion\\DefaultProductKey"; + static const char HKLM_WindowsNT_DPK2[] = "SOFTWARE\\Microsoft\\Windows " + "NT\\CurrentVersion\\DefaultProductKey2"; len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT, - "DigitalProductId", &buf.DigitalProductId, - &len) == ERROR_SUCCESS && + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT, "DigitalProductId", &buf.DigitalProductId, &len) == + ERROR_SUCCESS && len > 42 && len < sizeof(buf)) { bootid_collect(&uuid, &buf.DigitalProductId, len); got_machineid = true; } len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK, - "DigitalProductId", &buf.DigitalProductId, - &len) == ERROR_SUCCESS && + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK, "DigitalProductId", &buf.DigitalProductId, &len) == + ERROR_SUCCESS && len > 42 && len < sizeof(buf)) { bootid_collect(&uuid, &buf.DigitalProductId, len); got_machineid = true; } len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK2, - "DigitalProductId", &buf.DigitalProductId, - &len) == ERROR_SUCCESS && + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK2, "DigitalProductId", &buf.DigitalProductId, &len) == + ERROR_SUCCESS && len > 42 && len < sizeof(buf)) { bootid_collect(&uuid, &buf.DigitalProductId, len); got_machineid = true; } } - static const char HKLM_PrefetcherParams[] = - "SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Memory " - "Management\\PrefetchParameters"; + static const char HKLM_PrefetcherParams[] = "SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Memory " + "Management\\PrefetchParameters"; len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BootId", - &buf.BootId, &len) == ERROR_SUCCESS && + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BootId", &buf.BootId, &len) == ERROR_SUCCESS && len > 1 && len < sizeof(buf)) { bootid_collect(&uuid, &buf.BootId, len); got_bootseq = true; } len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BaseTime", - &buf.BaseTime, &len) == ERROR_SUCCESS && + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BaseTime", &buf.BaseTime, &len) == ERROR_SUCCESS && len >= sizeof(buf.BaseTime) && buf.BaseTime) { bootid_collect(&uuid, &buf.BaseTime, len); got_boottime = true; } /* BootTime from SYSTEM_TIMEOFDAY_INFORMATION */ - NTSTATUS status = NtQuerySystemInformation( - 0x03 /* SystemTmeOfDayInformation */, &buf.SysTimeOfDayInfo, - sizeof(buf.SysTimeOfDayInfo), &len); + NTSTATUS status = NtQuerySystemInformation(0x03 /* SystemTmeOfDayInformation */, &buf.SysTimeOfDayInfo, + sizeof(buf.SysTimeOfDayInfo), &len); if (NT_SUCCESS(status) && len >= offsetof(union buf, SysTimeOfDayInfoHacked.BootTimeBias) + sizeof(buf.SysTimeOfDayInfoHacked.BootTimeBias) && buf.SysTimeOfDayInfoHacked.BootTime.QuadPart) { const uint64_t UnbiasedBootTime = - buf.SysTimeOfDayInfoHacked.BootTime.QuadPart - - buf.SysTimeOfDayInfoHacked.BootTimeBias; + buf.SysTimeOfDayInfoHacked.BootTime.QuadPart - buf.SysTimeOfDayInfoHacked.BootTimeBias; if (UnbiasedBootTime) { bootid_collect(&uuid, &UnbiasedBootTime, sizeof(UnbiasedBootTime)); got_boottime = true; @@ -3384,13 +3151,11 @@ __cold static bin128_t osal_bootid(void) { if (0x1CCCCCC > now.dwHighDateTime) #else struct timespec mono, real; - if (clock_gettime(CLOCK_MONOTONIC, &mono) || - clock_gettime(CLOCK_REALTIME, &real) || + if (clock_gettime(CLOCK_MONOTONIC, &mono) || clock_gettime(CLOCK_REALTIME, &real) || /* wrong time, RTC is mad or absent */ 1555555555l > real.tv_sec || /* seems no adjustment by RTC/NTP, i.e. a fake time */ - real.tv_sec < mono.tv_sec || 1234567890l > real.tv_sec - mono.tv_sec || - (real.tv_sec - mono.tv_sec) % 900u == 0) + real.tv_sec < mono.tv_sec || 1234567890l > real.tv_sec - mono.tv_sec || (real.tv_sec - mono.tv_sec) % 900u == 0) #endif goto lack; } @@ -3398,8 +3163,7 @@ __cold static bin128_t osal_bootid(void) { return uuid; } -__cold int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, - intptr_t *avail_pages) { +__cold int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, intptr_t *avail_pages) { if (!page_size && !total_pages && !avail_pages) return LOG_IFERR(MDBX_EINVAL); if (total_pages) @@ -3437,8 +3201,7 @@ __cold int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, if (total_ram_Kb == -1) return LOG_IFERR(errno); const intptr_t total_ram_pages = (total_ram_Kb << 10) >> log2page; -#elif defined(HW_USERMEM) || defined(HW_PHYSMEM64) || defined(HW_MEMSIZE) || \ - defined(HW_PHYSMEM) +#elif defined(HW_USERMEM) || defined(HW_PHYSMEM64) || defined(HW_MEMSIZE) || defined(HW_PHYSMEM) size_t ram, len = sizeof(ram); static const int mib[] = {CTL_HW, #if defined(HW_USERMEM) @@ -3480,8 +3243,7 @@ __cold int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, mach_msg_type_number_t count = HOST_VM_INFO_COUNT; vm_statistics_data_t vmstat; mach_port_t mport = mach_host_self(); - kern_return_t kerr = host_statistics(mach_host_self(), HOST_VM_INFO, - (host_info_t)&vmstat, &count); + kern_return_t kerr = host_statistics(mach_host_self(), HOST_VM_INFO, (host_info_t)&vmstat, &count); mach_port_deallocate(mach_task_self(), mport); if (unlikely(kerr != KERN_SUCCESS)) return LOG_IFERR(MDBX_ENOSYS); @@ -3525,9 +3287,8 @@ __cold int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, #ifdef __IPHONE_OS_VERSION_MIN_REQUIRED #include -#elif __GLIBC_PREREQ(2, 25) || defined(__FreeBSD__) || defined(__NetBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || __has_include() +#elif __GLIBC_PREREQ(2, 25) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || __has_include() #include #endif /* sys/random.h */ @@ -3558,23 +3319,19 @@ MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *env) { #endif /* FreeBSD */ #if defined(_WIN32) || defined(_WIN64) - if (imports.CoCreateGuid && imports.CoCreateGuid(&uuid) == 0 && - check_uuid(uuid)) + if (imports.CoCreateGuid && imports.CoCreateGuid(&uuid) == 0 && check_uuid(uuid)) return uuid; HCRYPTPROV hCryptProv = 0; - if (CryptAcquireContextW(&hCryptProv, nullptr, nullptr, PROV_RSA_FULL, - CRYPT_VERIFYCONTEXT | CRYPT_SILENT)) { - const BOOL ok = - CryptGenRandom(hCryptProv, sizeof(uuid), (unsigned char *)&uuid); + if (CryptAcquireContextW(&hCryptProv, nullptr, nullptr, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT | CRYPT_SILENT)) { + const BOOL ok = CryptGenRandom(hCryptProv, sizeof(uuid), (unsigned char *)&uuid); CryptReleaseContext(hCryptProv, 0); if (ok && check_uuid(uuid)) return uuid; } #elif defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && defined(__IPHONE_8_0) #if __IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_8_0 - if (CCRandomGenerateBytes(&uuid, sizeof(uuid)) == kCCSuccess && - check_uuid(uuid)) + if (CCRandomGenerateBytes(&uuid, sizeof(uuid)) == kCCSuccess && check_uuid(uuid)) return uuid; #endif /* iOS >= 8.x */ #else @@ -3587,14 +3344,13 @@ MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *env) { if (len == sizeof(uuid) && check_uuid(uuid)) return uuid; } -#if (__GLIBC_PREREQ(2, 25) || defined(__FreeBSD__) || defined(__NetBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__)) && \ +#if (__GLIBC_PREREQ(2, 25) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__)) && \ !defined(__APPLE__) && !defined(__ANDROID_API__) if (getrandom(&uuid, sizeof(uuid), 0) == sizeof(uuid) && check_uuid(uuid)) return uuid; -#elif defined(__OpenBSD__) || (defined(__sun) && defined(__SVR4)) || \ - (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && \ - __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200) +#elif defined(__OpenBSD__) || (defined(__sun) && defined(__SVR4)) || \ + (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200) if (getentropy(&uuid, sizeof(uuid)) == 0 && check_uuid(uuid)) return uuid; #endif /* getrandom() / getentropy() */ @@ -3633,13 +3389,11 @@ void osal_ctor(void) { #else globals.sys_pagesize = sysconf(_SC_PAGE_SIZE); globals.sys_allocation_granularity = (MDBX_WORDBITS > 32) ? 65536 : 4096; - globals.sys_allocation_granularity = - (globals.sys_allocation_granularity > globals.sys_pagesize) - ? globals.sys_allocation_granularity - : globals.sys_pagesize; + globals.sys_allocation_granularity = (globals.sys_allocation_granularity > globals.sys_pagesize) + ? globals.sys_allocation_granularity + : globals.sys_pagesize; #endif - assert(globals.sys_pagesize > 0 && - (globals.sys_pagesize & (globals.sys_pagesize - 1)) == 0); + assert(globals.sys_pagesize > 0 && (globals.sys_pagesize & (globals.sys_pagesize - 1)) == 0); assert(globals.sys_allocation_granularity >= globals.sys_pagesize && globals.sys_allocation_granularity % globals.sys_pagesize == 0); globals.sys_pagesize_ln2 = log2n_powerof2(globals.sys_pagesize); @@ -3659,12 +3413,10 @@ void osal_ctor(void) { uint32_t proba = UINT32_MAX; while (true) { - unsigned time_conversion_checkup = - osal_monotime_to_16dot16(osal_16dot16_to_monotime(proba)); + unsigned time_conversion_checkup = osal_monotime_to_16dot16(osal_16dot16_to_monotime(proba)); unsigned one_more = (proba < UINT32_MAX) ? proba + 1 : proba; unsigned one_less = (proba > 0) ? proba - 1 : proba; - ENSURE(nullptr, time_conversion_checkup >= one_less && - time_conversion_checkup <= one_more); + ENSURE(nullptr, time_conversion_checkup >= one_less && time_conversion_checkup <= one_more); if (proba == 0) break; proba >>= 1; diff --git a/src/osal.h b/src/osal.h index 813baa9a..5a049a24 100644 --- a/src/osal.h +++ b/src/osal.h @@ -12,9 +12,8 @@ #if __has_include() #include -#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS should have explicit cache control */ #include #endif @@ -28,11 +27,9 @@ MDBX_MAYBE_UNUSED static inline void osal_compiler_barrier(void) { __memory_barrier(); #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __compiler_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __fence(); #else #error "Could not guess the kind of compiler, please report to us." @@ -60,11 +57,9 @@ MDBX_MAYBE_UNUSED static inline void osal_memory_barrier(void) { #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __machine_rw_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_mf(); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __lwsync(); #else #error "Could not guess the kind of compiler, please report to us." @@ -101,9 +96,7 @@ typedef CRITICAL_SECTION osal_fastmutex_t; #if MDBX_WITHOUT_MSVC_CRT #ifndef osal_malloc -static inline void *osal_malloc(size_t bytes) { - return HeapAlloc(GetProcessHeap(), 0, bytes); -} +static inline void *osal_malloc(size_t bytes) { return HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_malloc */ #ifndef osal_calloc @@ -114,8 +107,7 @@ static inline void *osal_calloc(size_t nelem, size_t size) { #ifndef osal_realloc static inline void *osal_realloc(void *ptr, size_t bytes) { - return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) - : HeapAlloc(GetProcessHeap(), 0, bytes); + return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) : HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_realloc */ @@ -208,7 +200,7 @@ typedef struct osal_mmap { #elif defined(__APPLE__) || defined(__MACH__) || defined(_DARWIN_C_SOURCE) -#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ +#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0 /* FIXME: add checks for IOS versions, etc */ #define MDBX_HAVE_PWRITEV 1 @@ -279,39 +271,29 @@ typedef struct osal_ioring { MDBX_INTERNAL int osal_ioring_create(osal_ioring_t * #if defined(_WIN32) || defined(_WIN64) , - bool enable_direct, - mdbx_filehandle_t overlapped_fd + bool enable_direct, mdbx_filehandle_t overlapped_fd #endif /* Windows */ ); MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *, size_t items); MDBX_INTERNAL void osal_ioring_destroy(osal_ioring_t *); MDBX_INTERNAL void osal_ioring_reset(osal_ioring_t *); -MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, - void *data, const size_t bytes); +MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, void *data, const size_t bytes); typedef struct osal_ioring_write_result { int err; unsigned wops; } osal_ioring_write_result_t; -MDBX_INTERNAL osal_ioring_write_result_t -osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); +MDBX_INTERNAL osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx, - void (*callback)(iov_ctx_t *ctx, - size_t offset, void *data, - size_t bytes)); + void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_left(const osal_ioring_t *ior) { - return ior->slots_left; -} +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_left(const osal_ioring_t *ior) { return ior->slots_left; } -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_used(const osal_ioring_t *ior) { +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_used(const osal_ioring_t *ior) { return ior->allocated - ior->slots_left; } -MDBX_MAYBE_UNUSED static inline int -osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { +MDBX_MAYBE_UNUSED static inline int osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { items = (items > 32) ? items : 32; #if defined(_WIN32) || defined(_WIN64) if (ior->direct) { @@ -330,13 +312,11 @@ osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { /*----------------------------------------------------------------------------*/ /* libc compatibility stuff */ -#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ - (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) #define osal_asprintf asprintf #define osal_vasprintf vasprintf #else -MDBX_MAYBE_UNUSED MDBX_INTERNAL - MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); +MDBX_MAYBE_UNUSED MDBX_INTERNAL MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); MDBX_INTERNAL int osal_vasprintf(char **strp, const char *fmt, va_list ap); #endif @@ -358,14 +338,12 @@ MDBX_MAYBE_UNUSED MDBX_INTERNAL void osal_jitter(bool tiny); #else #define MAX_WRITE UINT32_C(0x3f000000) -#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && \ - !defined(__ANDROID_API__) +#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && !defined(__ANDROID_API__) #define MDBX_F_SETLK F_SETLK64 #define MDBX_F_SETLKW F_SETLKW64 #define MDBX_F_GETLK F_GETLK64 -#if (__GLIBC_PREREQ(2, 28) && \ - (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ - defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ +#if (__GLIBC_PREREQ(2, 28) && (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ + defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ defined(fcntl64) #define MDBX_FCNTL fcntl64 #else @@ -383,8 +361,7 @@ MDBX_MAYBE_UNUSED MDBX_INTERNAL void osal_jitter(bool tiny); #define MDBX_STRUCT_FLOCK struct flock #endif /* MDBX_F_SETLK, MDBX_F_SETLKW, MDBX_F_GETLK */ -#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) +#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) #define MDBX_F_OFD_SETLK F_OFD_SETLK64 #define MDBX_F_OFD_SETLKW F_OFD_SETLKW64 #define MDBX_F_OFD_GETLK F_OFD_GETLK64 @@ -393,8 +370,7 @@ MDBX_MAYBE_UNUSED MDBX_INTERNAL void osal_jitter(bool tiny); #define MDBX_F_OFD_SETLKW F_OFD_SETLKW #define MDBX_F_OFD_GETLK F_OFD_GETLK #ifndef OFF_T_MAX -#define OFF_T_MAX \ - (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) +#define OFF_T_MAX (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) #endif /* OFF_T_MAX */ #endif /* MDBX_F_OFD_SETLK64, MDBX_F_OFD_SETLKW64, MDBX_F_OFD_GETLK64 */ @@ -414,8 +390,7 @@ MDBX_MAYBE_UNUSED static inline int osal_get_errno(void) { } #ifndef osal_memalign_alloc -MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, - void **result); +MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, void **result); #endif #ifndef osal_memalign_free MDBX_INTERNAL void osal_memalign_free(void *ptr); @@ -433,19 +408,13 @@ MDBX_INTERNAL int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); MDBX_INTERNAL int osal_fastmutex_release(osal_fastmutex_t *fastmutex); MDBX_INTERNAL int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, - size_t sgvcnt, uint64_t offset); -MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, - uint64_t offset); -MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, - size_t count, uint64_t offset); -MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, - size_t count); +MDBX_INTERNAL int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset); +MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, size_t count); -MDBX_INTERNAL int -osal_thread_create(osal_thread_t *thread, - THREAD_RESULT(THREAD_CALL *start_routine)(void *), - void *arg); +MDBX_INTERNAL int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); MDBX_INTERNAL int osal_thread_join(osal_thread_t thread); enum osal_syncmode_bits { @@ -456,8 +425,7 @@ enum osal_syncmode_bits { MDBX_SYNC_IODQ = 8 }; -MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, - const enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, const enum osal_syncmode_bits mode_bits); MDBX_INTERNAL int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); MDBX_INTERNAL int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); MDBX_INTERNAL int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); @@ -483,14 +451,11 @@ MDBX_MAYBE_UNUSED static inline bool osal_isdirsep(pathchar_t c) { c == '/'; } -MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, - size_t len); +MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len); MDBX_INTERNAL pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len); MDBX_INTERNAL int osal_fileexists(const pathchar_t *pathname); -MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, - const MDBX_env *env, const pathchar_t *pathname, - mdbx_filehandle_t *fd, - mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env, + const pathchar_t *pathname, mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits); MDBX_INTERNAL int osal_closefile(mdbx_filehandle_t fd); MDBX_INTERNAL int osal_removefile(const pathchar_t *pathname); MDBX_INTERNAL int osal_removedirectory(const pathchar_t *pathname); @@ -499,26 +464,21 @@ MDBX_INTERNAL int osal_lockfile(mdbx_filehandle_t fd, bool wait); #define MMAP_OPTION_TRUNCATE 1 #define MMAP_OPTION_SEMAPHORE 2 -MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, - const size_t limit, const unsigned options); +MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, const unsigned options); MDBX_INTERNAL int osal_munmap(osal_mmap_t *map); #define MDBX_MRESIZE_MAY_MOVE 0x00000100 #define MDBX_MRESIZE_MAY_UNMAP 0x00000200 -MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, - size_t limit); +MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit); #if defined(_WIN32) || defined(_WIN64) typedef struct { unsigned limit, count; HANDLE handles[31]; } mdbx_handle_array_t; -MDBX_INTERNAL int -osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL int osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); MDBX_INTERNAL int osal_resume_threads_after_remap(mdbx_handle_array_t *array); #endif /* Windows */ -MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, - size_t length, enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, - const pathchar_t *pathname, int err); +MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, size_t length, enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err); MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle); MDBX_MAYBE_UNUSED static inline uint32_t osal_getpid(void) { @@ -549,8 +509,7 @@ MDBX_INTERNAL int osal_check_tid4bionic(void); static inline int osal_check_tid4bionic(void) { return 0; } #endif /* __ANDROID_API__ || ANDROID) || BIONIC */ -MDBX_MAYBE_UNUSED static inline int -osal_pthread_mutex_lock(pthread_mutex_t *mutex) { +MDBX_MAYBE_UNUSED static inline int osal_pthread_mutex_lock(pthread_mutex_t *mutex) { int err = osal_check_tid4bionic(); return unlikely(err) ? err : pthread_mutex_lock(mutex); } @@ -561,8 +520,7 @@ MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults); MDBX_INTERNAL uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); MDBX_INTERNAL uint32_t osal_monotime_to_16dot16(uint64_t monotime); -MDBX_MAYBE_UNUSED static inline uint32_t -osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { +MDBX_MAYBE_UNUSED static inline uint32_t osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { uint32_t seconds_16dot16 = osal_monotime_to_16dot16(monotime); return seconds_16dot16 ? seconds_16dot16 : /* fix underflow */ (monotime > 0); } @@ -589,10 +547,8 @@ MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *); /*----------------------------------------------------------------------------*/ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t -osal_bswap64(uint64_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap64) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t osal_bswap64(uint64_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap64) return __builtin_bswap64(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_uint64(v); @@ -601,19 +557,14 @@ osal_bswap64(uint64_t v) { #elif defined(bswap_64) return bswap_64(v); #else - return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | - ((v << 24) & UINT64_C(0x0000ff0000000000)) | - ((v << 8) & UINT64_C(0x000000ff00000000)) | - ((v >> 8) & UINT64_C(0x00000000ff000000)) | - ((v >> 24) & UINT64_C(0x0000000000ff0000)) | - ((v >> 40) & UINT64_C(0x000000000000ff00)); + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | ((v << 24) & UINT64_C(0x0000ff0000000000)) | + ((v << 8) & UINT64_C(0x000000ff00000000)) | ((v >> 8) & UINT64_C(0x00000000ff000000)) | + ((v >> 24) & UINT64_C(0x0000000000ff0000)) | ((v >> 40) & UINT64_C(0x000000000000ff00)); #endif } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t -osal_bswap32(uint32_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap32) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t osal_bswap32(uint32_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap32) return __builtin_bswap32(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_ulong(v); @@ -622,7 +573,6 @@ osal_bswap32(uint32_t v) { #elif defined(bswap_32) return bswap_32(v); #else - return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | - ((v >> 8) & UINT32_C(0x0000ff00)); + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | ((v >> 8) & UINT32_C(0x0000ff00)); #endif } diff --git a/src/page-get.c b/src/page-get.c index 57f2e177..a07768a7 100644 --- a/src/page-get.c +++ b/src/page-get.c @@ -3,17 +3,14 @@ #include "internals.h" -__cold int MDBX_PRINTF_ARGS(2, 3) - bad_page(const page_t *mp, const char *fmt, ...) { +__cold int MDBX_PRINTF_ARGS(2, 3) bad_page(const page_t *mp, const char *fmt, ...) { if (LOG_ENABLED(MDBX_LOG_ERROR)) { static const page_t *prev; if (prev != mp) { char buf4unknown[16]; prev = mp; - debug_log(MDBX_LOG_ERROR, "badpage", 0, - "corrupted %s-page #%u, mod-txnid %" PRIaTXN "\n", - pagetype_caption(page_type(mp), buf4unknown), mp->pgno, - mp->txnid); + debug_log(MDBX_LOG_ERROR, "badpage", 0, "corrupted %s-page #%u, mod-txnid %" PRIaTXN "\n", + pagetype_caption(page_type(mp), buf4unknown), mp->pgno, mp->txnid); } va_list args; @@ -24,17 +21,14 @@ __cold int MDBX_PRINTF_ARGS(2, 3) return MDBX_CORRUPTED; } -__cold void MDBX_PRINTF_ARGS(2, 3) - poor_page(const page_t *mp, const char *fmt, ...) { +__cold void MDBX_PRINTF_ARGS(2, 3) poor_page(const page_t *mp, const char *fmt, ...) { if (LOG_ENABLED(MDBX_LOG_NOTICE)) { static const page_t *prev; if (prev != mp) { char buf4unknown[16]; prev = mp; - debug_log(MDBX_LOG_NOTICE, "poorpage", 0, - "suboptimal %s-page #%u, mod-txnid %" PRIaTXN "\n", - pagetype_caption(page_type(mp), buf4unknown), mp->pgno, - mp->txnid); + debug_log(MDBX_LOG_NOTICE, "poorpage", 0, "suboptimal %s-page #%u, mod-txnid %" PRIaTXN "\n", + pagetype_caption(page_type(mp), buf4unknown), mp->pgno, mp->txnid); } va_list args; @@ -63,21 +57,17 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) { const ptrdiff_t offset = ptr_dist(mp, env->dxb_mmap.base); unsigned flags_mask = P_ILL_BITS; unsigned flags_expected = 0; - if (offset < 0 || - offset > (ptrdiff_t)(pgno2bytes(env, mc->txn->geo.first_unallocated) - - ((mp->flags & P_SUBP) ? PAGEHDRSZ + 1 : env->ps))) { + if (offset < 0 || offset > (ptrdiff_t)(pgno2bytes(env, mc->txn->geo.first_unallocated) - + ((mp->flags & P_SUBP) ? PAGEHDRSZ + 1 : env->ps))) { /* should be dirty page without MDBX_WRITEMAP, or a subpage of. */ flags_mask -= P_SUBP; - if ((env->flags & MDBX_WRITEMAP) != 0 || - (!is_shadowed(mc->txn, mp) && !(mp->flags & P_SUBP))) - rc = bad_page(mp, "invalid page-address %p, offset %zi\n", - __Wpedantic_format_voidptr(mp), offset); + if ((env->flags & MDBX_WRITEMAP) != 0 || (!is_shadowed(mc->txn, mp) && !(mp->flags & P_SUBP))) + rc = bad_page(mp, "invalid page-address %p, offset %zi\n", __Wpedantic_format_voidptr(mp), offset); } else if (offset & (env->ps - 1)) flags_expected = P_SUBP; if (unlikely((mp->flags & flags_mask) != flags_expected)) - rc = bad_page(mp, "unknown/extra page-flags (have 0x%x, expect 0x%x)\n", - mp->flags & flags_mask, flags_expected); + rc = bad_page(mp, "unknown/extra page-flags (have 0x%x, expect 0x%x)\n", mp->flags & flags_mask, flags_expected); cASSERT(mc, (mc->checking & z_dupfix) == 0 || (mc->flags & z_inner) != 0); const uint8_t type = page_type(mp); @@ -86,82 +76,62 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) { return bad_page(mp, "invalid type (%u)\n", type); case P_LARGE: if (unlikely(mc->flags & z_inner)) - rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "large", - "nested dupsort tree", mc->tree->flags); + rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "large", "nested dupsort tree", mc->tree->flags); const pgno_t npages = mp->pages; if (unlikely(npages < 1 || npages >= MAX_PAGENO / 2)) rc = bad_page(mp, "invalid n-pages (%u) for large-page\n", npages); if (unlikely(mp->pgno + npages > mc->txn->geo.first_unallocated)) - rc = bad_page( - mp, "end of large-page beyond (%u) allocated space (%u next-pgno)\n", - mp->pgno + npages, mc->txn->geo.first_unallocated); + rc = bad_page(mp, "end of large-page beyond (%u) allocated space (%u next-pgno)\n", mp->pgno + npages, + mc->txn->geo.first_unallocated); return rc; //-------------------------- end of large/overflow page handling case P_LEAF | P_SUBP: if (unlikely(mc->tree->height != 1)) - rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", - "leaf-sub", "nested dupsort db", mc->tree->flags); + rc = + bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "leaf-sub", "nested dupsort db", mc->tree->flags); /* fall through */ __fallthrough; case P_LEAF: if (unlikely((mc->checking & z_dupfix) != 0)) - rc = bad_page(mp, - "unexpected leaf-page for dupfix subtree (db-lags 0x%x)\n", - mc->tree->flags); + rc = bad_page(mp, "unexpected leaf-page for dupfix subtree (db-lags 0x%x)\n", mc->tree->flags); break; case P_LEAF | P_DUPFIX | P_SUBP: if (unlikely(mc->tree->height != 1)) - rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", - "leaf2-sub", "nested dupsort db", mc->tree->flags); + rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "leaf2-sub", "nested dupsort db", + mc->tree->flags); /* fall through */ __fallthrough; case P_LEAF | P_DUPFIX: if (unlikely((mc->checking & z_dupfix) == 0)) - rc = bad_page( - mp, - "unexpected leaf2-page for non-dupfix (sub)tree (db-flags 0x%x)\n", - mc->tree->flags); + rc = bad_page(mp, "unexpected leaf2-page for non-dupfix (sub)tree (db-flags 0x%x)\n", mc->tree->flags); break; case P_BRANCH: break; } - if (unlikely(mp->upper < mp->lower || (mp->lower & 1) || - PAGEHDRSZ + mp->upper > env->ps)) - rc = bad_page(mp, "invalid page lower(%u)/upper(%u) with limit %zu\n", - mp->lower, mp->upper, page_space(env)); + if (unlikely(mp->upper < mp->lower || (mp->lower & 1) || PAGEHDRSZ + mp->upper > env->ps)) + rc = bad_page(mp, "invalid page lower(%u)/upper(%u) with limit %zu\n", mp->lower, mp->upper, page_space(env)); const char *const end_of_page = ptr_disp(mp, env->ps); const size_t nkeys = page_numkeys(mp); STATIC_ASSERT(P_BRANCH == 1); if (unlikely(nkeys <= (uint8_t)(mp->flags & P_BRANCH))) { if ((!(mc->flags & z_inner) || mc->tree->items) && - (!(mc->checking & z_updating) || - !(is_modifable(mc->txn, mp) || (mp->flags & P_SUBP)))) - rc = - bad_page(mp, "%s-page nkeys (%zu) < %u\n", - is_branch(mp) ? "branch" : "leaf", nkeys, 1 + is_branch(mp)); + (!(mc->checking & z_updating) || !(is_modifable(mc->txn, mp) || (mp->flags & P_SUBP)))) + rc = bad_page(mp, "%s-page nkeys (%zu) < %u\n", is_branch(mp) ? "branch" : "leaf", nkeys, 1 + is_branch(mp)); } const size_t ksize_max = keysize_max(env->ps, 0); const size_t leaf2_ksize = mp->dupfix_ksize; if (is_dupfix_leaf(mp)) { - if (unlikely((mc->flags & z_inner) == 0 || - (mc->tree->flags & MDBX_DUPFIXED) == 0)) - rc = bad_page(mp, "unexpected leaf2-page (db-flags 0x%x)\n", - mc->tree->flags); + if (unlikely((mc->flags & z_inner) == 0 || (mc->tree->flags & MDBX_DUPFIXED) == 0)) + rc = bad_page(mp, "unexpected leaf2-page (db-flags 0x%x)\n", mc->tree->flags); else if (unlikely(leaf2_ksize != mc->tree->dupfix_size)) rc = bad_page(mp, "invalid leaf2_ksize %zu\n", leaf2_ksize); else if (unlikely(((leaf2_ksize & nkeys) ^ mp->upper) & 1)) - rc = bad_page( - mp, "invalid page upper (%u) for nkeys %zu with leaf2-length %zu\n", - mp->upper, nkeys, leaf2_ksize); + rc = bad_page(mp, "invalid page upper (%u) for nkeys %zu with leaf2-length %zu\n", mp->upper, nkeys, leaf2_ksize); } else { - if (unlikely((mp->upper & 1) || - PAGEHDRSZ + mp->upper + nkeys * sizeof(node_t) + nkeys - 1 > - env->ps)) - rc = - bad_page(mp, "invalid page upper (%u) for nkeys %zu with limit %zu\n", - mp->upper, nkeys, page_space(env)); + if (unlikely((mp->upper & 1) || PAGEHDRSZ + mp->upper + nkeys * sizeof(node_t) + nkeys - 1 > env->ps)) + rc = bad_page(mp, "invalid page upper (%u) for nkeys %zu with limit %zu\n", mp->upper, nkeys, page_space(env)); } MDBX_val here, prev = {0, 0}; @@ -170,17 +140,14 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) { if (is_dupfix_leaf(mp)) { const char *const key = page_dupfix_ptr(mp, i, mc->tree->dupfix_size); if (unlikely(end_of_page < key + leaf2_ksize)) { - rc = bad_page(mp, "leaf2-item beyond (%zu) page-end\n", - key + leaf2_ksize - end_of_page); + rc = bad_page(mp, "leaf2-item beyond (%zu) page-end\n", key + leaf2_ksize - end_of_page); continue; } if (unlikely(leaf2_ksize != mc->clc->k.lmin)) { - if (unlikely(leaf2_ksize < mc->clc->k.lmin || - leaf2_ksize > mc->clc->k.lmax)) - rc = bad_page(mp, - "leaf2-item size (%zu) <> min/max length (%zu/%zu)\n", - leaf2_ksize, mc->clc->k.lmin, mc->clc->k.lmax); + if (unlikely(leaf2_ksize < mc->clc->k.lmin || leaf2_ksize > mc->clc->k.lmax)) + rc = bad_page(mp, "leaf2-item size (%zu) <> min/max length (%zu/%zu)\n", leaf2_ksize, mc->clc->k.lmin, + mc->clc->k.lmax); else mc->clc->k.lmin = mc->clc->k.lmax = leaf2_ksize; } @@ -188,16 +155,14 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) { here.iov_base = (void *)key; here.iov_len = leaf2_ksize; if (prev.iov_base && unlikely(mc->clc->k.cmp(&prev, &here) >= 0)) - rc = bad_page(mp, "leaf2-item #%zu wrong order (%s >= %s)\n", i, - DKEY(&prev), DVAL(&here)); + rc = bad_page(mp, "leaf2-item #%zu wrong order (%s >= %s)\n", i, DKEY(&prev), DVAL(&here)); prev = here; } } else { const node_t *const node = page_node(mp, i); const char *const node_end = ptr_disp(node, NODESIZE); if (unlikely(node_end > end_of_page)) { - rc = bad_page(mp, "node[%zu] (%zu) beyond page-end\n", i, - node_end - end_of_page); + rc = bad_page(mp, "node[%zu] (%zu) beyond page-end\n", i, node_end - end_of_page); continue; } const size_t ksize = node_ks(node); @@ -205,44 +170,36 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) { rc = bad_page(mp, "node[%zu] too long key (%zu)\n", i, ksize); const char *const key = node_key(node); if (unlikely(end_of_page < key + ksize)) { - rc = bad_page(mp, "node[%zu] key (%zu) beyond page-end\n", i, - key + ksize - end_of_page); + rc = bad_page(mp, "node[%zu] key (%zu) beyond page-end\n", i, key + ksize - end_of_page); continue; } if ((is_leaf(mp) || i > 0)) { if (unlikely(ksize < mc->clc->k.lmin || ksize > mc->clc->k.lmax)) - rc = bad_page( - mp, "node[%zu] key size (%zu) <> min/max key-length (%zu/%zu)\n", - i, ksize, mc->clc->k.lmin, mc->clc->k.lmax); + rc = bad_page(mp, "node[%zu] key size (%zu) <> min/max key-length (%zu/%zu)\n", i, ksize, mc->clc->k.lmin, + mc->clc->k.lmax); if ((mc->checking & z_ignord) == 0) { here.iov_base = (void *)key; here.iov_len = ksize; if (prev.iov_base && unlikely(mc->clc->k.cmp(&prev, &here) >= 0)) - rc = bad_page(mp, "node[%zu] key wrong order (%s >= %s)\n", i, - DKEY(&prev), DVAL(&here)); + rc = bad_page(mp, "node[%zu] key wrong order (%s >= %s)\n", i, DKEY(&prev), DVAL(&here)); prev = here; } } if (is_branch(mp)) { if ((mc->checking & z_updating) == 0 && i == 0 && unlikely(ksize != 0)) - rc = bad_page(mp, "branch-node[%zu] wrong 0-node key-length (%zu)\n", - i, ksize); + rc = bad_page(mp, "branch-node[%zu] wrong 0-node key-length (%zu)\n", i, ksize); const pgno_t ref = node_pgno(node); - if (unlikely(ref < MIN_PAGENO) || - (unlikely(ref >= mc->txn->geo.first_unallocated) && - (unlikely(ref >= mc->txn->geo.now) || - !(mc->checking & z_retiring)))) + if (unlikely(ref < MIN_PAGENO) || (unlikely(ref >= mc->txn->geo.first_unallocated) && + (unlikely(ref >= mc->txn->geo.now) || !(mc->checking & z_retiring)))) rc = bad_page(mp, "branch-node[%zu] wrong pgno (%u)\n", i, ref); if (unlikely(node_flags(node))) - rc = bad_page(mp, "branch-node[%zu] wrong flags (%u)\n", i, - node_flags(node)); + rc = bad_page(mp, "branch-node[%zu] wrong flags (%u)\n", i, node_flags(node)); continue; } switch (node_flags(node)) { default: - rc = - bad_page(mp, "invalid node[%zu] flags (%u)\n", i, node_flags(node)); + rc = bad_page(mp, "invalid node[%zu] flags (%u)\n", i, node_flags(node)); break; case N_BIG /* data on large-page */: case 0 /* usual */: @@ -256,46 +213,36 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) { const char *const data = node_data(node); if (node_flags(node) & N_BIG) { if (unlikely(end_of_page < data + sizeof(pgno_t))) { - rc = bad_page( - mp, "node-%s(%zu of %zu, %zu bytes) beyond (%zu) page-end\n", - "bigdata-pgno", i, nkeys, dsize, data + dsize - end_of_page); + rc = bad_page(mp, "node-%s(%zu of %zu, %zu bytes) beyond (%zu) page-end\n", "bigdata-pgno", i, nkeys, dsize, + data + dsize - end_of_page); continue; } if (unlikely(dsize <= v_clc.lmin || dsize > v_clc.lmax)) - rc = bad_page( - mp, - "big-node data size (%zu) <> min/max value-length (%zu/%zu)\n", - dsize, v_clc.lmin, v_clc.lmax); - if (unlikely(node_size_len(node_ks(node), dsize) <= - mc->txn->env->leaf_nodemax) && + rc = bad_page(mp, "big-node data size (%zu) <> min/max value-length (%zu/%zu)\n", dsize, v_clc.lmin, + v_clc.lmax); + if (unlikely(node_size_len(node_ks(node), dsize) <= mc->txn->env->leaf_nodemax) && mc->tree != &mc->txn->dbs[FREE_DBI]) poor_page(mp, "too small data (%zu bytes) for bigdata-node", dsize); if ((mc->checking & z_retiring) == 0) { - const pgr_t lp = - page_get_large(mc, node_largedata_pgno(node), mp->txnid); + const pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->txnid); if (unlikely(lp.err != MDBX_SUCCESS)) return lp.err; cASSERT(mc, page_type(lp.page) == P_LARGE); const unsigned npages = largechunk_npages(env, dsize); if (unlikely(lp.page->pages != npages)) { if (lp.page->pages < npages) - rc = bad_page(lp.page, - "too less n-pages %u for bigdata-node (%zu bytes)", - lp.page->pages, dsize); + rc = bad_page(lp.page, "too less n-pages %u for bigdata-node (%zu bytes)", lp.page->pages, dsize); else if (mc->tree != &mc->txn->dbs[FREE_DBI]) - poor_page(lp.page, - "extra n-pages %u for bigdata-node (%zu bytes)", - lp.page->pages, dsize); + poor_page(lp.page, "extra n-pages %u for bigdata-node (%zu bytes)", lp.page->pages, dsize); } } continue; } if (unlikely(end_of_page < data + dsize)) { - rc = bad_page(mp, - "node-%s(%zu of %zu, %zu bytes) beyond (%zu) page-end\n", - "data", i, nkeys, dsize, data + dsize - end_of_page); + rc = bad_page(mp, "node-%s(%zu of %zu, %zu bytes) beyond (%zu) page-end\n", "data", i, nkeys, dsize, + data + dsize - end_of_page); continue; } @@ -305,9 +252,7 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) { continue; case 0 /* usual */: if (unlikely(dsize < v_clc.lmin || dsize > v_clc.lmax)) { - rc = bad_page( - mp, "node-data size (%zu) <> min/max value-length (%zu/%zu)\n", - dsize, v_clc.lmin, v_clc.lmax); + rc = bad_page(mp, "node-data size (%zu) <> min/max value-length (%zu/%zu)\n", dsize, v_clc.lmin, v_clc.lmax); continue; } break; @@ -319,15 +264,13 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) { break; case N_TREE | N_DUP /* dupsorted sub-tree */: if (unlikely(dsize != sizeof(tree_t))) { - rc = bad_page(mp, "invalid nested-db record size (%zu, expect %zu)\n", - dsize, sizeof(tree_t)); + rc = bad_page(mp, "invalid nested-db record size (%zu, expect %zu)\n", dsize, sizeof(tree_t)); continue; } break; case N_DUP /* short sub-page */: if (unlikely(dsize <= PAGEHDRSZ)) { - rc = bad_page(mp, "invalid nested/sub-page record size (%zu)\n", - dsize); + rc = bad_page(mp, "invalid nested/sub-page record size (%zu)\n", dsize); continue; } else { const page_t *const sp = (page_t *)data; @@ -337,28 +280,23 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) { case P_LEAF | P_DUPFIX | P_SUBP: break; default: - rc = bad_page(mp, "invalid nested/sub-page flags (0x%02x)\n", - sp->flags); + rc = bad_page(mp, "invalid nested/sub-page flags (0x%02x)\n", sp->flags); continue; } const char *const end_of_subpage = data + dsize; const intptr_t nsubkeys = page_numkeys(sp); - if (unlikely(nsubkeys == 0) && !(mc->checking & z_updating) && - mc->tree->items) - rc = bad_page(mp, "no keys on a %s-page\n", - is_dupfix_leaf(sp) ? "leaf2-sub" : "leaf-sub"); + if (unlikely(nsubkeys == 0) && !(mc->checking & z_updating) && mc->tree->items) + rc = bad_page(mp, "no keys on a %s-page\n", is_dupfix_leaf(sp) ? "leaf2-sub" : "leaf-sub"); MDBX_val sub_here, sub_prev = {0, 0}; for (int ii = 0; ii < nsubkeys; ii++) { if (is_dupfix_leaf(sp)) { /* DUPFIX pages have no entries[] or node headers */ const size_t sub_ksize = sp->dupfix_ksize; - const char *const sub_key = - page_dupfix_ptr(sp, ii, mc->tree->dupfix_size); + const char *const sub_key = page_dupfix_ptr(sp, ii, mc->tree->dupfix_size); if (unlikely(end_of_subpage < sub_key + sub_ksize)) { - rc = bad_page(mp, "nested-leaf2-key beyond (%zu) nested-page\n", - sub_key + sub_ksize - end_of_subpage); + rc = bad_page(mp, "nested-leaf2-key beyond (%zu) nested-page\n", sub_key + sub_ksize - end_of_subpage); continue; } @@ -374,24 +312,20 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) { if ((mc->checking & z_ignord) == 0) { sub_here.iov_base = (void *)sub_key; sub_here.iov_len = sub_ksize; - if (sub_prev.iov_base && - unlikely(v_clc.cmp(&sub_prev, &sub_here) >= 0)) - rc = bad_page(mp, - "nested-leaf2-key #%u wrong order (%s >= %s)\n", - ii, DKEY(&sub_prev), DVAL(&sub_here)); + if (sub_prev.iov_base && unlikely(v_clc.cmp(&sub_prev, &sub_here) >= 0)) + rc = bad_page(mp, "nested-leaf2-key #%u wrong order (%s >= %s)\n", ii, DKEY(&sub_prev), + DVAL(&sub_here)); sub_prev = sub_here; } } else { const node_t *const sub_node = page_node(sp, ii); const char *const sub_node_end = ptr_disp(sub_node, NODESIZE); if (unlikely(sub_node_end > end_of_subpage)) { - rc = bad_page(mp, "nested-node beyond (%zu) nested-page\n", - end_of_subpage - sub_node_end); + rc = bad_page(mp, "nested-node beyond (%zu) nested-page\n", end_of_subpage - sub_node_end); continue; } if (unlikely(node_flags(sub_node) != 0)) - rc = bad_page(mp, "nested-node invalid flags (%u)\n", - node_flags(sub_node)); + rc = bad_page(mp, "nested-node invalid flags (%u)\n", node_flags(sub_node)); const size_t sub_ksize = node_ks(sub_node); const char *const sub_key = node_key(sub_node); @@ -406,19 +340,15 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) { if ((mc->checking & z_ignord) == 0) { sub_here.iov_base = (void *)sub_key; sub_here.iov_len = sub_ksize; - if (sub_prev.iov_base && - unlikely(v_clc.cmp(&sub_prev, &sub_here) >= 0)) - rc = bad_page(mp, - "nested-node-key #%u wrong order (%s >= %s)\n", - ii, DKEY(&sub_prev), DVAL(&sub_here)); + if (sub_prev.iov_base && unlikely(v_clc.cmp(&sub_prev, &sub_here) >= 0)) + rc = bad_page(mp, "nested-node-key #%u wrong order (%s >= %s)\n", ii, DKEY(&sub_prev), + DVAL(&sub_here)); sub_prev = sub_here; } if (unlikely(sub_dsize != 0)) - rc = bad_page(mp, "nested-node non-empty data size (%zu)\n", - sub_dsize); + rc = bad_page(mp, "nested-node non-empty data size (%zu)\n", sub_dsize); if (unlikely(end_of_subpage < sub_key + sub_ksize)) - rc = bad_page(mp, "nested-node-key beyond (%zu) nested-page\n", - sub_key + sub_ksize - end_of_subpage); + rc = bad_page(mp, "nested-node-key beyond (%zu) nested-page\n", sub_key + sub_ksize - end_of_subpage); } } } @@ -429,9 +359,7 @@ __cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) { return rc; } -static __always_inline int check_page_header(const uint16_t ILL, - const page_t *page, - MDBX_txn *const txn, +static __always_inline int check_page_header(const uint16_t ILL, const page_t *page, MDBX_txn *const txn, const txnid_t front) { if (unlikely(page->flags & ILL)) { if (ILL == P_ILL_BITS || (page->flags & P_ILL_BITS)) @@ -439,59 +367,44 @@ static __always_inline int check_page_header(const uint16_t ILL, else if (ILL & P_LARGE) { assert((ILL & (P_BRANCH | P_LEAF | P_DUPFIX)) == 0); assert(page->flags & (P_BRANCH | P_LEAF | P_DUPFIX)); - return bad_page(page, "unexpected %s instead of %s (%u)\n", - "large/overflow", "branch/leaf/leaf2", page->flags); + return bad_page(page, "unexpected %s instead of %s (%u)\n", "large/overflow", "branch/leaf/leaf2", page->flags); } else if (ILL & (P_BRANCH | P_LEAF | P_DUPFIX)) { assert((ILL & P_BRANCH) && (ILL & P_LEAF) && (ILL & P_DUPFIX)); assert(page->flags & (P_BRANCH | P_LEAF | P_DUPFIX)); - return bad_page(page, "unexpected %s instead of %s (%u)\n", - "branch/leaf/leaf2", "large/overflow", page->flags); + return bad_page(page, "unexpected %s instead of %s (%u)\n", "branch/leaf/leaf2", "large/overflow", page->flags); } else { assert(false); } } - if (unlikely(page->txnid > front) && - unlikely(page->txnid > txn->front_txnid || front < txn->txnid)) - return bad_page( - page, - "invalid page' txnid (%" PRIaTXN ") for %s' txnid (%" PRIaTXN ")\n", - page->txnid, - (front == txn->front_txnid && front != txn->txnid) ? "front-txn" - : "parent-page", - front); + if (unlikely(page->txnid > front) && unlikely(page->txnid > txn->front_txnid || front < txn->txnid)) + return bad_page(page, "invalid page' txnid (%" PRIaTXN ") for %s' txnid (%" PRIaTXN ")\n", page->txnid, + (front == txn->front_txnid && front != txn->txnid) ? "front-txn" : "parent-page", front); - if (((ILL & P_LARGE) || !is_largepage(page)) && - (ILL & (P_BRANCH | P_LEAF | P_DUPFIX)) == 0) { + if (((ILL & P_LARGE) || !is_largepage(page)) && (ILL & (P_BRANCH | P_LEAF | P_DUPFIX)) == 0) { /* Контроль четности page->upper тут либо приводит к ложным ошибкам, * либо слишком дорог по количеству операций. Заковырка в том, что upper * может быть нечетным на DUPFIX-страницах, при нечетном количестве * элементов нечетной длины. Поэтому четность page->upper здесь не * проверяется, но соответствующие полные проверки есть в page_check(). */ - if (unlikely(page->upper < page->lower || (page->lower & 1) || - PAGEHDRSZ + page->upper > txn->env->ps)) - return bad_page(page, - "invalid page' lower(%u)/upper(%u) with limit %zu\n", - page->lower, page->upper, page_space(txn->env)); + if (unlikely(page->upper < page->lower || (page->lower & 1) || PAGEHDRSZ + page->upper > txn->env->ps)) + return bad_page(page, "invalid page' lower(%u)/upper(%u) with limit %zu\n", page->lower, page->upper, + page_space(txn->env)); } else if ((ILL & P_LARGE) == 0) { const pgno_t npages = page->pages; if (unlikely(npages < 1) || unlikely(npages >= MAX_PAGENO / 2)) return bad_page(page, "invalid n-pages (%u) for large-page\n", npages); if (unlikely(page->pgno + npages > txn->geo.first_unallocated)) - return bad_page( - page, - "end of large-page beyond (%u) allocated space (%u next-pgno)\n", - page->pgno + npages, txn->geo.first_unallocated); + return bad_page(page, "end of large-page beyond (%u) allocated space (%u next-pgno)\n", page->pgno + npages, + txn->geo.first_unallocated); } else { assert(false); } return MDBX_SUCCESS; } -__cold static __noinline pgr_t check_page_complete(const uint16_t ILL, - page_t *page, - const MDBX_cursor *const mc, +__cold static __noinline pgr_t check_page_complete(const uint16_t ILL, page_t *page, const MDBX_cursor *const mc, const txnid_t front) { pgr_t r = {page, check_page_header(ILL, page, mc->txn, front)}; if (likely(r.err == MDBX_SUCCESS)) @@ -501,9 +414,7 @@ __cold static __noinline pgr_t check_page_complete(const uint16_t ILL, return r; } -static __always_inline pgr_t page_get_inline(const uint16_t ILL, - const MDBX_cursor *const mc, - const pgno_t pgno, +static __always_inline pgr_t page_get_inline(const uint16_t ILL, const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front) { MDBX_txn *const txn = mc->txn; tASSERT(txn, front <= txn->front_txnid); @@ -527,8 +438,7 @@ static __always_inline pgr_t page_get_inline(const uint16_t ILL, * because the dirty list got full. Bring this page * back in from the map (but don't unspill it here, * leave that unless page_touch happens again). */ - if (unlikely(spiller->flags & MDBX_TXN_SPILLS) && - spill_search(spiller, pgno)) + if (unlikely(spiller->flags & MDBX_TXN_SPILLS) && spill_search(spiller, pgno)) break; const size_t i = dpl_search(spiller, pgno); @@ -543,9 +453,7 @@ static __always_inline pgr_t page_get_inline(const uint16_t ILL, } if (unlikely(r.page->pgno != pgno)) { - r.err = bad_page( - r.page, "pgno mismatch (%" PRIaPGNO ") != expected (%" PRIaPGNO ")\n", - r.page->pgno, pgno); + r.err = bad_page(r.page, "pgno mismatch (%" PRIaPGNO ") != expected (%" PRIaPGNO ")\n", r.page->pgno, pgno); goto bailout; } @@ -562,18 +470,14 @@ static __always_inline pgr_t page_get_inline(const uint16_t ILL, return r; } -pgr_t page_get_any(const MDBX_cursor *const mc, const pgno_t pgno, - const txnid_t front) { +pgr_t page_get_any(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front) { return page_get_inline(P_ILL_BITS, mc, pgno, front); } -__hot pgr_t page_get_three(const MDBX_cursor *const mc, const pgno_t pgno, - const txnid_t front) { +__hot pgr_t page_get_three(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front) { return page_get_inline(P_ILL_BITS | P_LARGE, mc, pgno, front); } -pgr_t page_get_large(const MDBX_cursor *const mc, const pgno_t pgno, - const txnid_t front) { - return page_get_inline(P_ILL_BITS | P_BRANCH | P_LEAF | P_DUPFIX, mc, pgno, - front); +pgr_t page_get_large(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front) { + return page_get_inline(P_ILL_BITS | P_BRANCH | P_LEAF | P_DUPFIX, mc, pgno, front); } diff --git a/src/page-iov.c b/src/page-iov.c index 700ff5d0..b6686017 100644 --- a/src/page-iov.c +++ b/src/page-iov.c @@ -3,17 +3,14 @@ #include "internals.h" -int iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, size_t items, size_t npages, - mdbx_filehandle_t fd, bool check_coherence) { +int iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, size_t items, size_t npages, mdbx_filehandle_t fd, + bool check_coherence) { ctx->env = txn->env; ctx->ior = &txn->env->ioring; ctx->fd = fd; ctx->coherency_timestamp = - (check_coherence || txn->env->lck->pgops.incoherence.weak) - ? 0 - : UINT64_MAX /* не выполнять сверку */; - ctx->err = osal_ioring_prepare(ctx->ior, items, - pgno_align2os_bytes(txn->env, npages)); + (check_coherence || txn->env->lck->pgops.incoherence.weak) ? 0 : UINT64_MAX /* не выполнять сверку */; + ctx->err = osal_ioring_prepare(ctx->ior, items, pgno_align2os_bytes(txn->env, npages)); if (likely(ctx->err == MDBX_SUCCESS)) { #if MDBX_NEED_WRITTEN_RANGE ctx->flush_begin = MAX_PAGENO; @@ -24,8 +21,7 @@ int iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, size_t items, size_t npages, return ctx->err; } -static void iov_callback4dirtypages(iov_ctx_t *ctx, size_t offset, void *data, - size_t bytes) { +static void iov_callback4dirtypages(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes) { MDBX_env *const env = ctx->env; eASSERT(env, (env->flags & MDBX_WRITEMAP) == 0); @@ -89,19 +85,15 @@ static void iov_callback4dirtypages(iov_ctx_t *ctx, size_t offset, void *data, #ifndef MDBX_FORCE_CHECK_MMAP_COHERENCY #define MDBX_FORCE_CHECK_MMAP_COHERENCY 0 #endif /* MDBX_FORCE_CHECK_MMAP_COHERENCY */ - if ((MDBX_FORCE_CHECK_MMAP_COHERENCY || - ctx->coherency_timestamp != UINT64_MAX) && + if ((MDBX_FORCE_CHECK_MMAP_COHERENCY || ctx->coherency_timestamp != UINT64_MAX) && unlikely(memcmp(wp, rp, bytes))) { ctx->coherency_timestamp = 0; env->lck->pgops.incoherence.weak = - (env->lck->pgops.incoherence.weak >= INT32_MAX) - ? INT32_MAX - : env->lck->pgops.incoherence.weak + 1; + (env->lck->pgops.incoherence.weak >= INT32_MAX) ? INT32_MAX : env->lck->pgops.incoherence.weak + 1; WARNING("catch delayed/non-arrived page %" PRIaPGNO " %s", wp->pgno, "(workaround for incoherent flaw of unified page/buffer cache)"); do - if (coherency_timeout(&ctx->coherency_timestamp, wp->pgno, env) != - MDBX_RESULT_TRUE) { + if (coherency_timeout(&ctx->coherency_timestamp, wp->pgno, env) != MDBX_RESULT_TRUE) { ctx->err = MDBX_PROBLEM; break; } @@ -160,8 +152,7 @@ int iov_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, size_t npages) { #if MDBX_AVOID_MSYNC doit:; #endif /* MDBX_AVOID_MSYNC */ - int err = osal_ioring_add(ctx->ior, pgno2bytes(env, dp->pgno), dp, - pgno2bytes(env, npages)); + int err = osal_ioring_add(ctx->ior, pgno2bytes(env, dp->pgno), dp, pgno2bytes(env, npages)); if (unlikely(err != MDBX_SUCCESS)) { ctx->err = err; if (unlikely(err != MDBX_RESULT_TRUE)) { @@ -171,8 +162,7 @@ int iov_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, size_t npages) { err = iov_write(ctx); tASSERT(txn, iov_empty(ctx)); if (likely(err == MDBX_SUCCESS)) { - err = osal_ioring_add(ctx->ior, pgno2bytes(env, dp->pgno), dp, - pgno2bytes(env, npages)); + err = osal_ioring_add(ctx->ior, pgno2bytes(env, dp->pgno), dp, pgno2bytes(env, npages)); if (unlikely(err != MDBX_SUCCESS)) { iov_complete(ctx); return ctx->err = err; @@ -188,11 +178,8 @@ int iov_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, size_t npages) { } #if MDBX_NEED_WRITTEN_RANGE - ctx->flush_begin = - (ctx->flush_begin < dp->pgno) ? ctx->flush_begin : dp->pgno; - ctx->flush_end = (ctx->flush_end > dp->pgno + (pgno_t)npages) - ? ctx->flush_end - : dp->pgno + (pgno_t)npages; + ctx->flush_begin = (ctx->flush_begin < dp->pgno) ? ctx->flush_begin : dp->pgno; + ctx->flush_end = (ctx->flush_end > dp->pgno + (pgno_t)npages) ? ctx->flush_end : dp->pgno + (pgno_t)npages; #endif /* MDBX_NEED_WRITTEN_RANGE */ return MDBX_SUCCESS; } diff --git a/src/page-iov.h b/src/page-iov.h index 397f6fbe..bf367fef 100644 --- a/src/page-iov.h +++ b/src/page-iov.h @@ -24,15 +24,11 @@ struct iov_ctx { uint64_t coherency_timestamp; }; -MDBX_INTERNAL __must_check_result int -iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, size_t items, size_t npages, - mdbx_filehandle_t fd, bool check_coherence); +MDBX_INTERNAL __must_check_result int iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, size_t items, size_t npages, + mdbx_filehandle_t fd, bool check_coherence); -static inline bool iov_empty(const iov_ctx_t *ctx) { - return osal_ioring_used(ctx->ior) == 0; -} +static inline bool iov_empty(const iov_ctx_t *ctx) { return osal_ioring_used(ctx->ior) == 0; } -MDBX_INTERNAL __must_check_result int iov_page(MDBX_txn *txn, iov_ctx_t *ctx, - page_t *dp, size_t npages); +MDBX_INTERNAL __must_check_result int iov_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, size_t npages); MDBX_INTERNAL __must_check_result int iov_write(iov_ctx_t *ctx); diff --git a/src/page-ops.c b/src/page-ops.c index d07cde07..6ca5f332 100644 --- a/src/page-ops.c +++ b/src/page-ops.c @@ -42,13 +42,11 @@ pgr_t page_new(MDBX_cursor *mc, const unsigned flags) { } pgr_t page_new_large(MDBX_cursor *mc, const size_t npages) { - pgr_t ret = likely(npages == 1) ? gc_alloc_single(mc) - : gc_alloc_ex(mc, npages, ALLOC_DEFAULT); + pgr_t ret = likely(npages == 1) ? gc_alloc_single(mc) : gc_alloc_ex(mc, npages, ALLOC_DEFAULT); if (unlikely(ret.err != MDBX_SUCCESS)) return ret; - DEBUG("dbi %zu allocated new large-page %" PRIaPGNO ", num %zu", - cursor_dbi(mc), ret.page->pgno, npages); + DEBUG("dbi %zu allocated new large-page %" PRIaPGNO ", num %zu", cursor_dbi(mc), ret.page->pgno, npages); ret.page->flags = P_LARGE; cASSERT(mc, *cursor_dbi_state(mc) & DBI_DIRTY); cASSERT(mc, mc->txn->flags & MDBX_TXN_DIRTY); @@ -62,8 +60,7 @@ pgr_t page_new_large(MDBX_cursor *mc, const size_t npages) { return ret; } -__hot void page_copy(page_t *const dst, const page_t *const src, - const size_t size) { +__hot void page_copy(page_t *const dst, const page_t *const src, const size_t size) { STATIC_ASSERT(UINT16_MAX > MDBX_MAX_PAGESIZE - PAGEHDRSZ); STATIC_ASSERT(MDBX_MIN_PAGESIZE > PAGEHDRSZ + NODESIZE * 4); void *copy_dst = dst; @@ -94,17 +91,14 @@ __hot void page_copy(page_t *const dst, const page_t *const src, bailout: if (src->flags & P_DUPFIX) - bad_page(src, "%s addr %p, n-keys %zu, ksize %u", - "invalid/corrupted source page", __Wpedantic_format_voidptr(src), + bad_page(src, "%s addr %p, n-keys %zu, ksize %u", "invalid/corrupted source page", __Wpedantic_format_voidptr(src), page_numkeys(src), src->dupfix_ksize); else - bad_page(src, "%s addr %p, upper %u", "invalid/corrupted source page", - __Wpedantic_format_voidptr(src), src->upper); + bad_page(src, "%s addr %p, upper %u", "invalid/corrupted source page", __Wpedantic_format_voidptr(src), src->upper); memset(dst, -1, size); } -__cold pgr_t __must_check_result page_unspill(MDBX_txn *const txn, - const page_t *const mp) { +__cold pgr_t __must_check_result page_unspill(MDBX_txn *const txn, const page_t *const mp) { VERBOSE("unspill page %" PRIaPGNO, mp->pgno); tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0); tASSERT(txn, is_spilled(txn, mp)); @@ -139,13 +133,11 @@ __cold pgr_t __must_check_result page_unspill(MDBX_txn *const txn, ret.page->flags |= (scan == txn) ? 0 : P_SPILLED; ret.err = MDBX_SUCCESS; return ret; - } while (likely((scan = scan->parent) != nullptr && - (scan->flags & MDBX_TXN_SPILLS) != 0)); - ERROR("Page %" PRIaPGNO " mod-txnid %" PRIaTXN - " not found in the spill-list(s), current txn %" PRIaTXN + } while (likely((scan = scan->parent) != nullptr && (scan->flags & MDBX_TXN_SPILLS) != 0)); + ERROR("Page %" PRIaPGNO " mod-txnid %" PRIaTXN " not found in the spill-list(s), current txn %" PRIaTXN " front %" PRIaTXN ", root txn %" PRIaTXN " front %" PRIaTXN, - mp->pgno, mp->txnid, txn->txnid, txn->front_txnid, - txn->env->basal_txn->txnid, txn->env->basal_txn->front_txnid); + mp->pgno, mp->txnid, txn->txnid, txn->front_txnid, txn->env->basal_txn->txnid, + txn->env->basal_txn->front_txnid); ret.err = MDBX_PROBLEM; ret.page = nullptr; return ret; @@ -157,8 +149,7 @@ __hot int page_touch_modifable(MDBX_txn *txn, const page_t *const mp) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); const size_t n = dpl_search(txn, mp->pgno); - if (MDBX_AVOID_MSYNC && - unlikely(txn->tw.dirtylist->items[n].pgno != mp->pgno)) { + if (MDBX_AVOID_MSYNC && unlikely(txn->tw.dirtylist->items[n].pgno != mp->pgno)) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP)); tASSERT(txn, n > 0 && n <= txn->tw.dirtylist->length + 1); VERBOSE("unspill page %" PRIaPGNO, mp->pgno); @@ -169,18 +160,15 @@ __hot int page_touch_modifable(MDBX_txn *txn, const page_t *const mp) { } tASSERT(txn, n > 0 && n <= txn->tw.dirtylist->length); - tASSERT(txn, txn->tw.dirtylist->items[n].pgno == mp->pgno && - txn->tw.dirtylist->items[n].ptr == mp); + tASSERT(txn, txn->tw.dirtylist->items[n].pgno == mp->pgno && txn->tw.dirtylist->items[n].ptr == mp); if (!MDBX_AVOID_MSYNC || (txn->flags & MDBX_WRITEMAP) == 0) { - size_t *const ptr = - ptr_disp(txn->tw.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t)); + size_t *const ptr = ptr_disp(txn->tw.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t)); *ptr = txn->tw.dirtylru; } return MDBX_SUCCESS; } -__hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, - const page_t *const mp) { +__hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *const mp) { tASSERT(txn, !is_modifable(txn, mp) && !is_largepage(mp)); if (is_subpage(mp)) { ((page_t *)mp)->txnid = txn->front_txnid; @@ -201,8 +189,7 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, goto fail; const pgno_t pgno = np->pgno; - DEBUG("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, cursor_dbi_dbg(mc), - mp->pgno, pgno); + DEBUG("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno, pgno); tASSERT(txn, mp->pgno != pgno); pnl_append_prereserved(txn->tw.retired_pages, mp->pgno); /* Update the parent page, if any, to point to the new page */ @@ -233,17 +220,14 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, if (unlikely(!txn->parent)) { ERROR("Unexpected not frozen/modifiable/spilled but shadowed %s " "page %" PRIaPGNO " mod-txnid %" PRIaTXN "," - " without parent transaction, current txn %" PRIaTXN - " front %" PRIaTXN, - is_branch(mp) ? "branch" : "leaf", mp->pgno, mp->txnid, - mc->txn->txnid, mc->txn->front_txnid); + " without parent transaction, current txn %" PRIaTXN " front %" PRIaTXN, + is_branch(mp) ? "branch" : "leaf", mp->pgno, mp->txnid, mc->txn->txnid, mc->txn->front_txnid); rc = MDBX_PROBLEM; goto fail; } DEBUG("clone db %d page %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno); - tASSERT(txn, - txn->tw.dirtylist->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE); + tASSERT(txn, txn->tw.dirtylist->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE); /* No - copy it */ np = page_shadow_alloc(txn, 1); if (unlikely(!np)) { @@ -299,8 +283,7 @@ page_t *page_shadow_alloc(MDBX_txn *txn, size_t num) { if (likely(num == 1 && np)) { eASSERT(env, env->shadow_reserve_len > 0); MDBX_ASAN_UNPOISON_MEMORY_REGION(np, size); - VALGRIND_MEMPOOL_ALLOC(env, ptr_disp(np, -(ptrdiff_t)sizeof(size_t)), - size + sizeof(size_t)); + VALGRIND_MEMPOOL_ALLOC(env, ptr_disp(np, -(ptrdiff_t)sizeof(size_t)), size + sizeof(size_t)); VALGRIND_MAKE_MEM_DEFINED(&page_next(np), sizeof(page_t *)); env->shadow_reserve = page_next(np); env->shadow_reserve_len -= 1; @@ -338,8 +321,7 @@ void page_shadow_release(MDBX_env *env, page_t *dp, size_t npages) { MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, pgno2bytes(env, npages)); if (unlikely(env->flags & MDBX_PAGEPERTURB)) memset(dp, -1, pgno2bytes(env, npages)); - if (likely(npages == 1 && - env->shadow_reserve_len < env->options.dp_reserve_limit)) { + if (likely(npages == 1 && env->shadow_reserve_len < env->options.dp_reserve_limit)) { MDBX_ASAN_POISON_MEMORY_REGION(dp, env->ps); MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(dp), sizeof(page_t *)); page_next(dp) = env->shadow_reserve; @@ -354,8 +336,7 @@ void page_shadow_release(MDBX_env *env, page_t *dp, size_t npages) { } } -__cold static void page_kill(MDBX_txn *txn, page_t *mp, pgno_t pgno, - size_t npages) { +__cold static void page_kill(MDBX_txn *txn, page_t *mp, pgno_t pgno, size_t npages) { MDBX_env *const env = txn->env; DEBUG("kill %zu page(s) %" PRIaPGNO, npages, pgno); eASSERT(env, pgno >= NUM_METAS && npages); @@ -391,8 +372,7 @@ static inline bool suitable4loose(const MDBX_txn *txn, pgno_t pgno) { return txn->tw.loose_count < txn->env->options.dp_loose_limit && (!MDBX_ENABLE_REFUND || /* skip pages near to the end in favor of compactification */ - txn->geo.first_unallocated > - pgno + txn->env->options.dp_loose_limit || + txn->geo.first_unallocated > pgno + txn->env->options.dp_loose_limit || txn->geo.first_unallocated <= txn->env->options.dp_loose_limit); } @@ -404,8 +384,7 @@ static inline bool suitable4loose(const MDBX_txn *txn, pgno_t pgno) { * * If the page wasn't dirtied in this txn, just add it * to this txn's free list. */ -int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, - page_t *mp /* maybe null */, +int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null */, unsigned pageflags /* maybe unknown/zero */) { int rc; MDBX_txn *const txn = mc->txn; @@ -423,13 +402,7 @@ int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, * So for flexibility and avoid extra internal dependencies we just * fallback to reading if dirty list was not allocated yet. */ size_t di = 0, si = 0, npages = 1; - enum page_status { - unknown, - frozen, - spilled, - shadowed, - modifable - } status = unknown; + enum page_status { unknown, frozen, spilled, shadowed, modifable } status = unknown; if (unlikely(!mp)) { if (ASSERT_ENABLED() && pageflags) { @@ -437,8 +410,7 @@ int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, check = page_get_any(mc, pgno, txn->front_txnid); if (unlikely(check.err != MDBX_SUCCESS)) return check.err; - tASSERT(txn, ((unsigned)check.page->flags & ~P_SPILLED) == - (pageflags & ~P_FROZEN)); + tASSERT(txn, ((unsigned)check.page->flags & ~P_SPILLED) == (pageflags & ~P_FROZEN)); tASSERT(txn, !(pageflags & P_FROZEN) || is_frozen(txn, check.page)); } if (pageflags & P_FROZEN) { @@ -540,8 +512,7 @@ status_done: /* Возврат страниц в нераспределенный "хвост" БД. * Содержимое страниц не уничтожается, а для вложенных транзакций граница * нераспределенного "хвоста" БД сдвигается только при их коммите. */ - if (MDBX_ENABLE_REFUND && - unlikely(pgno + npages == txn->geo.first_unallocated)) { + if (MDBX_ENABLE_REFUND && unlikely(pgno + npages == txn->geo.first_unallocated)) { const char *kind = nullptr; if (status == modifable) { /* Страница испачкана в этой транзакции, но до этого могла быть @@ -589,8 +560,7 @@ status_done: if (status == modifable) { /* Dirty page from this transaction */ /* If suitable we can reuse it through loose list */ - if (likely(npages == 1 && suitable4loose(txn, pgno)) && - (di || !txn->tw.dirtylist)) { + if (likely(npages == 1 && suitable4loose(txn, pgno)) && (di || !txn->tw.dirtylist)) { DEBUG("loosen dirty page %" PRIaPGNO, pgno); if (MDBX_DEBUG != 0 || unlikely(txn->env->flags & MDBX_PAGEPERTURB)) memset(page_data(mp), -1, txn->env->ps - PAGEHDRSZ); @@ -600,9 +570,7 @@ status_done: txn->tw.loose_pages = mp; txn->tw.loose_count++; #if MDBX_ENABLE_REFUND - txn->tw.loose_refund_wl = (pgno + 2 > txn->tw.loose_refund_wl) - ? pgno + 2 - : txn->tw.loose_refund_wl; + txn->tw.loose_refund_wl = (pgno + 2 > txn->tw.loose_refund_wl) ? pgno + 2 : txn->tw.loose_refund_wl; #endif /* MDBX_ENABLE_REFUND */ VALGRIND_MAKE_MEM_NOACCESS(page_data(mp), txn->env->ps - PAGEHDRSZ); MDBX_ASAN_POISON_MEMORY_REGION(page_data(mp), txn->env->ps - PAGEHDRSZ); @@ -617,9 +585,7 @@ status_done: * в том числе, позже выгружена и затем снова загружена и изменена. * В обоих случаях её нельзя затирать на диске и помечать недоступной * в asan и/или valgrind */ - for (MDBX_txn *parent = txn->parent; - parent && (parent->flags & MDBX_TXN_SPILLS); - parent = parent->parent) { + for (MDBX_txn *parent = txn->parent; parent && (parent->flags & MDBX_TXN_SPILLS); parent = parent->parent) { if (spill_intersect(parent, pgno, npages)) goto skip_invalidate; if (dpl_intersect(parent, pgno, npages)) @@ -631,11 +597,8 @@ status_done: #endif page_kill(txn, mp, pgno, npages); if ((txn->flags & MDBX_WRITEMAP) == 0) { - VALGRIND_MAKE_MEM_NOACCESS(page_data(pgno2page(txn->env, pgno)), - pgno2bytes(txn->env, npages) - PAGEHDRSZ); - MDBX_ASAN_POISON_MEMORY_REGION(page_data(pgno2page(txn->env, pgno)), - pgno2bytes(txn->env, npages) - - PAGEHDRSZ); + VALGRIND_MAKE_MEM_NOACCESS(page_data(pgno2page(txn->env, pgno)), pgno2bytes(txn->env, npages) - PAGEHDRSZ); + MDBX_ASAN_POISON_MEMORY_REGION(page_data(pgno2page(txn->env, pgno)), pgno2bytes(txn->env, npages) - PAGEHDRSZ); } } skip_invalidate: @@ -646,9 +609,7 @@ status_done: reclaim: DEBUG("reclaim %zu %s page %" PRIaPGNO, npages, "dirty", pgno); rc = pnl_insert_span(&txn->tw.relist, pgno, npages); - tASSERT(txn, - pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - - MDBX_ENABLE_REFUND)); + tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); tASSERT(txn, dpl_check(txn)); return rc; } @@ -675,8 +636,7 @@ status_done: if (ASSERT_ENABLED()) { const page_t *parent_dp = nullptr; /* Check parent(s)'s dirty lists. */ - for (MDBX_txn *parent = txn->parent; parent && !parent_dp; - parent = parent->parent) { + for (MDBX_txn *parent = txn->parent; parent && !parent_dp; parent = parent->parent) { tASSERT(txn, !spill_search(parent, pgno)); parent_dp = debug_dpl_find(parent, pgno); } @@ -697,8 +657,7 @@ status_done: goto retire; } -__hot int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, - size_t npages) { +__hot int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, size_t npages) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); mp->txnid = txn->front_txnid; if (!txn->tw.dirtylist) { @@ -756,37 +715,27 @@ void recalculate_subpage_thresholds(MDBX_env *env) { size_t whole = env->leaf_nodemax - NODESIZE; env->subpage_limit = (whole * env->options.subpage.limit + 32767) >> 16; whole = env->subpage_limit; - env->subpage_reserve_limit = - (whole * env->options.subpage.reserve_limit + 32767) >> 16; + env->subpage_reserve_limit = (whole * env->options.subpage.reserve_limit + 32767) >> 16; eASSERT(env, env->leaf_nodemax >= env->subpage_limit + NODESIZE); eASSERT(env, env->subpage_limit >= env->subpage_reserve_limit); whole = env->leaf_nodemax; - env->subpage_room_threshold = - (whole * env->options.subpage.room_threshold + 32767) >> 16; - env->subpage_reserve_prereq = - (whole * env->options.subpage.reserve_prereq + 32767) >> 16; - if (env->subpage_room_threshold + env->subpage_reserve_limit > - (intptr_t)page_space(env)) + env->subpage_room_threshold = (whole * env->options.subpage.room_threshold + 32767) >> 16; + env->subpage_reserve_prereq = (whole * env->options.subpage.reserve_prereq + 32767) >> 16; + if (env->subpage_room_threshold + env->subpage_reserve_limit > (intptr_t)page_space(env)) env->subpage_reserve_prereq = page_space(env); - else if (env->subpage_reserve_prereq < - env->subpage_room_threshold + env->subpage_reserve_limit) - env->subpage_reserve_prereq = - env->subpage_room_threshold + env->subpage_reserve_limit; - eASSERT(env, env->subpage_reserve_prereq > - env->subpage_room_threshold + env->subpage_reserve_limit); + else if (env->subpage_reserve_prereq < env->subpage_room_threshold + env->subpage_reserve_limit) + env->subpage_reserve_prereq = env->subpage_room_threshold + env->subpage_reserve_limit; + eASSERT(env, env->subpage_reserve_prereq > env->subpage_room_threshold + env->subpage_reserve_limit); } -size_t page_subleaf2_reserve(const MDBX_env *env, size_t host_page_room, - size_t subpage_len, size_t item_len) { +size_t page_subleaf2_reserve(const MDBX_env *env, size_t host_page_room, size_t subpage_len, size_t item_len) { eASSERT(env, (subpage_len & 1) == 0); eASSERT(env, env->leaf_nodemax >= env->subpage_limit + NODESIZE); size_t reserve = 0; - for (size_t n = 0; - n < 5 && reserve + item_len <= env->subpage_reserve_limit && - EVEN_CEIL(subpage_len + item_len) <= env->subpage_limit && - host_page_room >= - env->subpage_reserve_prereq + EVEN_CEIL(subpage_len + item_len); + for (size_t n = 0; n < 5 && reserve + item_len <= env->subpage_reserve_limit && + EVEN_CEIL(subpage_len + item_len) <= env->subpage_limit && + host_page_room >= env->subpage_reserve_prereq + EVEN_CEIL(subpage_len + item_len); ++n) { subpage_len += item_len; reserve += item_len; diff --git a/src/page-ops.h b/src/page-ops.h index 63cdd0b5..bb4ebd16 100644 --- a/src/page-ops.h +++ b/src/page-ops.h @@ -5,9 +5,7 @@ #include "essentials.h" -MDBX_INTERNAL int __must_check_result tree_search_finalize(MDBX_cursor *mc, - const MDBX_val *key, - int flags); +MDBX_INTERNAL int __must_check_result tree_search_finalize(MDBX_cursor *mc, const MDBX_val *key, int flags); MDBX_INTERNAL int tree_search_lowest(MDBX_cursor *mc); enum page_search_flags { @@ -16,64 +14,47 @@ enum page_search_flags { Z_FIRST = 4, Z_LAST = 8, }; -MDBX_INTERNAL int __must_check_result tree_search(MDBX_cursor *mc, - const MDBX_val *key, - int flags); +MDBX_INTERNAL int __must_check_result tree_search(MDBX_cursor *mc, const MDBX_val *key, int flags); #define MDBX_SPLIT_REPLACE MDBX_APPENDDUP /* newkey is not new */ -MDBX_INTERNAL int __must_check_result page_split(MDBX_cursor *mc, - const MDBX_val *const newkey, - MDBX_val *const newdata, - pgno_t newpgno, - const unsigned naf); +MDBX_INTERNAL int __must_check_result page_split(MDBX_cursor *mc, const MDBX_val *const newkey, MDBX_val *const newdata, + pgno_t newpgno, const unsigned naf); /*----------------------------------------------------------------------------*/ -MDBX_INTERNAL int MDBX_PRINTF_ARGS(2, 3) - bad_page(const page_t *mp, const char *fmt, ...); +MDBX_INTERNAL int MDBX_PRINTF_ARGS(2, 3) bad_page(const page_t *mp, const char *fmt, ...); -MDBX_INTERNAL void MDBX_PRINTF_ARGS(2, 3) - poor_page(const page_t *mp, const char *fmt, ...); +MDBX_INTERNAL void MDBX_PRINTF_ARGS(2, 3) poor_page(const page_t *mp, const char *fmt, ...); -MDBX_NOTHROW_PURE_FUNCTION static inline bool is_frozen(const MDBX_txn *txn, - const page_t *mp) { +MDBX_NOTHROW_PURE_FUNCTION static inline bool is_frozen(const MDBX_txn *txn, const page_t *mp) { return mp->txnid < txn->txnid; } -MDBX_NOTHROW_PURE_FUNCTION static inline bool is_spilled(const MDBX_txn *txn, - const page_t *mp) { +MDBX_NOTHROW_PURE_FUNCTION static inline bool is_spilled(const MDBX_txn *txn, const page_t *mp) { return mp->txnid == txn->txnid; } -MDBX_NOTHROW_PURE_FUNCTION static inline bool is_shadowed(const MDBX_txn *txn, - const page_t *mp) { +MDBX_NOTHROW_PURE_FUNCTION static inline bool is_shadowed(const MDBX_txn *txn, const page_t *mp) { return mp->txnid > txn->txnid; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool -is_correct(const MDBX_txn *txn, const page_t *mp) { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_correct(const MDBX_txn *txn, const page_t *mp) { return mp->txnid <= txn->front_txnid; } -MDBX_NOTHROW_PURE_FUNCTION static inline bool is_modifable(const MDBX_txn *txn, - const page_t *mp) { +MDBX_NOTHROW_PURE_FUNCTION static inline bool is_modifable(const MDBX_txn *txn, const page_t *mp) { return mp->txnid == txn->front_txnid; } -MDBX_INTERNAL int __must_check_result page_check(const MDBX_cursor *const mc, - const page_t *const mp); +MDBX_INTERNAL int __must_check_result page_check(const MDBX_cursor *const mc, const page_t *const mp); -MDBX_INTERNAL pgr_t page_get_any(const MDBX_cursor *const mc, const pgno_t pgno, - const txnid_t front); +MDBX_INTERNAL pgr_t page_get_any(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front); -MDBX_INTERNAL pgr_t page_get_three(const MDBX_cursor *const mc, - const pgno_t pgno, const txnid_t front); +MDBX_INTERNAL pgr_t page_get_three(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front); -MDBX_INTERNAL pgr_t page_get_large(const MDBX_cursor *const mc, - const pgno_t pgno, const txnid_t front); +MDBX_INTERNAL pgr_t page_get_large(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front); -static inline int __must_check_result page_get(const MDBX_cursor *mc, - const pgno_t pgno, page_t **mp, +static inline int __must_check_result page_get(const MDBX_cursor *mc, const pgno_t pgno, page_t **mp, const txnid_t front) { pgr_t ret = page_get_three(mc, pgno, front); *mp = ret.page; @@ -82,21 +63,18 @@ static inline int __must_check_result page_get(const MDBX_cursor *mc, /*----------------------------------------------------------------------------*/ -MDBX_INTERNAL int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, - size_t npages); +MDBX_INTERNAL int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, size_t npages); MDBX_INTERNAL pgr_t page_new(MDBX_cursor *mc, const unsigned flags); MDBX_INTERNAL pgr_t page_new_large(MDBX_cursor *mc, const size_t npages); MDBX_INTERNAL int page_touch_modifable(MDBX_txn *txn, const page_t *const mp); -MDBX_INTERNAL int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, - const page_t *const mp); +MDBX_INTERNAL int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *const mp); static inline int page_touch(MDBX_cursor *mc) { page_t *const mp = mc->pg[mc->top]; MDBX_txn *txn = mc->txn; tASSERT(txn, mc->txn->flags & MDBX_TXN_DIRTY); - tASSERT(txn, - F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY)); + tASSERT(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY)); tASSERT(txn, !is_largepage(mp)); if (ASSERT_ENABLED()) { if (mc->flags & z_inner) { @@ -119,40 +97,31 @@ static inline int page_touch(MDBX_cursor *mc) { return page_touch_unmodifable(txn, mc, mp); } -MDBX_INTERNAL void page_copy(page_t *const dst, const page_t *const src, - const size_t size); -MDBX_INTERNAL pgr_t __must_check_result page_unspill(MDBX_txn *const txn, - const page_t *const mp); +MDBX_INTERNAL void page_copy(page_t *const dst, const page_t *const src, const size_t size); +MDBX_INTERNAL pgr_t __must_check_result page_unspill(MDBX_txn *const txn, const page_t *const mp); MDBX_INTERNAL page_t *page_shadow_alloc(MDBX_txn *txn, size_t num); -MDBX_INTERNAL void page_shadow_release(MDBX_env *env, page_t *dp, - size_t npages); +MDBX_INTERNAL void page_shadow_release(MDBX_env *env, page_t *dp, size_t npages); -MDBX_INTERNAL int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, - page_t *mp /* maybe null */, +MDBX_INTERNAL int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null */, unsigned pageflags /* maybe unknown/zero */); -static inline int page_retire(MDBX_cursor *mc, page_t *mp) { - return page_retire_ex(mc, mp->pgno, mp, mp->flags); -} +static inline int page_retire(MDBX_cursor *mc, page_t *mp) { return page_retire_ex(mc, mp->pgno, mp, mp->flags); } -static inline void page_wash(MDBX_txn *txn, size_t di, page_t *const mp, - const size_t npages) { +static inline void page_wash(MDBX_txn *txn, size_t di, page_t *const mp, const size_t npages) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); mp->txnid = INVALID_TXNID; mp->flags = P_BAD; if (txn->tw.dirtylist) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - tASSERT(txn, - MDBX_AVOID_MSYNC || (di && txn->tw.dirtylist->items[di].ptr == mp)); + tASSERT(txn, MDBX_AVOID_MSYNC || (di && txn->tw.dirtylist->items[di].ptr == mp)); if (!MDBX_AVOID_MSYNC || di) { dpl_remove_ex(txn, di, npages); txn->tw.dirtyroom++; tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom - : txn->env->options.dp_limit)); + (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP)) { page_shadow_release(txn->env, mp, npages); return; @@ -160,20 +129,14 @@ static inline void page_wash(MDBX_txn *txn, size_t di, page_t *const mp, } } else { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) && !MDBX_AVOID_MSYNC && !di); - txn->tw.writemap_dirty_npages -= (txn->tw.writemap_dirty_npages > npages) - ? npages - : txn->tw.writemap_dirty_npages; + txn->tw.writemap_dirty_npages -= (txn->tw.writemap_dirty_npages > npages) ? npages : txn->tw.writemap_dirty_npages; } VALGRIND_MAKE_MEM_UNDEFINED(mp, PAGEHDRSZ); - VALGRIND_MAKE_MEM_NOACCESS(page_data(mp), - pgno2bytes(txn->env, npages) - PAGEHDRSZ); - MDBX_ASAN_POISON_MEMORY_REGION(page_data(mp), - pgno2bytes(txn->env, npages) - PAGEHDRSZ); + VALGRIND_MAKE_MEM_NOACCESS(page_data(mp), pgno2bytes(txn->env, npages) - PAGEHDRSZ); + MDBX_ASAN_POISON_MEMORY_REGION(page_data(mp), pgno2bytes(txn->env, npages) - PAGEHDRSZ); } -MDBX_INTERNAL size_t page_subleaf2_reserve(const MDBX_env *env, - size_t host_page_room, - size_t subpage_len, size_t item_len); +MDBX_INTERNAL size_t page_subleaf2_reserve(const MDBX_env *env, size_t host_page_room, size_t subpage_len, + size_t item_len); -#define page_next(mp) \ - (*(page_t **)ptr_disp((mp)->entries, sizeof(void *) - sizeof(uint32_t))) +#define page_next(mp) (*(page_t **)ptr_disp((mp)->entries, sizeof(void *) - sizeof(uint32_t))) diff --git a/src/page-search.c b/src/page-search.c index 127e9ba0..c19a9274 100644 --- a/src/page-search.c +++ b/src/page-search.c @@ -54,14 +54,11 @@ __hot int tree_search(MDBX_cursor *mc, const MDBX_val *key, int flags) { cASSERT(mc, root >= NUM_METAS && root < mc->txn->geo.first_unallocated); if (mc->top < 0 || mc->pg[0]->pgno != root) { txnid_t pp_txnid = mc->tree->mod_txnid; - pp_txnid = /* tree->mod_txnid maybe zero in a legacy DB */ pp_txnid - ? pp_txnid - : mc->txn->txnid; + pp_txnid = /* tree->mod_txnid maybe zero in a legacy DB */ pp_txnid ? pp_txnid : mc->txn->txnid; if ((mc->txn->flags & MDBX_TXN_RDONLY) == 0) { MDBX_txn *scan = mc->txn; do - if ((scan->flags & MDBX_TXN_DIRTY) && - (dbi == MAIN_DBI || (scan->dbi_state[dbi] & DBI_DIRTY))) { + if ((scan->flags & MDBX_TXN_DIRTY) && (dbi == MAIN_DBI || (scan->dbi_state[dbi] & DBI_DIRTY))) { /* После коммита вложенных тразакций может быть mod_txnid > front */ pp_txnid = scan->front_txnid; break; @@ -75,8 +72,7 @@ __hot int tree_search(MDBX_cursor *mc, const MDBX_val *key, int flags) { mc->top = 0; mc->ki[0] = (flags & Z_LAST) ? page_numkeys(mc->pg[0]) - 1 : 0; - DEBUG("db %d root page %" PRIaPGNO " has flags 0x%X", cursor_dbi_dbg(mc), - root, mc->pg[0]->flags); + DEBUG("db %d root page %" PRIaPGNO " has flags 0x%X", cursor_dbi_dbg(mc), root, mc->pg[0]->flags); if (flags & Z_MODIFY) { err = page_touch(mc); @@ -90,8 +86,7 @@ __hot int tree_search(MDBX_cursor *mc, const MDBX_val *key, int flags) { return tree_search_finalize(mc, key, flags); } -__hot __noinline int tree_search_finalize(MDBX_cursor *mc, const MDBX_val *key, - int flags) { +__hot __noinline int tree_search_finalize(MDBX_cursor *mc, const MDBX_val *key, int flags) { cASSERT(mc, !is_poor(mc)); DKBUF_DEBUG; int err; @@ -128,16 +123,14 @@ __hot __noinline int tree_search_finalize(MDBX_cursor *mc, const MDBX_val *key, } if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->pgno, mp->flags); + ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags); err = MDBX_CORRUPTED; bailout: be_poor(mc); return err; } - DEBUG("found leaf page %" PRIaPGNO " for key [%s]", mp->pgno, - DKEY_DEBUG(key)); + DEBUG("found leaf page %" PRIaPGNO " for key [%s]", mp->pgno, DKEY_DEBUG(key)); /* Логически верно, но (в текущем понимании) нет необходимости. Однако, стоит ещё по-проверять/по-тестировать. Возможно есть сценарий, в котором очистка флагов всё-таки требуется. diff --git a/src/pnl.c b/src/pnl.c index e8825c6d..d40fe7e5 100644 --- a/src/pnl.c +++ b/src/pnl.c @@ -25,14 +25,11 @@ MDBX_INTERNAL void pnl_free(pnl_t pnl) { MDBX_INTERNAL void pnl_shrink(pnl_t __restrict *__restrict ppnl) { assert(pnl_bytes2size(pnl_size2bytes(MDBX_PNL_INITIAL)) >= MDBX_PNL_INITIAL && - pnl_bytes2size(pnl_size2bytes(MDBX_PNL_INITIAL)) < - MDBX_PNL_INITIAL * 3 / 2); - assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && - MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); + pnl_bytes2size(pnl_size2bytes(MDBX_PNL_INITIAL)) < MDBX_PNL_INITIAL * 3 / 2); + assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); MDBX_PNL_SETSIZE(*ppnl, 0); if (unlikely(MDBX_PNL_ALLOCLEN(*ppnl) > - MDBX_PNL_INITIAL * (MDBX_PNL_PREALLOC_FOR_RADIXSORT ? 8 : 4) - - MDBX_CACHELINE_SIZE / sizeof(pgno_t))) { + MDBX_PNL_INITIAL * (MDBX_PNL_PREALLOC_FOR_RADIXSORT ? 8 : 4) - MDBX_CACHELINE_SIZE / sizeof(pgno_t))) { size_t bytes = pnl_size2bytes(MDBX_PNL_INITIAL * 2); pnl_t pnl = osal_realloc(*ppnl - 1, bytes); if (likely(pnl)) { @@ -45,11 +42,9 @@ MDBX_INTERNAL void pnl_shrink(pnl_t __restrict *__restrict ppnl) { } } -MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, - const size_t wanna) { +MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, const size_t wanna) { const size_t allocated = MDBX_PNL_ALLOCLEN(*ppnl); - assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && - MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); + assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); if (likely(allocated >= wanna)) return MDBX_SUCCESS; @@ -58,9 +53,7 @@ MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, return MDBX_TXN_FULL; } - const size_t size = (wanna + wanna - allocated < PAGELIST_LIMIT) - ? wanna + wanna - allocated - : PAGELIST_LIMIT; + const size_t size = (wanna + wanna - allocated < PAGELIST_LIMIT) ? wanna + wanna - allocated : PAGELIST_LIMIT; size_t bytes = pnl_size2bytes(size); pnl_t pnl = osal_realloc(*ppnl - 1, bytes); if (likely(pnl)) { @@ -75,8 +68,8 @@ MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, return MDBX_ENOMEM; } -static __always_inline int __must_check_result pnl_append_stepped( - unsigned step, __restrict pnl_t *ppnl, pgno_t pgno, size_t n) { +static __always_inline int __must_check_result pnl_append_stepped(unsigned step, __restrict pnl_t *ppnl, pgno_t pgno, + size_t n) { assert(n > 0); int rc = pnl_need(ppnl, n); if (unlikely(rc != MDBX_SUCCESS)) @@ -106,18 +99,15 @@ static __always_inline int __must_check_result pnl_append_stepped( return MDBX_SUCCESS; } -__hot MDBX_INTERNAL int __must_check_result -spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) { +__hot MDBX_INTERNAL int __must_check_result spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) { return pnl_append_stepped(2, ppnl, pgno << 1, n); } -__hot MDBX_INTERNAL int __must_check_result -pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) { +__hot MDBX_INTERNAL int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) { return pnl_append_stepped(1, ppnl, pgno, n); } -__hot MDBX_INTERNAL int __must_check_result -pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) { +__hot MDBX_INTERNAL int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) { assert(n > 0); int rc = pnl_need(ppnl, n); if (unlikely(rc != MDBX_SUCCESS)) @@ -135,8 +125,7 @@ pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) { return MDBX_SUCCESS; } -__hot __noinline MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, - const size_t limit) { +__hot __noinline MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, const size_t limit) { assert(limit >= MIN_PAGENO - MDBX_ENABLE_REFUND); if (likely(MDBX_PNL_GETSIZE(pnl))) { if (unlikely(MDBX_PNL_GETSIZE(pnl) > PAGELIST_LIMIT)) @@ -146,8 +135,7 @@ __hot __noinline MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, if (unlikely(MDBX_PNL_MOST(pnl) >= limit)) return false; - if ((!MDBX_DISABLE_VALIDATION || AUDIT_ENABLED()) && - likely(MDBX_PNL_GETSIZE(pnl) > 1)) { + if ((!MDBX_DISABLE_VALIDATION || AUDIT_ENABLED()) && likely(MDBX_PNL_GETSIZE(pnl) > 1)) { const pgno_t *scan = MDBX_PNL_BEGIN(pnl); const pgno_t *const end = MDBX_PNL_END(pnl); pgno_t prev = *scan++; @@ -161,10 +149,9 @@ __hot __noinline MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, return true; } -static __always_inline void -pnl_merge_inner(pgno_t *__restrict dst, const pgno_t *__restrict src_a, - const pgno_t *__restrict src_b, - const pgno_t *__restrict const src_b_detent) { +static __always_inline void pnl_merge_inner(pgno_t *__restrict dst, const pgno_t *__restrict src_a, + const pgno_t *__restrict src_b, + const pgno_t *__restrict const src_b_detent) { do { #if MDBX_HAVE_CMOV const bool flag = MDBX_PNL_ORDERED(*src_b, *src_a); @@ -203,14 +190,11 @@ __hot MDBX_INTERNAL size_t pnl_merge(pnl_t dst, const pnl_t src) { total += src_len; if (!MDBX_DEBUG && total < (MDBX_HAVE_CMOV ? 21 : 12)) goto avoid_call_libc_for_short_cases; - if (dst_len == 0 || - MDBX_PNL_ORDERED(MDBX_PNL_LAST(dst), MDBX_PNL_FIRST(src))) + if (dst_len == 0 || MDBX_PNL_ORDERED(MDBX_PNL_LAST(dst), MDBX_PNL_FIRST(src))) memcpy(MDBX_PNL_END(dst), MDBX_PNL_BEGIN(src), src_len * sizeof(pgno_t)); else if (MDBX_PNL_ORDERED(MDBX_PNL_LAST(src), MDBX_PNL_FIRST(dst))) { - memmove(MDBX_PNL_BEGIN(dst) + src_len, MDBX_PNL_BEGIN(dst), - dst_len * sizeof(pgno_t)); - memcpy(MDBX_PNL_BEGIN(dst), MDBX_PNL_BEGIN(src), - src_len * sizeof(pgno_t)); + memmove(MDBX_PNL_BEGIN(dst) + src_len, MDBX_PNL_BEGIN(dst), dst_len * sizeof(pgno_t)); + memcpy(MDBX_PNL_BEGIN(dst), MDBX_PNL_BEGIN(src), src_len * sizeof(pgno_t)); } else { avoid_call_libc_for_short_cases: dst[0] = /* the detent */ (MDBX_PNL_ASCENDING ? 0 : P_INVALID); @@ -227,8 +211,7 @@ __hot MDBX_INTERNAL size_t pnl_merge(pnl_t dst, const pnl_t src) { #else #define MDBX_PNL_EXTRACT_KEY(ptr) (P_INVALID - *(ptr)) #endif -RADIXSORT_IMPL(pgno, pgno_t, MDBX_PNL_EXTRACT_KEY, - MDBX_PNL_PREALLOC_FOR_RADIXSORT, 0) +RADIXSORT_IMPL(pgno, pgno_t, MDBX_PNL_EXTRACT_KEY, MDBX_PNL_PREALLOC_FOR_RADIXSORT, 0) SORT_IMPL(pgno_sort, false, pgno_t, MDBX_PNL_ORDERED) @@ -240,8 +223,7 @@ __hot __noinline MDBX_INTERNAL void pnl_sort_nochk(pnl_t pnl) { SEARCH_IMPL(pgno_bsearch, pgno_t, pgno_t, MDBX_PNL_ORDERED) -__hot __noinline MDBX_INTERNAL size_t pnl_search_nochk(const pnl_t pnl, - pgno_t pgno) { +__hot __noinline MDBX_INTERNAL size_t pnl_search_nochk(const pnl_t pnl, pgno_t pgno) { const pgno_t *begin = MDBX_PNL_BEGIN(pnl); const pgno_t *it = pgno_bsearch(begin, MDBX_PNL_GETSIZE(pnl), pgno); const pgno_t *end = begin + MDBX_PNL_GETSIZE(pnl); diff --git a/src/pnl.h b/src/pnl.h index 8995b54d..ba033775 100644 --- a/src/pnl.h +++ b/src/pnl.h @@ -26,16 +26,15 @@ typedef const pgno_t *const_pnl_t; #define MDBX_PNL_GRANULATE_LOG2 10 #define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) -#define MDBX_PNL_INITIAL \ - (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_PNL_INITIAL (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) #define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) #define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) -#define MDBX_PNL_SETSIZE(pl, size) \ - do { \ - const size_t __size = size; \ - assert(__size < INT_MAX); \ - (pl)[0] = (pgno_t)__size; \ +#define MDBX_PNL_SETSIZE(pl, size) \ + do { \ + const size_t __size = size; \ + assert(__size < INT_MAX); \ + (pl)[0] = (pgno_t)__size; \ } while (0) #define MDBX_PNL_FIRST(pl) ((pl)[1]) #define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) @@ -62,13 +61,10 @@ MDBX_MAYBE_UNUSED static inline size_t pnl_size2bytes(size_t size) { size += size; #endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + - (PAGELIST_LIMIT * (MDBX_PNL_PREALLOC_FOR_RADIXSORT + 1) + - MDBX_PNL_GRANULATE + 3) * - sizeof(pgno_t) < + (PAGELIST_LIMIT * (MDBX_PNL_PREALLOC_FOR_RADIXSORT + 1) + MDBX_PNL_GRANULATE + 3) * sizeof(pgno_t) < SIZE_MAX / 4 * 3); size_t bytes = - ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(pgno_t) * (size + 3), - MDBX_PNL_GRANULATE * sizeof(pgno_t)) - + ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(pgno_t) * (size + 3), MDBX_PNL_GRANULATE * sizeof(pgno_t)) - MDBX_ASSUME_MALLOC_OVERHEAD; return bytes; } @@ -87,21 +83,16 @@ MDBX_INTERNAL pnl_t pnl_alloc(size_t size); MDBX_INTERNAL void pnl_free(pnl_t pnl); -MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, - const size_t wanna); +MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, const size_t wanna); -MDBX_MAYBE_UNUSED static inline int __must_check_result -pnl_need(pnl_t __restrict *__restrict ppnl, size_t num) { - assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && - MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); +MDBX_MAYBE_UNUSED static inline int __must_check_result pnl_need(pnl_t __restrict *__restrict ppnl, size_t num) { + assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); assert(num <= PAGELIST_LIMIT); const size_t wanna = MDBX_PNL_GETSIZE(*ppnl) + num; - return likely(MDBX_PNL_ALLOCLEN(*ppnl) >= wanna) ? MDBX_SUCCESS - : pnl_reserve(ppnl, wanna); + return likely(MDBX_PNL_ALLOCLEN(*ppnl) >= wanna) ? MDBX_SUCCESS : pnl_reserve(ppnl, wanna); } -MDBX_MAYBE_UNUSED static inline void -pnl_append_prereserved(__restrict pnl_t pnl, pgno_t pgno) { +MDBX_MAYBE_UNUSED static inline void pnl_append_prereserved(__restrict pnl_t pnl, pgno_t pgno) { assert(MDBX_PNL_GETSIZE(pnl) < MDBX_PNL_ALLOCLEN(pnl)); if (AUDIT_ENABLED()) { for (size_t i = MDBX_PNL_GETSIZE(pnl); i > 0; --i) @@ -113,14 +104,11 @@ pnl_append_prereserved(__restrict pnl_t pnl, pgno_t pgno) { MDBX_INTERNAL void pnl_shrink(pnl_t __restrict *__restrict ppnl); -MDBX_INTERNAL int __must_check_result spill_append_span(__restrict pnl_t *ppnl, - pgno_t pgno, size_t n); +MDBX_INTERNAL int __must_check_result spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); -MDBX_INTERNAL int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, - pgno_t pgno, size_t n); +MDBX_INTERNAL int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); -MDBX_INTERNAL int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, - pgno_t pgno, size_t n); +MDBX_INTERNAL int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); MDBX_INTERNAL size_t pnl_search_nochk(const pnl_t pnl, pgno_t pgno); @@ -128,10 +116,8 @@ MDBX_INTERNAL void pnl_sort_nochk(pnl_t pnl); MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, const size_t limit); -MDBX_MAYBE_UNUSED static inline bool pnl_check_allocated(const const_pnl_t pnl, - const size_t limit) { - return pnl == nullptr || (MDBX_PNL_ALLOCLEN(pnl) >= MDBX_PNL_GETSIZE(pnl) && - pnl_check(pnl, limit)); +MDBX_MAYBE_UNUSED static inline bool pnl_check_allocated(const const_pnl_t pnl, const size_t limit) { + return pnl == nullptr || (MDBX_PNL_ALLOCLEN(pnl) >= MDBX_PNL_GETSIZE(pnl) && pnl_check(pnl, limit)); } MDBX_MAYBE_UNUSED static inline void pnl_sort(pnl_t pnl, size_t limit4check) { @@ -140,8 +126,7 @@ MDBX_MAYBE_UNUSED static inline void pnl_sort(pnl_t pnl, size_t limit4check) { (void)limit4check; } -MDBX_MAYBE_UNUSED static inline size_t pnl_search(const pnl_t pnl, pgno_t pgno, - size_t limit) { +MDBX_MAYBE_UNUSED static inline size_t pnl_search(const pnl_t pnl, pgno_t pgno, size_t limit) { assert(pnl_check_allocated(pnl, limit)); if (MDBX_HAVE_CMOV) { /* cmov-ускоренный бинарный поиск может читать (но не использовать) один diff --git a/src/preface.h b/src/preface.h index 8bffbf6e..962c7ae7 100644 --- a/src/preface.h +++ b/src/preface.h @@ -4,8 +4,7 @@ #pragma once /* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ -#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || \ - (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS) +#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS) #undef NDEBUG #ifndef MDBX_DEBUG /* Чтобы избежать включения отладки только из-за включения assert-проверок */ @@ -29,8 +28,7 @@ #endif /* MDBX_DISABLE_GNU_SOURCE */ /* Should be defined before any includes */ -#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \ - !defined(ANDROID) +#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && !defined(ANDROID) #define _FILE_OFFSET_BITS 64 #endif /* _FILE_OFFSET_BITS */ @@ -38,8 +36,7 @@ #define _DARWIN_C_SOURCE #endif /* _DARWIN_C_SOURCE */ -#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \ - !defined(__USE_MINGW_ANSI_STDIO) +#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO) #define __USE_MINGW_ANSI_STDIO 1 #endif /* MinGW */ @@ -56,8 +53,7 @@ #define UNICODE #endif /* UNICODE */ -#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ - !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT #define _NO_CRT_STDIO_INLINE #endif /* _NO_CRT_STDIO_INLINE */ @@ -72,8 +68,7 @@ #endif /* NOMINMAX */ /* Workaround for modern libstdc++ with CLANG < 4.x */ -#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && \ - defined(__clang__) && __clang_major__ < 4 +#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && defined(__clang__) && __clang_major__ < 4 #define __GLIBCXX_BITSIZE_INT_N_0 128 #define __GLIBCXX_TYPE_INT_N_0 __int128 #endif /* Workaround for modern libstdc++ with CLANG < 4.x */ @@ -107,8 +102,7 @@ * and how to and where you can obtain the latest "Visual Studio 2015" build * with all fixes. */ -#error \ - "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +#error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." #endif #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ @@ -117,9 +111,8 @@ #pragma warning(disable : 5045) /* will insert Spectre mitigation... */ #endif #if _MSC_VER > 1914 -#pragma warning( \ - disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ - producing 'defined' has undefined behavior */ +#pragma warning(disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ + producing 'defined' has undefined behavior */ #endif #if _MSC_VER < 1920 /* avoid "error C2219: syntax error: type qualifier must be after '*'" */ @@ -127,33 +120,32 @@ #endif #if _MSC_VER > 1930 #pragma warning(disable : 6235) /* is always a constant */ -#pragma warning(disable : 6237) /* is never evaluated and might \ +#pragma warning(disable : 6237) /* is never evaluated and might \ have side effects */ #endif #pragma warning(disable : 4710) /* 'xyz': function not inlined */ -#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ inline expansion */ -#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ +#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ struct/union */ #pragma warning(disable : 4702) /* unreachable code */ #pragma warning(disable : 4706) /* assignment within conditional expression */ #pragma warning(disable : 4127) /* conditional expression is constant */ -#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ alignment specifier */ #pragma warning(disable : 4310) /* cast truncates constant value */ -#pragma warning(disable : 4820) /* bytes padding added after data member for \ +#pragma warning(disable : 4820) /* bytes padding added after data member for \ alignment */ -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ unaligned */ -#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ array in struct/union */ -#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ aggregate initializer */ -#pragma warning( \ - disable : 4505) /* unreferenced local function has been removed */ -#endif /* _MSC_VER (warnings) */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ #if defined(__GNUC__) && __GNUC__ < 9 #pragma GCC diagnostic ignored "-Wattributes" @@ -166,12 +158,12 @@ #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #endif /* _MSC_VER (warnings) */ @@ -232,8 +224,7 @@ #ifndef __GNUC_PREREQ #if defined(__GNUC__) && defined(__GNUC_MINOR__) -#define __GNUC_PREREQ(maj, min) \ - ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +#define __GNUC_PREREQ(maj, min) ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GNUC_PREREQ(maj, min) (0) #endif @@ -241,8 +232,7 @@ #ifndef __CLANG_PREREQ #ifdef __clang__ -#define __CLANG_PREREQ(maj, min) \ - ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +#define __CLANG_PREREQ(maj, min) ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) #else #define __CLANG_PREREQ(maj, min) (0) #endif @@ -250,8 +240,7 @@ #ifndef __GLIBC_PREREQ #if defined(__GLIBC__) && defined(__GLIBC_MINOR__) -#define __GLIBC_PREREQ(maj, min) \ - ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +#define __GLIBC_PREREQ(maj, min) ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GLIBC_PREREQ(maj, min) (0) #endif @@ -261,8 +250,7 @@ /* pre-requirements */ #if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF -#error \ - "Sanity checking failed: Two's complement, reasonably sized integer types" +#error "Sanity checking failed: Two's complement, reasonably sized integer types" #endif #ifndef SSIZE_MAX @@ -294,8 +282,7 @@ #endif #ifdef __SANITIZE_THREAD__ -#warning \ - "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." #endif /* __SANITIZE_THREAD__ */ /*----------------------------------------------------------------------------*/ @@ -327,8 +314,7 @@ #endif #endif /* __extern_C */ -#if !defined(nullptr) && !defined(__cplusplus) || \ - (__cplusplus < 201103L && !defined(_MSC_VER)) +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) #define nullptr NULL #endif @@ -340,9 +326,8 @@ #endif #endif /* Apple OSX & iOS */ -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) #include #include #include @@ -359,8 +344,7 @@ #endif #else #include -#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ - defined(_WIN32) || defined(_WIN64)) +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(_WIN32) || defined(_WIN64)) #include #endif /* !Solaris */ #endif /* !xBSD */ @@ -469,43 +453,38 @@ __extern_C key_t ftok(const char *, int); /*----------------------------------------------------------------------------*/ /* Byteorder */ -#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ - defined(i486) || defined(__i486) || defined(__i486__) || defined(i586) || \ - defined(__i586) || defined(__i586__) || defined(i686) || \ - defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ - defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ - defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ - defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || defined(i486) || defined(__i486) || \ + defined(__i486__) || defined(i586) || defined(__i586) || defined(__i586__) || defined(i686) || defined(__i686) || \ + defined(__i686__) || defined(_M_IX86) || defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ + defined(_M_X64) || defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) #ifndef __ia32__ /* LY: define neutral __ia32__ for x86 and x86-64 */ #define __ia32__ 1 #endif /* __ia32__ */ -#if !defined(__amd64__) && \ - (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64)) +#if !defined(__amd64__) && \ + (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64) || defined(_M_AMD64)) /* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ #define __amd64__ 1 #endif /* __amd64__ */ #endif /* all x86 */ -#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ - !defined(__ORDER_BIG_ENDIAN__) +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || !defined(__ORDER_BIG_ENDIAN__) -#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || \ - defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || __has_include() +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || \ + __has_include() #include -#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ - defined(HAVE_MACHINE_ENDIAN_H) || __has_include() +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || defined(HAVE_MACHINE_ENDIAN_H) || \ + __has_include() #include #elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include() #include -#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ (__has_include() && __has_include()) #include #include -#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ - defined(__NetBSD__) || defined(HAVE_SYS_PARAM_H) || __has_include() +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include() #include #endif /* OS */ @@ -521,27 +500,19 @@ __extern_C key_t ftok(const char *, int); #define __ORDER_LITTLE_ENDIAN__ 1234 #define __ORDER_BIG_ENDIAN__ 4321 -#if defined(__LITTLE_ENDIAN__) || \ - (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ - defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ - defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ - defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ - defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ - defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ - defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ - defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ - defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ - defined(__WINDOWS__) +#if defined(__LITTLE_ENDIAN__) || (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || defined(__ARMEL__) || \ + defined(__THUMBEL__) || defined(__AARCH64EL__) || defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || defined(__elbrus_4c__) || defined(__elbrus_8c__) || \ + defined(__bfin__) || defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || defined(__IA64__) || \ + defined(__ia64) || defined(_M_IA64) || defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || defined(__WINDOWS__) #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#elif defined(__BIG_ENDIAN__) || \ - (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ - defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ - defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ - defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ - defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ - defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ - defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#elif defined(__BIG_ENDIAN__) || (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || defined(__ARMEB__) || \ + defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \ + defined(__sparc__) || defined(__sparc) || defined(__370__) || defined(__THW_370__) || defined(__s390__) || \ + defined(__s390x__) || defined(__SYSC_ZARCH__) #define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ #else @@ -561,17 +532,14 @@ __extern_C key_t ftok(const char *, int); #define MDBX_HAVE_CMOV 1 #elif defined(__thumb__) || defined(__thumb) || defined(__TARGET_ARCH_THUMB) #define MDBX_HAVE_CMOV 0 -#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || \ - defined(__aarch64) || defined(__arm__) || defined(__arm) || \ - defined(__CC_ARM) +#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || defined(__aarch64) || defined(__arm__) || \ + defined(__arm) || defined(__CC_ARM) #define MDBX_HAVE_CMOV 1 -#elif (defined(__riscv__) || defined(__riscv64)) && \ - (defined(__riscv_b) || defined(__riscv_bitmanip)) +#elif (defined(__riscv__) || defined(__riscv64)) && (defined(__riscv_b) || defined(__riscv_bitmanip)) #define MDBX_HAVE_CMOV 1 -#elif defined(i686) || defined(__i686) || defined(__i686__) || \ - (defined(_M_IX86) && _M_IX86 > 600) || defined(__x86_64) || \ - defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64) +#elif defined(i686) || defined(__i686) || defined(__i686__) || (defined(_M_IX86) && _M_IX86 > 600) || \ + defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) #define MDBX_HAVE_CMOV 1 #else #define MDBX_HAVE_CMOV 0 @@ -597,8 +565,7 @@ __extern_C key_t ftok(const char *, int); #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) #include -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) #include #elif defined(__IBMC__) && defined(__powerpc) #include @@ -620,29 +587,26 @@ __extern_C key_t ftok(const char *, int); #endif /* Compiler */ #if !defined(__noop) && !defined(_MSC_VER) -#define __noop \ - do { \ +#define __noop \ + do { \ } while (0) #endif /* __noop */ -#if defined(__fallthrough) && \ - (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) +#if defined(__fallthrough) && (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) #undef __fallthrough #endif /* __fallthrough workaround for MinGW */ #ifndef __fallthrough -#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ - (!defined(__clang__) || __clang__ > 4)) || \ +#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && (!defined(__clang__) || __clang__ > 4)) || \ __cplusplus >= 201703L #define __fallthrough [[fallthrough]] #elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L #define __fallthrough [[fallthrough]] -#elif __GNUC_PREREQ(7, 0) && \ - (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ - (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +#elif __GNUC_PREREQ(7, 0) && (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) #define __fallthrough __attribute__((__fallthrough__)) -#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && \ - __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && __has_feature(cxx_attributes) && \ + __has_warning("-Wimplicit-fallthrough") #define __fallthrough [[clang::fallthrough]] #else #define __fallthrough @@ -655,8 +619,8 @@ __extern_C key_t ftok(const char *, int); #elif defined(_MSC_VER) #define __unreachable() __assume(0) #else -#define __unreachable() \ - do { \ +#define __unreachable() \ + do { \ } while (1) #endif #endif /* __unreachable */ @@ -665,9 +629,9 @@ __extern_C key_t ftok(const char *, int); #if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) #define __prefetch(ptr) __builtin_prefetch(ptr) #else -#define __prefetch(ptr) \ - do { \ - (void)(ptr); \ +#define __prefetch(ptr) \ + do { \ + (void)(ptr); \ } while (0) #endif #endif /* __prefetch */ @@ -677,8 +641,7 @@ __extern_C key_t ftok(const char *, int); #endif /* offsetof */ #ifndef container_of -#define container_of(ptr, type, member) \ - ((type *)((char *)(ptr) - offsetof(type, member))) +#define container_of(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member))) #endif /* container_of */ /*----------------------------------------------------------------------------*/ @@ -750,8 +713,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __hot #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__hot__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__hot__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put frequently used functions in separate section */ #define __hot __attribute__((__section__("text.hot"))) __optimize("O3") @@ -767,8 +729,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __cold #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__cold__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__cold__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put infrequently used functions in separate section */ #define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") @@ -791,8 +752,7 @@ __extern_C key_t ftok(const char *, int); #endif /* __flatten */ #ifndef likely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define likely(cond) __builtin_expect(!!(cond), 1) #else #define likely(x) (!!(x)) @@ -800,8 +760,7 @@ __extern_C key_t ftok(const char *, int); #endif /* likely */ #ifndef unlikely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define unlikely(cond) __builtin_expect(!!(cond), 0) #else #define unlikely(x) (!!(x)) @@ -821,8 +780,7 @@ __extern_C key_t ftok(const char *, int); #define MDBX_WEAK_IMPORT_ATTRIBUTE WEAK_IMPORT_ATTRIBUTE #elif __has_attribute(__weak__) && __has_attribute(__weak_import__) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__, __weak_import__)) -#elif __has_attribute(__weak__) || \ - (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) +#elif __has_attribute(__weak__) || (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__)) #else #define MDBX_WEAK_IMPORT_ATTRIBUTE @@ -835,9 +793,7 @@ __extern_C key_t ftok(const char *, int); #ifndef MDBX_EXCLUDE_FOR_GPROF #ifdef ENABLE_GPROF -#define MDBX_EXCLUDE_FOR_GPROF \ - __attribute__((__no_instrument_function__, \ - __no_profile_instrument_function__)) +#define MDBX_EXCLUDE_FOR_GPROF __attribute__((__no_instrument_function__, __no_profile_instrument_function__)) #else #define MDBX_EXCLUDE_FOR_GPROF #endif /* ENABLE_GPROF */ @@ -846,10 +802,9 @@ __extern_C key_t ftok(const char *, int); /*----------------------------------------------------------------------------*/ #ifndef expect_with_probability -#if defined(__builtin_expect_with_probability) || \ - __has_builtin(__builtin_expect_with_probability) || __GNUC_PREREQ(9, 0) -#define expect_with_probability(expr, value, prob) \ - __builtin_expect_with_probability(expr, value, prob) +#if defined(__builtin_expect_with_probability) || __has_builtin(__builtin_expect_with_probability) || \ + __GNUC_PREREQ(9, 0) +#define expect_with_probability(expr, value, prob) __builtin_expect_with_probability(expr, value, prob) #else #define expect_with_probability(expr, value, prob) (expr) #endif @@ -866,11 +821,9 @@ __extern_C key_t ftok(const char *, int); #if MDBX_GOOFY_MSVC_STATIC_ANALYZER || (defined(_MSC_VER) && _MSC_VER > 1919) #define MDBX_ANALYSIS_ASSUME(expr) __analysis_assume(expr) #ifdef _PREFAST_ -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(prefast(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(prefast(suppress : warn_id)) #else -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(warning(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(warning(suppress : warn_id)) #endif #else #define MDBX_ANALYSIS_ASSUME(expr) assert(expr) @@ -878,8 +831,7 @@ __extern_C key_t ftok(const char *, int); #endif /* MDBX_GOOFY_MSVC_STATIC_ANALYZER */ #ifndef FLEXIBLE_ARRAY_MEMBERS -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || (!defined(__cplusplus) && defined(_MSC_VER)) #define FLEXIBLE_ARRAY_MEMBERS 1 #else #define FLEXIBLE_ARRAY_MEMBERS 0 @@ -938,8 +890,7 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define CONCAT(a, b) a##b #define XCONCAT(a, b) CONCAT(a, b) -#define MDBX_TETRAD(a, b, c, d) \ - ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) +#define MDBX_TETRAD(a, b, c, d) ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) #define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) @@ -953,14 +904,13 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #elif defined(_MSC_VER) #include #define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) -#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \ - __has_feature(c_static_assert) +#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __has_feature(c_static_assert) #define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) #else -#define STATIC_ASSERT_MSG(expr, msg) \ - switch (0) { \ - case 0: \ - case (expr):; \ +#define STATIC_ASSERT_MSG(expr, msg) \ + switch (0) { \ + case 0: \ + case (expr):; \ } #endif #endif /* STATIC_ASSERT */ diff --git a/src/proto.h b/src/proto.h index 28562eb2..d4cc67f4 100644 --- a/src/proto.h +++ b/src/proto.h @@ -8,39 +8,26 @@ /* Internal prototypes */ /* audit.c */ -MDBX_INTERNAL int audit_ex(MDBX_txn *txn, size_t retired_stored, - bool dont_filter_gc); +MDBX_INTERNAL int audit_ex(MDBX_txn *txn, size_t retired_stored, bool dont_filter_gc); /* mvcc-readers.c */ MDBX_INTERNAL bsr_t mvcc_bind_slot(MDBX_env *env); -MDBX_MAYBE_UNUSED MDBX_INTERNAL pgno_t mvcc_largest_this(MDBX_env *env, - pgno_t largest); -MDBX_INTERNAL txnid_t mvcc_shapshot_oldest(MDBX_env *const env, - const txnid_t steady); -MDBX_INTERNAL pgno_t mvcc_snapshot_largest(const MDBX_env *env, - pgno_t last_used_page); -MDBX_INTERNAL txnid_t mvcc_kick_laggards(MDBX_env *env, - const txnid_t straggler); +MDBX_MAYBE_UNUSED MDBX_INTERNAL pgno_t mvcc_largest_this(MDBX_env *env, pgno_t largest); +MDBX_INTERNAL txnid_t mvcc_shapshot_oldest(MDBX_env *const env, const txnid_t steady); +MDBX_INTERNAL pgno_t mvcc_snapshot_largest(const MDBX_env *env, pgno_t last_used_page); +MDBX_INTERNAL txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler); MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rlocked, int *dead); MDBX_INTERNAL txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t laggard); /* dxb.c */ -MDBX_INTERNAL int dxb_setup(MDBX_env *env, const int lck_rc, - const mdbx_mode_t mode_bits); -MDBX_INTERNAL int __must_check_result -dxb_read_header(MDBX_env *env, meta_t *meta, const int lck_exclusive, - const mdbx_mode_t mode_bits); +MDBX_INTERNAL int dxb_setup(MDBX_env *env, const int lck_rc, const mdbx_mode_t mode_bits); +MDBX_INTERNAL int __must_check_result dxb_read_header(MDBX_env *env, meta_t *meta, const int lck_exclusive, + const mdbx_mode_t mode_bits); enum resize_mode { implicit_grow, impilict_shrink, explicit_resize }; -MDBX_INTERNAL int __must_check_result dxb_resize(MDBX_env *const env, - const pgno_t used_pgno, - const pgno_t size_pgno, - pgno_t limit_pgno, - const enum resize_mode mode); -MDBX_INTERNAL int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, - const bool enable, const bool force_whole); -MDBX_INTERNAL int __must_check_result dxb_sync_locked(MDBX_env *env, - unsigned flags, - meta_t *const pending, +MDBX_INTERNAL int __must_check_result dxb_resize(MDBX_env *const env, const pgno_t used_pgno, const pgno_t size_pgno, + pgno_t limit_pgno, const enum resize_mode mode); +MDBX_INTERNAL int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, const bool enable, const bool force_whole); +MDBX_INTERNAL int __must_check_result dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, troika_t *const troika); #if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__) MDBX_INTERNAL void dxb_sanitize_tail(MDBX_env *env, MDBX_txn *txn); @@ -60,9 +47,8 @@ MDBX_INTERNAL int txn_park(MDBX_txn *txn, bool autounpark); MDBX_INTERNAL int txn_unpark(MDBX_txn *txn); MDBX_INTERNAL int txn_check_badbits_parked(const MDBX_txn *txn, int bad_bits); -#define TXN_END_NAMES \ - {"committed", "empty-commit", "abort", "reset", \ - "fail-begin", "fail-beginchild", "ousted", nullptr} +#define TXN_END_NAMES \ + {"committed", "empty-commit", "abort", "reset", "fail-begin", "fail-beginchild", "ousted", nullptr} enum { /* txn_end operation number, for logging */ TXN_END_COMMITTED, @@ -84,8 +70,7 @@ MDBX_INTERNAL int txn_write(MDBX_txn *txn, iov_ctx_t *ctx); /* env.c */ MDBX_INTERNAL int env_open(MDBX_env *env, mdbx_mode_t mode); -MDBX_INTERNAL int env_info(const MDBX_env *env, const MDBX_txn *txn, - MDBX_envinfo *out, size_t bytes, troika_t *troika); +MDBX_INTERNAL int env_info(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out, size_t bytes, troika_t *troika); MDBX_INTERNAL int env_sync(MDBX_env *env, bool force, bool nonblock); MDBX_INTERNAL int env_close(MDBX_env *env, bool resurrect_after_fork); MDBX_INTERNAL bool env_txn0_owned(const MDBX_env *env); @@ -97,27 +82,17 @@ MDBX_INTERNAL unsigned env_setup_pagesize(MDBX_env *env, const size_t pagesize); /* tree.c */ MDBX_INTERNAL int tree_drop(MDBX_cursor *mc, const bool may_have_tables); MDBX_INTERNAL int __must_check_result tree_rebalance(MDBX_cursor *mc); -MDBX_INTERNAL int __must_check_result tree_propagate_key(MDBX_cursor *mc, - const MDBX_val *key); +MDBX_INTERNAL int __must_check_result tree_propagate_key(MDBX_cursor *mc, const MDBX_val *key); MDBX_INTERNAL void recalculate_merge_thresholds(MDBX_env *env); MDBX_INTERNAL void recalculate_subpage_thresholds(MDBX_env *env); /* table.c */ MDBX_INTERNAL int __must_check_result tbl_fetch(MDBX_txn *txn, size_t dbi); -MDBX_INTERNAL int __must_check_result tbl_setup(const MDBX_env *env, - kvx_t *const kvx, - const tree_t *const db); +MDBX_INTERNAL int __must_check_result tbl_setup(const MDBX_env *env, kvx_t *const kvx, const tree_t *const db); /* coherency.c */ -MDBX_INTERNAL bool coherency_check_meta(const MDBX_env *env, - const volatile meta_t *meta, - bool report); -MDBX_INTERNAL int coherency_fetch_head(MDBX_txn *txn, const meta_ptr_t head, - uint64_t *timestamp); -MDBX_INTERNAL int coherency_check_written(const MDBX_env *env, - const txnid_t txnid, - const volatile meta_t *meta, - const intptr_t pgno, - uint64_t *timestamp); -MDBX_INTERNAL int coherency_timeout(uint64_t *timestamp, intptr_t pgno, - const MDBX_env *env); +MDBX_INTERNAL bool coherency_check_meta(const MDBX_env *env, const volatile meta_t *meta, bool report); +MDBX_INTERNAL int coherency_fetch_head(MDBX_txn *txn, const meta_ptr_t head, uint64_t *timestamp); +MDBX_INTERNAL int coherency_check_written(const MDBX_env *env, const txnid_t txnid, const volatile meta_t *meta, + const intptr_t pgno, uint64_t *timestamp); +MDBX_INTERNAL int coherency_timeout(uint64_t *timestamp, intptr_t pgno, const MDBX_env *env); diff --git a/src/range-estimate.c b/src/range-estimate.c index 2deb3905..ea093088 100644 --- a/src/range-estimate.c +++ b/src/range-estimate.c @@ -10,20 +10,17 @@ typedef struct diff_result { } diff_t; /* calculates: r = x - y */ -__hot static int cursor_diff(const MDBX_cursor *const __restrict x, - const MDBX_cursor *const __restrict y, +__hot static int cursor_diff(const MDBX_cursor *const __restrict x, const MDBX_cursor *const __restrict y, diff_t *const __restrict r) { r->diff = 0; r->level = 0; r->root_nkeys = 0; if (unlikely(x->signature != cur_signature_live)) - return (x->signature == cur_signature_ready4dispose) ? MDBX_EINVAL - : MDBX_EBADSIGN; + return (x->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN; if (unlikely(y->signature != cur_signature_live)) - return (y->signature == cur_signature_ready4dispose) ? MDBX_EINVAL - : MDBX_EBADSIGN; + return (y->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN; int rc = check_txn(x->txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) @@ -86,8 +83,7 @@ __hot static int cursor_diff(const MDBX_cursor *const __restrict x, return MDBX_SUCCESS; } -__hot static ptrdiff_t estimate(const tree_t *tree, - diff_t *const __restrict dr) { +__hot static ptrdiff_t estimate(const tree_t *tree, diff_t *const __restrict dr) { /* root: branch-page => scale = leaf-factor * branch-factor^(N-1) * level-1: branch-page(s) => scale = leaf-factor * branch-factor^2 * level-2: branch-page(s) => scale = leaf-factor * branch-factor @@ -98,8 +94,7 @@ __hot static ptrdiff_t estimate(const tree_t *tree, if (btree_power < 0) return dr->diff; - ptrdiff_t estimated = - (ptrdiff_t)tree->items * dr->diff / (ptrdiff_t)tree->leaf_pages; + ptrdiff_t estimated = (ptrdiff_t)tree->items * dr->diff / (ptrdiff_t)tree->leaf_pages; if (btree_power == 0) return estimated; @@ -112,9 +107,7 @@ __hot static ptrdiff_t estimate(const tree_t *tree, total(branch_entries) = leaf_pages + branch_pages - 1 (root page) */ const size_t log2_fixedpoint = sizeof(size_t) - 1; const size_t half = UINT64_C(1) << (log2_fixedpoint - 1); - const size_t factor = - ((tree->leaf_pages + tree->branch_pages - 1) << log2_fixedpoint) / - tree->branch_pages; + const size_t factor = ((tree->leaf_pages + tree->branch_pages - 1) << log2_fixedpoint) / tree->branch_pages; while (1) { switch ((size_t)btree_power) { default: { @@ -149,11 +142,8 @@ __hot static ptrdiff_t estimate(const tree_t *tree, } } -__hot int mdbx_estimate_distance(const MDBX_cursor *first, - const MDBX_cursor *last, - ptrdiff_t *distance_items) { - if (unlikely(first == nullptr || last == nullptr || - distance_items == nullptr)) +__hot int mdbx_estimate_distance(const MDBX_cursor *first, const MDBX_cursor *last, ptrdiff_t *distance_items) { + if (unlikely(first == nullptr || last == nullptr || distance_items == nullptr)) return LOG_IFERR(MDBX_EINVAL); *distance_items = 0; @@ -177,17 +167,14 @@ __hot int mdbx_estimate_distance(const MDBX_cursor *first, return MDBX_SUCCESS; } -__hot int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, - MDBX_val *data, MDBX_cursor_op move_op, +__hot int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, MDBX_cursor_op move_op, ptrdiff_t *distance_items) { - if (unlikely(cursor == nullptr || distance_items == nullptr || - move_op == MDBX_GET_CURRENT || move_op == MDBX_GET_MULTIPLE)) + if (unlikely(cursor == nullptr || distance_items == nullptr || move_op == MDBX_GET_CURRENT || + move_op == MDBX_GET_MULTIPLE)) return LOG_IFERR(MDBX_EINVAL); if (unlikely(cursor->signature != cur_signature_live)) - return LOG_IFERR((cursor->signature == cur_signature_ready4dispose) - ? MDBX_EINVAL - : MDBX_EBADSIGN); + return LOG_IFERR((cursor->signature == cur_signature_ready4dispose) ? MDBX_EINVAL : MDBX_EBADSIGN); int rc = check_txn(cursor->txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) @@ -209,8 +196,7 @@ __hot int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, MDBX_val stub_data; if (data == nullptr) { - const unsigned mask = - 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY; + const unsigned mask = 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY; if (unlikely(mask & (1 << move_op))) return LOG_IFERR(MDBX_EINVAL); stub_data.iov_base = nullptr; @@ -220,9 +206,8 @@ __hot int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, MDBX_val stub_key; if (key == nullptr) { - const unsigned mask = 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | - 1 << MDBX_SET_KEY | 1 << MDBX_SET | - 1 << MDBX_SET_RANGE; + const unsigned mask = + 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY | 1 << MDBX_SET | 1 << MDBX_SET_RANGE; if (unlikely(mask & (1 << move_op))) return LOG_IFERR(MDBX_EINVAL); stub_key.iov_base = nullptr; @@ -232,8 +217,7 @@ __hot int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, next.outer.signature = cur_signature_live; rc = cursor_ops(&next.outer, key, data, move_op); - if (unlikely(rc != MDBX_SUCCESS && - (rc != MDBX_NOTFOUND || !is_pointed(&next.outer)))) + if (unlikely(rc != MDBX_SUCCESS && (rc != MDBX_NOTFOUND || !is_pointed(&next.outer)))) return LOG_IFERR(rc); if (move_op == MDBX_LAST) { @@ -243,11 +227,8 @@ __hot int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, return mdbx_estimate_distance(cursor, &next.outer, distance_items); } -__hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, - const MDBX_val *begin_key, - const MDBX_val *begin_data, - const MDBX_val *end_key, const MDBX_val *end_data, - ptrdiff_t *size_items) { +__hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *begin_key, const MDBX_val *begin_data, + const MDBX_val *end_key, const MDBX_val *end_data, ptrdiff_t *size_items) { int rc = check_txn(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); @@ -255,8 +236,7 @@ __hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, if (unlikely(!size_items)) return LOG_IFERR(MDBX_EINVAL); - if (unlikely(begin_data && - (begin_key == nullptr || begin_key == MDBX_EPSILON))) + if (unlikely(begin_data && (begin_key == nullptr || begin_key == MDBX_EPSILON))) return LOG_IFERR(MDBX_EINVAL); if (unlikely(end_data && (end_key == nullptr || end_key == MDBX_EPSILON))) @@ -285,20 +265,14 @@ __hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, rc = outer_first(&begin.outer, nullptr, nullptr); if (unlikely(end_key == MDBX_EPSILON)) { /* LY: FIRST..+epsilon case */ - return LOG_IFERR( - (rc == MDBX_SUCCESS) - ? mdbx_cursor_count(&begin.outer, (size_t *)size_items) - : rc); + return LOG_IFERR((rc == MDBX_SUCCESS) ? mdbx_cursor_count(&begin.outer, (size_t *)size_items) : rc); } } else { if (unlikely(begin_key == MDBX_EPSILON)) { if (end_key == nullptr) { /* LY: -epsilon..LAST case */ rc = outer_last(&begin.outer, nullptr, nullptr); - return LOG_IFERR( - (rc == MDBX_SUCCESS) - ? mdbx_cursor_count(&begin.outer, (size_t *)size_items) - : rc); + return LOG_IFERR((rc == MDBX_SUCCESS) ? mdbx_cursor_count(&begin.outer, (size_t *)size_items) : rc); } /* LY: -epsilon..value case */ assert(end_key != MDBX_EPSILON); @@ -309,22 +283,19 @@ __hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, end_key = begin_key; } if (end_key && !begin_data && !end_data && - (begin_key == end_key || - begin.outer.clc->k.cmp(begin_key, end_key) == 0)) { + (begin_key == end_key || begin.outer.clc->k.cmp(begin_key, end_key) == 0)) { /* LY: single key case */ - rc = cursor_seek(&begin.outer, (MDBX_val *)begin_key, nullptr, MDBX_SET) - .err; + rc = cursor_seek(&begin.outer, (MDBX_val *)begin_key, nullptr, MDBX_SET).err; if (unlikely(rc != MDBX_SUCCESS)) { *size_items = 0; return LOG_IFERR((rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc); } *size_items = 1; if (inner_pointed(&begin.outer)) - *size_items = - (sizeof(*size_items) >= sizeof(begin.inner.nested_tree.items) || - begin.inner.nested_tree.items <= PTRDIFF_MAX) - ? (size_t)begin.inner.nested_tree.items - : PTRDIFF_MAX; + *size_items = (sizeof(*size_items) >= sizeof(begin.inner.nested_tree.items) || + begin.inner.nested_tree.items <= PTRDIFF_MAX) + ? (size_t)begin.inner.nested_tree.items + : PTRDIFF_MAX; return MDBX_SUCCESS; } else { @@ -332,9 +303,7 @@ __hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val proxy_data = {nullptr, 0}; if (begin_data) proxy_data = *begin_data; - rc = LOG_IFERR(cursor_seek(&begin.outer, &proxy_key, &proxy_data, - MDBX_SET_LOWERBOUND) - .err); + rc = LOG_IFERR(cursor_seek(&begin.outer, &proxy_key, &proxy_data, MDBX_SET_LOWERBOUND).err); } } @@ -356,8 +325,7 @@ __hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val proxy_data = {nullptr, 0}; if (end_data) proxy_data = *end_data; - rc = cursor_seek(&end.outer, &proxy_key, &proxy_data, MDBX_SET_LOWERBOUND) - .err; + rc = cursor_seek(&end.outer, &proxy_key, &proxy_data, MDBX_SET_LOWERBOUND).err; } if (unlikely(rc != MDBX_SUCCESS)) { if (rc != MDBX_NOTFOUND || !is_pointed(&end.outer)) @@ -367,10 +335,9 @@ __hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, rc = mdbx_estimate_distance(&begin.outer, &end.outer, size_items); if (unlikely(rc != MDBX_SUCCESS)) return LOG_IFERR(rc); - assert(*size_items >= -(ptrdiff_t)begin.outer.tree->items && - *size_items <= (ptrdiff_t)begin.outer.tree->items); + assert(*size_items >= -(ptrdiff_t)begin.outer.tree->items && *size_items <= (ptrdiff_t)begin.outer.tree->items); -#if 0 /* LY: Was decided to returns as-is (i.e. negative) the estimation \ +#if 0 /* LY: Was decided to returns as-is (i.e. negative) the estimation \ * results for an inverted ranges. */ /* Commit 8ddfd1f34ad7cf7a3c4aa75d2e248ca7e639ed63 diff --git a/src/refund.c b/src/refund.c index 3742e569..2d1ef607 100644 --- a/src/refund.c +++ b/src/refund.c @@ -8,8 +8,7 @@ static void refund_reclaimed(MDBX_txn *txn) { /* Scanning in descend order */ pgno_t first_unallocated = txn->geo.first_unallocated; const pnl_t pnl = txn->tw.relist; - tASSERT(txn, - MDBX_PNL_GETSIZE(pnl) && MDBX_PNL_MOST(pnl) == first_unallocated - 1); + tASSERT(txn, MDBX_PNL_GETSIZE(pnl) && MDBX_PNL_MOST(pnl) == first_unallocated - 1); #if MDBX_PNL_ASCENDING size_t i = MDBX_PNL_GETSIZE(pnl); tASSERT(txn, pnl[i] == first_unallocated - 1); @@ -26,12 +25,10 @@ static void refund_reclaimed(MDBX_txn *txn) { for (size_t move = 0; move < len; ++move) pnl[1 + move] = pnl[i + move]; #endif - VERBOSE("refunded %" PRIaPGNO " pages: %" PRIaPGNO " -> %" PRIaPGNO, - txn->geo.first_unallocated - first_unallocated, + VERBOSE("refunded %" PRIaPGNO " pages: %" PRIaPGNO " -> %" PRIaPGNO, txn->geo.first_unallocated - first_unallocated, txn->geo.first_unallocated, first_unallocated); txn->geo.first_unallocated = first_unallocated; - tASSERT(txn, - pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - 1)); + tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - 1)); } static void refund_loose(MDBX_txn *txn) { @@ -58,18 +55,14 @@ static void refund_loose(MDBX_txn *txn) { } /* Collect loose-pages which may be refunded. */ - tASSERT(txn, - txn->geo.first_unallocated >= MIN_PAGENO + txn->tw.loose_count); + tASSERT(txn, txn->geo.first_unallocated >= MIN_PAGENO + txn->tw.loose_count); pgno_t most = MIN_PAGENO; size_t w = 0; for (const page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) { tASSERT(txn, lp->flags == P_LOOSE); tASSERT(txn, txn->geo.first_unallocated > lp->pgno); - if (likely(txn->geo.first_unallocated - txn->tw.loose_count <= - lp->pgno)) { - tASSERT(txn, - w < ((suitable == onstack) ? pnl_bytes2size(sizeof(onstack)) - : MDBX_PNL_ALLOCLEN(suitable))); + if (likely(txn->geo.first_unallocated - txn->tw.loose_count <= lp->pgno)) { + tASSERT(txn, w < ((suitable == onstack) ? pnl_bytes2size(sizeof(onstack)) : MDBX_PNL_ALLOCLEN(suitable))); suitable[++w] = lp->pgno; most = (lp->pgno > most) ? lp->pgno : most; } @@ -84,10 +77,8 @@ static void refund_loose(MDBX_txn *txn) { /* Scanning in descend order */ const intptr_t step = MDBX_PNL_ASCENDING ? -1 : 1; - const intptr_t begin = - MDBX_PNL_ASCENDING ? MDBX_PNL_GETSIZE(suitable) : 1; - const intptr_t end = - MDBX_PNL_ASCENDING ? 0 : MDBX_PNL_GETSIZE(suitable) + 1; + const intptr_t begin = MDBX_PNL_ASCENDING ? MDBX_PNL_GETSIZE(suitable) : 1; + const intptr_t end = MDBX_PNL_ASCENDING ? 0 : MDBX_PNL_GETSIZE(suitable) + 1; tASSERT(txn, suitable[begin] >= suitable[end - step]); tASSERT(txn, most == suitable[begin]); @@ -97,8 +88,7 @@ static void refund_loose(MDBX_txn *txn) { most -= 1; } const size_t refunded = txn->geo.first_unallocated - most; - DEBUG("refund-suitable %zu pages %" PRIaPGNO " -> %" PRIaPGNO, refunded, - most, txn->geo.first_unallocated); + DEBUG("refund-suitable %zu pages %" PRIaPGNO " -> %" PRIaPGNO, refunded, most, txn->geo.first_unallocated); txn->geo.first_unallocated = most; txn->tw.loose_count -= refunded; if (dl) { @@ -126,22 +116,19 @@ static void refund_loose(MDBX_txn *txn) { } dpl_setlen(dl, w); tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom - : txn->env->options.dp_limit)); + (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); } goto unlink_loose; } } else { /* Dirtylist is mostly sorted, just refund loose pages at the end. */ dpl_sort(txn); - tASSERT(txn, - dl->length < 2 || dl->items[1].pgno < dl->items[dl->length].pgno); + tASSERT(txn, dl->length < 2 || dl->items[1].pgno < dl->items[dl->length].pgno); tASSERT(txn, dl->sorted == dl->length); /* Scan dirtylist tail-forward and cutoff suitable pages. */ size_t n; - for (n = dl->length; dl->items[n].pgno == txn->geo.first_unallocated - 1 && - dl->items[n].ptr->flags == P_LOOSE; + for (n = dl->length; dl->items[n].pgno == txn->geo.first_unallocated - 1 && dl->items[n].ptr->flags == P_LOOSE; --n) { tASSERT(txn, n > 0); page_t *dp = dl->items[n].ptr; @@ -158,8 +145,7 @@ static void refund_loose(MDBX_txn *txn) { txn->tw.dirtyroom += refunded; dl->pages_including_loose -= refunded; tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom - : txn->env->options.dp_limit)); + (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); /* Filter-out loose chain & dispose refunded pages. */ unlink_loose: @@ -188,18 +174,15 @@ static void refund_loose(MDBX_txn *txn) { bool txn_refund(MDBX_txn *txn) { const pgno_t before = txn->geo.first_unallocated; - if (txn->tw.loose_pages && - txn->tw.loose_refund_wl > txn->geo.first_unallocated) + if (txn->tw.loose_pages && txn->tw.loose_refund_wl > txn->geo.first_unallocated) refund_loose(txn); while (true) { - if (MDBX_PNL_GETSIZE(txn->tw.relist) == 0 || - MDBX_PNL_MOST(txn->tw.relist) != txn->geo.first_unallocated - 1) + if (MDBX_PNL_GETSIZE(txn->tw.relist) == 0 || MDBX_PNL_MOST(txn->tw.relist) != txn->geo.first_unallocated - 1) break; refund_reclaimed(txn); - if (!txn->tw.loose_pages || - txn->tw.loose_refund_wl <= txn->geo.first_unallocated) + if (!txn->tw.loose_pages || txn->tw.loose_refund_wl <= txn->geo.first_unallocated) break; const pgno_t memo = txn->geo.first_unallocated; diff --git a/src/sort.h b/src/sort.h index 3169e317..824c6bf5 100644 --- a/src/sort.h +++ b/src/sort.h @@ -14,21 +14,21 @@ * Thanks to John M. Gamble for the http://pages.ripco.net/~jgamble/nw.html */ #if MDBX_HAVE_CMOV -#define SORT_CMP_SWAP(TYPE, CMP, a, b) \ - do { \ - const TYPE swap_tmp = (a); \ - const bool swap_cmp = expect_with_probability(CMP(swap_tmp, b), 0, .5); \ - (a) = swap_cmp ? swap_tmp : b; \ - (b) = swap_cmp ? b : swap_tmp; \ +#define SORT_CMP_SWAP(TYPE, CMP, a, b) \ + do { \ + const TYPE swap_tmp = (a); \ + const bool swap_cmp = expect_with_probability(CMP(swap_tmp, b), 0, .5); \ + (a) = swap_cmp ? swap_tmp : b; \ + (b) = swap_cmp ? b : swap_tmp; \ } while (0) #else -#define SORT_CMP_SWAP(TYPE, CMP, a, b) \ - do \ - if (expect_with_probability(!CMP(a, b), 0, .5)) { \ - const TYPE swap_tmp = (a); \ - (a) = (b); \ - (b) = swap_tmp; \ - } \ +#define SORT_CMP_SWAP(TYPE, CMP, a, b) \ + do \ + if (expect_with_probability(!CMP(a, b), 0, .5)) { \ + const TYPE swap_tmp = (a); \ + (a) = (b); \ + (b) = swap_tmp; \ + } \ while (0) #endif @@ -42,11 +42,11 @@ // [[1,2]] // [[0,2]] // [[0,1]] -#define SORT_NETWORK_3(TYPE, CMP, begin) \ - do { \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ +#define SORT_NETWORK_3(TYPE, CMP, begin) \ + do { \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ } while (0) // 5 comparators, 3 parallel operations @@ -61,13 +61,13 @@ // [[0,1],[2,3]] // [[0,2],[1,3]] // [[1,2]] -#define SORT_NETWORK_4(TYPE, CMP, begin) \ - do { \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ +#define SORT_NETWORK_4(TYPE, CMP, begin) \ + do { \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ } while (0) // 9 comparators, 5 parallel operations @@ -86,17 +86,17 @@ // [[2,4],[0,1]] // [[2,3],[1,4]] // [[1,2],[3,4]] -#define SORT_NETWORK_5(TYPE, CMP, begin) \ - do { \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ +#define SORT_NETWORK_5(TYPE, CMP, begin) \ + do { \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ } while (0) // 12 comparators, 6 parallel operations @@ -118,20 +118,20 @@ // [[0,3],[1,4]] // [[2,4],[1,3]] // [[2,3]] -#define SORT_NETWORK_6(TYPE, CMP, begin) \ - do { \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ +#define SORT_NETWORK_6(TYPE, CMP, begin) \ + do { \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ } while (0) // 16 comparators, 6 parallel operations @@ -155,24 +155,24 @@ // [[2,3],[4,5]] // [[1,4],[3,6]] // [[1,2],[3,4],[5,6]] -#define SORT_NETWORK_7(TYPE, CMP, begin) \ - do { \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[6]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[6]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[6]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[6]); \ +#define SORT_NETWORK_7(TYPE, CMP, begin) \ + do { \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[6]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[6]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[6]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[6]); \ } while (0) // 19 comparators, 6 parallel operations @@ -198,237 +198,236 @@ // [[2,3],[4,5]] // [[1,4],[3,6]] // [[1,2],[3,4],[5,6]] -#define SORT_NETWORK_8(TYPE, CMP, begin) \ - do { \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[6]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[7]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[6]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[7]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[6], begin[7]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[6]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[6]); \ +#define SORT_NETWORK_8(TYPE, CMP, begin) \ + do { \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[6]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[7]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[6]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[7]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[6], begin[7]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[6]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[6]); \ } while (0) -#define SORT_INNER(TYPE, CMP, begin, end, len) \ - switch (len) { \ - default: \ - assert(false); \ - __unreachable(); \ - case 0: \ - case 1: \ - break; \ - case 2: \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ - break; \ - case 3: \ - SORT_NETWORK_3(TYPE, CMP, begin); \ - break; \ - case 4: \ - SORT_NETWORK_4(TYPE, CMP, begin); \ - break; \ - case 5: \ - SORT_NETWORK_5(TYPE, CMP, begin); \ - break; \ - case 6: \ - SORT_NETWORK_6(TYPE, CMP, begin); \ - break; \ - case 7: \ - SORT_NETWORK_7(TYPE, CMP, begin); \ - break; \ - case 8: \ - SORT_NETWORK_8(TYPE, CMP, begin); \ - break; \ +#define SORT_INNER(TYPE, CMP, begin, end, len) \ + switch (len) { \ + default: \ + assert(false); \ + __unreachable(); \ + case 0: \ + case 1: \ + break; \ + case 2: \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ + break; \ + case 3: \ + SORT_NETWORK_3(TYPE, CMP, begin); \ + break; \ + case 4: \ + SORT_NETWORK_4(TYPE, CMP, begin); \ + break; \ + case 5: \ + SORT_NETWORK_5(TYPE, CMP, begin); \ + break; \ + case 6: \ + SORT_NETWORK_6(TYPE, CMP, begin); \ + break; \ + case 7: \ + SORT_NETWORK_7(TYPE, CMP, begin); \ + break; \ + case 8: \ + SORT_NETWORK_8(TYPE, CMP, begin); \ + break; \ } -#define SORT_SWAP(TYPE, a, b) \ - do { \ - const TYPE swap_tmp = (a); \ - (a) = (b); \ - (b) = swap_tmp; \ +#define SORT_SWAP(TYPE, a, b) \ + do { \ + const TYPE swap_tmp = (a); \ + (a) = (b); \ + (b) = swap_tmp; \ } while (0) -#define SORT_PUSH(low, high) \ - do { \ - top->lo = (low); \ - top->hi = (high); \ - ++top; \ +#define SORT_PUSH(low, high) \ + do { \ + top->lo = (low); \ + top->hi = (high); \ + ++top; \ } while (0) -#define SORT_POP(low, high) \ - do { \ - --top; \ - low = top->lo; \ - high = top->hi; \ +#define SORT_POP(low, high) \ + do { \ + --top; \ + low = top->lo; \ + high = top->hi; \ } while (0) -#define SORT_IMPL(NAME, EXPECT_LOW_CARDINALITY_OR_PRESORTED, TYPE, CMP) \ - \ - static inline bool NAME##_is_sorted(const TYPE *first, const TYPE *last) { \ - while (++first <= last) \ - if (expect_with_probability(CMP(first[0], first[-1]), 1, .1)) \ - return false; \ - return true; \ - } \ - \ - typedef struct { \ - TYPE *lo, *hi; \ - } NAME##_stack; \ - \ - __hot static void NAME(TYPE *const __restrict begin, \ - TYPE *const __restrict end) { \ - NAME##_stack stack[sizeof(size_t) * CHAR_BIT], *__restrict top = stack; \ - \ - TYPE *__restrict hi = end - 1; \ - TYPE *__restrict lo = begin; \ - while (true) { \ - const ptrdiff_t len = hi - lo; \ - if (len < 8) { \ - SORT_INNER(TYPE, CMP, lo, hi + 1, len + 1); \ - if (unlikely(top == stack)) \ - break; \ - SORT_POP(lo, hi); \ - continue; \ - } \ - \ - TYPE *__restrict mid = lo + (len >> 1); \ - SORT_CMP_SWAP(TYPE, CMP, *lo, *mid); \ - SORT_CMP_SWAP(TYPE, CMP, *mid, *hi); \ - SORT_CMP_SWAP(TYPE, CMP, *lo, *mid); \ - \ - TYPE *right = hi - 1; \ - TYPE *left = lo + 1; \ - while (1) { \ - while (expect_with_probability(CMP(*left, *mid), 0, .5)) \ - ++left; \ - while (expect_with_probability(CMP(*mid, *right), 0, .5)) \ - --right; \ - if (unlikely(left > right)) { \ - if (EXPECT_LOW_CARDINALITY_OR_PRESORTED) { \ - if (NAME##_is_sorted(lo, right)) \ - lo = right + 1; \ - if (NAME##_is_sorted(left, hi)) \ - hi = left; \ - } \ - break; \ - } \ - SORT_SWAP(TYPE, *left, *right); \ - mid = (mid == left) ? right : (mid == right) ? left : mid; \ - ++left; \ - --right; \ - } \ - \ - if (right - lo > hi - left) { \ - SORT_PUSH(lo, right); \ - lo = left; \ - } else { \ - SORT_PUSH(left, hi); \ - hi = right; \ - } \ - } \ - \ - if (AUDIT_ENABLED()) { \ - for (TYPE *scan = begin + 1; scan < end; ++scan) \ - assert(CMP(scan[-1], scan[0])); \ - } \ +#define SORT_IMPL(NAME, EXPECT_LOW_CARDINALITY_OR_PRESORTED, TYPE, CMP) \ + \ + static inline bool NAME##_is_sorted(const TYPE *first, const TYPE *last) { \ + while (++first <= last) \ + if (expect_with_probability(CMP(first[0], first[-1]), 1, .1)) \ + return false; \ + return true; \ + } \ + \ + typedef struct { \ + TYPE *lo, *hi; \ + } NAME##_stack; \ + \ + __hot static void NAME(TYPE *const __restrict begin, TYPE *const __restrict end) { \ + NAME##_stack stack[sizeof(size_t) * CHAR_BIT], *__restrict top = stack; \ + \ + TYPE *__restrict hi = end - 1; \ + TYPE *__restrict lo = begin; \ + while (true) { \ + const ptrdiff_t len = hi - lo; \ + if (len < 8) { \ + SORT_INNER(TYPE, CMP, lo, hi + 1, len + 1); \ + if (unlikely(top == stack)) \ + break; \ + SORT_POP(lo, hi); \ + continue; \ + } \ + \ + TYPE *__restrict mid = lo + (len >> 1); \ + SORT_CMP_SWAP(TYPE, CMP, *lo, *mid); \ + SORT_CMP_SWAP(TYPE, CMP, *mid, *hi); \ + SORT_CMP_SWAP(TYPE, CMP, *lo, *mid); \ + \ + TYPE *right = hi - 1; \ + TYPE *left = lo + 1; \ + while (1) { \ + while (expect_with_probability(CMP(*left, *mid), 0, .5)) \ + ++left; \ + while (expect_with_probability(CMP(*mid, *right), 0, .5)) \ + --right; \ + if (unlikely(left > right)) { \ + if (EXPECT_LOW_CARDINALITY_OR_PRESORTED) { \ + if (NAME##_is_sorted(lo, right)) \ + lo = right + 1; \ + if (NAME##_is_sorted(left, hi)) \ + hi = left; \ + } \ + break; \ + } \ + SORT_SWAP(TYPE, *left, *right); \ + mid = (mid == left) ? right : (mid == right) ? left : mid; \ + ++left; \ + --right; \ + } \ + \ + if (right - lo > hi - left) { \ + SORT_PUSH(lo, right); \ + lo = left; \ + } else { \ + SORT_PUSH(left, hi); \ + hi = right; \ + } \ + } \ + \ + if (AUDIT_ENABLED()) { \ + for (TYPE *scan = begin + 1; scan < end; ++scan) \ + assert(CMP(scan[-1], scan[0])); \ + } \ } /*------------------------------------------------------------------------------ * LY: radix sort for large chunks */ -#define RADIXSORT_IMPL(NAME, TYPE, EXTRACT_KEY, BUFFER_PREALLOCATED, END_GAP) \ - \ - __hot static bool NAME##_radixsort(TYPE *const begin, const size_t length) { \ - TYPE *tmp; \ - if (BUFFER_PREALLOCATED) { \ - tmp = begin + length + END_GAP; \ - /* memset(tmp, 0xDeadBeef, sizeof(TYPE) * length); */ \ - } else { \ - tmp = osal_malloc(sizeof(TYPE) * length); \ - if (unlikely(!tmp)) \ - return false; \ - } \ - \ - size_t key_shift = 0, key_diff_mask; \ - do { \ - struct { \ - pgno_t a[256], b[256]; \ - } counters; \ - memset(&counters, 0, sizeof(counters)); \ - \ - key_diff_mask = 0; \ - size_t prev_key = EXTRACT_KEY(begin) >> key_shift; \ - TYPE *r = begin, *end = begin + length; \ - do { \ - const size_t key = EXTRACT_KEY(r) >> key_shift; \ - counters.a[key & 255]++; \ - counters.b[(key >> 8) & 255]++; \ - key_diff_mask |= prev_key ^ key; \ - prev_key = key; \ - } while (++r != end); \ - \ - pgno_t ta = 0, tb = 0; \ - for (size_t i = 0; i < 256; ++i) { \ - const pgno_t ia = counters.a[i]; \ - counters.a[i] = ta; \ - ta += ia; \ - const pgno_t ib = counters.b[i]; \ - counters.b[i] = tb; \ - tb += ib; \ - } \ - \ - r = begin; \ - do { \ - const size_t key = EXTRACT_KEY(r) >> key_shift; \ - tmp[counters.a[key & 255]++] = *r; \ - } while (++r != end); \ - \ - if (unlikely(key_diff_mask < 256)) { \ - memcpy(begin, tmp, ptr_dist(end, begin)); \ - break; \ - } \ - end = (r = tmp) + length; \ - do { \ - const size_t key = EXTRACT_KEY(r) >> key_shift; \ - begin[counters.b[(key >> 8) & 255]++] = *r; \ - } while (++r != end); \ - \ - key_shift += 16; \ - } while (key_diff_mask >> 16); \ - \ - if (!(BUFFER_PREALLOCATED)) \ - osal_free(tmp); \ - return true; \ +#define RADIXSORT_IMPL(NAME, TYPE, EXTRACT_KEY, BUFFER_PREALLOCATED, END_GAP) \ + \ + __hot static bool NAME##_radixsort(TYPE *const begin, const size_t length) { \ + TYPE *tmp; \ + if (BUFFER_PREALLOCATED) { \ + tmp = begin + length + END_GAP; \ + /* memset(tmp, 0xDeadBeef, sizeof(TYPE) * length); */ \ + } else { \ + tmp = osal_malloc(sizeof(TYPE) * length); \ + if (unlikely(!tmp)) \ + return false; \ + } \ + \ + size_t key_shift = 0, key_diff_mask; \ + do { \ + struct { \ + pgno_t a[256], b[256]; \ + } counters; \ + memset(&counters, 0, sizeof(counters)); \ + \ + key_diff_mask = 0; \ + size_t prev_key = EXTRACT_KEY(begin) >> key_shift; \ + TYPE *r = begin, *end = begin + length; \ + do { \ + const size_t key = EXTRACT_KEY(r) >> key_shift; \ + counters.a[key & 255]++; \ + counters.b[(key >> 8) & 255]++; \ + key_diff_mask |= prev_key ^ key; \ + prev_key = key; \ + } while (++r != end); \ + \ + pgno_t ta = 0, tb = 0; \ + for (size_t i = 0; i < 256; ++i) { \ + const pgno_t ia = counters.a[i]; \ + counters.a[i] = ta; \ + ta += ia; \ + const pgno_t ib = counters.b[i]; \ + counters.b[i] = tb; \ + tb += ib; \ + } \ + \ + r = begin; \ + do { \ + const size_t key = EXTRACT_KEY(r) >> key_shift; \ + tmp[counters.a[key & 255]++] = *r; \ + } while (++r != end); \ + \ + if (unlikely(key_diff_mask < 256)) { \ + memcpy(begin, tmp, ptr_dist(end, begin)); \ + break; \ + } \ + end = (r = tmp) + length; \ + do { \ + const size_t key = EXTRACT_KEY(r) >> key_shift; \ + begin[counters.b[(key >> 8) & 255]++] = *r; \ + } while (++r != end); \ + \ + key_shift += 16; \ + } while (key_diff_mask >> 16); \ + \ + if (!(BUFFER_PREALLOCATED)) \ + osal_free(tmp); \ + return true; \ } /*------------------------------------------------------------------------------ * LY: Binary search */ #if defined(__clang__) && __clang_major__ > 4 && defined(__ia32__) -#define WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(size, flag) \ - do \ - __asm __volatile("" \ - : "+r"(size) \ - : "r" /* the `b` constraint is more suitable here, but \ - cause CLANG to allocate and push/pop an one more \ - register, so using the `r` which avoids this. */ \ - (flag)); \ +#define WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(size, flag) \ + do \ + __asm __volatile("" \ + : "+r"(size) \ + : "r" /* the `b` constraint is more suitable here, but \ + cause CLANG to allocate and push/pop an one more \ + register, so using the `r` which avoids this. */ \ + (flag)); \ while (0) #else -#define WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(size, flag) \ - do { \ - /* nope for non-clang or non-x86 */; \ +#define WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(size, flag) \ + do { \ + /* nope for non-clang or non-x86 */; \ } while (0) #endif /* Workaround for CLANG */ diff --git a/src/spill.c b/src/spill.c index 0a02ad52..261adb78 100644 --- a/src/spill.c +++ b/src/spill.c @@ -4,33 +4,25 @@ #include "internals.h" void spill_remove(MDBX_txn *txn, size_t idx, size_t npages) { - tASSERT(txn, idx > 0 && idx <= MDBX_PNL_GETSIZE(txn->tw.spilled.list) && - txn->tw.spilled.least_removed > 0); - txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed) - ? idx - : txn->tw.spilled.least_removed; + tASSERT(txn, idx > 0 && idx <= MDBX_PNL_GETSIZE(txn->tw.spilled.list) && txn->tw.spilled.least_removed > 0); + txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed) ? idx : txn->tw.spilled.least_removed; txn->tw.spilled.list[idx] |= 1; MDBX_PNL_SETSIZE(txn->tw.spilled.list, - MDBX_PNL_GETSIZE(txn->tw.spilled.list) - - (idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list))); + MDBX_PNL_GETSIZE(txn->tw.spilled.list) - (idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list))); while (unlikely(npages > 1)) { const pgno_t pgno = (txn->tw.spilled.list[idx] >> 1) + 1; if (MDBX_PNL_ASCENDING) { - if (++idx > MDBX_PNL_GETSIZE(txn->tw.spilled.list) || - (txn->tw.spilled.list[idx] >> 1) != pgno) + if (++idx > MDBX_PNL_GETSIZE(txn->tw.spilled.list) || (txn->tw.spilled.list[idx] >> 1) != pgno) return; } else { if (--idx < 1 || (txn->tw.spilled.list[idx] >> 1) != pgno) return; - txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed) - ? idx - : txn->tw.spilled.least_removed; + txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed) ? idx : txn->tw.spilled.least_removed; } txn->tw.spilled.list[idx] |= 1; MDBX_PNL_SETSIZE(txn->tw.spilled.list, - MDBX_PNL_GETSIZE(txn->tw.spilled.list) - - (idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list))); + MDBX_PNL_GETSIZE(txn->tw.spilled.list) - (idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list))); --npages; } } @@ -57,8 +49,7 @@ pnl_t spill_purge(MDBX_txn *txn) { /*----------------------------------------------------------------------------*/ -static int spill_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, - const size_t npages) { +static int spill_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, const size_t npages) { tASSERT(txn, !(txn->flags & MDBX_WRITEMAP)); #if MDBX_ENABLE_PGOP_STAT txn->env->lck->pgops.spill.weak += npages; @@ -72,8 +63,7 @@ static int spill_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, /* Set unspillable LRU-label for dirty pages watched by txn. * Returns the number of pages marked as unspillable. */ -static size_t spill_cursor_keep(const MDBX_txn *const txn, - const MDBX_cursor *mc) { +static size_t spill_cursor_keep(const MDBX_txn *const txn, const MDBX_cursor *mc) { tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); size_t keep = 0; while (!is_poor(mc)) { @@ -87,8 +77,7 @@ static size_t spill_cursor_keep(const MDBX_txn *const txn, size_t const n = dpl_search(txn, mp->pgno); if (txn->tw.dirtylist->items[n].pgno == mp->pgno && /* не считаем дважды */ dpl_age(txn, n)) { - size_t *const ptr = ptr_disp(txn->tw.dirtylist->items[n].ptr, - -(ptrdiff_t)sizeof(size_t)); + size_t *const ptr = ptr_disp(txn->tw.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t)); *ptr = txn->tw.dirtylru; tASSERT(txn, dpl_age(txn, n) == 0); ++keep; @@ -112,8 +101,7 @@ static size_t spill_txn_keep(MDBX_txn *txn, MDBX_cursor *m0) { size_t keep = m0 ? spill_cursor_keep(txn, m0) : 0; TXN_FOREACH_DBI_ALL(txn, dbi) { - if (F_ISSET(txn->dbi_state[dbi], DBI_DIRTY | DBI_VALID) && - txn->dbs[dbi].root != P_INVALID) + if (F_ISSET(txn->dbi_state[dbi], DBI_DIRTY | DBI_VALID) && txn->dbs[dbi].root != P_INVALID) for (MDBX_cursor *mc = txn->cursors[dbi]; mc; mc = mc->next) if (mc != m0) keep += spill_cursor_keep(txn, mc); @@ -126,8 +114,7 @@ static size_t spill_txn_keep(MDBX_txn *txn, MDBX_cursor *m0) { * 0 = should be spilled; * ... * > 255 = must not be spilled. */ -MDBX_NOTHROW_PURE_FUNCTION static unsigned -spill_prio(const MDBX_txn *txn, const size_t i, const uint32_t reciprocal) { +MDBX_NOTHROW_PURE_FUNCTION static unsigned spill_prio(const MDBX_txn *txn, const size_t i, const uint32_t reciprocal) { dpl_t *const dl = txn->tw.dirtylist; const uint32_t age = dpl_age(txn, i); const size_t npages = dpl_npages(dl, i); @@ -139,8 +126,7 @@ spill_prio(const MDBX_txn *txn, const size_t i, const uint32_t reciprocal) { page_t *const dp = dl->items[i].ptr; if (dp->flags & (P_LOOSE | P_SPILLED)) { - DEBUG("skip %s %zu page %" PRIaPGNO, - (dp->flags & P_LOOSE) ? "loose" : "parent-spilled", npages, pgno); + DEBUG("skip %s %zu page %" PRIaPGNO, (dp->flags & P_LOOSE) ? "loose" : "parent-spilled", npages, pgno); return 256; } @@ -175,67 +161,49 @@ spill_prio(const MDBX_txn *txn, const size_t i, const uint32_t reciprocal) { return prio = (unsigned)factor; } -static size_t spill_gate(const MDBX_env *env, intptr_t part, - const size_t total) { - const intptr_t spill_min = - env->options.spill_min_denominator - ? (total + env->options.spill_min_denominator - 1) / - env->options.spill_min_denominator - : 1; +static size_t spill_gate(const MDBX_env *env, intptr_t part, const size_t total) { + const intptr_t spill_min = env->options.spill_min_denominator + ? (total + env->options.spill_min_denominator - 1) / env->options.spill_min_denominator + : 1; const intptr_t spill_max = - total - (env->options.spill_max_denominator - ? total / env->options.spill_max_denominator - : 0); + total - (env->options.spill_max_denominator ? total / env->options.spill_max_denominator : 0); part = (part < spill_max) ? part : spill_max; part = (part > spill_min) ? part : spill_min; eASSERT(env, part >= 0 && (size_t)part <= total); return (size_t)part; } -__cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, - const intptr_t wanna_spill_entries, - const intptr_t wanna_spill_npages, - const size_t need) { +__cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intptr_t wanna_spill_entries, + const intptr_t wanna_spill_npages, const size_t need) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); int rc = MDBX_SUCCESS; if (unlikely(txn->tw.loose_count >= - (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose - : txn->tw.writemap_dirty_npages))) + (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages))) goto done; - const size_t dirty_entries = - txn->tw.dirtylist ? (txn->tw.dirtylist->length - txn->tw.loose_count) : 1; + const size_t dirty_entries = txn->tw.dirtylist ? (txn->tw.dirtylist->length - txn->tw.loose_count) : 1; const size_t dirty_npages = - (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose - : txn->tw.writemap_dirty_npages) - + (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages) - txn->tw.loose_count; - const size_t need_spill_entries = - spill_gate(txn->env, wanna_spill_entries, dirty_entries); - const size_t need_spill_npages = - spill_gate(txn->env, wanna_spill_npages, dirty_npages); + const size_t need_spill_entries = spill_gate(txn->env, wanna_spill_entries, dirty_entries); + const size_t need_spill_npages = spill_gate(txn->env, wanna_spill_npages, dirty_npages); - const size_t need_spill = (need_spill_entries > need_spill_npages) - ? need_spill_entries - : need_spill_npages; + const size_t need_spill = (need_spill_entries > need_spill_npages) ? need_spill_entries : need_spill_npages; if (!need_spill) goto done; if (txn->flags & MDBX_WRITEMAP) { - NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "msync", - dirty_entries, dirty_npages); + NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "msync", dirty_entries, dirty_npages); const MDBX_env *env = txn->env; tASSERT(txn, txn->tw.spilled.list == nullptr); - rc = osal_msync(&txn->env->dxb_mmap, 0, - pgno_align2os_bytes(env, txn->geo.first_unallocated), - MDBX_SYNC_KICK); + rc = osal_msync(&txn->env->dxb_mmap, 0, pgno_align2os_bytes(env, txn->geo.first_unallocated), MDBX_SYNC_KICK); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; #if MDBX_AVOID_MSYNC MDBX_ANALYSIS_ASSUME(txn->tw.dirtylist != nullptr); tASSERT(txn, dpl_check(txn)); - env->lck->unsynced_pages.weak += - txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count; + env->lck->unsynced_pages.weak += txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count; dpl_clear(txn->tw.dirtylist); txn->tw.dirtyroom = env->options.dp_limit - txn->tw.loose_count; for (page_t *lp = txn->tw.loose_pages; lp != nullptr; lp = page_next(lp)) { @@ -256,12 +224,10 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, goto done; } - NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "write", - need_spill_entries, need_spill_npages); + NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "write", need_spill_entries, need_spill_npages); MDBX_ANALYSIS_ASSUME(txn->tw.dirtylist != nullptr); tASSERT(txn, txn->tw.dirtylist->length - txn->tw.loose_count >= 1); - tASSERT(txn, txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count >= - need_spill_npages); + tASSERT(txn, txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count >= need_spill_npages); if (!txn->tw.spilled.list) { txn->tw.spilled.least_removed = INT_MAX; txn->tw.spilled.list = pnl_alloc(need_spill); @@ -338,10 +304,8 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, for (size_t i = 1; i <= dl->length; ++i) { const unsigned prio = spill_prio(txn, i, reciprocal); size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t)); - TRACE("page %" PRIaPGNO - ", lru %zu, is_multi %c, npages %u, age %u of %u, prio %u", - dl->items[i].pgno, *ptr, (dl->items[i].npages > 1) ? 'Y' : 'N', - dpl_npages(dl, i), dpl_age(txn, i), age_max, prio); + TRACE("page %" PRIaPGNO ", lru %zu, is_multi %c, npages %u, age %u of %u, prio %u", dl->items[i].pgno, *ptr, + (dl->items[i].npages > 1) ? 'Y' : 'N', dpl_npages(dl, i), dpl_age(txn, i), age_max, prio); if (prio < 256) { radix_entries[prio] += 1; spillable_entries += 1; @@ -354,20 +318,16 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, tASSERT(txn, spillable_npages >= spillable_entries); pgno_t spilled_entries = 0, spilled_npages = 0; if (likely(spillable_entries > 0)) { - size_t prio2spill = 0, prio2adjacent = 128, - amount_entries = radix_entries[0], amount_npages = radix_npages[0]; + size_t prio2spill = 0, prio2adjacent = 128, amount_entries = radix_entries[0], amount_npages = radix_npages[0]; for (size_t i = 1; i < 256; i++) { - if (amount_entries < need_spill_entries || - amount_npages < need_spill_npages) { + if (amount_entries < need_spill_entries || amount_npages < need_spill_npages) { prio2spill = i; prio2adjacent = i + (257 - i) / 2; amount_entries += radix_entries[i]; amount_npages += radix_npages[i]; - } else if (amount_entries + amount_entries < - spillable_entries + need_spill_entries + } else if (amount_entries + amount_entries < spillable_entries + need_spill_entries /* РАВНОЗНАЧНО: amount - need_spill < spillable - amount */ - || amount_npages + amount_npages < - spillable_npages + need_spill_npages) { + || amount_npages + amount_npages < spillable_npages + need_spill_npages) { prio2adjacent = i; amount_entries += radix_entries[i]; amount_npages += radix_npages[i]; @@ -377,44 +337,38 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, VERBOSE("prio2spill %zu, prio2adjacent %zu, spillable %zu/%zu," " wanna-spill %zu/%zu, amount %zu/%zu", - prio2spill, prio2adjacent, spillable_entries, spillable_npages, - need_spill_entries, need_spill_npages, amount_entries, - amount_npages); + prio2spill, prio2adjacent, spillable_entries, spillable_npages, need_spill_entries, need_spill_npages, + amount_entries, amount_npages); tASSERT(txn, prio2spill < prio2adjacent && prio2adjacent <= 256); iov_ctx_t ctx; - rc = iov_init( - txn, &ctx, amount_entries, amount_npages, + rc = iov_init(txn, &ctx, amount_entries, amount_npages, #if defined(_WIN32) || defined(_WIN64) - txn->env->ioring.overlapped_fd ? txn->env->ioring.overlapped_fd : + txn->env->ioring.overlapped_fd ? txn->env->ioring.overlapped_fd : #endif - txn->env->lazy_fd, - true); + txn->env->lazy_fd, + true); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; size_t r = 0, w = 0; pgno_t last = 0; - while (r < dl->length && (spilled_entries < need_spill_entries || - spilled_npages < need_spill_npages)) { + while (r < dl->length && (spilled_entries < need_spill_entries || spilled_npages < need_spill_npages)) { dl->items[++w] = dl->items[++r]; unsigned prio = spill_prio(txn, w, reciprocal); - if (prio > prio2spill && - (prio >= prio2adjacent || last != dl->items[w].pgno)) + if (prio > prio2spill && (prio >= prio2adjacent || last != dl->items[w].pgno)) continue; const size_t e = w; last = dpl_endpgno(dl, w); - while (--w && dpl_endpgno(dl, w) == dl->items[w + 1].pgno && - spill_prio(txn, w, reciprocal) < prio2adjacent) + while (--w && dpl_endpgno(dl, w) == dl->items[w + 1].pgno && spill_prio(txn, w, reciprocal) < prio2adjacent) ; for (size_t i = w; ++i <= e;) { const unsigned npages = dpl_npages(dl, i); prio = spill_prio(txn, i, reciprocal); - DEBUG("%sspill[%zu] %u page %" PRIaPGNO " (age %d, prio %u)", - (prio > prio2spill) ? "co-" : "", i, npages, dl->items[i].pgno, - dpl_age(txn, i), prio); + DEBUG("%sspill[%zu] %u page %" PRIaPGNO " (age %d, prio %u)", (prio > prio2spill) ? "co-" : "", i, npages, + dl->items[i].pgno, dpl_age(txn, i), prio); tASSERT(txn, prio < 256); ++spilled_entries; spilled_npages += npages; @@ -424,8 +378,7 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, } } - VERBOSE("spilled entries %u, spilled npages %u", spilled_entries, - spilled_npages); + VERBOSE("spilled entries %u, spilled npages %u", spilled_entries, spilled_npages); tASSERT(txn, spillable_entries == 0 || spilled_entries > 0); tASSERT(txn, spilled_npages >= spilled_entries); @@ -449,16 +402,14 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, txn->env->lck->unsynced_pages.weak += spilled_npages; pnl_sort(txn->tw.spilled.list, (size_t)txn->geo.first_unallocated << 1); txn->flags |= MDBX_TXN_SPILLS; - NOTICE("spilled %u dirty-entries, %u dirty-npages, now have %zu dirty-room", - spilled_entries, spilled_npages, txn->tw.dirtyroom); + NOTICE("spilled %u dirty-entries, %u dirty-npages, now have %zu dirty-room", spilled_entries, spilled_npages, + txn->tw.dirtyroom); } else { tASSERT(txn, rc == MDBX_SUCCESS); for (size_t i = 1; i <= dl->length; ++i) { page_t *dp = dl->items[i].ptr; - VERBOSE( - "unspillable[%zu]: pgno %u, npages %u, flags 0x%04X, age %u, prio %u", - i, dp->pgno, dpl_npages(dl, i), dp->flags, dpl_age(txn, i), - spill_prio(txn, i, reciprocal)); + VERBOSE("unspillable[%zu]: pgno %u, npages %u, flags 0x%04X, age %u, prio %u", i, dp->pgno, dpl_npages(dl, i), + dp->flags, dpl_age(txn, i), spill_prio(txn, i, reciprocal)); } } @@ -468,17 +419,13 @@ __cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, "needed %zu, spillable %zu; " "spilled %u dirty-entries, now have %zu dirty-room", dl->length + spilled_entries, dl->length, - (txn->parent && txn->parent->tw.dirtylist) - ? (intptr_t)txn->parent->tw.dirtylist->length - : -1, - txn->tw.loose_count, need, spillable_entries, spilled_entries, - txn->tw.dirtyroom); + (txn->parent && txn->parent->tw.dirtylist) ? (intptr_t)txn->parent->tw.dirtylist->length : -1, + txn->tw.loose_count, need, spillable_entries, spilled_entries, txn->tw.dirtyroom); ENSURE(txn->env, txn->tw.loose_count + txn->tw.dirtyroom > need / 2); #endif /* xMDBX_DEBUG_SPILLING */ done: - return likely(txn->tw.dirtyroom + txn->tw.loose_count > - ((need > CURSOR_STACK_SIZE) ? CURSOR_STACK_SIZE : need)) + return likely(txn->tw.dirtyroom + txn->tw.loose_count > ((need > CURSOR_STACK_SIZE) ? CURSOR_STACK_SIZE : need)) ? MDBX_SUCCESS : MDBX_TXN_FULL; } diff --git a/src/spill.h b/src/spill.h index f4c427dd..32a8c9b2 100644 --- a/src/spill.h +++ b/src/spill.h @@ -7,10 +7,8 @@ MDBX_INTERNAL void spill_remove(MDBX_txn *txn, size_t idx, size_t npages); MDBX_INTERNAL pnl_t spill_purge(MDBX_txn *txn); -MDBX_INTERNAL int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, - const intptr_t wanna_spill_entries, - const intptr_t wanna_spill_npages, - const size_t need); +MDBX_INTERNAL int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intptr_t wanna_spill_entries, + const intptr_t wanna_spill_npages, const size_t need); /*----------------------------------------------------------------------------*/ static inline size_t spill_search(const MDBX_txn *txn, pgno_t pgno) { @@ -23,8 +21,7 @@ static inline size_t spill_search(const MDBX_txn *txn, pgno_t pgno) { return (n <= MDBX_PNL_GETSIZE(pnl) && pnl[n] == pgno) ? n : 0; } -static inline bool spill_intersect(const MDBX_txn *txn, pgno_t pgno, - size_t npages) { +static inline bool spill_intersect(const MDBX_txn *txn, pgno_t pgno, size_t npages) { const pnl_t pnl = txn->tw.spilled.list; if (likely(!pnl)) return false; @@ -32,23 +29,18 @@ static inline bool spill_intersect(const MDBX_txn *txn, pgno_t pgno, if (LOG_ENABLED(MDBX_LOG_EXTRA)) { DEBUG_EXTRA("PNL len %zu [", len); for (size_t i = 1; i <= len; ++i) - DEBUG_EXTRA_PRINT(" %li", (pnl[i] & 1) ? -(long)(pnl[i] >> 1) - : (long)(pnl[i] >> 1)); + DEBUG_EXTRA_PRINT(" %li", (pnl[i] & 1) ? -(long)(pnl[i] >> 1) : (long)(pnl[i] >> 1)); DEBUG_EXTRA_PRINT("%s\n", "]"); } const pgno_t spilled_range_begin = pgno << 1; const pgno_t spilled_range_last = ((pgno + (pgno_t)npages) << 1) - 1; #if MDBX_PNL_ASCENDING - const size_t n = - pnl_search(pnl, spilled_range_begin, (size_t)(MAX_PAGENO + 1) << 1); - tASSERT(txn, n && (n == MDBX_PNL_GETSIZE(pnl) + 1 || - spilled_range_begin <= pnl[n])); + const size_t n = pnl_search(pnl, spilled_range_begin, (size_t)(MAX_PAGENO + 1) << 1); + tASSERT(txn, n && (n == MDBX_PNL_GETSIZE(pnl) + 1 || spilled_range_begin <= pnl[n])); const bool rc = n <= MDBX_PNL_GETSIZE(pnl) && pnl[n] <= spilled_range_last; #else - const size_t n = - pnl_search(pnl, spilled_range_last, (size_t)MAX_PAGENO + MAX_PAGENO + 1); - tASSERT(txn, n && (n == MDBX_PNL_GETSIZE(pnl) + 1 || - spilled_range_last >= pnl[n])); + const size_t n = pnl_search(pnl, spilled_range_last, (size_t)MAX_PAGENO + MAX_PAGENO + 1); + tASSERT(txn, n && (n == MDBX_PNL_GETSIZE(pnl) + 1 || spilled_range_last >= pnl[n])); const bool rc = n <= MDBX_PNL_GETSIZE(pnl) && pnl[n] >= spilled_range_begin; #endif if (ASSERT_ENABLED()) { @@ -60,17 +52,13 @@ static inline bool spill_intersect(const MDBX_txn *txn, pgno_t pgno, return rc; } -static inline int txn_spill(MDBX_txn *const txn, MDBX_cursor *const m0, - const size_t need) { +static inline int txn_spill(MDBX_txn *const txn, MDBX_cursor *const m0, const size_t need) { tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); tASSERT(txn, !m0 || cursor_is_tracked(m0)); - const intptr_t wanna_spill_entries = - txn->tw.dirtylist ? (need - txn->tw.dirtyroom - txn->tw.loose_count) : 0; + const intptr_t wanna_spill_entries = txn->tw.dirtylist ? (need - txn->tw.dirtyroom - txn->tw.loose_count) : 0; const intptr_t wanna_spill_npages = - need + - (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose - : txn->tw.writemap_dirty_npages) - + need + (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages) - txn->tw.loose_count - txn->env->options.dp_limit; /* production mode */ diff --git a/src/table.c b/src/table.c index c37f72a5..c9a1e980 100644 --- a/src/table.c +++ b/src/table.c @@ -19,11 +19,8 @@ int tbl_setup(const MDBX_env *env, kvx_t *const kvx, const tree_t *const db) { kvx->clc.v.lmax = env_valsize_max(env, db->flags); if ((db->flags & (MDBX_DUPFIXED | MDBX_INTEGERDUP)) != 0 && db->dupfix_size) { - if (!MDBX_DISABLE_VALIDATION && - unlikely(db->dupfix_size < kvx->clc.v.lmin || - db->dupfix_size > kvx->clc.v.lmax)) { - ERROR("db.dupfix_size (%u) <> min/max value-length (%zu/%zu)", - db->dupfix_size, kvx->clc.v.lmin, kvx->clc.v.lmax); + if (!MDBX_DISABLE_VALIDATION && unlikely(db->dupfix_size < kvx->clc.v.lmin || db->dupfix_size > kvx->clc.v.lmax)) { + ERROR("db.dupfix_size (%u) <> min/max value-length (%zu/%zu)", db->dupfix_size, kvx->clc.v.lmin, kvx->clc.v.lmax); return MDBX_CORRUPTED; } kvx->clc.v.lmin = kvx->clc.v.lmax = db->dupfix_size; @@ -41,10 +38,8 @@ int tbl_fetch(MDBX_txn *txn, size_t dbi) { rc = tree_search(&couple.outer, &kvx->name, 0); if (unlikely(rc != MDBX_SUCCESS)) { bailout: - NOTICE("dbi %zu refs to inaccessible table `%*s` for txn %" PRIaTXN - " (err %d)", - dbi, (int)kvx->name.iov_len, (const char *)kvx->name.iov_base, - txn->txnid, rc); + NOTICE("dbi %zu refs to inaccessible table `%*s` for txn %" PRIaTXN " (err %d)", dbi, (int)kvx->name.iov_len, + (const char *)kvx->name.iov_base, txn->txnid, rc); return (rc == MDBX_NOTFOUND) ? MDBX_BAD_DBI : rc; } @@ -55,21 +50,18 @@ int tbl_fetch(MDBX_txn *txn, size_t dbi) { goto bailout; } if (unlikely((node_flags(nsr.node) & (N_DUP | N_TREE)) != N_TREE)) { - NOTICE("dbi %zu refs to not a named table `%*s` for txn %" PRIaTXN " (%s)", - dbi, (int)kvx->name.iov_len, (const char *)kvx->name.iov_base, - txn->txnid, "wrong flags"); + NOTICE("dbi %zu refs to not a named table `%*s` for txn %" PRIaTXN " (%s)", dbi, (int)kvx->name.iov_len, + (const char *)kvx->name.iov_base, txn->txnid, "wrong flags"); return MDBX_INCOMPATIBLE; /* not a named DB */ } - rc = node_read(&couple.outer, nsr.node, &data, - couple.outer.pg[couple.outer.top]); + rc = node_read(&couple.outer, nsr.node, &data, couple.outer.pg[couple.outer.top]); if (unlikely(rc != MDBX_SUCCESS)) return rc; if (unlikely(data.iov_len != sizeof(tree_t))) { - NOTICE("dbi %zu refs to not a named table `%*s` for txn %" PRIaTXN " (%s)", - dbi, (int)kvx->name.iov_len, (const char *)kvx->name.iov_base, - txn->txnid, "wrong rec-size"); + NOTICE("dbi %zu refs to not a named table `%*s` for txn %" PRIaTXN " (%s)", dbi, (int)kvx->name.iov_len, + (const char *)kvx->name.iov_base, txn->txnid, "wrong rec-size"); return MDBX_INCOMPATIBLE; /* not a named DB */ } @@ -80,8 +72,8 @@ int tbl_fetch(MDBX_txn *txn, size_t dbi) { if (unlikely((db->flags & DB_PERSISTENT_FLAGS) != flags)) { NOTICE("dbi %zu refs to the re-created table `%*s` for txn %" PRIaTXN " with different flags (present 0x%X != wanna 0x%X)", - dbi, (int)kvx->name.iov_len, (const char *)kvx->name.iov_base, - txn->txnid, db->flags & DB_PERSISTENT_FLAGS, flags); + dbi, (int)kvx->name.iov_len, (const char *)kvx->name.iov_base, txn->txnid, db->flags & DB_PERSISTENT_FLAGS, + flags); return MDBX_INCOMPATIBLE; } @@ -90,8 +82,7 @@ int tbl_fetch(MDBX_txn *txn, size_t dbi) { const txnid_t pp_txnid = couple.outer.pg[couple.outer.top]->txnid; tASSERT(txn, txn->front_txnid >= pp_txnid); if (unlikely(db->mod_txnid > pp_txnid)) { - ERROR("db.mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")", - db->mod_txnid, pp_txnid); + ERROR("db.mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")", db->mod_txnid, pp_txnid); return MDBX_CORRUPTED; } #endif /* !MDBX_DISABLE_VALIDATION */ diff --git a/src/tls.c b/src/tls.c index cdfdda3e..7590c65f 100644 --- a/src/tls.c +++ b/src/tls.c @@ -29,21 +29,17 @@ static int uniq_peek(const osal_mmap_t *pending, osal_mmap_t *scan) { bait = 0 /* hush MSVC warning */; rc = osal_msync(scan, 0, sizeof(lck_t), MDBX_SYNC_DATA); if (rc == MDBX_SUCCESS) - rc = osal_pread(pending->fd, &bait, sizeof(scan_lck->bait_uniqueness), - offsetof(lck_t, bait_uniqueness)); + rc = osal_pread(pending->fd, &bait, sizeof(scan_lck->bait_uniqueness), offsetof(lck_t, bait_uniqueness)); } - if (likely(rc == MDBX_SUCCESS) && - bait == atomic_load64(&scan_lck->bait_uniqueness, mo_AcquireRelease)) + if (likely(rc == MDBX_SUCCESS) && bait == atomic_load64(&scan_lck->bait_uniqueness, mo_AcquireRelease)) rc = MDBX_RESULT_TRUE; - TRACE("uniq-peek: %s, bait 0x%016" PRIx64 ",%s rc %d", - pending_lck ? "mem" : "file", bait, + TRACE("uniq-peek: %s, bait 0x%016" PRIx64 ",%s rc %d", pending_lck ? "mem" : "file", bait, (rc == MDBX_RESULT_TRUE) ? " found," : (rc ? " FAILED," : ""), rc); return rc; } -static int uniq_poke(const osal_mmap_t *pending, osal_mmap_t *scan, - uint64_t *abra) { +static int uniq_poke(const osal_mmap_t *pending, osal_mmap_t *scan, uint64_t *abra) { if (*abra == 0) { const uintptr_t tid = osal_thread_self(); uintptr_t uit = 0; @@ -51,9 +47,7 @@ static int uniq_poke(const osal_mmap_t *pending, osal_mmap_t *scan, *abra = rrxmrrxmsx_0(osal_monotime() + UINT64_C(5873865991930747) * uit); } const uint64_t cadabra = - rrxmrrxmsx_0(*abra + UINT64_C(7680760450171793) * (unsigned)osal_getpid()) - << 24 | - *abra >> 40; + rrxmrrxmsx_0(*abra + UINT64_C(7680760450171793) * (unsigned)osal_getpid()) << 24 | *abra >> 40; lck_t *const scan_lck = scan->lck; atomic_store64(&scan_lck->bait_uniqueness, cadabra, mo_AcquireRelease); *abra = *abra * UINT64_C(6364136223846793005) + 1; @@ -67,14 +61,12 @@ __cold int rthc_uniq_check(const osal_mmap_t *pending, MDBX_env **found) { MDBX_env *const scan = rthc_table[i].env; if (!scan->lck_mmap.lck || &scan->lck_mmap == pending) continue; - int err = - atomic_load64(&scan->lck_mmap.lck->bait_uniqueness, mo_AcquireRelease) - ? uniq_peek(pending, &scan->lck_mmap) - : uniq_poke(pending, &scan->lck_mmap, &salt); + int err = atomic_load64(&scan->lck_mmap.lck->bait_uniqueness, mo_AcquireRelease) + ? uniq_peek(pending, &scan->lck_mmap) + : uniq_poke(pending, &scan->lck_mmap, &salt); if (err == MDBX_ENODATA) { uint64_t length = 0; - if (likely(osal_filesize(pending->fd, &length) == MDBX_SUCCESS && - length == 0)) { + if (likely(osal_filesize(pending->fd, &length) == MDBX_SUCCESS && length == 0)) { /* LY: skip checking since LCK-file is empty, i.e. just created. */ DEBUG("%s", "unique (new/empty lck)"); return MDBX_SUCCESS; @@ -114,8 +106,7 @@ static osal_thread_key_t rthc_key; static mdbx_atomic_uint32_t rthc_pending; static inline uint64_t rthc_signature(const void *addr, uint8_t kind) { - uint64_t salt = osal_thread_self() * UINT64_C(0xA2F0EEC059629A17) ^ - UINT64_C(0x01E07C6FDB596497) * (uintptr_t)(addr); + uint64_t salt = osal_thread_self() * UINT64_C(0xA2F0EEC059629A17) ^ UINT64_C(0x01E07C6FDB596497) * (uintptr_t)(addr); #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return salt << 8 | kind; #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ @@ -128,45 +119,36 @@ static inline uint64_t rthc_signature(const void *addr, uint8_t kind) { #define MDBX_THREAD_RTHC_REGISTERED(addr) rthc_signature(addr, 0x0D) #define MDBX_THREAD_RTHC_COUNTED(addr) rthc_signature(addr, 0xC0) static __thread uint64_t rthc_thread_state -#if __has_attribute(tls_model) && \ - (defined(__PIC__) || defined(__pic__) || MDBX_BUILD_SHARED_LIBRARY) +#if __has_attribute(tls_model) && (defined(__PIC__) || defined(__pic__) || MDBX_BUILD_SHARED_LIBRARY) __attribute__((tls_model("local-dynamic"))) #endif ; -#if defined(__APPLE__) && defined(__SANITIZE_ADDRESS__) && \ - !defined(MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS) +#if defined(__APPLE__) && defined(__SANITIZE_ADDRESS__) && !defined(MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS) /* Avoid ASAN-trap due the target TLS-variable feed by Darwin's tlv_free() */ -#define MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS \ - __attribute__((__no_sanitize_address__, __noinline__)) +#define MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((__no_sanitize_address__, __noinline__)) #else #define MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS inline #endif -MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS static uint64_t rthc_read(const void *rthc) { - return *(volatile uint64_t *)rthc; -} +MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS static uint64_t rthc_read(const void *rthc) { return *(volatile uint64_t *)rthc; } -MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS static uint64_t -rthc_compare_and_clean(const void *rthc, const uint64_t signature) { +MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS static uint64_t rthc_compare_and_clean(const void *rthc, const uint64_t signature) { #if MDBX_64BIT_CAS return atomic_cas64((mdbx_atomic_uint64_t *)rthc, signature, 0); #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return atomic_cas32((mdbx_atomic_uint32_t *)rthc, (uint32_t)signature, 0); #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - return atomic_cas32((mdbx_atomic_uint32_t *)rthc, (uint32_t)(signature >> 32), - 0); + return atomic_cas32((mdbx_atomic_uint32_t *)rthc, (uint32_t)(signature >> 32), 0); #else #error "FIXME: Unsupported byte order" #endif } -static inline int rthc_atexit(void (*dtor)(void *), void *obj, - void *dso_symbol) { +static inline int rthc_atexit(void (*dtor)(void *), void *obj, void *dso_symbol) { #ifndef MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL -#if defined(LIBCXXABI_HAS_CXA_THREAD_ATEXIT_IMPL) || \ - defined(HAVE___CXA_THREAD_ATEXIT_IMPL) || __GLIBC_PREREQ(2, 18) || \ - defined(BIONIC) +#if defined(LIBCXXABI_HAS_CXA_THREAD_ATEXIT_IMPL) || defined(HAVE___CXA_THREAD_ATEXIT_IMPL) || \ + __GLIBC_PREREQ(2, 18) || defined(BIONIC) #define MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL 1 #else #define MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL 0 @@ -174,11 +156,9 @@ static inline int rthc_atexit(void (*dtor)(void *), void *obj, #endif /* MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL */ #ifndef MDBX_HAVE_CXA_THREAD_ATEXIT -#if defined(LIBCXXABI_HAS_CXA_THREAD_ATEXIT) || \ - defined(HAVE___CXA_THREAD_ATEXIT) +#if defined(LIBCXXABI_HAS_CXA_THREAD_ATEXIT) || defined(HAVE___CXA_THREAD_ATEXIT) #define MDBX_HAVE_CXA_THREAD_ATEXIT 1 -#elif !MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL && \ - (defined(__linux__) || defined(__gnu_linux__)) +#elif !MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL && (defined(__linux__) || defined(__gnu_linux__)) #define MDBX_HAVE_CXA_THREAD_ATEXIT 1 #else #define MDBX_HAVE_CXA_THREAD_ATEXIT 0 @@ -190,13 +170,11 @@ static inline int rthc_atexit(void (*dtor)(void *), void *obj, #define __cxa_thread_atexit __cxa_thread_atexit_impl #endif #if MDBX_HAVE_CXA_THREAD_ATEXIT || defined(__cxa_thread_atexit) - extern int __cxa_thread_atexit(void (*dtor)(void *), void *obj, - void *dso_symbol) MDBX_WEAK_IMPORT_ATTRIBUTE; + extern int __cxa_thread_atexit(void (*dtor)(void *), void *obj, void *dso_symbol) MDBX_WEAK_IMPORT_ATTRIBUTE; if (&__cxa_thread_atexit) rc = __cxa_thread_atexit(dtor, obj, dso_symbol); #elif defined(__APPLE__) || defined(_DARWIN_C_SOURCE) - extern void _tlv_atexit(void (*termfunc)(void *objAddr), void *objAddr) - MDBX_WEAK_IMPORT_ATTRIBUTE; + extern void _tlv_atexit(void (*termfunc)(void *objAddr), void *objAddr) MDBX_WEAK_IMPORT_ATTRIBUTE; if (&_tlv_atexit) { (void)dso_symbol; _tlv_atexit(dtor, obj); @@ -250,8 +228,7 @@ static inline int thread_key_create(osal_thread_key_t *key) { #else rc = pthread_key_create(key, nullptr); #endif - TRACE("&key = %p, value %" PRIuPTR ", rc %d", __Wpedantic_format_voidptr(key), - (uintptr_t)*key, rc); + TRACE("&key = %p, value %" PRIuPTR ", rc %d", __Wpedantic_format_voidptr(key), (uintptr_t)*key, rc); return rc; } @@ -259,21 +236,17 @@ void thread_rthc_set(osal_thread_key_t key, const void *value) { #if defined(_WIN32) || defined(_WIN64) ENSURE(nullptr, TlsSetValue(key, (void *)value)); #else - const uint64_t sign_registered = - MDBX_THREAD_RTHC_REGISTERED(&rthc_thread_state); + const uint64_t sign_registered = MDBX_THREAD_RTHC_REGISTERED(&rthc_thread_state); const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(&rthc_thread_state); - if (value && unlikely(rthc_thread_state != sign_registered && - rthc_thread_state != sign_counted)) { + if (value && unlikely(rthc_thread_state != sign_registered && rthc_thread_state != sign_counted)) { rthc_thread_state = sign_registered; TRACE("thread registered 0x%" PRIxPTR, osal_thread_self()); - if (rthc_atexit(rthc_thread_dtor, &rthc_thread_state, - (void *)&mdbx_version /* dso_anchor */)) { + if (rthc_atexit(rthc_thread_dtor, &rthc_thread_state, (void *)&mdbx_version /* dso_anchor */)) { ENSURE(nullptr, pthread_setspecific(rthc_key, &rthc_thread_state) == 0); rthc_thread_state = sign_counted; const unsigned count_before = atomic_add32(&rthc_pending, 1); ENSURE(nullptr, count_before < INT_MAX); - NOTICE("fallback to pthreads' tsd, key %" PRIuPTR ", count %u", - (uintptr_t)rthc_key, count_before); + NOTICE("fallback to pthreads' tsd, key %" PRIuPTR ", count %u", (uintptr_t)rthc_key, count_before); (void)count_before; } } @@ -286,11 +259,9 @@ __cold void rthc_thread_dtor(void *rthc) { rthc_lock(); const uint32_t current_pid = osal_getpid(); #if defined(_WIN32) || defined(_WIN64) - TRACE(">> pid %d, thread 0x%" PRIxPTR ", module %p", current_pid, - osal_thread_self(), rthc); + TRACE(">> pid %d, thread 0x%" PRIxPTR ", module %p", current_pid, osal_thread_self(), rthc); #else - TRACE(">> pid %d, thread 0x%" PRIxPTR ", rthc %p", current_pid, - osal_thread_self(), rthc); + TRACE(">> pid %d, thread 0x%" PRIxPTR ", rthc %p", current_pid, osal_thread_self(), rthc); #endif for (size_t i = 0; i < rthc_count; ++i) { @@ -306,22 +277,18 @@ __cold void rthc_thread_dtor(void *rthc) { continue; #if !defined(_WIN32) && !defined(_WIN64) if (pthread_setspecific(env->me_txkey, nullptr) != 0) { - TRACE("== thread 0x%" PRIxPTR - ", rthc %p: ignore race with tsd-key deletion", - osal_thread_self(), __Wpedantic_format_voidptr(reader)); + TRACE("== thread 0x%" PRIxPTR ", rthc %p: ignore race with tsd-key deletion", osal_thread_self(), + __Wpedantic_format_voidptr(reader)); continue /* ignore race with tsd-key deletion by mdbx_env_close() */; } #endif - TRACE("== thread 0x%" PRIxPTR - ", rthc %p, [%zi], %p ... %p (%+i), rtch-pid %i, " + TRACE("== thread 0x%" PRIxPTR ", rthc %p, [%zi], %p ... %p (%+i), rtch-pid %i, " "current-pid %i", - osal_thread_self(), __Wpedantic_format_voidptr(reader), i, - __Wpedantic_format_voidptr(begin), __Wpedantic_format_voidptr(end), - (int)(reader - begin), reader->pid.weak, current_pid); + osal_thread_self(), __Wpedantic_format_voidptr(reader), i, __Wpedantic_format_voidptr(begin), + __Wpedantic_format_voidptr(end), (int)(reader - begin), reader->pid.weak, current_pid); if (atomic_load32(&reader->pid, mo_Relaxed) == current_pid) { - TRACE("==== thread 0x%" PRIxPTR ", rthc %p, cleanup", osal_thread_self(), - __Wpedantic_format_voidptr(reader)); + TRACE("==== thread 0x%" PRIxPTR ", rthc %p, cleanup", osal_thread_self(), __Wpedantic_format_voidptr(reader)); (void)atomic_cas32(&reader->pid, current_pid, 0); atomic_store32(&env->lck->rdt_refresh_flag, true, mo_Relaxed); } @@ -334,26 +301,20 @@ __cold void rthc_thread_dtor(void *rthc) { const uint64_t sign_registered = MDBX_THREAD_RTHC_REGISTERED(rthc); const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(rthc); const uint64_t state = rthc_read(rthc); - if (state == sign_registered && - rthc_compare_and_clean(rthc, sign_registered)) { - TRACE("== thread 0x%" PRIxPTR - ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", - osal_thread_self(), rthc, osal_getpid(), "registered", state); - } else if (state == sign_counted && - rthc_compare_and_clean(rthc, sign_counted)) { - TRACE("== thread 0x%" PRIxPTR - ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", - osal_thread_self(), rthc, osal_getpid(), "counted", state); + if (state == sign_registered && rthc_compare_and_clean(rthc, sign_registered)) { + TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), rthc, + osal_getpid(), "registered", state); + } else if (state == sign_counted && rthc_compare_and_clean(rthc, sign_counted)) { + TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), rthc, + osal_getpid(), "counted", state); ENSURE(nullptr, atomic_sub32(&rthc_pending, 1) > 0); } else { - WARNING("thread 0x%" PRIxPTR - ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", - osal_thread_self(), rthc, osal_getpid(), "wrong", state); + WARNING("thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), rthc, + osal_getpid(), "wrong", state); } if (atomic_load32(&rthc_pending, mo_AcquireRelease) == 0) { - TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, wake", osal_thread_self(), - rthc, osal_getpid()); + TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, wake", osal_thread_self(), rthc, osal_getpid()); ENSURE(nullptr, pthread_cond_broadcast(&rthc_cond) == 0); } @@ -367,8 +328,7 @@ __cold void rthc_thread_dtor(void *rthc) { } __cold int rthc_register(MDBX_env *const env) { - TRACE(">> env %p, rthc_count %u, rthc_limit %u", - __Wpedantic_format_voidptr(env), rthc_count, rthc_limit); + TRACE(">> env %p, rthc_count %u, rthc_limit %u", __Wpedantic_format_voidptr(env), rthc_count, rthc_limit); int rc = MDBX_SUCCESS; for (size_t i = 0; i < rthc_count; ++i) @@ -380,8 +340,7 @@ __cold int rthc_register(MDBX_env *const env) { env->me_txkey = 0; if (unlikely(rthc_count == rthc_limit)) { rthc_entry_t *new_table = - osal_realloc((rthc_table == rthc_table_static) ? nullptr : rthc_table, - sizeof(rthc_entry_t) * rthc_limit * 2); + osal_realloc((rthc_table == rthc_table_static) ? nullptr : rthc_table, sizeof(rthc_entry_t) * rthc_limit * 2); if (unlikely(new_table == nullptr)) { rc = MDBX_ENOMEM; goto bailout; @@ -400,14 +359,12 @@ __cold int rthc_register(MDBX_env *const env) { } rthc_table[rthc_count].env = env; - TRACE("== [%i] = env %p, key %" PRIuPTR, rthc_count, - __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey); + TRACE("== [%i] = env %p, key %" PRIuPTR, rthc_count, __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey); ++rthc_count; bailout: - TRACE("<< env %p, key %" PRIuPTR ", rthc_count %u, rthc_limit %u, rc %d", - __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey, rthc_count, - rthc_limit, rc); + TRACE("<< env %p, key %" PRIuPTR ", rthc_count %u, rthc_limit %u, rc %d", __Wpedantic_format_voidptr(env), + (uintptr_t)env->me_txkey, rthc_count, rthc_limit, rc); return rc; } @@ -418,10 +375,8 @@ __cold static int rthc_drown(MDBX_env *const env) { if (likely(env->lck_mmap.lck && current_pid == env->pid)) { reader_slot_t *const begin = &env->lck_mmap.lck->rdt[0]; reader_slot_t *const end = &env->lck_mmap.lck->rdt[env->max_readers]; - TRACE("== %s env %p pid %d, readers %p ...%p, current-pid %d", - (current_pid == env->pid) ? "cleanup" : "skip", - __Wpedantic_format_voidptr(env), env->pid, - __Wpedantic_format_voidptr(begin), __Wpedantic_format_voidptr(end), + TRACE("== %s env %p pid %d, readers %p ...%p, current-pid %d", (current_pid == env->pid) ? "cleanup" : "skip", + __Wpedantic_format_voidptr(env), env->pid, __Wpedantic_format_voidptr(begin), __Wpedantic_format_voidptr(end), current_pid); bool cleaned = false; for (reader_slot_t *r = begin; r < end; ++r) { @@ -434,8 +389,7 @@ __cold static int rthc_drown(MDBX_env *const env) { if (cleaned) atomic_store32(&env->lck_mmap.lck->rdt_refresh_flag, true, mo_Relaxed); rc = rthc_uniq_check(&env->lck_mmap, &inprocess_neighbor); - if (!inprocess_neighbor && env->registered_reader_pid && - env->lck_mmap.fd != INVALID_HANDLE_VALUE) { + if (!inprocess_neighbor && env->registered_reader_pid && env->lck_mmap.fd != INVALID_HANDLE_VALUE) { int err = lck_rpid_clear(env); rc = rc ? rc : err; } @@ -446,9 +400,8 @@ __cold static int rthc_drown(MDBX_env *const env) { } __cold int rthc_remove(MDBX_env *const env) { - TRACE(">>> env %p, key %zu, rthc_count %u, rthc_limit %u", - __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey, rthc_count, - rthc_limit); + TRACE(">>> env %p, key %zu, rthc_count %u, rthc_limit %u", __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey, + rthc_count, rthc_limit); int rc = MDBX_SUCCESS; if (likely(env->pid)) @@ -469,9 +422,8 @@ __cold int rthc_remove(MDBX_env *const env) { } } - TRACE("<<< %p, key %zu, rthc_count %u, rthc_limit %u", - __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey, rthc_count, - rthc_limit); + TRACE("<<< %p, key %zu, rthc_count %u, rthc_limit %u", __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey, + rthc_count, rthc_limit); return rc; } @@ -508,8 +460,8 @@ __cold void rthc_ctor(void) { #else ENSURE(nullptr, pthread_atfork(nullptr, nullptr, rthc_afterfork) == 0); ENSURE(nullptr, pthread_key_create(&rthc_key, rthc_thread_dtor) == 0); - TRACE("pid %d, &mdbx_rthc_key = %p, value 0x%x", osal_getpid(), - __Wpedantic_format_voidptr(&rthc_key), (unsigned)rthc_key); + TRACE("pid %d, &mdbx_rthc_key = %p, value 0x%x", osal_getpid(), __Wpedantic_format_voidptr(&rthc_key), + (unsigned)rthc_key); #endif } @@ -517,33 +469,23 @@ __cold void rthc_dtor(const uint32_t current_pid) { rthc_lock(); #if !defined(_WIN32) && !defined(_WIN64) uint64_t *rthc = pthread_getspecific(rthc_key); - TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status 0x%08" PRIx64 - ", left %d", - osal_thread_self(), __Wpedantic_format_voidptr(rthc), current_pid, - rthc ? rthc_read(rthc) : ~UINT64_C(0), + TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status 0x%08" PRIx64 ", left %d", osal_thread_self(), + __Wpedantic_format_voidptr(rthc), current_pid, rthc ? rthc_read(rthc) : ~UINT64_C(0), atomic_load32(&rthc_pending, mo_Relaxed)); if (rthc) { const uint64_t sign_registered = MDBX_THREAD_RTHC_REGISTERED(rthc); const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(rthc); const uint64_t state = rthc_read(rthc); - if (state == sign_registered && - rthc_compare_and_clean(rthc, sign_registered)) { - TRACE("== thread 0x%" PRIxPTR - ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", - osal_thread_self(), __Wpedantic_format_voidptr(rthc), current_pid, - "registered", state); - } else if (state == sign_counted && - rthc_compare_and_clean(rthc, sign_counted)) { - TRACE("== thread 0x%" PRIxPTR - ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", - osal_thread_self(), __Wpedantic_format_voidptr(rthc), current_pid, - "counted", state); + if (state == sign_registered && rthc_compare_and_clean(rthc, sign_registered)) { + TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), + __Wpedantic_format_voidptr(rthc), current_pid, "registered", state); + } else if (state == sign_counted && rthc_compare_and_clean(rthc, sign_counted)) { + TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), + __Wpedantic_format_voidptr(rthc), current_pid, "counted", state); ENSURE(nullptr, atomic_sub32(&rthc_pending, 1) > 0); } else { - WARNING("thread 0x%" PRIxPTR - ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", - osal_thread_self(), __Wpedantic_format_voidptr(rthc), current_pid, - "wrong", state); + WARNING("thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), + __Wpedantic_format_voidptr(rthc), current_pid, "wrong", state); } } @@ -558,8 +500,7 @@ __cold void rthc_dtor(const uint32_t current_pid) { abstime.tv_sec += 600; #endif - for (unsigned left; - (left = atomic_load32(&rthc_pending, mo_AcquireRelease)) > 0;) { + for (unsigned left; (left = atomic_load32(&rthc_pending, mo_AcquireRelease)) > 0;) { NOTICE("tls-cleanup: pid %d, pending %u, wait for...", current_pid, left); const int rc = pthread_cond_timedwait(&rthc_cond, &rthc_mutex, &abstime); if (rc && rc != EINTR) @@ -581,9 +522,8 @@ __cold void rthc_dtor(const uint32_t current_pid) { for (reader_slot_t *reader = begin; reader < end; ++reader) { TRACE("== [%zi] = key %" PRIuPTR ", %p ... %p, rthc %p (%+i), " "rthc-pid %i, current-pid %i", - i, (uintptr_t)env->me_txkey, __Wpedantic_format_voidptr(begin), - __Wpedantic_format_voidptr(end), __Wpedantic_format_voidptr(reader), - (int)(reader - begin), reader->pid.weak, current_pid); + i, (uintptr_t)env->me_txkey, __Wpedantic_format_voidptr(begin), __Wpedantic_format_voidptr(end), + __Wpedantic_format_voidptr(reader), (int)(reader - begin), reader->pid.weak, current_pid); if (atomic_load32(&reader->pid, mo_Relaxed) == current_pid) { (void)atomic_cas32(&reader->pid, current_pid, 0); TRACE("== cleanup %p", __Wpedantic_format_voidptr(reader)); diff --git a/src/tools/chk.c b/src/tools/chk.c index 22e38460..a22fd57b 100644 --- a/src/tools/chk.c +++ b/src/tools/chk.c @@ -30,8 +30,7 @@ static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { static uint64_t GetMilliseconds(void) { LARGE_INTEGER Counter, Frequency; - return (QueryPerformanceFrequency(&Frequency) && - QueryPerformanceCounter(&Counter)) + return (QueryPerformanceFrequency(&Frequency) && QueryPerformanceCounter(&Counter)) ? Counter.QuadPart * 1000ul / Frequency.QuadPart : 0; } @@ -93,9 +92,8 @@ static void lf_flush(void) { } static bool silently(enum MDBX_chk_severity severity) { - int cutoff = - chk.scope ? chk.scope->verbosity >> MDBX_chk_severity_prio_shift - : verbose + (MDBX_chk_result >> MDBX_chk_severity_prio_shift); + int cutoff = chk.scope ? chk.scope->verbosity >> MDBX_chk_severity_prio_shift + : verbose + (MDBX_chk_result >> MDBX_chk_severity_prio_shift); int prio = (severity >> MDBX_chk_severity_prio_shift); if (chk.scope && chk.scope->stage == MDBX_chk_tables && verbose < 2) prio += 1; @@ -125,11 +123,9 @@ static FILE *prefix(enum MDBX_chk_severity severity) { " ////// " // F +2 }; - const bool nl = - line_struct.scope_depth != chk.scope_nesting || - (line_struct.severity != severity && - (line_struct.severity != MDBX_chk_processing || - severity < MDBX_chk_result || severity > MDBX_chk_resolution)); + const bool nl = line_struct.scope_depth != chk.scope_nesting || + (line_struct.severity != severity && (line_struct.severity != MDBX_chk_processing || + severity < MDBX_chk_result || severity > MDBX_chk_resolution)); if (nl) lf(); if (severity < MDBX_chk_warning) @@ -157,8 +153,7 @@ static void suffix(size_t cookie, const char *str) { } } -static size_t MDBX_PRINTF_ARGS(2, 3) - print(enum MDBX_chk_severity severity, const char *msg, ...) { +static size_t MDBX_PRINTF_ARGS(2, 3) print(enum MDBX_chk_severity severity, const char *msg, ...) { FILE *out = prefix(severity); if (out) { va_list args; @@ -171,8 +166,7 @@ static size_t MDBX_PRINTF_ARGS(2, 3) return 0; } -static FILE *MDBX_PRINTF_ARGS(2, 3) - print_ln(enum MDBX_chk_severity severity, const char *msg, ...) { +static FILE *MDBX_PRINTF_ARGS(2, 3) print_ln(enum MDBX_chk_severity severity, const char *msg, ...) { FILE *out = prefix(severity); if (out) { va_list args; @@ -185,15 +179,12 @@ static FILE *MDBX_PRINTF_ARGS(2, 3) return out; } -static void logger(MDBX_log_level_t level, const char *function, int line, - const char *fmt, va_list args) { +static void logger(MDBX_log_level_t level, const char *function, int line, const char *fmt, va_list args) { if (level <= MDBX_LOG_ERROR) mdbx_env_chk_encount_problem(&chk); - const unsigned kind = (level > MDBX_LOG_NOTICE) - ? level - MDBX_LOG_NOTICE + - (MDBX_chk_extra & MDBX_chk_severity_kind_mask) - : level; + const unsigned kind = + (level > MDBX_LOG_NOTICE) ? level - MDBX_LOG_NOTICE + (MDBX_chk_extra & MDBX_chk_severity_kind_mask) : level; const unsigned prio = kind << MDBX_chk_severity_prio_shift; enum MDBX_chk_severity severity = prio + kind; FILE *out = prefix(severity); @@ -204,8 +195,8 @@ static void logger(MDBX_log_level_t level, const char *function, int line, if (have_lf) for (size_t i = 0; i < line_struct.scope_depth; ++i) fputs(" ", out); - fprintf(out, have_lf ? " %s(), %u" : " (%s:%u)", - function + (strncmp(function, "mdbx_", 5) ? 0 : 5), line); + fprintf(out, have_lf ? " %s(), %u" : " (%s:%u)", function + (strncmp(function, "mdbx_", 5) ? 0 : 5), + line); lf(); } else if (have_lf) { line_struct.empty = true; @@ -249,8 +240,8 @@ static bool check_break(MDBX_chk_context_t *ctx) { return true; } -static int scope_push(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *scope, - MDBX_chk_scope_t *inner, const char *fmt, va_list args) { +static int scope_push(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *scope, MDBX_chk_scope_t *inner, const char *fmt, + va_list args) { (void)scope; if (fmt && *fmt) { FILE *out = prefix(MDBX_chk_processing); @@ -264,22 +255,19 @@ static int scope_push(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *scope, return MDBX_SUCCESS; } -static void scope_pop(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *scope, - MDBX_chk_scope_t *inner) { +static void scope_pop(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *scope, MDBX_chk_scope_t *inner) { (void)ctx; (void)scope; suffix(inner->usr_o.number, inner->subtotal_issues ? "error(s)" : "done"); flush(); } -static MDBX_chk_user_table_cookie_t *table_filter(MDBX_chk_context_t *ctx, - const MDBX_val *name, +static MDBX_chk_user_table_cookie_t *table_filter(MDBX_chk_context_t *ctx, const MDBX_val *name, MDBX_db_flags_t flags) { (void)ctx; (void)flags; return (!only_table.iov_base || - (only_table.iov_len == name->iov_len && - memcmp(only_table.iov_base, name->iov_base, name->iov_len) == 0)) + (only_table.iov_len == name->iov_len && memcmp(only_table.iov_base, name->iov_base, name->iov_len) == 0)) ? (void *)(intptr_t)-1 : nullptr; } @@ -293,8 +281,7 @@ static int stage_begin(MDBX_chk_context_t *ctx, enum MDBX_chk_stage stage) { } static int conclude(MDBX_chk_context_t *ctx); -static int stage_end(MDBX_chk_context_t *ctx, enum MDBX_chk_stage stage, - int err) { +static int stage_end(MDBX_chk_context_t *ctx, enum MDBX_chk_stage stage, int err) { if (stage == MDBX_chk_conclude && !err) err = conclude(ctx); suffix(anchor_lineno, err ? "error(s)" : "done"); @@ -303,14 +290,12 @@ static int stage_end(MDBX_chk_context_t *ctx, enum MDBX_chk_stage stage, return err; } -static MDBX_chk_line_t *print_begin(MDBX_chk_context_t *ctx, - enum MDBX_chk_severity severity) { +static MDBX_chk_line_t *print_begin(MDBX_chk_context_t *ctx, enum MDBX_chk_severity severity) { (void)ctx; if (silently(severity)) return nullptr; if (line_struct.ctx) { - if (line_struct.severity == MDBX_chk_processing && - severity >= MDBX_chk_result && severity <= MDBX_chk_resolution && + if (line_struct.severity == MDBX_chk_processing && severity >= MDBX_chk_result && severity <= MDBX_chk_resolution && line_output) fputc(' ', line_output); else @@ -356,39 +341,36 @@ static const MDBX_chk_callbacks_t cb = {.check_break = check_break, .print_format = print_format}; static void usage(char *prog) { - fprintf( - stderr, - "usage: %s " - "[-V] [-v] [-q] [-c] [-0|1|2] [-w] [-d] [-i] [-s table] [-u|U] dbpath\n" - " -V\t\tprint version and exit\n" - " -v\t\tmore verbose, could be repeated upto 9 times for extra details\n" - " -q\t\tbe quiet\n" - " -c\t\tforce cooperative mode (don't try exclusive)\n" - " -w\t\twrite-mode checking\n" - " -d\t\tdisable page-by-page traversal of B-tree\n" - " -i\t\tignore wrong order errors (for custom comparators case)\n" - " -s table\tprocess a specific subdatabase only\n" - " -u\t\twarmup database before checking\n" - " -U\t\twarmup and try lock database pages in memory before checking\n" - " -0|1|2\tforce using specific meta-page 0, or 2 for checking\n" - " -t\t\tturn to a specified meta-page on successful check\n" - " -T\t\tturn to a specified meta-page EVEN ON UNSUCCESSFUL CHECK!\n", - prog); + fprintf(stderr, + "usage: %s " + "[-V] [-v] [-q] [-c] [-0|1|2] [-w] [-d] [-i] [-s table] [-u|U] dbpath\n" + " -V\t\tprint version and exit\n" + " -v\t\tmore verbose, could be repeated upto 9 times for extra details\n" + " -q\t\tbe quiet\n" + " -c\t\tforce cooperative mode (don't try exclusive)\n" + " -w\t\twrite-mode checking\n" + " -d\t\tdisable page-by-page traversal of B-tree\n" + " -i\t\tignore wrong order errors (for custom comparators case)\n" + " -s table\tprocess a specific subdatabase only\n" + " -u\t\twarmup database before checking\n" + " -U\t\twarmup and try lock database pages in memory before checking\n" + " -0|1|2\tforce using specific meta-page 0, or 2 for checking\n" + " -t\t\tturn to a specified meta-page on successful check\n" + " -T\t\tturn to a specified meta-page EVEN ON UNSUCCESSFUL CHECK!\n", + prog); exit(EXIT_INTERRUPTED); } static int conclude(MDBX_chk_context_t *ctx) { int err = MDBX_SUCCESS; if (ctx->result.total_problems == 1 && ctx->result.problems_meta == 1 && - (chk_flags & - (MDBX_CHK_SKIP_BTREE_TRAVERSAL | MDBX_CHK_SKIP_KV_TRAVERSAL)) == 0 && - (env_flags & MDBX_RDONLY) == 0 && !only_table.iov_base && - stuck_meta < 0 && ctx->result.steady_txnid < ctx->result.recent_txnid) { - const size_t step_lineno = - print(MDBX_chk_resolution, - "Perform sync-to-disk for make steady checkpoint" - " at txn-id #%" PRIi64 "...", - ctx->result.recent_txnid); + (chk_flags & (MDBX_CHK_SKIP_BTREE_TRAVERSAL | MDBX_CHK_SKIP_KV_TRAVERSAL)) == 0 && + (env_flags & MDBX_RDONLY) == 0 && !only_table.iov_base && stuck_meta < 0 && + ctx->result.steady_txnid < ctx->result.recent_txnid) { + const size_t step_lineno = print(MDBX_chk_resolution, + "Perform sync-to-disk for make steady checkpoint" + " at txn-id #%" PRIi64 "...", + ctx->result.recent_txnid); flush(); err = error_fn("walk_pages", mdbx_env_sync_ex(ctx->env, true, false)); if (err == MDBX_SUCCESS) { @@ -398,19 +380,13 @@ static int conclude(MDBX_chk_context_t *ctx) { } } - if (turn_meta && stuck_meta >= 0 && - (chk_flags & - (MDBX_CHK_SKIP_BTREE_TRAVERSAL | MDBX_CHK_SKIP_KV_TRAVERSAL)) == 0 && - !only_table.iov_base && - (env_flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == MDBX_EXCLUSIVE) { - const bool successful_check = - (err | ctx->result.total_problems | ctx->result.problems_meta) == 0; + if (turn_meta && stuck_meta >= 0 && (chk_flags & (MDBX_CHK_SKIP_BTREE_TRAVERSAL | MDBX_CHK_SKIP_KV_TRAVERSAL)) == 0 && + !only_table.iov_base && (env_flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == MDBX_EXCLUSIVE) { + const bool successful_check = (err | ctx->result.total_problems | ctx->result.problems_meta) == 0; if (successful_check || force_turn_meta) { - const size_t step_lineno = print( - MDBX_chk_resolution, - "Performing turn to the specified meta-page (%d) due to %s!", - stuck_meta, - successful_check ? "successful check" : "the -T option was given"); + const size_t step_lineno = + print(MDBX_chk_resolution, "Performing turn to the specified meta-page (%d) due to %s!", stuck_meta, + successful_check ? "successful check" : "the -T option was given"); flush(); err = mdbx_env_turn_for_recovery(ctx->env, stuck_meta); if (err != MDBX_SUCCESS) @@ -475,12 +451,9 @@ int main(int argc, char *argv[]) { " - build: %s for %s by %s\n" " - flags: %s\n" " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.patch, - mdbx_version.tweak, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); + mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor, + mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options); return EXIT_SUCCESS; case 'v': if (verbose >= 9 && 0) @@ -546,8 +519,7 @@ int main(int argc, char *argv[]) { break; case 'U': warmup = true; - warmup_flags = - MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock; + warmup_flags = MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock; break; default: usage(prog); @@ -566,21 +538,17 @@ int main(int argc, char *argv[]) { } if (turn_meta) { if (stuck_meta < 0) { - error_fmt( - "meta-page must be specified (by -0, -1 or -2 options) to turn to " - "it."); + error_fmt("meta-page must be specified (by -0, -1 or -2 options) to turn to " + "it."); rc = EXIT_INTERRUPTED; } if (env_flags & MDBX_RDONLY) { - error_fmt( - "write-mode must be enabled to turn to the specified meta-page."); + error_fmt("write-mode must be enabled to turn to the specified meta-page."); rc = EXIT_INTERRUPTED; } - if (only_table.iov_base || (chk_flags & (MDBX_CHK_SKIP_BTREE_TRAVERSAL | - MDBX_CHK_SKIP_KV_TRAVERSAL))) { - error_fmt( - "whole database checking with b-tree traversal are required to turn " - "to the specified meta-page."); + if (only_table.iov_base || (chk_flags & (MDBX_CHK_SKIP_BTREE_TRAVERSAL | MDBX_CHK_SKIP_KV_TRAVERSAL))) { + error_fmt("whole database checking with b-tree traversal are required to turn " + "to the specified meta-page."); rc = EXIT_INTERRUPTED; } } @@ -604,20 +572,15 @@ int main(int argc, char *argv[]) { print(MDBX_chk_result, "mdbx_chk %s (%s, T-%s)\nRunning for %s in 'read-%s' mode with " "verbosity level %u (%s)...", - mdbx_version.git.describe, mdbx_version.git.datetime, - mdbx_version.git.tree, envname, + mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_version.git.tree, envname, (env_flags & MDBX_RDONLY) ? "only" : "write", verbose, (verbose > 8) - ? (MDBX_DEBUG ? "extra details for debugging" - : "same as 8 for non-debug builds with MDBX_DEBUG=0") + ? (MDBX_DEBUG ? "extra details for debugging" : "same as 8 for non-debug builds with MDBX_DEBUG=0") : "of 0..9"); lf_flush(); - mdbx_setup_debug((verbose + MDBX_LOG_WARN < MDBX_LOG_TRACE) - ? (MDBX_log_level_t)(verbose + MDBX_LOG_WARN) - : MDBX_LOG_TRACE, - MDBX_DBG_DUMP | MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | - MDBX_DBG_LEGACY_OVERLAP | MDBX_DBG_DONT_UPGRADE, - logger); + mdbx_setup_debug( + (verbose + MDBX_LOG_WARN < MDBX_LOG_TRACE) ? (MDBX_log_level_t)(verbose + MDBX_LOG_WARN) : MDBX_LOG_TRACE, + MDBX_DBG_DUMP | MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_LEGACY_OVERLAP | MDBX_DBG_DONT_UPGRADE, logger); rc = mdbx_env_create(&env); if (rc) { @@ -632,18 +595,16 @@ int main(int argc, char *argv[]) { } if (stuck_meta >= 0) { - rc = mdbx_env_open_for_recovery(env, envname, stuck_meta, - (env_flags & MDBX_RDONLY) ? false : true); + rc = mdbx_env_open_for_recovery(env, envname, stuck_meta, (env_flags & MDBX_RDONLY) ? false : true); } else { rc = mdbx_env_open(env, envname, env_flags, 0); - if ((env_flags & MDBX_EXCLUSIVE) && - (rc == MDBX_BUSY || + if ((env_flags & MDBX_EXCLUSIVE) && (rc == MDBX_BUSY || #if defined(_WIN32) || defined(_WIN64) - rc == ERROR_LOCK_VIOLATION || rc == ERROR_SHARING_VIOLATION + rc == ERROR_LOCK_VIOLATION || rc == ERROR_SHARING_VIOLATION #else - rc == EBUSY || rc == EAGAIN + rc == EBUSY || rc == EAGAIN #endif - )) { + )) { env_flags &= ~MDBX_EXCLUSIVE; rc = mdbx_env_open(env, envname, env_flags | MDBX_ACCEDE, 0); } @@ -652,13 +613,10 @@ int main(int argc, char *argv[]) { if (rc) { error_fn("mdbx_env_open", rc); if (rc == MDBX_WANNA_RECOVERY && (env_flags & MDBX_RDONLY)) - print_ln(MDBX_chk_result, - "Please run %s in the read-write mode (with '-w' option).", - prog); + print_ln(MDBX_chk_result, "Please run %s in the read-write mode (with '-w' option).", prog); goto bailout; } - print_ln(MDBX_chk_verbose, "%s mode", - (env_flags & MDBX_EXCLUSIVE) ? "monopolistic" : "cooperative"); + print_ln(MDBX_chk_verbose, "%s mode", (env_flags & MDBX_EXCLUSIVE) ? "monopolistic" : "cooperative"); if (warmup) { anchor_lineno = print(MDBX_chk_verbose, "warming up..."); @@ -671,9 +629,7 @@ int main(int argc, char *argv[]) { suffix(anchor_lineno, rc ? "timeout" : "done"); } - rc = mdbx_env_chk(env, &cb, &chk, chk_flags, - MDBX_chk_result + (verbose << MDBX_chk_severity_prio_shift), - 0); + rc = mdbx_env_chk(env, &cb, &chk, chk_flags, MDBX_chk_result + (verbose << MDBX_chk_severity_prio_shift), 0); if (rc) { if (chk.result.total_problems == 0) error_fn("mdbx_env_chk", rc); @@ -683,8 +639,7 @@ int main(int argc, char *argv[]) { bailout: if (env) { - const bool dont_sync = rc != 0 || chk.result.total_problems || - (chk_flags & MDBX_CHK_READWRITE) == 0; + const bool dont_sync = rc != 0 || chk.result.total_problems || (chk_flags & MDBX_CHK_READWRITE) == 0; mdbx_env_close_ex(env, dont_sync); } flush(); @@ -702,21 +657,17 @@ bailout: error_fn("clock_gettime", errno); return EXIT_FAILURE_SYS; } - elapsed = timestamp_finish.tv_sec - timestamp_start.tv_sec + - (timestamp_finish.tv_nsec - timestamp_start.tv_nsec) * 1e-9; + elapsed = + timestamp_finish.tv_sec - timestamp_start.tv_sec + (timestamp_finish.tv_nsec - timestamp_start.tv_nsec) * 1e-9; #endif /* !WINDOWS */ if (chk.result.total_problems) { - print_ln(MDBX_chk_result, - "Total %" PRIuSIZE " error%s detected, elapsed %.3f seconds.", - chk.result.total_problems, + print_ln(MDBX_chk_result, "Total %" PRIuSIZE " error%s detected, elapsed %.3f seconds.", chk.result.total_problems, (chk.result.total_problems > 1) ? "s are" : " is", elapsed); - if (chk.result.problems_meta || chk.result.problems_kv || - chk.result.problems_gc) + if (chk.result.problems_meta || chk.result.problems_kv || chk.result.problems_gc) return EXIT_FAILURE_CHECK_MAJOR; return EXIT_FAILURE_CHECK_MINOR; } - print_ln(MDBX_chk_result, "No error is detected, elapsed %.3f seconds.", - elapsed); + print_ln(MDBX_chk_result, "No error is detected, elapsed %.3f seconds.", elapsed); return EXIT_SUCCESS; } diff --git a/src/tools/copy.c b/src/tools/copy.c index 4441a982..122bea72 100644 --- a/src/tools/copy.c +++ b/src/tools/copy.c @@ -37,20 +37,19 @@ static void signal_handler(int sig) { #endif /* !WINDOWS */ static void usage(const char *prog) { - fprintf( - stderr, - "usage: %s [-V] [-q] [-c] [-d] [-p] [-u|U] src_path [dest_path]\n" - " -V\t\tprint version and exit\n" - " -q\t\tbe quiet\n" - " -c\t\tenable compactification (skip unused pages)\n" - " -d\t\tenforce copy to be a dynamic size DB\n" - " -p\t\tusing transaction parking/ousting during copying MVCC-snapshot\n" - " \t\tto avoid stopping recycling and overflowing the DB\n" - " -u\t\twarmup database before copying\n" - " -U\t\twarmup and try lock database pages in memory before copying\n" - " src_path\tsource database\n" - " dest_path\tdestination (stdout if not specified)\n", - prog); + fprintf(stderr, + "usage: %s [-V] [-q] [-c] [-d] [-p] [-u|U] src_path [dest_path]\n" + " -V\t\tprint version and exit\n" + " -q\t\tbe quiet\n" + " -c\t\tenable compactification (skip unused pages)\n" + " -d\t\tenforce copy to be a dynamic size DB\n" + " -p\t\tusing transaction parking/ousting during copying MVCC-snapshot\n" + " \t\tto avoid stopping recycling and overflowing the DB\n" + " -u\t\twarmup database before copying\n" + " -U\t\twarmup and try lock database pages in memory before copying\n" + " src_path\tsource database\n" + " dest_path\tdestination (stdout if not specified)\n", + prog); exit(EXIT_FAILURE); } @@ -79,10 +78,8 @@ int main(int argc, char *argv[]) { warmup = true; else if (argv[1][1] == 'U' && argv[1][2] == '\0') { warmup = true; - warmup_flags = - MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock; - } else if ((argv[1][1] == 'h' && argv[1][2] == '\0') || - strcmp(argv[1], "--help") == 0) + warmup_flags = MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock; + } else if ((argv[1][1] == 'h' && argv[1][2] == '\0') || strcmp(argv[1], "--help") == 0) usage(progname); else if (argv[1][1] == 'V' && argv[1][2] == '\0') { printf("mdbx_copy version %d.%d.%d.%d\n" @@ -91,12 +88,9 @@ int main(int argc, char *argv[]) { " - build: %s for %s by %s\n" " - flags: %s\n" " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.patch, - mdbx_version.tweak, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); + mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor, + mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options); return EXIT_SUCCESS; } else argc = 0; @@ -119,10 +113,9 @@ int main(int argc, char *argv[]) { #endif /* !WINDOWS */ if (!quiet) { - fprintf((argc == 2) ? stderr : stdout, - "mdbx_copy %s (%s, T-%s)\nRunning for copy %s to %s...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, - mdbx_version.git.tree, argv[1], (argc == 2) ? "stdout" : argv[2]); + fprintf((argc == 2) ? stderr : stdout, "mdbx_copy %s (%s, T-%s)\nRunning for copy %s to %s...\n", + mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_version.git.tree, argv[1], + (argc == 2) ? "stdout" : argv[2]); fflush(nullptr); } @@ -150,8 +143,7 @@ int main(int argc, char *argv[]) { rc = mdbx_env_copy(env, argv[2], cpflags); } if (rc) - fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc, - mdbx_strerror(rc)); + fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc, mdbx_strerror(rc)); mdbx_env_close(env); return rc ? EXIT_FAILURE : EXIT_SUCCESS; diff --git a/src/tools/drop.c b/src/tools/drop.c index 8de80cd9..9f5cfbfd 100644 --- a/src/tools/drop.c +++ b/src/tools/drop.c @@ -54,8 +54,7 @@ static void usage(void) { static void error(const char *func, int rc) { if (!quiet) - fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, - mdbx_strerror(rc)); + fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc)); } int main(int argc, char *argv[]) { @@ -86,12 +85,9 @@ int main(int argc, char *argv[]) { " - build: %s for %s by %s\n" " - flags: %s\n" " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.patch, - mdbx_version.tweak, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); + mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor, + mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options); return EXIT_SUCCESS; case 'q': quiet = true; @@ -127,8 +123,7 @@ int main(int argc, char *argv[]) { envname = argv[optind]; if (!quiet) { - printf("mdbx_drop %s (%s, T-%s)\nRunning for %s/%s...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, + printf("mdbx_drop %s (%s, T-%s)\nRunning for %s/%s...\n", mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_version.git.tree, envname, subname ? subname : "@MAIN"); fflush(nullptr); } diff --git a/src/tools/dump.c b/src/tools/dump.c index de93422f..f7c1a49d 100644 --- a/src/tools/dump.c +++ b/src/tools/dump.c @@ -95,8 +95,7 @@ bool quiet = false, rescue = false; const char *prog; static void error(const char *func, int rc) { if (!quiet) - fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, - mdbx_strerror(rc)); + fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc)); } /* Dump in BDB-compatible format */ @@ -126,10 +125,8 @@ static int dump_tbl(MDBX_txn *txn, MDBX_dbi dbi, char *name) { if (mode & GLOBAL) { mode -= GLOBAL; if (info.mi_geo.upper != info.mi_geo.lower) - printf("geometry=l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64 - ",g%" PRIu64 "\n", - info.mi_geo.lower, info.mi_geo.current, info.mi_geo.upper, - info.mi_geo.shrink, info.mi_geo.grow); + printf("geometry=l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64 ",g%" PRIu64 "\n", info.mi_geo.lower, + info.mi_geo.current, info.mi_geo.upper, info.mi_geo.shrink, info.mi_geo.grow); printf("mapsize=%" PRIu64 "\n", info.mi_geo.upper); printf("maxreaders=%u\n", info.mi_maxreaders); @@ -140,8 +137,7 @@ static int dump_tbl(MDBX_txn *txn, MDBX_dbi dbi, char *name) { return rc; } if (canary.v) - printf("canary=v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64 "\n", - canary.v, canary.x, canary.y, canary.z); + printf("canary=v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64 "\n", canary.v, canary.x, canary.y, canary.z); } printf("format=%s\n", mode & PRINT ? "print" : "bytevalue"); if (name) @@ -153,10 +149,7 @@ static int dump_tbl(MDBX_txn *txn, MDBX_dbi dbi, char *name) { else if (!name) printf("txnid=%" PRIaTXN "\n", mdbx_txn_id(txn)); */ - printf("duplicates=%d\n", (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | - MDBX_INTEGERDUP | MDBX_REVERSEDUP)) - ? 1 - : 0); + printf("duplicates=%d\n", (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP)) ? 1 : 0); for (int i = 0; dbflags[i].bit; i++) if (flags & dbflags[i].bit) printf("%s=1\n", dbflags[i].name); @@ -187,8 +180,7 @@ static int dump_tbl(MDBX_txn *txn, MDBX_dbi dbi, char *name) { } } - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == - MDBX_SUCCESS) { + while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == MDBX_SUCCESS) { if (user_break) { rc = MDBX_EINTR; break; @@ -212,31 +204,27 @@ static int dump_tbl(MDBX_txn *txn, MDBX_dbi dbi, char *name) { } static void usage(void) { - fprintf( - stderr, - "usage: %s " - "[-V] [-q] [-f file] [-l] [-p] [-r] [-a|-s table] [-u|U] " - "dbpath\n" - " -V\t\tprint version and exit\n" - " -q\t\tbe quiet\n" - " -f\t\twrite to file instead of stdout\n" - " -l\t\tlist tables and exit\n" - " -p\t\tuse printable characters\n" - " -r\t\trescue mode (ignore errors to dump corrupted DB)\n" - " -a\t\tdump main DB and all tables\n" - " -s name\tdump only the specified named table\n" - " -u\t\twarmup database before dumping\n" - " -U\t\twarmup and try lock database pages in memory before dumping\n" - " \t\tby default dump only the main DB\n", - prog); + fprintf(stderr, + "usage: %s " + "[-V] [-q] [-f file] [-l] [-p] [-r] [-a|-s table] [-u|U] " + "dbpath\n" + " -V\t\tprint version and exit\n" + " -q\t\tbe quiet\n" + " -f\t\twrite to file instead of stdout\n" + " -l\t\tlist tables and exit\n" + " -p\t\tuse printable characters\n" + " -r\t\trescue mode (ignore errors to dump corrupted DB)\n" + " -a\t\tdump main DB and all tables\n" + " -s name\tdump only the specified named table\n" + " -u\t\twarmup database before dumping\n" + " -U\t\twarmup and try lock database pages in memory before dumping\n" + " \t\tby default dump only the main DB\n", + prog); exit(EXIT_FAILURE); } static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) { - return (a->iov_len == b->iov_len && - memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) - ? 0 - : 1; + return (a->iov_len == b->iov_len && memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) ? 0 : 1; } int main(int argc, char *argv[]) { @@ -274,12 +262,9 @@ int main(int argc, char *argv[]) { " - build: %s for %s by %s\n" " - flags: %s\n" " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.patch, - mdbx_version.tweak, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); + mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor, + mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options); return EXIT_SUCCESS; case 'l': list = true; @@ -292,8 +277,7 @@ int main(int argc, char *argv[]) { break; case 'f': if (freopen(optarg, "w", stdout) == nullptr) { - fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, - mdbx_strerror(errno)); + fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, mdbx_strerror(errno)); exit(EXIT_FAILURE); } break; @@ -318,8 +302,7 @@ int main(int argc, char *argv[]) { break; case 'U': warmup = true; - warmup_flags = - MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock; + warmup_flags = MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock; break; default: usage(); @@ -344,9 +327,8 @@ int main(int argc, char *argv[]) { envname = argv[optind]; if (!quiet) { - fprintf(stderr, "mdbx_dump %s (%s, T-%s)\nRunning for %s...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, - mdbx_version.git.tree, envname); + fprintf(stderr, "mdbx_dump %s (%s, T-%s)\nRunning for %s...\n", mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.tree, envname); fflush(nullptr); } @@ -364,11 +346,8 @@ int main(int argc, char *argv[]) { } } - err = mdbx_env_open( - env, envname, - envflags | (rescue ? MDBX_RDONLY | MDBX_EXCLUSIVE | MDBX_VALIDATION - : MDBX_RDONLY), - 0); + err = mdbx_env_open(env, envname, envflags | (rescue ? MDBX_RDONLY | MDBX_EXCLUSIVE | MDBX_VALIDATION : MDBX_RDONLY), + 0); if (unlikely(err != MDBX_SUCCESS)) { error("mdbx_env_open", err); goto env_close; @@ -414,8 +393,7 @@ int main(int argc, char *argv[]) { bool have_raw = false; int count = 0; MDBX_val key; - while (MDBX_SUCCESS == - (err = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) { + while (MDBX_SUCCESS == (err = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) { if (user_break) { err = MDBX_EINTR; break; @@ -434,8 +412,7 @@ int main(int argc, char *argv[]) { subname[key.iov_len] = '\0'; MDBX_dbi sub_dbi; - err = mdbx_dbi_open_ex(txn, subname, MDBX_DB_ACCEDE, &sub_dbi, - rescue ? equal_or_greater : nullptr, + err = mdbx_dbi_open_ex(txn, subname, MDBX_DB_ACCEDE, &sub_dbi, rescue ? equal_or_greater : nullptr, rescue ? equal_or_greater : nullptr); if (unlikely(err != MDBX_SUCCESS)) { if (err == MDBX_INCOMPATIBLE) { @@ -455,8 +432,7 @@ int main(int argc, char *argv[]) { if (!rescue) break; if (!quiet) - fprintf(stderr, "%s: %s: ignore %s for `%s` and continue\n", prog, - envname, mdbx_strerror(err), subname); + fprintf(stderr, "%s: %s: ignore %s for `%s` and continue\n", prog, envname, mdbx_strerror(err), subname); /* Here is a hack for rescue mode, don't do that: * - we should restart transaction in case error due * database corruption; @@ -491,8 +467,7 @@ int main(int argc, char *argv[]) { err = dump_tbl(txn, MAIN_DBI, nullptr); else if (!count) { if (!quiet) - fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, - envname); + fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, envname); err = MDBX_NOTFOUND; } } else { diff --git a/src/tools/load.c b/src/tools/load.c index 9182926f..efdd50c9 100644 --- a/src/tools/load.c +++ b/src/tools/load.c @@ -44,11 +44,10 @@ static size_t lineno; static void error(const char *func, int rc) { if (!quiet) { if (lineno) - fprintf(stderr, "%s: at input line %" PRIiSIZE ": %s() error %d, %s\n", - prog, lineno, func, rc, mdbx_strerror(rc)); - else - fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, + fprintf(stderr, "%s: at input line %" PRIiSIZE ": %s() error %d, %s\n", prog, lineno, func, rc, mdbx_strerror(rc)); + else + fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc)); } } @@ -60,9 +59,7 @@ static char *valstr(char *line, const char *item) { if (line[len] > ' ') return nullptr; if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", prog, - lineno, item); + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", prog, lineno, item); exit(EXIT_FAILURE); } char *ptr = strchr(line, '\n'); @@ -80,9 +77,7 @@ static bool valnum(char *line, const char *item, uint64_t *value) { *value = strtoull(str, &end, 0); if (end && *end) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unexpected number format for '%s'\n", - prog, lineno, item); + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected number format for '%s'\n", prog, lineno, item); exit(EXIT_FAILURE); } return true; @@ -95,8 +90,7 @@ static bool valbool(char *line, const char *item, bool *value) { if (u64 > 1) { if (!quiet) - fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected value for '%s'\n", - prog, lineno, item); + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected value for '%s'\n", prog, lineno, item); exit(EXIT_FAILURE); } *value = u64 != 0; @@ -129,11 +123,10 @@ typedef struct flagbit { #define S(s) STRLENOF(s), s -flagbit dbflags[] = { - {MDBX_REVERSEKEY, S("reversekey")}, {MDBX_DUPSORT, S("duplicates")}, - {MDBX_DUPSORT, S("dupsort")}, {MDBX_INTEGERKEY, S("integerkey")}, - {MDBX_DUPFIXED, S("dupfix")}, {MDBX_INTEGERDUP, S("integerdup")}, - {MDBX_REVERSEDUP, S("reversedup")}, {0, 0, nullptr}}; +flagbit dbflags[] = {{MDBX_REVERSEKEY, S("reversekey")}, {MDBX_DUPSORT, S("duplicates")}, + {MDBX_DUPSORT, S("dupsort")}, {MDBX_INTEGERKEY, S("integerkey")}, + {MDBX_DUPFIXED, S("dupfix")}, {MDBX_INTEGERDUP, S("integerdup")}, + {MDBX_REVERSEDUP, S("reversedup")}, {0, 0, nullptr}}; static int readhdr(void) { /* reset parameters */ @@ -158,10 +151,8 @@ static int readhdr(void) { if (valnum(dbuf.iov_base, "VERSION", &u64)) { if (u64 != 3) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unsupported value %" PRIu64 - " for %s\n", - prog, lineno, u64, "VERSION"); + fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported value %" PRIu64 " for %s\n", prog, lineno, u64, + "VERSION"); exit(EXIT_FAILURE); } continue; @@ -170,16 +161,12 @@ static int readhdr(void) { if (valnum(dbuf.iov_base, "db_pagesize", &u64)) { if (!(mode & GLOBAL) && envinfo.mi_dxb_pagesize != u64) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore value %" PRIu64 - " for '%s' in non-global context\n", - prog, lineno, u64, "db_pagesize"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore value %" PRIu64 " for '%s' in non-global context\n", prog, + lineno, u64, "db_pagesize"); } else if (u64 < MDBX_MIN_PAGESIZE || u64 > MDBX_MAX_PAGESIZE) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value %" PRIu64 - " for %s\n", - prog, lineno, u64, "db_pagesize"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value %" PRIu64 " for %s\n", prog, lineno, u64, + "db_pagesize"); } else envinfo.mi_dxb_pagesize = (uint32_t)u64; continue; @@ -196,9 +183,7 @@ static int readhdr(void) { continue; } if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", prog, - lineno, str, "format"); + fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", prog, lineno, str, "format"); exit(EXIT_FAILURE); } @@ -220,9 +205,7 @@ static int readhdr(void) { if (str) { if (strcmp(str, "btree") != 0) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", - prog, lineno, str, "type"); + fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", prog, lineno, str, "type"); free(subname); exit(EXIT_FAILURE); } @@ -232,10 +215,8 @@ static int readhdr(void) { if (valnum(dbuf.iov_base, "mapaddr", &u64)) { if (u64) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 - " for %s\n", - prog, lineno, u64, "mapaddr"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 " for %s\n", prog, lineno, u64, + "mapaddr"); } continue; } @@ -243,16 +224,12 @@ static int readhdr(void) { if (valnum(dbuf.iov_base, "mapsize", &u64)) { if (!(mode & GLOBAL)) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore value %" PRIu64 - " for '%s' in non-global context\n", - prog, lineno, u64, "mapsize"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore value %" PRIu64 " for '%s' in non-global context\n", prog, + lineno, u64, "mapsize"); } else if (u64 < MIN_MAPSIZE || u64 > MAX_MAPSIZE64) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 - " for %s\n", - prog, lineno, u64, "mapsize"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 " for %s\n", prog, lineno, u64, + "mapsize"); } else envinfo.mi_mapsize = (size_t)u64; continue; @@ -261,16 +238,12 @@ static int readhdr(void) { if (valnum(dbuf.iov_base, "maxreaders", &u64)) { if (!(mode & GLOBAL)) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore value %" PRIu64 - " for '%s' in non-global context\n", - prog, lineno, u64, "maxreaders"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore value %" PRIu64 " for '%s' in non-global context\n", prog, + lineno, u64, "maxreaders"); } else if (u64 < 1 || u64 > MDBX_READERS_LIMIT) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 - " for %s\n", - prog, lineno, u64, "maxreaders"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 " for %s\n", prog, lineno, u64, + "maxreaders"); } else envinfo.mi_maxreaders = (int)u64; continue; @@ -279,10 +252,8 @@ static int readhdr(void) { if (valnum(dbuf.iov_base, "txnid", &u64)) { if (u64 < MIN_TXNID || u64 > MAX_TXNID) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 - " for %s\n", - prog, lineno, u64, "txnid"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 " for %s\n", prog, lineno, u64, + "txnid"); } else txnid = u64; continue; @@ -301,16 +272,11 @@ static int readhdr(void) { "%s: line %" PRIiSIZE ": ignore values %s" " for '%s' in non-global context\n", prog, lineno, str, "geometry"); - } else if (sscanf(str, - "l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64 - ",g%" PRIu64, - &envinfo.mi_geo.lower, &envinfo.mi_geo.current, - &envinfo.mi_geo.upper, &envinfo.mi_geo.shrink, + } else if (sscanf(str, "l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64 ",g%" PRIu64, &envinfo.mi_geo.lower, + &envinfo.mi_geo.current, &envinfo.mi_geo.upper, &envinfo.mi_geo.shrink, &envinfo.mi_geo.grow) != 5) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", - prog, lineno, "geometry"); + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", prog, lineno, "geometry"); exit(EXIT_FAILURE); } continue; @@ -324,12 +290,10 @@ static int readhdr(void) { "%s: line %" PRIiSIZE ": ignore values %s" " for '%s' in non-global context\n", prog, lineno, str, "canary"); - } else if (sscanf(str, "v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64, - &canary.v, &canary.x, &canary.y, &canary.z) != 4) { + } else if (sscanf(str, "v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64, &canary.v, &canary.x, &canary.y, + &canary.z) != 4) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", - prog, lineno, "canary"); + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", prog, lineno, "canary"); exit(EXIT_FAILURE); } continue; @@ -353,9 +317,8 @@ static int readhdr(void) { } if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n", - prog, lineno, (char *)dbuf.iov_base); + fprintf(stderr, "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n", prog, lineno, + (char *)dbuf.iov_base); next:; } return EOF; @@ -363,8 +326,7 @@ static int readhdr(void) { static int badend(void) { if (!quiet) - fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog, - lineno); + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog, lineno); return errno ? errno : MDBX_ENODATA; } @@ -416,9 +378,7 @@ __hot static int readline(MDBX_val *out, MDBX_val *buf) { buf->iov_base = osal_realloc(buf->iov_base, buf->iov_len * 2); if (!buf->iov_base) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": out of memory, line too long\n", prog, - lineno); + fprintf(stderr, "%s: line %" PRIiSIZE ": out of memory, line too long\n", prog, lineno); return MDBX_ENOMEM; } c1 = buf->iov_base; @@ -490,10 +450,7 @@ static void usage(void) { } static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) { - return (a->iov_len == b->iov_len && - memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) - ? 0 - : 1; + return (a->iov_len == b->iov_len && memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) ? 0 : 1; } int main(int argc, char *argv[]) { @@ -530,12 +487,9 @@ int main(int argc, char *argv[]) { " - build: %s for %s by %s\n" " - flags: %s\n" " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.patch, - mdbx_version.tweak, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); + mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor, + mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options); return EXIT_SUCCESS; case 'a': putflags |= MDBX_APPEND; @@ -543,8 +497,7 @@ int main(int argc, char *argv[]) { case 'f': if (freopen(optarg, "r", stdin) == nullptr) { if (!quiet) - fprintf(stderr, "%s: %s: open: %s\n", prog, optarg, - mdbx_strerror(errno)); + fprintf(stderr, "%s: %s: open: %s\n", prog, optarg, mdbx_strerror(errno)); exit(EXIT_FAILURE); } break; @@ -592,8 +545,7 @@ int main(int argc, char *argv[]) { envname = argv[optind]; if (!quiet) - printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, + printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n", mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_version.git.tree, envname); fflush(nullptr); @@ -638,25 +590,22 @@ int main(int argc, char *argv[]) { if (envinfo.mi_geo.current | envinfo.mi_mapsize) { if (envinfo.mi_geo.current) { - err = mdbx_env_set_geometry( - env, (intptr_t)envinfo.mi_geo.lower, (intptr_t)envinfo.mi_geo.current, - (intptr_t)envinfo.mi_geo.upper, (intptr_t)envinfo.mi_geo.shrink, - (intptr_t)envinfo.mi_geo.grow, - envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1); + err = mdbx_env_set_geometry(env, (intptr_t)envinfo.mi_geo.lower, (intptr_t)envinfo.mi_geo.current, + (intptr_t)envinfo.mi_geo.upper, (intptr_t)envinfo.mi_geo.shrink, + (intptr_t)envinfo.mi_geo.grow, + envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1); } else { if (envinfo.mi_mapsize > MAX_MAPSIZE) { if (!quiet) - fprintf( - stderr, - "Database size is too large for current system (mapsize=%" PRIu64 - " is great than system-limit %zu)\n", - envinfo.mi_mapsize, (size_t)MAX_MAPSIZE); + fprintf(stderr, + "Database size is too large for current system (mapsize=%" PRIu64 + " is great than system-limit %zu)\n", + envinfo.mi_mapsize, (size_t)MAX_MAPSIZE); goto bailout; } - err = mdbx_env_set_geometry( - env, (intptr_t)envinfo.mi_mapsize, (intptr_t)envinfo.mi_mapsize, - (intptr_t)envinfo.mi_mapsize, 0, 0, - envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1); + err = mdbx_env_set_geometry(env, (intptr_t)envinfo.mi_mapsize, (intptr_t)envinfo.mi_mapsize, + (intptr_t)envinfo.mi_mapsize, 0, 0, + envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1); } if (unlikely(err != MDBX_SUCCESS)) { error("mdbx_env_set_geometry", err); @@ -673,8 +622,7 @@ int main(int argc, char *argv[]) { kbuf.iov_len = mdbx_env_get_maxvalsize_ex(env, 0) + (size_t)1; if (kbuf.iov_len >= INTPTR_MAX / 2) { if (!quiet) - fprintf(stderr, "mdbx_env_get_maxkeysize() failed, returns %zu\n", - kbuf.iov_len); + fprintf(stderr, "mdbx_env_get_maxkeysize() failed, returns %zu\n", kbuf.iov_len); goto bailout; } @@ -709,10 +657,9 @@ int main(int argc, char *argv[]) { } const char *const dbi_name = subname ? subname : "@MAIN"; - err = - mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi, - (putflags & MDBX_APPEND) ? equal_or_greater : nullptr, - (putflags & MDBX_APPEND) ? equal_or_greater : nullptr); + err = mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi, + (putflags & MDBX_APPEND) ? equal_or_greater : nullptr, + (putflags & MDBX_APPEND) ? equal_or_greater : nullptr); if (unlikely(err != MDBX_SUCCESS)) { error("mdbx_dbi_open_ex", err); goto bailout; @@ -726,9 +673,7 @@ int main(int argc, char *argv[]) { } if (present_sequence > sequence) { if (!quiet) - fprintf(stderr, - "present sequence for '%s' value (%" PRIu64 - ") is greater than loaded (%" PRIu64 ")\n", + fprintf(stderr, "present sequence for '%s' value (%" PRIu64 ") is greater than loaded (%" PRIu64 ")\n", dbi_name, present_sequence, sequence); err = MDBX_RESULT_TRUE; goto bailout; @@ -750,8 +695,7 @@ int main(int argc, char *argv[]) { } if (putflags & MDBX_APPEND) - putflags = (dbi_flags & MDBX_DUPSORT) ? putflags | MDBX_APPENDDUP - : putflags & ~MDBX_APPENDDUP; + putflags = (dbi_flags & MDBX_DUPSORT) ? putflags | MDBX_APPENDDUP : putflags & ~MDBX_APPENDDUP; err = mdbx_cursor_open(txn, dbi, &mc); if (unlikely(err != MDBX_SUCCESS)) { @@ -770,8 +714,7 @@ int main(int argc, char *argv[]) { err = readline(&data, &dbuf); if (err) { if (!quiet) - fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n", - prog, lineno); + fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n", prog, lineno); goto bailout; } @@ -780,8 +723,7 @@ int main(int argc, char *argv[]) { continue; if (err == MDBX_BAD_VALSIZE && rescue) { if (!quiet) - fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno, - mdbx_strerror(err)); + fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno, mdbx_strerror(err)); continue; } if (unlikely(err != MDBX_SUCCESS)) { diff --git a/src/tools/stat.c b/src/tools/stat.c index f8808caa..57c99b45 100644 --- a/src/tools/stat.c +++ b/src/tools/stat.c @@ -61,27 +61,24 @@ static void usage(const char *prog) { exit(EXIT_FAILURE); } -static int reader_list_func(void *ctx, int num, int slot, mdbx_pid_t pid, - mdbx_tid_t thread, uint64_t txnid, uint64_t lag, - size_t bytes_used, size_t bytes_retained) { +static int reader_list_func(void *ctx, int num, int slot, mdbx_pid_t pid, mdbx_tid_t thread, uint64_t txnid, + uint64_t lag, size_t bytes_used, size_t bytes_retained) { (void)ctx; if (num == 1) printf("Reader Table\n" " #\tslot\t%6s %*s %20s %10s %13s %13s\n", - "pid", (int)sizeof(size_t) * 2, "thread", "txnid", "lag", "used", - "retained"); + "pid", (int)sizeof(size_t) * 2, "thread", "txnid", "lag", "used", "retained"); if (thread < (mdbx_tid_t)((intptr_t)MDBX_TID_TXN_OUSTED)) - printf(" %3d)\t[%d]\t%6" PRIdSIZE " %*" PRIxPTR, num, slot, (size_t)pid, - (int)sizeof(size_t) * 2, (uintptr_t)thread); + printf(" %3d)\t[%d]\t%6" PRIdSIZE " %*" PRIxPTR, num, slot, (size_t)pid, (int)sizeof(size_t) * 2, + (uintptr_t)thread); else printf(" %3d)\t[%d]\t%6" PRIdSIZE " %sed", num, slot, (size_t)pid, - (thread == (mdbx_tid_t)((uintptr_t)MDBX_TID_TXN_PARKED)) ? "park" - : "oust"); + (thread == (mdbx_tid_t)((uintptr_t)MDBX_TID_TXN_PARKED)) ? "park" : "oust"); if (txnid) - printf(" %20" PRIu64 " %10" PRIu64 " %12.1fM %12.1fM\n", txnid, lag, - bytes_used / 1048576.0, bytes_retained / 1048576.0); + printf(" %20" PRIu64 " %10" PRIu64 " %12.1fM %12.1fM\n", txnid, lag, bytes_used / 1048576.0, + bytes_retained / 1048576.0); else printf(" %20s %10s %13s %13s\n", "-", "0", "0", "0"); @@ -92,8 +89,7 @@ const char *prog; bool quiet = false; static void error(const char *func, int rc) { if (!quiet) - fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, - mdbx_strerror(rc)); + fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc)); } int main(int argc, char *argv[]) { @@ -129,12 +125,9 @@ int main(int argc, char *argv[]) { " - build: %s for %s by %s\n" " - flags: %s\n" " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.patch, - mdbx_version.tweak, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); + mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor, + mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options); return EXIT_SUCCESS; case 'q': quiet = true; @@ -187,8 +180,7 @@ int main(int argc, char *argv[]) { envname = argv[optind]; envname = argv[optind]; if (!quiet) { - printf("mdbx_stat %s (%s, T-%s)\nRunning for %s...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, + printf("mdbx_stat %s (%s, T-%s)\nRunning for %s...\n", mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_version.git.tree, envname); fflush(nullptr); } @@ -232,39 +224,27 @@ int main(int argc, char *argv[]) { if (pgop) { printf("Page Operations (for current session):\n"); - printf(" New: %8" PRIu64 "\t// quantity of a new pages added\n", - mei.mi_pgop_stat.newly); - printf(" CoW: %8" PRIu64 - "\t// quantity of pages copied for altering\n", - mei.mi_pgop_stat.cow); + printf(" New: %8" PRIu64 "\t// quantity of a new pages added\n", mei.mi_pgop_stat.newly); + printf(" CoW: %8" PRIu64 "\t// quantity of pages copied for altering\n", mei.mi_pgop_stat.cow); printf(" Clone: %8" PRIu64 "\t// quantity of parent's dirty pages " "clones for nested transactions\n", mei.mi_pgop_stat.clone); - printf(" Split: %8" PRIu64 - "\t// page splits during insertions or updates\n", - mei.mi_pgop_stat.split); - printf(" Merge: %8" PRIu64 - "\t// page merges during deletions or updates\n", - mei.mi_pgop_stat.merge); + printf(" Split: %8" PRIu64 "\t// page splits during insertions or updates\n", mei.mi_pgop_stat.split); + printf(" Merge: %8" PRIu64 "\t// page merges during deletions or updates\n", mei.mi_pgop_stat.merge); printf(" Spill: %8" PRIu64 "\t// quantity of spilled/ousted `dirty` " "pages during large transactions\n", mei.mi_pgop_stat.spill); printf(" Unspill: %8" PRIu64 "\t// quantity of unspilled/redone `dirty` " "pages during large transactions\n", mei.mi_pgop_stat.unspill); - printf(" WOP: %8" PRIu64 - "\t// number of explicit write operations (not a pages) to a disk\n", + printf(" WOP: %8" PRIu64 "\t// number of explicit write operations (not a pages) to a disk\n", mei.mi_pgop_stat.wops); - printf(" PreFault: %8" PRIu64 - "\t// number of prefault write operations (not a pages)\n", + printf(" PreFault: %8" PRIu64 "\t// number of prefault write operations (not a pages)\n", mei.mi_pgop_stat.prefault); - printf(" mInCore: %8" PRIu64 "\t// number of mincore() calls\n", - mei.mi_pgop_stat.mincore); - printf(" mSync: %8" PRIu64 - "\t// number of explicit msync-to-disk operations (not a pages)\n", + printf(" mInCore: %8" PRIu64 "\t// number of mincore() calls\n", mei.mi_pgop_stat.mincore); + printf(" mSync: %8" PRIu64 "\t// number of explicit msync-to-disk operations (not a pages)\n", mei.mi_pgop_stat.msync); - printf(" fSync: %8" PRIu64 - "\t// number of explicit fsync-to-disk operations (not a pages)\n", + printf(" fSync: %8" PRIu64 "\t// number of explicit fsync-to-disk operations (not a pages)\n", mei.mi_pgop_stat.fsync); } @@ -272,18 +252,15 @@ int main(int argc, char *argv[]) { printf("Environment Info\n"); printf(" Pagesize: %u\n", mei.mi_dxb_pagesize); if (mei.mi_geo.lower != mei.mi_geo.upper) { - printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64 - "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64 - "/-%" PRIu64 ")\n", - mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow, - mei.mi_geo.shrink, mei.mi_geo.lower / mei.mi_dxb_pagesize, - mei.mi_geo.upper / mei.mi_dxb_pagesize, - mei.mi_geo.grow / mei.mi_dxb_pagesize, - mei.mi_geo.shrink / mei.mi_dxb_pagesize); - printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n", - mei.mi_mapsize, mei.mi_mapsize / mei.mi_dxb_pagesize); - printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", - mei.mi_geo.current, mei.mi_geo.current / mei.mi_dxb_pagesize); + printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64 "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 + " pages (+%" PRIu64 "/-%" PRIu64 ")\n", + mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow, mei.mi_geo.shrink, + mei.mi_geo.lower / mei.mi_dxb_pagesize, mei.mi_geo.upper / mei.mi_dxb_pagesize, + mei.mi_geo.grow / mei.mi_dxb_pagesize, mei.mi_geo.shrink / mei.mi_dxb_pagesize); + printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n", mei.mi_mapsize, + mei.mi_mapsize / mei.mi_dxb_pagesize); + printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", mei.mi_geo.current, + mei.mi_geo.current / mei.mi_dxb_pagesize); #if defined(_WIN32) || defined(_WIN64) if (mei.mi_geo.shrink && mei.mi_geo.current != mei.mi_geo.upper) printf(" WARNING: Due Windows system limitations a " @@ -293,12 +270,11 @@ int main(int argc, char *argv[]) { "until it will be closed or reopened in read-write mode.\n"); #endif } else { - printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", - mei.mi_geo.current, mei.mi_geo.current / mei.mi_dxb_pagesize); + printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", mei.mi_geo.current, + mei.mi_geo.current / mei.mi_dxb_pagesize); } printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid); - printf(" Latter reader transaction ID: %" PRIu64 " (%" PRIi64 ")\n", - mei.mi_latter_reader_txnid, + printf(" Latter reader transaction ID: %" PRIu64 " (%" PRIi64 ")\n", mei.mi_latter_reader_txnid, mei.mi_latter_reader_txnid - mei.mi_recent_txnid); printf(" Max readers: %u\n", mei.mi_maxreaders); printf(" Number of reader slots uses: %u\n", mei.mi_numreaders); @@ -352,8 +328,7 @@ int main(int argc, char *argv[]) { pgno_t pages = 0, *iptr; pgno_t reclaimable = 0; MDBX_val key, data; - while (MDBX_SUCCESS == - (rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT))) { + while (MDBX_SUCCESS == (rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT))) { if (user_break) { rc = MDBX_EINTR; break; @@ -367,29 +342,23 @@ int main(int argc, char *argv[]) { if (freinfo > 1) { char *bad = ""; - pgno_t prev = - MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)mei.mi_last_pgno + 1; + pgno_t prev = MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)mei.mi_last_pgno + 1; pgno_t span = 1; for (unsigned i = 0; i < number; ++i) { pgno_t pg = iptr[i]; if (MDBX_PNL_DISORDERED(prev, pg)) bad = " [bad sequence]"; prev = pg; - while (i + span < number && - iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) - : pgno_sub(pg, span))) + while (i + span < number && iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) : pgno_sub(pg, span))) ++span; } - printf(" Transaction %" PRIaTXN ", %" PRIaPGNO - " pages, maxspan %" PRIaPGNO "%s\n", - *(txnid_t *)key.iov_base, number, span, bad); + printf(" Transaction %" PRIaTXN ", %" PRIaPGNO " pages, maxspan %" PRIaPGNO "%s\n", *(txnid_t *)key.iov_base, + number, span, bad); if (freinfo > 2) { for (unsigned i = 0; i < number; i += span) { const pgno_t pg = iptr[i]; for (span = 1; - i + span < number && - iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) - : pgno_sub(pg, span)); + i + span < number && iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) : pgno_sub(pg, span)); ++span) ; if (span > 1) @@ -443,8 +412,7 @@ int main(int argc, char *argv[]) { value = reclaimable; printf(" Reclaimable: %" PRIu64 " %.1f%%\n", value, value / percent); - value = mei.mi_mapsize / mei.mi_dxb_pagesize - (mei.mi_last_pgno + 1) + - reclaimable; + value = mei.mi_mapsize / mei.mi_dxb_pagesize - (mei.mi_last_pgno + 1) + reclaimable; printf(" Available: %" PRIu64 " %.1f%%\n", value, value / percent); } else printf(" GC: %" PRIaPGNO " pages\n", pages); @@ -474,8 +442,7 @@ int main(int argc, char *argv[]) { } MDBX_val key; - while (MDBX_SUCCESS == - (rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) { + while (MDBX_SUCCESS == (rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) { MDBX_dbi xdbi; if (memchr(key.iov_base, '\0', key.iov_len)) continue; diff --git a/src/tools/wingetopt.c b/src/tools/wingetopt.c index 4f27d648..96210cc6 100644 --- a/src/tools/wingetopt.c +++ b/src/tools/wingetopt.c @@ -11,12 +11,12 @@ #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #if !defined(_CRT_SECURE_NO_WARNINGS) #define _CRT_SECURE_NO_WARNINGS @@ -70,8 +70,7 @@ int getopt(int argc, char *const argv[], const char *opts) { if (argv[optind][sp + 1] != '\0') optarg = &argv[optind++][sp + 1]; else if (++optind >= argc) { - fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", - c); + fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", c); sp = 1; return '?'; } else diff --git a/src/tree.c b/src/tree.c index 0f430749..67b69ddc 100644 --- a/src/tree.c +++ b/src/tree.c @@ -5,8 +5,7 @@ #include "internals.h" -static MDBX_cursor *cursor_clone(const MDBX_cursor *csrc, - cursor_couple_t *couple) { +static MDBX_cursor *cursor_clone(const MDBX_cursor *csrc, cursor_couple_t *couple) { cASSERT(csrc, csrc->txn->txnid >= csrc->txn->env->lck->cached_oldest.weak); couple->outer.next = nullptr; couple->outer.backup = nullptr; @@ -40,13 +39,10 @@ static MDBX_cursor *cursor_clone(const MDBX_cursor *csrc, void recalculate_merge_thresholds(MDBX_env *env) { const size_t bytes = page_space(env); - env->merge_threshold = - (uint16_t)(bytes - - (bytes * env->options.merge_threshold_16dot16_percent >> 16)); + env->merge_threshold = (uint16_t)(bytes - (bytes * env->options.merge_threshold_16dot16_percent >> 16)); env->merge_threshold_gc = - (uint16_t)(bytes - ((env->options.merge_threshold_16dot16_percent > 19005) - ? bytes / 3 /* 33 % */ - : bytes / 4 /* 25 % */)); + (uint16_t)(bytes - ((env->options.merge_threshold_16dot16_percent > 19005) ? bytes / 3 /* 33 % */ + : bytes / 4 /* 25 % */)); } int tree_drop(MDBX_cursor *mc, const bool may_have_tables) { @@ -60,9 +56,8 @@ int tree_drop(MDBX_cursor *mc, const bool may_have_tables) { if (!(may_have_tables | mc->tree->large_pages)) cursor_pop(mc); - rc = pnl_need(&txn->tw.retired_pages, (size_t)mc->tree->branch_pages + - (size_t)mc->tree->leaf_pages + - (size_t)mc->tree->large_pages); + rc = pnl_need(&txn->tw.retired_pages, + (size_t)mc->tree->branch_pages + (size_t)mc->tree->leaf_pages + (size_t)mc->tree->large_pages); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; @@ -100,9 +95,7 @@ int tree_drop(MDBX_cursor *mc, const bool may_have_tables) { cASSERT(mc, mc->top + 1 < mc->tree->height); mc->checking |= z_retiring; const unsigned pagetype = (is_frozen(txn, mp) ? P_FROZEN : 0) + - ((mc->top + 2 == mc->tree->height) - ? (mc->checking & (P_LEAF | P_DUPFIX)) - : P_BRANCH); + ((mc->top + 2 == mc->tree->height) ? (mc->checking & (P_LEAF | P_DUPFIX)) : P_BRANCH); for (size_t i = 0; i < nkeys; i++) { node_t *node = page_node(mp, i); tASSERT(txn, (node_flags(node) & (N_BIG | N_TREE | N_DUP)) == 0); @@ -153,8 +146,7 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) { cASSERT(csrc, csrc->top == cdst->top); if (unlikely(page_type(psrc) != page_type(pdst))) { bailout: - ERROR("Wrong or mismatch pages's types (src %d, dst %d) to move node", - page_type(psrc), page_type(pdst)); + ERROR("Wrong or mismatch pages's types (src %d, dst %d) to move node", page_type(psrc), page_type(pdst)); csrc->txn->flags |= MDBX_TXN_ERROR; return MDBX_PROBLEM; } @@ -225,8 +217,7 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) { mn->top = top; mn->ki[mn->top] = 0; - const intptr_t delta = EVEN_CEIL(key.iov_len) - - EVEN_CEIL(node_ks(page_node(mn->pg[mn->top], 0))); + const intptr_t delta = EVEN_CEIL(key.iov_len) - EVEN_CEIL(node_ks(page_node(mn->pg[mn->top], 0))); const intptr_t needed = branch_size(cdst->txn->env, &key4move) + delta; const intptr_t have = page_room(pdst); if (unlikely(needed > have)) @@ -255,10 +246,8 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) { pdst = cdst->pg[cdst->top]; } - DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO - " to node %u on page %" PRIaPGNO, - "branch", csrc->ki[csrc->top], DKEY_DEBUG(&key4move), psrc->pgno, - cdst->ki[cdst->top], pdst->pgno); + DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO " to node %u on page %" PRIaPGNO, "branch", csrc->ki[csrc->top], + DKEY_DEBUG(&key4move), psrc->pgno, cdst->ki[cdst->top], pdst->pgno); /* Add the node to the destination page. */ rc = node_add_branch(cdst, cdst->ki[cdst->top], &key4move, srcpg); } break; @@ -275,13 +264,10 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) { data.iov_base = node_data(srcnode); key4move.iov_len = node_ks(srcnode); key4move.iov_base = node_key(srcnode); - DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO - " to node %u on page %" PRIaPGNO, - "leaf", csrc->ki[csrc->top], DKEY_DEBUG(&key4move), psrc->pgno, - cdst->ki[cdst->top], pdst->pgno); + DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO " to node %u on page %" PRIaPGNO, "leaf", csrc->ki[csrc->top], + DKEY_DEBUG(&key4move), psrc->pgno, cdst->ki[cdst->top], pdst->pgno); /* Add the node to the destination page. */ - rc = node_add_leaf(cdst, cdst->ki[cdst->top], &key4move, &data, - node_flags(srcnode)); + rc = node_add_leaf(cdst, cdst->ki[cdst->top], &key4move, &data, node_flags(srcnode)); } break; case P_LEAF | P_DUPFIX: { @@ -290,12 +276,9 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) { return rc; psrc = csrc->pg[csrc->top]; pdst = cdst->pg[cdst->top]; - key4move = - page_dupfix_key(psrc, csrc->ki[csrc->top], csrc->tree->dupfix_size); - DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO - " to node %u on page %" PRIaPGNO, - "leaf2", csrc->ki[csrc->top], DKEY_DEBUG(&key4move), psrc->pgno, - cdst->ki[cdst->top], pdst->pgno); + key4move = page_dupfix_key(psrc, csrc->ki[csrc->top], csrc->tree->dupfix_size); + DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO " to node %u on page %" PRIaPGNO, "leaf2", csrc->ki[csrc->top], + DKEY_DEBUG(&key4move), psrc->pgno, cdst->ki[cdst->top], pdst->pgno); /* Add the node to the destination page. */ rc = node_add_dupfix(cdst, cdst->ki[cdst->top], &key4move); } break; @@ -329,13 +312,11 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) { if (!is_related(csrc, m3)) continue; - if (m3 != cdst && m3->pg[csrc->top] == pdst && - m3->ki[csrc->top] >= cdst->ki[csrc->top]) { + if (m3 != cdst && m3->pg[csrc->top] == pdst && m3->ki[csrc->top] >= cdst->ki[csrc->top]) { m3->ki[csrc->top] += 1; } - if (/* m3 != csrc && */ m3->pg[csrc->top] == psrc && - m3->ki[csrc->top] == csrc->ki[csrc->top]) { + if (/* m3 != csrc && */ m3->pg[csrc->top] == psrc && m3->ki[csrc->top] == csrc->ki[csrc->top]) { m3->pg[csrc->top] = pdst; m3->ki[csrc->top] = cdst->ki[cdst->top]; cASSERT(csrc, csrc->top > 0); @@ -387,8 +368,7 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) { key.iov_len = node_ks(srcnode); key.iov_base = node_key(srcnode); } - DEBUG("update separator for source page %" PRIaPGNO " to [%s]", - psrc->pgno, DKEY_DEBUG(&key)); + DEBUG("update separator for source page %" PRIaPGNO " to [%s]", psrc->pgno, DKEY_DEBUG(&key)); cursor_couple_t couple; MDBX_cursor *const mn = cursor_clone(csrc, &couple); @@ -423,8 +403,7 @@ static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) { key.iov_len = node_ks(srcnode); key.iov_base = node_key(srcnode); } - DEBUG("update separator for destination page %" PRIaPGNO " to [%s]", - pdst->pgno, DKEY_DEBUG(&key)); + DEBUG("update separator for destination page %" PRIaPGNO " to [%s]", pdst->pgno, DKEY_DEBUG(&key)); cursor_couple_t couple; MDBX_cursor *const mn = cursor_clone(cdst, &couple); cASSERT(cdst, mn->top > 0); @@ -465,12 +444,10 @@ static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) { cASSERT(csrc, csrc->clc == cdst->clc && csrc->tree == cdst->tree); cASSERT(csrc, csrc->top > 0); /* can't merge root page */ cASSERT(cdst, cdst->top > 0); - cASSERT(cdst, cdst->top + 1 < cdst->tree->height || - is_leaf(cdst->pg[cdst->tree->height - 1])); - cASSERT(csrc, csrc->top + 1 < csrc->tree->height || - is_leaf(csrc->pg[csrc->tree->height - 1])); - cASSERT(cdst, csrc->txn->env->options.prefer_waf_insteadof_balance || - page_room(pdst) >= page_used(cdst->txn->env, psrc)); + cASSERT(cdst, cdst->top + 1 < cdst->tree->height || is_leaf(cdst->pg[cdst->tree->height - 1])); + cASSERT(csrc, csrc->top + 1 < csrc->tree->height || is_leaf(csrc->pg[csrc->tree->height - 1])); + cASSERT(cdst, + csrc->txn->env->options.prefer_waf_insteadof_balance || page_room(pdst) >= page_used(cdst->txn->env, psrc)); const int pagetype = page_type(psrc); /* Move all nodes from src to dst */ @@ -560,10 +537,8 @@ static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) { } pdst = cdst->pg[cdst->top]; - DEBUG("dst page %" PRIaPGNO " now has %zu keys (%u.%u%% filled)", - pdst->pgno, page_numkeys(pdst), - page_fill_percentum_x10(cdst->txn->env, pdst) / 10, - page_fill_percentum_x10(cdst->txn->env, pdst) % 10); + DEBUG("dst page %" PRIaPGNO " now has %zu keys (%u.%u%% filled)", pdst->pgno, page_numkeys(pdst), + page_fill_percentum_x10(cdst->txn->env, pdst) / 10, page_fill_percentum_x10(cdst->txn->env, pdst) % 10); cASSERT(csrc, psrc == csrc->pg[csrc->top]); cASSERT(cdst, pdst == cdst->pg[cdst->top]); @@ -598,11 +573,8 @@ static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) { m3->pg[csrc->top] = pdst; m3->ki[csrc->top] += (indx_t)dst_nkeys; m3->ki[csrc->top - 1] = cdst->ki[csrc->top - 1]; - } else if (m3->pg[csrc->top - 1] == csrc->pg[csrc->top - 1] && - m3->ki[csrc->top - 1] > csrc->ki[csrc->top - 1]) { - cASSERT(m3, m3->ki[csrc->top - 1] > 0 && - m3->ki[csrc->top - 1] <= - page_numkeys(m3->pg[csrc->top - 1])); + } else if (m3->pg[csrc->top - 1] == csrc->pg[csrc->top - 1] && m3->ki[csrc->top - 1] > csrc->ki[csrc->top - 1]) { + cASSERT(m3, m3->ki[csrc->top - 1] > 0 && m3->ki[csrc->top - 1] <= page_numkeys(m3->pg[csrc->top - 1])); m3->ki[csrc->top - 1] -= 1; } @@ -641,8 +613,7 @@ static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) { if (is_leaf(cdst->pg[cdst->top])) { /* LY: don't touch cursor if top-page is a LEAF */ - cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || - page_type(cdst->pg[cdst->top]) == pagetype); + cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype); return MDBX_SUCCESS; } @@ -656,8 +627,7 @@ static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) { if (top_page == cdst->pg[cdst->top]) { /* LY: don't touch cursor if prev top-page already on the top */ cASSERT(cdst, cdst->ki[cdst->top] == top_indx); - cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || - page_type(cdst->pg[cdst->top]) == pagetype); + cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype); return MDBX_SUCCESS; } @@ -671,18 +641,15 @@ static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) { cASSERT(cdst, cdst->ki[new_top] == top_indx); /* LY: restore cursor stack */ cdst->top = (int8_t)new_top; - cASSERT(cdst, cdst->top + 1 < cdst->tree->height || - is_leaf(cdst->pg[cdst->tree->height - 1])); - cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || - page_type(cdst->pg[cdst->top]) == pagetype); + cASSERT(cdst, cdst->top + 1 < cdst->tree->height || is_leaf(cdst->pg[cdst->tree->height - 1])); + cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype); return MDBX_SUCCESS; } page_t *const stub_page = (page_t *)(~(uintptr_t)top_page); const indx_t stub_indx = top_indx; - if (save_height > cdst->tree->height && - ((cdst->pg[save_top] == top_page && cdst->ki[save_top] == top_indx) || - (cdst->pg[save_top] == stub_page && cdst->ki[save_top] == stub_indx))) { + if (save_height > cdst->tree->height && ((cdst->pg[save_top] == top_page && cdst->ki[save_top] == top_indx) || + (cdst->pg[save_top] == stub_page && cdst->ki[save_top] == stub_indx))) { /* LY: restore cursor stack */ cdst->pg[new_top] = top_page; cdst->ki[new_top] = top_indx; @@ -691,10 +658,8 @@ static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) { cdst->ki[new_top + 1] = INT16_MAX; #endif cdst->top = (int8_t)new_top; - cASSERT(cdst, cdst->top + 1 < cdst->tree->height || - is_leaf(cdst->pg[cdst->tree->height - 1])); - cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || - page_type(cdst->pg[cdst->top]) == pagetype); + cASSERT(cdst, cdst->top + 1 < cdst->tree->height || is_leaf(cdst->pg[cdst->tree->height - 1])); + cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype); return MDBX_SUCCESS; } @@ -707,8 +672,7 @@ bailout: int tree_rebalance(MDBX_cursor *mc) { cASSERT(mc, cursor_is_tracked(mc)); cASSERT(mc, mc->top >= 0); - cASSERT(mc, mc->top + 1 < mc->tree->height || - is_leaf(mc->pg[mc->tree->height - 1])); + cASSERT(mc, mc->top + 1 < mc->tree->height || is_leaf(mc->pg[mc->tree->height - 1])); const page_t *const tp = mc->pg[mc->top]; const uint8_t pagetype = page_type(tp); @@ -716,29 +680,22 @@ int tree_rebalance(MDBX_cursor *mc) { const size_t minkeys = (pagetype & P_BRANCH) + (size_t)1; /* Pages emptier than this are candidates for merging. */ - size_t room_threshold = likely(mc->tree != &mc->txn->dbs[FREE_DBI]) - ? mc->txn->env->merge_threshold - : mc->txn->env->merge_threshold_gc; + size_t room_threshold = + likely(mc->tree != &mc->txn->dbs[FREE_DBI]) ? mc->txn->env->merge_threshold : mc->txn->env->merge_threshold_gc; const size_t numkeys = page_numkeys(tp); const size_t room = page_room(tp); - DEBUG("rebalancing %s page %" PRIaPGNO - " (has %zu keys, fill %u.%u%%, used %zu, room %zu bytes)", - is_leaf(tp) ? "leaf" : "branch", tp->pgno, numkeys, - page_fill_percentum_x10(mc->txn->env, tp) / 10, - page_fill_percentum_x10(mc->txn->env, tp) % 10, - page_used(mc->txn->env, tp), room); + DEBUG("rebalancing %s page %" PRIaPGNO " (has %zu keys, fill %u.%u%%, used %zu, room %zu bytes)", + is_leaf(tp) ? "leaf" : "branch", tp->pgno, numkeys, page_fill_percentum_x10(mc->txn->env, tp) / 10, + page_fill_percentum_x10(mc->txn->env, tp) % 10, page_used(mc->txn->env, tp), room); cASSERT(mc, is_modifable(mc->txn, tp)); if (unlikely(numkeys < minkeys)) { - DEBUG("page %" PRIaPGNO " must be merged due keys < %zu threshold", - tp->pgno, minkeys); + DEBUG("page %" PRIaPGNO " must be merged due keys < %zu threshold", tp->pgno, minkeys); } else if (unlikely(room > room_threshold)) { - DEBUG("page %" PRIaPGNO " should be merged due room %zu > %zu threshold", - tp->pgno, room, room_threshold); + DEBUG("page %" PRIaPGNO " should be merged due room %zu > %zu threshold", tp->pgno, room, room_threshold); } else { - DEBUG("no need to rebalance page %" PRIaPGNO ", room %zu < %zu threshold", - tp->pgno, room, room_threshold); + DEBUG("no need to rebalance page %" PRIaPGNO ", room %zu < %zu threshold", tp->pgno, room, room_threshold); cASSERT(mc, mc->tree->items > 0); return MDBX_SUCCESS; } @@ -752,11 +709,9 @@ int tree_rebalance(MDBX_cursor *mc) { DEBUG("%s", "tree is completely empty"); cASSERT(mc, is_leaf(mp)); cASSERT(mc, (*cursor_dbi_state(mc) & DBI_DIRTY) != 0); - cASSERT(mc, mc->tree->branch_pages == 0 && mc->tree->large_pages == 0 && - mc->tree->leaf_pages == 1); + cASSERT(mc, mc->tree->branch_pages == 0 && mc->tree->large_pages == 0 && mc->tree->leaf_pages == 1); /* Adjust cursors pointing to mp */ - for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; - m2 = m2->next) { + for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) { MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2; if (!is_poor(m3) && m3->pg[0] == mp) { be_poor(m3); @@ -790,8 +745,7 @@ int tree_rebalance(MDBX_cursor *mc) { } /* Adjust other cursors pointing to mp */ - for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; - m2 = m2->next) { + for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) { MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2; if (is_related(mc, m3) && m3->pg[0] == mp) { for (intptr_t i = 0; i < mc->tree->height; i++) { @@ -801,14 +755,11 @@ int tree_rebalance(MDBX_cursor *mc) { m3->top -= 1; } } - cASSERT(mc, is_leaf(mc->pg[mc->top]) || - page_type(mc->pg[mc->top]) == pagetype); - cASSERT(mc, mc->top + 1 < mc->tree->height || - is_leaf(mc->pg[mc->tree->height - 1])); + cASSERT(mc, is_leaf(mc->pg[mc->top]) || page_type(mc->pg[mc->top]) == pagetype); + cASSERT(mc, mc->top + 1 < mc->tree->height || is_leaf(mc->pg[mc->tree->height - 1])); return page_retire(mc, mp); } - DEBUG("root page %" PRIaPGNO " doesn't need rebalancing (flags 0x%x)", - mp->pgno, mp->flags); + DEBUG("root page %" PRIaPGNO " doesn't need rebalancing (flags 0x%x)", mp->pgno, mp->flags); return MDBX_SUCCESS; } @@ -829,17 +780,14 @@ int tree_rebalance(MDBX_cursor *mc) { page_t *left = nullptr, *right = nullptr; if (mn->ki[pre_top] > 0) { - rc = - page_get(mn, node_pgno(page_node(mn->pg[pre_top], mn->ki[pre_top] - 1)), - &left, mc->pg[mc->top]->txnid); + rc = page_get(mn, node_pgno(page_node(mn->pg[pre_top], mn->ki[pre_top] - 1)), &left, mc->pg[mc->top]->txnid); if (unlikely(rc != MDBX_SUCCESS)) return rc; cASSERT(mc, page_type(left) == page_type(mc->pg[mc->top])); } if (mn->ki[pre_top] + (size_t)1 < page_numkeys(mn->pg[pre_top])) { - rc = page_get( - mn, node_pgno(page_node(mn->pg[pre_top], mn->ki[pre_top] + (size_t)1)), - &right, mc->pg[mc->top]->txnid); + rc = page_get(mn, node_pgno(page_node(mn->pg[pre_top], mn->ki[pre_top] + (size_t)1)), &right, + mc->pg[mc->top]->txnid); if (unlikely(rc != MDBX_SUCCESS)) return rc; cASSERT(mc, page_type(right) == page_type(mc->pg[mc->top])); @@ -857,8 +805,7 @@ int tree_rebalance(MDBX_cursor *mc) { bool involve = !(left && right); retry: cASSERT(mc, mc->top > 0); - if (left_room > room_threshold && left_room >= right_room && - (is_modifable(mc->txn, left) || involve)) { + if (left_room > room_threshold && left_room >= right_room && (is_modifable(mc->txn, left) || involve)) { /* try merge with left */ cASSERT(mc, left_nkeys >= minkeys); mn->pg[mn->top] = left; @@ -878,8 +825,7 @@ retry: return rc; } } - if (right_room > room_threshold && - (is_modifable(mc->txn, right) || involve)) { + if (right_room > room_threshold && (is_modifable(mc->txn, right) || involve)) { /* try merge with right */ cASSERT(mc, right_nkeys >= minkeys); mn->pg[mn->top] = right; @@ -897,8 +843,7 @@ retry: } } - if (left_nkeys > minkeys && - (right_nkeys <= left_nkeys || right_room >= left_room) && + if (left_nkeys > minkeys && (right_nkeys <= left_nkeys || right_room >= left_room) && (is_modifable(mc->txn, left) || involve)) { /* try move from left */ mn->pg[mn->top] = left; @@ -939,16 +884,13 @@ retry: return MDBX_SUCCESS; } - if (mc->txn->env->options.prefer_waf_insteadof_balance && - likely(room_threshold > 0)) { + if (mc->txn->env->options.prefer_waf_insteadof_balance && likely(room_threshold > 0)) { room_threshold = 0; goto retry; } if (likely(!involve) && - (likely(mc->tree != &mc->txn->dbs[FREE_DBI]) || mc->txn->tw.loose_pages || - MDBX_PNL_GETSIZE(mc->txn->tw.relist) || - (mc->flags & z_gcu_preparation) || (mc->txn->flags & txn_gc_drained) || - room_threshold)) { + (likely(mc->tree != &mc->txn->dbs[FREE_DBI]) || mc->txn->tw.loose_pages || MDBX_PNL_GETSIZE(mc->txn->tw.relist) || + (mc->flags & z_gcu_preparation) || (mc->txn->flags & txn_gc_drained) || room_threshold)) { involve = true; goto retry; } @@ -957,17 +899,14 @@ retry: goto retry; } - ERROR("Unable to merge/rebalance %s page %" PRIaPGNO - " (has %zu keys, fill %u.%u%%, used %zu, room %zu bytes)", - is_leaf(tp) ? "leaf" : "branch", tp->pgno, numkeys, - page_fill_percentum_x10(mc->txn->env, tp) / 10, - page_fill_percentum_x10(mc->txn->env, tp) % 10, - page_used(mc->txn->env, tp), room); + ERROR("Unable to merge/rebalance %s page %" PRIaPGNO " (has %zu keys, fill %u.%u%%, used %zu, room %zu bytes)", + is_leaf(tp) ? "leaf" : "branch", tp->pgno, numkeys, page_fill_percentum_x10(mc->txn->env, tp) / 10, + page_fill_percentum_x10(mc->txn->env, tp) % 10, page_used(mc->txn->env, tp), room); return MDBX_PROBLEM; } -int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, - MDBX_val *const newdata, pgno_t newpgno, const unsigned naf) { +int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, MDBX_val *const newdata, pgno_t newpgno, + const unsigned naf) { unsigned flags; int rc = MDBX_SUCCESS, foliage = 0; MDBX_env *const env = mc->txn->env; @@ -988,11 +927,8 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, STATIC_ASSERT(P_BRANCH == 1); const size_t minkeys = (mp->flags & P_BRANCH) + (size_t)1; - DEBUG(">> splitting %s-page %" PRIaPGNO - " and adding %zu+%zu [%s] at %i, nkeys %zi", - is_leaf(mp) ? "leaf" : "branch", mp->pgno, newkey->iov_len, - newdata ? newdata->iov_len : 0, DKEY_DEBUG(newkey), mc->ki[mc->top], - nkeys); + DEBUG(">> splitting %s-page %" PRIaPGNO " and adding %zu+%zu [%s] at %i, nkeys %zi", is_leaf(mp) ? "leaf" : "branch", + mp->pgno, newkey->iov_len, newdata ? newdata->iov_len : 0, DKEY_DEBUG(newkey), mc->ki[mc->top], nkeys); cASSERT(mc, nkeys + 1 >= minkeys * 2); /* Create a new sibling page. */ @@ -1057,10 +993,8 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, mn->ki[mn->top] = 0; mn->ki[prev_top] = mc->ki[prev_top] + 1; - size_t split_indx = - (newindx < nkeys) - ? /* split at the middle */ (nkeys + 1) >> 1 - : /* split at the end (i.e. like append-mode ) */ nkeys - minkeys + 1; + size_t split_indx = (newindx < nkeys) ? /* split at the middle */ (nkeys + 1) >> 1 + : /* split at the end (i.e. like append-mode ) */ nkeys - minkeys + 1; eASSERT(env, split_indx >= minkeys && split_indx <= nkeys - minkeys + 1); cASSERT(mc, !is_branch(mp) || newindx > 0); @@ -1094,11 +1028,9 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, if (foliage) { TRACE("pure-left: foliage %u, top %i, ptop %zu, split_indx %zi, " "minkeys %zi, sepkey %s, parent-room %zu, need4split %zu", - foliage, mc->top, prev_top, split_indx, minkeys, - DKEY_DEBUG(&sepkey), page_room(mc->pg[prev_top]), + foliage, mc->top, prev_top, split_indx, minkeys, DKEY_DEBUG(&sepkey), page_room(mc->pg[prev_top]), branch_size(env, &sepkey)); - TRACE("pure-left: newkey %s, newdata %s, newindx %zu", - DKEY_DEBUG(newkey), DVAL_DEBUG(newdata), newindx); + TRACE("pure-left: newkey %s, newdata %s, newindx %zu", DKEY_DEBUG(newkey), DVAL_DEBUG(newdata), newindx); } } } @@ -1112,8 +1044,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, sepkey = *newkey; } else if (unlikely(pure_left)) { /* newindx == split_indx == 0 */ - TRACE("pure-left: no-split, but add new pure page at the %s", - "left/before"); + TRACE("pure-left: no-split, but add new pure page at the %s", "left/before"); cASSERT(mc, newindx == 0 && split_indx == 0 && minkeys == 1); TRACE("pure-left: old-first-key is %s", DKEY_DEBUG(&sepkey)); } else { @@ -1139,8 +1070,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, void *const ins = page_dupfix_ptr(mp, mc->ki[mc->top], ksize); memcpy(sister->entries, split, rsize); sepkey.iov_base = sister->entries; - memmove(ptr_disp(ins, ksize), ins, - (split_indx - mc->ki[mc->top]) * ksize); + memmove(ptr_disp(ins, ksize), ins, (split_indx - mc->ki[mc->top]) * ksize); memcpy(ins, newkey->iov_base, ksize); cASSERT(mc, UINT16_MAX - mp->lower >= (int)sizeof(indx_t)); mp->lower += sizeof(indx_t); @@ -1151,16 +1081,14 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, memcpy(sister->entries, split, distance * ksize); void *const ins = page_dupfix_ptr(sister, distance, ksize); memcpy(ins, newkey->iov_base, ksize); - memcpy(ptr_disp(ins, ksize), ptr_disp(split, distance * ksize), - rsize - distance * ksize); + memcpy(ptr_disp(ins, ksize), ptr_disp(split, distance * ksize), rsize - distance * ksize); cASSERT(mc, UINT16_MAX - sister->lower >= (int)sizeof(indx_t)); sister->lower += sizeof(indx_t); cASSERT(mc, sister->upper >= ksize - sizeof(indx_t)); sister->upper -= (indx_t)(ksize - sizeof(indx_t)); cASSERT(mc, distance <= (int)UINT16_MAX); mc->ki[mc->top] = (indx_t)distance; - cASSERT(mc, - (((ksize & page_numkeys(sister)) ^ sister->upper) & 1) == 0); + cASSERT(mc, (((ksize & page_numkeys(sister)) ^ sister->upper) & 1) == 0); } if (AUDIT_ENABLED()) { @@ -1180,8 +1108,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, } const size_t max_space = page_space(env); - const size_t new_size = is_leaf(mp) ? leaf_size(env, newkey, newdata) - : branch_size(env, newkey); + const size_t new_size = is_leaf(mp) ? leaf_size(env, newkey, newdata) : branch_size(env, newkey); /* prepare to insert */ size_t i = 0; @@ -1218,8 +1145,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, split_indx += mp->flags & P_BRANCH; } eASSERT(env, split_indx >= minkeys && split_indx <= nkeys + 1 - minkeys); - const size_t dim_nodes = - (newindx >= split_indx) ? split_indx : nkeys - split_indx; + const size_t dim_nodes = (newindx >= split_indx) ? split_indx : nkeys - split_indx; const size_t dim_used = (sizeof(indx_t) + NODESIZE + 1) * dim_nodes; if (new_size >= dim_used) { /* Search for best acceptable split point */ @@ -1239,15 +1165,13 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, node_t *node = ptr_disp(mp, tmp_ki_copy->entries[i] + PAGEHDRSZ); size = NODESIZE + node_ks(node) + sizeof(indx_t); if (is_leaf(mp)) - size += - (node_flags(node) & N_BIG) ? sizeof(pgno_t) : node_ds(node); + size += (node_flags(node) & N_BIG) ? sizeof(pgno_t) : node_ds(node); size = EVEN_CEIL(size); } before += size; after -= size; - TRACE("step %zu, size %zu, before %zu, after %zu, max %zu", i, size, - before, after, max_space); + TRACE("step %zu, size %zu, before %zu, after %zu, max %zu", i, size, before, after, max_space); if (before <= max_space && after <= max_space) { const size_t split = i + (dir > 0); @@ -1271,8 +1195,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, sepkey = *newkey; if (split_indx != newindx) { - node_t *node = - ptr_disp(mp, tmp_ki_copy->entries[split_indx] + PAGEHDRSZ); + node_t *node = ptr_disp(mp, tmp_ki_copy->entries[split_indx] + PAGEHDRSZ); sepkey.iov_len = node_ks(node); sepkey.iov_base = node_key(node); } @@ -1308,8 +1231,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, /* Right page might now have changed parent. * Check if left page also changed parent. */ - if (mn->pg[prev_top] != mc->pg[prev_top] && - mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) { + if (mn->pg[prev_top] != mc->pg[prev_top] && mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) { for (intptr_t i = 0; i < prev_top; i++) { mc->pg[i] = mn->pg[i]; mc->ki[i] = mn->ki[i]; @@ -1334,14 +1256,11 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, page_t *ptop_page = mc->pg[prev_top]; TRACE("pure-left: adding to parent page %u node[%u] left-leaf page #%u key " "%s", - ptop_page->pgno, mc->ki[prev_top], sister->pgno, - DKEY(mc->ki[prev_top] ? newkey : nullptr)); + ptop_page->pgno, mc->ki[prev_top], sister->pgno, DKEY(mc->ki[prev_top] ? newkey : nullptr)); assert(mc->top == prev_top + 1); mc->top = (uint8_t)prev_top; - rc = node_add_branch(mc, mc->ki[prev_top], - mc->ki[prev_top] ? newkey : nullptr, sister->pgno); - cASSERT(mc, mp == mc->pg[prev_top + 1] && newindx == mc->ki[prev_top + 1] && - prev_top == mc->top); + rc = node_add_branch(mc, mc->ki[prev_top], mc->ki[prev_top] ? newkey : nullptr, sister->pgno); + cASSERT(mc, mp == mc->pg[prev_top + 1] && newindx == mc->ki[prev_top + 1] && prev_top == mc->top); if (likely(rc == MDBX_SUCCESS) && mc->ki[prev_top] == 0) { node_t *node = page_node(mc->pg[prev_top], 1); @@ -1351,12 +1270,10 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, mc->ki[prev_top] = 1; rc = tree_propagate_key(mc, &sepkey); cASSERT(mc, mc->top == prev_top && mc->ki[prev_top] == 1); - cASSERT(mc, - mp == mc->pg[prev_top + 1] && newindx == mc->ki[prev_top + 1]); + cASSERT(mc, mp == mc->pg[prev_top + 1] && newindx == mc->ki[prev_top + 1]); mc->ki[prev_top] = 0; } else { - TRACE("pure-left: no-need-update prev-first key on parent %s", - DKEY(&sepkey)); + TRACE("pure-left: no-need-update prev-first key on parent %s", DKEY(&sepkey)); } mc->top++; @@ -1367,8 +1284,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, cASSERT(mc, node_pgno(node) == mp->pgno && mc->pg[prev_top] == ptop_page); } else { mn->top -= 1; - TRACE("add-to-parent the right-entry[%u] for new sibling-page", - mn->ki[prev_top]); + TRACE("add-to-parent the right-entry[%u] for new sibling-page", mn->ki[prev_top]); rc = node_add_branch(mn, mn->ki[prev_top], &sepkey, sister->pgno); mn->top += 1; if (unlikely(rc != MDBX_SUCCESS)) @@ -1403,8 +1319,8 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, sepkey = get_key(page_node(mc->pg[mc->top - i], mc->ki[mc->top - i])); if (mc->clc->k.cmp(newkey, &sepkey) < 0) { mc->top -= (int8_t)i; - DEBUG("pure-left: update new-first on parent [%i] page %u key %s", - mc->ki[mc->top], mc->pg[mc->top]->pgno, DKEY(newkey)); + DEBUG("pure-left: update new-first on parent [%i] page %u key %s", mc->ki[mc->top], mc->pg[mc->top]->pgno, + DKEY(newkey)); rc = tree_propagate_key(mc, newkey); mc->top += (int8_t)i; if (unlikely(rc != MDBX_SUCCESS)) @@ -1474,16 +1390,14 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, } } while (ii != split_indx); - TRACE("ii %zu, nkeys %zu, n %zu, pgno #%u", ii, nkeys, n, - mc->pg[mc->top]->pgno); + TRACE("ii %zu, nkeys %zu, n %zu, pgno #%u", ii, nkeys, n, mc->pg[mc->top]->pgno); nkeys = page_numkeys(tmp_ki_copy); for (size_t i = 0; i < nkeys; i++) mp->entries[i] = tmp_ki_copy->entries[i]; mp->lower = tmp_ki_copy->lower; mp->upper = tmp_ki_copy->upper; - memcpy(page_node(mp, nkeys - 1), page_node(tmp_ki_copy, nkeys - 1), - env->ps - tmp_ki_copy->upper - PAGEHDRSZ); + memcpy(page_node(mp, nkeys - 1), page_node(tmp_ki_copy, nkeys - 1), env->ps - tmp_ki_copy->upper - PAGEHDRSZ); /* reset back to original page */ if (newindx < split_indx) { @@ -1492,8 +1406,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, mc->pg[mc->top] = sister; mc->ki[prev_top]++; /* Make sure ki is still valid. */ - if (mn->pg[prev_top] != mc->pg[prev_top] && - mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) { + if (mn->pg[prev_top] != mc->pg[prev_top] && mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) { for (intptr_t i = 0; i <= prev_top; i++) { mc->pg[i] = mn->pg[i]; mc->ki[i] = mn->ki[i]; @@ -1504,8 +1417,7 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, mc->pg[mc->top] = sister; mc->ki[prev_top]++; /* Make sure ki is still valid. */ - if (mn->pg[prev_top] != mc->pg[prev_top] && - mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) { + if (mn->pg[prev_top] != mc->pg[prev_top] && mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) { for (intptr_t i = 0; i <= prev_top; i++) { mc->pg[i] = mn->pg[i]; mc->ki[i] = mn->ki[i]; @@ -1545,16 +1457,14 @@ int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, m3->pg[i] = mn->pg[i]; } } - } else if (!did_split_parent && m3->top >= prev_top && - m3->pg[prev_top] == mc->pg[prev_top] && + } else if (!did_split_parent && m3->top >= prev_top && m3->pg[prev_top] == mc->pg[prev_top] && m3->ki[prev_top] >= mc->ki[prev_top]) { m3->ki[prev_top]++; /* also for the `pure-left` case */ } if (inner_pointed(m3) && is_leaf(mp)) cursor_inner_refresh(m3, m3->pg[mc->top], m3->ki[mc->top]); } - TRACE("mp #%u left: %zd, sister #%u left: %zd", mp->pgno, page_room(mp), - sister->pgno, page_room(sister)); + TRACE("mp #%u left: %zd, sister #%u left: %zd", mp->pgno, page_room(mp), sister->pgno, page_room(sister)); done: if (tmp_ki_copy) @@ -1596,8 +1506,8 @@ int tree_propagate_key(MDBX_cursor *mc, const MDBX_val *key) { MDBX_val k2; k2.iov_base = node_key(node); k2.iov_len = node_ks(node); - DEBUG("update key %zi (offset %zu) [%s] to [%s] on page %" PRIaPGNO, indx, - ptr, DVAL_DEBUG(&k2), DKEY_DEBUG(key), mp->pgno); + DEBUG("update key %zi (offset %zu) [%s] to [%s] on page %" PRIaPGNO, indx, ptr, DVAL_DEBUG(&k2), DKEY_DEBUG(key), + mp->pgno); #endif /* MDBX_DEBUG */ /* Sizes must be 2-byte aligned. */ diff --git a/src/txl.c b/src/txl.c index aca3758d..024b099f 100644 --- a/src/txl.c +++ b/src/txl.c @@ -6,8 +6,7 @@ static inline size_t txl_size2bytes(const size_t size) { assert(size > 0 && size <= txl_max * 2); size_t bytes = - ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(txnid_t) * (size + 2), - txl_granulate * sizeof(txnid_t)) - + ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(txnid_t) * (size + 2), txl_granulate * sizeof(txnid_t)) - MDBX_ASSUME_MALLOC_OVERHEAD; return bytes; } @@ -38,11 +37,9 @@ MDBX_INTERNAL void txl_free(txl_t txl) { osal_free(txl - 1); } -MDBX_INTERNAL int txl_reserve(txl_t __restrict *__restrict ptxl, - const size_t wanna) { +MDBX_INTERNAL int txl_reserve(txl_t __restrict *__restrict ptxl, const size_t wanna) { const size_t allocated = (size_t)MDBX_PNL_ALLOCLEN(*ptxl); - assert(MDBX_PNL_GETSIZE(*ptxl) <= txl_max && - MDBX_PNL_ALLOCLEN(*ptxl) >= MDBX_PNL_GETSIZE(*ptxl)); + assert(MDBX_PNL_GETSIZE(*ptxl) <= txl_max && MDBX_PNL_ALLOCLEN(*ptxl) >= MDBX_PNL_GETSIZE(*ptxl)); if (likely(allocated >= wanna)) return MDBX_SUCCESS; @@ -51,9 +48,7 @@ MDBX_INTERNAL int txl_reserve(txl_t __restrict *__restrict ptxl, return MDBX_TXN_FULL; } - const size_t size = (wanna + wanna - allocated < txl_max) - ? wanna + wanna - allocated - : txl_max; + const size_t size = (wanna + wanna - allocated < txl_max) ? wanna + wanna - allocated : txl_max; size_t bytes = txl_size2bytes(size); txl_t txl = osal_realloc(*ptxl - 1, bytes); if (likely(txl)) { @@ -68,14 +63,11 @@ MDBX_INTERNAL int txl_reserve(txl_t __restrict *__restrict ptxl, return MDBX_ENOMEM; } -static __always_inline int __must_check_result -txl_need(txl_t __restrict *__restrict ptxl, size_t num) { - assert(MDBX_PNL_GETSIZE(*ptxl) <= txl_max && - MDBX_PNL_ALLOCLEN(*ptxl) >= MDBX_PNL_GETSIZE(*ptxl)); +static __always_inline int __must_check_result txl_need(txl_t __restrict *__restrict ptxl, size_t num) { + assert(MDBX_PNL_GETSIZE(*ptxl) <= txl_max && MDBX_PNL_ALLOCLEN(*ptxl) >= MDBX_PNL_GETSIZE(*ptxl)); assert(num <= PAGELIST_LIMIT); const size_t wanna = (size_t)MDBX_PNL_GETSIZE(*ptxl) + num; - return likely(MDBX_PNL_ALLOCLEN(*ptxl) >= wanna) ? MDBX_SUCCESS - : txl_reserve(ptxl, wanna); + return likely(MDBX_PNL_ALLOCLEN(*ptxl) >= wanna) ? MDBX_SUCCESS : txl_reserve(ptxl, wanna); } static __always_inline void txl_xappend(txl_t __restrict txl, txnid_t id) { @@ -86,12 +78,9 @@ static __always_inline void txl_xappend(txl_t __restrict txl, txnid_t id) { #define TXNID_SORT_CMP(first, last) ((first) > (last)) SORT_IMPL(txnid_sort, false, txnid_t, TXNID_SORT_CMP) -MDBX_INTERNAL void txl_sort(txl_t txl) { - txnid_sort(MDBX_PNL_BEGIN(txl), MDBX_PNL_END(txl)); -} +MDBX_INTERNAL void txl_sort(txl_t txl) { txnid_sort(MDBX_PNL_BEGIN(txl), MDBX_PNL_END(txl)); } -MDBX_INTERNAL int __must_check_result txl_append(txl_t __restrict *ptxl, - txnid_t id) { +MDBX_INTERNAL int __must_check_result txl_append(txl_t __restrict *ptxl, txnid_t id) { if (unlikely(MDBX_PNL_GETSIZE(*ptxl) == MDBX_PNL_ALLOCLEN(*ptxl))) { int rc = txl_need(ptxl, txl_granulate); if (unlikely(rc != MDBX_SUCCESS)) diff --git a/src/txl.h b/src/txl.h index a17fbee6..e80db522 100644 --- a/src/txl.h +++ b/src/txl.h @@ -11,8 +11,7 @@ typedef const txnid_t *const_txl_t; enum txl_rules { txl_granulate = 32, - txl_initial = - txl_granulate - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t), + txl_initial = txl_granulate - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t), txl_max = (1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t) }; @@ -20,7 +19,6 @@ MDBX_INTERNAL txl_t txl_alloc(void); MDBX_INTERNAL void txl_free(txl_t txl); -MDBX_INTERNAL int __must_check_result txl_append(txl_t __restrict *ptxl, - txnid_t id); +MDBX_INTERNAL int __must_check_result txl_append(txl_t __restrict *ptxl, txnid_t id); MDBX_INTERNAL void txl_sort(txl_t txl); diff --git a/src/txn.c b/src/txn.c index f6ac98d5..166e7682 100644 --- a/src/txn.c +++ b/src/txn.c @@ -4,8 +4,7 @@ #include "internals.h" __hot txnid_t txn_snapshot_oldest(const MDBX_txn *const txn) { - return mvcc_shapshot_oldest( - txn->env, txn->tw.troika.txnid[txn->tw.troika.prefer_steady]); + return mvcc_shapshot_oldest(txn->env, txn->tw.troika.txnid[txn->tw.troika.prefer_steady]); } static void done_cursors(MDBX_txn *txn, const bool merge) { @@ -59,16 +58,14 @@ int txn_write(MDBX_txn *txn, iov_ctx_t *ctx) { dl->sorted = dpl_setlen(dl, w); txn->tw.dirtyroom += r - 1 - w; tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom - : txn->env->options.dp_limit)); + (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); tASSERT(txn, txn->tw.dirtylist->length == txn->tw.loose_count); tASSERT(txn, txn->tw.dirtylist->pages_including_loose == txn->tw.loose_count); return rc; } /* Merge child txn into parent */ -static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, - const size_t parent_retired_len) { +static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t parent_retired_len) { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0); dpl_t *const src = dpl_sort(txn); @@ -84,8 +81,7 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, parent->tw.dirtyroom += dst->sorted - n; dst->sorted = dpl_setlen(dst, n); tASSERT(parent, parent->tw.dirtyroom + parent->tw.dirtylist->length == - (parent->parent ? parent->parent->tw.dirtyroom - : parent->env->options.dp_limit)); + (parent->parent ? parent->parent->tw.dirtyroom : parent->env->options.dp_limit)); } /* Remove reclaimed pages from parent's dirty list */ @@ -94,8 +90,7 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, /* Move retired pages from parent's dirty & spilled list to reclaimed */ size_t r, w, d, s, l; - for (r = w = parent_retired_len; - ++r <= MDBX_PNL_GETSIZE(parent->tw.retired_pages);) { + for (r = w = parent_retired_len; ++r <= MDBX_PNL_GETSIZE(parent->tw.retired_pages);) { const pgno_t pgno = parent->tw.retired_pages[r]; const size_t di = dpl_exist(parent, pgno); const size_t si = !di ? spill_search(parent, pgno) : 0; @@ -103,8 +98,7 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const char *kind; if (di) { page_t *dp = dst->items[di].ptr; - tASSERT(parent, (dp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | - P_SPILLED)) == 0); + tASSERT(parent, (dp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | P_SPILLED)) == 0); npages = dpl_npages(dst, di); page_wash(parent, di, dp, npages); kind = "dirty"; @@ -128,8 +122,7 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, break; } #else - while (w > parent_retired_len && - parent->tw.retired_pages[w - 1] == pgno + l) { + while (w > parent_retired_len && parent->tw.retired_pages[w - 1] == pgno + l) { --w; if (++l == npages) break; @@ -145,22 +138,19 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, continue; } - DEBUG("reclaim retired parent's %u -> %zu %s page %" PRIaPGNO, npages, l, - kind, pgno); + DEBUG("reclaim retired parent's %u -> %zu %s page %" PRIaPGNO, npages, l, kind, pgno); int err = pnl_insert_span(&parent->tw.relist, pgno, l); ENSURE(txn->env, err == MDBX_SUCCESS); } MDBX_PNL_SETSIZE(parent->tw.retired_pages, w); /* Filter-out parent spill list */ - if (parent->tw.spilled.list && - MDBX_PNL_GETSIZE(parent->tw.spilled.list) > 0) { + if (parent->tw.spilled.list && MDBX_PNL_GETSIZE(parent->tw.spilled.list) > 0) { const pnl_t sl = spill_purge(parent); size_t len = MDBX_PNL_GETSIZE(sl); if (len) { /* Remove refunded pages from parent's spill list */ - if (MDBX_ENABLE_REFUND && - MDBX_PNL_MOST(sl) >= (parent->geo.first_unallocated << 1)) { + if (MDBX_ENABLE_REFUND && MDBX_PNL_MOST(sl) >= (parent->geo.first_unallocated << 1)) { #if MDBX_PNL_ASCENDING size_t i = MDBX_PNL_GETSIZE(sl); assert(MDBX_PNL_MOST(sl) == MDBX_PNL_LAST(sl)); @@ -182,8 +172,7 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, memmove(sl + 1, sl + 1 + i, len * sizeof(sl[0])); #endif } - tASSERT(txn, pnl_check_allocated(sl, (size_t)parent->geo.first_unallocated - << 1)); + tASSERT(txn, pnl_check_allocated(sl, (size_t)parent->geo.first_unallocated << 1)); /* Remove reclaimed pages from parent's spill list */ s = MDBX_PNL_GETSIZE(sl), r = MDBX_PNL_GETSIZE(reclaimed_list); @@ -200,8 +189,7 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, s -= !cmp; r -= cmp; } else { - DEBUG("remove reclaimed parent's spilled page %" PRIaPGNO, - reclaimed_pgno); + DEBUG("remove reclaimed parent's spilled page %" PRIaPGNO, reclaimed_pgno); spill_remove(parent, s, 1); --s; --r; @@ -231,8 +219,7 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, continue; } - DEBUG("remove dirtied parent's spilled %u page %" PRIaPGNO, npages, - dirty_pgno_form); + DEBUG("remove dirtied parent's spilled %u page %" PRIaPGNO, npages, dirty_pgno_form); spill_remove(parent, s, 1); s += step; } @@ -244,27 +231,22 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, /* Remove anything in our spill list from parent's dirty list */ if (txn->tw.spilled.list) { - tASSERT(txn, - pnl_check_allocated(txn->tw.spilled.list, - (size_t)parent->geo.first_unallocated << 1)); + tASSERT(txn, pnl_check_allocated(txn->tw.spilled.list, (size_t)parent->geo.first_unallocated << 1)); dpl_sift(parent, txn->tw.spilled.list, true); tASSERT(parent, parent->tw.dirtyroom + parent->tw.dirtylist->length == - (parent->parent ? parent->parent->tw.dirtyroom - : parent->env->options.dp_limit)); + (parent->parent ? parent->parent->tw.dirtyroom : parent->env->options.dp_limit)); } /* Find length of merging our dirty list with parent's and release * filter-out pages */ for (l = 0, d = dst->length, s = src->length; d > 0 && s > 0;) { page_t *sp = src->items[s].ptr; - tASSERT(parent, (sp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | - P_LOOSE | P_SPILLED)) == 0); + tASSERT(parent, (sp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | P_LOOSE | P_SPILLED)) == 0); const unsigned s_npages = dpl_npages(src, s); const pgno_t s_pgno = src->items[s].pgno; page_t *dp = dst->items[d].ptr; - tASSERT(parent, (dp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | - P_SPILLED)) == 0); + tASSERT(parent, (dp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | P_SPILLED)) == 0); const unsigned d_npages = dpl_npages(dst, d); const pgno_t d_pgno = dst->items[d].pgno; @@ -289,8 +271,7 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, while (s > 0) { page_t *sp = src->items[s].ptr; - tASSERT(parent, (sp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | - P_LOOSE | P_SPILLED)) == 0); + tASSERT(parent, (sp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | P_LOOSE | P_SPILLED)) == 0); if (sp->flags != P_LOOSE) { sp->txnid = parent->front_txnid; sp->flags &= ~P_SPILLED; @@ -318,9 +299,7 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, } assert(l > d); if (dst->items[d].ptr) { - dst->items[l--] = (dst->items[d].pgno > src->items[s].pgno) - ? dst->items[d--] - : src->items[s--]; + dst->items[l--] = (dst->items[d].pgno > src->items[s].pgno) ? dst->items[d--] : src->items[s--]; } else --d; } @@ -360,9 +339,7 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, } assert(l < d); if (dst->items[d].ptr) { - dst->items[l++] = (dst->items[d].pgno < src->items[s].pgno) - ? dst->items[d++] - : src->items[s++]; + dst->items[l++] = (dst->items[d].pgno < src->items[s].pgno) ? dst->items[d++] : src->items[s++]; } else ++d; } @@ -412,8 +389,7 @@ static void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, parent->flags &= ~MDBX_TXN_HAS_CHILD; if (parent->tw.spilled.list) { - assert(pnl_check_allocated(parent->tw.spilled.list, - (size_t)parent->geo.first_unallocated << 1)); + assert(pnl_check_allocated(parent->tw.spilled.list, (size_t)parent->geo.first_unallocated << 1)); if (MDBX_PNL_GETSIZE(parent->tw.spilled.list)) parent->flags |= MDBX_TXN_SPILLS; } @@ -424,19 +400,15 @@ static void take_gcprof(MDBX_txn *txn, MDBX_commit_latency *latency) { if (MDBX_ENABLE_PROFGC) { pgop_stat_t *const ptr = &env->lck->pgops; latency->gc_prof.work_counter = ptr->gc_prof.work.spe_counter; - latency->gc_prof.work_rtime_monotonic = - osal_monotime_to_16dot16(ptr->gc_prof.work.rtime_monotonic); - latency->gc_prof.work_xtime_cpu = - osal_monotime_to_16dot16(ptr->gc_prof.work.xtime_cpu); + latency->gc_prof.work_rtime_monotonic = osal_monotime_to_16dot16(ptr->gc_prof.work.rtime_monotonic); + latency->gc_prof.work_xtime_cpu = osal_monotime_to_16dot16(ptr->gc_prof.work.xtime_cpu); latency->gc_prof.work_rsteps = ptr->gc_prof.work.rsteps; latency->gc_prof.work_xpages = ptr->gc_prof.work.xpages; latency->gc_prof.work_majflt = ptr->gc_prof.work.majflt; latency->gc_prof.self_counter = ptr->gc_prof.self.spe_counter; - latency->gc_prof.self_rtime_monotonic = - osal_monotime_to_16dot16(ptr->gc_prof.self.rtime_monotonic); - latency->gc_prof.self_xtime_cpu = - osal_monotime_to_16dot16(ptr->gc_prof.self.xtime_cpu); + latency->gc_prof.self_rtime_monotonic = osal_monotime_to_16dot16(ptr->gc_prof.self.rtime_monotonic); + latency->gc_prof.self_xtime_cpu = osal_monotime_to_16dot16(ptr->gc_prof.self.xtime_cpu); latency->gc_prof.self_rsteps = ptr->gc_prof.self.rsteps; latency->gc_prof.self_xpages = ptr->gc_prof.self.xpages; latency->gc_prof.self_majflt = ptr->gc_prof.self.majflt; @@ -453,8 +425,7 @@ static void take_gcprof(MDBX_txn *txn, MDBX_commit_latency *latency) { } int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { - STATIC_ASSERT(MDBX_TXN_FINISHED == MDBX_TXN_BLOCKED - MDBX_TXN_HAS_CHILD - - MDBX_TXN_ERROR - MDBX_TXN_PARKED); + STATIC_ASSERT(MDBX_TXN_FINISHED == MDBX_TXN_BLOCKED - MDBX_TXN_HAS_CHILD - MDBX_TXN_ERROR - MDBX_TXN_PARKED); const uint64_t ts_0 = latency ? osal_monotime() : 0; uint64_t ts_1 = 0, ts_2 = 0, ts_3 = 0, ts_4 = 0, ts_5 = 0, gc_cputime = 0; @@ -483,13 +454,11 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { } /* txn_end() mode for a commit which writes nothing */ - unsigned end_mode = - TXN_END_PURE_COMMIT | TXN_END_UPDATE | TXN_END_SLOT | TXN_END_FREE; + unsigned end_mode = TXN_END_PURE_COMMIT | TXN_END_UPDATE | TXN_END_SLOT | TXN_END_FREE; if (unlikely(txn->flags & MDBX_TXN_RDONLY)) goto done; - if ((txn->flags & MDBX_NOSTICKYTHREADS) && - unlikely(txn->owner != osal_thread_self())) { + if ((txn->flags & MDBX_NOSTICKYTHREADS) && unlikely(txn->owner != osal_thread_self())) { rc = MDBX_THREAD_MISMATCH; goto fail; } @@ -512,25 +481,19 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { eASSERT(env, txn != env->basal_txn); MDBX_txn *const parent = txn->parent; eASSERT(env, parent->signature == txn_signature); - eASSERT(env, - parent->nested == txn && (parent->flags & MDBX_TXN_HAS_CHILD) != 0); + eASSERT(env, parent->nested == txn && (parent->flags & MDBX_TXN_HAS_CHILD) != 0); eASSERT(env, dpl_check(txn)); - if (txn->tw.dirtylist->length == 0 && !(txn->flags & MDBX_TXN_DIRTY) && - parent->n_dbi == txn->n_dbi) { + if (txn->tw.dirtylist->length == 0 && !(txn->flags & MDBX_TXN_DIRTY) && parent->n_dbi == txn->n_dbi) { TXN_FOREACH_DBI_ALL(txn, i) { tASSERT(txn, (txn->dbi_state[i] & DBI_DIRTY) == 0); - if ((txn->dbi_state[i] & DBI_STALE) && - !(parent->dbi_state[i] & DBI_STALE)) - tASSERT(txn, - memcmp(&parent->dbs[i], &txn->dbs[i], sizeof(tree_t)) == 0); + if ((txn->dbi_state[i] & DBI_STALE) && !(parent->dbi_state[i] & DBI_STALE)) + tASSERT(txn, memcmp(&parent->dbs[i], &txn->dbs[i], sizeof(tree_t)) == 0); } tASSERT(txn, memcmp(&parent->geo, &txn->geo, sizeof(parent->geo)) == 0); - tASSERT(txn, memcmp(&parent->canary, &txn->canary, - sizeof(parent->canary)) == 0); - tASSERT(txn, !txn->tw.spilled.list || - MDBX_PNL_GETSIZE(txn->tw.spilled.list) == 0); + tASSERT(txn, memcmp(&parent->canary, &txn->canary, sizeof(parent->canary)) == 0); + tASSERT(txn, !txn->tw.spilled.list || MDBX_PNL_GETSIZE(txn->tw.spilled.list) == 0); tASSERT(txn, txn->tw.loose_count == 0); /* fast completion of pure nested transaction */ @@ -543,8 +506,7 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { * if allocation fails. */ const size_t parent_retired_len = (uintptr_t)parent->tw.retired_pages; tASSERT(txn, parent_retired_len <= MDBX_PNL_GETSIZE(txn->tw.retired_pages)); - const size_t retired_delta = - MDBX_PNL_GETSIZE(txn->tw.retired_pages) - parent_retired_len; + const size_t retired_delta = MDBX_PNL_GETSIZE(txn->tw.retired_pages) - parent_retired_len; if (retired_delta) { rc = pnl_need(&txn->tw.relist, retired_delta); if (unlikely(rc != MDBX_SUCCESS)) @@ -553,18 +515,15 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { if (txn->tw.spilled.list) { if (parent->tw.spilled.list) { - rc = pnl_need(&parent->tw.spilled.list, - MDBX_PNL_GETSIZE(txn->tw.spilled.list)); + rc = pnl_need(&parent->tw.spilled.list, MDBX_PNL_GETSIZE(txn->tw.spilled.list)); if (unlikely(rc != MDBX_SUCCESS)) goto fail; } spill_purge(txn); } - if (unlikely(txn->tw.dirtylist->length + parent->tw.dirtylist->length > - parent->tw.dirtylist->detent && - !dpl_reserve(parent, txn->tw.dirtylist->length + - parent->tw.dirtylist->length))) { + if (unlikely(txn->tw.dirtylist->length + parent->tw.dirtylist->length > parent->tw.dirtylist->detent && + !dpl_reserve(parent, txn->tw.dirtylist->length + parent->tw.dirtylist->length))) { rc = MDBX_ENOMEM; goto fail; } @@ -604,17 +563,12 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { if (txn->dbi_state[dbi] & (DBI_CREAT | DBI_FRESH | DBI_DIRTY)) { parent->dbs[dbi] = txn->dbs[dbi]; /* preserve parent's status */ - const uint8_t state = - txn->dbi_state[dbi] | - (parent->dbi_state[dbi] & (DBI_CREAT | DBI_FRESH | DBI_DIRTY)); - DEBUG("dbi %zu dbi-state %s 0x%02x -> 0x%02x", dbi, - (parent->dbi_state[dbi] != state) ? "update" : "still", + const uint8_t state = txn->dbi_state[dbi] | (parent->dbi_state[dbi] & (DBI_CREAT | DBI_FRESH | DBI_DIRTY)); + DEBUG("dbi %zu dbi-state %s 0x%02x -> 0x%02x", dbi, (parent->dbi_state[dbi] != state) ? "update" : "still", parent->dbi_state[dbi], state); parent->dbi_state[dbi] = state; } else { - eASSERT(env, - txn->dbi_state[dbi] == (parent->dbi_state[dbi] & - ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY))); + eASSERT(env, txn->dbi_state[dbi] == (parent->dbi_state[dbi] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY))); } } @@ -635,15 +589,13 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { if (ASSERT_ENABLED()) { /* Check parent's loose pages not suitable for refund */ for (page_t *lp = parent->tw.loose_pages; lp; lp = page_next(lp)) { - tASSERT(parent, lp->pgno < parent->tw.loose_refund_wl && - lp->pgno + 1 < parent->geo.first_unallocated); + tASSERT(parent, lp->pgno < parent->tw.loose_refund_wl && lp->pgno + 1 < parent->geo.first_unallocated); MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *)); VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); } /* Check parent's reclaimed pages not suitable for refund */ if (MDBX_PNL_GETSIZE(parent->tw.relist)) - tASSERT(parent, MDBX_PNL_MOST(parent->tw.relist) + 1 < - parent->geo.first_unallocated); + tASSERT(parent, MDBX_PNL_MOST(parent->tw.relist) + 1 < parent->geo.first_unallocated); } #endif /* MDBX_ENABLE_REFUND */ @@ -659,17 +611,14 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { } else { tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom - : env->options.dp_limit)); + (txn->parent ? txn->parent->tw.dirtyroom : env->options.dp_limit)); } done_cursors(txn, false); end_mode |= TXN_END_EOTDONE; if ((!txn->tw.dirtylist || txn->tw.dirtylist->length == 0) && (txn->flags & (MDBX_TXN_DIRTY | MDBX_TXN_SPILLS)) == 0) { - TXN_FOREACH_DBI_ALL(txn, i) { - tASSERT(txn, !(txn->dbi_state[i] & DBI_DIRTY)); - } + TXN_FOREACH_DBI_ALL(txn, i) { tASSERT(txn, !(txn->dbi_state[i] & DBI_DIRTY)); } #if defined(MDBX_NOSUCCESS_EMPTY_COMMIT) && MDBX_NOSUCCESS_EMPTY_COMMIT rc = txn_end(txn, end_mode); if (unlikely(rc != MDBX_SUCCESS)) @@ -681,10 +630,8 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { #endif /* MDBX_NOSUCCESS_EMPTY_COMMIT */ } - DEBUG("committing txn %" PRIaTXN " %p on env %p, root page %" PRIaPGNO - "/%" PRIaPGNO, - txn->txnid, (void *)txn, (void *)env, txn->dbs[MAIN_DBI].root, - txn->dbs[FREE_DBI].root); + DEBUG("committing txn %" PRIaTXN " %p on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, txn->txnid, (void *)txn, + (void *)env, txn->dbs[MAIN_DBI].root, txn->dbs[FREE_DBI].root); if (txn->n_dbi > CORE_DBS) { /* Update table root pointers */ @@ -698,9 +645,7 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { if ((txn->dbi_state[i] & DBI_DIRTY) == 0) continue; tree_t *const db = &txn->dbs[i]; - DEBUG("update main's entry for sub-db %zu, mod_txnid %" PRIaTXN - " -> %" PRIaTXN, - i, db->mod_txnid, txn->txnid); + DEBUG("update main's entry for sub-db %zu, mod_txnid %" PRIaTXN " -> %" PRIaTXN, i, db->mod_txnid, txn->txnid); /* Может быть mod_txnid > front после коммита вложенных тразакций */ db->mod_txnid = txn->txnid; MDBX_val data = {db, sizeof(tree_t)}; @@ -726,13 +671,9 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { goto fail; tASSERT(txn, txn->tw.loose_count == 0); - txn->dbs[FREE_DBI].mod_txnid = (txn->dbi_state[FREE_DBI] & DBI_DIRTY) - ? txn->txnid - : txn->dbs[FREE_DBI].mod_txnid; + txn->dbs[FREE_DBI].mod_txnid = (txn->dbi_state[FREE_DBI] & DBI_DIRTY) ? txn->txnid : txn->dbs[FREE_DBI].mod_txnid; - txn->dbs[MAIN_DBI].mod_txnid = (txn->dbi_state[MAIN_DBI] & DBI_DIRTY) - ? txn->txnid - : txn->dbs[MAIN_DBI].mod_txnid; + txn->dbs[MAIN_DBI].mod_txnid = (txn->dbi_state[MAIN_DBI] & DBI_DIRTY) ? txn->txnid : txn->dbs[MAIN_DBI].mod_txnid; ts_2 = latency ? osal_monotime() : 0; ts_3 = ts_2; @@ -745,8 +686,7 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { bool need_flush_for_nometasync = false; const meta_ptr_t head = meta_recent(env, &txn->tw.troika); - const uint32_t meta_sync_txnid = - atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed); + const uint32_t meta_sync_txnid = atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed); /* sync prev meta */ if (head.is_steady && meta_sync_txnid != (uint32_t)head.txnid) { /* Исправление унаследованного от LMDB недочета: @@ -767,10 +707,8 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { * sync-операцией выполняемой после записи данных текущей транзакции. * Соответственно, требуется явно обновлять мета-страницу, что полностью * уничтожает выгоду от NOMETASYNC. */ - const uint32_t txnid_dist = - ((txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC) - ? MDBX_NOMETASYNC_LAZY_FD - : MDBX_NOMETASYNC_LAZY_WRITEMAP; + const uint32_t txnid_dist = ((txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC) ? MDBX_NOMETASYNC_LAZY_FD + : MDBX_NOMETASYNC_LAZY_WRITEMAP; /* Смысл "магии" в том, чтобы избежать отдельного вызова fdatasync() * или msync() для гарантированной фиксации на диске мета-страницы, * которая была "лениво" отправлена на запись в предыдущей транзакции, @@ -807,8 +745,7 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { #endif /* Windows */ iov_ctx_t write_ctx; - rc = iov_init(txn, &write_ctx, txn->tw.dirtylist->length, - txn->tw.dirtylist->pages_including_loose, fd, false); + rc = iov_init(txn, &write_ctx, txn->tw.dirtylist->length, txn->tw.dirtylist->pages_including_loose, fd, false); if (unlikely(rc != MDBX_SUCCESS)) { ERROR("txn-%s: error %d", "iov-init", rc); goto fail; @@ -835,8 +772,7 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { meta.validator_id = head.ptr_c->validator_id; meta.extra_pagehdr = head.ptr_c->extra_pagehdr; unaligned_poke_u64(4, meta.pages_retired, - unaligned_peek_u64(4, head.ptr_c->pages_retired) + - MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + unaligned_peek_u64(4, head.ptr_c->pages_retired) + MDBX_PNL_GETSIZE(txn->tw.retired_pages)); meta.geometry = txn->geo; meta.trees.gc = txn->dbs[FREE_DBI]; meta.trees.main = txn->dbs[MAIN_DBI]; @@ -847,15 +783,13 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { #if MDBX_ENABLE_BIGFOOT if (gcu_ctx.bigfoot > txn->txnid) { commit_txnid = gcu_ctx.bigfoot; - TRACE("use @%" PRIaTXN " (+%zu) for commit bigfoot-txn", commit_txnid, - (size_t)(commit_txnid - txn->txnid)); + TRACE("use @%" PRIaTXN " (+%zu) for commit bigfoot-txn", commit_txnid, (size_t)(commit_txnid - txn->txnid)); } #endif meta.unsafe_sign = DATASIGN_NONE; meta_set_txnid(env, &meta, commit_txnid); - rc = dxb_sync_locked(env, env->flags | txn->flags | txn_shrink_allowed, &meta, - &txn->tw.troika); + rc = dxb_sync_locked(env, env->flags | txn->flags | txn_shrink_allowed, &meta, &txn->tw.troika); ts_5 = latency ? osal_monotime() : 0; if (unlikely(rc != MDBX_SUCCESS)) { @@ -874,8 +808,7 @@ done: provide_latency: if (latency) { latency->preparation = ts_1 ? osal_monotime_to_16dot16(ts_1 - ts_0) : 0; - latency->gc_wallclock = - (ts_2 > ts_1) ? osal_monotime_to_16dot16(ts_2 - ts_1) : 0; + latency->gc_wallclock = (ts_2 > ts_1) ? osal_monotime_to_16dot16(ts_2 - ts_1) : 0; latency->gc_cputime = gc_cputime ? osal_monotime_to_16dot16(gc_cputime) : 0; latency->audit = (ts_3 > ts_2) ? osal_monotime_to_16dot16(ts_3 - ts_2) : 0; latency->write = (ts_4 > ts_3) ? osal_monotime_to_16dot16(ts_4 - ts_3) : 0; @@ -897,8 +830,7 @@ fail: int txn_abort(MDBX_txn *txn) { if (txn->flags & MDBX_TXN_RDONLY) /* LY: don't close DBI-handles */ - return txn_end(txn, TXN_END_ABORT | TXN_END_UPDATE | TXN_END_SLOT | - TXN_END_FREE); + return txn_end(txn, TXN_END_ABORT | TXN_END_UPDATE | TXN_END_SLOT | TXN_END_FREE); if (unlikely(txn->flags & MDBX_TXN_FINISHED)) return MDBX_BAD_TXN; @@ -923,8 +855,7 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { flags |= env->flags & (MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP); if (flags & MDBX_TXN_RDONLY) { - eASSERT(env, (flags & ~(txn_ro_begin_flags | MDBX_WRITEMAP | - MDBX_NOSTICKYTHREADS)) == 0); + eASSERT(env, (flags & ~(txn_ro_begin_flags | MDBX_WRITEMAP | MDBX_NOSTICKYTHREADS)) == 0); txn->flags = flags; reader_slot_t *r = txn->to.reader; STATIC_ASSERT(sizeof(uintptr_t) <= sizeof(r->tid)); @@ -932,8 +863,7 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { eASSERT(env, !(env->flags & MDBX_NOSTICKYTHREADS)); r = thread_rthc_get(env->me_txkey); if (likely(r)) { - if (unlikely(!r->pid.weak) && - (globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN)) { + if (unlikely(!r->pid.weak) && (globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN)) { thread_rthc_set(env->me_txkey, nullptr); r = nullptr; } else { @@ -946,8 +876,7 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { } if (likely(r)) { - if (unlikely(r->pid.weak != env->pid || - r->txnid.weak < SAFE64_INVALID_THRESHOLD)) + if (unlikely(r->pid.weak != env->pid || r->txnid.weak < SAFE64_INVALID_THRESHOLD)) return MDBX_BAD_RSLOT; } else if (env->lck_mmap.lck) { bsr_t brs = mvcc_bind_slot(env); @@ -970,8 +899,7 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { return MDBX_SUCCESS; } txn->owner = (uintptr_t)r->tid.weak; - if ((env->flags & MDBX_NOSTICKYTHREADS) == 0 && env->txn && - unlikely(env->basal_txn->owner == txn->owner) && + if ((env->flags & MDBX_NOSTICKYTHREADS) == 0 && env->txn && unlikely(env->basal_txn->owner == txn->owner) && (globals.runtime_flags & MDBX_DBG_LEGACY_OVERLAP) == 0) return MDBX_TXN_OVERLAPPING; @@ -980,26 +908,18 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { size_t loop = 0; troika_t troika = meta_tap(env); while (1) { - const meta_ptr_t head = - likely(env->stuck_meta < 0) - ? /* regular */ meta_recent(env, &troika) - : /* recovery mode */ meta_ptr(env, env->stuck_meta); + const meta_ptr_t head = likely(env->stuck_meta < 0) ? /* regular */ meta_recent(env, &troika) + : /* recovery mode */ meta_ptr(env, env->stuck_meta); if (likely(r != nullptr)) { safe64_reset(&r->txnid, true); - atomic_store32(&r->snapshot_pages_used, - head.ptr_v->geometry.first_unallocated, mo_Relaxed); - atomic_store64( - &r->snapshot_pages_retired, - unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired), - mo_Relaxed); + atomic_store32(&r->snapshot_pages_used, head.ptr_v->geometry.first_unallocated, mo_Relaxed); + atomic_store64(&r->snapshot_pages_retired, unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired), + mo_Relaxed); safe64_write(&r->txnid, head.txnid); eASSERT(env, r->pid.weak == osal_getpid()); - eASSERT(env, r->tid.weak == ((env->flags & MDBX_NOSTICKYTHREADS) - ? 0 - : osal_thread_self())); + eASSERT(env, r->tid.weak == ((env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self())); eASSERT(env, r->txnid.weak == head.txnid || - (r->txnid.weak >= SAFE64_INVALID_THRESHOLD && - head.txnid < env->lck->cached_oldest.weak)); + (r->txnid.weak >= SAFE64_INVALID_THRESHOLD && head.txnid < env->lck->cached_oldest.weak)); atomic_store32(&env->lck->rdt_refresh_flag, true, mo_AcquireRelease); } else { /* exclusive mode without lck */ @@ -1013,8 +933,7 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { timestamp = 0; continue; } - ERROR("bailout waiting for valid snapshot (%s)", - "meta-pages are too volatile"); + ERROR("bailout waiting for valid snapshot (%s)", "meta-pages are too volatile"); rc = MDBX_PROBLEM; goto read_failed; } @@ -1029,8 +948,7 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { goto read_failed; } - const uint64_t snap_oldest = - atomic_load64(&env->lck->cached_oldest, mo_AcquireRelease); + const uint64_t snap_oldest = atomic_load64(&env->lck->cached_oldest, mo_AcquireRelease); if (unlikely(txn->txnid < snap_oldest)) { if (env->stuck_meta < 0) goto retry; @@ -1041,8 +959,7 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { goto read_failed; } - if (likely(r != nullptr) && - unlikely(txn->txnid != atomic_load64(&r->txnid, mo_Relaxed))) + if (likely(r != nullptr) && unlikely(txn->txnid != atomic_load64(&r->txnid, mo_Relaxed))) goto retry; break; } @@ -1058,23 +975,19 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { } tASSERT(txn, rc == MDBX_SUCCESS); - ENSURE(env, - txn->txnid >= - /* paranoia is appropriate here */ env->lck->cached_oldest.weak); + ENSURE(env, txn->txnid >= + /* paranoia is appropriate here */ env->lck->cached_oldest.weak); tASSERT(txn, txn->dbs[FREE_DBI].flags == MDBX_INTEGERKEY); tASSERT(txn, check_table_flags(txn->dbs[MAIN_DBI].flags)); } else { - eASSERT(env, (flags & ~(txn_rw_begin_flags | MDBX_TXN_SPILLS | - MDBX_WRITEMAP | MDBX_NOSTICKYTHREADS)) == 0); + eASSERT(env, (flags & ~(txn_rw_begin_flags | MDBX_TXN_SPILLS | MDBX_WRITEMAP | MDBX_NOSTICKYTHREADS)) == 0); const uintptr_t tid = osal_thread_self(); if (unlikely(txn->owner == tid || /* not recovery mode */ env->stuck_meta >= 0)) return MDBX_BUSY; lck_t *const lck = env->lck_mmap.lck; - if (lck && (env->flags & MDBX_NOSTICKYTHREADS) == 0 && - (globals.runtime_flags & MDBX_DBG_LEGACY_OVERLAP) == 0) { - const size_t snap_nreaders = - atomic_load32(&lck->rdt_length, mo_AcquireRelease); + if (lck && (env->flags & MDBX_NOSTICKYTHREADS) == 0 && (globals.runtime_flags & MDBX_DBG_LEGACY_OVERLAP) == 0) { + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); for (size_t i = 0; i < snap_nreaders; ++i) { if (atomic_load32(&lck->rdt[i].pid, mo_Relaxed) == env->pid && unlikely(atomic_load64(&lck->rdt[i].tid, mo_Relaxed) == tid)) { @@ -1153,8 +1066,7 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { eASSERT(env, txn->tw.writemap_spilled_npages == 0); } - txn->front_txnid = - txn->txnid + ((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) == 0); + txn->front_txnid = txn->txnid + ((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) == 0); /* Setup db info */ tASSERT(txn, txn->dbs[FREE_DBI].flags == MDBX_INTEGERKEY); @@ -1162,10 +1074,8 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { VALGRIND_MAKE_MEM_UNDEFINED(txn->dbi_state, env->max_dbi); #if MDBX_ENABLE_DBI_SPARSE txn->n_dbi = CORE_DBS; - VALGRIND_MAKE_MEM_UNDEFINED( - txn->dbi_sparse, - ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) / - CHAR_BIT); + VALGRIND_MAKE_MEM_UNDEFINED(txn->dbi_sparse, + ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) / CHAR_BIT); txn->dbi_sparse[0] = (1 << CORE_DBS) - 1; #else txn->n_dbi = (env->n_dbi < 8) ? env->n_dbi : 8; @@ -1177,13 +1087,10 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { txn->cursors[FREE_DBI] = nullptr; txn->cursors[MAIN_DBI] = nullptr; txn->dbi_seqs[FREE_DBI] = 0; - txn->dbi_seqs[MAIN_DBI] = - atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease); + txn->dbi_seqs[MAIN_DBI] = atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease); - if (unlikely(env->dbs_flags[MAIN_DBI] != - (DB_VALID | txn->dbs[MAIN_DBI].flags))) { - const bool need_txn_lock = - env->basal_txn && env->basal_txn->owner != osal_thread_self(); + if (unlikely(env->dbs_flags[MAIN_DBI] != (DB_VALID | txn->dbs[MAIN_DBI].flags))) { + const bool need_txn_lock = env->basal_txn && env->basal_txn->owner != osal_thread_self(); bool should_unlock = false; if (need_txn_lock) { rc = lck_txn_lock(env, true); @@ -1202,10 +1109,8 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { * то следующая будет ждать на dbi_lock */ !env->txn) { if (env->dbs_flags[MAIN_DBI] != 0 || MDBX_DEBUG) - NOTICE("renew MainDB for %s-txn %" PRIaTXN - " since db-flags changes 0x%x -> 0x%x", - (txn->flags & MDBX_TXN_RDONLY) ? "ro" : "rw", txn->txnid, - env->dbs_flags[MAIN_DBI] & ~DB_VALID, + NOTICE("renew MainDB for %s-txn %" PRIaTXN " since db-flags changes 0x%x -> 0x%x", + (txn->flags & MDBX_TXN_RDONLY) ? "ro" : "rw", txn->txnid, env->dbs_flags[MAIN_DBI] & ~DB_VALID, txn->dbs[MAIN_DBI].flags); env->dbs_flags[MAIN_DBI] = DB_POISON; atomic_store32(&env->dbi_seqs[MAIN_DBI], seq, mo_AcquireRelease); @@ -1213,14 +1118,12 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { if (likely(rc == MDBX_SUCCESS)) { seq = dbi_seq_next(env, MAIN_DBI); env->dbs_flags[MAIN_DBI] = DB_VALID | txn->dbs[MAIN_DBI].flags; - txn->dbi_seqs[MAIN_DBI] = atomic_store32(&env->dbi_seqs[MAIN_DBI], - seq, mo_AcquireRelease); + txn->dbi_seqs[MAIN_DBI] = atomic_store32(&env->dbi_seqs[MAIN_DBI], seq, mo_AcquireRelease); } } else { ERROR("MainDB db-flags changes 0x%x -> 0x%x ahead of read-txn " "%" PRIaTXN, - txn->dbs[MAIN_DBI].flags, env->dbs_flags[MAIN_DBI] & ~DB_VALID, - txn->txnid); + txn->dbs[MAIN_DBI].flags, env->dbs_flags[MAIN_DBI] & ~DB_VALID, txn->txnid); rc = MDBX_INCOMPATIBLE; } } @@ -1235,8 +1138,7 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { } if (unlikely(txn->dbs[FREE_DBI].flags != MDBX_INTEGERKEY)) { - ERROR("unexpected/invalid db-flags 0x%x for %s", txn->dbs[FREE_DBI].flags, - "GC/FreeDB"); + ERROR("unexpected/invalid db-flags 0x%x for %s", txn->dbs[FREE_DBI].flags, "GC/FreeDB"); rc = MDBX_INCOMPATIBLE; goto bailout; } @@ -1249,8 +1151,7 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { } else { const size_t size_bytes = pgno2bytes(env, txn->geo.end_pgno); const size_t used_bytes = pgno2bytes(env, txn->geo.first_unallocated); - const size_t required_bytes = - (txn->flags & MDBX_TXN_RDONLY) ? used_bytes : size_bytes; + const size_t required_bytes = (txn->flags & MDBX_TXN_RDONLY) ? used_bytes : size_bytes; eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current); if (unlikely(required_bytes > env->dxb_mmap.current)) { /* Размер БД (для пишущих транзакций) или используемых данных (для @@ -1259,13 +1160,11 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { * границы размера БД и отображения. В читающих транзакциях нельзя * изменять размер файла, который может быть больше необходимого этой * транзакции. */ - if (txn->geo.upper > MAX_PAGENO + 1 || - bytes2pgno(env, pgno2bytes(env, txn->geo.upper)) != txn->geo.upper) { + if (txn->geo.upper > MAX_PAGENO + 1 || bytes2pgno(env, pgno2bytes(env, txn->geo.upper)) != txn->geo.upper) { rc = MDBX_UNABLE_EXTEND_MAPSIZE; goto bailout; } - rc = dxb_resize(env, txn->geo.first_unallocated, txn->geo.end_pgno, - txn->geo.upper, implicit_grow); + rc = dxb_resize(env, txn->geo.first_unallocated, txn->geo.end_pgno, txn->geo.upper, implicit_grow); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current); @@ -1300,9 +1199,7 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { eASSERT(env, env->dxb_mmap.filesize >= required_bytes); if (env->dxb_mmap.current > env->dxb_mmap.filesize) env->dxb_mmap.current = - (env->dxb_mmap.limit < env->dxb_mmap.filesize) - ? env->dxb_mmap.limit - : (size_t)env->dxb_mmap.filesize; + (env->dxb_mmap.limit < env->dxb_mmap.filesize) ? env->dxb_mmap.limit : (size_t)env->dxb_mmap.filesize; } #if defined(_WIN32) || defined(_WIN64) imports.srwl_ReleaseShared(&env->remap_guard); @@ -1315,8 +1212,7 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { if (unlikely(rc != MDBX_SUCCESS)) goto bailout; } - eASSERT(env, pgno2bytes(env, txn->geo.first_unallocated) <= - env->dxb_mmap.current); + eASSERT(env, pgno2bytes(env, txn->geo.first_unallocated) <= env->dxb_mmap.current); eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current); if (txn->flags & MDBX_TXN_RDONLY) { #if defined(_WIN32) || defined(_WIN64) @@ -1326,8 +1222,7 @@ int txn_renew(MDBX_txn *txn, unsigned flags) { * since Wine don't support section extending, * i.e. in both cases unmap+map are required. */ used_bytes < env->geo_in_bytes.upper && env->geo_in_bytes.grow)) && - /* avoid recursive use SRW */ (txn->flags & MDBX_NOSTICKYTHREADS) == - 0) { + /* avoid recursive use SRW */ (txn->flags & MDBX_NOSTICKYTHREADS) == 0) { txn->flags |= txn_shrink_allowed; imports.srwl_AcquireShared(&env->remap_guard); } @@ -1352,11 +1247,9 @@ int txn_end(MDBX_txn *txn, unsigned mode) { MDBX_env *env = txn->env; static const char *const names[] = TXN_END_NAMES; - DEBUG("%s txn %" PRIaTXN "%c-0x%X %p on env %p, root page %" PRIaPGNO - "/%" PRIaPGNO, - names[mode & TXN_END_OPMASK], txn->txnid, - (txn->flags & MDBX_TXN_RDONLY) ? 'r' : 'w', txn->flags, (void *)txn, - (void *)env, txn->dbs[MAIN_DBI].root, txn->dbs[FREE_DBI].root); + DEBUG("%s txn %" PRIaTXN "%c-0x%X %p on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, names[mode & TXN_END_OPMASK], + txn->txnid, (txn->flags & MDBX_TXN_RDONLY) ? 'r' : 'w', txn->flags, (void *)txn, (void *)env, + txn->dbs[MAIN_DBI].root, txn->dbs[FREE_DBI].root); if (!(mode & TXN_END_EOTDONE)) /* !(already closed cursors) */ done_cursors(txn, false); @@ -1369,21 +1262,17 @@ int txn_end(MDBX_txn *txn, unsigned mode) { if (likely(!(txn->flags & MDBX_TXN_FINISHED))) { if (likely((txn->flags & MDBX_TXN_PARKED) == 0)) { ENSURE(env, txn->txnid >= - /* paranoia is appropriate here */ env->lck - ->cached_oldest.weak); - eASSERT(env, txn->txnid == slot->txnid.weak && - slot->txnid.weak >= env->lck->cached_oldest.weak); + /* paranoia is appropriate here */ env->lck->cached_oldest.weak); + eASSERT(env, txn->txnid == slot->txnid.weak && slot->txnid.weak >= env->lck->cached_oldest.weak); } else { - if ((mode & TXN_END_OPMASK) != TXN_END_OUSTED && - safe64_read(&slot->tid) == MDBX_TID_TXN_OUSTED) + if ((mode & TXN_END_OPMASK) != TXN_END_OUSTED && safe64_read(&slot->tid) == MDBX_TID_TXN_OUSTED) mode = (mode & ~TXN_END_OPMASK) | TXN_END_OUSTED; do { safe64_reset(&slot->txnid, false); atomic_store64(&slot->tid, txn->owner, mo_AcquireRelease); atomic_yield(); } while ( - unlikely(safe64_read(&slot->txnid) < SAFE64_INVALID_THRESHOLD || - safe64_read(&slot->tid) != txn->owner)); + unlikely(safe64_read(&slot->txnid) < SAFE64_INVALID_THRESHOLD || safe64_read(&slot->tid) != txn->owner)); } dxb_sanitize_tail(env, nullptr); atomic_store32(&slot->snapshot_pages_used, 0, mo_Relaxed); @@ -1404,14 +1293,12 @@ int txn_end(MDBX_txn *txn, unsigned mode) { imports.srwl_ReleaseShared(&env->remap_guard); #endif txn->n_dbi = 0; /* prevent further DBI activity */ - txn->flags = ((mode & TXN_END_OPMASK) != TXN_END_OUSTED) - ? MDBX_TXN_RDONLY | MDBX_TXN_FINISHED - : MDBX_TXN_RDONLY | MDBX_TXN_FINISHED | MDBX_TXN_OUSTED; + txn->flags = ((mode & TXN_END_OPMASK) != TXN_END_OUSTED) ? MDBX_TXN_RDONLY | MDBX_TXN_FINISHED + : MDBX_TXN_RDONLY | MDBX_TXN_FINISHED | MDBX_TXN_OUSTED; txn->owner = 0; } else if (!(txn->flags & MDBX_TXN_FINISHED)) { - ENSURE(env, - txn->txnid >= - /* paranoia is appropriate here */ env->lck->cached_oldest.weak); + ENSURE(env, txn->txnid >= + /* paranoia is appropriate here */ env->lck->cached_oldest.weak); if (txn == env->basal_txn) dxb_sanitize_tail(env, nullptr); @@ -1433,28 +1320,20 @@ int txn_end(MDBX_txn *txn, unsigned mode) { eASSERT(env, txn->parent != nullptr); MDBX_txn *const parent = txn->parent; eASSERT(env, parent->signature == txn_signature); - eASSERT(env, parent->nested == txn && - (parent->flags & MDBX_TXN_HAS_CHILD) != 0); - eASSERT(env, - pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - - MDBX_ENABLE_REFUND)); - eASSERT(env, memcmp(&txn->tw.troika, &parent->tw.troika, - sizeof(troika_t)) == 0); + eASSERT(env, parent->nested == txn && (parent->flags & MDBX_TXN_HAS_CHILD) != 0); + eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + eASSERT(env, memcmp(&txn->tw.troika, &parent->tw.troika, sizeof(troika_t)) == 0); txn->owner = 0; if (txn->tw.gc.reclaimed) { - eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) >= - (uintptr_t)parent->tw.gc.reclaimed); - MDBX_PNL_SETSIZE(txn->tw.gc.reclaimed, - (uintptr_t)parent->tw.gc.reclaimed); + eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.gc.reclaimed) >= (uintptr_t)parent->tw.gc.reclaimed); + MDBX_PNL_SETSIZE(txn->tw.gc.reclaimed, (uintptr_t)parent->tw.gc.reclaimed); parent->tw.gc.reclaimed = txn->tw.gc.reclaimed; } if (txn->tw.retired_pages) { - eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.retired_pages) >= - (uintptr_t)parent->tw.retired_pages); - MDBX_PNL_SETSIZE(txn->tw.retired_pages, - (uintptr_t)parent->tw.retired_pages); + eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.retired_pages) >= (uintptr_t)parent->tw.retired_pages); + MDBX_PNL_SETSIZE(txn->tw.retired_pages, (uintptr_t)parent->tw.retired_pages); parent->tw.retired_pages = txn->tw.retired_pages; } @@ -1467,18 +1346,15 @@ int txn_end(MDBX_txn *txn, unsigned mode) { dpl_free(txn); pnl_free(txn->tw.relist); - if (parent->geo.upper != txn->geo.upper || - parent->geo.now != txn->geo.now) { + if (parent->geo.upper != txn->geo.upper || parent->geo.now != txn->geo.now) { /* undo resize performed by child txn */ - rc = dxb_resize(env, parent->geo.first_unallocated, parent->geo.now, - parent->geo.upper, impilict_shrink); + rc = dxb_resize(env, parent->geo.first_unallocated, parent->geo.now, parent->geo.upper, impilict_shrink); if (rc == MDBX_EPERM) { /* unable undo resize (it is regular for Windows), * therefore promote size changes from child to the parent txn */ WARNING("unable undo resize performed by child txn, promote to " "the parent (%u->%u, %u->%u)", - txn->geo.now, parent->geo.now, txn->geo.upper, - parent->geo.upper); + txn->geo.now, parent->geo.now, txn->geo.upper, parent->geo.upper); parent->geo.now = txn->geo.now; parent->geo.upper = txn->geo.upper; parent->flags |= MDBX_TXN_DIRTY; @@ -1524,13 +1400,10 @@ int mdbx_txn_renew(MDBX_txn *txn) { int rc = txn_renew(txn, MDBX_TXN_RDONLY); if (rc == MDBX_SUCCESS) { - tASSERT(txn, txn->owner == (txn->flags & MDBX_NOSTICKYTHREADS) - ? 0 - : osal_thread_self()); - DEBUG("renew txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO - "/%" PRIaPGNO, - txn->txnid, (txn->flags & MDBX_TXN_RDONLY) ? 'r' : 'w', (void *)txn, - (void *)txn->env, txn->dbs[MAIN_DBI].root, txn->dbs[FREE_DBI].root); + tASSERT(txn, txn->owner == (txn->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self()); + DEBUG("renew txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, txn->txnid, + (txn->flags & MDBX_TXN_RDONLY) ? 'r' : 'w', (void *)txn, (void *)txn->env, txn->dbs[MAIN_DBI].root, + txn->dbs[FREE_DBI].root); } return rc; } @@ -1544,12 +1417,9 @@ int mdbx_txn_set_userctx(MDBX_txn *txn, void *ctx) { return MDBX_SUCCESS; } -void *mdbx_txn_get_userctx(const MDBX_txn *txn) { - return check_txn(txn, MDBX_TXN_FINISHED) ? nullptr : txn->userctx; -} +void *mdbx_txn_get_userctx(const MDBX_txn *txn) { return check_txn(txn, MDBX_TXN_FINISHED) ? nullptr : txn->userctx; } -int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, - MDBX_txn **ret, void *context) { +int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, MDBX_txn **ret, void *context) { if (unlikely(!ret)) return MDBX_EINVAL; *ret = nullptr; @@ -1567,23 +1437,19 @@ int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, MDBX_txn *txn = nullptr; if (parent) { /* Nested transactions: Max 1 child, write txns only, no writemap */ - rc = check_txn_rw(parent, - MDBX_TXN_RDONLY | MDBX_WRITEMAP | MDBX_TXN_BLOCKED); + rc = check_txn_rw(parent, MDBX_TXN_RDONLY | MDBX_WRITEMAP | MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return rc; if (env->options.spill_parent4child_denominator) { /* Spill dirty-pages of parent to provide dirtyroom for child txn */ - rc = txn_spill(parent, nullptr, - parent->tw.dirtylist->length / - env->options.spill_parent4child_denominator); + rc = txn_spill(parent, nullptr, parent->tw.dirtylist->length / env->options.spill_parent4child_denominator); if (unlikely(rc != MDBX_SUCCESS)) return rc; } tASSERT(parent, audit_ex(parent, 0, false) == 0); - flags |= parent->flags & (txn_rw_begin_flags | MDBX_TXN_SPILLS | - MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP); + flags |= parent->flags & (txn_rw_begin_flags | MDBX_TXN_SPILLS | MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP); } else if ((flags & MDBX_TXN_RDONLY) == 0) { /* Reuse preallocated write txn. However, do not touch it until * txn_renew() succeeds, since it currently may be active. */ @@ -1593,22 +1459,16 @@ int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, const intptr_t bitmap_bytes = #if MDBX_ENABLE_DBI_SPARSE - ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) / - CHAR_BIT; + ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) / CHAR_BIT; #else 0; #endif /* MDBX_ENABLE_DBI_SPARSE */ STATIC_ASSERT(sizeof(txn->tw) > sizeof(txn->to)); - const size_t base = (flags & MDBX_TXN_RDONLY) - ? sizeof(MDBX_txn) - sizeof(txn->tw) + sizeof(txn->to) - : sizeof(MDBX_txn); - const size_t size = - base + - ((flags & MDBX_TXN_RDONLY) - ? (size_t)bitmap_bytes + env->max_dbi * sizeof(txn->dbi_seqs[0]) - : 0) + - env->max_dbi * (sizeof(txn->dbs[0]) + sizeof(txn->cursors[0]) + - sizeof(txn->dbi_state[0])); + const size_t base = + (flags & MDBX_TXN_RDONLY) ? sizeof(MDBX_txn) - sizeof(txn->tw) + sizeof(txn->to) : sizeof(MDBX_txn); + const size_t size = base + + ((flags & MDBX_TXN_RDONLY) ? (size_t)bitmap_bytes + env->max_dbi * sizeof(txn->dbi_seqs[0]) : 0) + + env->max_dbi * (sizeof(txn->dbs[0]) + sizeof(txn->cursors[0]) + sizeof(txn->dbi_state[0])); txn = osal_malloc(size); if (unlikely(txn == nullptr)) { DEBUG("calloc: %s", "failed"); @@ -1619,15 +1479,13 @@ int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, VALGRIND_MAKE_MEM_UNDEFINED(txn, size); #endif /* MDBX_DEBUG */ MDBX_ANALYSIS_ASSUME(size > base); - memset(txn, 0, - (MDBX_GOOFY_MSVC_STATIC_ANALYZER && base > size) ? size : base); + memset(txn, 0, (MDBX_GOOFY_MSVC_STATIC_ANALYZER && base > size) ? size : base); txn->dbs = ptr_disp(txn, base); txn->cursors = ptr_disp(txn->dbs, env->max_dbi * sizeof(txn->dbs[0])); #if MDBX_DEBUG txn->cursors[FREE_DBI] = nullptr; /* avoid SIGSEGV in an assertion later */ #endif - txn->dbi_state = - ptr_disp(txn, size - env->max_dbi * sizeof(txn->dbi_state[0])); + txn->dbi_state = ptr_disp(txn, size - env->max_dbi * sizeof(txn->dbi_state[0])); txn->flags = flags; txn->env = env; @@ -1640,10 +1498,8 @@ int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, txn->geo = parent->geo; rc = dpl_alloc(txn); if (likely(rc == MDBX_SUCCESS)) { - const size_t len = - MDBX_PNL_GETSIZE(parent->tw.relist) + parent->tw.loose_count; - txn->tw.relist = - pnl_alloc((len > MDBX_PNL_INITIAL) ? len : MDBX_PNL_INITIAL); + const size_t len = MDBX_PNL_GETSIZE(parent->tw.relist) + parent->tw.loose_count; + txn->tw.relist = pnl_alloc((len > MDBX_PNL_INITIAL) ? len : MDBX_PNL_INITIAL); if (unlikely(!txn->tw.relist)) rc = MDBX_ENOMEM; } @@ -1682,28 +1538,22 @@ int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, if (parent->tw.spilled.list) spill_purge(parent); - tASSERT(txn, MDBX_PNL_ALLOCLEN(txn->tw.relist) >= - MDBX_PNL_GETSIZE(parent->tw.relist)); - memcpy(txn->tw.relist, parent->tw.relist, - MDBX_PNL_SIZEOF(parent->tw.relist)); - eASSERT(env, pnl_check_allocated( - txn->tw.relist, - (txn->geo.first_unallocated /* LY: intentional assignment - here, only for assertion */ - = parent->geo.first_unallocated) - - MDBX_ENABLE_REFUND)); + tASSERT(txn, MDBX_PNL_ALLOCLEN(txn->tw.relist) >= MDBX_PNL_GETSIZE(parent->tw.relist)); + memcpy(txn->tw.relist, parent->tw.relist, MDBX_PNL_SIZEOF(parent->tw.relist)); + eASSERT(env, pnl_check_allocated(txn->tw.relist, (txn->geo.first_unallocated /* LY: intentional assignment + here, only for assertion */ + = parent->geo.first_unallocated) - + MDBX_ENABLE_REFUND)); txn->tw.gc.time_acc = parent->tw.gc.time_acc; txn->tw.gc.last_reclaimed = parent->tw.gc.last_reclaimed; if (parent->tw.gc.reclaimed) { txn->tw.gc.reclaimed = parent->tw.gc.reclaimed; - parent->tw.gc.reclaimed = - (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->tw.gc.reclaimed); + parent->tw.gc.reclaimed = (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->tw.gc.reclaimed); } txn->tw.retired_pages = parent->tw.retired_pages; - parent->tw.retired_pages = - (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->tw.retired_pages); + parent->tw.retired_pages = (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->tw.retired_pages); txn->txnid = parent->txnid; txn->front_txnid = parent->front_txnid + 1; @@ -1719,25 +1569,18 @@ int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, txn->cursors[FREE_DBI] = nullptr; txn->cursors[MAIN_DBI] = nullptr; - txn->dbi_state[FREE_DBI] = - parent->dbi_state[FREE_DBI] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY); - txn->dbi_state[MAIN_DBI] = - parent->dbi_state[MAIN_DBI] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY); - memset(txn->dbi_state + CORE_DBS, 0, - (txn->n_dbi = parent->n_dbi) - CORE_DBS); + txn->dbi_state[FREE_DBI] = parent->dbi_state[FREE_DBI] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY); + txn->dbi_state[MAIN_DBI] = parent->dbi_state[MAIN_DBI] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY); + memset(txn->dbi_state + CORE_DBS, 0, (txn->n_dbi = parent->n_dbi) - CORE_DBS); memcpy(txn->dbs, parent->dbs, sizeof(txn->dbs[0]) * CORE_DBS); tASSERT(parent, parent->tw.dirtyroom + parent->tw.dirtylist->length == - (parent->parent ? parent->parent->tw.dirtyroom - : parent->env->options.dp_limit)); + (parent->parent ? parent->parent->tw.dirtyroom : parent->env->options.dp_limit)); tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->parent ? txn->parent->tw.dirtyroom - : txn->env->options.dp_limit)); + (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); env->txn = txn; tASSERT(parent, parent->cursors[FREE_DBI] == nullptr); - rc = parent->cursors[MAIN_DBI] - ? cursor_shadow(parent->cursors[MAIN_DBI], txn, MAIN_DBI) - : MDBX_SUCCESS; + rc = parent->cursors[MAIN_DBI] ? cursor_shadow(parent->cursors[MAIN_DBI], txn, MAIN_DBI) : MDBX_SUCCESS; if (AUDIT_ENABLED() && ASSERT_ENABLED()) { txn->signature = txn_signature; tASSERT(txn, audit_ex(txn, 0, false) == 0); @@ -1745,8 +1588,7 @@ int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, if (unlikely(rc != MDBX_SUCCESS)) txn_end(txn, TXN_END_FAIL_BEGINCHILD); } else { /* MDBX_TXN_RDONLY */ - txn->dbi_seqs = - ptr_disp(txn->cursors, env->max_dbi * sizeof(txn->cursors[0])); + txn->dbi_seqs = ptr_disp(txn->cursors, env->max_dbi * sizeof(txn->cursors[0])); #if MDBX_ENABLE_DBI_SPARSE txn->dbi_sparse = ptr_disp(txn->dbi_state, -bitmap_bytes); #endif /* MDBX_ENABLE_DBI_SPARSE */ @@ -1761,22 +1603,19 @@ int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, if (flags & (MDBX_TXN_RDONLY_PREPARE - MDBX_TXN_RDONLY)) eASSERT(env, txn->flags == (MDBX_TXN_RDONLY | MDBX_TXN_FINISHED)); else if (flags & MDBX_TXN_RDONLY) - eASSERT(env, (txn->flags & - ~(MDBX_NOSTICKYTHREADS | MDBX_TXN_RDONLY | MDBX_WRITEMAP | - /* Win32: SRWL flag */ txn_shrink_allowed)) == 0); + eASSERT(env, (txn->flags & ~(MDBX_NOSTICKYTHREADS | MDBX_TXN_RDONLY | MDBX_WRITEMAP | + /* Win32: SRWL flag */ txn_shrink_allowed)) == 0); else { - eASSERT(env, (txn->flags & ~(MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP | - txn_shrink_allowed | MDBX_NOMETASYNC | + eASSERT(env, (txn->flags & ~(MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP | txn_shrink_allowed | MDBX_NOMETASYNC | MDBX_SAFE_NOSYNC | MDBX_TXN_SPILLS)) == 0); assert(!txn->tw.spilled.list && !txn->tw.spilled.least_removed); } txn->signature = txn_signature; txn->userctx = context; *ret = txn; - DEBUG("begin txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO - "/%" PRIaPGNO, - txn->txnid, (flags & MDBX_TXN_RDONLY) ? 'r' : 'w', (void *)txn, - (void *)env, txn->dbs[MAIN_DBI].root, txn->dbs[FREE_DBI].root); + DEBUG("begin txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, txn->txnid, + (flags & MDBX_TXN_RDONLY) ? 'r' : 'w', (void *)txn, (void *)env, txn->dbs[MAIN_DBI].root, + txn->dbs[FREE_DBI].root); } return rc; @@ -1811,43 +1650,34 @@ int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) { head_retired = unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired); info->txn_space_limit_soft = pgno2bytes(env, head.ptr_v->geometry.now); info->txn_space_limit_hard = pgno2bytes(env, head.ptr_v->geometry.upper); - info->txn_space_leftover = - pgno2bytes(env, head.ptr_v->geometry.now - - head.ptr_v->geometry.first_unallocated); + info->txn_space_leftover = pgno2bytes(env, head.ptr_v->geometry.now - head.ptr_v->geometry.first_unallocated); } while (unlikely(meta_should_retry(env, &troika))); info->txn_reader_lag = head.txnid - info->txn_id; info->txn_space_dirty = info->txn_space_retired = 0; uint64_t reader_snapshot_pages_retired = 0; if (txn->to.reader && - ((txn->flags & MDBX_TXN_PARKED) == 0 || - safe64_read(&txn->to.reader->tid) != MDBX_TID_TXN_OUSTED) && + ((txn->flags & MDBX_TXN_PARKED) == 0 || safe64_read(&txn->to.reader->tid) != MDBX_TID_TXN_OUSTED) && head_retired > - (reader_snapshot_pages_retired = atomic_load64( - &txn->to.reader->snapshot_pages_retired, mo_Relaxed))) { - info->txn_space_dirty = info->txn_space_retired = pgno2bytes( - env, (pgno_t)(head_retired - reader_snapshot_pages_retired)); + (reader_snapshot_pages_retired = atomic_load64(&txn->to.reader->snapshot_pages_retired, mo_Relaxed))) { + info->txn_space_dirty = info->txn_space_retired = + pgno2bytes(env, (pgno_t)(head_retired - reader_snapshot_pages_retired)); size_t retired_next_reader = 0; lck_t *const lck = env->lck_mmap.lck; if (scan_rlt && info->txn_reader_lag > 1 && lck) { /* find next more recent reader */ txnid_t next_reader = head.txnid; - const size_t snap_nreaders = - atomic_load32(&lck->rdt_length, mo_AcquireRelease); + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); for (size_t i = 0; i < snap_nreaders; ++i) { retry: if (atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease)) { jitter4testing(true); const uint64_t snap_tid = safe64_read(&lck->rdt[i].tid); const txnid_t snap_txnid = safe64_read(&lck->rdt[i].txnid); - const uint64_t snap_retired = atomic_load64( - &lck->rdt[i].snapshot_pages_retired, mo_AcquireRelease); - if (unlikely(snap_retired != - atomic_load64(&lck->rdt[i].snapshot_pages_retired, - mo_Relaxed)) || - snap_txnid != safe64_read(&lck->rdt[i].txnid) || - snap_tid != safe64_read(&lck->rdt[i].tid)) + const uint64_t snap_retired = atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_AcquireRelease); + if (unlikely(snap_retired != atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_Relaxed)) || + snap_txnid != safe64_read(&lck->rdt[i].txnid) || snap_tid != safe64_read(&lck->rdt[i].tid)) goto retry; if (snap_txnid <= txn->txnid) { retired_next_reader = 0; @@ -1856,10 +1686,7 @@ int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) { if (snap_txnid < next_reader && snap_tid >= MDBX_TID_TXN_OUSTED) { next_reader = snap_txnid; retired_next_reader = pgno2bytes( - env, (pgno_t)(snap_retired - - atomic_load64( - &txn->to.reader->snapshot_pages_retired, - mo_Relaxed))); + env, (pgno_t)(snap_retired - atomic_load64(&txn->to.reader->snapshot_pages_retired, mo_Relaxed))); } } } @@ -1870,27 +1697,23 @@ int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) { info->txn_space_limit_soft = pgno2bytes(env, txn->geo.now); info->txn_space_limit_hard = pgno2bytes(env, txn->geo.upper); info->txn_space_retired = - pgno2bytes(env, txn->nested ? (size_t)txn->tw.retired_pages - : MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + pgno2bytes(env, txn->nested ? (size_t)txn->tw.retired_pages : MDBX_PNL_GETSIZE(txn->tw.retired_pages)); info->txn_space_leftover = pgno2bytes(env, txn->tw.dirtyroom); - info->txn_space_dirty = pgno2bytes( - env, txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose - : (txn->tw.writemap_dirty_npages + - txn->tw.writemap_spilled_npages)); + info->txn_space_dirty = + pgno2bytes(env, txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose + : (txn->tw.writemap_dirty_npages + txn->tw.writemap_spilled_npages)); info->txn_reader_lag = INT64_MAX; lck_t *const lck = env->lck_mmap.lck; if (scan_rlt && lck) { txnid_t oldest_snapshot = txn->txnid; - const size_t snap_nreaders = - atomic_load32(&lck->rdt_length, mo_AcquireRelease); + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); if (snap_nreaders) { oldest_snapshot = txn_snapshot_oldest(txn); if (oldest_snapshot == txn->txnid - 1) { /* check if there is at least one reader */ bool exists = false; for (size_t i = 0; i < snap_nreaders; ++i) { - if (atomic_load32(&lck->rdt[i].pid, mo_Relaxed) && - txn->txnid > safe64_read(&lck->rdt[i].txnid)) { + if (atomic_load32(&lck->rdt[i].pid, mo_Relaxed) && txn->txnid > safe64_read(&lck->rdt[i].txnid)) { exists = true; break; } @@ -1906,8 +1729,7 @@ int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) { } MDBX_env *mdbx_txn_env(const MDBX_txn *txn) { - if (unlikely(!txn || txn->signature != txn_signature || - txn->env->signature.weak != env_signature)) + if (unlikely(!txn || txn->signature != txn_signature || txn->env->signature.weak != env_signature)) return nullptr; return txn->env; } @@ -1920,10 +1742,8 @@ uint64_t mdbx_txn_id(const MDBX_txn *txn) { MDBX_txn_flags_t mdbx_txn_flags(const MDBX_txn *txn) { STATIC_ASSERT( - (MDBX_TXN_INVALID & - (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | - MDBX_TXN_HAS_CHILD | txn_gc_drained | txn_shrink_allowed | - txn_rw_begin_flags | txn_ro_begin_flags)) == 0); + (MDBX_TXN_INVALID & (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | MDBX_TXN_HAS_CHILD | + txn_gc_drained | txn_shrink_allowed | txn_rw_begin_flags | txn_ro_begin_flags)) == 0); if (unlikely(!txn || txn->signature != txn_signature)) return MDBX_TXN_INVALID; assert(0 == (int)(txn->flags & MDBX_TXN_INVALID)); @@ -1975,8 +1795,7 @@ int mdbx_txn_abort(MDBX_txn *txn) { if (unlikely(rc != MDBX_SUCCESS)) return rc; - if ((txn->flags & (MDBX_TXN_RDONLY | MDBX_NOSTICKYTHREADS)) == - MDBX_NOSTICKYTHREADS && + if ((txn->flags & (MDBX_TXN_RDONLY | MDBX_NOSTICKYTHREADS)) == MDBX_NOSTICKYTHREADS && unlikely(txn->owner != osal_thread_self())) { mdbx_txn_break(txn); return MDBX_THREAD_MISMATCH; @@ -2028,11 +1847,9 @@ int txn_check_badbits_parked(const MDBX_txn *txn, int bad_bits) { * - получается что транзакцию можно припарковать, потом поломать вызвав * mdbx_txn_break(), но далее любое её использование приведет к завершению * при распарковке. */ - if ((txn->flags & (bad_bits | MDBX_TXN_AUTOUNPARK)) != - (MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK)) + if ((txn->flags & (bad_bits | MDBX_TXN_AUTOUNPARK)) != (MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK)) return MDBX_BAD_TXN; - tASSERT(txn, bad_bits == MDBX_TXN_BLOCKED || - bad_bits == MDBX_TXN_BLOCKED - MDBX_TXN_ERROR); + tASSERT(txn, bad_bits == MDBX_TXN_BLOCKED || bad_bits == MDBX_TXN_BLOCKED - MDBX_TXN_ERROR); return mdbx_txn_unpark((MDBX_txn *)txn, false); } diff --git a/src/unaligned.h b/src/unaligned.h index 722e084a..4085b51d 100644 --- a/src/unaligned.h +++ b/src/unaligned.h @@ -6,22 +6,17 @@ /*------------------------------------------------------------------------------ * Unaligned access */ -MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t -field_alignment(size_t alignment_baseline, size_t field_offset) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t field_alignment(size_t alignment_baseline, + size_t field_offset) { size_t merge = alignment_baseline | (size_t)field_offset; return merge & -(int)merge; } /* read-thunk for UB-sanitizer */ -MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t -peek_u8(const uint8_t *__restrict ptr) { - return *ptr; -} +MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t peek_u8(const uint8_t *__restrict ptr) { return *ptr; } /* write-thunk for UB-sanitizer */ -static inline void poke_u8(uint8_t *__restrict ptr, const uint8_t v) { - *ptr = v; -} +static inline void poke_u8(uint8_t *__restrict ptr, const uint8_t v) { *ptr = v; } static inline void *bcopy_2(void *__restrict dst, const void *__restrict src) { uint8_t *__restrict d = (uint8_t *)dst; @@ -31,8 +26,7 @@ static inline void *bcopy_2(void *__restrict dst, const void *__restrict src) { return d; } -static inline void *bcopy_4(void *const __restrict dst, - const void *const __restrict src) { +static inline void *bcopy_4(void *const __restrict dst, const void *const __restrict src) { uint8_t *__restrict d = (uint8_t *)dst; const uint8_t *__restrict s = (uint8_t *)src; d[0] = s[0]; @@ -42,8 +36,7 @@ static inline void *bcopy_4(void *const __restrict dst, return d; } -static inline void *bcopy_8(void *const __restrict dst, - const void *const __restrict src) { +static inline void *bcopy_8(void *const __restrict dst, const void *const __restrict src) { uint8_t *__restrict d = (uint8_t *)dst; const uint8_t *__restrict s = (uint8_t *)src; d[0] = s[0]; @@ -57,14 +50,13 @@ static inline void *bcopy_8(void *const __restrict dst, return d; } -MDBX_NOTHROW_PURE_FUNCTION static inline uint16_t -unaligned_peek_u16(const size_t expected_alignment, const void *const ptr) { +MDBX_NOTHROW_PURE_FUNCTION static inline uint16_t unaligned_peek_u16(const size_t expected_alignment, + const void *const ptr) { assert((uintptr_t)ptr % expected_alignment == 0); if (MDBX_UNALIGNED_OK >= 2 || (expected_alignment % sizeof(uint16_t)) == 0) return *(const uint16_t *)ptr; else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) return *(const __unaligned uint16_t *)ptr; #else uint16_t v; @@ -74,15 +66,12 @@ unaligned_peek_u16(const size_t expected_alignment, const void *const ptr) { } } -static inline void unaligned_poke_u16(const size_t expected_alignment, - void *const __restrict ptr, - const uint16_t v) { +static inline void unaligned_poke_u16(const size_t expected_alignment, void *const __restrict ptr, const uint16_t v) { assert((uintptr_t)ptr % expected_alignment == 0); if (MDBX_UNALIGNED_OK >= 2 || (expected_alignment % sizeof(v)) == 0) *(uint16_t *)ptr = v; else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) *((uint16_t __unaligned *)ptr) = v; #else bcopy_2((uint8_t *)ptr, (const uint8_t *)&v); @@ -90,21 +79,17 @@ static inline void unaligned_poke_u16(const size_t expected_alignment, } } -MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t -unaligned_peek_u32(const size_t expected_alignment, - const void *const __restrict ptr) { +MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t unaligned_peek_u32(const size_t expected_alignment, + const void *const __restrict ptr) { assert((uintptr_t)ptr % expected_alignment == 0); if (MDBX_UNALIGNED_OK >= 4 || (expected_alignment % sizeof(uint32_t)) == 0) return *(const uint32_t *)ptr; else if ((expected_alignment % sizeof(uint16_t)) == 0) { - const uint16_t lo = - ((const uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__]; - const uint16_t hi = - ((const uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__]; + const uint16_t lo = ((const uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__]; + const uint16_t hi = ((const uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__]; return lo | (uint32_t)hi << 16; } else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) return *(const __unaligned uint32_t *)ptr; #else uint32_t v; @@ -114,19 +99,15 @@ unaligned_peek_u32(const size_t expected_alignment, } } -static inline void unaligned_poke_u32(const size_t expected_alignment, - void *const __restrict ptr, - const uint32_t v) { +static inline void unaligned_poke_u32(const size_t expected_alignment, void *const __restrict ptr, const uint32_t v) { assert((uintptr_t)ptr % expected_alignment == 0); if (MDBX_UNALIGNED_OK >= 4 || (expected_alignment % sizeof(v)) == 0) *(uint32_t *)ptr = v; else if ((expected_alignment % sizeof(uint16_t)) == 0) { ((uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint16_t)v; - ((uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] = - (uint16_t)(v >> 16); + ((uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] = (uint16_t)(v >> 16); } else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) *((uint32_t __unaligned *)ptr) = v; #else bcopy_4((uint8_t *)ptr, (const uint8_t *)&v); @@ -134,21 +115,17 @@ static inline void unaligned_poke_u32(const size_t expected_alignment, } } -MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t -unaligned_peek_u64(const size_t expected_alignment, - const void *const __restrict ptr) { +MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t unaligned_peek_u64(const size_t expected_alignment, + const void *const __restrict ptr) { assert((uintptr_t)ptr % expected_alignment == 0); if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(uint64_t)) == 0) return *(const uint64_t *)ptr; else if ((expected_alignment % sizeof(uint32_t)) == 0) { - const uint32_t lo = - ((const uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__]; - const uint32_t hi = - ((const uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__]; + const uint32_t lo = ((const uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__]; + const uint32_t hi = ((const uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__]; return lo | (uint64_t)hi << 32; } else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) return *(const __unaligned uint64_t *)ptr; #else uint64_t v; @@ -158,40 +135,32 @@ unaligned_peek_u64(const size_t expected_alignment, } } -static inline uint64_t -unaligned_peek_u64_volatile(const size_t expected_alignment, - const volatile void *const __restrict ptr) { +static inline uint64_t unaligned_peek_u64_volatile(const size_t expected_alignment, + const volatile void *const __restrict ptr) { assert((uintptr_t)ptr % expected_alignment == 0); assert(expected_alignment % sizeof(uint32_t) == 0); if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(uint64_t)) == 0) return *(const volatile uint64_t *)ptr; else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) return *(const volatile __unaligned uint64_t *)ptr; #else - const uint32_t lo = ((const volatile uint32_t *) - ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__]; - const uint32_t hi = ((const volatile uint32_t *) - ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__]; + const uint32_t lo = ((const volatile uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__]; + const uint32_t hi = ((const volatile uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__]; return lo | (uint64_t)hi << 32; #endif /* _MSC_VER || __unaligned */ } } -static inline void unaligned_poke_u64(const size_t expected_alignment, - void *const __restrict ptr, - const uint64_t v) { +static inline void unaligned_poke_u64(const size_t expected_alignment, void *const __restrict ptr, const uint64_t v) { assert((uintptr_t)ptr % expected_alignment == 0); if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(v)) == 0) *(uint64_t *)ptr = v; else if ((expected_alignment % sizeof(uint32_t)) == 0) { ((uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint32_t)v; - ((uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] = - (uint32_t)(v >> 32); + ((uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] = (uint32_t)(v >> 32); } else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) *((uint64_t __unaligned *)ptr) = v; #else bcopy_8((uint8_t *)ptr, (const uint8_t *)&v); @@ -199,28 +168,22 @@ static inline void unaligned_poke_u64(const size_t expected_alignment, } } -#define UNALIGNED_PEEK_8(ptr, struct, field) \ - peek_u8(ptr_disp(ptr, offsetof(struct, field))) -#define UNALIGNED_POKE_8(ptr, struct, field, value) \ - poke_u8(ptr_disp(ptr, offsetof(struct, field)), value) +#define UNALIGNED_PEEK_8(ptr, struct, field) peek_u8(ptr_disp(ptr, offsetof(struct, field))) +#define UNALIGNED_POKE_8(ptr, struct, field, value) poke_u8(ptr_disp(ptr, offsetof(struct, field)), value) -#define UNALIGNED_PEEK_16(ptr, struct, field) \ - unaligned_peek_u16(1, ptr_disp(ptr, offsetof(struct, field))) -#define UNALIGNED_POKE_16(ptr, struct, field, value) \ +#define UNALIGNED_PEEK_16(ptr, struct, field) unaligned_peek_u16(1, ptr_disp(ptr, offsetof(struct, field))) +#define UNALIGNED_POKE_16(ptr, struct, field, value) \ unaligned_poke_u16(1, ptr_disp(ptr, offsetof(struct, field)), value) -#define UNALIGNED_PEEK_32(ptr, struct, field) \ - unaligned_peek_u32(1, ptr_disp(ptr, offsetof(struct, field))) -#define UNALIGNED_POKE_32(ptr, struct, field, value) \ +#define UNALIGNED_PEEK_32(ptr, struct, field) unaligned_peek_u32(1, ptr_disp(ptr, offsetof(struct, field))) +#define UNALIGNED_POKE_32(ptr, struct, field, value) \ unaligned_poke_u32(1, ptr_disp(ptr, offsetof(struct, field)), value) -#define UNALIGNED_PEEK_64(ptr, struct, field) \ - unaligned_peek_u64(1, ptr_disp(ptr, offsetof(struct, field))) -#define UNALIGNED_POKE_64(ptr, struct, field, value) \ +#define UNALIGNED_PEEK_64(ptr, struct, field) unaligned_peek_u64(1, ptr_disp(ptr, offsetof(struct, field))) +#define UNALIGNED_POKE_64(ptr, struct, field, value) \ unaligned_poke_u64(1, ptr_disp(ptr, offsetof(struct, field)), value) -MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t -peek_pgno(const void *const __restrict ptr) { +MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t peek_pgno(const void *const __restrict ptr) { if (sizeof(pgno_t) == sizeof(uint32_t)) return (pgno_t)unaligned_peek_u32(1, ptr); else if (sizeof(pgno_t) == sizeof(uint64_t)) diff --git a/src/utils.c b/src/utils.c index 317b3dd7..05b7a21e 100644 --- a/src/utils.c +++ b/src/utils.c @@ -3,10 +3,8 @@ #include "internals.h" -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL unsigned -log2n_powerof2(size_t value_uintptr) { - assert(value_uintptr > 0 && value_uintptr < INT32_MAX && - is_powerof2(value_uintptr)); +MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL unsigned log2n_powerof2(size_t value_uintptr) { + assert(value_uintptr > 0 && value_uintptr < INT32_MAX && is_powerof2(value_uintptr)); assert((value_uintptr & -(intptr_t)value_uintptr) == value_uintptr); const uint32_t value_uint32 = (uint32_t)value_uintptr; #if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctz) @@ -18,9 +16,8 @@ log2n_powerof2(size_t value_uintptr) { _BitScanForward(&index, value_uint32); return index; #else - static const uint8_t debruijn_ctz32[32] = { - 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, - 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; + static const uint8_t debruijn_ctz32[32] = {0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; return debruijn_ctz32[(uint32_t)(value_uint32 * 0x077CB531ul) >> 27]; #endif } diff --git a/src/utils.h b/src/utils.h index 9f51099a..85563b09 100644 --- a/src/utils.h +++ b/src/utils.h @@ -27,46 +27,36 @@ /* Pointer distance as signed number of bytes */ #define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) -#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_POISON_MEMORY_REGION(addr, size); \ +#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_POISON_MEMORY_REGION(addr, size); \ } while (0) -#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_UNPOISON_MEMORY_REGION(addr, size); \ +#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_UNPOISON_MEMORY_REGION(addr, size); \ } while (0) -MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t -branchless_abs(intptr_t value) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t branchless_abs(intptr_t value) { assert(value > INT_MIN); - const size_t expanded_sign = - (size_t)(value >> (sizeof(value) * CHAR_BIT - 1)); + const size_t expanded_sign = (size_t)(value >> (sizeof(value) * CHAR_BIT - 1)); return ((size_t)value + expanded_sign) ^ expanded_sign; } -MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline bool -is_powerof2(size_t x) { - return (x & (x - 1)) == 0; -} +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline bool is_powerof2(size_t x) { return (x & (x - 1)) == 0; } -MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t -floor_powerof2(size_t value, size_t granularity) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t floor_powerof2(size_t value, size_t granularity) { assert(is_powerof2(granularity)); return value & ~(granularity - 1); } -MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t -ceil_powerof2(size_t value, size_t granularity) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t ceil_powerof2(size_t value, size_t granularity) { return floor_powerof2(value + granularity - 1, granularity); } -MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL unsigned -log2n_powerof2(size_t value_uintptr); +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL unsigned log2n_powerof2(size_t value_uintptr); MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL uint64_t rrxmrrxmsx_0(uint64_t v); @@ -75,8 +65,7 @@ struct monotime_cache { int expire_countdown; }; -MDBX_MAYBE_UNUSED static inline uint64_t -monotime_since_cached(uint64_t begin_timestamp, struct monotime_cache *cache) { +MDBX_MAYBE_UNUSED static inline uint64_t monotime_since_cached(uint64_t begin_timestamp, struct monotime_cache *cache) { if (cache->expire_countdown) cache->expire_countdown -= 1; else { diff --git a/src/version.c.in b/src/version.c.in index 19513c68..5c585eae 100644 --- a/src/version.c.in +++ b/src/version.c.in @@ -3,8 +3,7 @@ #include "internals.h" -#if MDBX_VERSION_MAJOR != ${MDBX_VERSION_MAJOR} || \ - MDBX_VERSION_MINOR != ${MDBX_VERSION_MINOR} +#if MDBX_VERSION_MAJOR != ${MDBX_VERSION_MAJOR} || MDBX_VERSION_MINOR != ${MDBX_VERSION_MINOR} #error "API version mismatch! Had `git fetch --tags` done?" #endif @@ -18,8 +17,7 @@ __dll_export #endif #ifdef __attribute_externally_visible__ __attribute_externally_visible__ -#elif (defined(__GNUC__) && !defined(__clang__)) || \ - __has_attribute(__externally_visible__) +#elif (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__externally_visible__) __attribute__((__externally_visible__)) #endif const struct MDBX_version_info mdbx_version = { @@ -29,8 +27,7 @@ __dll_export ${MDBX_VERSION_TWEAK}, "@MDBX_VERSION_PRERELEASE@", /* pre-release suffix of SemVer @MDBX_VERSION_PURE@ */ - {"@MDBX_GIT_TIMESTAMP@", "@MDBX_GIT_TREE@", "@MDBX_GIT_COMMIT@", - "@MDBX_GIT_DESCRIBE@"}, + {"@MDBX_GIT_TIMESTAMP@", "@MDBX_GIT_TREE@", "@MDBX_GIT_COMMIT@", "@MDBX_GIT_DESCRIBE@"}, sourcery}; __dll_export @@ -41,8 +38,7 @@ __dll_export #endif #ifdef __attribute_externally_visible__ __attribute_externally_visible__ -#elif (defined(__GNUC__) && !defined(__clang__)) || \ - __has_attribute(__externally_visible__) +#elif (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__externally_visible__) __attribute__((__externally_visible__)) #endif const char *const mdbx_sourcery_anchor = sourcery; diff --git a/src/walk.c b/src/walk.c index 265e7da8..585c350d 100644 --- a/src/walk.c +++ b/src/walk.c @@ -41,19 +41,16 @@ static page_type_t walk_subpage_type(const page_t *sp) { } /* Depth-first tree traversal. */ -__cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, - txnid_t parent_txnid) { +__cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, txnid_t parent_txnid) { assert(pgno != P_INVALID); page_t *mp = nullptr; int err = page_get(ctx->cursor, pgno, &mp, parent_txnid); const page_type_t type = walk_page_type(mp); const size_t nentries = mp ? page_numkeys(mp) : 0; - size_t header_size = - (mp && !is_dupfix_leaf(mp)) ? PAGEHDRSZ + mp->lower : PAGEHDRSZ; + size_t header_size = (mp && !is_dupfix_leaf(mp)) ? PAGEHDRSZ + mp->lower : PAGEHDRSZ; size_t payload_size = 0; - size_t unused_size = - (mp ? page_room(mp) : ctx->txn->env->ps - header_size) - payload_size; + size_t unused_size = (mp ? page_room(mp) : ctx->txn->env->ps - header_size) - payload_size; size_t align_bytes = 0; for (size_t i = 0; err == MDBX_SUCCESS && i < nentries; ++i) { @@ -89,12 +86,10 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, assert(err == MDBX_SUCCESS); pgr_t lp = page_get_large(ctx->cursor, large_pgno, mp->txnid); - const size_t npages = - ((err = lp.err) == MDBX_SUCCESS) ? lp.page->pages : 1; + const size_t npages = ((err = lp.err) == MDBX_SUCCESS) ? lp.page->pages : 1; const size_t pagesize = pgno2bytes(ctx->txn->env, npages); const size_t over_unused = pagesize - over_payload - over_header; - const int rc = ctx->visitor(large_pgno, npages, ctx->userctx, ctx->deep, - tbl, pagesize, page_large, err, 1, + const int rc = ctx->visitor(large_pgno, npages, ctx->userctx, ctx->deep, tbl, pagesize, page_large, err, 1, over_payload, over_header, over_unused); if (unlikely(rc != MDBX_SUCCESS)) return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; @@ -104,8 +99,7 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, case N_TREE /* sub-db */: { if (unlikely(node_data_size != sizeof(tree_t))) { - ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid table node size", (unsigned)node_data_size); + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid table node size", (unsigned)node_data_size); assert(err == MDBX_CORRUPTED); err = MDBX_CORRUPTED; } @@ -115,8 +109,7 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, case N_TREE | N_DUP /* dupsorted sub-tree */: if (unlikely(node_data_size != sizeof(tree_t))) { - ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid sub-tree node size", (unsigned)node_data_size); + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-tree node size", (unsigned)node_data_size); assert(err == MDBX_CORRUPTED); err = MDBX_CORRUPTED; } @@ -126,8 +119,7 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, case N_DUP /* short sub-page */: { if (unlikely(node_data_size <= PAGEHDRSZ || (node_data_size & 1))) { - ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid sub-page node size", (unsigned)node_data_size); + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-page node size", (unsigned)node_data_size); assert(err == MDBX_CORRUPTED); err = MDBX_CORRUPTED; break; @@ -137,14 +129,12 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, const page_type_t subtype = walk_subpage_type(sp); const size_t nsubkeys = page_numkeys(sp); if (unlikely(subtype == page_sub_broken)) { - ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid sub-page flags", sp->flags); + ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-page flags", sp->flags); assert(err == MDBX_CORRUPTED); err = MDBX_CORRUPTED; } - size_t subheader_size = - is_dupfix_leaf(sp) ? PAGEHDRSZ : PAGEHDRSZ + sp->lower; + size_t subheader_size = is_dupfix_leaf(sp) ? PAGEHDRSZ : PAGEHDRSZ + sp->lower; size_t subunused_size = page_room(sp); size_t subpayload_size = 0; size_t subalign_bytes = 0; @@ -161,18 +151,15 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, subpayload_size += subnode_size; subalign_bytes += subnode_size & 1; if (unlikely(node_flags(subnode) != 0)) { - ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "unexpected sub-node flags", node_flags(subnode)); + ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, "unexpected sub-node flags", node_flags(subnode)); assert(err == MDBX_CORRUPTED); err = MDBX_CORRUPTED; } } } - const int rc = - ctx->visitor(pgno, 0, ctx->userctx, ctx->deep + 1, tbl, - node_data_size, subtype, err, nsubkeys, subpayload_size, - subheader_size, subunused_size + subalign_bytes); + const int rc = ctx->visitor(pgno, 0, ctx->userctx, ctx->deep + 1, tbl, node_data_size, subtype, err, nsubkeys, + subpayload_size, subheader_size, subunused_size + subalign_bytes); if (unlikely(rc != MDBX_SUCCESS)) return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; header_size += subheader_size; @@ -182,16 +169,14 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, } break; default: - ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid node flags", node_flags(node)); + ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid node flags", node_flags(node)); assert(err == MDBX_CORRUPTED); err = MDBX_CORRUPTED; } } - const int rc = ctx->visitor( - pgno, 1, ctx->userctx, ctx->deep, tbl, ctx->txn->env->ps, type, err, - nentries, payload_size, header_size, unused_size + align_bytes); + const int rc = ctx->visitor(pgno, 1, ctx->userctx, ctx->deep, tbl, ctx->txn->env->ps, type, err, nentries, + payload_size, header_size, unused_size + align_bytes); if (unlikely(rc != MDBX_SUCCESS)) return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; @@ -220,8 +205,7 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, case N_TREE /* sub-db */: if (unlikely(node_ds(node) != sizeof(tree_t))) { - ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid sub-tree node size", (unsigned)node_ds(node)); + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-tree node size", (unsigned)node_ds(node)); assert(err == MDBX_CORRUPTED); err = MDBX_CORRUPTED; } else { @@ -238,8 +222,8 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, case N_TREE | N_DUP /* dupsorted sub-tree */: if (unlikely(node_ds(node) != sizeof(tree_t))) { - ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, - "invalid dupsort sub-tree node size", (unsigned)node_ds(node)); + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid dupsort sub-tree node size", + (unsigned)node_ds(node)); assert(err == MDBX_CORRUPTED); err = MDBX_CORRUPTED; } else { @@ -248,8 +232,7 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, assert(err == MDBX_SUCCESS); err = cursor_dupsort_setup(ctx->cursor, node, mp); if (likely(err == MDBX_SUCCESS)) { - assert(ctx->cursor->subcur == - &container_of(ctx->cursor, cursor_couple_t, outer)->inner); + assert(ctx->cursor->subcur == &container_of(ctx->cursor, cursor_couple_t, outer)->inner); ctx->cursor = &ctx->cursor->subcur->cursor; ctx->deep += 1; tbl->nested = &aligned_db; @@ -257,8 +240,7 @@ __cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, tbl->nested = nullptr; ctx->deep -= 1; subcur_t *inner_xcursor = container_of(ctx->cursor, subcur_t, cursor); - cursor_couple_t *couple = - container_of(inner_xcursor, cursor_couple_t, inner); + cursor_couple_t *couple = container_of(inner_xcursor, cursor_couple_t, inner); ctx->cursor = &couple->outer; } } @@ -280,30 +262,24 @@ __cold static int walk_tbl(walk_ctx_t *ctx, walk_tbl_t *tbl) { if (unlikely(rc != MDBX_SUCCESS)) return rc; - const uint8_t cursor_checking = (ctx->options & dont_check_keys_ordering) - ? z_pagecheck | z_ignord - : z_pagecheck; + const uint8_t cursor_checking = (ctx->options & dont_check_keys_ordering) ? z_pagecheck | z_ignord : z_pagecheck; couple.outer.checking |= cursor_checking; couple.inner.cursor.checking |= cursor_checking; couple.outer.next = ctx->cursor; couple.outer.top_and_flags = z_disable_tree_search_fastpath; ctx->cursor = &couple.outer; - rc = walk_pgno(ctx, tbl, db->root, - db->mod_txnid ? db->mod_txnid : ctx->txn->txnid); + rc = walk_pgno(ctx, tbl, db->root, db->mod_txnid ? db->mod_txnid : ctx->txn->txnid); ctx->cursor = couple.outer.next; return rc; } -__cold int walk_pages(MDBX_txn *txn, walk_func *visitor, void *user, - walk_options_t options) { +__cold int walk_pages(MDBX_txn *txn, walk_func *visitor, void *user, walk_options_t options) { int rc = check_txn(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return rc; - walk_ctx_t ctx = { - .txn = txn, .userctx = user, .visitor = visitor, .options = options}; - walk_tbl_t tbl = {.name = {.iov_base = MDBX_CHK_GC}, - .internal = &txn->dbs[FREE_DBI]}; + walk_ctx_t ctx = {.txn = txn, .userctx = user, .visitor = visitor, .options = options}; + walk_tbl_t tbl = {.name = {.iov_base = MDBX_CHK_GC}, .internal = &txn->dbs[FREE_DBI]}; rc = walk_tbl(&ctx, &tbl); if (!MDBX_IS_ERROR(rc)) { tbl.name.iov_base = MDBX_CHK_MAIN; diff --git a/src/walk.h b/src/walk.h index ef79c70b..be6dc507 100644 --- a/src/walk.h +++ b/src/walk.h @@ -10,14 +10,11 @@ typedef struct walk_tbl { tree_t *internal, *nested; } walk_tbl_t; -typedef int walk_func(const size_t pgno, const unsigned number, void *const ctx, - const int deep, const walk_tbl_t *table, - const size_t page_size, const page_type_t page_type, - const MDBX_error_t err, const size_t nentries, - const size_t payload_bytes, const size_t header_bytes, - const size_t unused_bytes); +typedef int walk_func(const size_t pgno, const unsigned number, void *const ctx, const int deep, + const walk_tbl_t *table, const size_t page_size, const page_type_t page_type, + const MDBX_error_t err, const size_t nentries, const size_t payload_bytes, + const size_t header_bytes, const size_t unused_bytes); typedef enum walk_options { dont_check_keys_ordering = 1 } walk_options_t; -MDBX_INTERNAL int walk_pages(MDBX_txn *txn, walk_func *visitor, void *user, - walk_options_t options); +MDBX_INTERNAL int walk_pages(MDBX_txn *txn, walk_func *visitor, void *user, walk_options_t options); diff --git a/src/windows-import.c b/src/windows-import.c index a401014c..ee0ea9c8 100644 --- a/src/windows-import.c +++ b/src/windows-import.c @@ -9,9 +9,7 @@ // Stub for slim read-write lock // Portion Copyright (C) 1995-2002 Brad Wilson -static void WINAPI stub_srwlock_Init(osal_srwlock_t *srwl) { - srwl->readerCount = srwl->writerCount = 0; -} +static void WINAPI stub_srwlock_Init(osal_srwlock_t *srwl) { srwl->readerCount = srwl->writerCount = 0; } static void WINAPI stub_srwlock_AcquireShared(osal_srwlock_t *srwl) { while (true) { @@ -76,8 +74,7 @@ static void WINAPI stub_srwlock_ReleaseExclusive(osal_srwlock_t *srwl) { static uint64_t WINAPI stub_GetTickCount64(void) { LARGE_INTEGER Counter, Frequency; - return (QueryPerformanceFrequency(&Frequency) && - QueryPerformanceCounter(&Counter)) + return (QueryPerformanceFrequency(&Frequency) && QueryPerformanceCounter(&Counter)) ? Counter.QuadPart * 1000ul / Frequency.QuadPart : 0; } @@ -91,8 +88,7 @@ struct libmdbx_imports imports; #pragma GCC diagnostic ignored "-Wcast-function-type" #endif /* GCC/MINGW */ -#define MDBX_IMPORT(HANDLE, ENTRY) \ - imports.ENTRY = (MDBX_##ENTRY)GetProcAddress(HANDLE, #ENTRY) +#define MDBX_IMPORT(HANDLE, ENTRY) imports.ENTRY = (MDBX_##ENTRY)GetProcAddress(HANDLE, #ENTRY) void windows_import(void) { const HINSTANCE hNtdll = GetModuleHandleA("ntdll.dll"); @@ -121,20 +117,13 @@ void windows_import(void) { } const osal_srwlock_t_function srwlock_init = - (osal_srwlock_t_function)(hKernel32dll - ? GetProcAddress(hKernel32dll, - "InitializeSRWLock") - : nullptr); + (osal_srwlock_t_function)(hKernel32dll ? GetProcAddress(hKernel32dll, "InitializeSRWLock") : nullptr); if (srwlock_init) { imports.srwl_Init = srwlock_init; - imports.srwl_AcquireShared = (osal_srwlock_t_function)GetProcAddress( - hKernel32dll, "AcquireSRWLockShared"); - imports.srwl_ReleaseShared = (osal_srwlock_t_function)GetProcAddress( - hKernel32dll, "ReleaseSRWLockShared"); - imports.srwl_AcquireExclusive = (osal_srwlock_t_function)GetProcAddress( - hKernel32dll, "AcquireSRWLockExclusive"); - imports.srwl_ReleaseExclusive = (osal_srwlock_t_function)GetProcAddress( - hKernel32dll, "ReleaseSRWLockExclusive"); + imports.srwl_AcquireShared = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "AcquireSRWLockShared"); + imports.srwl_ReleaseShared = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "ReleaseSRWLockShared"); + imports.srwl_AcquireExclusive = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "AcquireSRWLockExclusive"); + imports.srwl_ReleaseExclusive = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "ReleaseSRWLockExclusive"); } else { imports.srwl_Init = stub_srwlock_Init; imports.srwl_AcquireShared = stub_srwlock_AcquireShared; diff --git a/src/windows-import.h b/src/windows-import.h index b7c461f9..0362e9b6 100644 --- a/src/windows-import.h +++ b/src/windows-import.h @@ -59,32 +59,27 @@ typedef struct _FILE_REMOTE_PROTOCOL_INFO { #endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */ -typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(_In_ HANDLE hFile, + _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( - _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, - _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, - _Out_opt_ LPDWORD lpMaximumComponentLength, - _Out_opt_ LPDWORD lpFileSystemFlags, - _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); + _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, _In_ DWORD nVolumeNameSize, + _Out_opt_ LPDWORD lpVolumeSerialNumber, _Out_opt_ LPDWORD lpMaximumComponentLength, + _Out_opt_ LPDWORD lpFileSystemFlags, _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); -typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, - _Out_ LPWSTR lpszFilePath, - _In_ DWORD cchFilePath, - _In_ DWORD dwFlags); +typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, _Out_ LPWSTR lpszFilePath, + _In_ DWORD cchFilePath, _In_ DWORD dwFlags); -typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(_In_ HANDLE hFile, + _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( - IN HANDLE FileHandle, IN OUT HANDLE Event, - IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, - OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, - IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, - OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); +typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(IN HANDLE FileHandle, IN OUT HANDLE Event, + IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, + OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, + IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, + OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); @@ -95,27 +90,21 @@ typedef struct _WIN32_MEMORY_RANGE_ENTRY { } WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; #endif /* Windows 8.x */ -typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( - HANDLE hProcess, ULONG_PTR NumberOfEntries, - PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); +typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(HANDLE hProcess, ULONG_PTR NumberOfEntries, + PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; -typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, - IN PLARGE_INTEGER NewSectionSize); +typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, IN PLARGE_INTEGER NewSectionSize); -typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, - LPCSTR lpValue, DWORD dwFlags, - LPDWORD pdwType, PVOID pvData, - LPDWORD pcbData); +typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, LPCSTR lpValue, DWORD dwFlags, LPDWORD pdwType, + PVOID pvData, LPDWORD pcbData); typedef long(WINAPI *MDBX_CoCreateGuid)(bin128_t *guid); NTSYSAPI ULONG RtlRandomEx(PULONG Seed); -typedef BOOL(WINAPI *MDBX_SetFileIoOverlappedRange)(HANDLE FileHandle, - PUCHAR OverlappedRangeStart, - ULONG Length); +typedef BOOL(WINAPI *MDBX_SetFileIoOverlappedRange)(HANDLE FileHandle, PUCHAR OverlappedRangeStart, ULONG Length); struct libmdbx_imports { osal_srwlock_t_function srwl_Init; diff --git a/test/append.c++ b/test/append.c++ index 064dbf3d..59f6d1ca 100644 --- a/test/append.c++ +++ b/test/append.c++ @@ -5,16 +5,14 @@ class testcase_append : public testcase { public: - testcase_append(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid) {} + testcase_append(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} bool run() override; static bool review_params(actor_params ¶ms, unsigned space_id) { if (!testcase::review_params(params, space_id)) return false; const bool ordered = !flipcoin_x3(); - log_notice("the '%s' key-generation mode is selected", - ordered ? "ordered/linear" : "unordered/non-linear"); + log_notice("the '%s' key-generation mode is selected", ordered ? "ordered/linear" : "unordered/non-linear"); if (ordered && !params.make_keygen_linear()) return false; return true; @@ -37,13 +35,10 @@ bool testcase_append::run() { keyvalue_maker.setup(config.params, 0 /* thread_number */); /* LY: тест наполнения таблиц в append-режиме, * при котором записи добавляются строго в конец (в порядке сортировки) */ - const MDBX_put_flags_t flags = - reverse - ? ((config.params.table_flags & MDBX_DUPSORT) ? MDBX_UPSERT - : MDBX_NOOVERWRITE) - : ((config.params.table_flags & MDBX_DUPSORT) - ? (flipcoin() ? MDBX_APPEND | MDBX_APPENDDUP : MDBX_APPENDDUP) - : MDBX_APPEND); + const MDBX_put_flags_t flags = reverse ? ((config.params.table_flags & MDBX_DUPSORT) ? MDBX_UPSERT : MDBX_NOOVERWRITE) + : ((config.params.table_flags & MDBX_DUPSORT) + ? (flipcoin() ? MDBX_APPEND | MDBX_APPENDDUP : MDBX_APPENDDUP) + : MDBX_APPEND); key = keygen::alloc(config.params.keylen_max); data = keygen::alloc(config.params.datalen_max); @@ -59,11 +54,9 @@ bool testcase_append::run() { simple_checksum committed_inserted_checksum = inserted_checksum; while (should_continue()) { const keygen::serial_t serial = serial_count; - const bool turn_key = (config.params.table_flags & MDBX_DUPSORT) == 0 || - flipcoin_n(config.params.keygen.split); - if (turn_key - ? !keyvalue_maker.increment_key_part(serial_count, reverse ? -1 : 1) - : !keyvalue_maker.increment(serial_count, reverse ? -1 : 1)) { + const bool turn_key = (config.params.table_flags & MDBX_DUPSORT) == 0 || flipcoin_n(config.params.keygen.split); + if (turn_key ? !keyvalue_maker.increment_key_part(serial_count, reverse ? -1 : 1) + : !keyvalue_maker.increment(serial_count, reverse ? -1 : 1)) { // дошли до границы пространства ключей break; } @@ -106,8 +99,7 @@ bool testcase_append::run() { break; case MDBX_APPENDDUP: assert((config.params.table_flags & MDBX_DUPSORT) != 0); - expect_key_mismatch = - mdbx_cmp(txn_guard.get(), dbi, &key->value, &ge_key) == 0; + expect_key_mismatch = mdbx_cmp(txn_guard.get(), dbi, &key->value, &ge_key) == 0; break; } } else if (err == MDBX_NOTFOUND /* all pair are less than */) { @@ -152,10 +144,9 @@ bool testcase_append::run() { const auto insertion_result = speculum.insert(item); if (!insertion_result.second) { char dump_key[32], dump_value[32]; - log_error( - "speculum.append: unexpected %s {%s, %s}", "MDBX_SUCCESS", - mdbx_dump_val(&key->value, dump_key, sizeof(dump_key)), - mdbx_dump_val(&data->value, dump_value, sizeof(dump_value))); + log_error("speculum.append: unexpected %s {%s, %s}", "MDBX_SUCCESS", + mdbx_dump_val(&key->value, dump_key, sizeof(dump_key)), + mdbx_dump_val(&data->value, dump_value, sizeof(dump_value))); return false; } } @@ -199,8 +190,7 @@ bool testcase_append::run() { cursor_renew(); MDBX_val check_key, check_data; - err = mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, - reverse ? MDBX_LAST : MDBX_FIRST); + err = mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, reverse ? MDBX_LAST : MDBX_FIRST); if (likely(inserted_number)) { if (unlikely(err != MDBX_SUCCESS)) failure_perror("mdbx_cursor_get(MDBX_FIRST)", err); @@ -213,19 +203,16 @@ bool testcase_append::run() { read_checksum.push((uint32_t)read_count, check_key); read_checksum.push(10639, check_data); - err = mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, - reverse ? MDBX_PREV : MDBX_NEXT); + err = mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, reverse ? MDBX_PREV : MDBX_NEXT); } if (unlikely(err != MDBX_NOTFOUND)) failure_perror("mdbx_cursor_get(MDBX_NEXT) != EOF", err); if (unlikely(read_count != inserted_number)) - failure("read_count(%" PRIu64 ") != inserted_number(%" PRIu64 ")", - read_count, inserted_number); + failure("read_count(%" PRIu64 ") != inserted_number(%" PRIu64 ")", read_count, inserted_number); - if (unlikely(read_checksum.value != inserted_checksum.value) && - !keyvalue_maker.is_unordered()) + if (unlikely(read_checksum.value != inserted_checksum.value) && !keyvalue_maker.is_unordered()) failure("read_checksum(0x%016" PRIu64 ") " "!= inserted_checksum(0x%016" PRIu64 ")", read_checksum.value, inserted_checksum.value); diff --git a/test/base.h++ b/test/base.h++ index a2d2cfbd..21af13a2 100644 --- a/test/base.h++ +++ b/test/base.h++ @@ -7,12 +7,12 @@ #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - mode specified; termination on exception \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + mode specified; termination on exception \ is not guaranteed. Specify /EHsc */ #endif /* _MSC_VER (warnings) */ @@ -71,24 +71,22 @@ #ifdef _MSC_VER #pragma warning(pop) -#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ +#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ struct/union */ #pragma warning(disable : 4127) /* conditional expression is constant */ #if _MSC_VER < 1900 -#pragma warning(disable : 4510) /* default constructor could \ +#pragma warning(disable : 4510) /* default constructor could \ not be generated */ -#pragma warning(disable : 4512) /* assignment operator could \ +#pragma warning(disable : 4512) /* assignment operator could \ not be generated */ #pragma warning(disable : 4610) /* user-defined constructor required */ #ifndef snprintf -#define snprintf(buffer, buffer_size, format, ...) \ - _snprintf_s(buffer, buffer_size, _TRUNCATE, format, __VA_ARGS__) +#define snprintf(buffer, buffer_size, format, ...) _snprintf_s(buffer, buffer_size, _TRUNCATE, format, __VA_ARGS__) #endif #ifndef vsnprintf -#define vsnprintf(buffer, buffer_size, format, args) \ - _vsnprintf_s(buffer, buffer_size, _TRUNCATE, format, args) +#define vsnprintf(buffer, buffer_size, format, args) _vsnprintf_s(buffer, buffer_size, _TRUNCATE, format, args) #endif -#pragma warning(disable : 4996) /* 'vsnprintf': This function or variable \ +#pragma warning(disable : 4996) /* 'vsnprintf': This function or variable \ may be unsafe */ #endif #endif /* _MSC_VER */ diff --git a/test/cases.c++ b/test/cases.c++ index 3d380f61..fdb99485 100644 --- a/test/cases.c++ +++ b/test/cases.c++ @@ -14,9 +14,7 @@ bool registry::add(const record *item) { auto const singleton = instance(); assert(singleton->name2id.count(std::string(item->name)) == 0); assert(singleton->id2record.count(item->id) == 0); - if (singleton->name2id.count(std::string(item->name)) + - singleton->id2record.count(item->id) == - 0) { + if (singleton->name2id.count(std::string(item->name)) + singleton->id2record.count(item->id) == 0) { singleton->name2id[std::string(item->name)] = item; singleton->id2record[item->id] = item; return true; @@ -24,28 +22,24 @@ bool registry::add(const record *item) { return false; } -testcase *registry::create_actor(const actor_config &config, - const mdbx_pid_t pid) { +testcase *registry::create_actor(const actor_config &config, const mdbx_pid_t pid) { return instance()->id2record.at(config.testcase)->constructor(config, pid); } -bool registry::review_actor_params(const actor_testcase id, - actor_params ¶ms, - const unsigned space_id) { +bool registry::review_actor_params(const actor_testcase id, actor_params ¶ms, const unsigned space_id) { return instance()->id2record.at(id)->review_params(params, space_id); } //----------------------------------------------------------------------------- -void configure_actor(unsigned &last_space_id, const actor_testcase testcase, - const char *space_id_cstr, actor_params params) { +void configure_actor(unsigned &last_space_id, const actor_testcase testcase, const char *space_id_cstr, + actor_params params) { unsigned wait4id = 0; if (params.waitfor_nops) { for (auto i = global::actors.rbegin(); i != global::actors.rend(); ++i) { if (i->is_waitable(params.waitfor_nops)) { if (i->signal_nops && i->signal_nops != params.waitfor_nops) - failure("Previous waitable actor (id=%u) already linked on %u-ops\n", - i->actor_id, i->signal_nops); + failure("Previous waitable actor (id=%u) already linked on %u-ops\n", i->actor_id, i->signal_nops); wait4id = i->actor_id; i->signal_nops = params.waitfor_nops; break; @@ -75,15 +69,12 @@ void configure_actor(unsigned &last_space_id, const actor_testcase testcase, failure("Actor config-review failed for space-id %lu\n", space_id); last_space_id = unsigned(space_id); - log_trace("configure_actor: space %lu for %s", space_id, - testcase2str(testcase)); - global::actors.emplace_back( - actor_config(testcase, params, unsigned(space_id), wait4id)); + log_trace("configure_actor: space %lu for %s", space_id, testcase2str(testcase)); + global::actors.emplace_back(actor_config(testcase, params, unsigned(space_id), wait4id)); global::databases.insert(params.pathname_db); } -void testcase_setup(const char *casename, const actor_params ¶ms, - unsigned &last_space_id) { +void testcase_setup(const char *casename, const actor_params ¶ms, unsigned &last_space_id) { if (strcmp(casename, "basic") == 0) { log_notice(">>> testcase_setup(%s)", casename); configure_actor(last_space_id, ac_nested, nullptr, params); @@ -111,8 +102,7 @@ void keycase_setup(const char *casename, actor_params ¶ms) { params.keygen.keycase = kc_random; // TODO log_notice("<<< keycase_setup(%s): done", casename); - } else if (strcmp(casename, "dashes") == 0 || - strcmp(casename, "aside") == 0) { + } else if (strcmp(casename, "dashes") == 0 || strcmp(casename, "aside") == 0) { log_notice(">>> keycase_setup(%s)", casename); params.keygen.keycase = kc_dashes; // TODO diff --git a/test/chrono.c++ b/test/chrono.c++ index 093d938c..daf285b1 100644 --- a/test/chrono.c++ +++ b/test/chrono.c++ @@ -18,9 +18,7 @@ uint32_t ns2fractional(uint32_t ns) { return uint32_t((uint64_t(ns) << 32) / NSEC_PER_SEC); } -uint32_t fractional2ns(uint32_t fractional) { - return uint32_t((fractional * uint64_t(NSEC_PER_SEC)) >> 32); -} +uint32_t fractional2ns(uint32_t fractional) { return uint32_t((fractional * uint64_t(NSEC_PER_SEC)) >> 32); } #ifndef USEC_PER_SEC #define USEC_PER_SEC 1000000u @@ -51,33 +49,27 @@ uint32_t ms2fractional(uint32_t ms) { return uint32_t((uint64_t(ms) << 32) / MSEC_PER_SEC); } -uint32_t fractional2ms(uint32_t fractional) { - return uint32_t((fractional * uint64_t(MSEC_PER_SEC)) >> 32); -} +uint32_t fractional2ms(uint32_t fractional) { return uint32_t((fractional * uint64_t(MSEC_PER_SEC)) >> 32); } time from_ns(uint64_t ns) { time result; - result.fixedpoint = - ((ns / NSEC_PER_SEC) << 32) | ns2fractional(uint32_t(ns % NSEC_PER_SEC)); + result.fixedpoint = ((ns / NSEC_PER_SEC) << 32) | ns2fractional(uint32_t(ns % NSEC_PER_SEC)); return result; } time from_us(uint64_t us) { time result; - result.fixedpoint = - ((us / USEC_PER_SEC) << 32) | us2fractional(uint32_t(us % USEC_PER_SEC)); + result.fixedpoint = ((us / USEC_PER_SEC) << 32) | us2fractional(uint32_t(us % USEC_PER_SEC)); return result; } time from_ms(uint64_t ms) { time result; - result.fixedpoint = - ((ms / MSEC_PER_SEC) << 32) | ms2fractional(uint32_t(ms % MSEC_PER_SEC)); + result.fixedpoint = ((ms / MSEC_PER_SEC) << 32) | ms2fractional(uint32_t(ms % MSEC_PER_SEC)); return result; } -#if __GNUC_PREREQ(8, 0) && \ - (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) +#if __GNUC_PREREQ(8, 0) && (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-function-type" #endif /* GCC/MINGW */ @@ -88,16 +80,14 @@ time now_realtime() { if (unlikely(!query_time)) { HMODULE hModule = GetModuleHandle(TEXT("kernel32.dll")); if (hModule) - query_time = (void(WINAPI *)(LPFILETIME))GetProcAddress( - hModule, "GetSystemTimePreciseAsFileTime"); + query_time = (void(WINAPI *)(LPFILETIME))GetProcAddress(hModule, "GetSystemTimePreciseAsFileTime"); if (!query_time) query_time = GetSystemTimeAsFileTime; } FILETIME filetime; query_time(&filetime); - uint64_t ns100 = - (uint64_t)filetime.dwHighDateTime << 32 | filetime.dwLowDateTime; + uint64_t ns100 = (uint64_t)filetime.dwHighDateTime << 32 | filetime.dwLowDateTime; return from_ns((ns100 - UINT64_C(116444736000000000)) * 100u); #else struct timespec ts; @@ -115,8 +105,7 @@ time now_monotonic() { if (reciprocal == 0) { if (!QueryPerformanceFrequency(&Frequency)) failure_perror("QueryPerformanceFrequency()", GetLastError()); - reciprocal = (((UINT64_C(1) << 48) + Frequency.QuadPart / 2 + 1) / - Frequency.QuadPart); + reciprocal = (((UINT64_C(1) << 48) + Frequency.QuadPart / 2 + 1) / Frequency.QuadPart); assert(reciprocal); } @@ -138,8 +127,7 @@ time now_monotonic() { #endif } -#if __GNUC_PREREQ(8, 0) && \ - (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) +#if __GNUC_PREREQ(8, 0) && (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) #pragma GCC diagnostic pop #endif /* GCC/MINGW */ diff --git a/test/chrono.h++ b/test/chrono.h++ index 5908509f..0749db8a 100644 --- a/test/chrono.h++ +++ b/test/chrono.h++ @@ -63,12 +63,10 @@ inline time infinite() { return result; } -#if defined(HAVE_TIMESPEC_TV_NSEC) || defined(__timespec_defined) || \ - defined(CLOCK_REALTIME) +#if defined(HAVE_TIMESPEC_TV_NSEC) || defined(__timespec_defined) || defined(CLOCK_REALTIME) inline time from_timespec(const struct timespec &ts) { time result; - result.fixedpoint = - ((uint64_t)ts.tv_sec << 32) | ns2fractional((uint32_t)ts.tv_nsec); + result.fixedpoint = ((uint64_t)ts.tv_sec << 32) | ns2fractional((uint32_t)ts.tv_nsec); return result; } #endif /* HAVE_TIMESPEC_TV_NSEC */ @@ -76,8 +74,7 @@ inline time from_timespec(const struct timespec &ts) { #if defined(HAVE_TIMEVAL_TV_USEC) || defined(_STRUCT_TIMEVAL) inline time from_timeval(const struct timeval &tv) { time result; - result.fixedpoint = - ((uint64_t)tv.tv_sec << 32) | us2fractional((uint32_t)tv.tv_usec); + result.fixedpoint = ((uint64_t)tv.tv_sec << 32) | us2fractional((uint32_t)tv.tv_usec); return result; } #endif /* HAVE_TIMEVAL_TV_USEC */ diff --git a/test/config.c++ b/test/config.c++ index 4732d95b..e35e74f2 100644 --- a/test/config.c++ +++ b/test/config.c++ @@ -9,8 +9,8 @@ namespace config { -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - const char **value, const char *default_value) { +bool parse_option(int argc, char *const argv[], int &narg, const char *option, const char **value, + const char *default_value) { assert(narg < argc); const char *current = argv[narg]; const size_t optlen = strlen(option); @@ -49,14 +49,11 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, failure("No value given for '--%s' option\n", option); } -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - std::string &value, bool allow_empty) { - return parse_option(argc, argv, narg, option, value, allow_empty, - allow_empty ? "" : nullptr); +bool parse_option(int argc, char *const argv[], int &narg, const char *option, std::string &value, bool allow_empty) { + return parse_option(argc, argv, narg, option, value, allow_empty, allow_empty ? "" : nullptr); } -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - std::string &value, bool allow_empty, +bool parse_option(int argc, char *const argv[], int &narg, const char *option, std::string &value, bool allow_empty, const char *default_value) { const char *value_cstr; if (!parse_option(argc, argv, narg, option, &value_cstr, default_value)) @@ -70,8 +67,7 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, } template <> -bool parse_option(int argc, char *const argv[], int &narg, - const char *option, unsigned &mask, +bool parse_option(int argc, char *const argv[], int &narg, const char *option, unsigned &mask, const option_verb *verbs) { const char *list; if (!parse_option(argc, argv, narg, option, &list)) @@ -95,8 +91,7 @@ bool parse_option(int argc, char *const argv[], int &narg, while (true) { if (!scan->verb) - failure("Unknown verb '%.*s', for option '--%s'\n", (int)len, list, - option); + failure("Unknown verb '%.*s', for option '--%s'\n", (int)len, list, option); if (strlen(scan->verb) == len && strncmp(list, scan->verb, len) == 0) { mask = strikethrough ? mask & ~scan->mask : mask | scan->mask; clear = strikethrough ? clear & ~scan->mask : clear | scan->mask; @@ -110,10 +105,8 @@ bool parse_option(int argc, char *const argv[], int &narg, return true; } -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - uint64_t &value, const scale_mode scale, - const uint64_t minval, const uint64_t maxval, - const uint64_t default_value) { +bool parse_option(int argc, char *const argv[], int &narg, const char *option, uint64_t &value, const scale_mode scale, + const uint64_t minval, const uint64_t maxval, const uint64_t default_value) { const char *value_cstr; if (!parse_option(argc, argv, narg, option, &value_cstr)) @@ -134,17 +127,13 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, return true; } - if (strcmp(value_cstr, "rnd") == 0 || strcmp(value_cstr, "rand") == 0 || - strcmp(value_cstr, "random") == 0) { + if (strcmp(value_cstr, "rnd") == 0 || strcmp(value_cstr, "rand") == 0 || strcmp(value_cstr, "random") == 0) { value = minval; if (maxval > minval) { - uint64_t salt = (scale != entropy) - ? prng64() ^ UINT64_C(44263400549519813) - : (chrono::now_monotonic().fixedpoint ^ - UINT64_C(0xD85794512ED321FD)) * - UINT64_C(0x9120038359EAF3) ^ - chrono::now_realtime().fixedpoint * - UINT64_C(0x2FE5232BDC8E5F); + uint64_t salt = (scale != entropy) ? prng64() ^ UINT64_C(44263400549519813) + : (chrono::now_monotonic().fixedpoint ^ UINT64_C(0xD85794512ED321FD)) * + UINT64_C(0x9120038359EAF3) ^ + chrono::now_realtime().fixedpoint * UINT64_C(0x2FE5232BDC8E5F); value += salt % (maxval - minval); } if (scale == intkey) @@ -161,43 +150,32 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, raw = strtoull(value_cstr, &suffix, 10); } if (errno) - failure("Option '--%s' expects a numeric value (%s)\n", option, - test_strerror(errno)); + failure("Option '--%s' expects a numeric value (%s)\n", option, test_strerror(errno)); uint64_t multiplier = 1; if (suffix && *suffix) { if (scale == no_scale || scale == intkey) - failure("Option '--%s' doesn't accepts suffixes, so '%s' is unexpected\n", - option, suffix); + failure("Option '--%s' doesn't accepts suffixes, so '%s' is unexpected\n", option, suffix); if (strcmp(suffix, "K") == 0 || strcasecmp(suffix, "Kilo") == 0) multiplier = (scale == decimal) ? UINT64_C(1000) : UINT64_C(1024); else if (strcmp(suffix, "M") == 0 || strcasecmp(suffix, "Mega") == 0) - multiplier = - (scale == decimal) ? UINT64_C(1000) * 1000 : UINT64_C(1024) * 1024; + multiplier = (scale == decimal) ? UINT64_C(1000) * 1000 : UINT64_C(1024) * 1024; else if (strcmp(suffix, "G") == 0 || strcasecmp(suffix, "Giga") == 0) - multiplier = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 - : UINT64_C(1024) * 1024 * 1024; + multiplier = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 : UINT64_C(1024) * 1024 * 1024; else if (strcmp(suffix, "T") == 0 || strcasecmp(suffix, "Tera") == 0) - multiplier = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 * 1000 - : UINT64_C(1024) * 1024 * 1024 * 1024; - else if (scale == duration && - (strcmp(suffix, "s") == 0 || strcasecmp(suffix, "Seconds") == 0)) + multiplier = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 * 1000 : UINT64_C(1024) * 1024 * 1024 * 1024; + else if (scale == duration && (strcmp(suffix, "s") == 0 || strcasecmp(suffix, "Seconds") == 0)) multiplier = 1; - else if (scale == duration && - (strcmp(suffix, "m") == 0 || strcasecmp(suffix, "Minutes") == 0)) + else if (scale == duration && (strcmp(suffix, "m") == 0 || strcasecmp(suffix, "Minutes") == 0)) multiplier = 60; - else if (scale == duration && - (strcmp(suffix, "h") == 0 || strcasecmp(suffix, "Hours") == 0)) + else if (scale == duration && (strcmp(suffix, "h") == 0 || strcasecmp(suffix, "Hours") == 0)) multiplier = 3600; - else if (scale == duration && - (strcmp(suffix, "d") == 0 || strcasecmp(suffix, "Days") == 0)) + else if (scale == duration && (strcmp(suffix, "d") == 0 || strcasecmp(suffix, "Days") == 0)) multiplier = 3600 * 24; else - failure( - "Option '--%s' expects a numeric value with Kilo/Mega/Giga/Tera %s" - "suffixes, but '%s' is unexpected\n", - option, (scale == duration) ? "or Seconds/Minutes/Hours/Days " : "", - suffix); + failure("Option '--%s' expects a numeric value with Kilo/Mega/Giga/Tera %s" + "suffixes, but '%s' is unexpected\n", + option, (scale == duration) ? "or Seconds/Minutes/Hours/Days " : "", suffix); } if (raw >= UINT64_MAX / multiplier) @@ -205,47 +183,38 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, value = raw * multiplier; if (maxval && value > maxval) - failure("The maximal value for option '--%s' is %" PRIu64 "\n", option, - maxval); + failure("The maximal value for option '--%s' is %" PRIu64 "\n", option, maxval); if (value < minval) - failure("The minimal value for option '--%s' is %" PRIu64 "\n", option, - minval); + failure("The minimal value for option '--%s' is %" PRIu64 "\n", option, minval); if (scale == intkey) value &= ~3u; return true; } -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - unsigned &value, const scale_mode scale, - const unsigned minval, const unsigned maxval, - const unsigned default_value) { +bool parse_option(int argc, char *const argv[], int &narg, const char *option, unsigned &value, const scale_mode scale, + const unsigned minval, const unsigned maxval, const unsigned default_value) { uint64_t huge; - if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval, - default_value)) + if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval, default_value)) return false; value = unsigned(huge); return true; } -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - uint8_t &value, const uint8_t minval, const uint8_t maxval, - const uint8_t default_value) { +bool parse_option(int argc, char *const argv[], int &narg, const char *option, uint8_t &value, const uint8_t minval, + const uint8_t maxval, const uint8_t default_value) { uint64_t huge; - if (!parse_option(argc, argv, narg, option, huge, no_scale, minval, maxval, - default_value)) + if (!parse_option(argc, argv, narg, option, huge, no_scale, minval, maxval, default_value)) return false; value = uint8_t(huge); return true; } -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - int64_t &value, const int64_t minval, const int64_t maxval, - const int64_t default_value) { +bool parse_option(int argc, char *const argv[], int &narg, const char *option, int64_t &value, const int64_t minval, + const int64_t maxval, const int64_t default_value) { uint64_t proxy = uint64_t(value); - if (parse_option(argc, argv, narg, option, proxy, config::binary, - uint64_t(minval), uint64_t(maxval), + if (parse_option(argc, argv, narg, option, proxy, config::binary, uint64_t(minval), uint64_t(maxval), uint64_t(default_value))) { value = int64_t(proxy); return true; @@ -253,12 +222,10 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, return false; } -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - int32_t &value, const int32_t minval, const int32_t maxval, - const int32_t default_value) { +bool parse_option(int argc, char *const argv[], int &narg, const char *option, int32_t &value, const int32_t minval, + const int32_t maxval, const int32_t default_value) { uint64_t proxy = uint64_t(value); - if (parse_option(argc, argv, narg, option, proxy, config::binary, - uint64_t(minval), uint64_t(maxval), + if (parse_option(argc, argv, narg, option, proxy, config::binary, uint64_t(minval), uint64_t(maxval), uint64_t(default_value))) { value = int32_t(proxy); return true; @@ -266,14 +233,12 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, return false; } -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - logging::loglevel &loglevel) { +bool parse_option(int argc, char *const argv[], int &narg, const char *option, logging::loglevel &loglevel) { const char *value_cstr; if (!parse_option(argc, argv, narg, option, &value_cstr)) return false; - if (strcmp(value_cstr, "min") == 0 || strcmp(value_cstr, "minimal") == 0 || - strcmp(value_cstr, "fatal") == 0) { + if (strcmp(value_cstr, "min") == 0 || strcmp(value_cstr, "minimal") == 0 || strcmp(value_cstr, "fatal") == 0) { loglevel = logging::failure; return true; } @@ -308,8 +273,7 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, return true; } - if (strcmp(value_cstr, "max") == 0 || strcmp(value_cstr, "maximal") == 0 || - strcmp(value_cstr, "extra") == 0) { + if (strcmp(value_cstr, "max") == 0 || strcmp(value_cstr, "maximal") == 0 || strcmp(value_cstr, "extra") == 0) { loglevel = logging::extra; return true; } @@ -329,8 +293,7 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, failure("Unknown log-level '%s', for option '--%s'\n", value_cstr, option); } -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - bool &value) { +bool parse_option(int argc, char *const argv[], int &narg, const char *option, bool &value) { const char *value_cstr = nullptr; if (!parse_option(argc, argv, narg, option, &value_cstr, "yes")) { const char *current = argv[narg]; @@ -338,8 +301,7 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, value = false; return true; } - if (strncmp(current, "--dont-", 7) == 0 && - strcmp(current + 7, option) == 0) { + if (strncmp(current, "--dont-", 7) == 0 && strcmp(current + 7, option) == 0) { value = false; return true; } @@ -361,41 +323,36 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, return true; } - failure( - "Option '--%s' expects a 'boolean' value Yes/No, so '%s' is unexpected\n", - option, value_cstr); + failure("Option '--%s' expects a 'boolean' value Yes/No, so '%s' is unexpected\n", option, value_cstr); } //----------------------------------------------------------------------------- -const struct option_verb mode_bits[] = { - {"rdonly", unsigned(MDBX_RDONLY)}, - {"nosync-utterly", unsigned(MDBX_UTTERLY_NOSYNC)}, - {"nosubdir", unsigned(MDBX_NOSUBDIR)}, - {"nosync-safe", unsigned(MDBX_SAFE_NOSYNC)}, - {"nometasync", unsigned(MDBX_NOMETASYNC)}, - {"writemap", unsigned(MDBX_WRITEMAP)}, - {"nostickythreads", unsigned(MDBX_NOSTICKYTHREADS)}, - {"no-sticky-threads", unsigned(MDBX_NOSTICKYTHREADS)}, - {"nordahead", unsigned(MDBX_NORDAHEAD)}, - {"nomeminit", unsigned(MDBX_NOMEMINIT)}, - {"lifo", unsigned(MDBX_LIFORECLAIM)}, - {"perturb", unsigned(MDBX_PAGEPERTURB)}, - {"accede", unsigned(MDBX_ACCEDE)}, - {"exclusive", unsigned(MDBX_EXCLUSIVE)}, - {nullptr, 0}}; +const struct option_verb mode_bits[] = {{"rdonly", unsigned(MDBX_RDONLY)}, + {"nosync-utterly", unsigned(MDBX_UTTERLY_NOSYNC)}, + {"nosubdir", unsigned(MDBX_NOSUBDIR)}, + {"nosync-safe", unsigned(MDBX_SAFE_NOSYNC)}, + {"nometasync", unsigned(MDBX_NOMETASYNC)}, + {"writemap", unsigned(MDBX_WRITEMAP)}, + {"nostickythreads", unsigned(MDBX_NOSTICKYTHREADS)}, + {"no-sticky-threads", unsigned(MDBX_NOSTICKYTHREADS)}, + {"nordahead", unsigned(MDBX_NORDAHEAD)}, + {"nomeminit", unsigned(MDBX_NOMEMINIT)}, + {"lifo", unsigned(MDBX_LIFORECLAIM)}, + {"perturb", unsigned(MDBX_PAGEPERTURB)}, + {"accede", unsigned(MDBX_ACCEDE)}, + {"exclusive", unsigned(MDBX_EXCLUSIVE)}, + {nullptr, 0}}; -const struct option_verb table_bits[] = { - {"key.reverse", unsigned(MDBX_REVERSEKEY)}, - {"key.integer", unsigned(MDBX_INTEGERKEY)}, - {"data.integer", unsigned(MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT)}, - {"data.fixed", unsigned(MDBX_DUPFIXED | MDBX_DUPSORT)}, - {"data.reverse", unsigned(MDBX_REVERSEDUP | MDBX_DUPSORT)}, - {"data.dups", unsigned(MDBX_DUPSORT)}, - {nullptr, 0}}; +const struct option_verb table_bits[] = {{"key.reverse", unsigned(MDBX_REVERSEKEY)}, + {"key.integer", unsigned(MDBX_INTEGERKEY)}, + {"data.integer", unsigned(MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT)}, + {"data.fixed", unsigned(MDBX_DUPFIXED | MDBX_DUPSORT)}, + {"data.reverse", unsigned(MDBX_REVERSEDUP | MDBX_DUPSORT)}, + {"data.dups", unsigned(MDBX_DUPSORT)}, + {nullptr, 0}}; -static void dump_verbs(const char *caption, size_t bits, - const struct option_verb *verbs) { +static void dump_verbs(const char *caption, size_t bits, const struct option_verb *verbs) { log_verbose("%s: 0x%" PRIx64 " = ", caption, (uint64_t)bits); const char *comma = ""; @@ -429,28 +386,21 @@ void dump(const char *title) { logging::local_suffix indent(title); for (auto i = global::actors.begin(); i != global::actors.end(); ++i) { - log_verbose("#%u, testcase %s, space_id/table %u\n", i->actor_id, - testcase2str(i->testcase), i->space_id); + log_verbose("#%u, testcase %s, space_id/table %u\n", i->actor_id, testcase2str(i->testcase), i->space_id); indent.push(); log_verbose("prng-seed: %u\n", i->params.prng_seed); if (i->params.loglevel) { log_verbose("log: level %u, %s\n", i->params.loglevel, - i->params.pathname_log.empty() - ? "console" - : i->params.pathname_log.c_str()); + i->params.pathname_log.empty() ? "console" : i->params.pathname_log.c_str()); } - log_verbose("database: %s, size %" PRIuPTR "[%" PRIiPTR "..%" PRIiPTR - ", %i %i, %i]\n", - i->params.pathname_db.c_str(), i->params.size_now, - i->params.size_lower, i->params.size_upper, - i->params.shrink_threshold, i->params.growth_step, - i->params.pagesize); + log_verbose("database: %s, size %" PRIuPTR "[%" PRIiPTR "..%" PRIiPTR ", %i %i, %i]\n", + i->params.pathname_db.c_str(), i->params.size_now, i->params.size_lower, i->params.size_upper, + i->params.shrink_threshold, i->params.growth_step, i->params.pagesize); dump_verbs("mode", i->params.mode_flags, mode_bits); - log_verbose("random-writemap: %s\n", - i->params.random_writemap ? "Yes" : "No"); + log_verbose("random-writemap: %s\n", i->params.random_writemap ? "Yes" : "No"); dump_verbs("table", i->params.table_flags, table_bits); if (i->params.test_nops) @@ -465,62 +415,46 @@ void dump(const char *title) { log_verbose("threads %u\n", i->params.nthreads); - log_verbose( - "keygen.params: case %s, width %u, mesh %u, rotate %u, offset %" PRIu64 - ", split %u/%u\n", - keygencase2str(i->params.keygen.keycase), i->params.keygen.width, - i->params.keygen.mesh, i->params.keygen.rotate, i->params.keygen.offset, - i->params.keygen.split, - i->params.keygen.width - i->params.keygen.split); - log_verbose("keygen.zerofill: %s\n", - i->params.keygen.zero_fill ? "Yes" : "No"); - log_verbose("key: minlen %u, maxlen %u\n", i->params.keylen_min, - i->params.keylen_max); - log_verbose("data: minlen %u, maxlen %u\n", i->params.datalen_min, - i->params.datalen_max); + log_verbose("keygen.params: case %s, width %u, mesh %u, rotate %u, offset %" PRIu64 ", split %u/%u\n", + keygencase2str(i->params.keygen.keycase), i->params.keygen.width, i->params.keygen.mesh, + i->params.keygen.rotate, i->params.keygen.offset, i->params.keygen.split, + i->params.keygen.width - i->params.keygen.split); + log_verbose("keygen.zerofill: %s\n", i->params.keygen.zero_fill ? "Yes" : "No"); + log_verbose("key: minlen %u, maxlen %u\n", i->params.keylen_min, i->params.keylen_max); + log_verbose("data: minlen %u, maxlen %u\n", i->params.datalen_min, i->params.datalen_max); - log_verbose("batch: read %u, write %u\n", i->params.batch_read, - i->params.batch_write); + log_verbose("batch: read %u, write %u\n", i->params.batch_read, i->params.batch_write); if (i->params.waitfor_nops) - log_verbose("wait: actor %u for %u ops\n", i->wait4id, - i->params.waitfor_nops); + log_verbose("wait: actor %u for %u ops\n", i->wait4id, i->params.waitfor_nops); else if (i->params.delaystart) dump_duration("delay", i->params.delaystart); else log_verbose("no-delay\n"); if (i->params.inject_writefaultn) - log_verbose("inject-writefault on %u ops\n", - i->params.inject_writefaultn); + log_verbose("inject-writefault on %u ops\n", i->params.inject_writefaultn); else log_verbose("no-inject-writefault\n"); - log_verbose("limits: readers %u, tables %u, txn-bytes %zu\n", - i->params.max_readers, i->params.max_tables, + log_verbose("limits: readers %u, tables %u, txn-bytes %zu\n", i->params.max_readers, i->params.max_tables, mdbx_limits_txnsize_max(i->params.pagesize)); log_verbose("drop table: %s\n", i->params.drop_table ? "Yes" : "No"); - log_verbose("ignore MDBX_MAP_FULL error: %s\n", - i->params.ignore_dbfull ? "Yes" : "No"); - log_verbose("verifying by speculum: %s\n", - i->params.speculum ? "Yes" : "No"); + log_verbose("ignore MDBX_MAP_FULL error: %s\n", i->params.ignore_dbfull ? "Yes" : "No"); + log_verbose("verifying by speculum: %s\n", i->params.speculum ? "Yes" : "No"); indent.pop(); } dump_duration("timeout", global::config::timeout_duration_seconds); - log_verbose("cleanup: before %s, after %s\n", - global::config::cleanup_before ? "Yes" : "No", + log_verbose("cleanup: before %s, after %s\n", global::config::cleanup_before ? "Yes" : "No", global::config::cleanup_after ? "Yes" : "No"); log_verbose("failfast: %s\n", global::config::failfast ? "Yes" : "No"); - log_verbose("progress indicator: %s\n", - global::config::progress_indicator ? "Yes" : "No"); - log_verbose("console mode: %s\n", - global::config::console_mode ? "Yes" : "No"); - log_verbose("geometry jitter: %s\n", - global::config::geometry_jitter ? "Yes" : "No"); + log_verbose("progress indicator: %s\n", global::config::progress_indicator ? "Yes" : "No"); + log_verbose("console mode: %s\n", global::config::console_mode ? "Yes" : "No"); + log_verbose("geometry jitter: %s\n", global::config::geometry_jitter ? "Yes" : "No"); } } /* namespace config */ @@ -529,11 +463,8 @@ void dump(const char *title) { using namespace config; -actor_config::actor_config(actor_testcase testcase, const actor_params ¶ms, - unsigned space_id, unsigned wait4id) - : actor_config_pod(1 + unsigned(global::actors.size()), testcase, space_id, - wait4id), - params(params) {} +actor_config::actor_config(actor_testcase testcase, const actor_params ¶ms, unsigned space_id, unsigned wait4id) + : actor_config_pod(1 + unsigned(global::actors.size()), testcase, space_id, wait4id), params(params) {} const std::string actor_config::serialize(const char *prefix) const { simple_checksum checksum; @@ -551,25 +482,19 @@ const std::string actor_config::serialize(const char *prefix) const { result.push_back('|'); #if __cplusplus > 201400 - static_assert(std::is_trivially_copyable::value, - "actor_params_pod should by POD"); + static_assert(std::is_trivially_copyable::value, "actor_params_pod should by POD"); #else - static_assert(std::is_standard_layout::value, - "actor_params_pod should by POD"); + static_assert(std::is_standard_layout::value, "actor_params_pod should by POD"); #endif - result.append(data2hex(static_cast(¶ms), - sizeof(actor_params_pod), checksum)); + result.append(data2hex(static_cast(¶ms), sizeof(actor_params_pod), checksum)); result.push_back('|'); #if __cplusplus > 201400 - static_assert(std::is_trivially_copyable::value, - "actor_config_pod should by POD"); + static_assert(std::is_trivially_copyable::value, "actor_config_pod should by POD"); #else - static_assert(std::is_standard_layout::value, - "actor_config_pod should by POD"); + static_assert(std::is_standard_layout::value, "actor_config_pod should by POD"); #endif - result.append(data2hex(static_cast(this), - sizeof(actor_config_pod), checksum)); + result.append(data2hex(static_cast(this), sizeof(actor_config_pod), checksum)); result.push_back('|'); result.push_back(global::config::progress_indicator ? 'Y' : 'N'); checksum.push(global::config::progress_indicator); @@ -615,16 +540,12 @@ bool actor_config::deserialize(const char *str, actor_config &config) { return false; } #if __cplusplus > 201400 - static_assert(std::is_trivially_copyable::value, - "actor_params_pod should by POD"); + static_assert(std::is_trivially_copyable::value, "actor_params_pod should by POD"); #else - static_assert(std::is_standard_layout::value, - "actor_params_pod should by POD"); + static_assert(std::is_standard_layout::value, "actor_params_pod should by POD"); #endif - if (!hex2data(str, slash, static_cast(&config.params), - sizeof(actor_params_pod), checksum)) { - TRACE("<< actor_config::deserialize: actor_params_pod(%.*s)\n", - (int)(slash - str), str); + if (!hex2data(str, slash, static_cast(&config.params), sizeof(actor_params_pod), checksum)) { + TRACE("<< actor_config::deserialize: actor_params_pod(%.*s)\n", (int)(slash - str), str); return false; } str = slash + 1; @@ -635,16 +556,12 @@ bool actor_config::deserialize(const char *str, actor_config &config) { return false; } #if __cplusplus > 201400 - static_assert(std::is_trivially_copyable::value, - "actor_config_pod should by POD"); + static_assert(std::is_trivially_copyable::value, "actor_config_pod should by POD"); #else - static_assert(std::is_standard_layout::value, - "actor_config_pod should by POD"); + static_assert(std::is_standard_layout::value, "actor_config_pod should by POD"); #endif - if (!hex2data(str, slash, static_cast(&config), - sizeof(actor_config_pod), checksum)) { - TRACE("<< actor_config::deserialize: actor_config_pod(%.*s)\n", - (int)(slash - str), str); + if (!hex2data(str, slash, static_cast(&config), sizeof(actor_config_pod), checksum)) { + TRACE("<< actor_config::deserialize: actor_config_pod(%.*s)\n", (int)(slash - str), str); return false; } str = slash + 1; @@ -654,8 +571,7 @@ bool actor_config::deserialize(const char *str, actor_config &config) { TRACE("<< actor_config::deserialize: slash-5\n"); return false; } - if ((str[0] == 'Y' || str[0] == 'N') && (str[1] == 'Y' || str[1] == 'N') && - (str[2] == 'Y' || str[2] == 'N')) { + if ((str[0] == 'Y' || str[0] == 'N') && (str[1] == 'Y' || str[1] == 'N') && (str[2] == 'Y' || str[2] == 'N')) { global::config::progress_indicator = str[0] == 'Y'; checksum.push(global::config::progress_indicator); global::config::console_mode = str[1] == 'Y'; @@ -690,21 +606,14 @@ bool actor_config::deserialize(const char *str, actor_config &config) { return true; } -unsigned actor_params::mdbx_keylen_min() const { - return unsigned(mdbx_limits_keysize_min(table_flags)); -} +unsigned actor_params::mdbx_keylen_min() const { return unsigned(mdbx_limits_keysize_min(table_flags)); } -unsigned actor_params::mdbx_keylen_max() const { - return unsigned(mdbx_limits_keysize_max(pagesize, table_flags)); -} +unsigned actor_params::mdbx_keylen_max() const { return unsigned(mdbx_limits_keysize_max(pagesize, table_flags)); } -unsigned actor_params::mdbx_datalen_min() const { - return unsigned(mdbx_limits_valsize_min(table_flags)); -} +unsigned actor_params::mdbx_datalen_min() const { return unsigned(mdbx_limits_valsize_min(table_flags)); } unsigned actor_params::mdbx_datalen_max() const { - return std::min(unsigned(UINT16_MAX), - unsigned(mdbx_limits_valsize_max(pagesize, table_flags))); + return std::min(unsigned(UINT16_MAX), unsigned(mdbx_limits_valsize_max(pagesize, table_flags))); } bool actor_params::make_keygen_linear() { @@ -713,26 +622,18 @@ bool actor_params::make_keygen_linear() { keygen.rotate = 0; keygen.offset = 0; const auto max_serial = serial_mask(keygen.width) + base; - const auto max_key_serial = (keygen.split && (table_flags & MDBX_DUPSORT)) - ? max_serial >> keygen.split - : max_serial; - const auto max_value_serial = (keygen.split && (table_flags & MDBX_DUPSORT)) - ? serial_mask(keygen.split) - : 0; + const auto max_key_serial = (keygen.split && (table_flags & MDBX_DUPSORT)) ? max_serial >> keygen.split : max_serial; + const auto max_value_serial = (keygen.split && (table_flags & MDBX_DUPSORT)) ? serial_mask(keygen.split) : 0; - while (keylen_min < 8 && - (keylen_min == 0 || serial_mask(keylen_min * 8) < max_key_serial)) { + while (keylen_min < 8 && (keylen_min == 0 || serial_mask(keylen_min * 8) < max_key_serial)) { keylen_min += (table_flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) ? 4 : 1; if (keylen_max < keylen_min) keylen_max = keylen_min; } if (table_flags & MDBX_DUPSORT) - while ( - datalen_min < 8 && - (datalen_min == 0 || serial_mask(datalen_min * 8) < max_value_serial)) { - datalen_min += - (table_flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) ? 4 : 1; + while (datalen_min < 8 && (datalen_min == 0 || serial_mask(datalen_min * 8) < max_value_serial)) { + datalen_min += (table_flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) ? 4 : 1; if (datalen_max < datalen_min) datalen_max = datalen_min; } diff --git a/test/config.h++ b/test/config.h++ index 45ab7cf4..71ff5c97 100644 --- a/test/config.h++ +++ b/test/config.h++ @@ -54,18 +54,16 @@ namespace config { enum scale_mode { no_scale, decimal, binary, duration, intkey, entropy }; -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - const char **value, const char *default_value = nullptr); +bool parse_option(int argc, char *const argv[], int &narg, const char *option, const char **value, + const char *default_value = nullptr); -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - std::string &value, bool allow_empty = false); +bool parse_option(int argc, char *const argv[], int &narg, const char *option, std::string &value, + bool allow_empty = false); -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - std::string &value, bool allow_empty, +bool parse_option(int argc, char *const argv[], int &narg, const char *option, std::string &value, bool allow_empty, const char *default_value); -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - bool &value); +bool parse_option(int argc, char *const argv[], int &narg, const char *option, bool &value); struct option_verb { const char *const verb; @@ -73,8 +71,7 @@ struct option_verb { }; template -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - MASK &mask, const option_verb *verbs) { +bool parse_option(int argc, char *const argv[], int &narg, const char *option, MASK &mask, const option_verb *verbs) { static_assert(sizeof(MASK) <= sizeof(unsigned), "WTF?"); unsigned u = unsigned(mask); if (parse_option(argc, argv, narg, option, u, verbs)) { @@ -85,49 +82,36 @@ bool parse_option(int argc, char *const argv[], int &narg, const char *option, } template <> -bool parse_option(int argc, char *const argv[], int &narg, - const char *option, unsigned &mask, +bool parse_option(int argc, char *const argv[], int &narg, const char *option, unsigned &mask, const option_verb *verbs); -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - uint64_t &value, const scale_mode scale, - const uint64_t minval = 0, const uint64_t maxval = INT64_MAX, - const uint64_t default_value = 0); +bool parse_option(int argc, char *const argv[], int &narg, const char *option, uint64_t &value, const scale_mode scale, + const uint64_t minval = 0, const uint64_t maxval = INT64_MAX, const uint64_t default_value = 0); -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - unsigned &value, const scale_mode scale, - const unsigned minval = 0, const unsigned maxval = INT32_MAX, - const unsigned default_value = 0); +bool parse_option(int argc, char *const argv[], int &narg, const char *option, unsigned &value, const scale_mode scale, + const unsigned minval = 0, const unsigned maxval = INT32_MAX, const unsigned default_value = 0); -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - uint8_t &value, const uint8_t minval = 0, +bool parse_option(int argc, char *const argv[], int &narg, const char *option, uint8_t &value, const uint8_t minval = 0, const uint8_t maxval = 255, const uint8_t default_value = 0); -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - int64_t &value, const int64_t minval, const int64_t maxval, - const int64_t default_value = -1); +bool parse_option(int argc, char *const argv[], int &narg, const char *option, int64_t &value, const int64_t minval, + const int64_t maxval, const int64_t default_value = -1); -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - int32_t &value, const int32_t minval, const int32_t maxval, - const int32_t default_value = -1); +bool parse_option(int argc, char *const argv[], int &narg, const char *option, int32_t &value, const int32_t minval, + const int32_t maxval, const int32_t default_value = -1); -inline bool parse_option_intptr(int argc, char *const argv[], int &narg, - const char *option, intptr_t &value, - const intptr_t minval, const intptr_t maxval, - const intptr_t default_value = -1) { +inline bool parse_option_intptr(int argc, char *const argv[], int &narg, const char *option, intptr_t &value, + const intptr_t minval, const intptr_t maxval, const intptr_t default_value = -1) { static_assert(sizeof(intptr_t) == 4 || sizeof(intptr_t) == 8, "WTF?"); if (sizeof(intptr_t) == 8) - return parse_option(argc, argv, narg, option, - *reinterpret_cast(&value), int64_t(minval), + return parse_option(argc, argv, narg, option, *reinterpret_cast(&value), int64_t(minval), int64_t(maxval), int64_t(default_value)); else - return parse_option(argc, argv, narg, option, - *reinterpret_cast(&value), int32_t(minval), + return parse_option(argc, argv, narg, option, *reinterpret_cast(&value), int32_t(minval), int32_t(maxval), int32_t(default_value)); } -bool parse_option(int argc, char *const argv[], int &narg, const char *option, - logging::loglevel &); +bool parse_option(int argc, char *const argv[], int &narg, const char *option, logging::loglevel &); //----------------------------------------------------------------------------- struct keygen_params_pod { @@ -295,10 +279,8 @@ struct actor_config_pod { unsigned signal_nops{0}; actor_config_pod() = default; - actor_config_pod(unsigned actor_id, actor_testcase testcase, - unsigned space_id, unsigned wait4id) - : actor_id(actor_id), space_id(space_id), testcase(testcase), - wait4id(wait4id) {} + actor_config_pod(unsigned actor_id, actor_testcase testcase, unsigned space_id, unsigned wait4id) + : actor_id(actor_id), space_id(space_id), testcase(testcase), wait4id(wait4id) {} }; extern const struct option_verb mode_bits[]; @@ -326,8 +308,7 @@ struct actor_config : public config::actor_config_pod { bool wanna_event4signalling() const { return true /* TODO ? */; } actor_config() = default; - actor_config(actor_testcase testcase, const actor_params ¶ms, - unsigned space_id, unsigned wait4id); + actor_config(actor_testcase testcase, const actor_params ¶ms, unsigned space_id, unsigned wait4id); actor_config(const char *str) : actor_config() { if (!deserialize(str, *this)) diff --git a/test/copy.c++ b/test/copy.c++ index e21a1318..b5d068d8 100644 --- a/test/copy.c++ +++ b/test/copy.c++ @@ -9,8 +9,7 @@ class testcase_copy : public testcase { public: testcase_copy(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid), - copy_pathname(config.params.pathname_db + "-copy") {} + : testcase(config, pid), copy_pathname(config.params.pathname_db + "-copy") {} bool run() override; }; REGISTER_TESTCASE(copy); @@ -21,14 +20,10 @@ void testcase_copy::copy_db(const bool with_compaction) { failure_perror("osal_removefile()", err); if (flipcoin()) { - err = mdbx_env_copy(db_guard.get(), copy_pathname.c_str(), - with_compaction ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS); - log_verbose("mdbx_env_copy(%s), err %d", with_compaction ? "true" : "false", - err); + err = mdbx_env_copy(db_guard.get(), copy_pathname.c_str(), with_compaction ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS); + log_verbose("mdbx_env_copy(%s), err %d", with_compaction ? "true" : "false", err); if (unlikely(err != MDBX_SUCCESS)) - failure_perror(with_compaction ? "mdbx_env_copy(MDBX_CP_COMPACT)" - : "mdbx_env_copy(MDBX_CP_ASIS)", - err); + failure_perror(with_compaction ? "mdbx_env_copy(MDBX_CP_COMPACT)" : "mdbx_env_copy(MDBX_CP_ASIS)", err); } else { do { const bool ro = mode_readonly() || flipcoin(); @@ -36,26 +31,20 @@ void testcase_copy::copy_db(const bool with_compaction) { const bool dynsize = flipcoin(); const bool flush = flipcoin(); const bool enable_renew = flipcoin(); - const MDBX_copy_flags_t flags = - (with_compaction ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | - (dynsize ? MDBX_CP_FORCE_DYNAMIC_SIZE : MDBX_CP_DEFAULTS) | - (throttle ? MDBX_CP_THROTTLE_MVCC : MDBX_CP_DEFAULTS) | - (flush ? MDBX_CP_DEFAULTS : MDBX_CP_DONT_FLUSH) | - (enable_renew ? MDBX_CP_RENEW_TXN : MDBX_CP_DEFAULTS); + const MDBX_copy_flags_t flags = (with_compaction ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | + (dynsize ? MDBX_CP_FORCE_DYNAMIC_SIZE : MDBX_CP_DEFAULTS) | + (throttle ? MDBX_CP_THROTTLE_MVCC : MDBX_CP_DEFAULTS) | + (flush ? MDBX_CP_DEFAULTS : MDBX_CP_DONT_FLUSH) | + (enable_renew ? MDBX_CP_RENEW_TXN : MDBX_CP_DEFAULTS); txn_begin(ro); - err = - mdbx_txn_copy2pathname(txn_guard.get(), copy_pathname.c_str(), flags); + err = mdbx_txn_copy2pathname(txn_guard.get(), copy_pathname.c_str(), flags); log_verbose("mdbx_txn_copy2pathname(flags=0x%X), err %d", flags, err); txn_end(err != MDBX_SUCCESS || flipcoin()); - if (unlikely( - err != MDBX_SUCCESS && !(throttle && err == MDBX_OUSTED) && - !(!enable_renew && err == MDBX_MVCC_RETARDED) && - !(err == MDBX_EINVAL && !ro && - (flags & (MDBX_CP_THROTTLE_MVCC | MDBX_CP_RENEW_TXN)) != 0))) - failure_perror(with_compaction - ? "mdbx_txn_copy2pathname(MDBX_CP_COMPACT)" - : "mdbx_txn_copy2pathname(MDBX_CP_ASIS)", - err); + if (unlikely(err != MDBX_SUCCESS && !(throttle && err == MDBX_OUSTED) && + !(!enable_renew && err == MDBX_MVCC_RETARDED) && + !(err == MDBX_EINVAL && !ro && (flags & (MDBX_CP_THROTTLE_MVCC | MDBX_CP_RENEW_TXN)) != 0))) + failure_perror( + with_compaction ? "mdbx_txn_copy2pathname(MDBX_CP_COMPACT)" : "mdbx_txn_copy2pathname(MDBX_CP_ASIS)", err); } while (err != MDBX_SUCCESS); } } diff --git a/test/dead.c++ b/test/dead.c++ index d4bbbc19..c34443eb 100644 --- a/test/dead.c++ +++ b/test/dead.c++ @@ -5,8 +5,7 @@ class testcase_deadread : public testcase { public: - testcase_deadread(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid) {} + testcase_deadread(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} bool run() override; }; REGISTER_TESTCASE(deadread); @@ -24,8 +23,7 @@ bool testcase_deadread::run() { class testcase_deadwrite : public testcase { public: - testcase_deadwrite(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid) {} + testcase_deadwrite(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} bool run() override; }; diff --git a/test/extra/crunched_delete.c++ b/test/extra/crunched_delete.c++ index d11b5528..fde48950 100644 --- a/test/extra/crunched_delete.c++ +++ b/test/extra/crunched_delete.c++ @@ -26,8 +26,7 @@ std::string format_va(const char *fmt, va_list ap) { result.reserve(size_t(needed + 1)); result.resize(size_t(needed), '\0'); assert(int(result.capacity()) > needed); - int actual = vsnprintf(const_cast(result.data()), result.capacity(), - fmt, ones); + int actual = vsnprintf(const_cast(result.data()), result.capacity(), fmt, ones); assert(actual == needed); (void)actual; va_end(ones); @@ -47,10 +46,8 @@ struct acase { unsigned vlen_min, vlen_max; unsigned dupmax_log2; - acase(unsigned klen_min, unsigned klen_max, unsigned vlen_min, - unsigned vlen_max, unsigned dupmax_log2) - : klen_min(klen_min), klen_max(klen_max), vlen_min(vlen_min), - vlen_max(vlen_max), dupmax_log2(dupmax_log2) {} + acase(unsigned klen_min, unsigned klen_max, unsigned vlen_min, unsigned vlen_max, unsigned dupmax_log2) + : klen_min(klen_min), klen_max(klen_max), vlen_min(vlen_min), vlen_max(vlen_max), dupmax_log2(dupmax_log2) {} }; // std::random_device rd; @@ -80,33 +77,26 @@ static mdbx::slice mk_val(mdbx::default_buffer &buf, const acase &thecase) { static std::string name(unsigned n) { return format("Commitment_%05u", n); } -static mdbx::map_handle create_and_fill(mdbx::txn txn, const acase &thecase, - const unsigned n) { +static mdbx::map_handle create_and_fill(mdbx::txn txn, const acase &thecase, const unsigned n) { auto map = txn.create_map(name(n), - (thecase.klen_min == thecase.klen_max && - (thecase.klen_min == 4 || thecase.klen_max == 8)) + (thecase.klen_min == thecase.klen_max && (thecase.klen_min == 4 || thecase.klen_max == 8)) ? mdbx::key_mode::ordinal : mdbx::key_mode::usual, - (thecase.vlen_min == thecase.vlen_max) - ? mdbx::value_mode::multi_samelength - : mdbx::value_mode::multi); + (thecase.vlen_min == thecase.vlen_max) ? mdbx::value_mode::multi_samelength + : mdbx::value_mode::multi); if (txn.get_map_stat(map).ms_entries < NN) { mdbx::default_buffer k, v; for (auto i = 0u; i < NN; i++) { mk_key(k, thecase); - for (auto ii = thecase.dupmax_log2 - ? 1u + (rnd() & ((2u << thecase.dupmax_log2) - 1u)) - : 1u; - ii > 0; --ii) + for (auto ii = thecase.dupmax_log2 ? 1u + (rnd() & ((2u << thecase.dupmax_log2) - 1u)) : 1u; ii > 0; --ii) txn.upsert(map, k, mk_val(v, thecase)); } } return map; } -static void chunched_delete(mdbx::txn txn, const acase &thecase, - const unsigned n) { +static void chunched_delete(mdbx::txn txn, const acase &thecase, const unsigned n) { // printf(">> %s, case #%i\n", __FUNCTION__, n); mdbx::default_buffer k, v; auto map = txn.open_map_accede(name(n)); @@ -121,27 +111,20 @@ static void chunched_delete(mdbx::txn txn, const acase &thecase, bool last_r; if (true == ((last_op = "MDBX_GET_BOTH"), - (last_r = cursor.find_multivalue( - mk_key(k, thecase), mk_val(v, thecase), false))) || + (last_r = cursor.find_multivalue(mk_key(k, thecase), mk_val(v, thecase), false))) || rnd() % 3 == 0 || - true == ((last_op = "MDBX_SET_RANGE"), - (last_r = cursor.lower_bound(mk_key(k, thecase), false)))) { + true == ((last_op = "MDBX_SET_RANGE"), (last_r = cursor.lower_bound(mk_key(k, thecase), false)))) { int i = int(rnd() % 7) - 3; // if (i) // printf(" %s -> %s\n", last_op, last_r ? "true" : "false"); // printf("== shift multi %i\n", i); try { - while (i < 0 && - true == ((last_op = "MDBX_PREV_DUP"), - (last_r = cursor.to_current_prev_multi(false)))) + while (i < 0 && true == ((last_op = "MDBX_PREV_DUP"), (last_r = cursor.to_current_prev_multi(false)))) ++i; - while (i > 0 && - true == ((last_op = "MDBX_NEXT_DUP"), - (last_r = cursor.to_current_next_multi(false)))) + while (i > 0 && true == ((last_op = "MDBX_NEXT_DUP"), (last_r = cursor.to_current_next_multi(false)))) --i; } catch (const mdbx::no_data &) { - printf("cursor_del() -> exception, last %s %s\n", last_op, - last_r ? "true" : "false"); + printf("cursor_del() -> exception, last %s %s\n", last_op, last_r ? "true" : "false"); continue; } } @@ -159,8 +142,7 @@ static void chunched_delete(mdbx::txn txn, const acase &thecase, // printf(" cursor_del() -> %s\n", last_r ? "true" : "false"); } while (cursor.to_next(false) && --i > 0); } catch (const mdbx::no_data &) { - printf("cursor_del() -> exception, last %s %s\n", last_op, - last_r ? "true" : "false"); + printf("cursor_del() -> exception, last %s %s\n", last_op, last_r ? "true" : "false"); } // (void) last_op; @@ -178,8 +160,8 @@ static void chunched_delete(mdbx::txn txn, const acase &thecase, static char log_buffer[1024]; -static void logger_nofmt(MDBX_log_level_t loglevel, const char *function, - int line, const char *msg, unsigned length) noexcept { +static void logger_nofmt(MDBX_log_level_t loglevel, const char *function, int line, const char *msg, + unsigned length) noexcept { (void)length; (void)loglevel; fprintf(stdout, "%s:%u %s", function, line, msg); @@ -187,12 +169,10 @@ static void logger_nofmt(MDBX_log_level_t loglevel, const char *function, bool outofrange_prev(mdbx::env env) { mdbx::cursor_managed cursor; - const std::array items = { - {{"k1", "v1"}, {"k1", "v2"}, {"k2", "v1"}, {"k2", "v2"}}}; + const std::array items = {{{"k1", "v1"}, {"k1", "v2"}, {"k2", "v1"}, {"k2", "v2"}}}; auto txn = env.start_write(); - auto multi = - txn.create_map("multi", mdbx::key_mode::usual, mdbx::value_mode::multi); + auto multi = txn.create_map("multi", mdbx::key_mode::usual, mdbx::value_mode::multi); auto simple = txn.create_map("simple"); txn.clear_map(multi); txn.clear_map(simple); @@ -242,12 +222,10 @@ bool outofrange_prev(mdbx::env env) { } bool next_prev_current(mdbx::env env) { - const std::array items = { - {{"k1", "v1"}, {"k1", "v2"}, {"k2", "v1"}, {"k2", "v2"}}}; + const std::array items = {{{"k1", "v1"}, {"k1", "v2"}, {"k2", "v1"}, {"k2", "v2"}}}; auto txn = env.start_write(); - auto map = - txn.create_map("multi", mdbx::key_mode::usual, mdbx::value_mode::multi); + auto map = txn.create_map("multi", mdbx::key_mode::usual, mdbx::value_mode::multi); txn.clear_map(map); for (const auto &i : items) txn.upsert(map, i); @@ -309,8 +287,7 @@ bool next_prev_current(mdbx::env env) { } bool simple(mdbx::env env) { - const std::array items = { - {{"k0", "v0"}, {"k1", "v1"}, {"k2", "v2"}}}; + const std::array items = {{{"k0", "v0"}, {"k1", "v1"}, {"k2", "v2"}}}; auto txn = env.start_write(); auto map = txn.create_map("simple"); @@ -376,14 +353,12 @@ int main(int argc, const char *argv[]) { (void)argc; (void)argv; - mdbx_setup_debug_nofmt(MDBX_LOG_NOTICE, MDBX_DBG_ASSERT, logger_nofmt, - log_buffer, sizeof(log_buffer)); + mdbx_setup_debug_nofmt(MDBX_LOG_NOTICE, MDBX_DBG_ASSERT, logger_nofmt, log_buffer, sizeof(log_buffer)); mdbx::path db_filename = "test-crunched-del"; mdbx::env::remove(db_filename); - mdbx::env_managed env(db_filename, mdbx::env_managed::create_parameters(), - mdbx::env::operate_parameters(42)); + mdbx::env_managed env(db_filename, mdbx::env_managed::create_parameters(), mdbx::env::operate_parameters(42)); if (!simple(env) || !next_prev_current(env) || !outofrange_prev(env)) return EXIT_FAILURE; @@ -392,8 +367,7 @@ int main(int argc, const char *argv[]) { // Значения разной длины от 100 до 1000 байт. testset.emplace_back(/* keylen_min */ 1, /* keylen_max */ 64, /* datalen_min */ 100, /* datalen_max */ - mdbx_env_get_valsize4page_max( - env, MDBX_db_flags_t(mdbx::value_mode::multi)), + mdbx_env_get_valsize4page_max(env, MDBX_db_flags_t(mdbx::value_mode::multi)), /* dups_log2 */ 6); // В одной таблице DupSort: path -> version_u64+data // path - это префикс в дереве. Самые частые длины: 1-5 байт и 32-36 байт. diff --git a/test/extra/cursor_closing.c++ b/test/extra/cursor_closing.c++ index 7b6967ef..045b7677 100644 --- a/test/extra/cursor_closing.c++ +++ b/test/extra/cursor_closing.c++ @@ -2,8 +2,8 @@ #include -static void logger_nofmt(MDBX_log_level_t loglevel, const char *function, - int line, const char *msg, unsigned length) noexcept { +static void logger_nofmt(MDBX_log_level_t loglevel, const char *function, int line, const char *msg, + unsigned length) noexcept { (void)length; (void)loglevel; std::cout << function << ":" << line << " " << msg; @@ -15,20 +15,17 @@ int main(int argc, const char *argv[]) { (void)argc; (void)argv; - mdbx_setup_debug_nofmt(MDBX_LOG_NOTICE, MDBX_DBG_ASSERT, logger_nofmt, - log_buffer, sizeof(log_buffer)); + mdbx_setup_debug_nofmt(MDBX_LOG_NOTICE, MDBX_DBG_ASSERT, logger_nofmt, log_buffer, sizeof(log_buffer)); mdbx::path db_filename = "test-cursor-closing"; mdbx::env::remove(db_filename); - mdbx::env_managed env( - db_filename, mdbx::env_managed::create_parameters(), - mdbx::env::operate_parameters(42, 0, mdbx::env::nested_transactions)); + mdbx::env_managed env(db_filename, mdbx::env_managed::create_parameters(), + mdbx::env::operate_parameters(42, 0, mdbx::env::nested_transactions)); { auto txn = env.start_write(); - auto table = txn.create_map("dummy", mdbx::key_mode::usual, - mdbx::value_mode::single); + auto table = txn.create_map("dummy", mdbx::key_mode::usual, mdbx::value_mode::single); auto cursor_1 = txn.open_cursor(table); auto cursor_2 = cursor_1.clone(); diff --git a/test/extra/dbi.c++ b/test/extra/dbi.c++ index 2a12be6a..9ed37c45 100644 --- a/test/extra/dbi.c++ +++ b/test/extra/dbi.c++ @@ -4,8 +4,8 @@ static char log_buffer[1024]; -static void logger_nofmt(MDBX_log_level_t loglevel, const char *function, - int line, const char *msg, unsigned length) noexcept { +static void logger_nofmt(MDBX_log_level_t loglevel, const char *function, int line, const char *msg, + unsigned length) noexcept { (void)length; (void)loglevel; fprintf(stdout, "%s:%u %s", function, line, msg); @@ -15,8 +15,7 @@ int main(int argc, const char *argv[]) { (void)argc; (void)argv; - mdbx_setup_debug_nofmt(MDBX_LOG_NOTICE, MDBX_DBG_ASSERT, logger_nofmt, - log_buffer, sizeof(log_buffer)); + mdbx_setup_debug_nofmt(MDBX_LOG_NOTICE, MDBX_DBG_ASSERT, logger_nofmt, log_buffer, sizeof(log_buffer)); mdbx::path db_filename = "test-dbi"; mdbx::env::remove(db_filename); @@ -26,14 +25,12 @@ int main(int argc, const char *argv[]) { { mdbx::env_managed env2(db_filename, createParameters, operateParameters); mdbx::txn_managed txn2 = env2.start_write(false); - /* mdbx::map_handle testHandle2 = */ txn2.create_map( - "fap1", mdbx::key_mode::reverse, mdbx::value_mode::single); + /* mdbx::map_handle testHandle2 = */ txn2.create_map("fap1", mdbx::key_mode::reverse, mdbx::value_mode::single); txn2.commit(); } mdbx::env_managed env(db_filename, createParameters, operateParameters); mdbx::txn_managed txn = env.start_write(false); - /* mdbx::map_handle testHandle = */ txn.create_map( - "fap1", mdbx::key_mode::usual, mdbx::value_mode::single); + /* mdbx::map_handle testHandle = */ txn.create_map("fap1", mdbx::key_mode::usual, mdbx::value_mode::single); txn.commit(); std::cout << "OK\n"; diff --git a/test/extra/doubtless_positioning.c++ b/test/extra/doubtless_positioning.c++ index 4cf710cd..28c37492 100644 --- a/test/extra/doubtless_positioning.c++ +++ b/test/extra/doubtless_positioning.c++ @@ -7,8 +7,7 @@ #include #include -static ::std::ostream &operator<<(::std::ostream &out, - const mdbx::cursor::move_operation op) { +static ::std::ostream &operator<<(::std::ostream &out, const mdbx::cursor::move_operation op) { static const char *const str[] = {"FIRST", "FIRST_DUP", "GET_BOTH", @@ -70,54 +69,45 @@ static buffer random_value() { return random(prng() % 47); } using predicate = std::function; -static bool probe(mdbx::txn txn, mdbx::map_handle dbi, - mdbx::cursor::move_operation op, predicate cmp, +static bool probe(mdbx::txn txn, mdbx::map_handle dbi, mdbx::cursor::move_operation op, predicate cmp, const buffer_pair &pair) { auto seeker = txn.open_cursor(dbi); auto scanner = seeker.clone(); - const bool scan_backward = - op == mdbx::cursor::key_lesser_than || - op == mdbx::cursor::key_lesser_or_equal || - op == mdbx::cursor::multi_exactkey_value_lesser_than || - op == mdbx::cursor::multi_exactkey_value_lesser_or_equal || - op == mdbx::cursor::pair_lesser_than || - op == mdbx::cursor::pair_lesser_or_equal; + const bool scan_backward = op == mdbx::cursor::key_lesser_than || op == mdbx::cursor::key_lesser_or_equal || + op == mdbx::cursor::multi_exactkey_value_lesser_than || + op == mdbx::cursor::multi_exactkey_value_lesser_or_equal || + op == mdbx::cursor::pair_lesser_than || op == mdbx::cursor::pair_lesser_or_equal; const bool is_multi = mdbx::is_multi(txn.get_handle_info(dbi).value_mode()); auto seek_result = seeker.move(op, pair.key, pair.value, false); - auto scan_result = scanner.fullscan( - [cmp, &pair](const mdbx::pair &scan) -> bool { return cmp(scan, pair); }, - scan_backward); + auto scan_result = + scanner.fullscan([cmp, &pair](const mdbx::pair &scan) -> bool { return cmp(scan, pair); }, scan_backward); if (seek_result.done == scan_result && (!scan_result || - seeker.is_same_position( - scanner, - op < mdbx::cursor::multi_exactkey_value_lesser_than && is_multi))) + seeker.is_same_position(scanner, op < mdbx::cursor::multi_exactkey_value_lesser_than && is_multi))) return true; std::cerr << std::endl; std::cerr << "bug:"; std::cerr << std::endl; - std::cerr << std::string(is_multi ? "multi" : "single") << "-map, op " << op - << ", key " << pair.key << ", value " << pair.value; + std::cerr << std::string(is_multi ? "multi" : "single") << "-map, op " << op << ", key " << pair.key << ", value " + << pair.value; std::cerr << std::endl; std::cerr << "\tscanner: "; if (scan_result) - std::cerr << " done, key " << scanner.current(false).key << ", value " - << scanner.current(false).value; + std::cerr << " done, key " << scanner.current(false).key << ", value " << scanner.current(false).value; else std::cerr << "not-found"; std::cerr << std::endl; - std::cerr << "\t seeker: " << (seek_result.done ? " done" : "not-found") - << ", key " << seek_result.key << ", value " << seek_result.value; + std::cerr << "\t seeker: " << (seek_result.done ? " done" : "not-found") << ", key " << seek_result.key + << ", value " << seek_result.value; std::cerr << std::endl; return false; } -static bool probe(mdbx::txn txn, mdbx::map_handle dbi, - mdbx::cursor::move_operation op, predicate cmp) { +static bool probe(mdbx::txn txn, mdbx::map_handle dbi, mdbx::cursor::move_operation op, predicate cmp) { const auto pair = buffer_pair(random_key(), random_value()); const bool ok = probe(txn, dbi, op, cmp, pair); #if MDBX_DEBUG @@ -159,32 +149,27 @@ static bool test(mdbx::txn txn, mdbx::map_handle dbi) { ok = probe(txn, dbi, mdbx::cursor::multi_exactkey_value_lesser_than, [txn, dbi](const mdbx::pair &l, const mdbx::pair &r) -> bool { - return mdbx_cmp(txn, dbi, l.key, r.key) == 0 && - mdbx_dcmp(txn, dbi, l.value, r.value) < 0; + return mdbx_cmp(txn, dbi, l.key, r.key) == 0 && mdbx_dcmp(txn, dbi, l.value, r.value) < 0; }) && ok; ok = probe(txn, dbi, mdbx::cursor::multi_exactkey_value_lesser_or_equal, [txn, dbi](const mdbx::pair &l, const mdbx::pair &r) -> bool { - return mdbx_cmp(txn, dbi, l.key, r.key) == 0 && - mdbx_dcmp(txn, dbi, l.value, r.value) <= 0; + return mdbx_cmp(txn, dbi, l.key, r.key) == 0 && mdbx_dcmp(txn, dbi, l.value, r.value) <= 0; }) && ok; ok = probe(txn, dbi, mdbx::cursor::multi_exactkey_value_equal, [txn, dbi](const mdbx::pair &l, const mdbx::pair &r) -> bool { - return mdbx_cmp(txn, dbi, l.key, r.key) == 0 && - mdbx_dcmp(txn, dbi, l.value, r.value) == 0; + return mdbx_cmp(txn, dbi, l.key, r.key) == 0 && mdbx_dcmp(txn, dbi, l.value, r.value) == 0; }) && ok; ok = probe(txn, dbi, mdbx::cursor::multi_exactkey_value_greater_or_equal, [txn, dbi](const mdbx::pair &l, const mdbx::pair &r) -> bool { - return mdbx_cmp(txn, dbi, l.key, r.key) == 0 && - mdbx_dcmp(txn, dbi, l.value, r.value) >= 0; + return mdbx_cmp(txn, dbi, l.key, r.key) == 0 && mdbx_dcmp(txn, dbi, l.value, r.value) >= 0; }) && ok; ok = probe(txn, dbi, mdbx::cursor::multi_exactkey_value_greater, [txn, dbi](const mdbx::pair &l, const mdbx::pair &r) -> bool { - return mdbx_cmp(txn, dbi, l.key, r.key) == 0 && - mdbx_dcmp(txn, dbi, l.value, r.value) > 0; + return mdbx_cmp(txn, dbi, l.key, r.key) == 0 && mdbx_dcmp(txn, dbi, l.value, r.value) > 0; }) && ok; @@ -205,9 +190,7 @@ static bool test(mdbx::txn txn, mdbx::map_handle dbi) { }) && ok; ok = probe(txn, dbi, mdbx::cursor::pair_equal, - [](const mdbx::pair &l, const mdbx::pair &r) -> bool { - return l == r; - }) && + [](const mdbx::pair &l, const mdbx::pair &r) -> bool { return l == r; }) && ok; ok = probe(txn, dbi, mdbx::cursor::pair_greater_or_equal, [txn, dbi](const mdbx::pair &l, const mdbx::pair &r) -> bool { @@ -234,14 +217,11 @@ int main(int argc, const char *argv[]) { mdbx::path db_filename = "test-posi"; mdbx::env_managed::remove(db_filename); - mdbx::env_managed env(db_filename, mdbx::env_managed::create_parameters(), - mdbx::env::operate_parameters(3)); + mdbx::env_managed env(db_filename, mdbx::env_managed::create_parameters(), mdbx::env::operate_parameters(3)); auto txn = env.start_write(); - auto single = - txn.create_map("single", mdbx::key_mode::usual, mdbx::value_mode::single); - auto multi = - txn.create_map("multi", mdbx::key_mode::usual, mdbx::value_mode::multi); + auto single = txn.create_map("single", mdbx::key_mode::usual, mdbx::value_mode::single); + auto multi = txn.create_map("multi", mdbx::key_mode::usual, mdbx::value_mode::multi); for (size_t i = 0; i < 1000; ++i) { auto key = random_key(); txn.upsert(single, key, random_value()); diff --git a/test/extra/dupfix_addodd.c b/test/extra/dupfix_addodd.c index 5b666af9..b2e6aeea 100644 --- a/test/extra/dupfix_addodd.c +++ b/test/extra/dupfix_addodd.c @@ -37,8 +37,7 @@ int main() { exit(EXIT_FAILURE); } - rc = mdbx_env_open(env, "./example-db", MDBX_NOSUBDIR | MDBX_LIFORECLAIM, - 0664); + rc = mdbx_env_open(env, "./example-db", MDBX_NOSUBDIR | MDBX_LIFORECLAIM, 0664); if (rc != MDBX_SUCCESS) { fprintf(stderr, "mdbx_env_open: (%d) %s\n", rc, mdbx_strerror(rc)); exit(EXIT_FAILURE); @@ -50,8 +49,7 @@ int main() { exit(EXIT_FAILURE); } - rc = mdbx_dbi_open(txn, "test", MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_CREATE, - &dbi); + rc = mdbx_dbi_open(txn, "test", MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_CREATE, &dbi); if (rc != MDBX_SUCCESS) { fprintf(stderr, "mdbx_dbi_open: (%d) %s\n", rc, mdbx_strerror(rc)); exit(EXIT_FAILURE); diff --git a/test/extra/dupfix_multiple.c++ b/test/extra/dupfix_multiple.c++ index 1b0ffc45..a8897314 100644 --- a/test/extra/dupfix_multiple.c++ +++ b/test/extra/dupfix_multiple.c++ @@ -8,14 +8,11 @@ int doit() { mdbx::path db_filename = "test-dupfix-multiple"; mdbx::env_managed::remove(db_filename); - mdbx::env_managed env(db_filename, mdbx::env_managed::create_parameters(), - mdbx::env::operate_parameters()); + mdbx::env_managed env(db_filename, mdbx::env_managed::create_parameters(), mdbx::env::operate_parameters()); - using buffer = - mdbx::buffer; + using buffer = mdbx::buffer; auto txn = env.start_write(); - auto map = txn.create_map(nullptr, mdbx::key_mode::ordinal, - mdbx::value_mode::multi_ordinal); + auto map = txn.create_map(nullptr, mdbx::key_mode::ordinal, mdbx::value_mode::multi_ordinal); txn.insert(map, buffer::key_from_u64(21), buffer::key_from_u64(18)); txn.insert(map, buffer::key_from_u64(7), buffer::key_from_u64(19)); @@ -29,15 +26,11 @@ int doit() { txn = env.start_read(); auto cursor = txn.open_cursor(map); - if (cursor.to_first().value.as_uint64() != 19 || - cursor.to_next().value.as_uint64() != 18 || - cursor.to_next().value.as_uint64() != 17 || - cursor.to_next().value.as_uint64() != 16 || - cursor.to_next().value.as_uint64() != 15 || - cursor.to_next().value.as_uint64() != 14 || - cursor.to_next().value.as_uint64() != 13 || - cursor.to_next().value.as_uint64() != 12 || cursor.to_next(false).done || - !cursor.eof()) { + if (cursor.to_first().value.as_uint64() != 19 || cursor.to_next().value.as_uint64() != 18 || + cursor.to_next().value.as_uint64() != 17 || cursor.to_next().value.as_uint64() != 16 || + cursor.to_next().value.as_uint64() != 15 || cursor.to_next().value.as_uint64() != 14 || + cursor.to_next().value.as_uint64() != 13 || cursor.to_next().value.as_uint64() != 12 || + cursor.to_next(false).done || !cursor.eof()) { std::cerr << "Fail\n"; return EXIT_FAILURE; } @@ -45,20 +38,13 @@ int doit() { const uint64_t array[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 42, 17, 99, 0, 33, 333}; txn = env.start_write(); - txn.put_multiple_samelength(map, buffer::key_from_u64(13), array + 3, 4, - mdbx::upsert); - txn.put_multiple_samelength(map, buffer::key_from_u64(10), array + 0, 1, - mdbx::upsert); - txn.put_multiple_samelength(map, buffer::key_from_u64(12), array + 2, 3, - mdbx::upsert); - txn.put_multiple_samelength(map, buffer::key_from_u64(15), array + 5, 6, - mdbx::upsert); - txn.put_multiple_samelength(map, buffer::key_from_u64(14), array + 4, 5, - mdbx::upsert); - txn.put_multiple_samelength(map, buffer::key_from_u64(11), array + 1, 2, - mdbx::upsert); - txn.put_multiple_samelength(map, buffer::key_from_u64(16), array + 6, 7, - mdbx::upsert); + txn.put_multiple_samelength(map, buffer::key_from_u64(13), array + 3, 4, mdbx::upsert); + txn.put_multiple_samelength(map, buffer::key_from_u64(10), array + 0, 1, mdbx::upsert); + txn.put_multiple_samelength(map, buffer::key_from_u64(12), array + 2, 3, mdbx::upsert); + txn.put_multiple_samelength(map, buffer::key_from_u64(15), array + 5, 6, mdbx::upsert); + txn.put_multiple_samelength(map, buffer::key_from_u64(14), array + 4, 5, mdbx::upsert); + txn.put_multiple_samelength(map, buffer::key_from_u64(11), array + 1, 2, mdbx::upsert); + txn.put_multiple_samelength(map, buffer::key_from_u64(16), array + 6, 7, mdbx::upsert); txn.commit(); txn = env.start_read(); @@ -69,42 +55,30 @@ int doit() { cursor.to_next().value.as_uint64() != 1 || /* key = 11: 2 элемента, пропуск 1 */ - cursor.to_next().value.as_uint64() != 2 || - cursor.to_next().value.as_uint64() != 3 || + cursor.to_next().value.as_uint64() != 2 || cursor.to_next().value.as_uint64() != 3 || /* key = 12: 3 элемента, пропуск 2 */ - cursor.to_next().value.as_uint64() != 3 || - cursor.to_next().value.as_uint64() != 4 || + cursor.to_next().value.as_uint64() != 3 || cursor.to_next().value.as_uint64() != 4 || cursor.to_next().value.as_uint64() != 5 || /* key = 13: 4 элемента, пропуск 3 */ - cursor.to_next().value.as_uint64() != 4 || - cursor.to_next().value.as_uint64() != 5 || - cursor.to_next().value.as_uint64() != 6 || - cursor.to_next().value.as_uint64() != 7 || + cursor.to_next().value.as_uint64() != 4 || cursor.to_next().value.as_uint64() != 5 || + cursor.to_next().value.as_uint64() != 6 || cursor.to_next().value.as_uint64() != 7 || /* key = 14: 5 элементов, пропуск 4 */ - cursor.to_next().value.as_uint64() != 5 || - cursor.to_next().value.as_uint64() != 6 || - cursor.to_next().value.as_uint64() != 7 || - cursor.to_next().value.as_uint64() != 8 || + cursor.to_next().value.as_uint64() != 5 || cursor.to_next().value.as_uint64() != 6 || + cursor.to_next().value.as_uint64() != 7 || cursor.to_next().value.as_uint64() != 8 || cursor.to_next().value.as_uint64() != 9 || /* key = 15: 6 элементов, пропуск 5 */ - cursor.to_next().value.as_uint64() != 6 || - cursor.to_next().value.as_uint64() != 7 || - cursor.to_next().value.as_uint64() != 8 || - cursor.to_next().value.as_uint64() != 9 || - cursor.to_next().value.as_uint64() != 17 || - cursor.to_next().value.as_uint64() != 42 || + cursor.to_next().value.as_uint64() != 6 || cursor.to_next().value.as_uint64() != 7 || + cursor.to_next().value.as_uint64() != 8 || cursor.to_next().value.as_uint64() != 9 || + cursor.to_next().value.as_uint64() != 17 || cursor.to_next().value.as_uint64() != 42 || /* key = 16: 7 элементов, пропуск 6 */ - cursor.to_next().value.as_uint64() != 0 || - cursor.to_next().value.as_uint64() != 7 || - cursor.to_next().value.as_uint64() != 8 || - cursor.to_next().value.as_uint64() != 9 || - cursor.to_next().value.as_uint64() != 17 || - cursor.to_next().value.as_uint64() != 42 || + cursor.to_next().value.as_uint64() != 0 || cursor.to_next().value.as_uint64() != 7 || + cursor.to_next().value.as_uint64() != 8 || cursor.to_next().value.as_uint64() != 9 || + cursor.to_next().value.as_uint64() != 17 || cursor.to_next().value.as_uint64() != 42 || cursor.to_next().value.as_uint64() != 99 || /* key = 21 */ cursor.to_next().value.as_uint64() != 18 || @@ -113,46 +87,35 @@ int doit() { /* key = 24 */ cursor.to_next().value.as_uint64() != 15 || /* key = 25 */ cursor.to_next().value.as_uint64() != 14 || /* key = 26 */ cursor.to_next().value.as_uint64() != 13 || - /* key = 27 */ cursor.to_next().value.as_uint64() != 12 || - cursor.to_next(false).done || !cursor.eof()) { + /* key = 27 */ cursor.to_next().value.as_uint64() != 12 || cursor.to_next(false).done || !cursor.eof()) { std::cerr << "Fail\n"; return EXIT_FAILURE; } txn.abort(); txn = env.start_write(); - txn.put_multiple_samelength(map, buffer::key_from_u64(7), array + 3, 4, - mdbx::update); + txn.put_multiple_samelength(map, buffer::key_from_u64(7), array + 3, 4, mdbx::update); txn.upsert(map, buffer::key_from_u64(10), buffer::key_from_u64(14)); - txn.put_multiple_samelength(map, buffer::key_from_u64(11), array + 4, 5, - mdbx::upsert); - txn.put_multiple_samelength(map, buffer::key_from_u64(12), array + 0, 1, - mdbx::update); + txn.put_multiple_samelength(map, buffer::key_from_u64(11), array + 4, 5, mdbx::upsert); + txn.put_multiple_samelength(map, buffer::key_from_u64(12), array + 0, 1, mdbx::update); txn.update(map, buffer::key_from_u64(13), buffer::key_from_u64(18)); - txn.put_multiple_samelength(map, buffer::key_from_u64(14), array + 2, 3, - mdbx::update); + txn.put_multiple_samelength(map, buffer::key_from_u64(14), array + 2, 3, mdbx::update); txn.update(map, buffer::key_from_u64(15), buffer::key_from_u64(13)); - txn.put_multiple_samelength(map, buffer::key_from_u64(16), array + 6, 9, - mdbx::update); + txn.put_multiple_samelength(map, buffer::key_from_u64(16), array + 6, 9, mdbx::update); txn.update(map, buffer::key_from_u64(21), buffer::key_from_u64(17)); txn.update(map, buffer::key_from_u64(22), buffer::key_from_u64(15)); - txn.put_multiple_samelength(map, buffer::key_from_u64(23), array + 1, 2, - mdbx::update); + txn.put_multiple_samelength(map, buffer::key_from_u64(23), array + 1, 2, mdbx::update); txn.update(map, buffer::key_from_u64(24), buffer::key_from_u64(16)); - txn.put_multiple_samelength(map, buffer::key_from_u64(25), array + 5, 6, - mdbx::update); + txn.put_multiple_samelength(map, buffer::key_from_u64(25), array + 5, 6, mdbx::update); txn.upsert(map, buffer::key_from_u64(26), buffer::key_from_u64(12)); - txn.put_multiple_samelength(map, buffer::key_from_u64(27), array + 12, 3, - mdbx::update); + txn.put_multiple_samelength(map, buffer::key_from_u64(27), array + 12, 3, mdbx::update); txn.commit(); txn = env.start_read(); cursor = txn.open_cursor(map); if (/* key = 7 */ - cursor.to_first().value.as_uint64() != 4 || - cursor.to_next().value.as_uint64() != 5 || - cursor.to_next().value.as_uint64() != 6 || - cursor.to_next().value.as_uint64() != 7 || + cursor.to_first().value.as_uint64() != 4 || cursor.to_next().value.as_uint64() != 5 || + cursor.to_next().value.as_uint64() != 6 || cursor.to_next().value.as_uint64() != 7 || /* key = 10: 1 элемент */ cursor.to_next().value.as_uint64() != 1 || @@ -160,13 +123,10 @@ int doit() { cursor.to_next().value.as_uint64() != 14 || /* key = 11: 2 элемента, пропуск 1 */ - cursor.to_next().value.as_uint64() != 2 || - cursor.to_next().value.as_uint64() != 3 || + cursor.to_next().value.as_uint64() != 2 || cursor.to_next().value.as_uint64() != 3 || /* +5 элементов, пропуск 4 */ - cursor.to_next().value.as_uint64() != 5 || - cursor.to_next().value.as_uint64() != 6 || - cursor.to_next().value.as_uint64() != 7 || - cursor.to_next().value.as_uint64() != 8 || + cursor.to_next().value.as_uint64() != 5 || cursor.to_next().value.as_uint64() != 6 || + cursor.to_next().value.as_uint64() != 7 || cursor.to_next().value.as_uint64() != 8 || cursor.to_next().value.as_uint64() != 9 || /* key = 12: 1 элемент */ @@ -174,44 +134,33 @@ int doit() { /* key = 13 */ cursor.to_next().value.as_uint64() != 18 || /* key = 14: 3 элемента, пропуск 2 */ - cursor.to_next().value.as_uint64() != 3 || - cursor.to_next().value.as_uint64() != 4 || + cursor.to_next().value.as_uint64() != 3 || cursor.to_next().value.as_uint64() != 4 || cursor.to_next().value.as_uint64() != 5 || /* key = 15 */ cursor.to_next().value.as_uint64() != 13 || /* key = 16: 9 элементов, пропуск 6 */ - cursor.to_next().value.as_uint64() != 0 || - cursor.to_next().value.as_uint64() != 7 || - cursor.to_next().value.as_uint64() != 8 || - cursor.to_next().value.as_uint64() != 9 || - cursor.to_next().value.as_uint64() != 17 || - cursor.to_next().value.as_uint64() != 33 || - cursor.to_next().value.as_uint64() != 42 || - cursor.to_next().value.as_uint64() != 99 || + cursor.to_next().value.as_uint64() != 0 || cursor.to_next().value.as_uint64() != 7 || + cursor.to_next().value.as_uint64() != 8 || cursor.to_next().value.as_uint64() != 9 || + cursor.to_next().value.as_uint64() != 17 || cursor.to_next().value.as_uint64() != 33 || + cursor.to_next().value.as_uint64() != 42 || cursor.to_next().value.as_uint64() != 99 || cursor.to_next().value.as_uint64() != 333 || /* key = 21 */ cursor.to_next().value.as_uint64() != 17 || /* key = 22 */ cursor.to_next().value.as_uint64() != 15 || /* key = 23: 2 элемента, пропуск 1 */ - cursor.to_next().value.as_uint64() != 2 || - cursor.to_next().value.as_uint64() != 3 || + cursor.to_next().value.as_uint64() != 2 || cursor.to_next().value.as_uint64() != 3 || /* key = 24 */ cursor.to_next().value.as_uint64() != 16 || /* key = 25: 6 элемента, пропуск 5 */ - cursor.to_next().value.as_uint64() != 6 || - cursor.to_next().value.as_uint64() != 7 || - cursor.to_next().value.as_uint64() != 8 || - cursor.to_next().value.as_uint64() != 9 || - cursor.to_next().value.as_uint64() != 17 || - cursor.to_next().value.as_uint64() != 42 || + cursor.to_next().value.as_uint64() != 6 || cursor.to_next().value.as_uint64() != 7 || + cursor.to_next().value.as_uint64() != 8 || cursor.to_next().value.as_uint64() != 9 || + cursor.to_next().value.as_uint64() != 17 || cursor.to_next().value.as_uint64() != 42 || /* key = 26, 1+1 upsert */ - cursor.to_next().value.as_uint64() != 12 || - cursor.to_next().value.as_uint64() != 13 || + cursor.to_next().value.as_uint64() != 12 || cursor.to_next().value.as_uint64() != 13 || /* key = 27: 3 элемента, пропуск 12 */ - cursor.to_next().value.as_uint64() != 0 || - cursor.to_next().value.as_uint64() != 33 || + cursor.to_next().value.as_uint64() != 0 || cursor.to_next().value.as_uint64() != 33 || cursor.to_next().value.as_uint64() != 333 || cursor.to_next(false).done || !cursor.eof()) { @@ -247,8 +196,7 @@ int doit() { txn = env.start_write(); txn.clear_map(map); - map = txn.create_map(nullptr, mdbx::key_mode::usual, - mdbx::value_mode::multi_samelength); + map = txn.create_map(nullptr, mdbx::key_mode::usual, mdbx::value_mode::multi_samelength); txn.upsert(map, mdbx::slice("key1"), mdbx::slice("val1")); txn.upsert(map, mdbx::pair("key1", "val2")); txn.upsert(map, mdbx::pair("key1", "val3")); diff --git a/test/extra/early_close_dbi.c++ b/test/extra/early_close_dbi.c++ index d9723107..42bb7adc 100644 --- a/test/extra/early_close_dbi.c++ +++ b/test/extra/early_close_dbi.c++ @@ -23,14 +23,11 @@ int main(int argc, char *argv[]) { // 1); assert(err == MDBX_SUCCESS); intptr_t lowerbound(0), size(0), upperbound(mdbx::env::geometry::GiB / 2); - intptr_t step(128 * mdbx::env::geometry::MiB), - shrink(256 * mdbx::env::geometry::MiB), pagesize(-1); - err = mdbx_env_set_geometry(environment, lowerbound, size, upperbound, step, - shrink, pagesize); + intptr_t step(128 * mdbx::env::geometry::MiB), shrink(256 * mdbx::env::geometry::MiB), pagesize(-1); + err = mdbx_env_set_geometry(environment, lowerbound, size, upperbound, step, shrink, pagesize); assert(err == MDBX_SUCCESS); - MDBX_env_flags_t flags(MDBX_NOSUBDIR | MDBX_WRITEMAP | MDBX_LIFORECLAIM | - MDBX_NORDAHEAD); + MDBX_env_flags_t flags(MDBX_NOSUBDIR | MDBX_WRITEMAP | MDBX_LIFORECLAIM | MDBX_NORDAHEAD); err = mdbx_env_openT(environment, db_filename.c_str(), flags, 0644); assert(err == MDBX_SUCCESS); diff --git a/test/extra/hex_base64_base58.c++ b/test/extra/hex_base64_base58.c++ index f2419ebd..7ae1fac4 100644 --- a/test/extra/hex_base64_base58.c++ +++ b/test/extra/hex_base64_base58.c++ @@ -27,20 +27,16 @@ static buffer random(size_t length) { static bool basic() { bool ok = true; const char *const hex_dump = "1D58fa\n2e46E3\nBd9c7A\nC0bF"; - const uint8_t native[] = {0x1D, 0x58, 0xfa, 0x2e, 0x46, 0xE3, - 0xBd, 0x9c, 0x7A, 0xC0, 0xbF}; + const uint8_t native[] = {0x1D, 0x58, 0xfa, 0x2e, 0x46, 0xE3, 0xBd, 0x9c, 0x7A, 0xC0, 0xbF}; if (mdbx::slice(hex_dump).hex_decode(true) != mdbx::slice::wrap(native)) std::cerr << "hex_decode() failed\n"; - else if (mdbx::slice::wrap(native).encode_hex(true, 4).hex_decode(true) != - mdbx::slice::wrap(native)) + else if (mdbx::slice::wrap(native).encode_hex(true, 4).hex_decode(true) != mdbx::slice::wrap(native)) std::cerr << "hex_encode(UPPERCASE) failed\n"; - else if (mdbx::slice::wrap(native).encode_hex(false).hex_decode(true) != - mdbx::slice::wrap(native)) + else if (mdbx::slice::wrap(native).encode_hex(false).hex_decode(true) != mdbx::slice::wrap(native)) std::cerr << "hex_encode(lowercase) failed\n"; - if (mdbx::slice("").as_base64_string() != "" || - mdbx::slice(" ").encode_base64().as_string() != "IA==" || + if (mdbx::slice("").as_base64_string() != "" || mdbx::slice(" ").encode_base64().as_string() != "IA==" || mdbx::slice("~0").encode_base64().as_string() != "fjA=" || mdbx::slice("A_z").encode_base64().as_string() != "QV96" || mdbx::slice("Ka9q").encode_base64().as_string() != "S2E5cQ==" || @@ -50,12 +46,9 @@ static bool basic() { } const uint8_t base58_rfc[] = {0x00, 0x00, 0x28, 0x7f, 0xb4, 0xcd}; - if (mdbx::slice("").as_base58_string() != "" || - mdbx::slice(" ").encode_base58().as_string() != "Z" || + if (mdbx::slice("").as_base58_string() != "" || mdbx::slice(" ").encode_base58().as_string() != "Z" || mdbx::slice("Hello World!").as_base58_string() != "2NEpo7TZRRrLZSi2U" || - mdbx::slice("The quick brown fox jumps over the lazy dog.") - .encode_base58() - .as_string() != + mdbx::slice("The quick brown fox jumps over the lazy dog.").encode_base58().as_string() != "USm3fpXnKG5EUBx2ndxBDMPVciP5hGey2Jh4NDv6gmeo1LkMeiKrLJUUBk6Z" || mdbx::slice::wrap(base58_rfc).as_base58_string() != "11233QC4" || mdbx::slice("~0").encode_base58().as_string() != "Aby" || @@ -66,15 +59,11 @@ static bool basic() { ok = false; } - if (mdbx::slice("").base58_decode() != mdbx::slice() || - mdbx::slice("Z").base58_decode() != mdbx::slice(" ") || + if (mdbx::slice("").base58_decode() != mdbx::slice() || mdbx::slice("Z").base58_decode() != mdbx::slice(" ") || mdbx::slice("2NEpo7TZRRrLZSi2U").base58_decode() != "Hello World!" || - mdbx::slice( - "USm3fpXnKG5EUBx2ndxBDMPVciP5hGey2Jh4NDv6gmeo1LkMeiKrLJUUBk6Z") - .base58_decode() != + mdbx::slice("USm3fpXnKG5EUBx2ndxBDMPVciP5hGey2Jh4NDv6gmeo1LkMeiKrLJUUBk6Z").base58_decode() != mdbx::slice("The quick brown fox jumps over the lazy dog.") || - mdbx::slice("11233QC4").base58_decode() != - mdbx::slice::wrap(base58_rfc) || + mdbx::slice("11233QC4").base58_decode() != mdbx::slice::wrap(base58_rfc) || mdbx::slice("Aby").base58_decode() != mdbx::slice("~0") || mdbx::slice("NxZw").base58_decode() != mdbx::slice("A_z") || mdbx::slice("2vkjDi").base58_decode() != mdbx::slice("Ka9q") || @@ -94,28 +83,19 @@ int main(int argc, const char *argv[]) { for (size_t n = 0; n < 1000; ++n) { for (size_t length = 0; ok && length < 111; ++length) { const auto pattern = random(length); - if (pattern != pattern.encode_hex(bool(prng() & 1), prng() % 111) - .hex_decode(true) - .encode_hex() - .hex_decode(false)) { - std::cerr << "hex encode/decode failed: n " << n << ", length " - << length << std::endl; + if (pattern != + pattern.encode_hex(bool(prng() & 1), prng() % 111).hex_decode(true).encode_hex().hex_decode(false)) { + std::cerr << "hex encode/decode failed: n " << n << ", length " << length << std::endl; ok = false; } - if (pattern != pattern.encode_base64(unsigned(prng() % 111)) - .base64_decode(true) - .encode_base64() - .base64_decode(false)) { - std::cerr << "base64 encode/decode failed: n " << n << ", length " - << length << std::endl; + if (pattern != + pattern.encode_base64(unsigned(prng() % 111)).base64_decode(true).encode_base64().base64_decode(false)) { + std::cerr << "base64 encode/decode failed: n " << n << ", length " << length << std::endl; ok = false; } - if (pattern != pattern.encode_base58(unsigned(prng() % 111)) - .base58_decode(true) - .encode_base58() - .base58_decode(false)) { - std::cerr << "base58 encode/decode failed: n " << n << ", length " - << length << std::endl; + if (pattern != + pattern.encode_base58(unsigned(prng() % 111)).base58_decode(true).encode_base58().base58_decode(false)) { + std::cerr << "base58 encode/decode failed: n " << n << ", length " << length << std::endl; ok = false; } } diff --git a/test/extra/maindb_ordinal.c++ b/test/extra/maindb_ordinal.c++ index dc3fd597..c766049f 100644 --- a/test/extra/maindb_ordinal.c++ +++ b/test/extra/maindb_ordinal.c++ @@ -10,14 +10,11 @@ int main(int argc, const char *argv[]) { mdbx::path db_filename = "test-dupfix-multiple"; mdbx::env_managed::remove(db_filename); - mdbx::env_managed env(db_filename, mdbx::env_managed::create_parameters(), - mdbx::env::operate_parameters()); + mdbx::env_managed env(db_filename, mdbx::env_managed::create_parameters(), mdbx::env::operate_parameters()); - using buffer = - mdbx::buffer; + using buffer = mdbx::buffer; auto txn = env.start_write(); - auto map = txn.create_map(nullptr, mdbx::key_mode::ordinal, - mdbx::value_mode::single); + auto map = txn.create_map(nullptr, mdbx::key_mode::ordinal, mdbx::value_mode::single); #if 0 /* workaround */ txn.commit(); env.close(); @@ -39,14 +36,10 @@ int main(int argc, const char *argv[]) { txn = env.start_read(); auto cursor = txn.open_cursor(map); #if defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L - if (cursor.to_first().value.string_view() == "a" && - cursor.to_next().value.string_view() == "b" && - cursor.to_next().value.string_view() == "c" && - cursor.to_next().value.string_view() == "d" && - cursor.to_next().value.string_view() == "e" && - cursor.to_next().value.string_view() == "f" && - cursor.to_next().value.string_view() == "g" && - cursor.to_next().value.string_view() == "h" && + if (cursor.to_first().value.string_view() == "a" && cursor.to_next().value.string_view() == "b" && + cursor.to_next().value.string_view() == "c" && cursor.to_next().value.string_view() == "d" && + cursor.to_next().value.string_view() == "e" && cursor.to_next().value.string_view() == "f" && + cursor.to_next().value.string_view() == "g" && cursor.to_next().value.string_view() == "h" && !cursor.to_next(false).done && cursor.eof()) { std::cout << "OK\n"; return EXIT_SUCCESS; diff --git a/test/extra/open.c++ b/test/extra/open.c++ index bc955c83..55b58c8c 100644 --- a/test/extra/open.c++ +++ b/test/extra/open.c++ @@ -18,8 +18,8 @@ int main(int argc, const char *argv[]) { static char log_buffer[1024]; -static void logger_nofmt(MDBX_log_level_t loglevel, const char *function, - int line, const char *msg, unsigned length) noexcept { +static void logger_nofmt(MDBX_log_level_t loglevel, const char *function, int line, const char *msg, + unsigned length) noexcept { (void)length; (void)loglevel; fprintf(stdout, "%s:%u %s", function, line, msg); @@ -29,8 +29,7 @@ int main(int argc, const char *argv[]) { (void)argc; (void)argv; - mdbx_setup_debug_nofmt(MDBX_LOG_VERBOSE, MDBX_DBG_ASSERT, logger_nofmt, - log_buffer, sizeof(log_buffer)); + mdbx_setup_debug_nofmt(MDBX_LOG_VERBOSE, MDBX_DBG_ASSERT, logger_nofmt, log_buffer, sizeof(log_buffer)); mdbx::path path = "test-open"; mdbx::env::remove(path); @@ -41,19 +40,16 @@ int main(int argc, const char *argv[]) { createParameters2.geometry.make_fixed(42 * mdbx::env::geometry::MiB); mdbx::env_managed env2(path, createParameters2, operateParameters2); mdbx::txn_managed txn2 = env2.start_write(false); - /* mdbx::map_handle testHandle2 = */ txn2.create_map( - "fap1", mdbx::key_mode::reverse, mdbx::value_mode::single); + /* mdbx::map_handle testHandle2 = */ txn2.create_map("fap1", mdbx::key_mode::reverse, mdbx::value_mode::single); txn2.commit(); } mdbx::env::operate_parameters operateParameters(100, 10); mdbx::env_managed::create_parameters createParameters; - createParameters.geometry.make_dynamic(21 * mdbx::env::geometry::MiB, - 84 * mdbx::env::geometry::MiB); + createParameters.geometry.make_dynamic(21 * mdbx::env::geometry::MiB, 84 * mdbx::env::geometry::MiB); mdbx::env_managed env(path, createParameters, operateParameters); mdbx::txn_managed txn = env.start_write(false); - /* mdbx::map_handle testHandle = */ txn.create_map( - "fap1", mdbx::key_mode::usual, mdbx::value_mode::single); + /* mdbx::map_handle testHandle = */ txn.create_map("fap1", mdbx::key_mode::usual, mdbx::value_mode::single); txn.commit(); std::latch starter(1); @@ -62,8 +58,7 @@ int main(int argc, const char *argv[]) { starter.wait(); // mdbx::env_managed env(path, createParameters, operateParameters); mdbx::txn_managed txn = env.start_write(false); - /* mdbx::map_handle testHandle = */ txn.create_map( - "fap1", mdbx::key_mode::usual, mdbx::value_mode::single); + /* mdbx::map_handle testHandle = */ txn.create_map("fap1", mdbx::key_mode::usual, mdbx::value_mode::single); txn.commit(); }); @@ -71,8 +66,7 @@ int main(int argc, const char *argv[]) { starter.wait(); // mdbx::env_managed env(path, createParameters, operateParameters); mdbx::txn_managed txn = env.start_write(false); - /* mdbx::map_handle testHandle = */ txn.create_map( - "fap1", mdbx::key_mode::usual, mdbx::value_mode::single); + /* mdbx::map_handle testHandle = */ txn.create_map("fap1", mdbx::key_mode::usual, mdbx::value_mode::single); txn.commit(); }); diff --git a/test/extra/pcrf/pcrf_test.c b/test/extra/pcrf/pcrf_test.c index 876bdf74..1d023fcf 100644 --- a/test/extra/pcrf/pcrf_test.c +++ b/test/extra/pcrf/pcrf_test.c @@ -32,9 +32,8 @@ #include #include -#define IP_PRINTF_ARG_HOST(addr) \ - (int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), \ - (int)((addr) & 0xff) +#define IP_PRINTF_ARG_HOST(addr) \ + (int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), (int)((addr) & 0xff) char opt_db_path[PATH_MAX] = "./mdbx_bench2"; static MDBX_env *env; @@ -88,14 +87,13 @@ static int64_t get_id_from_pool() { return id; } -#define MDBX_CHECK(x) \ - do { \ - const int rc = (x); \ - if (rc != MDBX_SUCCESS) { \ - printf("Error [%d] %s in %s at %s:%d\n", rc, mdbx_strerror(rc), #x, \ - __FILE__, __LINE__); \ - exit(EXIT_FAILURE); \ - } \ +#define MDBX_CHECK(x) \ + do { \ + const int rc = (x); \ + if (rc != MDBX_SUCCESS) { \ + printf("Error [%d] %s in %s at %s:%d\n", rc, mdbx_strerror(rc), #x, __FILE__, __LINE__); \ + exit(EXIT_FAILURE); \ + } \ } while (0) static void db_connect() { @@ -105,13 +103,10 @@ static void db_connect() { MDBX_dbi dbi_ip; MDBX_CHECK(mdbx_env_create(&env)); - MDBX_CHECK(mdbx_env_set_geometry( - env, 0, 0, REC_COUNT * sizeof(session_data_t) * 10, -1, -1, -1)); + MDBX_CHECK(mdbx_env_set_geometry(env, 0, 0, REC_COUNT * sizeof(session_data_t) * 10, -1, -1, -1)); MDBX_CHECK(mdbx_env_set_maxdbs(env, 30)); - MDBX_CHECK(mdbx_env_open(env, opt_db_path, - MDBX_CREATE | MDBX_WRITEMAP | MDBX_UTTERLY_NOSYNC | - MDBX_LIFORECLAIM, - 0664)); + MDBX_CHECK( + mdbx_env_open(env, opt_db_path, MDBX_CREATE | MDBX_WRITEMAP | MDBX_UTTERLY_NOSYNC | MDBX_LIFORECLAIM, 0664)); MDBX_txn *txn; // transaction init @@ -135,16 +130,11 @@ static void create_record(uint64_t record_id) { MDBX_txn *txn; session_data_t data; // transaction init - snprintf(data.session_id1, sizeof(data.session_id1), - "prefix%02u_%02u.fill.fill.fill.fill.fill.fill;%" PRIu64, - (unsigned)(record_id % 3) + 1, (unsigned)(record_id % 9) + 1, - record_id); - snprintf(data.session_id2, sizeof(data.session_id2), - "dprefix%" PRIu64 ";%" PRIu64 ".fill.fill.;suffix", record_id, - (record_id + UINT64_C(1442695040888963407)) % - UINT64_C(6364136223846793005)); - snprintf(data.ip, sizeof(data.ip), "%d.%d.%d.%d", - IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF)); + snprintf(data.session_id1, sizeof(data.session_id1), "prefix%02u_%02u.fill.fill.fill.fill.fill.fill;%" PRIu64, + (unsigned)(record_id % 3) + 1, (unsigned)(record_id % 9) + 1, record_id); + snprintf(data.session_id2, sizeof(data.session_id2), "dprefix%" PRIu64 ";%" PRIu64 ".fill.fill.;suffix", record_id, + (record_id + UINT64_C(1442695040888963407)) % UINT64_C(6364136223846793005)); + snprintf(data.ip, sizeof(data.ip), "%d.%d.%d.%d", IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF)); event.obj_id = record_id; event.event_type = 1; @@ -152,8 +142,7 @@ static void create_record(uint64_t record_id) { MDBX_val _session_id2_rec = {data.session_id2, strlen(data.session_id2)}; MDBX_val _ip_rec = {data.ip, strlen(data.ip)}; MDBX_val _obj_id_rec = {&record_id, sizeof(record_id)}; - MDBX_val _data_rec = {&data, offsetof(session_data_t, fill) + - (rand() % sizeof(data.fill))}; + MDBX_val _data_rec = {&data, offsetof(session_data_t, fill) + (rand() % sizeof(data.fill))}; MDBX_val _event_rec = {&event, sizeof(event)}; uint64_t start = getClockUs(); @@ -162,20 +151,16 @@ static void create_record(uint64_t record_id) { MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id)); MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event)); MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip)); - MDBX_CHECK(mdbx_put(txn, dbi_session, &_obj_id_rec, &_data_rec, - MDBX_NOOVERWRITE | MDBX_NODUPDATA)); - MDBX_CHECK(mdbx_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec, - MDBX_NOOVERWRITE | MDBX_NODUPDATA)); - MDBX_CHECK(mdbx_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec, - MDBX_NOOVERWRITE | MDBX_NODUPDATA)); + MDBX_CHECK(mdbx_put(txn, dbi_session, &_obj_id_rec, &_data_rec, MDBX_NOOVERWRITE | MDBX_NODUPDATA)); + MDBX_CHECK(mdbx_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec, MDBX_NOOVERWRITE | MDBX_NODUPDATA)); + MDBX_CHECK(mdbx_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec, MDBX_NOOVERWRITE | MDBX_NODUPDATA)); MDBX_CHECK(mdbx_put(txn, dbi_ip, &_ip_rec, &_obj_id_rec, 0)); MDBX_CHECK(mdbx_put(txn, dbi_event, &_event_rec, &_obj_id_rec, 0)); MDBX_CHECK(mdbx_txn_commit(txn)); mdbx_data_size += (_data_rec.iov_len + _obj_id_rec.iov_len * 4); - mdbx_key_size += - (_obj_id_rec.iov_len + _session_id1_rec.iov_len + - _session_id2_rec.iov_len + _ip_rec.iov_len + _event_rec.iov_len); + mdbx_key_size += (_obj_id_rec.iov_len + _session_id1_rec.iov_len + _session_id2_rec.iov_len + _ip_rec.iov_len + + _event_rec.iov_len); // transaction commit mdbx_add_count++; @@ -218,9 +203,8 @@ static void delete_record(int64_t record_id) { MDBX_CHECK(mdbx_del(txn, dbi_session, &_obj_id_rec, NULL)); mdbx_data_size -= (_data_rec.iov_len + _obj_id_rec.iov_len * 4); - mdbx_key_size -= - (_obj_id_rec.iov_len + _session_id1_rec.iov_len + - _session_id2_rec.iov_len + _ip_rec.iov_len + _event_rec.iov_len); + mdbx_key_size -= (_obj_id_rec.iov_len + _session_id1_rec.iov_len + _session_id2_rec.iov_len + _ip_rec.iov_len + + _event_rec.iov_len); // transaction commit MDBX_CHECK(mdbx_txn_commit(txn)); @@ -233,8 +217,7 @@ static void db_disconnect() { printf("Connection closed\n"); } -static void get_db_stat(const char *db, int64_t *ms_branch_pages, - int64_t *ms_leaf_pages) { +static void get_db_stat(const char *db, int64_t *ms_branch_pages, int64_t *ms_leaf_pages) { MDBX_txn *txn; MDBX_stat stat; MDBX_dbi dbi; @@ -243,10 +226,8 @@ static void get_db_stat(const char *db, int64_t *ms_branch_pages, MDBX_CHECK(mdbx_dbi_open(txn, db, MDBX_CREATE, &dbi)); MDBX_CHECK(mdbx_dbi_stat(txn, dbi, &stat, sizeof(stat))); mdbx_txn_abort(txn); - printf("%15s | %15" PRIu64 " | %5u | %10" PRIu64 " | %10" PRIu64 - " | %11" PRIu64 " |\n", - db, stat.ms_branch_pages, stat.ms_depth, stat.ms_entries, - stat.ms_leaf_pages, stat.ms_overflow_pages); + printf("%15s | %15" PRIu64 " | %5u | %10" PRIu64 " | %10" PRIu64 " | %11" PRIu64 " |\n", db, stat.ms_branch_pages, + stat.ms_depth, stat.ms_entries, stat.ms_leaf_pages, stat.ms_overflow_pages); (*ms_branch_pages) += stat.ms_branch_pages; (*ms_leaf_pages) += stat.ms_leaf_pages; } @@ -261,25 +242,20 @@ static void periodic_stat(void) { printf("Environment Info\n"); printf(" Pagesize: %u\n", mst.ms_psize); if (mei.mi_geo.lower != mei.mi_geo.upper) { - printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64 - "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64 - "/-%" PRIu64 ")\n", - mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow, - mei.mi_geo.shrink, mei.mi_geo.lower / mst.ms_psize, - mei.mi_geo.upper / mst.ms_psize, mei.mi_geo.grow / mst.ms_psize, - mei.mi_geo.shrink / mst.ms_psize); - printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", - mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize); + printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64 "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 + " pages (+%" PRIu64 "/-%" PRIu64 ")\n", + mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow, mei.mi_geo.shrink, mei.mi_geo.lower / mst.ms_psize, + mei.mi_geo.upper / mst.ms_psize, mei.mi_geo.grow / mst.ms_psize, mei.mi_geo.shrink / mst.ms_psize); + printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", mei.mi_geo.current, + mei.mi_geo.current / mst.ms_psize); } else { - printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", - mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize); + printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", mei.mi_geo.current, + mei.mi_geo.current / mst.ms_psize); } - printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n", - mei.mi_mapsize, mei.mi_mapsize / mst.ms_psize); + printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n", mei.mi_mapsize, mei.mi_mapsize / mst.ms_psize); printf(" Number of pages used: %" PRIu64 "\n", mei.mi_last_pgno + 1); printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid); - printf(" Tail transaction ID: %" PRIu64 " (%" PRIi64 ")\n", - mei.mi_latter_reader_txnid, + printf(" Tail transaction ID: %" PRIu64 " (%" PRIi64 ")\n", mei.mi_latter_reader_txnid, mei.mi_latter_reader_txnid - mei.mi_recent_txnid); printf(" Max readers: %u\n", mei.mi_maxreaders); printf(" Number of readers used: %u\n", mei.mi_numreaders); @@ -290,8 +266,8 @@ static void periodic_stat(void) { get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages); get_db_stat("event", &ms_branch_pages, &ms_leaf_pages); get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages); - printf("%15s | %15" PRIu64 " | %5s | %10s | %10" PRIu64 " | %11s |\n", "", - ms_branch_pages, "", "", ms_leaf_pages, ""); + printf("%15s | %15" PRIu64 " | %5s | %10s | %10" PRIu64 " | %11s |\n", "", ms_branch_pages, "", "", ms_leaf_pages, + ""); static int64_t prev_add_count; static int64_t prev_del_count; @@ -300,32 +276,23 @@ static void periodic_stat(void) { static int64_t t = -1; if (t > 0) { int64_t delta = (getClockUs() - t); - printf("CPS: add %" PRIu64 ", delete %" PRIu64 - ", items processed - %" PRIu64 "K data=%" PRIu64 "K key=%" PRIu64 + printf("CPS: add %" PRIu64 ", delete %" PRIu64 ", items processed - %" PRIu64 "K data=%" PRIu64 "K key=%" PRIu64 "K\n", - (mdbx_add_count - prev_add_count) * 1000000 / delta, - (mdbx_del_count - prev_del_count) * 1000000 / delta, obj_id / 1024, - mdbx_data_size / 1024, mdbx_key_size / 1024); + (mdbx_add_count - prev_add_count) * 1000000 / delta, (mdbx_del_count - prev_del_count) * 1000000 / delta, + obj_id / 1024, mdbx_data_size / 1024, mdbx_key_size / 1024); printf("usage data=%" PRIu64 "%%", - ((mdbx_data_size + mdbx_key_size) * 100) / - ((ms_leaf_pages + ms_branch_pages) * 4096)); + ((mdbx_data_size + mdbx_key_size) * 100) / ((ms_leaf_pages + ms_branch_pages) * 4096)); if (prev_add_time != mdbx_add_time) { - printf(" Add : %" PRIu64 " c/s", (mdbx_add_count - prev_add_count) * - 1000000 / - (mdbx_add_time - prev_add_time)); + printf(" Add : %" PRIu64 " c/s", (mdbx_add_count - prev_add_count) * 1000000 / (mdbx_add_time - prev_add_time)); } if (prev_del_time != mdbx_del_time) { - printf(" Del : %" PRIu64 " c/s", (mdbx_del_count - prev_del_count) * - 1000000 / - (mdbx_del_time - prev_del_time)); + printf(" Del : %" PRIu64 " c/s", (mdbx_del_count - prev_del_count) * 1000000 / (mdbx_del_time - prev_del_time)); } if (mdbx_add_time) { - printf(" tAdd : %" PRIu64 " c/s", - mdbx_add_count * 1000000 / mdbx_add_time); + printf(" tAdd : %" PRIu64 " c/s", mdbx_add_count * 1000000 / mdbx_add_time); } if (mdbx_del_time) { - printf(" tDel : %" PRIu64 " c/s", - mdbx_del_count * 1000000 / mdbx_del_time); + printf(" tDel : %" PRIu64 " c/s", mdbx_del_count * 1000000 / mdbx_del_time); } puts(""); } diff --git a/test/extra/probe.c++ b/test/extra/probe.c++ index c80f5f87..9dca781c 100644 --- a/test/extra/probe.c++ +++ b/test/extra/probe.c++ @@ -5,7 +5,6 @@ int main(int argc, const char *argv[]) { (void)argc; (void)argv; - std::cout - << "OK (but this is do-nothing test just for a check for compilation)\n"; + std::cout << "OK (but this is do-nothing test just for a check for compilation)\n"; return EXIT_SUCCESS; } diff --git a/test/extra/upsert_alldups.c b/test/extra/upsert_alldups.c index 7999f081..d6e69a93 100644 --- a/test/extra/upsert_alldups.c +++ b/test/extra/upsert_alldups.c @@ -17,8 +17,8 @@ static int dump(MDBX_cursor *cur) { int rc = mdbx_cursor_get(cur, &key, &data, MDBX_FIRST); while (rc == 0) { - printf("(%.*s) = (%.*s)\n", (int)key.iov_len, (const char *)key.iov_base, - (int)data.iov_len, (const char *)data.iov_base); + printf("(%.*s) = (%.*s)\n", (int)key.iov_len, (const char *)key.iov_base, (int)data.iov_len, + (const char *)data.iov_base); rc = mdbx_cursor_get(cur, &key, &data, MDBX_NEXT); } return rc; @@ -38,8 +38,7 @@ static int clear(MDBX_cursor *cur) { return (rc == MDBX_NOTFOUND) ? 0 : rc; } -static int put(MDBX_txn *txn, MDBX_dbi dbi, const char *k, const char *v, - MDBX_put_flags_t flags) { +static int put(MDBX_txn *txn, MDBX_dbi dbi, const char *k, const char *v, MDBX_put_flags_t flags) { MDBX_val key = {.iov_base = (void *)k, .iov_len = strlen(k)}; MDBX_val data = {.iov_base = (void *)v, .iov_len = strlen(v)}; return mdbx_put(txn, dbi, &key, &data, flags); @@ -79,21 +78,21 @@ int main(int argc, const char *argv[]) { goto Fail; } -#define DUMP() \ - do { \ - if ((rc = dump(cur)) && rc != MDBX_NOTFOUND) { \ - errmsg = "failed to mdbx_cursor_get(FIRST): %s\n"; \ - goto Fail; \ - } \ - puts(""); \ +#define DUMP() \ + do { \ + if ((rc = dump(cur)) && rc != MDBX_NOTFOUND) { \ + errmsg = "failed to mdbx_cursor_get(FIRST): %s\n"; \ + goto Fail; \ + } \ + puts(""); \ } while (0) -#define PUTVAL(k, v, flags) \ - do { \ - if ((rc = put(txn, dbi, k, v, flags))) { \ - errmsg = "failed to mdbx_put: %s\n"; \ - goto Fail; \ - } \ +#define PUTVAL(k, v, flags) \ + do { \ + if ((rc = put(txn, dbi, k, v, flags))) { \ + errmsg = "failed to mdbx_put: %s\n"; \ + goto Fail; \ + } \ } while (0) puts("TEST WITH MULTIPLE KEYS ===================="); diff --git a/test/fork.c++ b/test/fork.c++ index 05fb250a..d546cf89 100644 --- a/test/fork.c++ +++ b/test/fork.c++ @@ -17,8 +17,7 @@ protected: unsigned dbi_state{0}; public: - testcase_smoke4fork(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid) {} + testcase_smoke4fork(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} virtual void txn_end(bool abort) override; bool run() override; virtual bool smoke() = 0; @@ -27,8 +26,7 @@ public: bool testcase_smoke4fork::open_dbi() { if (!dbi || dbi_invalid) { - if (dbi_stable || - (mdbx_txn_flags(txn_guard.get()) & MDBX_TXN_RDONLY) == 0) { + if (dbi_stable || (mdbx_txn_flags(txn_guard.get()) & MDBX_TXN_RDONLY) == 0) { dbi = db_table_open(!dbi_stable); dbi_invalid = false; } @@ -37,8 +35,7 @@ bool testcase_smoke4fork::open_dbi() { dbi_state = 0; if (dbi && !dbi_invalid) { unsigned unused_dbi_flags; - int err = - mdbx_dbi_flags_ex(txn_guard.get(), dbi, &unused_dbi_flags, &dbi_state); + int err = mdbx_dbi_flags_ex(txn_guard.get(), dbi, &unused_dbi_flags, &dbi_state); if (unlikely(err != MDBX_SUCCESS)) failure_perror("mdbx_dbi_flags_ex()", err); if ((dbi_state & (MDBX_DBI_CREAT | MDBX_DBI_FRESH)) == 0) @@ -69,8 +66,7 @@ bool testcase_smoke4fork::run() { if (history.empty() || current_pid != history.front()) { history.push_back(current_pid); if (history.size() > /* TODO: add test option */ 2) { - log_notice("force exit to avoid fork-bomb: deep %zu, pid stack", - history.size()); + log_notice("force exit to avoid fork-bomb: deep %zu, pid stack", history.size()); for (const auto pid : history) logging::feed(" %d", pid); logging::ln(); @@ -82,23 +78,19 @@ bool testcase_smoke4fork::run() { int err = db_open__begin__table_create_open_clean(dbi); if (unlikely(err != MDBX_SUCCESS)) { - log_notice("fork[deep %d, pid %d]: bailout-prepare due '%s'", deep, - current_pid, mdbx_strerror(err)); + log_notice("fork[deep %d, pid %d]: bailout-prepare due '%s'", deep, current_pid, mdbx_strerror(err)); return false; } open_dbi(); if (flipcoin()) { if (!smoke()) { - log_notice("%s[deep %d, pid %d] probe %s", "pre-fork", deep, current_pid, - "failed"); + log_notice("%s[deep %d, pid %d] probe %s", "pre-fork", deep, current_pid, "failed"); return false; } - log_verbose("%s[deep %d, pid %d] probe %s", "pre-fork", deep, current_pid, - "done"); + log_verbose("%s[deep %d, pid %d] probe %s", "pre-fork", deep, current_pid, "done"); } else { - log_verbose("%s[deep %d, pid %d] probe %s", "pre-fork", deep, current_pid, - "skipped"); + log_verbose("%s[deep %d, pid %d] probe %s", "pre-fork", deep, current_pid, "skipped"); #ifdef __SANITIZE_ADDRESS__ const bool commit_txn_to_avoid_memleak = true; #else @@ -115,13 +107,12 @@ bool testcase_smoke4fork::run() { if (child == 0) { const pid_t new_pid = getpid(); - log_verbose(">>> %s, deep %d, parent-pid %d, child-pid %d", - "mdbx_env_resurrect_after_fork()", deep, current_pid, new_pid); + log_verbose(">>> %s, deep %d, parent-pid %d, child-pid %d", "mdbx_env_resurrect_after_fork()", deep, current_pid, + new_pid); log_flush(); int err = mdbx_env_resurrect_after_fork(db_guard.get()); - log_verbose("<<< %s, deep %d, parent-pid %d, child-pid %d, err %d", - "mdbx_env_resurrect_after_fork()", deep, current_pid, new_pid, - err); + log_verbose("<<< %s, deep %d, parent-pid %d, child-pid %d, err %d", "mdbx_env_resurrect_after_fork()", deep, + current_pid, new_pid, err); log_flush(); if (err != MDBX_SUCCESS) failure_perror("mdbx_env_resurrect_after_fork()", err); @@ -134,12 +125,10 @@ bool testcase_smoke4fork::run() { mdbx_txn_abort(txn_guard.release()); } if (!smoke()) { - log_notice("%s[deep %d, pid %d] probe %s", "fork-child", deep, new_pid, - "failed"); + log_notice("%s[deep %d, pid %d] probe %s", "fork-child", deep, new_pid, "failed"); return false; } - log_verbose("%s[deep %d, pid %d] probe %s", "fork-child", deep, new_pid, - "done"); + log_verbose("%s[deep %d, pid %d] probe %s", "fork-child", deep, new_pid, "done"); log_flush(); return true; } @@ -154,12 +143,10 @@ bool testcase_smoke4fork::run() { if (WIFEXITED(status)) { const int code = WEXITSTATUS(status); if (code != EXIT_SUCCESS) { - log_notice("%s[deep %d, pid %d] child-pid %d failed, err %d", - "fork-child", deep, current_pid, child, code); + log_notice("%s[deep %d, pid %d] child-pid %d failed, err %d", "fork-child", deep, current_pid, child, code); return false; } - log_notice("%s[deep %d, pid %d] child-pid %d done", "fork-child", deep, - current_pid, child); + log_notice("%s[deep %d, pid %d] child-pid %d done", "fork-child", deep, current_pid, child); } else if (WIFSIGNALED(status)) { const int sig = WTERMSIG(status); switch (sig) { @@ -168,12 +155,12 @@ bool testcase_smoke4fork::run() { case SIGFPE: case SIGILL: case SIGSEGV: - log_notice("%s[deep %d, pid %d] child-pid %d %s by SIG%s", "fork-child", - deep, current_pid, child, "terminated", signal_name(sig)); + log_notice("%s[deep %d, pid %d] child-pid %d %s by SIG%s", "fork-child", deep, current_pid, child, "terminated", + signal_name(sig)); break; default: - log_notice("%s[deep %d, pid %d] child-id %d %s by SIG%s", "fork-child", - deep, current_pid, child, "killed", signal_name(sig)); + log_notice("%s[deep %d, pid %d] child-id %d %s by SIG%s", "fork-child", deep, current_pid, child, "killed", + signal_name(sig)); } return false; } else { @@ -181,12 +168,10 @@ bool testcase_smoke4fork::run() { } if (!smoke()) { - log_notice("%s[deep %d, pid %d] probe %s", "post-fork", deep, current_pid, - "failed"); + log_notice("%s[deep %d, pid %d] probe %s", "post-fork", deep, current_pid, "failed"); return false; } - log_verbose("%s[deep %d, pid %d] probe %s", "post-fork", deep, current_pid, - "done"); + log_verbose("%s[deep %d, pid %d] probe %s", "post-fork", deep, current_pid, "done"); return true; } @@ -196,16 +181,14 @@ class testcase_forkread : public testcase_smoke4fork { using inherited = testcase_smoke4fork; public: - testcase_forkread(const actor_config &config, const mdbx_pid_t pid) - : testcase_smoke4fork(config, pid) {} + testcase_forkread(const actor_config &config, const mdbx_pid_t pid) : testcase_smoke4fork(config, pid) {} bool smoke() override; }; REGISTER_TESTCASE(forkread); bool testcase_forkread::smoke() { MDBX_envinfo env_info; - int err = mdbx_env_info_ex(db_guard.get(), txn_guard.get(), &env_info, - sizeof(env_info)); + int err = mdbx_env_info_ex(db_guard.get(), txn_guard.get(), &env_info, sizeof(env_info)); if (err) failure_perror("mdbx_env_info_ex()", err); @@ -217,8 +200,7 @@ bool testcase_forkread::smoke() { if (err) failure_perror("mdbx_txn_info()", err); fetch_canary(); - err = mdbx_env_info_ex(db_guard.get(), txn_guard.get(), &env_info, - sizeof(env_info)); + err = mdbx_env_info_ex(db_guard.get(), txn_guard.get(), &env_info, sizeof(env_info)); if (err) failure_perror("mdbx_env_info_ex()", err); @@ -226,15 +208,13 @@ bool testcase_forkread::smoke() { if (dbi_invalid) { err = mdbx_dbi_sequence(txn_guard.get(), dbi, &seq, 0); if (unlikely(err != (dbi ? MDBX_BAD_DBI : MDBX_SUCCESS))) - failure("unexpected '%s' from mdbx_dbi_sequence(get, bad_dbi %d)", - mdbx_strerror(err), dbi); + failure("unexpected '%s' from mdbx_dbi_sequence(get, bad_dbi %d)", mdbx_strerror(err), dbi); open_dbi(); } if (!dbi_invalid) { err = mdbx_dbi_sequence(txn_guard.get(), dbi, &seq, 0); if (unlikely(err != MDBX_SUCCESS)) - failure("unexpected '%s' from mdbx_dbi_sequence(get, dbi %d)", - mdbx_strerror(err), dbi); + failure("unexpected '%s' from mdbx_dbi_sequence(get, dbi %d)", mdbx_strerror(err), dbi); } txn_end(false); return true; @@ -246,8 +226,7 @@ class testcase_forkwrite : public testcase_forkread { using inherited = testcase_forkread; public: - testcase_forkwrite(const actor_config &config, const mdbx_pid_t pid) - : testcase_forkread(config, pid) {} + testcase_forkwrite(const actor_config &config, const mdbx_pid_t pid) : testcase_forkread(config, pid) {} bool smoke() override; }; REGISTER_TESTCASE(forkwrite); @@ -266,15 +245,13 @@ bool testcase_forkwrite::smoke() { if (dbi_invalid) { int err = mdbx_dbi_sequence(txn_guard.get(), dbi, &seq, 0); if (unlikely(err != (dbi ? MDBX_BAD_DBI : MDBX_EACCESS))) - failure("unexpected '%s' from mdbx_dbi_sequence(get, bad_dbi %d)", - mdbx_strerror(err), dbi); + failure("unexpected '%s' from mdbx_dbi_sequence(get, bad_dbi %d)", mdbx_strerror(err), dbi); open_dbi(); } if (!dbi_invalid) { int err = mdbx_dbi_sequence(txn_guard.get(), dbi, &seq, 1); if (unlikely(err != MDBX_SUCCESS)) - failure("unexpected '%s' from mdbx_dbi_sequence(inc, dbi %d)", - mdbx_strerror(err), dbi); + failure("unexpected '%s' from mdbx_dbi_sequence(inc, dbi %d)", mdbx_strerror(err), dbi); } txn_end(false); diff --git a/test/hill.c++ b/test/hill.c++ index 6689a0db..ac6254e9 100644 --- a/test/hill.c++ +++ b/test/hill.c++ @@ -25,8 +25,7 @@ class testcase_hill : public testcase { public: - testcase_hill(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid) {} + testcase_hill(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} bool run() override; }; REGISTER_TESTCASE(hill); @@ -50,13 +49,9 @@ bool testcase_hill::run() { keygen::buffer b_data = keygen::alloc(config.params.datalen_max); const MDBX_put_flags_t insert_flags = - (config.params.table_flags & MDBX_DUPSORT) - ? MDBX_NODUPDATA - : MDBX_NODUPDATA | MDBX_NOOVERWRITE; + (config.params.table_flags & MDBX_DUPSORT) ? MDBX_NODUPDATA : MDBX_NODUPDATA | MDBX_NOOVERWRITE; const MDBX_put_flags_t update_flags = - (config.params.table_flags & MDBX_DUPSORT) - ? MDBX_CURRENT | MDBX_NODUPDATA | MDBX_NOOVERWRITE - : MDBX_NODUPDATA; + (config.params.table_flags & MDBX_DUPSORT) ? MDBX_CURRENT | MDBX_NODUPDATA | MDBX_NOOVERWRITE : MDBX_NODUPDATA; uint64_t serial_count = 0; uint64_t committed_serial = serial_count; @@ -80,8 +75,7 @@ bool testcase_hill::run() { // создаем первую запись из пары const keygen::serial_t age_shift = keyvalue_maker.remix_age(a_serial); - log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift, - a_serial); + log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift, a_serial); generate_pair(a_serial, a_key, a_data_1, age_shift); err = insert(a_key, a_data_1, insert_flags); @@ -154,8 +148,7 @@ bool testcase_hill::run() { } // обновляем данные в первой записи - log_trace("uphill: update-a (age %" PRIu64 "->0) %" PRIu64, age_shift, - a_serial); + log_trace("uphill: update-a (age %" PRIu64 "->0) %" PRIu64, age_shift, a_serial); generate_pair(a_serial, a_key, a_data_0, 0); checkdata("uphill: update-a", dbi, a_key->value, a_data_1->value); err = replace(a_key, a_data_0, a_data_1, update_flags); @@ -271,8 +264,7 @@ bool testcase_hill::run() { if (str.back() == '-') str.append(std::to_string(prev)); - log_notice("hill: reached %d tree depth & %s sub-tree depth(s)", - stat.ms_depth, str.c_str()); + log_notice("hill: reached %d tree depth & %s sub-tree depth(s)", stat.ms_depth, str.c_str()); } if ((config.params.table_flags & MDBX_DUPSORT) == 0) { @@ -292,16 +284,14 @@ bool testcase_hill::run() { // обновляем первую запись из пары const keygen::serial_t age_shift = keyvalue_maker.remix_age(a_serial); - log_trace("downhill: update-a (age 0->%" PRIu64 ") %" PRIu64, age_shift, - a_serial); + log_trace("downhill: update-a (age 0->%" PRIu64 ") %" PRIu64, age_shift, a_serial); generate_pair(a_serial, a_key, a_data_0, 0); generate_pair(a_serial, a_key, a_data_1, age_shift); checkdata("downhill: update-a", dbi, a_key->value, a_data_0->value); err = replace(a_key, a_data_1, a_data_0, update_flags); if (unlikely(err != MDBX_SUCCESS)) { if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("downhill: bailout at update-a due '%s'", - mdbx_strerror(err)); + log_notice("downhill: bailout at update-a due '%s'", mdbx_strerror(err)); txn_end(true); speculum = speculum_committed; break; @@ -334,8 +324,7 @@ bool testcase_hill::run() { err = insert(b_key, b_data, insert_flags); if (unlikely(err != MDBX_SUCCESS)) { if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("downhill: bailout at insert-a due '%s'", - mdbx_strerror(err)); + log_notice("downhill: bailout at insert-a due '%s'", mdbx_strerror(err)); txn_end(true); speculum = speculum_committed; break; @@ -363,14 +352,12 @@ bool testcase_hill::run() { } // удаляем первую запись - log_trace("downhill: delete-a (age %" PRIu64 ") %" PRIu64, age_shift, - a_serial); + log_trace("downhill: delete-a (age %" PRIu64 ") %" PRIu64, age_shift, a_serial); checkdata("downhill: delete-a", dbi, a_key->value, a_data_1->value); err = remove(a_key, a_data_1); if (unlikely(err != MDBX_SUCCESS)) { if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("downhill: bailout at delete-a due '%s'", - mdbx_strerror(err)); + log_notice("downhill: bailout at delete-a due '%s'", mdbx_strerror(err)); txn_end(true); speculum = speculum_committed; break; @@ -403,8 +390,7 @@ bool testcase_hill::run() { err = remove(b_key, b_data); if (unlikely(err != MDBX_SUCCESS)) { if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { - log_notice("downhill: bailout at delete-b due '%s'", - mdbx_strerror(err)); + log_notice("downhill: bailout at delete-b due '%s'", mdbx_strerror(err)); txn_end(true); speculum = speculum_committed; break; diff --git a/test/jitter.c++ b/test/jitter.c++ index 1f56978c..3bf75100 100644 --- a/test/jitter.c++ +++ b/test/jitter.c++ @@ -8,8 +8,7 @@ protected: void check_dbi_error(int expect, const char *stage); public: - testcase_jitter(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid) {} + testcase_jitter(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} bool run() override; }; REGISTER_TESTCASE(jitter); @@ -18,8 +17,7 @@ void testcase_jitter::check_dbi_error(int expect, const char *stage) { MDBX_stat stat; int err = mdbx_dbi_stat(txn_guard.get(), dbi, &stat, sizeof(stat)); if (err != expect) - failure("unexpected result for %s dbi-handle: expect %d, got %d", stage, - expect, err); + failure("unexpected result for %s dbi-handle: expect %d, got %d", stage, expect, err); } bool testcase_jitter::run() { @@ -31,8 +29,7 @@ bool testcase_jitter::run() { tablename_buf buffer; const char *const tablename = db_tablename(buffer); tablename_buf buffer_renamed; - const char *const tablename_renamed = - db_tablename(buffer_renamed, ".renamed"); + const char *const tablename_renamed = db_tablename(buffer_renamed, ".renamed"); while (should_continue()) { jitter_delay(); @@ -81,9 +78,8 @@ bool testcase_jitter::run() { // restore DBI dbi = db_table_open(false, renamed); if (renamed) { - err = mdbx_dbi_open( - txn_guard.get(), tablename_renamed, - flipcoin() ? MDBX_DB_ACCEDE : config.params.table_flags, &dbi); + err = mdbx_dbi_open(txn_guard.get(), tablename_renamed, flipcoin() ? MDBX_DB_ACCEDE : config.params.table_flags, + &dbi); if (unlikely(err != MDBX_SUCCESS)) failure_perror("open-renamed", err); err = mdbx_dbi_rename(txn_guard.get(), dbi, tablename); @@ -100,13 +96,10 @@ bool testcase_jitter::run() { if (upper_limit < 1) { MDBX_envinfo info; - err = mdbx_env_info_ex(db_guard.get(), txn_guard.get(), &info, - sizeof(info)); + err = mdbx_env_info_ex(db_guard.get(), txn_guard.get(), &info, sizeof(info)); if (err) failure_perror("mdbx_env_info_ex()", err); - upper_limit = (info.mi_geo.upper < INTPTR_MAX) - ? (intptr_t)info.mi_geo.upper - : INTPTR_MAX; + upper_limit = (info.mi_geo.upper < INTPTR_MAX) ? (intptr_t)info.mi_geo.upper : INTPTR_MAX; } if (flipcoin()) { @@ -156,29 +149,26 @@ bool testcase_jitter::run() { fetch_canary(); update_canary(1); if (global::config::geometry_jitter) { - err = mdbx_env_set_geometry( - db_guard.get(), -1, -1, - coin4size ? upper_limit * 2 / 3 : upper_limit * 3 / 2, -1, -1, -1); - if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE && - err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE && err != MDBX_EPERM) + err = mdbx_env_set_geometry(db_guard.get(), -1, -1, coin4size ? upper_limit * 2 / 3 : upper_limit * 3 / 2, -1, + -1, -1); + if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE && err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE && + err != MDBX_EPERM) failure_perror("mdbx_env_set_geometry-1", err); } } if (flipcoin()) { uint64_t unused; - err = mdbx_dbi_sequence(txn_guard.get(), MAIN_DBI, &unused, - mode_readonly() ? 0 : 1); + err = mdbx_dbi_sequence(txn_guard.get(), MAIN_DBI, &unused, mode_readonly() ? 0 : 1); if (err) failure_perror("mdbx_dbi_sequence()", err); } txn_end(flipcoin()); if (global::config::geometry_jitter) { - err = mdbx_env_set_geometry( - db_guard.get(), -1, -1, - !coin4size ? upper_limit * 2 / 3 : upper_limit * 3 / 2, -1, -1, -1); - if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE && - err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE && err != MDBX_EPERM) + err = mdbx_env_set_geometry(db_guard.get(), -1, -1, !coin4size ? upper_limit * 2 / 3 : upper_limit * 3 / 2, -1, + -1, -1); + if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE && err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE && + err != MDBX_EPERM) failure_perror("mdbx_env_set_geometry-2", err); } @@ -191,18 +181,16 @@ bool testcase_jitter::run() { if (global::config::geometry_jitter) { jitter_delay(); - err = mdbx_env_set_geometry(db_guard.get(), -1, -1, upper_limit, -1, -1, - -1); - if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE && - err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE && err != MDBX_EPERM) + err = mdbx_env_set_geometry(db_guard.get(), -1, -1, upper_limit, -1, -1, -1); + if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE && err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE && + err != MDBX_EPERM) failure_perror("mdbx_env_set_geometry-3", err); } db_close(); /* just 'align' nops with other tests with batching */ - const auto batching = - std::max(config.params.batch_read, config.params.batch_write); + const auto batching = std::max(config.params.batch_read, config.params.batch_write); report(std::max(1u, batching / 2)); } return true; diff --git a/test/keygen.c++ b/test/keygen.c++ index a3879770..8ac4987d 100644 --- a/test/keygen.c++ +++ b/test/keygen.c++ @@ -7,25 +7,18 @@ static const uint64_t primes[64] = { /* */ 0, 1, 3, 7, 13, 31, 61, 127, 251, 509, 1021, 2039, 4093, 8191, 16381, /* */ - UINT64_C(32749), UINT64_C(65521), UINT64_C(131071), UINT64_C(262139), - UINT64_C(524287), UINT64_C(1048573), UINT64_C(2097143), UINT64_C(4194301), - UINT64_C(8388593), UINT64_C(16777213), UINT64_C(33554393), - UINT64_C(67108859), UINT64_C(134217689), UINT64_C(268435399), - UINT64_C(536870909), UINT64_C(1073741789), UINT64_C(2147483647), - UINT64_C(4294967291), UINT64_C(8589934583), UINT64_C(17179869143), - UINT64_C(34359738337), UINT64_C(68719476731), UINT64_C(137438953447), - UINT64_C(274877906899), UINT64_C(549755813881), UINT64_C(1099511627689), - UINT64_C(2199023255531), UINT64_C(4398046511093), UINT64_C(8796093022151), - UINT64_C(17592186044399), UINT64_C(35184372088777), - UINT64_C(70368744177643), UINT64_C(140737488355213), - UINT64_C(281474976710597), UINT64_C(562949953421231), - UINT64_C(1125899906842597), UINT64_C(2251799813685119), - UINT64_C(4503599627370449), UINT64_C(9007199254740881), - UINT64_C(18014398509481951), UINT64_C(36028797018963913), - UINT64_C(72057594037927931), UINT64_C(144115188075855859), - UINT64_C(288230376151711717), UINT64_C(576460752303423433), - UINT64_C(1152921504606846883), UINT64_C(2305843009213693951), - UINT64_C(4611686018427387847), UINT64_C(9223372036854775783)}; + UINT64_C(32749), UINT64_C(65521), UINT64_C(131071), UINT64_C(262139), UINT64_C(524287), UINT64_C(1048573), + UINT64_C(2097143), UINT64_C(4194301), UINT64_C(8388593), UINT64_C(16777213), UINT64_C(33554393), UINT64_C(67108859), + UINT64_C(134217689), UINT64_C(268435399), UINT64_C(536870909), UINT64_C(1073741789), UINT64_C(2147483647), + UINT64_C(4294967291), UINT64_C(8589934583), UINT64_C(17179869143), UINT64_C(34359738337), UINT64_C(68719476731), + UINT64_C(137438953447), UINT64_C(274877906899), UINT64_C(549755813881), UINT64_C(1099511627689), + UINT64_C(2199023255531), UINT64_C(4398046511093), UINT64_C(8796093022151), UINT64_C(17592186044399), + UINT64_C(35184372088777), UINT64_C(70368744177643), UINT64_C(140737488355213), UINT64_C(281474976710597), + UINT64_C(562949953421231), UINT64_C(1125899906842597), UINT64_C(2251799813685119), UINT64_C(4503599627370449), + UINT64_C(9007199254740881), UINT64_C(18014398509481951), UINT64_C(36028797018963913), UINT64_C(72057594037927931), + UINT64_C(144115188075855859), UINT64_C(288230376151711717), UINT64_C(576460752303423433), + UINT64_C(1152921504606846883), UINT64_C(2305843009213693951), UINT64_C(4611686018427387847), + UINT64_C(9223372036854775783)}; /* static unsigned supid_log2(uint64_t v) { unsigned r = 0; @@ -39,9 +32,7 @@ static const uint64_t primes[64] = { namespace keygen { /* LY: https://en.wikipedia.org/wiki/Injective_function */ -serial_t injective(const serial_t serial, - const unsigned bits /* at least serial_minwith (8) */, - const serial_t salt) { +serial_t injective(const serial_t serial, const unsigned bits /* at least serial_minwith (8) */, const serial_t salt) { assert(bits >= serial_minwith && bits <= serial_maxwith); /* LY: All these "magic" prime numbers were found @@ -49,26 +40,21 @@ serial_t injective(const serial_t serial, static const uint64_t m[64 - serial_minwith + 1] = { /* 8 - 24 */ - 113, 157, 397, 653, 1753, 5641, 9697, 23873, 25693, 80833, 105953, 316937, - 309277, 834497, 1499933, 4373441, 10184137, + 113, 157, 397, 653, 1753, 5641, 9697, 23873, 25693, 80833, 105953, 316937, 309277, 834497, 1499933, 4373441, + 10184137, /* 25 - 64 */ - 10184137, 17279209, 33990377, 67295161, 284404553, 1075238767, 6346721573, - 6924051577, 19204053433, 45840188887, 53625693977, 73447827913, - 141638870249, 745683604649, 1283334050489, 1100828289853, 2201656586197, - 5871903036137, 11238507001417, 45264020802263, 105008404482889, - 81921776907059, 199987980256399, 307207457507641, 946769023178273, - 2420886491930041, 3601632139991929, 11984491914483833, 21805846439714153, - 23171543400565993, 53353226456762893, 155627817337932409, - 227827205384840249, 816509268558278821, 576933057762605689, - 2623957345935638441, 5048241705479929949, 4634245581946485653, - 4613509448041658233, 4952535426879925961}; - static const uint8_t s[64 - serial_minwith + 1] = { - /* 8 - 24 */ - 2, 3, 4, 4, 2, 4, 3, 3, 7, 3, 3, 4, 8, 3, 10, 3, 11, - /* 25 - 64 */ - 11, 9, 9, 9, 11, 10, 5, 14, 11, 16, 14, 12, 13, 16, 19, 10, 10, 21, 7, 20, - 10, 14, 22, 19, 3, 21, 18, 19, 26, 24, 2, 21, 25, 29, 24, 10, 11, 14, 20, - 19}; + 10184137, 17279209, 33990377, 67295161, 284404553, 1075238767, 6346721573, 6924051577, 19204053433, 45840188887, + 53625693977, 73447827913, 141638870249, 745683604649, 1283334050489, 1100828289853, 2201656586197, 5871903036137, + 11238507001417, 45264020802263, 105008404482889, 81921776907059, 199987980256399, 307207457507641, + 946769023178273, 2420886491930041, 3601632139991929, 11984491914483833, 21805846439714153, 23171543400565993, + 53353226456762893, 155627817337932409, 227827205384840249, 816509268558278821, 576933057762605689, + 2623957345935638441, 5048241705479929949, 4634245581946485653, 4613509448041658233, 4952535426879925961}; + static const uint8_t s[64 - serial_minwith + 1] = {/* 8 - 24 */ + 2, 3, 4, 4, 2, 4, 3, 3, 7, 3, 3, 4, 8, 3, 10, 3, 11, + /* 25 - 64 */ + 11, 9, 9, 9, 11, 10, 5, 14, 11, 16, 14, 12, 13, 16, 19, 10, 10, 21, + 7, 20, 10, 14, 22, 19, 3, 21, 18, 19, 26, 24, 2, 21, 25, 29, 24, + 10, 11, 14, 20, 19}; const auto mask = actor_params::serial_mask(bits); const auto mult = m[bits - 8]; @@ -83,49 +69,40 @@ serial_t injective(const serial_t serial, result ^= (result & mask) >> shift; result &= mask; - log_trace("keygen-injective: serial %" PRIu64 "/%u @%" PRIx64 ",%u,%" PRIu64 - " => %" PRIu64 "/%u", - serial, bits, mult, shift, salt, result, bits); + log_trace("keygen-injective: serial %" PRIu64 "/%u @%" PRIx64 ",%u,%" PRIu64 " => %" PRIu64 "/%u", serial, bits, mult, + shift, salt, result, bits); return result; } -void __hot maker::pair(serial_t serial, const buffer &key, buffer &value, - serial_t value_age, const bool keylen_changeable) { +void __hot maker::pair(serial_t serial, const buffer &key, buffer &value, serial_t value_age, + const bool keylen_changeable) { assert(mapping.width >= serial_minwith && mapping.width <= serial_maxwith); assert(mapping.split <= mapping.width); assert(mapping.mesh <= mapping.width); assert(mapping.rotate <= mapping.width); assert(mapping.offset <= actor_params::serial_mask(mapping.width)); assert(!(key_essentials.flags & - ~(essentials::prng_fill_flag | - unsigned(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT)))); + ~(essentials::prng_fill_flag | unsigned(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT)))); assert(!(value_essentials.flags & - ~(essentials::prng_fill_flag | - unsigned(MDBX_INTEGERDUP | MDBX_REVERSEDUP | MDBX_DUPFIXED)))); + ~(essentials::prng_fill_flag | unsigned(MDBX_INTEGERDUP | MDBX_REVERSEDUP | MDBX_DUPFIXED)))); - log_trace("keygen-pair: serial %" PRIu64 ", data-age %" PRIu64, serial, - value_age); + log_trace("keygen-pair: serial %" PRIu64 ", data-age %" PRIu64, serial, value_age); if (mapping.mesh >= serial_minwith) { - serial = (serial & ~actor_params::serial_mask(mapping.mesh)) | - injective(serial, mapping.mesh, salt); + serial = (serial & ~actor_params::serial_mask(mapping.mesh)) | injective(serial, mapping.mesh, salt); log_trace("keygen-pair: mesh@%u => %" PRIu64, mapping.mesh, serial); } if (mapping.rotate) { const unsigned right = mapping.rotate; const unsigned left = mapping.width - right; - serial = (serial << left) | - ((serial & actor_params::serial_mask(mapping.width)) >> right); - log_trace("keygen-pair: rotate@%u => %" PRIu64 ", 0x%" PRIx64, - mapping.rotate, serial, serial); + serial = (serial << left) | ((serial & actor_params::serial_mask(mapping.width)) >> right); + log_trace("keygen-pair: rotate@%u => %" PRIu64 ", 0x%" PRIx64, mapping.rotate, serial, serial); } if (mapping.offset) { - serial = - (serial + mapping.offset) & actor_params::serial_mask(mapping.width); - log_trace("keygen-pair: offset@%" PRIu64 " => %" PRIu64, mapping.offset, - serial); + serial = (serial + mapping.offset) & actor_params::serial_mask(mapping.width); + log_trace("keygen-pair: offset@%" PRIu64 " => %" PRIu64, mapping.offset, serial); } if (base) { serial += base; @@ -144,16 +121,13 @@ void __hot maker::pair(serial_t serial, const buffer &key, buffer &value, * Поэтому key_serial не трогаем, а в value_serial нелинейно вмешиваем * запрошенное количество бит из serial */ value_serial += - (serial ^ (serial >> mapping.split) * UINT64_C(57035339200100753)) & - actor_params::serial_mask(mapping.split); + (serial ^ (serial >> mapping.split) * UINT64_C(57035339200100753)) & actor_params::serial_mask(mapping.split); } - log_trace("keygen-pair: split@%u => k%" PRIu64 ", v%" PRIu64, mapping.split, - key_serial, value_serial); + log_trace("keygen-pair: split@%u => k%" PRIu64 ", v%" PRIu64, mapping.split, key_serial, value_serial); } - log_trace("keygen-pair: key %" PRIu64 ", value %" PRIu64, key_serial, - value_serial); + log_trace("keygen-pair: key %" PRIu64 ", value %" PRIu64, key_serial, value_serial); key_serial = mk_begin(key_serial, key_essentials, *key); value_serial = mk_begin(value_serial, value_essentials, *value); @@ -210,50 +184,39 @@ void __hot maker::pair(serial_t serial, const buffer &key, buffer &value, log_pair(logging::trace, "kv", key, value); } -void maker::setup(const config::actor_params_pod &actor, - unsigned thread_number) { +void maker::setup(const config::actor_params_pod &actor, unsigned thread_number) { #if CONSTEXPR_ENUM_FLAGS_OPERATIONS - static_assert(unsigned(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT | - MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP) < - UINT16_MAX, + static_assert(unsigned(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | + MDBX_REVERSEDUP) < UINT16_MAX, "WTF?"); #else - assert(unsigned(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT | - MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP) < - UINT16_MAX); + assert(unsigned(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | + MDBX_REVERSEDUP) < UINT16_MAX); #endif - key_essentials.flags = uint16_t( - actor.table_flags & - MDBX_db_flags_t(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT)); + key_essentials.flags = + uint16_t(actor.table_flags & MDBX_db_flags_t(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT)); assert(actor.keylen_min <= UINT16_MAX); key_essentials.minlen = uint16_t(actor.keylen_min); assert(actor.keylen_max <= UINT32_MAX); - key_essentials.maxlen = std::min( - uint32_t(actor.keylen_max), - uint32_t(mdbx_limits_keysize_max(actor.pagesize, actor.table_flags))); - key_essentials.bits = (key_essentials.maxlen < sizeof(serial_t)) - ? key_essentials.maxlen * CHAR_BIT - : sizeof(serial_t) * CHAR_BIT; + key_essentials.maxlen = + std::min(uint32_t(actor.keylen_max), uint32_t(mdbx_limits_keysize_max(actor.pagesize, actor.table_flags))); + key_essentials.bits = + (key_essentials.maxlen < sizeof(serial_t)) ? key_essentials.maxlen * CHAR_BIT : sizeof(serial_t) * CHAR_BIT; key_essentials.mask = actor_params::serial_mask(key_essentials.bits); - assert(key_essentials.bits > 63 || - key_essentials.mask > primes[key_essentials.bits]); + assert(key_essentials.bits > 63 || key_essentials.mask > primes[key_essentials.bits]); - value_essentials.flags = uint16_t( - actor.table_flags & - MDBX_db_flags_t(MDBX_INTEGERDUP | MDBX_REVERSEDUP | MDBX_DUPFIXED)); + value_essentials.flags = + uint16_t(actor.table_flags & MDBX_db_flags_t(MDBX_INTEGERDUP | MDBX_REVERSEDUP | MDBX_DUPFIXED)); assert(actor.datalen_min <= UINT16_MAX); value_essentials.minlen = uint16_t(actor.datalen_min); assert(actor.datalen_max <= UINT32_MAX); - value_essentials.maxlen = std::min( - uint32_t(actor.datalen_max), - uint32_t(mdbx_limits_valsize_max(actor.pagesize, actor.table_flags))); - value_essentials.bits = (value_essentials.maxlen < sizeof(serial_t)) - ? value_essentials.maxlen * CHAR_BIT - : sizeof(serial_t) * CHAR_BIT; + value_essentials.maxlen = + std::min(uint32_t(actor.datalen_max), uint32_t(mdbx_limits_valsize_max(actor.pagesize, actor.table_flags))); + value_essentials.bits = + (value_essentials.maxlen < sizeof(serial_t)) ? value_essentials.maxlen * CHAR_BIT : sizeof(serial_t) * CHAR_BIT; value_essentials.mask = actor_params::serial_mask(value_essentials.bits); - assert(value_essentials.bits > 63 || - value_essentials.mask > primes[value_essentials.bits]); + assert(value_essentials.bits > 63 || value_essentials.mask > primes[value_essentials.bits]); if (!actor.keygen.zero_fill) { key_essentials.flags |= essentials::prng_fill_flag; @@ -262,43 +225,31 @@ void maker::setup(const config::actor_params_pod &actor, mapping = actor.keygen; const auto split = mapping.split; - while (mapping.split > - value_essentials.bits - essentials::value_age_minwidth || - mapping.split >= mapping.width) + while (mapping.split > value_essentials.bits - essentials::value_age_minwidth || mapping.split >= mapping.width) mapping.split -= 1; if (split != mapping.split) - log_notice("keygen: reduce mapping-split from %u to %u", split, - mapping.split); + log_notice("keygen: reduce mapping-split from %u to %u", split, mapping.split); const auto width = mapping.width; - while (unsigned((actor.table_flags & MDBX_DUPSORT) - ? mapping.width - mapping.split - : mapping.width) > key_essentials.bits) + while (unsigned((actor.table_flags & MDBX_DUPSORT) ? mapping.width - mapping.split : mapping.width) > + key_essentials.bits) mapping.width -= 1; if (width != mapping.width) - log_notice("keygen: reduce mapping-width from %u to %u", width, - mapping.width); + log_notice("keygen: reduce mapping-width from %u to %u", width, mapping.width); value_age_bits = value_essentials.bits - mapping.split; value_age_mask = actor_params::serial_mask(value_age_bits); assert(value_age_bits >= essentials::value_age_minwidth); - salt = (prng_state ^ - (thread_number * 1575554837) * UINT64_C(59386707711075671)) * - UINT64_C(14653293970879851569); + salt = (prng_state ^ (thread_number * 1575554837) * UINT64_C(59386707711075671)) * UINT64_C(14653293970879851569); base = actor.serial_base(); } bool maker::is_unordered() const { - return mapping.rotate || - mapping.mesh > ((MDBX_db_flags_t(key_essentials.flags) & MDBX_DUPSORT) - ? 0 - : mapping.split); + return mapping.rotate || mapping.mesh > ((MDBX_db_flags_t(key_essentials.flags) & MDBX_DUPSORT) ? 0 : mapping.split); } -void maker::seek2end(serial_t &serial) const { - serial = actor_params::serial_mask(mapping.width) - 1; -} +void maker::seek2end(serial_t &serial) const { serial = actor_params::serial_mask(mapping.width) - 1; } bool maker::increment(serial_t &serial, int64_t delta) const { if (serial > actor_params::serial_mask(mapping.width)) { @@ -308,17 +259,12 @@ bool maker::increment(serial_t &serial, int64_t delta) const { } serial_t target = serial + delta; - if (target > actor_params::serial_mask(mapping.width) || - ((delta > 0) ? target < serial : target > serial)) { - log_extra("keygen-increment: %" PRIu64 "%-" PRId64 " => %" PRIu64 - ", overflow", - serial, delta, target); + if (target > actor_params::serial_mask(mapping.width) || ((delta > 0) ? target < serial : target > serial)) { + log_extra("keygen-increment: %" PRIu64 "%-" PRId64 " => %" PRIu64 ", overflow", serial, delta, target); return false; } - log_extra("keygen-increment: %" PRIu64 "%-" PRId64 " => %" PRIu64 - ", continue", - serial, delta, target); + log_extra("keygen-increment: %" PRIu64 "%-" PRId64 " => %" PRIu64 ", continue", serial, delta, target); serial = target; return true; } @@ -363,8 +309,7 @@ buffer alloc(size_t limit) { return buffer(ptr); } -serial_t __hot maker::mk_begin(serial_t serial, const essentials ¶ms, - result &out) { +serial_t __hot maker::mk_begin(serial_t serial, const essentials ¶ms, result &out) { assert(out.limit >= params.maxlen); assert(params.maxlen >= params.minlen); assert(serial <= params.mask); @@ -374,9 +319,8 @@ serial_t __hot maker::mk_begin(serial_t serial, const essentials ¶ms, assert(params.mask > primes[params.bits]); #else const serial_t maxbits = params.maxlen * CHAR_BIT; - serial ^= (serial >> maxbits / 2) * - serial_t((sizeof(serial_t) > 4) ? UINT64_C(40719303417517073) - : UINT32_C(3708688457)); + serial ^= + (serial >> maxbits / 2) * serial_t((sizeof(serial_t) > 4) ? UINT64_C(40719303417517073) : UINT32_C(3708688457)); serial &= params.mask; #endif assert(params.maxlen >= length(serial)); @@ -388,9 +332,7 @@ serial_t __hot maker::mk_begin(serial_t serial, const essentials ¶ms, if (serial % (variation + serial_t(1))) { auto refix = serial * UINT64_C(48835288005252737); refix ^= refix >> 32; - out.value.iov_len = - std::max(out.value.iov_len, - params.minlen + size_t(1) + size_t(refix) % variation); + out.value.iov_len = std::max(out.value.iov_len, params.minlen + size_t(1) + size_t(refix) % variation); } } @@ -400,18 +342,14 @@ serial_t __hot maker::mk_begin(serial_t serial, const essentials ¶ms, return serial; } -void __hot maker::mk_continue(const serial_t serial, const essentials ¶ms, - result &out) { +void __hot maker::mk_continue(const serial_t serial, const essentials ¶ms, result &out) { #if CONSTEXPR_ENUM_FLAGS_OPERATIONS - static_assert( - (essentials::prng_fill_flag & - unsigned(MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERKEY | - MDBX_INTEGERDUP | MDBX_REVERSEKEY | MDBX_REVERSEDUP)) == 0, - "WTF?"); + static_assert((essentials::prng_fill_flag & unsigned(MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERKEY | + MDBX_INTEGERDUP | MDBX_REVERSEKEY | MDBX_REVERSEDUP)) == 0, + "WTF?"); #else - assert((essentials::prng_fill_flag & - unsigned(MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERKEY | - MDBX_INTEGERDUP | MDBX_REVERSEKEY | MDBX_REVERSEDUP)) == 0); + assert((essentials::prng_fill_flag & unsigned(MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERKEY | MDBX_INTEGERDUP | + MDBX_REVERSEKEY | MDBX_REVERSEDUP)) == 0); #endif assert(length(serial) <= out.value.iov_len); out.value.iov_base = out.bytes; @@ -423,8 +361,7 @@ void __hot maker::mk_continue(const serial_t serial, const essentials ¶ms, if (!is_byteorder_le() && out.value.iov_len != 8) out.u32 = uint32_t(serial); } else { - const auto prefix = - std::max(std::min(unsigned(params.minlen), 8u), length(serial)); + const auto prefix = std::max(std::min(unsigned(params.minlen), 8u), length(serial)); out.u64 = htobe64(serial); out.value.iov_base = out.bytes + 8 - prefix; if (out.value.iov_len > prefix) { @@ -434,28 +371,22 @@ void __hot maker::mk_continue(const serial_t serial, const essentials ¶ms, } else memset(out.bytes + 8, '\0', out.value.iov_len - prefix); } - if (unlikely(MDBX_db_flags_t(params.flags) & - (MDBX_REVERSEKEY | MDBX_REVERSEDUP))) - std::reverse((char *)out.value.iov_base, - (char *)out.value.iov_base + out.value.iov_len); + if (unlikely(MDBX_db_flags_t(params.flags) & (MDBX_REVERSEKEY | MDBX_REVERSEDUP))) + std::reverse((char *)out.value.iov_base, (char *)out.value.iov_base + out.value.iov_len); } assert(out.value.iov_len >= params.minlen); assert(out.value.iov_len <= params.maxlen); assert(out.value.iov_len >= length(serial)); assert(out.value.iov_base >= out.bytes); - assert((char *)out.value.iov_base + out.value.iov_len <= - (char *)&out.bytes + out.limit); + assert((char *)out.value.iov_base + out.value.iov_len <= (char *)&out.bytes + out.limit); } -void log_pair(logging::loglevel level, const char *prefix, const buffer &key, - buffer &value) { +void log_pair(logging::loglevel level, const char *prefix, const buffer &key, buffer &value) { if (log_enabled(level)) { char dump_key[4096], dump_value[4096]; - logging::output( - level, "%s-pair: key %s, value %s", prefix, - mdbx_dump_val(&key->value, dump_key, sizeof(dump_key)), - mdbx_dump_val(&value->value, dump_value, sizeof(dump_value))); + logging::output(level, "%s-pair: key %s, value %s", prefix, mdbx_dump_val(&key->value, dump_key, sizeof(dump_key)), + mdbx_dump_val(&value->value, dump_value, sizeof(dump_value))); } } diff --git a/test/keygen.h++ b/test/keygen.h++ index a9d69dfe..389cab4d 100644 --- a/test/keygen.h++ +++ b/test/keygen.h++ @@ -60,11 +60,7 @@ namespace keygen { typedef uint64_t serial_t; -enum : serial_t { - serial_minwith = 8, - serial_maxwith = sizeof(serial_t) * 8, - serial_allones = ~(serial_t)0u -}; +enum : serial_t { serial_minwith = 8, serial_maxwith = sizeof(serial_t) * 8, serial_allones = ~(serial_t)0u }; struct result { MDBX_val value; @@ -75,9 +71,7 @@ struct result { uint64_t u64; }; - std::string as_string() const { - return std::string((const char *)value.iov_base, value.iov_len); - } + std::string as_string() const { return std::string((const char *)value.iov_base, value.iov_len); } }; //----------------------------------------------------------------------------- @@ -106,21 +100,17 @@ class maker { unsigned value_age_bits{0}; serial_t value_age_mask{0}; - static serial_t mk_begin(serial_t serial, const essentials ¶ms, - result &out); - static void mk_continue(const serial_t serial, const essentials ¶ms, - result &out); + static serial_t mk_begin(serial_t serial, const essentials ¶ms, result &out); + static void mk_continue(const serial_t serial, const essentials ¶ms, result &out); public: - void pair(serial_t serial, const buffer &key, buffer &value, - serial_t value_age, const bool keylen_changeable); + void pair(serial_t serial, const buffer &key, buffer &value, serial_t value_age, const bool keylen_changeable); void setup(const config::actor_params_pod &actor, unsigned thread_number); bool is_unordered() const; void seek2end(serial_t &serial) const; bool increment(serial_t &serial, int64_t delta) const; - bool increment_key_part(serial_t &serial, int64_t delta, - bool reset_value_part = true) const { + bool increment_key_part(serial_t &serial, int64_t delta, bool reset_value_part = true) const { if (reset_value_part) { serial_t value_part_bits = ((serial_t(1) << mapping.split) - 1); serial |= value_part_bits; @@ -131,12 +121,10 @@ public: } serial_t remix_age(serial_t serial) const { - return (UINT64_C(768097847591) * (serial ^ UINT64_C(768097847591))) & - value_age_mask; + return (UINT64_C(768097847591) * (serial ^ UINT64_C(768097847591))) & value_age_mask; } }; -void log_pair(logging::loglevel level, const char *prefix, const buffer &key, - buffer &value); +void log_pair(logging::loglevel level, const char *prefix, const buffer &key, buffer &value); } /* namespace keygen */ diff --git a/test/log.c++ b/test/log.c++ index 0a3d4355..9b952785 100644 --- a/test/log.c++ +++ b/test/log.c++ @@ -26,15 +26,13 @@ MDBX_NORETURN void failure_perror(const char *what, int errnum) { //----------------------------------------------------------------------------- -static void mdbx_logger(MDBX_log_level_t priority, const char *function, - int line, const char *fmt, +static void mdbx_logger(MDBX_log_level_t priority, const char *function, int line, const char *fmt, va_list args) MDBX_CXX17_NOEXCEPT { if (function) { if (priority == MDBX_LOG_FATAL) log_error("mdbx: fatal failure: %s, %d", function, line); - logging::output_nocheckloglevel( - logging::loglevel(priority), - strncmp(function, "mdbx_", 5) == 0 ? "%s: " : "mdbx %s: ", function); + logging::output_nocheckloglevel(logging::loglevel(priority), + strncmp(function, "mdbx_", 5) == 0 ? "%s: " : "mdbx %s: ", function); logging::feed_ap(fmt, args); } else logging::feed_ap(fmt, args); @@ -58,9 +56,7 @@ static FILE *flow; void setlevel(loglevel priority) { level = priority; int rc = mdbx_setup_debug(MDBX_log_level_t(priority), - MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER | - MDBX_DBG_DUMP, - mdbx_logger); + MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER | MDBX_DBG_DUMP, mdbx_logger); log_trace("set mdbx debug-opts: 0x%02x", rc); } @@ -117,8 +113,7 @@ void ln() { } } -void output_nocheckloglevel_ap(const logging::loglevel priority, - const char *format, va_list ap) { +void output_nocheckloglevel_ap(const logging::loglevel priority, const char *format, va_list ap) { ln(); chrono::time now = chrono::now_realtime(); struct tm tm; @@ -134,11 +129,9 @@ void output_nocheckloglevel_ap(const logging::loglevel priority, if (rc != MDBX_SUCCESS) failure_perror("localtime_r()", rc); - fprintf(stdout, - "[ %02d%02d%02d-%02d:%02d:%02d.%06d_%05lu %-10s %.4s ] %s" /* TODO */, - tm.tm_year - 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, - tm.tm_sec, chrono::fractional2us(now.fractional), (long)osal_getpid(), - prefix_buf, level2str(priority), suffix_ptr); + fprintf(stdout, "[ %02d%02d%02d-%02d:%02d:%02d.%06d_%05lu %-10s %.4s ] %s" /* TODO */, tm.tm_year - 100, + tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, chrono::fractional2us(now.fractional), + (long)osal_getpid(), prefix_buf, level2str(priority), suffix_ptr); va_list ones; memset(&ones, 0, sizeof(ones)) /* zap MSVC and other goofy compilers */; @@ -171,8 +164,7 @@ void output_nocheckloglevel_ap(const logging::loglevel priority, if (same_or_higher(priority, error)) { if (flow) flow = stderr; - fprintf(stderr, "[ %05lu %-10s %.4s ] %s", (long)osal_getpid(), prefix_buf, - level2str(priority), suffix_ptr); + fprintf(stderr, "[ %05lu %-10s %.4s ] %s", (long)osal_getpid(), prefix_buf, level2str(priority), suffix_ptr); vfprintf(stderr, format, ones); va_end(ones); } @@ -206,14 +198,12 @@ bool feed(const char *format, ...) { return true; } -local_suffix::local_suffix(const char *c_str) - : trim_pos(suffix_buf.size()), indent(0) { +local_suffix::local_suffix(const char *c_str) : trim_pos(suffix_buf.size()), indent(0) { suffix_buf.append(c_str); suffix_ptr = suffix_buf.c_str(); } -local_suffix::local_suffix(const std::string &str) - : trim_pos(suffix_buf.size()), indent(0) { +local_suffix::local_suffix(const std::string &str) : trim_pos(suffix_buf.size()), indent(0) { suffix_buf.append(str); suffix_ptr = suffix_buf.c_str(); } @@ -242,8 +232,7 @@ void progress_canary(bool active) { static chrono::time progress_timestamp; chrono::time now = chrono::now_monotonic(); - if (now.fixedpoint - progress_timestamp.fixedpoint < - chrono::from_ms(42).fixedpoint) + if (now.fixedpoint - progress_timestamp.fixedpoint < chrono::from_ms(42).fixedpoint) return; if (osal_progress_push(active)) { @@ -262,20 +251,17 @@ void progress_canary(bool active) { progress_timestamp = now; fprintf(stderr, "%c\b", "-\\|/"[last_point = point]); } - } else if (now.fixedpoint - progress_timestamp.fixedpoint > - chrono::from_seconds(2).fixedpoint) { + } else if (now.fixedpoint - progress_timestamp.fixedpoint > chrono::from_seconds(2).fixedpoint) { progress_timestamp = now; fprintf(stderr, "%c\b", "@*"[now.utc & 1]); } } else { static int count; - if (active && now.fixedpoint - progress_timestamp.fixedpoint > - chrono::from_seconds(1).fixedpoint) { + if (active && now.fixedpoint - progress_timestamp.fixedpoint > chrono::from_seconds(1).fixedpoint) { putc('.', stderr); progress_timestamp = now; ++count; - } else if (now.fixedpoint - progress_timestamp.fixedpoint > - chrono::from_seconds(5).fixedpoint) { + } else if (now.fixedpoint - progress_timestamp.fixedpoint > chrono::from_seconds(5).fixedpoint) { putc("@*"[now.utc & 1], stderr); progress_timestamp = now; ++count; @@ -364,9 +350,7 @@ void log_trouble(const char *where, const char *what, int errnum) { log_error("%s: %s %s", where, what, test_strerror(errnum)); } -bool log_enabled(const logging::loglevel priority) { - return logging::same_or_higher(priority, logging::level); -} +bool log_enabled(const logging::loglevel priority) { return logging::same_or_higher(priority, logging::level); } void log_flush(void) { logging::ln(); diff --git a/test/log.h++ b/test/log.h++ index 838e8de1..bb92783e 100644 --- a/test/log.h++ +++ b/test/log.h++ @@ -29,25 +29,20 @@ inline bool lower(loglevel left, loglevel right) { return left > right; } -inline bool same_or_higher(loglevel left, loglevel right) { - return left <= right; -} +inline bool same_or_higher(loglevel left, loglevel right) { return left <= right; } const char *level2str(const loglevel level); void setup(loglevel priority, const std::string &prefix); void setup(const std::string &prefix); void setlevel(loglevel priority); -void output_nocheckloglevel_ap(const loglevel priority, const char *format, - va_list ap); -bool MDBX_PRINTF_ARGS(2, 3) - output(const loglevel priority, const char *format, ...); +void output_nocheckloglevel_ap(const loglevel priority, const char *format, va_list ap); +bool MDBX_PRINTF_ARGS(2, 3) output(const loglevel priority, const char *format, ...); bool feed_ap(const char *format, va_list ap); bool MDBX_PRINTF_ARGS(1, 2) feed(const char *format, ...); void ln(); -void inline MDBX_PRINTF_ARGS(2, 3) - output_nocheckloglevel(const loglevel priority, const char *format, ...) { +void inline MDBX_PRINTF_ARGS(2, 3) output_nocheckloglevel(const loglevel priority, const char *format, ...) { va_list ap; va_start(ap, format); output_nocheckloglevel_ap(priority, format, ap); @@ -75,9 +70,7 @@ public: } // namespace logging -void MDBX_PRINTF_ARGS(1, 2) static inline log_null(const char *msg, ...) { - return (void)msg; -} +void MDBX_PRINTF_ARGS(1, 2) static inline log_null(const char *msg, ...) { return (void)msg; } void MDBX_PRINTF_ARGS(1, 2) log_extra(const char *msg, ...); void MDBX_PRINTF_ARGS(1, 2) log_trace(const char *msg, ...); void MDBX_PRINTF_ARGS(1, 2) log_debug(const char *msg, ...); diff --git a/test/main.c++ b/test/main.c++ index 9a7fb4df..90c3701e 100644 --- a/test/main.c++ +++ b/test/main.c++ @@ -9,104 +9,103 @@ #endif /* !Windows */ MDBX_NORETURN void usage(void) { - puts( - "usage:\n" - " --help or -h Show this text\n" - "Common parameters:\n" - " --loglevel=[0-7]|[fatal..extra]s" - " --pathname=... Path and/or name of database files\n" - " --repeat=N Set repeat counter\n" - " --threads=N Number of thread (unsupported for now)\n" - " --timeout=N[s|m|h|d] Set timeout in seconds/minutes/hours/days\n" - " --failfast[=YES/no] Lill all actors on first failure/error\n" - " --max-readers=N See mdbx_env_set_maxreaders() description\n" - " --max-tables=N Se mdbx_env_set_maxdbs() description\n" - " --dump-config[=YES/no] Dump entire test config before run\n" - " --progress[=YES/no] Enable/disable progress `canary`\n" - " --console[=yes/no] Enable/disable console-like output\n" - " --cleanup-before[=YES/no] Cleanup/remove and re-create database\n" - " --cleanup-after[=YES/no] Cleanup/remove database after completion\n" - " --prng-seed=N Seed PRNG\n" - "Database size control:\n" - " --pagesize=... Database page size: min, max, 256..65536\n" - " --size-lower=N[K|M|G|T] Lower-bound of size in Kb/Mb/Gb/Tb\n" - " --size-upper Upper-bound of size in Kb/Mb/Gb/Tb\n" - " --size Initial size in Kb/Mb/Gb/Tb\n" - " --shrink-threshold Shrink threshold in Kb/Mb/Gb/Tb\n" - " --growth-step Grow step in Kb/Mb/Gb/Tb\n" - "Predefined complex scenarios/cases:\n" - " --case=... Only `basic` scenario implemented for now\n" - " basic == Simultaneous multi-process execution\n" - " of test-actors: nested,hill,ttl,copy,append,jitter,try\n" - "Test actors:\n" - " --hill Fill-up and empty-down\n" - " by CRUD-operation quads\n" - " --ttl Stochastic time-to-live simulation\n" - " --nested Nested transactionы\n" - " with stochastic-size bellows\n" - " --jitter Jitter/delays simulation\n" - " --try Try write-transaction, no more\n" - " --copy Online copy/backup\n" - " --append Append-mode insertions\n" - " --dead.reader Dead-reader simulator\n" - " --dead.writer Dead-writer simulator\n" + puts("usage:\n" + " --help or -h Show this text\n" + "Common parameters:\n" + " --loglevel=[0-7]|[fatal..extra]s" + " --pathname=... Path and/or name of database files\n" + " --repeat=N Set repeat counter\n" + " --threads=N Number of thread (unsupported for now)\n" + " --timeout=N[s|m|h|d] Set timeout in seconds/minutes/hours/days\n" + " --failfast[=YES/no] Lill all actors on first failure/error\n" + " --max-readers=N See mdbx_env_set_maxreaders() description\n" + " --max-tables=N Se mdbx_env_set_maxdbs() description\n" + " --dump-config[=YES/no] Dump entire test config before run\n" + " --progress[=YES/no] Enable/disable progress `canary`\n" + " --console[=yes/no] Enable/disable console-like output\n" + " --cleanup-before[=YES/no] Cleanup/remove and re-create database\n" + " --cleanup-after[=YES/no] Cleanup/remove database after completion\n" + " --prng-seed=N Seed PRNG\n" + "Database size control:\n" + " --pagesize=... Database page size: min, max, 256..65536\n" + " --size-lower=N[K|M|G|T] Lower-bound of size in Kb/Mb/Gb/Tb\n" + " --size-upper Upper-bound of size in Kb/Mb/Gb/Tb\n" + " --size Initial size in Kb/Mb/Gb/Tb\n" + " --shrink-threshold Shrink threshold in Kb/Mb/Gb/Tb\n" + " --growth-step Grow step in Kb/Mb/Gb/Tb\n" + "Predefined complex scenarios/cases:\n" + " --case=... Only `basic` scenario implemented for now\n" + " basic == Simultaneous multi-process execution\n" + " of test-actors: nested,hill,ttl,copy,append,jitter,try\n" + "Test actors:\n" + " --hill Fill-up and empty-down\n" + " by CRUD-operation quads\n" + " --ttl Stochastic time-to-live simulation\n" + " --nested Nested transactionы\n" + " with stochastic-size bellows\n" + " --jitter Jitter/delays simulation\n" + " --try Try write-transaction, no more\n" + " --copy Online copy/backup\n" + " --append Append-mode insertions\n" + " --dead.reader Dead-reader simulator\n" + " --dead.writer Dead-writer simulator\n" #if !defined(_WIN32) && !defined(_WIN64) - " --fork.reader After-fork reader\n" - " --fork.writer After-fork writer\n" + " --fork.reader After-fork reader\n" + " --fork.writer After-fork writer\n" #endif /* Windows */ - "Actor options:\n" - " --batch.read=N Read-operations batch size\n" - " --batch.write=N Write-operations batch size\n" - " --delay=N | --no-delay (no)Delay test-actor before start\n" - " --wait4ops=N | --no-wait4ops (no)Wait for previous test-actor\n" - " completes # ops before start\n" - " --duration=N[s|m|h|d] Define running duration\n" - " --nops=N[K|M|G|T] Define number of operations/steps\n" - " --inject-writefault[=yes|NO] TBD (see the source code)\n" - " --drop[=yes|NO] Drop key-value space/table on " - "completion\n" - " --ignore-dbfull[=yes|NO] Ignore MDBX_MAP_FULL error\n" - " --speculum[=yes|NO] Use internal `speculum` to check " - "dataset\n" - " --geometry-jitter[=YES|no] Use jitter for geometry upper-limit\n" - "Keys and Value:\n" - " --keylen.min=N Minimal keys length\n" - " --keylen.max=N Miximal keys length\n" - " --keylen=N Set both min/max for keys length\n" - " --datalen.min=N Minimal data length\n" - " --datalen.max=N Miximal data length\n" - " --datalen=N Set both min/max for data length\n" - " --keygen.width=N TBD (see the source code)\n" - " --keygen.mesh=N TBD (see the source code)\n" - " --keygen.zerofill=yes|NO TBD (see the source code)\n" - " --keygen.split=N TBD (see the source code)\n" - " --keygen.rotate=N TBD (see the source code)\n" - " --keygen.offset=N TBD (see the source code)\n" - " --keygen.case=random Generator case (only `random` for now)\n" - "Database operation mode:\n" - " --mode={[+-]FLAG}[,[+-]FLAG]...\n" - " nosubdir == MDBX_NOSUBDIR\n" - " rdonly == MDBX_RDONLY\n" - " exclusive == MDBX_EXCLUSIVE\n" - " accede == MDBX_ACCEDE\n" - " nometasync == MDBX_NOMETASYNC\n" - " lifo == MDBX_LIFORECLAIM\n" - " nosync-safe == MDBX_SAFE_NOSYNC\n" - " writemap == MDBX_WRITEMAP\n" - " nosync-utterly == MDBX_UTTERLY_NOSYNC\n" - " perturb == MDBX_PAGEPERTURB\n" - " nostickythreads== MDBX_NOSTICKYTHREADS\n" - " nordahead == MDBX_NORDAHEAD\n" - " nomeminit == MDBX_NOMEMINIT\n" - " --random-writemap[=YES|no] Toggle MDBX_WRITEMAP randomly\n" - "Key-value space/table options:\n" - " --table={[+-]FLAG}[,[+-]FLAG]...\n" - " key.reverse == MDBX_REVERSEKEY\n" - " key.integer == MDBX_INTEGERKEY\n" - " data.dups == MDBX_DUPSORT\n" - " data.integer == MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT\n" - " data.fixed == MDBX_DUPFIXED | MDBX_DUPSORT\n" - " data.reverse == MDBX_REVERSEDUP | MDBX_DUPSORT\n"); + "Actor options:\n" + " --batch.read=N Read-operations batch size\n" + " --batch.write=N Write-operations batch size\n" + " --delay=N | --no-delay (no)Delay test-actor before start\n" + " --wait4ops=N | --no-wait4ops (no)Wait for previous test-actor\n" + " completes # ops before start\n" + " --duration=N[s|m|h|d] Define running duration\n" + " --nops=N[K|M|G|T] Define number of operations/steps\n" + " --inject-writefault[=yes|NO] TBD (see the source code)\n" + " --drop[=yes|NO] Drop key-value space/table on " + "completion\n" + " --ignore-dbfull[=yes|NO] Ignore MDBX_MAP_FULL error\n" + " --speculum[=yes|NO] Use internal `speculum` to check " + "dataset\n" + " --geometry-jitter[=YES|no] Use jitter for geometry upper-limit\n" + "Keys and Value:\n" + " --keylen.min=N Minimal keys length\n" + " --keylen.max=N Miximal keys length\n" + " --keylen=N Set both min/max for keys length\n" + " --datalen.min=N Minimal data length\n" + " --datalen.max=N Miximal data length\n" + " --datalen=N Set both min/max for data length\n" + " --keygen.width=N TBD (see the source code)\n" + " --keygen.mesh=N TBD (see the source code)\n" + " --keygen.zerofill=yes|NO TBD (see the source code)\n" + " --keygen.split=N TBD (see the source code)\n" + " --keygen.rotate=N TBD (see the source code)\n" + " --keygen.offset=N TBD (see the source code)\n" + " --keygen.case=random Generator case (only `random` for now)\n" + "Database operation mode:\n" + " --mode={[+-]FLAG}[,[+-]FLAG]...\n" + " nosubdir == MDBX_NOSUBDIR\n" + " rdonly == MDBX_RDONLY\n" + " exclusive == MDBX_EXCLUSIVE\n" + " accede == MDBX_ACCEDE\n" + " nometasync == MDBX_NOMETASYNC\n" + " lifo == MDBX_LIFORECLAIM\n" + " nosync-safe == MDBX_SAFE_NOSYNC\n" + " writemap == MDBX_WRITEMAP\n" + " nosync-utterly == MDBX_UTTERLY_NOSYNC\n" + " perturb == MDBX_PAGEPERTURB\n" + " nostickythreads== MDBX_NOSTICKYTHREADS\n" + " nordahead == MDBX_NORDAHEAD\n" + " nomeminit == MDBX_NOMEMINIT\n" + " --random-writemap[=YES|no] Toggle MDBX_WRITEMAP randomly\n" + "Key-value space/table options:\n" + " --table={[+-]FLAG}[,[+-]FLAG]...\n" + " key.reverse == MDBX_REVERSEKEY\n" + " key.integer == MDBX_INTEGERKEY\n" + " data.dups == MDBX_DUPSORT\n" + " data.integer == MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT\n" + " data.fixed == MDBX_DUPFIXED | MDBX_DUPSORT\n" + " data.reverse == MDBX_REVERSEDUP | MDBX_DUPSORT\n"); exit(EXIT_FAILURE); } @@ -209,9 +208,7 @@ bool geometry_jitter; const char global::thunk_param_prefix[] = "--execute="; -std::string thunk_param(const actor_config &config) { - return config.serialize(global::thunk_param_prefix); -} +std::string thunk_param(const actor_config &config) { return config.serialize(global::thunk_param_prefix); } void cleanup() { log_trace(">> cleanup"); @@ -234,9 +231,7 @@ static void fixup4qemu(actor_params ¶ms) { safe4qemu_limit >>= 1; #endif /* __SANITIZE_ADDRESS__ */ - if (params.size_lower > safe4qemu_limit || - params.size_now > safe4qemu_limit || - params.size_upper > safe4qemu_limit) { + if (params.size_lower > safe4qemu_limit || params.size_now > safe4qemu_limit || params.size_upper > safe4qemu_limit) { params.size_upper = std::min(params.size_upper, safe4qemu_limit); params.size_now = std::min(params.size_now, params.size_upper); params.size_lower = std::min(params.size_lower, params.size_now); @@ -246,13 +241,12 @@ static void fixup4qemu(actor_params ¶ms) { } #endif /* MDBX_WORDBITS == 32 */ -#if defined(__alpha__) || defined(__alpha) || defined(__sparc__) || \ - defined(__sparc) || defined(__sparc64__) || defined(__sparc64) +#if defined(__alpha__) || defined(__alpha) || defined(__sparc__) || defined(__sparc) || defined(__sparc64__) || \ + defined(__sparc64) if (params.size_lower != params.size_upper) { - log_notice( - "workaround: for conformance Alpha/Sparc build with QEMU/ASAN/Valgrind " - "enforce fixed database size %zu megabytes", - params.size_upper >> 20); + log_notice("workaround: for conformance Alpha/Sparc build with QEMU/ASAN/Valgrind " + "enforce fixed database size %zu megabytes", + params.size_upper >> 20); params.size_lower = params.size_now = params.size_upper; } #endif /* Alpha || Sparc */ @@ -282,15 +276,10 @@ int main(int argc, char *const argv[]) { if (argc < 2) failure("No parameters given. Try --help\n"); - if (argc == 2 && strncmp(argv[1], global::thunk_param_prefix, - strlen(global::thunk_param_prefix)) == 0) - return test_execute( - actor_config(argv[1] + strlen(global::thunk_param_prefix))) - ? EXIT_SUCCESS - : EXIT_FAILURE; + if (argc == 2 && strncmp(argv[1], global::thunk_param_prefix, strlen(global::thunk_param_prefix)) == 0) + return test_execute(actor_config(argv[1] + strlen(global::thunk_param_prefix))) ? EXIT_SUCCESS : EXIT_FAILURE; - if (argc == 2 && - (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-h") == 0)) + if (argc == 2 && (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-h") == 0)) usage(); actor_params params; @@ -300,30 +289,22 @@ int main(int argc, char *const argv[]) { unsigned last_space_id = 0; for (int narg = 1; narg < argc; ++narg) { - if (config::parse_option(argc, argv, narg, "dump-config", - global::config::dump_config)) + if (config::parse_option(argc, argv, narg, "dump-config", global::config::dump_config)) continue; - if (config::parse_option(argc, argv, narg, "cleanup-before", - global::config::cleanup_before)) + if (config::parse_option(argc, argv, narg, "cleanup-before", global::config::cleanup_before)) continue; - if (config::parse_option(argc, argv, narg, "cleanup-after", - global::config::cleanup_after)) + if (config::parse_option(argc, argv, narg, "cleanup-after", global::config::cleanup_after)) continue; - if (config::parse_option(argc, argv, narg, "failfast", - global::config::failfast)) + if (config::parse_option(argc, argv, narg, "failfast", global::config::failfast)) continue; - if (config::parse_option(argc, argv, narg, "progress", - global::config::progress_indicator)) + if (config::parse_option(argc, argv, narg, "progress", global::config::progress_indicator)) continue; - if (config::parse_option(argc, argv, narg, "console", - global::config::console_mode)) + if (config::parse_option(argc, argv, narg, "console", global::config::console_mode)) continue; - if (config::parse_option(argc, argv, narg, "geometry-jitter", - global::config::geometry_jitter)) + if (config::parse_option(argc, argv, narg, "geometry-jitter", global::config::geometry_jitter)) continue; - if (config::parse_option(argc, argv, narg, "timeout", - global::config::timeout_duration_seconds, - config::duration, 1)) + if (config::parse_option(argc, argv, narg, "timeout", global::config::timeout_duration_seconds, config::duration, + 1)) continue; logging::loglevel loglevel; @@ -341,19 +322,15 @@ int main(int argc, char *const argv[]) { } if (config::parse_option(argc, argv, narg, "pathname", params.pathname_db)) continue; - if (config::parse_option(argc, argv, narg, "mode", params.mode_flags, - config::mode_bits)) + if (config::parse_option(argc, argv, narg, "mode", params.mode_flags, config::mode_bits)) continue; - if (config::parse_option(argc, argv, narg, "random-writemap", - params.random_writemap)) + if (config::parse_option(argc, argv, narg, "random-writemap", params.random_writemap)) continue; - if (config::parse_option(argc, argv, narg, "table", params.table_flags, - config::table_bits)) { + if (config::parse_option(argc, argv, narg, "table", params.table_flags, config::table_bits)) { if ((params.table_flags & MDBX_DUPFIXED) == 0) params.table_flags &= ~MDBX_INTEGERDUP; if ((params.table_flags & MDBX_DUPSORT) == 0) - params.table_flags &= - ~(MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP); + params.table_flags &= ~(MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP); const unsigned keylen_max = params.mdbx_keylen_max(); if (params.keylen_min > keylen_max) params.keylen_min = keylen_max; @@ -378,8 +355,7 @@ int main(int argc, char *const argv[]) { continue; } - if (config::parse_option(argc, argv, narg, "pagesize", params.pagesize, - int(mdbx_limits_pgsize_min()), + if (config::parse_option(argc, argv, narg, "pagesize", params.pagesize, int(mdbx_limits_pgsize_min()), int(mdbx_limits_pgsize_max()))) { const unsigned keylen_max = params.mdbx_keylen_max(); if (params.keylen_min > keylen_max) @@ -393,159 +369,116 @@ int main(int argc, char *const argv[]) { params.datalen_max = datalen_max; continue; } - if (config::parse_option(argc, argv, narg, "repeat", params.nrepeat, - config::entropy)) + if (config::parse_option(argc, argv, narg, "repeat", params.nrepeat, config::entropy)) continue; - if (config::parse_option(argc, argv, narg, "threads", params.nthreads, - config::no_scale, 1, 64)) + if (config::parse_option(argc, argv, narg, "threads", params.nthreads, config::no_scale, 1, 64)) continue; - if (config::parse_option_intptr(argc, argv, narg, "size-lower", - params.size_lower, - mdbx_limits_dbsize_min(params.pagesize), - mdbx_limits_dbsize_max(params.pagesize))) + if (config::parse_option_intptr(argc, argv, narg, "size-lower", params.size_lower, + mdbx_limits_dbsize_min(params.pagesize), mdbx_limits_dbsize_max(params.pagesize))) continue; int64_t i64 = params.size_upper; - if (config::parse_option(argc, argv, narg, "size-upper-upto", i64, - int64_t(mdbx_limits_dbsize_min(params.pagesize)), + if (config::parse_option(argc, argv, narg, "size-upper-upto", i64, int64_t(mdbx_limits_dbsize_min(params.pagesize)), INT64_MAX, -1)) { if (i64 > mdbx_limits_dbsize_max(params.pagesize)) i64 = mdbx_limits_dbsize_max(params.pagesize); params.size_upper = intptr_t(i64); continue; } - if (config::parse_option_intptr(argc, argv, narg, "size-upper", - params.size_upper, - mdbx_limits_dbsize_min(params.pagesize), + if (config::parse_option_intptr(argc, argv, narg, "size-upper", params.size_upper, + mdbx_limits_dbsize_min(params.pagesize), mdbx_limits_dbsize_max(params.pagesize))) + continue; + if (config::parse_option_intptr(argc, argv, narg, "size", params.size_now, mdbx_limits_dbsize_min(params.pagesize), mdbx_limits_dbsize_max(params.pagesize))) continue; - if (config::parse_option_intptr(argc, argv, narg, "size", params.size_now, - mdbx_limits_dbsize_min(params.pagesize), - mdbx_limits_dbsize_max(params.pagesize))) + if (config::parse_option(argc, argv, narg, "shrink-threshold", params.shrink_threshold, 0, + (int)std::min((intptr_t)INT_MAX, mdbx_limits_dbsize_max(params.pagesize) - + mdbx_limits_dbsize_min(params.pagesize)))) continue; - if (config::parse_option( - argc, argv, narg, "shrink-threshold", params.shrink_threshold, 0, - (int)std::min((intptr_t)INT_MAX, - mdbx_limits_dbsize_max(params.pagesize) - - mdbx_limits_dbsize_min(params.pagesize)))) - continue; - if (config::parse_option( - argc, argv, narg, "growth-step", params.growth_step, 0, - (int)std::min((intptr_t)INT_MAX, - mdbx_limits_dbsize_max(params.pagesize) - - mdbx_limits_dbsize_min(params.pagesize)))) + if (config::parse_option(argc, argv, narg, "growth-step", params.growth_step, 0, + (int)std::min((intptr_t)INT_MAX, mdbx_limits_dbsize_max(params.pagesize) - + mdbx_limits_dbsize_min(params.pagesize)))) continue; - if (config::parse_option(argc, argv, narg, "keygen.width", - params.keygen.width, 8, 64)) + if (config::parse_option(argc, argv, narg, "keygen.width", params.keygen.width, 8, 64)) continue; - if (config::parse_option(argc, argv, narg, "keygen.mesh", - params.keygen.mesh, 0, 64)) + if (config::parse_option(argc, argv, narg, "keygen.mesh", params.keygen.mesh, 0, 64)) continue; - if (config::parse_option(argc, argv, narg, "prng-seed", params.prng_seed, - config::entropy)) { + if (config::parse_option(argc, argv, narg, "prng-seed", params.prng_seed, config::entropy)) { prng_seed(params.prng_seed); continue; } - if (config::parse_option(argc, argv, narg, "keygen.zerofill", - params.keygen.zero_fill)) + if (config::parse_option(argc, argv, narg, "keygen.zerofill", params.keygen.zero_fill)) continue; - if (config::parse_option(argc, argv, narg, "keygen.split", - params.keygen.split, 0, 63)) + if (config::parse_option(argc, argv, narg, "keygen.split", params.keygen.split, 0, 63)) continue; - if (config::parse_option(argc, argv, narg, "keygen.rotate", - params.keygen.rotate, 0, 63)) + if (config::parse_option(argc, argv, narg, "keygen.rotate", params.keygen.rotate, 0, 63)) continue; - if (config::parse_option(argc, argv, narg, "keygen.offset", - params.keygen.offset, config::binary)) + if (config::parse_option(argc, argv, narg, "keygen.offset", params.keygen.offset, config::binary)) continue; if (config::parse_option(argc, argv, narg, "keygen.case", &value)) { keycase_setup(value, params); continue; } - if (config::parse_option( - argc, argv, narg, "keylen.min", params.keylen_min, - (params.table_flags & MDBX_INTEGERKEY) ? config::intkey - : config::no_scale, - params.mdbx_keylen_min(), params.mdbx_keylen_max())) { - if ((params.table_flags & MDBX_INTEGERKEY) || - params.keylen_max < params.keylen_min) + if (config::parse_option(argc, argv, narg, "keylen.min", params.keylen_min, + (params.table_flags & MDBX_INTEGERKEY) ? config::intkey : config::no_scale, + params.mdbx_keylen_min(), params.mdbx_keylen_max())) { + if ((params.table_flags & MDBX_INTEGERKEY) || params.keylen_max < params.keylen_min) params.keylen_max = params.keylen_min; continue; } - if (config::parse_option( - argc, argv, narg, "keylen.max", params.keylen_max, - (params.table_flags & MDBX_INTEGERKEY) ? config::intkey - : config::no_scale, - params.mdbx_keylen_min(), params.mdbx_keylen_max())) { - if ((params.table_flags & MDBX_INTEGERKEY) || - params.keylen_min > params.keylen_max) + if (config::parse_option(argc, argv, narg, "keylen.max", params.keylen_max, + (params.table_flags & MDBX_INTEGERKEY) ? config::intkey : config::no_scale, + params.mdbx_keylen_min(), params.mdbx_keylen_max())) { + if ((params.table_flags & MDBX_INTEGERKEY) || params.keylen_min > params.keylen_max) params.keylen_min = params.keylen_max; continue; } - if (config::parse_option( - argc, argv, narg, "keylen", params.keylen_min, - (params.table_flags & MDBX_INTEGERKEY) ? config::intkey - : config::no_scale, - params.mdbx_keylen_min(), params.mdbx_keylen_max())) { + if (config::parse_option(argc, argv, narg, "keylen", params.keylen_min, + (params.table_flags & MDBX_INTEGERKEY) ? config::intkey : config::no_scale, + params.mdbx_keylen_min(), params.mdbx_keylen_max())) { params.keylen_max = params.keylen_min; continue; } - if (config::parse_option( - argc, argv, narg, "datalen.min", params.datalen_min, - (params.table_flags & MDBX_INTEGERDUP) ? config::intkey - : config::no_scale, - params.mdbx_datalen_min(), params.mdbx_datalen_max())) { - if ((params.table_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) || - params.datalen_max < params.datalen_min) + if (config::parse_option(argc, argv, narg, "datalen.min", params.datalen_min, + (params.table_flags & MDBX_INTEGERDUP) ? config::intkey : config::no_scale, + params.mdbx_datalen_min(), params.mdbx_datalen_max())) { + if ((params.table_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) || params.datalen_max < params.datalen_min) params.datalen_max = params.datalen_min; continue; } - if (config::parse_option( - argc, argv, narg, "datalen.max", params.datalen_max, - (params.table_flags & MDBX_INTEGERDUP) ? config::intkey - : config::no_scale, - params.mdbx_datalen_min(), params.mdbx_datalen_max())) { - if ((params.table_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) || - params.datalen_min > params.datalen_max) + if (config::parse_option(argc, argv, narg, "datalen.max", params.datalen_max, + (params.table_flags & MDBX_INTEGERDUP) ? config::intkey : config::no_scale, + params.mdbx_datalen_min(), params.mdbx_datalen_max())) { + if ((params.table_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) || params.datalen_min > params.datalen_max) params.datalen_min = params.datalen_max; continue; } - if (config::parse_option( - argc, argv, narg, "datalen", params.datalen_min, - (params.table_flags & MDBX_INTEGERDUP) ? config::intkey - : config::no_scale, - params.mdbx_datalen_min(), params.mdbx_datalen_max())) { + if (config::parse_option(argc, argv, narg, "datalen", params.datalen_min, + (params.table_flags & MDBX_INTEGERDUP) ? config::intkey : config::no_scale, + params.mdbx_datalen_min(), params.mdbx_datalen_max())) { params.datalen_max = params.datalen_min; continue; } - if (config::parse_option(argc, argv, narg, "batch.read", params.batch_read, - config::no_scale, 1)) + if (config::parse_option(argc, argv, narg, "batch.read", params.batch_read, config::no_scale, 1)) continue; - if (config::parse_option(argc, argv, narg, "batch.write", - params.batch_write, config::no_scale, 1)) + if (config::parse_option(argc, argv, narg, "batch.write", params.batch_write, config::no_scale, 1)) continue; - if (config::parse_option(argc, argv, narg, "delay", params.delaystart, - config::duration)) + if (config::parse_option(argc, argv, narg, "delay", params.delaystart, config::duration)) continue; - if (config::parse_option(argc, argv, narg, "wait4ops", params.waitfor_nops, - config::decimal)) + if (config::parse_option(argc, argv, narg, "wait4ops", params.waitfor_nops, config::decimal)) continue; - if (config::parse_option(argc, argv, narg, "inject-writefault", - params.inject_writefaultn, config::decimal)) + if (config::parse_option(argc, argv, narg, "inject-writefault", params.inject_writefaultn, config::decimal)) continue; if (config::parse_option(argc, argv, narg, "drop", params.drop_table)) continue; - if (config::parse_option(argc, argv, narg, "ignore-dbfull", - params.ignore_dbfull)) + if (config::parse_option(argc, argv, narg, "ignore-dbfull", params.ignore_dbfull)) continue; if (config::parse_option(argc, argv, narg, "speculum", params.speculum)) continue; - if (config::parse_option(argc, argv, narg, "max-readers", - params.max_readers, config::no_scale, 1, 255)) + if (config::parse_option(argc, argv, narg, "max-readers", params.max_readers, config::no_scale, 1, 255)) continue; - if (config::parse_option(argc, argv, narg, "max-tables", params.max_tables, - config::no_scale, 1, INT16_MAX)) + if (config::parse_option(argc, argv, narg, "max-tables", params.max_tables, config::no_scale, 1, INT16_MAX)) continue; if (config::parse_option(argc, argv, narg, "no-delay", nullptr)) { @@ -556,13 +489,11 @@ int main(int argc, char *const argv[]) { params.waitfor_nops = 0; continue; } - if (config::parse_option(argc, argv, narg, "duration", params.test_duration, - config::duration, 1)) { + if (config::parse_option(argc, argv, narg, "duration", params.test_duration, config::duration, 1)) { params.test_nops = 0; continue; } - if (config::parse_option(argc, argv, narg, "nops", params.test_nops, - config::decimal, 1)) { + if (config::parse_option(argc, argv, narg, "nops", params.test_nops, config::decimal, 1)) { params.test_duration = 0; continue; } @@ -647,8 +578,7 @@ int main(int argc, char *const argv[]) { (global::config::timeout_duration_seconds == 0) ? chrono::infinite().fixedpoint : global::start_monotonic.fixedpoint + - chrono::from_seconds(global::config::timeout_duration_seconds) - .fixedpoint; + chrono::from_seconds(global::config::timeout_duration_seconds).fixedpoint; if (global::config::cleanup_before) cleanup(); @@ -675,8 +605,7 @@ int main(int argc, char *const argv[]) { log_trace(">> killall_actors: (%s)", "start failed"); osal_killall_actors(); log_trace("<< killall_actors"); - failure("Failed to start actor #%u (%s)\n", a.actor_id, - test_strerror(rc)); + failure("Failed to start actor #%u (%s)\n", a.actor_id, test_strerror(rc)); } global::pid2actor[pid] = &a; } @@ -696,8 +625,7 @@ int main(int argc, char *const argv[]) { timeout_seconds_left = 0; else { chrono::time left_monotonic; - left_monotonic.fixedpoint = - global::deadline_monotonic.fixedpoint - now_monotonic.fixedpoint; + left_monotonic.fixedpoint = global::deadline_monotonic.fixedpoint - now_monotonic.fixedpoint; timeout_seconds_left = left_monotonic.seconds(); } @@ -713,8 +641,8 @@ int main(int argc, char *const argv[]) { continue; if (status > as_running) { - log_notice("actor #%u, id %d, pid %ld: %s\n", actor->actor_id, - actor->space_id, (long)pid, status2str(status)); + log_notice("actor #%u, id %d, pid %ld: %s\n", actor->actor_id, actor->space_id, (long)pid, + status2str(status)); left -= 1; if (status != as_successful) { if (global::config::failfast && !failed) { @@ -725,8 +653,8 @@ int main(int argc, char *const argv[]) { failed = true; } } else { - log_verbose("actor #%u, id %d, pid %ld: %s\n", actor->actor_id, - actor->space_id, (long)pid, status2str(status)); + log_verbose("actor #%u, id %d, pid %ld: %s\n", actor->actor_id, actor->space_id, (long)pid, + status2str(status)); } } else { if (timeout_seconds_left == 0) @@ -738,8 +666,7 @@ int main(int argc, char *const argv[]) { if (!failed) { MDBX_envinfo info; - int err = - mdbx_preopen_snapinfo(params.pathname_db.c_str(), &info, sizeof(info)); + int err = mdbx_preopen_snapinfo(params.pathname_db.c_str(), &info, sizeof(info)); if (err != MDBX_SUCCESS) failure_perror("mdbx_preopen_snapinfo()", err); } @@ -755,15 +682,12 @@ int main(int argc, char *const argv[]) { #if !(defined(_WIN32) || defined(_WIN64)) struct rusage spent; if (!getrusage(global::singlemode ? RUSAGE_SELF : RUSAGE_CHILDREN, &spent)) { - log_notice("%6s: user %f, system %f", "CPU", - spent.ru_utime.tv_sec + spent.ru_utime.tv_usec * 1e-6, + log_notice("%6s: user %f, system %f", "CPU", spent.ru_utime.tv_sec + spent.ru_utime.tv_usec * 1e-6, spent.ru_stime.tv_sec + spent.ru_stime.tv_usec * 1e-6); -#if defined(__linux__) || defined(__gnu_linux__) || defined(__FreeBSD__) || \ - defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || \ - defined(__bsdi__) || defined(__DragonFly__) || defined(__APPLE__) || \ +#if defined(__linux__) || defined(__gnu_linux__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || defined(__APPLE__) || \ defined(__MACH__) || defined(__sun) - log_notice("%6s: read %ld, write %ld", "IOPs", spent.ru_inblock, - spent.ru_oublock); + log_notice("%6s: read %ld, write %ld", "IOPs", spent.ru_inblock, spent.ru_oublock); if (spent.ru_maxrss > 0) log_notice("%6s: %ld Kb", "RAM", spent.ru_maxrss @@ -773,8 +697,7 @@ int main(int argc, char *const argv[]) { / 1024u #endif ); - log_notice("%6s: reclaims %ld, faults %ld, swaps %ld", "Paging", - spent.ru_minflt, spent.ru_majflt, spent.ru_nswap); + log_notice("%6s: reclaims %ld, faults %ld, swaps %ld", "Paging", spent.ru_minflt, spent.ru_majflt, spent.ru_nswap); #endif /* Linux */ } #endif /* !Windows */ diff --git a/test/nested.c++ b/test/nested.c++ index a90c4d37..dda73f92 100644 --- a/test/nested.c++ +++ b/test/nested.c++ @@ -38,16 +38,12 @@ class testcase_nested : public testcase_ttl { bool trim_tail(unsigned window_width); bool grow_head(unsigned head_count); bool pop_txn(bool abort); - bool pop_txn() { - return pop_txn(inherited::is_nested_txn_available() ? flipcoin_x3() - : flipcoin_x2()); - } + bool pop_txn() { return pop_txn(inherited::is_nested_txn_available() ? flipcoin_x3() : flipcoin_x2()); } void push_txn(); bool stochastic_breakable_restart_with_nested(bool force_restart = false); public: - testcase_nested(const actor_config &config, const mdbx_pid_t pid) - : inherited(config, pid) {} + testcase_nested(const actor_config &config, const mdbx_pid_t pid) : inherited(config, pid) {} bool setup() override; bool run() override; bool teardown() override; @@ -84,8 +80,7 @@ bool testcase_nested::teardown() { txn_begin(false); db_table_drop(dbi); int err = breakable_commit(); - if (unlikely(err != MDBX_SUCCESS) && - (err != MDBX_MAP_FULL || !config.params.ignore_dbfull)) { + if (unlikely(err != MDBX_SUCCESS) && (err != MDBX_MAP_FULL || !config.params.ignore_dbfull)) { log_notice("nested: bailout-clean due '%s'", mdbx_strerror(err)); ok = false; } @@ -101,8 +96,7 @@ bool testcase_nested::teardown() { void testcase_nested::push_txn() { MDBX_txn *nested_txn; - MDBX_txn_flags_t flags = MDBX_txn_flags_t( - prng32() & uint32_t(MDBX_TXN_NOSYNC | MDBX_TXN_NOMETASYNC)); + MDBX_txn_flags_t flags = MDBX_txn_flags_t(prng32() & uint32_t(MDBX_TXN_NOSYNC | MDBX_TXN_NOMETASYNC)); int err = mdbx_txn_begin(db_guard.get(), txn_guard.get(), flags, &nested_txn); if (unlikely(err != MDBX_SUCCESS)) failure_perror("mdbx_txn_begin(nested)", err); @@ -114,10 +108,9 @@ void testcase_nested::push_txn() { scoped_txn_guard nested_txn_guard(nested_txn); txn_guard.swap(nested_txn_guard); SET speculum_snapshot(speculum); - stack.emplace(std::move(nested_txn_guard), serial, fifo, - std::move(speculum_snapshot)); - log_verbose("begin level#%zu txn #%" PRIu64 ", flags 0x%x, serial %" PRIu64, - stack.size(), mdbx_txn_id(nested_txn), flags, serial); + stack.emplace(std::move(nested_txn_guard), serial, fifo, std::move(speculum_snapshot)); + log_verbose("begin level#%zu txn #%" PRIu64 ", flags 0x%x, serial %" PRIu64, stack.size(), mdbx_txn_id(nested_txn), + flags, serial); if (!dbi && stack.size() == 1) dbi = db_table_open(true); } @@ -128,18 +121,16 @@ bool testcase_nested::pop_txn(bool abort) { MDBX_txn *txn = txn_guard.release(); bool committed = false; if (abort) { - log_verbose( - "abort level#%zu txn #%" PRIu64 ", undo serial %" PRIu64 " <- %" PRIu64, - stack.size(), mdbx_txn_id(txn), serial, std::get<1>(stack.top())); - if (dbi > 0 && stack.size() == 1 && - is_handle_created_in_current_txn(dbi, txn)) + log_verbose("abort level#%zu txn #%" PRIu64 ", undo serial %" PRIu64 " <- %" PRIu64, stack.size(), mdbx_txn_id(txn), + serial, std::get<1>(stack.top())); + if (dbi > 0 && stack.size() == 1 && is_handle_created_in_current_txn(dbi, txn)) dbi = 0; int err = mdbx_txn_abort(txn); if (unlikely(err != MDBX_SUCCESS)) failure_perror("mdbx_txn_abort()", err); } else { - log_verbose("commit level#%zu txn, nested serial %" PRIu64 " -> %" PRIu64, - stack.size(), serial, std::get<1>(stack.top())); + log_verbose("commit level#%zu txn, nested serial %" PRIu64 " -> %" PRIu64, stack.size(), serial, + std::get<1>(stack.top())); int err = mdbx_txn_commit(txn); if (likely(err == MDBX_SUCCESS)) committed = true; @@ -147,8 +138,7 @@ bool testcase_nested::pop_txn(bool abort) { should_continue = false; if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { err = mdbx_txn_abort(txn); - if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH && - err != MDBX_BAD_TXN)) + if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH && err != MDBX_BAD_TXN)) failure_perror("mdbx_txn_abort()", err); } else failure_perror("mdbx_txn_commit()", err); @@ -165,18 +155,15 @@ bool testcase_nested::pop_txn(bool abort) { return should_continue; } -bool testcase_nested::stochastic_breakable_restart_with_nested( - bool force_restart) { - log_trace(">> stochastic_breakable_restart_with_nested%s", - force_restart ? ": force_restart" : ""); +bool testcase_nested::stochastic_breakable_restart_with_nested(bool force_restart) { + log_trace(">> stochastic_breakable_restart_with_nested%s", force_restart ? ": force_restart" : ""); if (force_restart) while (txn_guard) pop_txn(true); bool should_continue = true; - while (!stack.empty() && - (flipcoin() || txn_underutilization_x256(txn_guard.get()) < 42)) + while (!stack.empty() && (flipcoin() || txn_underutilization_x256(txn_guard.get()) < 42)) should_continue &= pop_txn(); if (flipcoin_x3()) { @@ -200,12 +187,10 @@ bool testcase_nested::stochastic_breakable_restart_with_nested( } if (should_continue) - while (stack.empty() || - (is_nested_txn_available() && flipcoin() && stack.size() < 5)) + while (stack.empty() || (is_nested_txn_available() && flipcoin() && stack.size() < 5)) push_txn(); - log_trace("<< stochastic_breakable_restart_with_nested: should_continue=%s", - should_continue ? "yes" : "no"); + log_trace("<< stochastic_breakable_restart_with_nested: should_continue=%s", should_continue ? "yes" : "no"); return should_continue; } @@ -215,8 +200,7 @@ bool testcase_nested::trim_tail(unsigned window_width) { while (fifo.size() > window_width) { uint64_t tail_serial = fifo.back().first; const unsigned tail_count = fifo.back().second; - log_verbose("nested: trim-tail (serial %" PRIu64 ", count %u)", - tail_serial, tail_count); + log_verbose("nested: trim-tail (serial %" PRIu64 ", count %u)", tail_serial, tail_count); fifo.pop_back(); for (unsigned n = 0; n < tail_count; ++n) { log_trace("nested: remove-tail %" PRIu64, tail_serial); @@ -235,9 +219,8 @@ bool testcase_nested::trim_tail(unsigned window_width) { report(tail_count); } } else if (!fifo.empty()) { - log_verbose("nested: purge state %" PRIu64 " - %" PRIu64 ", fifo-items %zu", - fifo.front().first, fifo.back().first + fifo.back().second, - fifo.size()); + log_verbose("nested: purge state %" PRIu64 " - %" PRIu64 ", fifo-items %zu", fifo.front().first, + fifo.back().first + fifo.back().second, fifo.size()); db_table_clear(dbi, txn_guard.get()); fifo.clear(); clear_wholetable_passed += 1; @@ -248,9 +231,7 @@ bool testcase_nested::trim_tail(unsigned window_width) { bool testcase_nested::grow_head(unsigned head_count) { const MDBX_put_flags_t insert_flags = - (config.params.table_flags & MDBX_DUPSORT) - ? MDBX_NODUPDATA - : MDBX_NODUPDATA | MDBX_NOOVERWRITE; + (config.params.table_flags & MDBX_DUPSORT) ? MDBX_NODUPDATA : MDBX_NODUPDATA | MDBX_NOOVERWRITE; retry: fifo.push_front(std::make_pair(serial, head_count)); for (unsigned n = 0; n < head_count; ++n) { @@ -289,12 +270,10 @@ bool testcase_nested::run() { unsigned loops = 0; while (true) { const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */; - const unsigned window_width = - (!should_continue() || flipcoin_x4()) ? 0 : edge2window(salt); + const unsigned window_width = (!should_continue() || flipcoin_x4()) ? 0 : edge2window(salt); const unsigned head_count = edge2count(salt); - log_debug("nested: step #%" PRIu64 " (serial %" PRIu64 - ", window %u, count %u) salt %" PRIu64, - nops_completed, serial, window_width, head_count, salt); + log_debug("nested: step #%" PRIu64 " (serial %" PRIu64 ", window %u, count %u) salt %" PRIu64, nops_completed, + serial, window_width, head_count, salt); if (!trim_tail(window_width)) return false; @@ -307,10 +286,8 @@ bool testcase_nested::run() { return false; } - if (!keyspace_overflow && (should_continue() || !clear_wholetable_passed || - !clear_stepbystep_passed)) { - unsigned underutilization_x256 = - txn_underutilization_x256(txn_guard.get()); + if (!keyspace_overflow && (should_continue() || !clear_wholetable_passed || !clear_stepbystep_passed)) { + unsigned underutilization_x256 = txn_underutilization_x256(txn_guard.get()); if (dbfull_passed > underutilization_x256) { log_notice("nested: skip head-grow to avoid one more dbfull (was %u, " "underutilization %.2f%%)", @@ -327,9 +304,7 @@ bool testcase_nested::run() { } loops += 1; } else if (fifo.empty()) { - log_notice("nested: done %u whole loops, %" PRIu64 " ops, %" PRIu64 - " items", - loops, nops_completed, serial); + log_notice("nested: done %u whole loops, %" PRIu64 " ops, %" PRIu64 " items", loops, nops_completed, serial); break; } else { log_notice("nested: done, wait for empty, skip head-grow"); diff --git a/test/osal-unix.c++ b/test/osal-unix.c++ index 2c099d85..3a6bf215 100644 --- a/test/osal-unix.c++ +++ b/test/osal-unix.c++ @@ -19,14 +19,12 @@ #error "Oops, MDBX_LOCKING is undefined!" #endif -#if defined(__APPLE__) && (MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008) +#if defined(__APPLE__) && (MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008) #include "stub/pthread_barrier.c" #endif /* __APPLE__ && MDBX_LOCKING >= MDBX_LOCKING_POSIX2001 */ -#if defined(__ANDROID_API__) && __ANDROID_API__ < 24 && \ - (MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008) +#if defined(__ANDROID_API__) && __ANDROID_API__ < 24 && \ + (MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008) #include "stub/pthread_barrier.c" #endif /* __ANDROID_API__ < 24 && MDBX_LOCKING >= MDBX_LOCKING_POSIX2001 */ @@ -40,9 +38,7 @@ #if __cplusplus >= 201103L #include -MDBX_MAYBE_UNUSED static inline int atomic_decrement(std::atomic_int *p) { - return std::atomic_fetch_sub(p, 1) - 1; -} +MDBX_MAYBE_UNUSED static inline int atomic_decrement(std::atomic_int *p) { return std::atomic_fetch_sub(p, 1) - 1; } #else MDBX_MAYBE_UNUSED static inline int atomic_decrement(volatile int *p) { #if defined(__GNUC__) || defined(__clang__) @@ -69,8 +65,7 @@ static void ipc_remove(void) { #else struct shared_t { -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 pthread_barrier_t barrier; pthread_mutex_t mutex; size_t count; @@ -105,18 +100,14 @@ void osal_wait4barrier(void) { op.sem_flg = 0; if (semop(ipc, &op, 1)) failure_perror("semop(wait)", errno); -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 assert(shared != nullptr && shared != MAP_FAILED); int err = pthread_barrier_wait(&shared->barrier); if (err != 0 && err != PTHREAD_BARRIER_SERIAL_THREAD) failure_perror("pthread_barrier_wait(shared)", err); #elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 assert(shared != nullptr && shared != MAP_FAILED); - int err = (atomic_decrement(&shared->barrier.countdown) > 0 && - sem_wait(&shared->barrier.sema)) - ? errno - : 0; + int err = (atomic_decrement(&shared->barrier.countdown) > 0 && sem_wait(&shared->barrier.sema)) ? errno : 0; if (err != 0) failure_perror("sem_wait(shared)", err); if (sem_post(&shared->barrier.sema)) @@ -149,22 +140,20 @@ void osal_setup(const std::vector &actors) { failure_perror("semctl(SETVAL.N, shared_sems)", errno); #else assert(shared == nullptr); - shared = (shared_t *)mmap( - nullptr, sizeof(shared_t) + actors.size() * sizeof(shared->events[0]), - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS + shared = + (shared_t *)mmap(nullptr, sizeof(shared_t) + actors.size() * sizeof(shared->events[0]), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS #ifdef MAP_HASSEMAPHORE - | MAP_HASSEMAPHORE + | MAP_HASSEMAPHORE #endif - , - -1, 0); + , + -1, 0); if (MAP_FAILED == (void *)shared) failure_perror("mmap(shared)", errno); shared->count = actors.size() + 1; -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 pthread_barrierattr_t barrierattr; int err = pthread_barrierattr_init(&barrierattr); if (err) @@ -173,8 +162,7 @@ void osal_setup(const std::vector &actors) { if (err) failure_perror("pthread_barrierattr_setpshared()", err); - err = pthread_barrier_init(&shared->barrier, &barrierattr, - unsigned(shared->count)); + err = pthread_barrier_init(&shared->barrier, &barrierattr, unsigned(shared->count)); if (err) failure_perror("pthread_barrier_init(shared)", err); pthread_barrierattr_destroy(&barrierattr); @@ -204,8 +192,7 @@ void osal_setup(const std::vector &actors) { err = pthread_cond_init(event, &condattr); if (err) failure_perror("pthread_cond_init(shared)", err); - log_trace("osal_setup: event(shared pthread_cond) %" PRIuPTR " -> %p", i, - __Wpedantic_format_voidptr(event)); + log_trace("osal_setup: event(shared pthread_cond) %" PRIuPTR " -> %p", i, __Wpedantic_format_voidptr(event)); } pthread_condattr_destroy(&condattr); pthread_mutexattr_destroy(&mutexattr); @@ -217,8 +204,7 @@ void osal_setup(const std::vector &actors) { sem_t *event = &shared->events[i]; if (sem_init(event, true, 0)) failure_perror("sem_init(shared.event)", errno); - log_trace("osal_setup: event(shared sem_init) %" PRIuPTR " -> %p", i, - __Wpedantic_format_voidptr(event)); + log_trace("osal_setup: event(shared sem_init) %" PRIuPTR " -> %p", i, __Wpedantic_format_voidptr(event)); } #else #error "FIXME" @@ -235,8 +221,7 @@ void osal_broadcast(unsigned id) { assert(shared != nullptr && shared != MAP_FAILED); if (id >= shared->count) failure("osal_broadcast: id > limit"); -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 int err = pthread_cond_broadcast(shared->events + id); if (err) failure_perror("pthread_cond_broadcast(shared)", err); @@ -261,8 +246,7 @@ int osal_waitfor(unsigned id) { if (id >= shared->count) failure("osal_waitfor: id > limit"); -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 int rc = pthread_mutex_lock(&shared->mutex); if (rc != 0) failure_perror("pthread_mutex_lock(shared)", rc); @@ -288,15 +272,13 @@ int osal_waitfor(unsigned id) { //----------------------------------------------------------------------------- -const std::string -actor_config::osal_serialize(simple_checksum &checksum) const { +const std::string actor_config::osal_serialize(simple_checksum &checksum) const { (void)checksum; /* not used in workload, but just for testing */ return "unix.fork"; } -bool actor_config::osal_deserialize(const char *str, const char *end, - simple_checksum &checksum) { +bool actor_config::osal_deserialize(const char *str, const char *end, simple_checksum &checksum) { (void)checksum; /* not used in workload, but just for testing */ return strncmp(str, "unix.fork", 9) == 0 && str + 9 == end; @@ -392,8 +374,7 @@ int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) { if (pid < 0) return errno; - log_trace("osal_actor_start: fork pid %ld for %u", (long)pid, - config.actor_id); + log_trace("osal_actor_start: fork pid %ld for %u", (long)pid, config.actor_id); children[pid] = as_running; return 0; } @@ -536,8 +517,7 @@ int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) { if (pid > 0) { if (WIFEXITED(status)) - children[pid] = - (WEXITSTATUS(status) == EXIT_SUCCESS) ? as_successful : as_failed; + children[pid] = (WEXITSTATUS(status) == EXIT_SUCCESS) ? as_successful : as_failed; else if (WIFSIGNALED(status)) { int sig = WTERMSIG(status); #ifdef WCOREDUMP @@ -551,13 +531,11 @@ int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) { case SIGFPE: case SIGILL: case SIGSEGV: - log_notice("child pid %lu %s by SIG%s", (long)pid, "terminated", - signal_name(sig)); + log_notice("child pid %lu %s by SIG%s", (long)pid, "terminated", signal_name(sig)); children[pid] = as_coredump; break; default: - log_notice("child pid %lu %s by SIG%s", (long)pid, "killed", - signal_name(sig)); + log_notice("child pid %lu %s by SIG%s", (long)pid, "killed", signal_name(sig)); children[pid] = as_killed; } } else if (WIFSTOPPED(status)) @@ -605,12 +583,10 @@ void osal_udelay(size_t us) { static size_t threshold_us; if (threshold_us == 0) { -#if defined(_POSIX_CPUTIME) && _POSIX_CPUTIME > -1 && \ - defined(CLOCK_PROCESS_CPUTIME_ID) +#if defined(_POSIX_CPUTIME) && _POSIX_CPUTIME > -1 && defined(CLOCK_PROCESS_CPUTIME_ID) if (clock_getres(CLOCK_PROCESS_CPUTIME_ID, &ts)) { int rc = errno; - log_warning("clock_getres(CLOCK_PROCESS_CPUTIME_ID), failed errno %d", - rc); + log_warning("clock_getres(CLOCK_PROCESS_CPUTIME_ID), failed errno %d", rc); } #endif /* CLOCK_PROCESS_CPUTIME_ID */ if (threshold_us == 0 && clock_getres(CLOCK_MONOTONIC, &ts)) { diff --git a/test/osal-windows.c++ b/test/osal-windows.c++ index 0ce04cc8..94f52b4a 100644 --- a/test/osal-windows.c++ +++ b/test/osal-windows.c++ @@ -30,13 +30,11 @@ void osal_wait4barrier(void) { DWORD rc = WaitForSingleObject(hBarrierSemaphore, 0); switch (rc) { default: - failure_perror("WaitForSingleObject(BarrierSemaphore)", - waitstatus2errcode(rc)); + failure_perror("WaitForSingleObject(BarrierSemaphore)", waitstatus2errcode(rc)); case WAIT_OBJECT_0: rc = WaitForSingleObject(hBarrierEvent, INFINITE); if (rc != WAIT_OBJECT_0) - failure_perror("WaitForSingleObject(BarrierEvent)", - waitstatus2errcode(rc)); + failure_perror("WaitForSingleObject(BarrierEvent)", waitstatus2errcode(rc)); break; case WAIT_TIMEOUT: if (!SetEvent(hBarrierEvent)) @@ -47,8 +45,7 @@ void osal_wait4barrier(void) { static HANDLE make_inheritable(HANDLE hHandle) { assert(hHandle != NULL && hHandle != INVALID_HANDLE_VALUE); - if (!DuplicateHandle(GetCurrentProcess(), hHandle, GetCurrentProcess(), - &hHandle, 0, TRUE, + if (!DuplicateHandle(GetCurrentProcess(), hHandle, GetCurrentProcess(), &hHandle, 0, TRUE, DUPLICATE_CLOSE_SOURCE | DUPLICATE_SAME_ACCESS)) failure_perror("DuplicateHandle()", GetLastError()); return hHandle; @@ -108,8 +105,7 @@ int osal_delay(unsigned seconds) { //----------------------------------------------------------------------------- -const std::string -actor_config::osal_serialize(simple_checksum &checksum) const { +const std::string actor_config::osal_serialize(simple_checksum &checksum) const { checksum.push(hBarrierSemaphore); checksum.push(hBarrierEvent); checksum.push(hProgressActiveEvent); @@ -127,12 +123,11 @@ actor_config::osal_serialize(simple_checksum &checksum) const { checksum.push(hSignal); } - return format("%p.%p.%p.%p.%p.%p", hBarrierSemaphore, hBarrierEvent, hWait, - hSignal, hProgressActiveEvent, hProgressPassiveEvent); + return format("%p.%p.%p.%p.%p.%p", hBarrierSemaphore, hBarrierEvent, hWait, hSignal, hProgressActiveEvent, + hProgressPassiveEvent); } -bool actor_config::osal_deserialize(const char *str, const char *end, - simple_checksum &checksum) { +bool actor_config::osal_deserialize(const char *str, const char *end, simple_checksum &checksum) { std::string copy(str, end - str); TRACE(">> osal_deserialize(%s)\n", copy.c_str()); @@ -144,9 +139,8 @@ bool actor_config::osal_deserialize(const char *str, const char *end, assert(events.empty()); HANDLE hWait, hSignal; - if (sscanf_s(copy.c_str(), "%p.%p.%p.%p.%p.%p", &hBarrierSemaphore, - &hBarrierEvent, &hWait, &hSignal, &hProgressActiveEvent, - &hProgressPassiveEvent) != 6) { + if (sscanf_s(copy.c_str(), "%p.%p.%p.%p.%p.%p", &hBarrierSemaphore, &hBarrierEvent, &hWait, &hSignal, + &hProgressActiveEvent, &hProgressPassiveEvent) != 6) { TRACE("<< osal_deserialize: failed\n"); return false; } @@ -175,23 +169,19 @@ bool actor_config::osal_deserialize(const char *str, const char *end, typedef std::pair child; static std::unordered_map children; -bool osal_multiactor_mode(void) { - return hProgressActiveEvent || hProgressPassiveEvent; -} +bool osal_multiactor_mode(void) { return hProgressActiveEvent || hProgressPassiveEvent; } bool osal_progress_push(bool active) { if (!children.empty()) { if (!SetEvent(active ? hProgressActiveEvent : hProgressPassiveEvent)) - failure_perror("osal_progress_push: SetEvent(overlord.progress)", - GetLastError()); + failure_perror("osal_progress_push: SetEvent(overlord.progress)", GetLastError()); return true; } return false; } -static void ArgvQuote(std::string &CommandLine, const std::string &Argument, - bool Force = false) +static void ArgvQuote(std::string &CommandLine, const std::string &Argument, bool Force = false) /*++ @@ -232,8 +222,7 @@ Environment: // parse quotes properly // - if (Force == false && Argument.empty() == false && - Argument.find_first_of(" \t\n\v\"") == Argument.npos) { + if (Force == false && Argument.empty() == false && Argument.find_first_of(" \t\n\v\"") == Argument.npos) { CommandLine.append(Argument); } else { CommandLine.push_back('"'); @@ -276,8 +265,7 @@ Environment: int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) { if (children.size() == MAXIMUM_WAIT_OBJECTS) - failure("Couldn't manage more that %u actors on Windows\n", - MAXIMUM_WAIT_OBJECTS); + failure("Couldn't manage more that %u actors on Windows\n", MAXIMUM_WAIT_OBJECTS); _flushall(); @@ -286,8 +274,7 @@ int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) { char exename[_MAX_PATH + 1]; DWORD exename_size = sizeof(exename); - if (!QueryFullProcessImageNameA(GetCurrentProcess(), 0, exename, - &exename_size)) + if (!QueryFullProcessImageNameA(GetCurrentProcess(), 0, exename, &exename_size)) failure_perror("QueryFullProcessImageName()", GetLastError()); if (exename[1] != ':') { @@ -383,8 +370,7 @@ int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) { while (true) { DWORD rc = - MsgWaitForMultipleObjectsEx((DWORD)handles.size(), &handles[0], - (timeout > 60) ? 60 * 1000 : timeout * 1000, + MsgWaitForMultipleObjectsEx((DWORD)handles.size(), &handles[0], (timeout > 60) ? 60 * 1000 : timeout * 1000, QS_ALLINPUT | QS_ALLPOSTMESSAGE, 0); if (rc == WAIT_OBJECT_0) { diff --git a/test/stub/pthread_barrier.c b/test/stub/pthread_barrier.c index 2e37900a..f8be732d 100644 --- a/test/stub/pthread_barrier.c +++ b/test/stub/pthread_barrier.c @@ -42,8 +42,7 @@ int pthread_barrierattr_destroy(pthread_barrierattr_t *attr) { return m ? m : c; } -int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr, - int *__restrict pshared) { +int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr, int *__restrict pshared) { return pthread_condattr_getpshared(&attr->cattr, pshared); } @@ -53,8 +52,7 @@ int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared) { return m ? m : c; } -int pthread_barrier_init(pthread_barrier_t *__restrict barrier, - const pthread_barrierattr_t *__restrict attr, +int pthread_barrier_init(pthread_barrier_t *__restrict barrier, const pthread_barrierattr_t *__restrict attr, unsigned count) { if (count == 0) return errno = EINVAL; diff --git a/test/stub/pthread_barrier.h b/test/stub/pthread_barrier.h index b9e0dd7c..5b8c6d86 100644 --- a/test/stub/pthread_barrier.h +++ b/test/stub/pthread_barrier.h @@ -61,12 +61,10 @@ typedef struct { int pthread_barrierattr_init(pthread_barrierattr_t *attr); int pthread_barrierattr_destroy(pthread_barrierattr_t *attr); -int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr, - int *__restrict pshared); +int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr, int *__restrict pshared); int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared); -int pthread_barrier_init(pthread_barrier_t *__restrict barrier, - const pthread_barrierattr_t *__restrict attr, +int pthread_barrier_init(pthread_barrier_t *__restrict barrier, const pthread_barrierattr_t *__restrict attr, unsigned int count); int pthread_barrier_destroy(pthread_barrier_t *barrier); diff --git a/test/test.c++ b/test/test.c++ index 9c827fc5..1d8db898 100644 --- a/test/test.c++ +++ b/test/test.c++ @@ -73,17 +73,14 @@ const char *keygencase2str(const keygen_case keycase) { //----------------------------------------------------------------------------- -int testcase::hsr_callback(const MDBX_env *env, const MDBX_txn *txn, - mdbx_pid_t pid, mdbx_tid_t tid, uint64_t laggard, - unsigned gap, size_t space, - int retry) MDBX_CXX17_NOEXCEPT { +int testcase::hsr_callback(const MDBX_env *env, const MDBX_txn *txn, mdbx_pid_t pid, mdbx_tid_t tid, uint64_t laggard, + unsigned gap, size_t space, int retry) MDBX_CXX17_NOEXCEPT { (void)txn; testcase *self = (testcase *)mdbx_env_get_userctx(env); if (retry == 0) - log_notice("hsr_callback: waitfor pid %lu, thread %" PRIuPTR - ", txn #%" PRIu64 ", gap %d, space %zu", - (long)pid, (size_t)tid, laggard, gap, space); + log_notice("hsr_callback: waitfor pid %lu, thread %" PRIuPTR ", txn #%" PRIu64 ", gap %d, space %zu", (long)pid, + (size_t)tid, laggard, gap, space); MDBX_envinfo info; int rc = mdbx_env_info_ex(env, txn, &info, sizeof(info)); @@ -91,8 +88,7 @@ int testcase::hsr_callback(const MDBX_env *env, const MDBX_txn *txn, return rc; if (self->should_continue(true) && - (space > size_t(info.mi_geo.grow) * 2 || - info.mi_geo.current >= info.mi_geo.upper)) { + (space > size_t(info.mi_geo.grow) * 2 || info.mi_geo.current >= info.mi_geo.upper)) { osal_yield(); if (retry > 0) osal_udelay(retry * size_t(100)); @@ -131,10 +127,8 @@ void testcase::db_prepare() { if (unlikely(rc != MDBX_SUCCESS)) failure_perror("mdbx_env_set_hsr()", rc); - rc = mdbx_env_set_geometry( - env, config.params.size_lower, config.params.size_now, - config.params.size_upper, config.params.growth_step, - config.params.shrink_threshold, config.params.pagesize); + rc = mdbx_env_set_geometry(env, config.params.size_lower, config.params.size_now, config.params.size_upper, + config.params.growth_step, config.params.shrink_threshold, config.params.pagesize); if (unlikely(rc != MDBX_SUCCESS)) failure_perror("mdbx_env_set_mapsize()", rc); @@ -153,8 +147,7 @@ void testcase::db_open() { if (config.params.random_writemap && flipcoin()) mode ^= MDBX_WRITEMAP; - int rc = mdbx_env_open(db_guard.get(), config.params.pathname_db.c_str(), - mode, 0640); + int rc = mdbx_env_open(db_guard.get(), config.params.pathname_db.c_str(), mode, 0640); if (unlikely(rc != MDBX_SUCCESS)) failure_perror("mdbx_env_open()", rc); @@ -185,32 +178,26 @@ void testcase::db_close() { void testcase::txn_begin(bool readonly, MDBX_txn_flags_t flags) { assert((flags & MDBX_TXN_RDONLY) == 0); - log_trace(">> txn_begin(%s, 0x%04X)", readonly ? "read-only" : "read-write", - flags); + log_trace(">> txn_begin(%s, 0x%04X)", readonly ? "read-only" : "read-write", flags); assert(!txn_guard); MDBX_txn *txn = nullptr; - int rc = mdbx_txn_begin(db_guard.get(), nullptr, - readonly ? flags | MDBX_TXN_RDONLY : flags, &txn); + int rc = mdbx_txn_begin(db_guard.get(), nullptr, readonly ? flags | MDBX_TXN_RDONLY : flags, &txn); if (unlikely(rc != MDBX_SUCCESS)) failure_perror("mdbx_txn_begin()", rc); txn_guard.reset(txn); need_speculum_assign = config.params.speculum && !readonly; - log_trace("<< txn_begin(%s, 0x%04X)", readonly ? "read-only" : "read-write", - flags); + log_trace("<< txn_begin(%s, 0x%04X)", readonly ? "read-only" : "read-write", flags); if (flipcoin_n(5)) { - const unsigned mask = - unsigned(MDBX_warmup_default | MDBX_warmup_force | MDBX_warmup_oomsafe | - MDBX_warmup_lock | MDBX_warmup_touchlimit); + const unsigned mask = unsigned(MDBX_warmup_default | MDBX_warmup_force | MDBX_warmup_oomsafe | MDBX_warmup_lock | + MDBX_warmup_touchlimit); static unsigned counter; - MDBX_warmup_flags_t warmup_flags = MDBX_warmup_flags_t( - (counter > MDBX_warmup_release) ? prng64() & mask : counter); + MDBX_warmup_flags_t warmup_flags = MDBX_warmup_flags_t((counter > MDBX_warmup_release) ? prng64() & mask : counter); counter += 1; int err = mdbx_env_warmup(db_guard.get(), txn, warmup_flags, 0); - log_trace("== counter %u, env_warmup(flags %u), rc %d", counter, - warmup_flags, err); + log_trace("== counter %u, env_warmup(flags %u), rc %d", counter, warmup_flags, err); } if (readonly && flipcoin()) @@ -226,8 +213,7 @@ int testcase::breakable_commit() { * during call mdbx_cmp() with zero txn. So it is the workaround for this: * - explicitly make copies of the `speculums`; * - explicitly move relevant copy after transaction commit. */ - SET speculum_committed_copy(ItemCompare(this)), - speculum_copy(ItemCompare(this)); + SET speculum_committed_copy(ItemCompare(this)), speculum_copy(ItemCompare(this)); if (need_speculum_assign) { speculum_committed_copy = speculum_committed; speculum_copy = speculum; @@ -236,8 +222,7 @@ int testcase::breakable_commit() { MDBX_txn *txn = txn_guard.release(); txn_inject_writefault(txn); int rc = mdbx_txn_commit(txn); - if (unlikely(rc != MDBX_SUCCESS) && - (rc != MDBX_MAP_FULL || !config.params.ignore_dbfull)) + if (unlikely(rc != MDBX_SUCCESS) && (rc != MDBX_MAP_FULL || !config.params.ignore_dbfull)) failure_perror("mdbx_txn_commit()", rc); if (need_speculum_assign) { @@ -259,8 +244,7 @@ unsigned testcase::txn_underutilization_x256(MDBX_txn *txn) const { if (unlikely(err != MDBX_SUCCESS)) failure_perror("mdbx_txn_info()", err); const size_t left = size_t(info.txn_space_leftover); - const size_t total = - size_t(info.txn_space_leftover) + size_t(info.txn_space_dirty); + const size_t total = size_t(info.txn_space_leftover) + size_t(info.txn_space_dirty); return (unsigned)(left / (total >> 8)); } return 0; @@ -357,9 +341,7 @@ void testcase::txn_inject_writefault(MDBX_txn *txn) { if (config.params.inject_writefaultn && txn) { if (config.params.inject_writefaultn <= nops_completed && (MDBX_txn_flags_t(mdbx_txn_flags(txn)) & MDBX_TXN_RDONLY) == 0) { - log_verbose( - "== txn_inject_writefault(): got %u nops or more, inject FAULT", - config.params.inject_writefaultn); + log_verbose("== txn_inject_writefault(): got %u nops or more, inject FAULT", config.params.inject_writefaultn); log_flush(); #if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) TerminateProcess(GetCurrentProcess(), 42); @@ -376,8 +358,7 @@ bool testcase::wait4start() { assert(!global::singlemode); int rc = osal_waitfor(config.wait4id); if (rc) { - log_trace("<< wait4start(%u), failed %s", config.wait4id, - test_strerror(rc)); + log_trace("<< wait4start(%u), failed %s", config.wait4id, test_strerror(rc)); return false; } } else { @@ -387,8 +368,7 @@ bool testcase::wait4start() { if (config.params.delaystart) { int rc = osal_delay(config.params.delaystart); if (rc) { - log_trace("<< delay(%u), failed %s", config.params.delaystart, - test_strerror(rc)); + log_trace("<< delay(%u), failed %s", config.params.delaystart, test_strerror(rc)); return false; } } else { @@ -410,13 +390,11 @@ void testcase::report(size_t nops_done) { return; nops_completed += nops_done; - log_debug("== complete +%" PRIuPTR " iteration, total %" PRIu64 " done", - nops_done, nops_completed); + log_debug("== complete +%" PRIuPTR " iteration, total %" PRIu64 " done", nops_done, nops_completed); kick_progress(true); - if (config.signal_nops && !signalled && - config.signal_nops <= nops_completed) { + if (config.signal_nops && !signalled && config.signal_nops <= nops_completed) { log_trace(">> signal(n-ops %" PRIu64 ")", nops_completed); if (!global::singlemode) osal_broadcast(config.actor_id); @@ -458,14 +436,12 @@ bool testcase::should_continue(bool check_timeout_only) const { if (config.params.test_duration) { chrono::time since; - since.fixedpoint = - chrono::now_monotonic().fixedpoint - start_timestamp.fixedpoint; + since.fixedpoint = chrono::now_monotonic().fixedpoint - start_timestamp.fixedpoint; if (since.seconds() >= config.params.test_duration) result = false; } - if (!check_timeout_only && config.params.test_nops && - nops_completed >= config.params.test_nops) + if (!check_timeout_only && config.params.test_nops && nops_completed >= config.params.test_nops) result = false; if (result) @@ -483,25 +459,18 @@ void testcase::fetch_canary() { failure_perror("mdbx_canary_get()", rc); if (canary_now.v < last.canary.v) - failure("fetch_canary: %" PRIu64 "(canary-now.v) < %" PRIu64 - "(canary-last.v)", - canary_now.v, last.canary.v); + failure("fetch_canary: %" PRIu64 "(canary-now.v) < %" PRIu64 "(canary-last.v)", canary_now.v, last.canary.v); if (canary_now.y < last.canary.y) - failure("fetch_canary: %" PRIu64 "(canary-now.y) < %" PRIu64 - "(canary-last.y)", - canary_now.y, last.canary.y); + failure("fetch_canary: %" PRIu64 "(canary-now.y) < %" PRIu64 "(canary-last.y)", canary_now.y, last.canary.y); last.canary = canary_now; - log_trace("<< fetch_canary: db-sequence %" PRIu64 - ", db-sequence.txnid %" PRIu64, - last.canary.y, last.canary.v); + log_trace("<< fetch_canary: db-sequence %" PRIu64 ", db-sequence.txnid %" PRIu64, last.canary.y, last.canary.v); } void testcase::update_canary(uint64_t increment) { MDBX_canary canary_now = last.canary; - log_trace(">> update_canary: sequence %" PRIu64 " += %" PRIu64, canary_now.y, - increment); + log_trace(">> update_canary: sequence %" PRIu64 " += %" PRIu64, canary_now.y, increment); canary_now.y += increment; int rc = mdbx_canary_put(txn_guard.get(), &canary_now); @@ -511,8 +480,7 @@ void testcase::update_canary(uint64_t increment) { log_trace("<< update_canary: sequence = %" PRIu64, canary_now.y); } -bool testcase::is_handle_created_in_current_txn(const MDBX_dbi handle, - MDBX_txn *txn) { +bool testcase::is_handle_created_in_current_txn(const MDBX_dbi handle, MDBX_txn *txn) { unsigned flags, state; int err = mdbx_dbi_flags_ex(txn, handle, &flags, &state); if (unlikely(err != MDBX_SUCCESS)) @@ -540,17 +508,14 @@ int testcase::db_open__begin__table_create_open_clean(MDBX_dbi &handle) { break; jitter_delay(true); } - log_notice("db_begin_table_create_open_clean: bailout due '%s'", - mdbx_strerror(err)); + log_notice("db_begin_table_create_open_clean: bailout due '%s'", mdbx_strerror(err)); return err; } -const char *testcase::db_tablename(tablename_buf &buffer, - const char *suffix) const { +const char *testcase::db_tablename(tablename_buf &buffer, const char *suffix) const { const char *tablename = nullptr; if (config.space_id) { - int rc = - snprintf(buffer, sizeof(buffer), "TBL%04u%s", config.space_id, suffix); + int rc = snprintf(buffer, sizeof(buffer), "TBL%04u%s", config.space_id, suffix); if (rc < 4 || rc >= (int)sizeof(tablename_buf) - 1) failure("snprintf(tablename): %d", rc); tablename = buffer; @@ -560,23 +525,20 @@ const char *testcase::db_tablename(tablename_buf &buffer, } MDBX_dbi testcase::db_table_open(bool create, bool expect_failure) { - log_trace(">> testcase::db_table_%s%s", create ? "create" : "open", - expect_failure ? "(expect_failure)" : ""); + log_trace(">> testcase::db_table_%s%s", create ? "create" : "open", expect_failure ? "(expect_failure)" : ""); tablename_buf buffer; const char *tablename = db_tablename(buffer); MDBX_dbi handle = 0; - int rc = mdbx_dbi_open( - txn_guard.get(), tablename, - create ? (MDBX_CREATE | config.params.table_flags) - : (flipcoin() ? MDBX_DB_ACCEDE - : MDBX_DB_DEFAULTS | config.params.table_flags), - &handle); + int rc = mdbx_dbi_open(txn_guard.get(), tablename, + create ? (MDBX_CREATE | config.params.table_flags) + : (flipcoin() ? MDBX_DB_ACCEDE : MDBX_DB_DEFAULTS | config.params.table_flags), + &handle); if (unlikely(expect_failure != (rc != MDBX_SUCCESS))) { char act[64]; - snprintf(act, sizeof(act), "mdbx_dbi_open(create=%s,expect_failure=%s)", - create ? "true" : "false", expect_failure ? "true" : "false"); + snprintf(act, sizeof(act), "mdbx_dbi_open(create=%s,expect_failure=%s)", create ? "true" : "false", + expect_failure ? "true" : "false"); failure_perror(act, rc); } @@ -617,11 +579,9 @@ void testcase::db_table_close(MDBX_dbi handle) { log_trace("<< testcase::db_table_close"); } -bool testcase::checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check, - MDBX_val expected_valued) { +bool testcase::checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check, MDBX_val expected_valued) { MDBX_val actual_value = expected_valued; - int err = mdbx_get_equal_or_great(txn_guard.get(), handle, &key2check, - &actual_value); + int err = mdbx_get_equal_or_great(txn_guard.get(), handle, &key2check, &actual_value); if (unlikely(err != MDBX_SUCCESS)) { if (!config.params.speculum || err != MDBX_RESULT_TRUE) failure_perror(step, (err == MDBX_RESULT_TRUE) ? MDBX_NOTFOUND : err); @@ -690,8 +650,7 @@ static void dump_stack(CONTEXT *ctx, FILE *out) { #error "FIXME" #endif , - process, thread, &stack, &ctxCopy, NULL, SymFunctionTableAccess64, - SymGetModuleBase64, NULL); + process, thread, &stack, &ctxCopy, NULL, SymFunctionTableAccess64, SymGetModuleBase64, NULL); if (!result) break; @@ -706,16 +665,14 @@ static void dump_stack(CONTEXT *ctx, FILE *out) { // try to get line if (SymGetLineFromAddr64(process, stack.AddrPC.Offset, &disp, &line)) { - fprintf(out, "\tat %s in %s: line: %lu: address: 0x%0" PRIx64 "\n", - pSymbol->Name, line.FileName, line.LineNumber, pSymbol->Address); + fprintf(out, "\tat %s in %s: line: %lu: address: 0x%0" PRIx64 "\n", pSymbol->Name, line.FileName, line.LineNumber, + pSymbol->Address); } else { // failed to get line - fprintf(out, "\tat %s, address 0x%0" PRIx64 ".\n", pSymbol->Name, - pSymbol->Address); + fprintf(out, "\tat %s, address 0x%0" PRIx64 ".\n", pSymbol->Name, pSymbol->Address); hModule = NULL; lstrcpyA(module, ""); - GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | - GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (LPCTSTR)(stack.AddrPC.Offset), &hModule); // at least print module name @@ -787,19 +744,16 @@ static LONG seh_filter(struct _EXCEPTION_POINTERS *ExInfo, FILE *out) { PVOID CodeAdress = ExInfo->ExceptionRecord->ExceptionAddress; fprintf(out, "****************************************************\n"); fprintf(out, "*** A Program Fault occurred:\n"); - fprintf(out, "*** Error code %08X: %s\n", - ExInfo->ExceptionRecord->ExceptionCode, caption); + fprintf(out, "*** Error code %08X: %s\n", ExInfo->ExceptionRecord->ExceptionCode, caption); fprintf(out, "****************************************************\n"); fprintf(out, "*** Address: %08zX\n", (intptr_t)CodeAdress); - fprintf(out, "*** Flags: %08X\n", - ExInfo->ExceptionRecord->ExceptionFlags); + fprintf(out, "*** Flags: %08X\n", ExInfo->ExceptionRecord->ExceptionFlags); dump_stack(ExInfo->ContextRecord, out); return EXCEPTION_EXECUTE_HANDLER; } #endif /* _MSC_VER */ -static bool execute_thunk(const actor_config *const_config, - const mdbx_pid_t pid) { +static bool execute_thunk(const actor_config *const_config, const mdbx_pid_t pid) { actor_config config = *const_config; try { if (global::singlemode) { @@ -837,8 +791,7 @@ static bool execute_thunk(const actor_config *const_config, log_verbose("test successfully"); else { if (config.params.nrepeat) - log_verbose("test successfully (iteration %zi of %zi)", iter, - size_t(config.params.nrepeat)); + log_verbose("test successfully (iteration %zi of %zi)", iter, size_t(config.params.nrepeat)); else log_verbose("test successfully (iteration %zi)", iter); } @@ -866,61 +819,44 @@ bool test_execute(const actor_config &config) { //----------------------------------------------------------------------------- -enum speculum_cursors : int { - lowerbound = 0, - prev = 1, - prev_prev = 2, - next = 3, - next_next = 4, - seek_check = 5 -}; +enum speculum_cursors : int { lowerbound = 0, prev = 1, prev_prev = 2, next = 3, next_next = 4, seek_check = 5 }; bool testcase::is_same(const Item &a, const Item &b) const { if (!is_samedata(dataview2iov(a.first), dataview2iov(b.first))) return false; - if ((config.params.table_flags & MDBX_DUPSORT) && - !is_samedata(dataview2iov(a.second), dataview2iov(b.second))) + if ((config.params.table_flags & MDBX_DUPSORT) && !is_samedata(dataview2iov(a.second), dataview2iov(b.second))) return false; return true; } -bool testcase::is_same(const testcase::SET::const_iterator &it, - const MDBX_val &k, const MDBX_val &v) const { +bool testcase::is_same(const testcase::SET::const_iterator &it, const MDBX_val &k, const MDBX_val &v) const { - return is_samedata(dataview2iov(it->first), k) && - is_samedata(dataview2iov(it->second), v); + return is_samedata(dataview2iov(it->first), k) && is_samedata(dataview2iov(it->second), v); } -void testcase::verbose(const char *where, const char *stage, - const testcase::SET::const_iterator &it) const { +void testcase::verbose(const char *where, const char *stage, const testcase::SET::const_iterator &it) const { if (it == speculum.end()) log_verbose("speculum-%s: %s expect END", where, stage); else { char dump_key[32], dump_value[32]; MDBX_val it_key = dataview2iov(it->first); MDBX_val it_data = dataview2iov(it->second); - log_verbose("speculum-%s: %s expect {%s, %s}", where, stage, - mdbx_dump_val(&it_key, dump_key, sizeof(dump_key)), + log_verbose("speculum-%s: %s expect {%s, %s}", where, stage, mdbx_dump_val(&it_key, dump_key, sizeof(dump_key)), mdbx_dump_val(&it_data, dump_value, sizeof(dump_value))); } } -void testcase::verbose(const char *where, const char *stage, const MDBX_val &k, - const MDBX_val &v, int err) const { +void testcase::verbose(const char *where, const char *stage, const MDBX_val &k, const MDBX_val &v, int err) const { char dump_key[32], dump_value[32]; if (err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE) - log_verbose("speculum-%s: %s cursor {%d, %s}", where, stage, err, - mdbx_strerror(err)); + log_verbose("speculum-%s: %s cursor {%d, %s}", where, stage, err, mdbx_strerror(err)); else - log_verbose("speculum-%s: %s cursor {%s, %s}", where, stage, - mdbx_dump_val(&k, dump_key, sizeof(dump_key)), + log_verbose("speculum-%s: %s cursor {%s, %s}", where, stage, mdbx_dump_val(&k, dump_key, sizeof(dump_key)), mdbx_dump_val(&v, dump_value, sizeof(dump_value))); } -bool testcase::speculum_check_iterator(const char *where, const char *stage, - const testcase::SET::const_iterator &it, - const MDBX_val &k, const MDBX_val &v, - MDBX_cursor *cursor) const { +bool testcase::speculum_check_iterator(const char *where, const char *stage, const testcase::SET::const_iterator &it, + const MDBX_val &k, const MDBX_val &v, MDBX_cursor *cursor) const { char dump_key[32], dump_value[32]; MDBX_val it_key = dataview2iov(it->first); MDBX_val it_data = dataview2iov(it->second); @@ -935,8 +871,8 @@ bool testcase::speculum_check_iterator(const char *where, const char *stage, } if (!is_samedata(it_data, v)) { speculum_render(it, cursor); - return failure("speculum-%s: %s data mismatch %s (must) != %s", where, - stage, mdbx_dump_val(&it_data, dump_key, sizeof(dump_key)), + return failure("speculum-%s: %s data mismatch %s (must) != %s", where, stage, + mdbx_dump_val(&it_data, dump_key, sizeof(dump_key)), mdbx_dump_val(&v, dump_value, sizeof(dump_value))); } return true; @@ -957,8 +893,7 @@ bool testcase::failure(const char *fmt, ...) const { #if SPECULUM_CURSORS -static void speculum_render_cursor(const MDBX_val &ikey, const MDBX_val &ival, - const MDBX_cursor *cursor, +static void speculum_render_cursor(const MDBX_val &ikey, const MDBX_val &ival, const MDBX_cursor *cursor, const MDBX_cursor *ref) { scoped_cursor_guard guard(mdbx_cursor_create(nullptr)); if (!guard) @@ -990,16 +925,14 @@ static void speculum_render_cursor(const MDBX_val &ikey, const MDBX_val &ival, if (mdbx_cursor_get(clone, &ckey, &cval, MDBX_GET_CURRENT) != MDBX_SUCCESS) *s++ = '!'; else { - const int kcmp = - mdbx_cmp(mdbx_cursor_txn(clone), mdbx_cursor_dbi(clone), &ikey, &ckey); + const int kcmp = mdbx_cmp(mdbx_cursor_txn(clone), mdbx_cursor_dbi(clone), &ikey, &ckey); if (kcmp < 0) *s++ = '<'; else if (kcmp > 0) *s++ = '>'; else { *s++ = '='; - const int vcmp = mdbx_dcmp(mdbx_cursor_txn(clone), mdbx_cursor_dbi(clone), - &ival, &cval); + const int vcmp = mdbx_dcmp(mdbx_cursor_txn(clone), mdbx_cursor_dbi(clone), &ival, &cval); if (vcmp < 0) *s++ = '<'; else if (vcmp > 0) @@ -1018,8 +951,7 @@ static void speculum_render_cursor(const MDBX_val &ikey, const MDBX_val &ival, printf(" | %-10.10s", status); } -void testcase::speculum_render(const testcase::SET::const_iterator &it, - const MDBX_cursor *ref) const { +void testcase::speculum_render(const testcase::SET::const_iterator &it, const MDBX_cursor *ref) const { char dump_key[32], dump_value[32]; auto top = it; @@ -1030,21 +962,18 @@ void testcase::speculum_render(const testcase::SET::const_iterator &it, } printf("## %-20.20s %-20.20s | %-10.10s | %-10.10s | %-10.10s | %-10.10s | " "%-10.10s | %-10.10s |\n", - "k0_1_2_3_4_5_6_7_8_9", "v0_1_2_3_4_5_6_7_8_9", "prev-prev", "prev", - "seek", "lowerbound", "next", "next-next"); + "k0_1_2_3_4_5_6_7_8_9", "v0_1_2_3_4_5_6_7_8_9", "prev-prev", "prev", "seek", "lowerbound", "next", + "next-next"); while (offset < 5 && top != speculum.end()) { const MDBX_val ikey = dataview2iov(top->first); const MDBX_val idata = dataview2iov(top->second); - printf("%+d) %20.20s %20.20s", offset, - mdbx_dump_val(&ikey, dump_key, sizeof(dump_key)), + printf("%+d) %20.20s %20.20s", offset, mdbx_dump_val(&ikey, dump_key, sizeof(dump_key)), mdbx_dump_val(&idata, dump_value, sizeof(dump_value))); speculum_render_cursor(ikey, idata, speculum_cursors[prev_prev].get(), ref); speculum_render_cursor(ikey, idata, speculum_cursors[prev].get(), ref); - speculum_render_cursor(ikey, idata, speculum_cursors[seek_check].get(), - ref); - speculum_render_cursor(ikey, idata, speculum_cursors[lowerbound].get(), - ref); + speculum_render_cursor(ikey, idata, speculum_cursors[seek_check].get(), ref); + speculum_render_cursor(ikey, idata, speculum_cursors[lowerbound].get(), ref); speculum_render_cursor(ikey, idata, speculum_cursors[next].get(), ref); speculum_render_cursor(ikey, idata, speculum_cursors[next_next].get(), ref); @@ -1054,29 +983,24 @@ void testcase::speculum_render(const testcase::SET::const_iterator &it, } } -bool testcase::speculum_check_cursor(const char *where, const char *stage, - const testcase::SET::const_iterator &it, - int cursor_err, const MDBX_val &cursor_key, - const MDBX_val &cursor_data, +bool testcase::speculum_check_cursor(const char *where, const char *stage, const testcase::SET::const_iterator &it, + int cursor_err, const MDBX_val &cursor_key, const MDBX_val &cursor_data, MDBX_cursor *cursor) const { // verbose(where, stage, cursor_key, cursor_data, cursor_err); // verbose(where, stage, it); - if (cursor_err != MDBX_SUCCESS && cursor_err != MDBX_NOTFOUND && - cursor_err != MDBX_RESULT_TRUE && cursor_err != MDBX_ENODATA) { + if (cursor_err != MDBX_SUCCESS && cursor_err != MDBX_NOTFOUND && cursor_err != MDBX_RESULT_TRUE && + cursor_err != MDBX_ENODATA) { speculum_render(it, cursor); - return failure("speculum-%s: %s %s %d %s", where, stage, "cursor-get", - cursor_err, mdbx_strerror(cursor_err)); + return failure("speculum-%s: %s %s %d %s", where, stage, "cursor-get", cursor_err, mdbx_strerror(cursor_err)); } char dump_key[32], dump_value[32]; - if (it == speculum.end() && cursor_err != MDBX_NOTFOUND && - cursor_err != MDBX_ENODATA) { + if (it == speculum.end() && cursor_err != MDBX_NOTFOUND && cursor_err != MDBX_ENODATA) { speculum_render(it, cursor); return failure("speculum-%s: %s extra pair {%s, %s}", where, stage, mdbx_dump_val(&cursor_key, dump_key, sizeof(dump_key)), mdbx_dump_val(&cursor_data, dump_value, sizeof(dump_value))); - } else if (it != speculum.end() && - (cursor_err == MDBX_NOTFOUND || cursor_err == MDBX_ENODATA)) { + } else if (it != speculum.end() && (cursor_err == MDBX_NOTFOUND || cursor_err == MDBX_ENODATA)) { speculum_render(it, cursor); MDBX_val it_key = dataview2iov(it->first); MDBX_val it_data = dataview2iov(it->second); @@ -1084,24 +1008,19 @@ bool testcase::speculum_check_cursor(const char *where, const char *stage, mdbx_dump_val(&it_key, dump_key, sizeof(dump_key)), mdbx_dump_val(&it_data, dump_value, sizeof(dump_value))); } else if (cursor_err == MDBX_SUCCESS || cursor_err == MDBX_RESULT_TRUE) - return speculum_check_iterator(where, stage, it, cursor_key, cursor_data, - cursor); + return speculum_check_iterator(where, stage, it, cursor_key, cursor_data, cursor); else { - assert(it == speculum.end() && - (cursor_err == MDBX_NOTFOUND || cursor_err == MDBX_ENODATA)); + assert(it == speculum.end() && (cursor_err == MDBX_NOTFOUND || cursor_err == MDBX_ENODATA)); return true; } } -bool testcase::speculum_check_cursor(const char *where, const char *stage, - const testcase::SET::const_iterator &it, - MDBX_cursor *cursor, - const MDBX_cursor_op op) const { +bool testcase::speculum_check_cursor(const char *where, const char *stage, const testcase::SET::const_iterator &it, + MDBX_cursor *cursor, const MDBX_cursor_op op) const { MDBX_val cursor_key = {0, 0}; MDBX_val cursor_data = {0, 0}; int err = mdbx_cursor_get(cursor, &cursor_key, &cursor_data, op); - return speculum_check_cursor(where, stage, it, err, cursor_key, cursor_data, - cursor); + return speculum_check_cursor(where, stage, it, err, cursor_key, cursor_data, cursor); } void testcase::speculum_prepare_cursors(const Item &item) { @@ -1109,8 +1028,7 @@ void testcase::speculum_prepare_cursors(const Item &item) { assert(config.params.speculum); if (speculum_cursors[lowerbound]) for (auto &guard : speculum_cursors) { - if (txn_guard.get() != mdbx_cursor_txn(guard.get()) || - dbi != mdbx_cursor_dbi(guard.get())) { + if (txn_guard.get() != mdbx_cursor_txn(guard.get()) || dbi != mdbx_cursor_dbi(guard.get())) { err = mdbx_cursor_bind(txn_guard.get(), guard.get(), dbi); if (unlikely(err != MDBX_SUCCESS)) failure_perror("mdbx_cursor_bind()", err); @@ -1127,81 +1045,64 @@ void testcase::speculum_prepare_cursors(const Item &item) { // mdbx_cursor_reset(speculum_cursors[seek_check].get()); const auto cursor_lowerbound = speculum_cursors[lowerbound].get(); - const MDBX_val item_key = dataview2iov(item.first), - item_data = dataview2iov(item.second); + const MDBX_val item_key = dataview2iov(item.first), item_data = dataview2iov(item.second); MDBX_val lowerbound_key = item_key; MDBX_val lowerbound_data = item_data; // verbose("prepare-cursors", "item", item_key, item_data); - err = mdbx_cursor_get(cursor_lowerbound, &lowerbound_key, &lowerbound_data, - MDBX_SET_LOWERBOUND); + err = mdbx_cursor_get(cursor_lowerbound, &lowerbound_key, &lowerbound_data, MDBX_SET_LOWERBOUND); // verbose("prepare-cursors", "lowerbound", lowerbound_key, lowerbound_data, // err); - if (unlikely(err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE && - err != MDBX_NOTFOUND)) - failure("speculum-%s: %s %s %d %s", "prepare-cursors", "lowerbound", - "cursor-get", err, mdbx_strerror(err)); + if (unlikely(err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE && err != MDBX_NOTFOUND)) + failure("speculum-%s: %s %s %d %s", "prepare-cursors", "lowerbound", "cursor-get", err, mdbx_strerror(err)); auto it_lowerbound = speculum.lower_bound(item); // verbose("prepare-cursors", "lowerbound", it_lowerbound); - speculum_check_cursor("prepare-cursors", "lowerbound", it_lowerbound, err, - lowerbound_key, lowerbound_data, cursor_lowerbound); + speculum_check_cursor("prepare-cursors", "lowerbound", it_lowerbound, err, lowerbound_key, lowerbound_data, + cursor_lowerbound); const auto cursor_prev = speculum_cursors[prev].get(); err = mdbx_cursor_copy(cursor_lowerbound, cursor_prev); if (unlikely(err != MDBX_SUCCESS)) - failure("speculum-%s: %s %s %d %s", "prepare-cursors", "prev", - "cursor-copy", err, mdbx_strerror(err)); + failure("speculum-%s: %s %s %d %s", "prepare-cursors", "prev", "cursor-copy", err, mdbx_strerror(err)); auto it_prev = it_lowerbound; if (it_prev != speculum.begin()) { - speculum_check_cursor("prepare-cursors", "prev", --it_prev, cursor_prev, - MDBX_PREV); + speculum_check_cursor("prepare-cursors", "prev", --it_prev, cursor_prev, MDBX_PREV); } else if ((err = mdbx_cursor_on_first(cursor_prev)) != MDBX_RESULT_TRUE) - failure("speculum-%s: %s on-first %d %s", "prepare-cursors", "prev", err, - mdbx_strerror(err)); + failure("speculum-%s: %s on-first %d %s", "prepare-cursors", "prev", err, mdbx_strerror(err)); const auto cursor_prev_prev = speculum_cursors[prev_prev].get(); err = mdbx_cursor_copy(cursor_prev, cursor_prev_prev); if (unlikely(err != MDBX_SUCCESS)) - failure("speculum-%s: %s %s %d %s", "prepare-cursors", "prev-prev", - "cursor-copy", err, mdbx_strerror(err)); + failure("speculum-%s: %s %s %d %s", "prepare-cursors", "prev-prev", "cursor-copy", err, mdbx_strerror(err)); auto it_prev_prev = it_prev; if (it_prev_prev != speculum.begin()) { - speculum_check_cursor("prepare-cursors", "prev-prev", --it_prev_prev, - cursor_prev_prev, MDBX_PREV); + speculum_check_cursor("prepare-cursors", "prev-prev", --it_prev_prev, cursor_prev_prev, MDBX_PREV); } else if ((err = mdbx_cursor_on_first(cursor_prev_prev)) != MDBX_RESULT_TRUE) - failure("speculum-%s: %s on-first %d %s", "prepare-cursors", "prev-prev", - err, mdbx_strerror(err)); + failure("speculum-%s: %s on-first %d %s", "prepare-cursors", "prev-prev", err, mdbx_strerror(err)); const auto cursor_next = speculum_cursors[next].get(); err = mdbx_cursor_copy(cursor_lowerbound, cursor_next); if (unlikely(err != MDBX_SUCCESS)) - failure("speculum-%s: %s %s %d %s", "prepare-cursors", "next", - "cursor-copy", err, mdbx_strerror(err)); + failure("speculum-%s: %s %s %d %s", "prepare-cursors", "next", "cursor-copy", err, mdbx_strerror(err)); auto it_next = it_lowerbound; if (it_next != speculum.end()) { - speculum_check_cursor("prepare-cursors", "next", ++it_next, cursor_next, - MDBX_NEXT); + speculum_check_cursor("prepare-cursors", "next", ++it_next, cursor_next, MDBX_NEXT); } else if ((err = mdbx_cursor_on_last(cursor_next)) != MDBX_RESULT_TRUE) - failure("speculum-%s: %s on-last %d %s", "prepare-cursors", "next", err, - mdbx_strerror(err)); + failure("speculum-%s: %s on-last %d %s", "prepare-cursors", "next", err, mdbx_strerror(err)); const auto cursor_next_next = speculum_cursors[next_next].get(); err = mdbx_cursor_copy(cursor_next, cursor_next_next); if (unlikely(err != MDBX_SUCCESS)) - failure("speculum-%s: %s %s %d %s", "prepare-cursors", "next-next", - "cursor-copy", err, mdbx_strerror(err)); + failure("speculum-%s: %s %s %d %s", "prepare-cursors", "next-next", "cursor-copy", err, mdbx_strerror(err)); auto it_next_next = it_next; if (it_next_next != speculum.end()) { - speculum_check_cursor("prepare-cursors", "next-next", ++it_next_next, - cursor_next_next, MDBX_NEXT); + speculum_check_cursor("prepare-cursors", "next-next", ++it_next_next, cursor_next_next, MDBX_NEXT); } else if ((err = mdbx_cursor_on_last(cursor_next_next)) != MDBX_RESULT_TRUE) - failure("speculum-%s: %s on-last %d %s", "prepare-cursors", "next-next", - err, mdbx_strerror(err)); + failure("speculum-%s: %s on-last %d %s", "prepare-cursors", "next-next", err, mdbx_strerror(err)); } #endif /* SPECULUM_CURSORS */ -int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata, - MDBX_put_flags_t flags) { +int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata, MDBX_put_flags_t flags) { int err; bool rc = true; Item item; @@ -1218,13 +1119,10 @@ int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata, check_seek_cursor = speculum_cursors[seek_check].get(); seek_check_key = akey->value; seek_check_data = adata->value; - seek_check_err = mdbx_cursor_get(check_seek_cursor, &seek_check_key, - &seek_check_data, MDBX_SET_LOWERBOUND); + seek_check_err = mdbx_cursor_get(check_seek_cursor, &seek_check_key, &seek_check_data, MDBX_SET_LOWERBOUND); // speculum_render(speculum.find(item), check_seek_cursor); - if (seek_check_err != MDBX_SUCCESS && seek_check_err != MDBX_NOTFOUND && - seek_check_err != MDBX_RESULT_TRUE) - failure("speculum-%s: %s pre-insert %d %s", "insert", "seek", - seek_check_err, mdbx_strerror(seek_check_err)); + if (seek_check_err != MDBX_SUCCESS && seek_check_err != MDBX_NOTFOUND && seek_check_err != MDBX_RESULT_TRUE) + failure("speculum-%s: %s pre-insert %d %s", "insert", "seek", seek_check_err, mdbx_strerror(seek_check_err)); #endif /* SPECULUM_CURSORS */ } @@ -1251,20 +1149,17 @@ int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata, #if SPECULUM_CURSORS if (insertion_result.second) { if (seek_check_err == MDBX_SUCCESS) { - log_error( - "speculum.pre-insert-seek: unexpected %d {%s, %s}", seek_check_err, - mdbx_dump_val(&seek_check_key, dump_key, sizeof(dump_key)), - mdbx_dump_val(&seek_check_data, dump_value, sizeof(dump_value))); + log_error("speculum.pre-insert-seek: unexpected %d {%s, %s}", seek_check_err, + mdbx_dump_val(&seek_check_key, dump_key, sizeof(dump_key)), + mdbx_dump_val(&seek_check_data, dump_value, sizeof(dump_value))); rc = false; } } else { if (seek_check_err != MDBX_SUCCESS) { - log_error( - "speculum.pre-insert-seek: unexpected %d {%s, %s}", seek_check_err, - mdbx_dump_val(&seek_check_key, dump_key, sizeof(dump_key)), - mdbx_dump_val(&seek_check_data, dump_value, sizeof(dump_value))); - speculum_check_iterator("insert", "pre-seek", insertion_result.first, - seek_check_key, seek_check_data, + log_error("speculum.pre-insert-seek: unexpected %d {%s, %s}", seek_check_err, + mdbx_dump_val(&seek_check_key, dump_key, sizeof(dump_key)), + mdbx_dump_val(&seek_check_data, dump_value, sizeof(dump_value))); + speculum_check_iterator("insert", "pre-seek", insertion_result.first, seek_check_key, seek_check_data, check_seek_cursor); rc = false; } @@ -1273,13 +1168,11 @@ int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata, if (insertion_result.first != speculum.begin()) { const auto cursor_prev = speculum_cursors[prev].get(); auto it_prev = insertion_result.first; - speculum_check_cursor("after-insert", "prev", --it_prev, cursor_prev, - MDBX_GET_CURRENT); + speculum_check_cursor("after-insert", "prev", --it_prev, cursor_prev, MDBX_GET_CURRENT); if (it_prev != speculum.begin()) { const auto cursor_prev_prev = speculum_cursors[prev_prev].get(); auto it_prev_prev = it_prev; - speculum_check_cursor("after-insert", "prev-prev", --it_prev_prev, - cursor_prev_prev, MDBX_GET_CURRENT); + speculum_check_cursor("after-insert", "prev-prev", --it_prev_prev, cursor_prev_prev, MDBX_GET_CURRENT); } } @@ -1288,20 +1181,17 @@ int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata, ++it_lowerbound; if (it_lowerbound != speculum.end()) { const auto cursor_lowerbound = speculum_cursors[lowerbound].get(); - speculum_check_cursor("after-insert", "lowerbound", it_lowerbound, - cursor_lowerbound, MDBX_GET_CURRENT); + speculum_check_cursor("after-insert", "lowerbound", it_lowerbound, cursor_lowerbound, MDBX_GET_CURRENT); auto it_next = it_lowerbound; if (++it_next != speculum.end()) { const auto cursor_next = speculum_cursors[next].get(); - speculum_check_cursor("after-insert", "next", it_next, cursor_next, - MDBX_GET_CURRENT); + speculum_check_cursor("after-insert", "next", it_next, cursor_next, MDBX_GET_CURRENT); auto it_next_next = it_next; if (++it_next_next != speculum.end()) { const auto cursor_next_next = speculum_cursors[next_next].get(); - speculum_check_cursor("after-insert", "next-next", it_next_next, - cursor_next_next, MDBX_GET_CURRENT); + speculum_check_cursor("after-insert", "next-next", it_next_next, cursor_next_next, MDBX_GET_CURRENT); } } } @@ -1313,10 +1203,8 @@ int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata, return rc ? MDBX_SUCCESS : MDBX_RESULT_TRUE; } -int testcase::replace(const keygen::buffer &akey, - const keygen::buffer &new_data, - const keygen::buffer &old_data, MDBX_put_flags_t flags, - bool hush_keygen_mistakes) { +int testcase::replace(const keygen::buffer &akey, const keygen::buffer &new_data, const keygen::buffer &old_data, + MDBX_put_flags_t flags, bool hush_keygen_mistakes) { int expected_err = MDBX_SUCCESS; if (config.params.speculum) { const auto S_key = iov2dataview(akey); @@ -1325,22 +1213,19 @@ int testcase::replace(const keygen::buffer &akey, const auto removed = speculum.erase(SET::key_type(S_key, S_old)); if (unlikely(!removed)) { char dump_key[128], dump_value[128]; - log_error( - "speculum-%s: no old pair {%s, %s} (keygen mistake)", "replace", - mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)), - mdbx_dump_val(&old_data->value, dump_value, sizeof(dump_value))); + log_error("speculum-%s: no old pair {%s, %s} (keygen mistake)", "replace", + mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)), + mdbx_dump_val(&old_data->value, dump_value, sizeof(dump_value))); expected_err = MDBX_NOTFOUND; } else if (unlikely(!speculum.emplace(S_key, S_new).second)) { char dump_key[128], dump_value[128]; - log_error( - "speculum-%s: %s {%s, %s}", "replace", "new pair not inserted", - mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)), - mdbx_dump_val(&new_data->value, dump_value, sizeof(dump_value))); + log_error("speculum-%s: %s {%s, %s}", "replace", "new pair not inserted", + mdbx_dump_val(&akey->value, dump_key, sizeof(dump_key)), + mdbx_dump_val(&new_data->value, dump_value, sizeof(dump_value))); expected_err = MDBX_KEYEXIST; } } - int err = mdbx_replace(txn_guard.get(), dbi, &akey->value, &new_data->value, - &old_data->value, flags); + int err = mdbx_replace(txn_guard.get(), dbi, &akey->value, &new_data->value, &old_data->value, flags); if (err && err == expected_err && hush_keygen_mistakes) { log_notice("speculum-%s: %s %d", "replace", "hust keygen mistake", err); err = MDBX_SUCCESS; @@ -1393,13 +1278,11 @@ int testcase::remove(const keygen::buffer &akey, const keygen::buffer &adata) { if (it_found != speculum.begin()) { const auto cursor_prev = speculum_cursors[prev].get(); auto it_prev = it_found; - speculum_check_cursor("after-remove", "prev", --it_prev, cursor_prev, - MDBX_GET_CURRENT); + speculum_check_cursor("after-remove", "prev", --it_prev, cursor_prev, MDBX_GET_CURRENT); if (it_prev != speculum.begin()) { const auto cursor_prev_prev = speculum_cursors[prev_prev].get(); auto it_prev_prev = it_prev; - speculum_check_cursor("after-remove", "prev-prev", --it_prev_prev, - cursor_prev_prev, MDBX_GET_CURRENT); + speculum_check_cursor("after-remove", "prev-prev", --it_prev_prev, cursor_prev_prev, MDBX_GET_CURRENT); } } @@ -1407,27 +1290,20 @@ int testcase::remove(const keygen::buffer &akey, const keygen::buffer &adata) { const auto cursor_next = speculum_cursors[next].get(); const auto cursor_lowerbound = speculum_cursors[lowerbound].get(); if (++it_next != speculum.end()) { - speculum_check_cursor("after-remove", "next", it_next, cursor_next, - MDBX_GET_CURRENT); - speculum_check_cursor("after-remove", "lowerbound", it_next, - cursor_lowerbound, MDBX_NEXT); + speculum_check_cursor("after-remove", "next", it_next, cursor_next, MDBX_GET_CURRENT); + speculum_check_cursor("after-remove", "lowerbound", it_next, cursor_lowerbound, MDBX_NEXT); auto it_next_next = it_next; const auto cursor_next_next = speculum_cursors[next_next].get(); if (++it_next_next != speculum.end()) { - speculum_check_cursor("after-remove", "next-next", it_next_next, - cursor_next_next, MDBX_GET_CURRENT); - } else if ((err = mdbx_cursor_on_last(cursor_next_next)) != - MDBX_RESULT_TRUE) - failure("speculum-%s: %s on-last %d %s", "after-remove", "next-next", - err, mdbx_strerror(err)); + speculum_check_cursor("after-remove", "next-next", it_next_next, cursor_next_next, MDBX_GET_CURRENT); + } else if ((err = mdbx_cursor_on_last(cursor_next_next)) != MDBX_RESULT_TRUE) + failure("speculum-%s: %s on-last %d %s", "after-remove", "next-next", err, mdbx_strerror(err)); } else { if ((err = mdbx_cursor_on_last(cursor_next)) != MDBX_RESULT_TRUE) - failure("speculum-%s: %s on-last %d %s", "after-remove", "next", err, - mdbx_strerror(err)); + failure("speculum-%s: %s on-last %d %s", "after-remove", "next", err, mdbx_strerror(err)); if ((err = mdbx_cursor_on_last(cursor_lowerbound)) != MDBX_RESULT_TRUE) - failure("speculum-%s: %s on-last %d %s", "after-remove", "lowerbound", - err, mdbx_strerror(err)); + failure("speculum-%s: %s on-last %d %s", "after-remove", "lowerbound", err, mdbx_strerror(err)); } #endif /* SPECULUM_CURSORS */ @@ -1477,8 +1353,8 @@ bool testcase::speculum_verify() { } else { eof = mdbx_cursor_eof(cursor); if (eof != MDBX_RESULT_FALSE) { - log_error("false-positive cursor-eof %u/%u: db{%s, %s}, rc %i", n, - extra, mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), + log_error("false-positive cursor-eof %u/%u: db{%s, %s}, rc %i", n, extra, + mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), eof); rc = false; } @@ -1489,38 +1365,31 @@ bool testcase::speculum_verify() { mkey = it->first; mvalue = it->second; } - if (err == MDBX_SUCCESS && it != speculum.cend() && S_key == it->first && - S_data == it->second) { + if (err == MDBX_SUCCESS && it != speculum.cend() && S_key == it->first && S_data == it->second) { ++it; err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT); } else if (err == MDBX_SUCCESS && - (it == speculum.cend() || S_key < it->first || - (S_key == it->first && S_data < it->second))) { + (it == speculum.cend() || S_key < it->first || (S_key == it->first && S_data < it->second))) { extra += 1; if (it != speculum.cend()) { - log_error("extra pair %u/%u: db{%s, %s} < mi{%s, %s}", n, extra, - mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), - mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), - mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), - mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); + log_error( + "extra pair %u/%u: db{%s, %s} < mi{%s, %s}", n, extra, mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), + mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), + mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); } else { - log_error("extra pair %u/%u: db{%s, %s} < mi.END", n, extra, - mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), + log_error("extra pair %u/%u: db{%s, %s} < mi.END", n, extra, mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), mdbx_dump_val(&avalue, dump_value, sizeof(dump_value))); } err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT); rc = false; } else if (it != speculum.cend() && - (err == MDBX_NOTFOUND || S_key > it->first || - (S_key == it->first && S_data > it->second))) { + (err == MDBX_NOTFOUND || S_key > it->first || (S_key == it->first && S_data > it->second))) { lost += 1; if (err == MDBX_NOTFOUND) { - log_error("lost pair %u/%u: db.END > mi{%s, %s}", n, lost, - mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), + log_error("lost pair %u/%u: db.END > mi{%s, %s}", n, lost, mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); } else { - log_error("lost pair %u/%u: db{%s, %s} > mi{%s, %s}", n, lost, - mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), + log_error("lost pair %u/%u: db{%s, %s} > mi{%s, %s}", n, lost, mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); @@ -1566,31 +1435,26 @@ bool testcase::check_batch_get() { bool rc = true; MDBX_val pairs[42]; size_t count = 0xDeadBeef; - batch_err = mdbx_cursor_get_batch(batch_cursor, &count, pairs, - ARRAY_LENGTH(pairs), MDBX_FIRST); + batch_err = mdbx_cursor_get_batch(batch_cursor, &count, pairs, ARRAY_LENGTH(pairs), MDBX_FIRST); size_t i, n = 0; while (batch_err == MDBX_SUCCESS || batch_err == MDBX_RESULT_TRUE) { for (i = 0; i < count; i += 2) { mdbx::slice k, v; - check_err = - mdbx_cursor_get(check_cursor, &k, &v, n ? MDBX_NEXT : MDBX_FIRST); + check_err = mdbx_cursor_get(check_cursor, &k, &v, n ? MDBX_NEXT : MDBX_FIRST); if (check_err != MDBX_SUCCESS) failure_perror("batch-verify: mdbx_cursor_get(MDBX_NEXT)", check_err); if (k != pairs[i] || v != pairs[i + 1]) { - log_error( - "batch-get pair mismatch %zu/%zu: sequential{%s, %s} != " - "batch{%s, %s}", - n + i / 2, i, mdbx_dump_val(&k, dump_key, sizeof(dump_key)), - mdbx_dump_val(&v, dump_value, sizeof(dump_value)), - mdbx_dump_val(&pairs[i], dump_key_batch, sizeof(dump_key_batch)), - mdbx_dump_val(&pairs[i + 1], dump_value_batch, - sizeof(dump_value_batch))); + log_error("batch-get pair mismatch %zu/%zu: sequential{%s, %s} != " + "batch{%s, %s}", + n + i / 2, i, mdbx_dump_val(&k, dump_key, sizeof(dump_key)), + mdbx_dump_val(&v, dump_value, sizeof(dump_value)), + mdbx_dump_val(&pairs[i], dump_key_batch, sizeof(dump_key_batch)), + mdbx_dump_val(&pairs[i + 1], dump_value_batch, sizeof(dump_value_batch))); rc = false; } ++n; } - batch_err = mdbx_cursor_get_batch(batch_cursor, &count, pairs, - ARRAY_LENGTH(pairs), MDBX_NEXT); + batch_err = mdbx_cursor_get_batch(batch_cursor, &count, pairs, ARRAY_LENGTH(pairs), MDBX_NEXT); } if (batch_err != MDBX_NOTFOUND) { log_error("mdbx_cursor_get_batch(), err %d", batch_err); @@ -1619,10 +1483,8 @@ bool testcase::check_batch_get() { } bool testcase::txn_probe_parking() { - MDBX_txn_flags_t state = - mdbx_txn_flags(txn_guard.get()) & - (MDBX_TXN_RDONLY | MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK | - MDBX_TXN_OUSTED | MDBX_TXN_BLOCKED); + MDBX_txn_flags_t state = mdbx_txn_flags(txn_guard.get()) & (MDBX_TXN_RDONLY | MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK | + MDBX_TXN_OUSTED | MDBX_TXN_BLOCKED); if (state != MDBX_TXN_RDONLY) return true; @@ -1635,42 +1497,35 @@ bool testcase::txn_probe_parking() { if (flipcoin()) { err = mdbx_txn_info(txn_guard.get(), &txn_info, flipcoin()); if (err != MDBX_SUCCESS) - failure("mdbx_txn_info(1), state 0x%x, err %d", - state = mdbx_txn_flags(txn_guard.get()), err); + failure("mdbx_txn_info(1), state 0x%x, err %d", state = mdbx_txn_flags(txn_guard.get()), err); } if (osal_multiactor_mode() && !mode_readonly()) { - while (flipcoin() && - ((state = mdbx_txn_flags(txn_guard.get())) & MDBX_TXN_OUSTED) == 0) + while (flipcoin() && ((state = mdbx_txn_flags(txn_guard.get())) & MDBX_TXN_OUSTED) == 0) osal_udelay(4242); } if (flipcoin()) { err = mdbx_txn_info(txn_guard.get(), &txn_info, flipcoin()); if (err != MDBX_SUCCESS) - failure("mdbx_txn_info(2), state 0x%x, err %d", - state = mdbx_txn_flags(txn_guard.get()), err); + failure("mdbx_txn_info(2), state 0x%x, err %d", state = mdbx_txn_flags(txn_guard.get()), err); } if (flipcoin()) { MDBX_envinfo env_info; - err = mdbx_env_info_ex(db_guard.get(), txn_guard.get(), &env_info, - sizeof(env_info)); + err = mdbx_env_info_ex(db_guard.get(), txn_guard.get(), &env_info, sizeof(env_info)); if (!autounpark) { if (err != MDBX_BAD_TXN) failure("mdbx_env_info_ex(autounpark=%s), flags 0x%x, unexpected err " "%d, must %d", autounpark ? "true" : "false", state, err, MDBX_BAD_TXN); } else if (err != MDBX_SUCCESS) { - if (err != MDBX_OUSTED || - ((state = mdbx_txn_flags(txn_guard.get())) & MDBX_TXN_OUSTED) == 0) - failure("mdbx_env_info_ex(autounpark=%s), flags 0x%x, err %d", - autounpark ? "true" : "false", state, err); + if (err != MDBX_OUSTED || ((state = mdbx_txn_flags(txn_guard.get())) & MDBX_TXN_OUSTED) == 0) + failure("mdbx_env_info_ex(autounpark=%s), flags 0x%x, err %d", autounpark ? "true" : "false", state, err); else { err = mdbx_txn_renew(txn_guard.get()); if (err != MDBX_SUCCESS) - failure("mdbx_txn_renew(), state 0x%x, err %d", - state = mdbx_txn_flags(txn_guard.get()), err); + failure("mdbx_txn_renew(), state 0x%x, err %d", state = mdbx_txn_flags(txn_guard.get()), err); } } } @@ -1679,20 +1534,17 @@ bool testcase::txn_probe_parking() { err = mdbx_txn_unpark(txn_guard.get(), autorestart); if (MDBX_IS_ERROR(err)) { if (err != MDBX_OUSTED || autorestart) - failure("mdbx_txn_unpark(autounpark=%s, autorestart=%s), err %d", - autounpark ? "true" : "false", autorestart ? "true" : "false", - err); + failure("mdbx_txn_unpark(autounpark=%s, autorestart=%s), err %d", autounpark ? "true" : "false", + autorestart ? "true" : "false", err); else { err = mdbx_txn_renew(txn_guard.get()); if (err != MDBX_SUCCESS) - failure("mdbx_txn_renew(), state 0x%x, err %d", - state = mdbx_txn_flags(txn_guard.get()), err); + failure("mdbx_txn_renew(), state 0x%x, err %d", state = mdbx_txn_flags(txn_guard.get()), err); } } state = mdbx_txn_flags(txn_guard.get()) & - (MDBX_TXN_RDONLY | MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK | - MDBX_TXN_OUSTED | MDBX_TXN_BLOCKED); + (MDBX_TXN_RDONLY | MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK | MDBX_TXN_OUSTED | MDBX_TXN_BLOCKED); if (state != MDBX_TXN_RDONLY) failure("unexpected txn-state 0x%x", state); return state == MDBX_TXN_RDONLY; diff --git a/test/test.h++ b/test/test.h++ index 39ce1118..861429ff 100644 --- a/test/test.h++ +++ b/test/test.h++ @@ -31,10 +31,9 @@ bool test_execute(const actor_config &config); std::string thunk_param(const actor_config &config); -void testcase_setup(const char *casename, const actor_params ¶ms, - unsigned &last_space_id); -void configure_actor(unsigned &last_space_id, const actor_testcase testcase, - const char *space_id_cstr, actor_params params); +void testcase_setup(const char *casename, const actor_params ¶ms, unsigned &last_space_id); +void configure_actor(unsigned &last_space_id, const actor_testcase testcase, const char *space_id_cstr, + actor_params params); void keycase_setup(const char *casename, actor_params ¶ms); namespace global { @@ -106,34 +105,26 @@ public: this->id = id; this->name = name; review_params = TESTCASE::review_params; - constructor = [](const actor_config &config, - const mdbx_pid_t pid) -> testcase * { + constructor = [](const actor_config &config, const mdbx_pid_t pid) -> testcase * { return new TESTCASE(config, pid); }; add(this); } }; - static bool review_actor_params(const actor_testcase id, actor_params ¶ms, - const unsigned space_id); - static testcase *create_actor(const actor_config &config, - const mdbx_pid_t pid); + static bool review_actor_params(const actor_testcase id, actor_params ¶ms, const unsigned space_id); + static testcase *create_actor(const actor_config &config, const mdbx_pid_t pid); }; -#define REGISTER_TESTCASE(NAME) \ - static registry::factory gRegister_##NAME( \ - ac_##NAME, MDBX_STRINGIFY(NAME)) +#define REGISTER_TESTCASE(NAME) \ + static registry::factory gRegister_##NAME(ac_##NAME, MDBX_STRINGIFY(NAME)) class testcase { protected: using data_view = mdbx::slice; static inline data_view iov2dataview(const MDBX_val &v) { - return (v.iov_base && v.iov_len) - ? data_view(static_cast(v.iov_base), v.iov_len) - : data_view(); - } - static inline data_view iov2dataview(const keygen::buffer &b) { - return iov2dataview(b->value); + return (v.iov_base && v.iov_len) ? data_view(static_cast(v.iov_base), v.iov_len) : data_view(); } + static inline data_view iov2dataview(const keygen::buffer &b) { return iov2dataview(b->value); } using Item = std::pair<::mdbx::buffer<>, ::mdbx::buffer<>>; @@ -145,16 +136,13 @@ protected: } struct ItemCompare { const testcase *context; - ItemCompare(const testcase *owner) : context(owner) { - /* The context->txn_guard may be empty/null here */ - } + ItemCompare(const testcase *owner) : context(owner) { /* The context->txn_guard may be empty/null here */ } bool operator()(const Item &a, const Item &b) const { MDBX_val va = dataview2iov(a.first), vb = dataview2iov(b.first); assert(context->txn_guard.get() != nullptr); int cmp = mdbx_cmp(context->txn_guard.get(), context->dbi, &va, &vb); - if (cmp == 0 && - (context->config.params.table_flags & MDBX_DUPSORT) != 0) { + if (cmp == 0 && (context->config.params.table_flags & MDBX_DUPSORT) != 0) { va = dataview2iov(a.second); vb = dataview2iov(b.second); cmp = mdbx_dcmp(context->txn_guard.get(), context->dbi, &va, &vb); @@ -194,61 +182,44 @@ protected: #if SPECULUM_CURSORS scoped_cursor_guard speculum_cursors[5 + 1]; void speculum_prepare_cursors(const Item &item); - bool speculum_check_cursor(const char *where, const char *stage, - const testcase::SET::const_iterator &it, - int cursor_err, const MDBX_val &cursor_key, - const MDBX_val &cursor_data, + bool speculum_check_cursor(const char *where, const char *stage, const testcase::SET::const_iterator &it, + int cursor_err, const MDBX_val &cursor_key, const MDBX_val &cursor_data, MDBX_cursor *cursor) const; - bool speculum_check_cursor(const char *where, const char *stage, - const testcase::SET::const_iterator &it, - MDBX_cursor *cursor, - const MDBX_cursor_op op) const; - void speculum_render(const testcase::SET::const_iterator &it, - const MDBX_cursor *ref) const; + bool speculum_check_cursor(const char *where, const char *stage, const testcase::SET::const_iterator &it, + MDBX_cursor *cursor, const MDBX_cursor_op op) const; + void speculum_render(const testcase::SET::const_iterator &it, const MDBX_cursor *ref) const; #endif /* SPECULUM_CURSORS */ - bool speculum_check_iterator(const char *where, const char *stage, - const testcase::SET::const_iterator &it, - const MDBX_val &k, const MDBX_val &v, - MDBX_cursor *cursor) const; + bool speculum_check_iterator(const char *where, const char *stage, const testcase::SET::const_iterator &it, + const MDBX_val &k, const MDBX_val &v, MDBX_cursor *cursor) const; - void verbose(const char *where, const char *stage, - const testcase::SET::const_iterator &it) const; - void verbose(const char *where, const char *stage, const MDBX_val &k, - const MDBX_val &v, int err = MDBX_SUCCESS) const; + void verbose(const char *where, const char *stage, const testcase::SET::const_iterator &it) const; + void verbose(const char *where, const char *stage, const MDBX_val &k, const MDBX_val &v, + int err = MDBX_SUCCESS) const; bool is_same(const Item &a, const Item &b) const; - bool is_same(const SET::const_iterator &it, const MDBX_val &k, - const MDBX_val &v) const; + bool is_same(const SET::const_iterator &it, const MDBX_val &k, const MDBX_val &v) const; bool speculum_verify(); bool check_batch_get(); - int insert(const keygen::buffer &akey, const keygen::buffer &adata, - MDBX_put_flags_t flags); - int replace(const keygen::buffer &akey, const keygen::buffer &new_value, - const keygen::buffer &old_value, MDBX_put_flags_t flags, - bool hush_keygen_mistakes = true); + int insert(const keygen::buffer &akey, const keygen::buffer &adata, MDBX_put_flags_t flags); + int replace(const keygen::buffer &akey, const keygen::buffer &new_value, const keygen::buffer &old_value, + MDBX_put_flags_t flags, bool hush_keygen_mistakes = true); int remove(const keygen::buffer &akey, const keygen::buffer &adata); - static int hsr_callback(const MDBX_env *env, const MDBX_txn *txn, - mdbx_pid_t pid, mdbx_tid_t tid, uint64_t laggard, - unsigned gap, size_t space, - int retry) MDBX_CXX17_NOEXCEPT; + static int hsr_callback(const MDBX_env *env, const MDBX_txn *txn, mdbx_pid_t pid, mdbx_tid_t tid, uint64_t laggard, + unsigned gap, size_t space, int retry) MDBX_CXX17_NOEXCEPT; MDBX_env_flags_t actual_env_mode{MDBX_ENV_DEFAULTS}; - bool is_nested_txn_available() const { - return (actual_env_mode & MDBX_WRITEMAP) == 0; - } + bool is_nested_txn_available() const { return (actual_env_mode & MDBX_WRITEMAP) == 0; } void kick_progress(bool active) const; void db_prepare(); void db_open(); void db_close(); - virtual void txn_begin(bool readonly, - MDBX_txn_flags_t flags = MDBX_TXN_READWRITE); + virtual void txn_begin(bool readonly, MDBX_txn_flags_t flags = MDBX_TXN_READWRITE); int breakable_commit(); virtual void txn_end(bool abort); int breakable_restart(); - void txn_restart(bool abort, bool readonly, - MDBX_txn_flags_t flags = MDBX_TXN_READWRITE); + void txn_restart(bool abort, bool readonly, MDBX_txn_flags_t flags = MDBX_TXN_READWRITE); void cursor_open(MDBX_dbi handle); void cursor_close(); void cursor_renew(); @@ -258,13 +229,11 @@ protected: void fetch_canary(); void update_canary(uint64_t increment); - bool checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check, - MDBX_val expected_valued); + bool checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check, MDBX_val expected_valued); unsigned txn_underutilization_x256(MDBX_txn *txn) const; using tablename_buf = char[32]; - const char *db_tablename(tablename_buf &buffer, - const char *suffix = "") const; + const char *db_tablename(tablename_buf &buffer, const char *suffix = "") const; MDBX_dbi db_table_open(bool create, bool expect_failure = false); void db_table_drop(MDBX_dbi handle); void db_table_clear(MDBX_dbi handle, MDBX_txn *txn = nullptr); @@ -278,22 +247,17 @@ protected: bool should_continue(bool check_timeout_only = false) const; bool MDBX_PRINTF_ARGS(2, 3) failure(const char *fmt, ...) const; - void generate_pair(const keygen::serial_t serial, keygen::buffer &out_key, - keygen::buffer &out_value, keygen::serial_t data_age) { + void generate_pair(const keygen::serial_t serial, keygen::buffer &out_key, keygen::buffer &out_value, + keygen::serial_t data_age) { keyvalue_maker.pair(serial, out_key, out_value, data_age, false); } - void generate_pair(const keygen::serial_t serial) { - keyvalue_maker.pair(serial, key, data, 0, true); - } + void generate_pair(const keygen::serial_t serial) { keyvalue_maker.pair(serial, key, data, 0, true); } - bool mode_readonly() const { - return (config.params.mode_flags & MDBX_RDONLY) ? true : false; - } + bool mode_readonly() const { return (config.params.mode_flags & MDBX_RDONLY) ? true : false; } public: - testcase(const actor_config &config, const mdbx_pid_t pid) - : config(config), pid(pid) { + testcase(const actor_config &config, const mdbx_pid_t pid) : config(config), pid(pid) { start_timestamp.reset(); memset(&last, 0, sizeof(last)); } @@ -301,11 +265,9 @@ public: static bool review_params(actor_params ¶ms, unsigned space_id) { // silently fix key/data length for fixed-length modes params.prng_seed += bleach32(space_id); - if ((params.table_flags & MDBX_INTEGERKEY) && - params.keylen_min != params.keylen_max) + if ((params.table_flags & MDBX_INTEGERKEY) && params.keylen_min != params.keylen_max) params.keylen_min = params.keylen_max; - if ((params.table_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) && - params.datalen_min != params.datalen_max) + if ((params.table_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) && params.datalen_min != params.datalen_max) params.datalen_min = params.datalen_max; return true; } @@ -330,8 +292,7 @@ protected: unsigned edge2count(uint64_t edge); public: - testcase_ttl(const actor_config &config, const mdbx_pid_t pid) - : inherited(config, pid) {} + testcase_ttl(const actor_config &config, const mdbx_pid_t pid) : inherited(config, pid) {} bool setup() override; bool run() override; }; diff --git a/test/try.c++ b/test/try.c++ index 708122ac..4333a996 100644 --- a/test/try.c++ +++ b/test/try.c++ @@ -5,8 +5,7 @@ class testcase_try : public testcase { public: - testcase_try(const actor_config &config, const mdbx_pid_t pid) - : testcase(config, pid) {} + testcase_try(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {} bool run() override; }; REGISTER_TESTCASE(try); diff --git a/test/ttl.c++ b/test/ttl.c++ index d22be384..a4482e1f 100644 --- a/test/ttl.c++ +++ b/test/ttl.c++ @@ -24,17 +24,14 @@ REGISTER_TESTCASE(ttl); unsigned testcase_ttl::edge2count(uint64_t edge) { const double rnd = u64_to_double1(prng64_map1_white(edge)); - const unsigned count = - unsigned(std::lrint(std::pow(sliding.max_step_size, rnd))); + const unsigned count = unsigned(std::lrint(std::pow(sliding.max_step_size, rnd))); // average value: (X - 1) / ln(X), where X = sliding.max_step_size return count; } unsigned testcase_ttl::edge2window(uint64_t edge) { const double rnd = u64_to_double1(bleach64(edge)); - const unsigned size = - sliding.max_window_size - - unsigned(std::lrint(std::pow(sliding.max_window_size, rnd))); + const unsigned size = sliding.max_window_size - unsigned(std::lrint(std::pow(sliding.max_window_size, rnd))); // average value: Y - (Y - 1) / ln(Y), where Y = sliding.max_window_size return size; } @@ -48,20 +45,17 @@ static inline double estimate(const double x, const double y) { } bool testcase_ttl::setup() { - const unsigned window_top_lower = - 7 /* нижний предел для верхней границы диапазона, в котором будет - стохастически колебаться размер окна */ + const unsigned window_top_lower = 7 /* нижний предел для верхней границы диапазона, в котором будет + стохастически колебаться размер окна */ ; - const unsigned count_top_lower = - 7 /* нижний предел для верхней границы диапазона, в котором будет - стохастически колебаться кол-во записей добавляемых на одном шаге */ + const unsigned count_top_lower = 7 /* нижний предел для верхней границы диапазона, в котором будет + стохастически колебаться кол-во записей добавляемых на одном шаге */ ; /* для параметризации используем подходящие параметры, * которые не имеют здесь смысла в первоначальном значении. */ - const double ratio = - double(config.params.batch_read ? config.params.batch_read : 1) / - double(config.params.batch_write ? config.params.batch_write : 1); + const double ratio = double(config.params.batch_read ? config.params.batch_read : 1) / + double(config.params.batch_write ? config.params.batch_write : 1); /* проще найти двоичным поиском (вариация метода Ньютона) */ double hi = config.params.test_nops, lo = 1; @@ -82,8 +76,7 @@ bool testcase_ttl::setup() { if (sliding.max_window_size < window_top_lower) sliding.max_window_size = window_top_lower; - while (estimate(sliding.max_step_size, sliding.max_window_size) > - config.params.test_nops * 2.0) { + while (estimate(sliding.max_step_size, sliding.max_window_size) > config.params.test_nops * 2.0) { if (ratio * sliding.max_step_size > sliding.max_window_size) { if (sliding.max_step_size < count_top_lower) break; @@ -95,8 +88,7 @@ bool testcase_ttl::setup() { } } - log_verbose("come up window_max %u from `batch_read`", - sliding.max_window_size); + log_verbose("come up window_max %u from `batch_read`", sliding.max_window_size); log_verbose("come up step_max %u from `batch_write`", sliding.max_step_size); return inherited::setup(); } @@ -113,9 +105,7 @@ bool testcase_ttl::run() { key = keygen::alloc(config.params.keylen_max); data = keygen::alloc(config.params.datalen_max); const MDBX_put_flags_t insert_flags = - (config.params.table_flags & MDBX_DUPSORT) - ? MDBX_NODUPDATA - : MDBX_NODUPDATA | MDBX_NOOVERWRITE; + (config.params.table_flags & MDBX_DUPSORT) ? MDBX_NODUPDATA : MDBX_NODUPDATA | MDBX_NOOVERWRITE; std::deque> fifo; uint64_t serial = 0; @@ -128,20 +118,17 @@ bool testcase_ttl::run() { while (true) { const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */; - const unsigned window_width = - (!should_continue() || flipcoin_x4()) ? 0 : edge2window(salt); + const unsigned window_width = (!should_continue() || flipcoin_x4()) ? 0 : edge2window(salt); unsigned head_count = edge2count(salt); - log_debug("ttl: step #%" PRIu64 " (serial %" PRIu64 - ", window %u, count %u) salt %" PRIu64, - nops_completed, serial, window_width, head_count, salt); + log_debug("ttl: step #%" PRIu64 " (serial %" PRIu64 ", window %u, count %u) salt %" PRIu64, nops_completed, serial, + window_width, head_count, salt); if (window_width || flipcoin()) { clear_stepbystep_passed += window_width == 0; while (fifo.size() > window_width) { uint64_t tail_serial = fifo.back().first; const unsigned tail_count = fifo.back().second; - log_trace("ttl: pop-tail (serial %" PRIu64 ", count %u)", tail_serial, - tail_count); + log_trace("ttl: pop-tail (serial %" PRIu64 ", count %u)", tail_serial, tail_count); fifo.pop_back(); for (unsigned n = 0; n < tail_count; ++n) { log_trace("ttl: remove-tail %" PRIu64, tail_serial); @@ -177,10 +164,8 @@ bool testcase_ttl::run() { return false; } - if (!keyspace_overflow && (should_continue() || !clear_wholetable_passed || - !clear_stepbystep_passed)) { - unsigned underutilization_x256 = - txn_underutilization_x256(txn_guard.get()); + if (!keyspace_overflow && (should_continue() || !clear_wholetable_passed || !clear_stepbystep_passed)) { + unsigned underutilization_x256 = txn_underutilization_x256(txn_guard.get()); if (dbfull_passed > underutilization_x256) { log_notice("ttl: skip head-grow to avoid one more dbfull (was %u, " "underutilization %.2f%%)", @@ -194,8 +179,7 @@ bool testcase_ttl::run() { generate_pair(serial); err = insert(key, data, insert_flags); if (unlikely(err != MDBX_SUCCESS)) { - if ((err == MDBX_TXN_FULL || err == MDBX_MAP_FULL) && - config.params.ignore_dbfull) { + if ((err == MDBX_TXN_FULL || err == MDBX_MAP_FULL) && config.params.ignore_dbfull) { log_notice("ttl: head-insert skip due '%s'", mdbx_strerror(err)); txn_restart(true, false); serial = fifo.front().first; @@ -227,8 +211,7 @@ bool testcase_ttl::run() { } loops += 1; } else if (fifo.empty()) { - log_notice("ttl: done %u whole loops, %" PRIu64 " ops, %" PRIu64 " items", - loops, nops_completed, serial); + log_notice("ttl: done %u whole loops, %" PRIu64 " ops, %" PRIu64 " items", loops, nops_completed, serial); rc = true; break; } else { diff --git a/test/utils.c++ b/test/utils.c++ index f96f4ffd..442e4a21 100644 --- a/test/utils.c++ +++ b/test/utils.c++ @@ -24,8 +24,7 @@ std::string format(const char *fmt, ...) { std::string result; result.reserve((size_t)needed + 1); result.resize((size_t)needed, '\0'); - MDBX_MAYBE_UNUSED int actual = - vsnprintf((char *)result.data(), result.capacity(), fmt, ones); + MDBX_MAYBE_UNUSED int actual = vsnprintf((char *)result.data(), result.capacity(), fmt, ones); assert(actual == needed); (void)actual; va_end(ones); @@ -50,8 +49,7 @@ std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum) { return result; } -bool hex2data(const char *hex_begin, const char *hex_end, void *ptr, - size_t bytes, simple_checksum &checksum) { +bool hex2data(const char *hex_begin, const char *hex_end, void *ptr, size_t bytes, simple_checksum &checksum) { if (bytes * 2 != (size_t)(hex_end - hex_begin)) return false; @@ -85,8 +83,7 @@ bool hex2data(const char *hex_begin, const char *hex_end, void *ptr, } bool is_samedata(const MDBX_val *a, const MDBX_val *b) { - return a->iov_len == b->iov_len && - memcmp(a->iov_base, b->iov_base, a->iov_len) == 0; + return a->iov_len == b->iov_len && memcmp(a->iov_base, b->iov_base, a->iov_len) == 0; } //----------------------------------------------------------------------------- @@ -96,13 +93,9 @@ uint64_t prng64_white(uint64_t &state) { return bleach64(state); } -uint32_t prng32_fast(uint64_t &state) { - return uint32_t(prng64_careless(state) >> 32); -} +uint32_t prng32_fast(uint64_t &state) { return uint32_t(prng64_careless(state) >> 32); } -uint32_t prng32_white(uint64_t &state) { - return bleach32(uint32_t(prng64_careless(state) >> 32)); -} +uint32_t prng32_white(uint64_t &state) { return bleach32(uint32_t(prng64_careless(state) >> 32)); } void prng_fill(uint64_t &state, void *ptr, size_t bytes) { uint32_t u32 = prng32_fast(state); @@ -174,9 +167,7 @@ bool flipcoin() { return prng32() & 1; } bool flipcoin_x2() { return (prng32() & 3) == 0; } bool flipcoin_x3() { return (prng32() & 7) == 0; } bool flipcoin_x4() { return (prng32() & 15) == 0; } -bool flipcoin_n(unsigned n) { - return (prng64() & ((UINT64_C(1) << n) - 1)) == 0; -} +bool flipcoin_n(unsigned n) { return (prng64() & ((UINT64_C(1) << n) - 1)) == 0; } bool jitter(unsigned probability_percent) { const uint32_t top = UINT32_MAX - UINT32_MAX % 100; @@ -201,8 +192,7 @@ void jitter_delay(bool extra) { osal_yield(); cpu_relax(); if (dice > 2) { - size_t us = - prng32() & (extra ? 0xffff /* 656 ms */ : 0x3ff /* 1 ms */); + size_t us = prng32() & (extra ? 0xffff /* 656 ms */ : 0x3ff /* 1 ms */); log_trace("== jitter.delay: %0.6f", us / 1000000.0); osal_udelay(us); } diff --git a/test/utils.h++ b/test/utils.h++ index 0dd7c4d8..caa34049 100644 --- a/test/utils.h++ +++ b/test/utils.h++ @@ -4,13 +4,11 @@ #pragma once #include "base.h++" -#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ - !defined(__ORDER_BIG_ENDIAN__) +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || !defined(__ORDER_BIG_ENDIAN__) #error __BYTE_ORDER__ should be defined. #endif -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \ - __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ #error Unsupported byte order. #endif @@ -21,16 +19,14 @@ #ifndef bswap32 #define bswap32(v) __builtin_bswap32(v) #endif -#if (__GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16)) && \ - !defined(bswap16) +#if (__GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16)) && !defined(bswap16) #define bswap16(v) __builtin_bswap16(v) #endif #elif defined(_MSC_VER) #if _MSC_FULL_VER < 190024215 -#pragma message( \ - "It is recommended to use Visual Studio 2015 (MSC 19.0) or newer.") +#pragma message("It is recommended to use Visual Studio 2015 (MSC 19.0) or newer.") #endif #define bswap64(v) _byteswap_uint64(v) @@ -60,12 +56,9 @@ #define bswap64(v) __bswap_64(v) #else static inline uint64_t bswap64(uint64_t v) { - return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | - ((v << 24) & UINT64_C(0x0000ff0000000000)) | - ((v << 8) & UINT64_C(0x000000ff00000000)) | - ((v >> 8) & UINT64_C(0x00000000ff0000000)) | - ((v >> 24) & UINT64_C(0x0000000000ff0000)) | - ((v >> 40) & UINT64_C(0x000000000000ff00)); + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | ((v << 24) & UINT64_C(0x0000ff0000000000)) | + ((v << 8) & UINT64_C(0x000000ff00000000)) | ((v >> 8) & UINT64_C(0x00000000ff0000000)) | + ((v >> 24) & UINT64_C(0x0000000000ff0000)) | ((v >> 40) & UINT64_C(0x000000000000ff00)); } #endif #endif /* bswap64 */ @@ -75,8 +68,7 @@ static inline uint64_t bswap64(uint64_t v) { #define bswap32(v) __bswap_32(v) #else static inline uint32_t bswap32(uint32_t v) { - return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | - ((v >> 8) & UINT32_C(0x0000ff00)); + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | ((v >> 8) & UINT32_C(0x0000ff00)); } #endif #endif /* bswap32 */ @@ -140,8 +132,7 @@ template static inline T load(const void *ptr) { if (MDBX_UNALIGNED_OK >= sizeof(T)) return *(const T *)ptr; else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) return *(const T __unaligned *)ptr; #else T local; @@ -155,8 +146,7 @@ template static inline void store(void *ptr, const T &value) { if (MDBX_UNALIGNED_OK >= sizeof(T)) *(T *)ptr = value; else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) *((T __unaligned *)ptr) = value; #else memcpy(ptr, &value, sizeof(T)); @@ -169,9 +159,7 @@ template static inline void store(void *ptr, const T &value) { //----------------------------------------------------------------------------- #ifndef rot64 -static inline uint64_t rot64(uint64_t v, unsigned s) { - return (v >> s) | (v << (64 - s)); -} +static inline uint64_t rot64(uint64_t v, unsigned s) { return (v >> s) | (v << (64 - s)); } #endif /* rot64 */ static inline bool is_power2(size_t x) { return (x & (x - 1)) == 0; } @@ -203,11 +191,9 @@ static inline void memory_barrier(void) { #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __machine_rw_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_mf(); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __lwsync(); #else #error "Could not guess the kind of compiler, please report to us." @@ -217,8 +203,7 @@ static inline void memory_barrier(void) { static inline void cpu_relax() { #if defined(__ia32__) _mm_pause(); -#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) || \ - defined(YieldProcessor) +#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) || defined(YieldProcessor) YieldProcessor(); #else /* nope */ @@ -243,9 +228,7 @@ struct simple_checksum { push((uint32_t)(data >> 32)); } - void push(const bool data) { - push(data ? UINT32_C(0x780E) : UINT32_C(0xFA18E)); - } + void push(const bool data) { push(data ? UINT32_C(0x780E) : UINT32_C(0xFA18E)); } void push(const void *ptr, size_t bytes) { const uint8_t *data = (const uint8_t *)ptr; @@ -269,12 +252,9 @@ struct simple_checksum { }; std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum); -bool hex2data(const char *hex_begin, const char *hex_end, void *ptr, - size_t bytes, simple_checksum &checksum); +bool hex2data(const char *hex_begin, const char *hex_end, void *ptr, size_t bytes, simple_checksum &checksum); bool is_samedata(const MDBX_val *a, const MDBX_val *b); -inline bool is_samedata(const MDBX_val &a, const MDBX_val &b) { - return is_samedata(&a, &b); -} +inline bool is_samedata(const MDBX_val &a, const MDBX_val &b) { return is_samedata(&a, &b); } std::string format(const char *fmt, ...); static inline uint64_t bleach64(uint64_t x) { @@ -300,22 +280,15 @@ static inline uint32_t bleach32(uint32_t x) { return x; } -static inline uint64_t prng64_map1_careless(uint64_t state) { - return state * UINT64_C(6364136223846793005) + 1; -} +static inline uint64_t prng64_map1_careless(uint64_t state) { return state * UINT64_C(6364136223846793005) + 1; } static inline uint64_t prng64_map2_careless(uint64_t state) { - return (state + UINT64_C(1442695040888963407)) * - UINT64_C(6364136223846793005); + return (state + UINT64_C(1442695040888963407)) * UINT64_C(6364136223846793005); } -static inline uint64_t prng64_map1_white(uint64_t state) { - return bleach64(prng64_map1_careless(state)); -} +static inline uint64_t prng64_map1_white(uint64_t state) { return bleach64(prng64_map1_careless(state)); } -static inline uint64_t prng64_map2_white(uint64_t state) { - return bleach64(prng64_map2_careless(state)); -} +static inline uint64_t prng64_map2_white(uint64_t state) { return bleach64(prng64_map2_careless(state)); } static inline uint64_t prng64_careless(uint64_t &state) { state = prng64_map1_careless(state);