From f7f9eaff95dee49a6cffef522ac8f14c7151442d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9B=D0=B5=D0=BE=D0=BD=D0=B8=D0=B4=20=D0=AE=D1=80=D1=8C?= =?UTF-8?q?=D0=B5=D0=B2=20=28Leonid=20Yuriev=29?= Date: Sat, 9 Jul 2022 00:29:17 +0300 Subject: [PATCH 1/5] mdbx-test: update Valgrind-suppressions for modern GCC. --- test/valgrind_suppress.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/valgrind_suppress.txt b/test/valgrind_suppress.txt index fbaf2bc5..1977becc 100644 --- a/test/valgrind_suppress.txt +++ b/test/valgrind_suppress.txt @@ -4,7 +4,7 @@ msync(start) fun:msync ... - fun:mdbx_sync_locked + fun:mdbx_sync_locked* } { msync-whole-mmap-2 @@ -12,7 +12,7 @@ msync(start) fun:msync ... - fun:mdbx_env_sync_internal + fun:mdbx_env_sync_internal* } { msync-whole-mmap-3 @@ -20,7 +20,7 @@ msync(start) fun:msync ... - fun:mdbx_mapresize + fun:mdbx_mapresize* } { msync-wipe-steady @@ -28,7 +28,7 @@ msync(start) fun:msync ... - fun:mdbx_wipe_steady + fun:mdbx_wipe_steady* } # memcmp() inside mdbx_iov_write() as workaround for todo4recovery://erased_by_github/libmdbx/issues/269 From e9a1042cc242fbc6c6bbac6f19b0fe1e35b448e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9B=D0=B5=D0=BE=D0=BD=D0=B8=D0=B4=20=D0=AE=D1=80=D1=8C?= =?UTF-8?q?=D0=B5=D0=B2=20=28Leonid=20Yuriev=29?= Date: Sat, 9 Jul 2022 10:31:47 +0300 Subject: [PATCH 2/5] mdbx-test: add `--pagesize NN` option to long-stotastic script. --- test/long_stochastic.sh | 63 ++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/test/long_stochastic.sh b/test/long_stochastic.sh index 04b9976e..cf961992 100755 --- a/test/long_stochastic.sh +++ b/test/long_stochastic.sh @@ -14,6 +14,7 @@ GEOMETRY_JITTER=yes BANNER="$(which banner 2>/dev/null | echo echo)" UNAME="$(uname -s 2>/dev/null || echo Unknown)" DB_UPTO_MB=17408 +PAGESIZE=min while [ -n "$1" ] do @@ -33,6 +34,7 @@ do echo "--dir PATH Specifies directory for test DB and other files (it will be cleared)" echo "--db-upto-mb NN Limits upper size of test DB to the NN megabytes" echo "--no-geometry-jitter Disable jitter for geometry upper-size" + echo "--pagesize NN Use specified page size (256 is minimal and used by default) " echo "--help Print this usage help and exit" exit -2 ;; @@ -109,6 +111,39 @@ do --no-geometry-jitter) GEOMETRY_JITTER=no ;; + --pagesize) + case "$2" in + min|max|256|512|1024|2048|4096|8192|16386|32768|65536) + PAGESIZE=$2 + ;; + 1|1k|1K|k|K) + PAGESIZE=$((1024*1)) + ;; + 2|2k|2K) + PAGESIZE=$((1024*2)) + ;; + 4|4k|4K) + PAGESIZE=$((1024*4)) + ;; + 8|8k|8K) + PAGESIZE=$((1024*8)) + ;; + 16|16k|16K) + PAGESIZE=$((1024*16)) + ;; + 32|32k|32K) + PAGESIZE=$((1024*32)) + ;; + 64|64k|64K) + PAGESIZE=$((1024*64)) + ;; + *) + echo "Invalig page size '$2'" + exit -2 + ;; + esac + shift + ;; *) echo "Unknown option '$1'" exit -2 @@ -346,65 +381,65 @@ for nops in 10 33 100 333 1000 3333 10000 33333 100000 333333 1000000 3333333 10 split=30 caption="Probe #$((++count)) int-key,with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --pagesize=$PAGESIZE --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \ --keygen.seed=${seed} caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --pagesize=$PAGESIZE --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \ --keygen.seed=${seed} caption="Probe #$((++count)) with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper-upto=${db_size_mb}M --table=+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --pagesize=$PAGESIZE --size-upper-upto=${db_size_mb}M --table=+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \ --keygen.seed=${seed} split=24 caption="Probe #$((++count)) int-key,with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --pagesize=$PAGESIZE --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \ --keygen.seed=${seed} caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --pagesize=$PAGESIZE --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \ --keygen.seed=${seed} caption="Probe #$((++count)) with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper-upto=${db_size_mb}M --table=+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --pagesize=$PAGESIZE --size-upper-upto=${db_size_mb}M --table=+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \ --keygen.seed=${seed} split=16 caption="Probe #$((++count)) int-key,w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ + --pagesize=$PAGESIZE --size-upper-upto=${db_size_mb}M --table=+key.integer,-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \ --keygen.seed=${seed} caption="Probe #$((++count)) int-key,with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --pagesize=$PAGESIZE --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \ --keygen.seed=${seed} caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --pagesize=$PAGESIZE --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \ --keygen.seed=${seed} caption="Probe #$((++count)) w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper-upto=${db_size_mb}M --table=-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ + --pagesize=$PAGESIZE --size-upper-upto=${db_size_mb}M --table=-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \ --keygen.seed=${seed} caption="Probe #$((++count)) with-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper-upto=${db_size_mb}M --table=+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --pagesize=$PAGESIZE --size-upper-upto=${db_size_mb}M --table=+data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \ --keygen.seed=${seed} split=4 caption="Probe #$((++count)) int-key,w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ + --pagesize=$PAGESIZE --size-upper-upto=${db_size_mb}M --table=+key.integer,-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \ --keygen.seed=${seed} caption="Probe #$((++count)) int-key,int-data, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --pagesize=$PAGESIZE --size-upper-upto=${db_size_mb}M --table=+key.integer,+data.integer --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \ --keygen.seed=${seed} caption="Probe #$((++count)) w/o-dups, split=${split}, case $((++subcase)) of ${cases}" probe \ - --pagesize=min --size-upper-upto=${db_size_mb}M --table=-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ + --pagesize=$PAGESIZE --size-upper-upto=${db_size_mb}M --table=-data.dups --keygen.split=${split} --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ --nops=$nops --batch.write=$wbatch --mode=$(bits2options $bits)${syncmodes[count%3]} \ --keygen.seed=${seed} done # options From 61d0d63ac2f2cbf5557b44845f73721f62bb8654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9B=D0=B5=D0=BE=D0=BD=D0=B8=D0=B4=20=D0=AE=D1=80=D1=8C?= =?UTF-8?q?=D0=B5=D0=B2=20=28Leonid=20Yuriev=29?= Date: Sat, 9 Jul 2022 13:22:35 +0300 Subject: [PATCH 3/5] mdbx-test: refine/fix batch-get testcase for large page size (i.e. when more pairs per page). --- test/test.cc | 66 ++++++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/test/test.cc b/test/test.cc index fd8f987d..6f620223 100644 --- a/test/test.cc +++ b/test/test.cc @@ -1189,28 +1189,29 @@ bool testcase::check_batch_get() { char dump_key[128], dump_value[128]; char dump_key_batch[128], dump_value_batch[128]; - MDBX_cursor *cursor; - int err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor); - if (err != MDBX_SUCCESS) - failure_perror("mdbx_cursor_open()", err); + MDBX_cursor *check_cursor; + int check_err = mdbx_cursor_open(txn_guard.get(), dbi, &check_cursor); + if (check_err != MDBX_SUCCESS) + failure_perror("mdbx_cursor_open()", check_err); MDBX_cursor *batch_cursor; - err = mdbx_cursor_open(txn_guard.get(), dbi, &batch_cursor); - if (err != MDBX_SUCCESS) - failure_perror("mdbx_cursor_open()", err); + int batch_err = mdbx_cursor_open(txn_guard.get(), dbi, &batch_cursor); + if (batch_err != MDBX_SUCCESS) + failure_perror("mdbx_cursor_open()", batch_err); + bool rc = true; MDBX_val pairs[42]; size_t count = 0xDeadBeef; - err = mdbx_cursor_get_batch(batch_cursor, &count, pairs, ARRAY_LENGTH(pairs), - MDBX_FIRST); - bool rc = true; + MDBX_cursor_op batch_op; + batch_err = mdbx_cursor_get_batch(batch_cursor, &count, pairs, + ARRAY_LENGTH(pairs), batch_op = MDBX_FIRST); size_t i, n = 0; - while (err == MDBX_SUCCESS) { + while (batch_err == MDBX_SUCCESS || batch_err == MDBX_RESULT_TRUE) { for (i = 0; i < count; i += 2) { mdbx::slice k, v; - int err2 = mdbx_cursor_get(cursor, &k, &v, MDBX_NEXT); - if (err2 != MDBX_SUCCESS) - failure_perror("mdbx_cursor_open()", err2); + check_err = mdbx_cursor_get(check_cursor, &k, &v, MDBX_NEXT); + if (check_err != MDBX_SUCCESS) + failure_perror("batch-verify: mdbx_cursor_get(MDBX_NEXT)", check_err); if (k != pairs[i] || v != pairs[i + 1]) { log_error( "batch-get pair mismatch %zu/%zu: sequential{%s, %s} != " @@ -1224,29 +1225,32 @@ bool testcase::check_batch_get() { } } n += i / 2; - err = mdbx_cursor_get_batch(batch_cursor, &count, pairs, - ARRAY_LENGTH(pairs), MDBX_NEXT); + batch_op = (batch_err == MDBX_RESULT_TRUE) ? MDBX_GET_CURRENT : MDBX_NEXT; + batch_err = mdbx_cursor_get_batch(batch_cursor, &count, pairs, + ARRAY_LENGTH(pairs), batch_op); } - if (err != MDBX_NOTFOUND) - failure_perror("mdbx_cursor_get_batch()", err); - - err = mdbx_cursor_eof(batch_cursor); - if (err != MDBX_RESULT_TRUE) { - log_error("batch-get %s cursor not-eof %d", "batch", err); - rc = false; - } - err = mdbx_cursor_on_last(batch_cursor); - if (err != MDBX_RESULT_TRUE) { - log_error("batch-get %s cursor not-on-last %d", "batch", err); + if (batch_err != MDBX_NOTFOUND) { + log_error("mdbx_cursor_get_batch(), op %u, err %d", batch_op, batch_err); rc = false; } - err = mdbx_cursor_on_last(cursor); - if (err != MDBX_RESULT_TRUE) { - log_error("batch-get %s cursor not-on-last %d", "checked", err); + batch_err = mdbx_cursor_eof(batch_cursor); + if (batch_err != MDBX_RESULT_TRUE) { + log_error("batch-get %s-cursor not-eof %d", "batch", batch_err); rc = false; } - mdbx_cursor_close(cursor); + batch_err = mdbx_cursor_on_last(batch_cursor); + if (batch_err != MDBX_RESULT_TRUE) { + log_error("batch-get %s-cursor not-on-last %d", "batch", batch_err); + rc = false; + } + + check_err = mdbx_cursor_on_last(check_cursor); + if (check_err != MDBX_RESULT_TRUE) { + log_error("batch-get %s-cursor not-on-last %d", "checked", check_err); + rc = false; + } + mdbx_cursor_close(check_cursor); mdbx_cursor_close(batch_cursor); return rc; } From 9d9df1150918e555e70f3f6ab3caa6c9360db740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9B=D0=B5=D0=BE=D0=BD=D0=B8=D0=B4=20=D0=AE=D1=80=D1=8C?= =?UTF-8?q?=D0=B5=D0=B2=20=28Leonid=20Yuriev=29?= Date: Sat, 9 Jul 2022 14:11:36 +0300 Subject: [PATCH 4/5] mdbx: update ChangeLog. --- ChangeLog.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 979ca3f1..af872b76 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,9 +1,10 @@ ChangeLog --------- -## v0.11.9 (scheduled to 2022-08-02) +## v0.11.9 (Чирчик-1992) scheduled to 2022-08-02 The stable bugfix release. +It is planned that this will be the last release of the v0.11 branch. New: @@ -20,12 +21,15 @@ Minors: - Fixed copy&paste typo inside `meta_checktxnid()`. - Minor fix `meta_checktxnid()` to avoid assertion in debug mode. - Minor fix `mdbx_env_set_geometry()` to avoid returning `EINVAL` in particular rare cases. + - Minor refine/fix batch-get testcase for large page size. + - Added `--pagesize NN` option to long-stotastic test script. + - Updated Valgrind-suppressions file for modern GCC. ------------------------------------------------------------------------------- -## v0.11.8 at 2022-06-12 +## v0.11.8 (Baked Apple) at 2022-06-12 The stable release with an important fixes and workaround for the critical macOS thread-local-storage issue. @@ -51,6 +55,7 @@ Fixes: - Fixed `mdbx_check_fs_local()` for CDROM case on Windows. - Fixed nasty typo of typename which caused false `MDBX_CORRUPTED` error in a rare execution path, when the size of the thread-ID type not equal to 8. + - Fixed Elbrus/E2K LCC 1.26 compiler warnings (memory model for atomic operations, etc). - Fixed write-after-free memory corruption on latest `macOS` during finalization/cleanup of thread(s) that executed read transaction(s). > The issue was suddenly discovered by a [CI](https://en.wikipedia.org/wiki/Continuous_integration) > after adding an iteration with macOS 11 "Big Sur", and then reproduced on recent release of macOS 12 "Monterey". @@ -62,7 +67,6 @@ Fixes: > This is unexpected crazy-like behavior since the order of resources releasing/destroying > is not the reverse of ones acquiring/construction order. Nonetheless such surprise > is now workarounded by using atomic compare-and-swap operations on a 64-bit signatures/cookies. - - Fixed Elbrus/E2K LCC 1.26 compiler warnings (memory model for atomic operations, etc). Minors: @@ -78,7 +82,7 @@ Minors: ------------------------------------------------------------------------------- -## v0.11.7 at 2022-04-22 +## v0.11.7 (Resurrected Sarmat) at 2022-04-22 The stable risen release after the Github's intentional malicious disaster. From 9bbf09b5c4597d05753b96d39580964ff5bca6ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9B=D0=B5=D0=BE=D0=BD=D0=B8=D0=B4=20=D0=AE=D1=80=D1=8C?= =?UTF-8?q?=D0=B5=D0=B2=20=28Leonid=20Yuriev=29?= Date: Sat, 9 Jul 2022 14:54:25 +0300 Subject: [PATCH 5/5] mdbx: minor update README for the locations of a source code. --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c4892b9b..3f78a26b 100644 --- a/README.md +++ b/README.md @@ -394,20 +394,20 @@ since release the version 1.0. _libmdbx_ provides two official ways for integration in source code form: -1. Using the amalgamated source code. - > The amalgamated source code includes all files required to build and +1. Using an amalgamated source code which available in the [releases section](https://gitflic.ru/project/erthink/libmdbx/release) on GitFlic. + > An amalgamated source code includes all files required to build and > use _libmdbx_, but not for testing _libmdbx_ itself. + > Beside the releases an amalgamated sources could be created any time from the original clone of git + > repository on Linux by executing `make dist`. As a result, the desired + > set of files will be formed in the `dist` subdirectory. -2. Adding the complete original source code as a `git submodule`. +2. Adding the complete source code as a `git submodule` from the [origin git repository](https://gitflic.ru/project/erthink/libmdbx) on GitFlic. > This allows you to build as _libmdbx_ and testing tool. > On the other hand, this way requires you to pull git tags, and use C++11 compiler for test tool. _**Please, avoid using any other techniques.**_ Otherwise, at least don't ask for support and don't name such chimeras `libmdbx`. -The amalgamated source code could be created from the original clone of git -repository on Linux by executing `make dist`. As a result, the desired -set of files will be formed in the `dist` subdirectory. ## Building and Testing