mdbx: release v0.10.5 (hotfix).

Acknowledgements:
-----------------
 - [Noel Kuntze](https://github.com/Thermi) for immediately bug reporting.

Fixes:
------
 - Fixed unaligned access regression after the `#pragma pack` fix for modern compilers.
 - Added UBSAN-test to CI to avoid a regression(s) similar to lately fixed.
 - Fixed possibility of meta-pages clashing after manually turn to a particular meta-page using `mdbx_chk` utility.

Minors:
-------
 - Refined handling of weak or invalid meta-pages while a DB opening.
 - Refined providing information for the @MAIN and @GC sub-databases of a last committed modification transaction's ID.

Signed-off-by: Leonid Yuriev <leo@yuriev.ru>

.circleci/config.yml

@@ -11,8 +11,7 @@ jobs:
       - TESTLOG: /tmp/test.log
     steps:
       - checkout
-      - run: make all
+      - run: ulimit -c unlimited && MDBX_BUILD_OPTIONS="-DNDEBUG=1 -DMDBX_FORCE_ASSERTIONS=1" make test-ubsan
-      - run: ulimit -c unlimited && make check
       - run:
           command: |
             mkdir -p /tmp/artifacts

ChangeLog.md

@@ -20,6 +20,24 @@ ChangeLog
  - Packages for [Astra Linux](https://astralinux.ru/), [ALT Linux](https://www.altlinux.org/), [ROSA Linux](https://www.rosalinux.ru/), etc.
 
 
+## v0.10.5 at 2021-10-13
+
+Acknowledgements:
+
+ - [Noel Kuntze](https://github.com/Thermi) for immediately bug reporting.
+
+Fixes:
+
+ - Fixed unaligned access regression after the `#pragma pack` fix for modern compilers.
+ - Added UBSAN-test to CI to avoid a regression(s) similar to lately fixed.
+ - Fixed possibility of meta-pages clashing after manually turn to a particular meta-page using `mdbx_chk` utility.
+
+Minors:
+
+ - Refined handling of weak or invalid meta-pages while a DB opening.
+ - Refined providing information for the @MAIN and @GC sub-databases of a last committed modification transaction's ID.
+
+
 ## v0.10.4 at 2021-10-10
 
 Acknowledgements:
@@ -29,7 +47,7 @@ Acknowledgements:
 
 Fixes:
 
- - Fixed possibility of looping update GC during transaction commit (no public issue since the problem was discovered inside [Positive Technologies](https://www.ptsecurity.ru).
+ - Fixed possibility of looping update GC during transaction commit (no public issue since the problem was discovered inside [Positive Technologies](https://www.ptsecurity.ru)).
  - Fixed `#pragma pack` to avoid provoking some compilers to generate code with [unaligned access](https://github.com/erthink/libmdbx/issues/235).
  - Fixed `noexcept` for potentially throwing `txn::put()` of C++ API.
 

appveyor.yml

@@ -1,4 +1,4 @@
-version: 0.10.4.{build}
+version: 0.10.5.{build}
 
 environment:
   matrix:

packages/buildroot/0001-package-libmdbx-new-package-library-database.patch

@@ -1,9 +1,9 @@
-From 0c0df833879b3f815959c8bd9b6cc27cb9d71b9e Mon Sep 17 00:00:00 2001
+From ce7b70a572cf77d8f37e5047d5d9c8361b2675ae Mon Sep 17 00:00:00 2001
 From: Leonid Yuriev <leo@yuriev.ru>
-Date: Tue, 3 Aug 2021 00:55:27 +0300
+Date: Sun, 10 Oct 2021 15:31:40 +0300
 Subject: [PATCH] package/libmdbx: new package (library/database).
 
-This patch adds libmdbx v0.10.2:
+This patch adds libmdbx v0.10.4:
  - libmdbx is one of the fastest compact embeddable key-value ACID database.
  - libmdbx has a specific set of properties and capabilities,
    focused on creating unique lightweight solutions.
@@ -103,18 +103,18 @@ index 0000000000..d13f73938f
 +		!BR2_TOOLCHAIN_GCC_AT_LEAST_4_4
 diff --git a/package/libmdbx/libmdbx.hash b/package/libmdbx/libmdbx.hash
 new file mode 100644
-index 0000000000..c8a28ada34
+index 0000000000..326cf57bb6
 --- /dev/null
 +++ b/package/libmdbx/libmdbx.hash
 @@ -0,0 +1,5 @@
 +# Hashes from: https://github.com/erthink/libmdbx/releases/
-+sha256  745555704df76626a6612ad0c6bc6b1a66bfab98b9245b07dfb82640aa46d6fa  libmdbx-amalgamated-0.10.2.tar.gz
++sha256  e11d5339a1e1cc34407898933b62a208936fd761a2cc31e11244d581d1d2b5d0  libmdbx-amalgamated-0.10.4.tar.gz
 +
 +# Locally calculated
 +sha256  310fe25c858a9515fc8c8d7d1f24a67c9496f84a91e0a0e41ea9975b1371e569  LICENSE
 diff --git a/package/libmdbx/libmdbx.mk b/package/libmdbx/libmdbx.mk
 new file mode 100644
-index 0000000000..60c5148625
+index 0000000000..f38e4a533b
 --- /dev/null
 +++ b/package/libmdbx/libmdbx.mk
 @@ -0,0 +1,42 @@
@@ -124,7 +124,7 @@ index 0000000000..60c5148625
 +#
 +################################################################################
 +
-+LIBMDBX_VERSION = 0.10.2
++LIBMDBX_VERSION = 0.10.4
 +LIBMDBX_SOURCE = libmdbx-amalgamated-$(LIBMDBX_VERSION).tar.gz
 +LIBMDBX_SITE = https://github.com/erthink/libmdbx/releases/download/v$(LIBMDBX_VERSION)
 +LIBMDBX_SUPPORTS_IN_SOURCE_BUILD = NO
@@ -161,5 +161,5 @@ index 0000000000..60c5148625
 +
 +$(eval $(cmake-package))
 -- 
-2.32.0
+2.33.0
 

src/core.c

@@ -10057,9 +10057,6 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) {
   ts_3 = latency ? mdbx_osal_monotime() : 0;
 
   if (likely(rc == MDBX_SUCCESS)) {
-    if (txn->mt_dbs[MAIN_DBI].md_flags & DBI_DIRTY)
-      txn->mt_dbs[MAIN_DBI].md_mod_txnid = txn->mt_txnid;
-    txn->mt_dbs[FREE_DBI].md_mod_txnid = txn->mt_txnid;
 
     MDBX_meta meta, *head = mdbx_meta_head(env);
     memcpy(meta.mm_magic_and_version, head->mm_magic_and_version, 8);
@@ -10072,7 +10069,15 @@ int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) {
 
     meta.mm_geo = txn->mt_geo;
     meta.mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI];
+    meta.mm_dbs[FREE_DBI].md_mod_txnid =
+        (txn->mt_dbistate[FREE_DBI] & DBI_DIRTY)
+            ? txn->mt_txnid
+            : txn->mt_dbs[FREE_DBI].md_mod_txnid;
     meta.mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI];
+    meta.mm_dbs[MAIN_DBI].md_mod_txnid =
+        (txn->mt_dbistate[MAIN_DBI] & DBI_DIRTY)
+            ? txn->mt_txnid
+            : txn->mt_dbs[MAIN_DBI].md_mod_txnid;
     meta.mm_canary = txn->mt_canary;
     mdbx_meta_set_txnid(env, &meta, txn->mt_txnid);
 
@@ -10431,10 +10436,9 @@ __cold static int mdbx_read_header(MDBX_env *env, MDBX_meta *dest,
   }
 
   if (dest->mm_psize == 0 ||
-      ((env->me_stuck_meta < 0)
+      (env->me_stuck_meta < 0 &&
-           ? (!META_IS_STEADY(dest) &&
+       !(META_IS_STEADY(dest) ||
-              !meta_weak_acceptable(env, dest, lck_exclusive))
+         meta_weak_acceptable(env, dest, lck_exclusive)))) {
-           : false)) {
     mdbx_error("%s", "no usable meta-pages, database is corrupted");
     if (rc == MDBX_SUCCESS) {
       /* TODO: try to restore the database by fully checking b-tree structure
@@ -10483,8 +10487,8 @@ __cold static MDBX_page *mdbx_meta_model(const MDBX_env *env, MDBX_page *model,
                        pages2pv(pv2pages(model_meta->mm_geo.shrink_pv)));
 
   model_meta->mm_psize = env->me_psize;
-  model_meta->mm_flags = (uint16_t)env->me_flags;
+  model_meta->mm_flags = (uint16_t)env->me_flags & DB_PERSISTENT_FLAGS;
-  model_meta->mm_flags |=
+  model_meta->mm_flags =
       MDBX_INTEGERKEY; /* this is mm_dbs[FREE_DBI].md_flags */
   model_meta->mm_dbs[FREE_DBI].md_root = P_INVALID;
   model_meta->mm_dbs[MAIN_DBI].md_root = P_INVALID;
@@ -11708,65 +11712,80 @@ __cold static int mdbx_setup_dxb(MDBX_env *env, const int lck_rc,
       mdbx_assert(env, lck_rc == MDBX_RESULT_TRUE);
       /* exclusive mode */
 
+      MDBX_meta clone;
       MDBX_meta const *const steady = mdbx_meta_steady(env);
-      const txnid_t steady_txnid = mdbx_meta_txnid_fluid(env, steady);
-      MDBX_meta steady_clone;
-      err = mdbx_validate_meta_copy(env, steady, &steady_clone);
-      if (unlikely(err != MDBX_SUCCESS)) {
-        mdbx_error("meta[%u] with %s txnid %" PRIaTXN
-                   " is corrupted, %s needed",
-                   bytes2pgno(env, (uint8_t *)steady - env->me_map), "steady",
-                   steady_txnid, "manual recovery");
-        return MDBX_CORRUPTED;
-      }
-
       MDBX_meta const *const head = mdbx_meta_head(env);
-      if (steady == head)
+      const txnid_t steady_txnid = mdbx_meta_txnid_fluid(env, steady);
-        break;
+      if (META_IS_STEADY(steady)) {
+        err = mdbx_validate_meta_copy(env, steady, &clone);
+        if (unlikely(err != MDBX_SUCCESS)) {
+          mdbx_error("meta[%u] with %s txnid %" PRIaTXN
+                     " is corrupted, %s needed",
+                     bytes2pgno(env, (uint8_t *)steady - env->me_map), "steady",
+                     steady_txnid, "manual recovery");
+          return MDBX_CORRUPTED;
+        }
+        if (steady == head)
+          break;
+      }
 
       const pgno_t pgno = bytes2pgno(env, (uint8_t *)head - env->me_map);
       const txnid_t head_txnid = mdbx_meta_txnid_fluid(env, head);
-      MDBX_meta head_clone;
       const bool head_valid =
-          mdbx_validate_meta_copy(env, head, &head_clone) == MDBX_SUCCESS;
+          mdbx_validate_meta_copy(env, head, &clone) == MDBX_SUCCESS;
+      mdbx_assert(env, !META_IS_STEADY(steady) || head_txnid != steady_txnid);
       if (unlikely(!head_valid)) {
-        mdbx_error("meta[%u] with %s txnid %" PRIaTXN
+        if (unlikely(!META_IS_STEADY(steady))) {
-                   " is corrupted, %s needed",
+          mdbx_error("%s for open or automatic rollback, %s",
-                   pgno, "last", head_txnid, "rollback");
+                     "there are no suitable meta-pages",
+                     "manual recovery is required");
+          return MDBX_CORRUPTED;
+        }
+        mdbx_warning("meta[%u] with last txnid %" PRIaTXN
+                     " is corrupted, rollback needed",
+                     pgno, head_txnid);
         goto purge_meta_head;
       }
 
-      mdbx_assert(env, head_txnid != head_txnid);
-      if (head_txnid == steady_txnid)
-        break;
-
-      mdbx_assert(env, META_IS_STEADY(steady) && !META_IS_STEADY(head));
       if (meta_bootid_match(head)) {
-        mdbx_warning(
+        if (env->me_flags & MDBX_RDONLY) {
-            "opening after an unclean shutdown, but boot-id(%016" PRIx64
+          mdbx_error("%s, but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH: "
-            "-%016" PRIx64
+                     "rollback NOT needed, steady-sync NEEDED%s",
-            ") is MATCH: rollback NOT needed, steady-sync NEEDED%s",
+                     "opening after an unclean shutdown", bootid.x, bootid.y,
-            bootid.x, bootid.y,
+                     ", but unable in read-only mode");
-            (env->me_flags & MDBX_RDONLY) ? ", but unable in read-only mode"
-                                          : "");
-        if (env->me_flags & MDBX_RDONLY)
           return MDBX_WANNA_RECOVERY;
-        meta = head_clone;
+        }
+        mdbx_warning("%s, but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH: "
+                     "rollback NOT needed, steady-sync NEEDED%s",
+                     "opening after an unclean shutdown", bootid.x, bootid.y,
+                     "");
+        meta = clone;
         atomic_store32(&env->me_lck->mti_unsynced_pages, meta.mm_geo.next,
                        mo_Relaxed);
         break;
       }
+      if (unlikely(!META_IS_STEADY(steady))) {
+        mdbx_error("%s, but %s for automatic rollback: %s",
+                   "opening after an unclean shutdown",
+                   "there are no suitable meta-pages",
+                   "manual recovery is required");
+        return MDBX_CORRUPTED;
+      }
       if (env->me_flags & MDBX_RDONLY) {
-        mdbx_error("rollback needed: (from head %" PRIaTXN
+        mdbx_error("%s and rollback needed: (from head %" PRIaTXN
-                   " to steady %" PRIaTXN "), but unable in read-only mode",
+                   " to steady %" PRIaTXN ")%s",
-                   head_txnid, steady_txnid);
+                   "opening after an unclean shutdown", head_txnid,
+                   steady_txnid, ", but unable in read-only mode");
         return MDBX_WANNA_RECOVERY;
       }
 
     purge_meta_head:
-      mdbx_notice("rollback: purge%s meta[%u] with%s txnid %" PRIaTXN,
+      mdbx_notice("%s and doing automatic rollback: "
+                  "purge%s meta[%u] with%s txnid %" PRIaTXN,
+                  "opening after an unclean shutdown",
                   head_valid ? "" : " invalid", pgno, head_valid ? " weak" : "",
                   head_txnid);
+      mdbx_ensure(env, META_IS_STEADY(steady));
       err = mdbx_override_meta(env, pgno, 0, head_valid ? head : steady);
       if (err) {
         mdbx_error("rollback: overwrite meta[%u] with txnid %" PRIaTXN
@@ -12235,7 +12254,7 @@ __cold int mdbx_env_turn_for_recovery(MDBX_env *env, unsigned target) {
     } else {
       txnid_t txnid = mdbx_meta_txnid_stable(env, &meta);
       if (new_txnid <= txnid)
-        safe64_txnid_next(new_txnid);
+        new_txnid = safe64_txnid_next(txnid);
     }
   }
 
@@ -13150,7 +13169,7 @@ dirty:
   }
 
   if (unlikely(ret.page->mp_txnid > front) &&
-      (ret.page->mp_txnid > txn->mt_front || front < txn->mt_txnid)) {
+      unlikely(ret.page->mp_txnid > txn->mt_front || front < txn->mt_txnid)) {
     bad_page(ret.page,
              "invalid page txnid (%" PRIaTXN ") for %s' txnid (%" PRIaTXN ")\n",
              ret.page->mp_txnid,
@@ -13404,11 +13423,13 @@ __hot static int mdbx_page_search(MDBX_cursor *mc, const MDBX_val *key,
                    : mc->mc_txn->mt_txnid;
     MDBX_txn *scan = mc->mc_txn;
     do
-      if (scan->mt_dbistate[mc->mc_dbi] & DBI_DIRTY) {
+      if ((scan->mt_flags & MDBX_TXN_DIRTY) &&
+          (mc->mc_dbi == MAIN_DBI ||
+           (scan->mt_dbistate[mc->mc_dbi] & DBI_DIRTY))) {
         pp_txnid = scan->mt_front;
         break;
       }
-    while ((scan = scan->mt_parent) != nullptr);
+    while (unlikely((scan = scan->mt_parent) != nullptr));
     if (unlikely((rc = mdbx_page_get(mc, root, &mc->mc_pg[0], pp_txnid) != 0)))
       return rc;
   }
@@ -19249,7 +19270,7 @@ __cold int mdbx_dbi_dupsort_depthmask(MDBX_txn *txn, MDBX_dbi dbi,
       break;
     case F_DUPDATA | F_SUBDATA:
       /* sub-tree */
-      *mask |= 1 << unaligned_peek_u16(1, &db->md_depth);
+      *mask |= 1 << UNALIGNED_PEEK_16(db, MDBX_db, md_depth);
       break;
     default:
       mdbx_error("wrong node-flags %u", flags);
@@ -19695,7 +19716,7 @@ static int dbi_open(MDBX_txn *txn, const char *table_name, unsigned user_flags,
 
   /* Got info, register DBI in this txn */
   memset(txn->mt_dbxs + slot, 0, sizeof(MDBX_dbx));
-  txn->mt_dbs[slot] = *(MDBX_db *)data.iov_base;
+  memcpy(&txn->mt_dbs[slot], data.iov_base, sizeof(MDBX_db));
   env->me_dbflags[slot] = 0;
   rc = mdbx_dbi_bind(txn, slot, user_flags, keycmp, datacmp);
   if (unlikely(rc != MDBX_SUCCESS)) {

src/internals.h

@@ -517,6 +517,8 @@ typedef struct MDBX_meta {
 
 } MDBX_meta;
 
+#pragma pack(1)
+
 /* Common header for all page types. The page type depends on mp_flags.
  *
  * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with