mdbx: release v0.11.1

Backward compatibility break:
-----------------------------

The database format signature has been changed to prevent
forward-interoperability with an previous releases, which may lead to a
[false positive diagnosis of database corruption](https://github.com/erthink/libmdbx/issues/238)
due to flaws of an old library versions.

This change is mostly invisible:

 - previously versions are unable to read/write a new DBs;
 - but the new release is able to handle an old DBs and will silently upgrade ones.

Acknowledgements:
-----------------
 - [Alex Sharov](https://github.com/AskAlexSharov) for reporting and testing.

Signed-off-by: Leonid Yuriev <leo@yuriev.ru>

.github/actions/spelling/expect.txt

@@ -709,6 +709,7 @@ inprocess
 INSTEADOF
 integerdup
 integerkey
+interoperability
 interprocedural
 intlimits
 intptr

ChangeLog.md

@@ -1,8 +1,6 @@
 ChangeLog
 ---------
 
-## v0.11.x (in development)
-
 ### TODO
 
  - [Engage an "overlapped I/O" on Windows](https://github.com/erthink/libmdbx/issues/224).
@@ -20,7 +18,34 @@ ChangeLog
  - Packages for [Astra Linux](https://astralinux.ru/), [ALT Linux](https://www.altlinux.org/), [ROSA Linux](https://www.rosalinux.ru/), etc.
 
 
-## v0.10.5 at 2021-10-13
+## v0.11.1 at 2021-10-23
+
+### Backward compatibility break:
+
+The database format signature has been changed to prevent
+forward-interoperability with an previous releases, which may lead to a
+[false positive diagnosis of database corruption](https://github.com/erthink/libmdbx/issues/238)
+due to flaws of an old library versions.
+
+This change is mostly invisible:
+
+ - previously versions are unable to read/write a new DBs;
+ - but the new release is able to handle an old DBs and will silently upgrade ones.
+
+Acknowledgements:
+
+ - [Alex Sharov](https://github.com/AskAlexSharov) for reporting and testing.
+
+
+## v0.10.5 at 2021-10-13 (obsolete, please use v0.11.1)
+
+Unfortunately, the `v0.10.5` accidentally comes not full-compatible with previous releases:
+
+ - `v0.10.5` can read/processing DBs created by previous releases, i.e. the backward-compatibility is provided;
+ - however, previous releases may lead to false-corrupted state with DB that was touched by `v0.10.5`, i.e. the forward-compatibility is broken for `v0.10.4` and earlier.
+
+This cannot be fixed, as it requires fixing past versions, which as a result we will just get a current version.
+Therefore, it is recommended to use `v0.11.1` instead of `v0.10.5`.
 
 Acknowledgements:
 

appveyor.yml

@@ -1,4 +1,4 @@
-version: 0.10.5.{build}
+version: 0.11.1.{build}
 
 environment:
   matrix:

mdbx.h

@@ -568,9 +568,9 @@ typedef mode_t mdbx_mode_t;
 extern "C" {
 #endif
 
-/* MDBX version 0.10.x */
+/* MDBX version 0.11.x */
 #define MDBX_VERSION_MAJOR 0
-#define MDBX_VERSION_MINOR 10
+#define MDBX_VERSION_MINOR 11
 
 #ifndef LIBMDBX_API
 #if defined(LIBMDBX_EXPORTS)

src/core.c

@@ -10124,7 +10124,8 @@ static int mdbx_validate_meta(MDBX_env *env, MDBX_meta *const meta,
   const uint64_t magic_and_version =
       unaligned_peek_u64(4, &meta->mm_magic_and_version);
   if (unlikely(magic_and_version != MDBX_DATA_MAGIC &&
-               magic_and_version != MDBX_DATA_MAGIC_DEVEL)) {
+               magic_and_version != MDBX_DATA_MAGIC_LEGACY_COMPAT &&
+               magic_and_version != MDBX_DATA_MAGIC_LEGACY_DEVEL)) {
     mdbx_error("meta[%u] has invalid magic/version %" PRIx64, meta_number,
                magic_and_version);
     return ((magic_and_version >> 8) != MDBX_MAGIC) ? MDBX_INVALID
@@ -11867,6 +11868,26 @@ __cold static int mdbx_setup_dxb(MDBX_env *env, const int lck_rc,
 
     atomic_store32(&env->me_lck->mti_discarded_tail,
                    bytes2pgno(env, used_aligned2os_bytes), mo_Relaxed);
+
+    if ((env->me_flags & MDBX_RDONLY) == 0 && env->me_stuck_meta < 0) {
+      for (int n = 0; n < 3; ++n) {
+        MDBX_meta *const meta = METAPAGE(env, n);
+        if (unlikely(unaligned_peek_u64(4, &meta->mm_magic_and_version) !=
+                     MDBX_DATA_MAGIC)) {
+          const txnid_t txnid = mdbx_meta_txnid_fluid(env, meta);
+          mdbx_notice("%s %s"
+                      "meta[%u], txnid %" PRIaTXN,
+                      "updating db-format signature for",
+                      META_IS_STEADY(meta) ? "stead-" : "weak-", n, txnid);
+          err = mdbx_override_meta(env, n, txnid, meta);
+          if (unlikely(err != MDBX_SUCCESS)) {
+            mdbx_error("%s meta[%u], txnid %" PRIaTXN ", error %d",
+                       "updating db-format signature for", n, txnid, err);
+            return err;
+          }
+        }
+      }
+    }
   } /* lck exclusive, lck_rc == MDBX_RESULT_TRUE */
 
   //---------------------------------------------------- setup madvise/readahead
@@ -13116,10 +13137,11 @@ mdbx_page_get_ex(MDBX_cursor *const mc, const pgno_t pgno,
   mdbx_tassert(txn, front <= txn->mt_front);
   if (unlikely(pgno >= txn->mt_next_pgno)) {
     mdbx_error("page #%" PRIaPGNO " beyond next-pgno", pgno);
+  notfound:
     ret.page = nullptr;
-  corrupted:
-    mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
     ret.err = MDBX_PAGE_NOTFOUND;
+  bailout:
+    mc->mc_txn->mt_flags |= MDBX_TXN_ERROR;
     return ret;
   }
 
@@ -13159,33 +13181,36 @@ dirty:
              "mismatch actual pgno (%" PRIaPGNO ") != expected (%" PRIaPGNO
              ")\n",
              ret.page->mp_pgno, pgno);
-    goto corrupted;
+    goto notfound;
   }
 
 #if !MDBX_DISABLE_PAGECHECKS
   if (unlikely(ret.page->mp_flags & P_ILL_BITS)) {
-    bad_page(ret.page, "invalid page's flags (%u)\n", ret.page->mp_flags);
-    goto corrupted;
+    ret.err =
+        bad_page(ret.page, "invalid page's flags (%u)\n", ret.page->mp_flags);
+    goto bailout;
   }
 
   if (unlikely(ret.page->mp_txnid > front) &&
       unlikely(ret.page->mp_txnid > txn->mt_front || front < txn->mt_txnid)) {
-    bad_page(ret.page,
-             "invalid page txnid (%" PRIaTXN ") for %s' txnid (%" PRIaTXN ")\n",
-             ret.page->mp_txnid,
-             (front == txn->mt_front && front != txn->mt_txnid) ? "front-txn"
-                                                                : "parent-page",
-             front);
-    goto corrupted;
+    ret.err = bad_page(
+        ret.page,
+        "invalid page txnid (%" PRIaTXN ") for %s' txnid (%" PRIaTXN ")\n",
+        ret.page->mp_txnid,
+        (front == txn->mt_front && front != txn->mt_txnid) ? "front-txn"
+                                                           : "parent-page",
+        front);
+    goto bailout;
   }
 
   if (unlikely((ret.page->mp_upper < ret.page->mp_lower ||
                 ((ret.page->mp_lower | ret.page->mp_upper) & 1) ||
                 PAGEHDRSZ + ret.page->mp_upper > env->me_psize) &&
                !IS_OVERFLOW(ret.page))) {
-    bad_page(ret.page, "invalid page lower(%u)/upper(%u) with limit (%u)\n",
-             ret.page->mp_lower, ret.page->mp_upper, page_space(env));
-    goto corrupted;
+    ret.err =
+        bad_page(ret.page, "invalid page lower(%u)/upper(%u) with limit (%u)\n",
+                 ret.page->mp_lower, ret.page->mp_upper, page_space(env));
+    goto bailout;
   }
 #endif /* !MDBX_DISABLE_PAGECHECKS */
 
@@ -13430,7 +13455,7 @@ __hot static int mdbx_page_search(MDBX_cursor *mc, const MDBX_val *key,
         break;
       }
     while (unlikely((scan = scan->mt_parent) != nullptr));
-    if (unlikely((rc = mdbx_page_get(mc, root, &mc->mc_pg[0], pp_txnid) != 0)))
+    if (unlikely((rc = mdbx_page_get(mc, root, &mc->mc_pg[0], pp_txnid)) != 0))
       return rc;
   }
 

src/internals.h

@@ -67,7 +67,7 @@
 #endif
 #ifndef _CRT_SECURE_NO_WARNINGS
 #define _CRT_SECURE_NO_WARNINGS
-#endif
+#endif /* _CRT_SECURE_NO_WARNINGS */
 #if _MSC_VER > 1800
 #pragma warning(disable : 4464) /* relative include path contains '..' */
 #endif
@@ -384,7 +384,7 @@ MDBX_MAYBE_UNUSED static
 #define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11)
 
 /* FROZEN: The version number for a database's datafile format. */
-#define MDBX_DATA_VERSION 2
+#define MDBX_DATA_VERSION 3
 /* The version number for a database's lockfile format. */
 #define MDBX_LOCK_VERSION 4
 
@@ -789,7 +789,11 @@ typedef struct MDBX_lockinfo {
 
 #define MDBX_DATA_MAGIC                                                        \
   ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION)
-#define MDBX_DATA_MAGIC_DEVEL ((MDBX_MAGIC << 8) + 255)
+
+#define MDBX_DATA_MAGIC_LEGACY_COMPAT                                          \
+  ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2)
+
+#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255)
 
 #define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION)
 

src/mdbx.c++

@@ -5,9 +5,9 @@
 // Non-inline part of the libmdbx C++ API (preliminary)
 //
 
-#ifdef _MSC_VER
+#if defined(_MSC_VER) && !defined(_CRT_SECURE_NO_WARNINGS)
 #define _CRT_SECURE_NO_WARNINGS
-#endif
+#endif /* _CRT_SECURE_NO_WARNINGS */
 
 #include "../mdbx.h++"
 

src/osal.h

@@ -33,7 +33,7 @@
 #if defined(_WIN32) || defined(_WIN64)
 #if !defined(_CRT_SECURE_NO_WARNINGS)
 #define _CRT_SECURE_NO_WARNINGS
-#endif
+#endif /* _CRT_SECURE_NO_WARNINGS */
 #if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY &&             \
     !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT
 #define _NO_CRT_STDIO_INLINE

test/base.h

@@ -23,7 +23,9 @@
 #define _WIN32_WINNT 0x0601 /* Windows 7 */
 #endif
 #ifdef _MSC_VER
+#ifndef _CRT_SECURE_NO_WARNINGS
 #define _CRT_SECURE_NO_WARNINGS
+#endif /* _CRT_SECURE_NO_WARNINGS */
 #pragma warning(push, 1)
 #pragma warning(disable : 4548) /* expression before comma has no effect;      \
                                    expected expression with side - effect */