mdbx: release v0.11.12 (Эребуни)

The stable bugfix release.

Fixes:
------

 - Fixed static assertion failure on platforms where the `off_t` type is wider
   than corresponding fields of `struct flock` used for file locking (backport).
   Now _libmdbx_ will use `fcntl64(F_GETLK64/F_SETLK64/F_SETLKW64)` if available.
 - Fixed assertion check inside `page_retire_ex()` (backport).

Minors:
-------

 - Fixed `-Wint-to-pointer-cast` warnings while casting to `mdbx_tid_t` (backport).
 - Removed needless `LockFileEx()` inside `mdbx_env_copy()` (backport).

11 files changed, 96 insertions(+), 49 deletions(-)
Signed-off-by: Леонид Юрьев (Leonid Yuriev) <leo@yuriev.ru>

ChangeLog.md

@@ -1,10 +1,50 @@
 ChangeLog
 ---------
 
+## v0.11.12 (Эребуни) at 2022-10-12
+
+The stable bugfix release.
+
+```
+11 files changed, 96 insertions(+), 49 deletions(-)
+Signed-off-by: Леонид Юрьев (Leonid Yuriev) <leo@yuriev.ru>
+```
+
+Fixes:
+
+ - Fixed static assertion failure on platforms where the `off_t` type is wider
+   than corresponding fields of `struct flock` used for file locking (backport).
+   Now _libmdbx_ will use `fcntl64(F_GETLK64/F_SETLK64/F_SETLKW64)` if available.
+ - Fixed assertion check inside `page_retire_ex()` (backport).
+
+Minors:
+
+ - Fixed `-Wint-to-pointer-cast` warnings while casting to `mdbx_tid_t` (backport).
+ - Removed needless `LockFileEx()` inside `mdbx_env_copy()` (backport).
+
+
+## v0.11.11 (Тендра-1790) at 2022-09-11
+
+The stable bugfix release.
+
+```
+10 files changed, 38 insertions(+), 21 deletions(-)
+Signed-off-by: Леонид Юрьев (Leonid Yuriev) <leo@yuriev.ru>
+```
+
+Fixes:
+
+ - Fixed an extra check for `MDBX_APPENDDUP` inside `mdbx_cursor_put()` which could result in returning `MDBX_EKEYMISMATCH` for valid cases.
+ - Fixed an extra ensure/assertion check of `oldest_reader` inside `mdbx_txn_end()`.
+ - Fixed derived C++ builds by removing `MDBX_INTERNAL_FUNC` for `mdbx_w2mb()` and `mdbx_mb2w()`.
+
+
+-------------------------------------------------------------------------------
+
+
 ## v0.11.10 (the TriColor) at 2022-08-22
 
 The stable bugfix release.
-It is planned that this will be the last release of the v0.11 branch.
 
 New:
 

src/core.c

@@ -4835,7 +4835,8 @@ static int mdbx_page_retire_ex(MDBX_cursor *mc, const pgno_t pgno,
       rc = mdbx_page_get(mc, pgno, &check, txn->mt_front);
       if (unlikely(rc != MDBX_SUCCESS))
         return rc;
-      mdbx_tassert(txn, (PAGETYPE(check) & ~P_LEAF2) == (pagetype & ~P_FROZEN));
+      mdbx_tassert(txn, (PAGETYPE(check) & ~(P_LEAF2 | P_SPILLED)) ==
+                            (pagetype & ~(P_LEAF2 | P_FROZEN)));
       mdbx_tassert(txn, !(pagetype & P_FROZEN) || IS_FROZEN(txn, check));
     }
     if (pagetype & P_FROZEN) {
@@ -8788,10 +8789,6 @@ static int mdbx_txn_end(MDBX_txn *txn, const unsigned mode) {
              (void *)env, txn->mt_dbs[MAIN_DBI].md_root,
              txn->mt_dbs[FREE_DBI].md_root);
 
-  mdbx_ensure(env, txn->mt_txnid >=
-                       /* paranoia is appropriate here */ env->me_lck
-                           ->mti_oldest_reader.weak);
-
   if (!(mode & MDBX_END_EOTDONE)) /* !(already closed cursors) */
     mdbx_cursors_eot(txn, false);
 
@@ -8801,6 +8798,9 @@ static int mdbx_txn_end(MDBX_txn *txn, const unsigned mode) {
       MDBX_reader *slot = txn->to.reader;
       mdbx_assert(env, slot->mr_pid.weak == env->me_pid);
       if (likely(!F_ISSET(txn->mt_flags, MDBX_TXN_FINISHED))) {
+        mdbx_ensure(env, txn->mt_txnid >=
+                             /* paranoia is appropriate here */ env->me_lck
+                                 ->mti_oldest_reader.weak);
         mdbx_assert(env, txn->mt_txnid == slot->mr_txnid.weak &&
                              slot->mr_txnid.weak >=
                                  env->me_lck->mti_oldest_reader.weak);
@@ -8828,7 +8828,10 @@ static int mdbx_txn_end(MDBX_txn *txn, const unsigned mode) {
     txn->mt_numdbs = 0; /* prevent further DBI activity */
     txn->mt_flags = MDBX_TXN_RDONLY | MDBX_TXN_FINISHED;
     txn->mt_owner = 0;
-  } else if (!F_ISSET(txn->mt_flags, MDBX_TXN_FINISHED)) {
+  } else if (!(txn->mt_flags & MDBX_TXN_FINISHED)) {
+    mdbx_ensure(env, txn->mt_txnid >=
+                         /* paranoia is appropriate here */ env->me_lck
+                             ->mti_oldest_reader.weak);
 #if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)
     if (txn == env->me_txn0)
       mdbx_txn_valgrind(env, nullptr);
@@ -15542,7 +15545,7 @@ int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data,
           mc->mc_ki[mc->mc_top]++; /* step forward for appending */
           rc = MDBX_NOTFOUND;
         } else {
-          if (unlikely(rc != MDBX_SUCCESS || !(flags & MDBX_APPENDDUP)))
+          if (unlikely(rc != MDBX_SUCCESS))
             /* new-key < last-key
              * or new-key == last-key without MDBX_APPENDDUP */
             return MDBX_EKEYMISMATCH;
@@ -19851,30 +19854,25 @@ __cold int mdbx_env_copy(MDBX_env *env, const char *dest_path,
 #endif
   );
 
-  if (rc == MDBX_SUCCESS) {
 #if defined(_WIN32) || defined(_WIN64)
-    OVERLAPPED ov;
+  /* no locking required since the file opened with ShareMode == 0 */
-    memset(&ov, 0, sizeof(ov));
-    if (!LockFileEx(newfd, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY,
-                    0, 0, INT32_MAX, &ov))
-      rc = GetLastError();
 #else
-    struct flock lock_op;
+  if (rc == MDBX_SUCCESS) {
+    MDBX_STRUCT_FLOCK lock_op;
     memset(&lock_op, 0, sizeof(lock_op));
     lock_op.l_type = F_WRLCK;
     lock_op.l_whence = SEEK_SET;
     lock_op.l_start = 0;
-    lock_op.l_len =
+    lock_op.l_len = OFF_T_MAX;
-        (sizeof(lock_op.l_len) > 4 ? INT64_MAX : INT32_MAX) & ~(size_t)0xffff;
+    if (MDBX_FCNTL(newfd, MDBX_F_SETLK, &lock_op)
-    if (fcntl(newfd, F_SETLK, &lock_op)
 #if (defined(__linux__) || defined(__gnu_linux__)) && defined(LOCK_EX) &&      \
     (!defined(__ANDROID_API__) || __ANDROID_API__ >= 24)
         || flock(newfd, LOCK_EX | LOCK_NB)
 #endif /* Linux */
     )
       rc = errno;
-#endif /* Windows / POSIX */
   }
+#endif /* Windows / POSIX */
 
   if (rc == MDBX_SUCCESS)
     rc = mdbx_env_copy2fd(env, newfd, flags);
@@ -20995,8 +20993,8 @@ __cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func,
                                                         reader_pages_retired))
                              : 0;
       }
-      rc = func(ctx, ++serial, i, pid, (mdbx_tid_t)tid, txnid, lag, bytes_used,
+      rc = func(ctx, ++serial, i, pid, (mdbx_tid_t)((intptr_t)tid), txnid, lag,
-                bytes_retained);
+                bytes_used, bytes_retained);
       if (unlikely(rc != MDBX_SUCCESS))
         break;
     }
@@ -21248,7 +21246,7 @@ __cold static txnid_t mdbx_kick_longlived_readers(MDBX_env *env,
             ? pgno2bytes(env, (pgno_t)(head_retired - oldest_retired))
             : 0;
     int rc = env->me_hsr_callback(
-        env, env->me_txn, pid, (mdbx_tid_t)tid, laggard,
+        env, env->me_txn, pid, (mdbx_tid_t)((intptr_t)tid), laggard,
         (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, space, retry);
     if (rc < 0)
       break;

src/lck-posix.c

@@ -158,26 +158,21 @@ __cold static void choice_fcntl(void) {
                            of reliability reasons */
 #endif                  /* linux */
   ) {
-    op_setlk = F_OFD_SETLK;
+    op_setlk = MDBX_F_OFD_SETLK;
-    op_setlkw = F_OFD_SETLKW;
+    op_setlkw = MDBX_F_OFD_SETLKW;
-    op_getlk = F_OFD_GETLK;
+    op_getlk = MDBX_F_OFD_GETLK;
     return;
   }
-  op_setlk = F_SETLK;
+  op_setlk = MDBX_F_SETLK;
-  op_setlkw = F_SETLKW;
+  op_setlkw = MDBX_F_SETLKW;
-  op_getlk = F_GETLK;
+  op_getlk = MDBX_F_GETLK;
 }
 #else
-#define op_setlk F_SETLK
+#define op_setlk MDBX_F_SETLK
-#define op_setlkw F_SETLKW
+#define op_setlkw MDBX_F_SETLKW
-#define op_getlk F_GETLK
+#define op_getlk MDBX_F_GETLK
 #endif /* MDBX_USE_OFDLOCKS */
 
-#ifndef OFF_T_MAX
-#define OFF_T_MAX                                                              \
-  (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xffff)
-#endif
-
 static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck,
                   const off_t offset, off_t len) {
   STATIC_ASSERT(sizeof(off_t) >= sizeof(void *) &&
@@ -200,7 +195,7 @@ static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck,
   assert((uint64_t)((off_t)((uint64_t)offset + (uint64_t)len)) ==
          ((uint64_t)offset + (uint64_t)len));
   for (;;) {
-    struct flock lock_op;
+    MDBX_STRUCT_FLOCK lock_op;
     STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(lock_op.l_start) &&
                           sizeof(off_t) <= sizeof(lock_op.l_len) &&
                           OFF_T_MAX == (off_t)OFF_T_MAX,
@@ -212,7 +207,7 @@ static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck,
     lock_op.l_whence = SEEK_SET;
     lock_op.l_start = offset;
     lock_op.l_len = len;
-    int rc = fcntl(fd, cmd, &lock_op);
+    int rc = MDBX_FCNTL(fd, cmd, &lock_op);
     mdbx_jitter4testing(true);
     if (rc != -1) {
       if (cmd == op_getlk) {
@@ -226,18 +221,18 @@ static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck,
     }
     rc = errno;
 #if MDBX_USE_OFDLOCKS
-    if (rc == EINVAL &&
+    if (rc == EINVAL && (cmd == MDBX_F_OFD_SETLK || cmd == MDBX_F_OFD_SETLKW ||
-        (cmd == F_OFD_SETLK || cmd == F_OFD_SETLKW || cmd == F_OFD_GETLK)) {
+                         cmd == MDBX_F_OFD_GETLK)) {
       /* fallback to non-OFD locks */
-      if (cmd == F_OFD_SETLK)
+      if (cmd == MDBX_F_OFD_SETLK)
-        cmd = F_SETLK;
+        cmd = MDBX_F_SETLK;
-      else if (cmd == F_OFD_SETLKW)
+      else if (cmd == MDBX_F_OFD_SETLKW)
-        cmd = F_SETLKW;
+        cmd = MDBX_F_SETLKW;
       else
-        cmd = F_GETLK;
+        cmd = MDBX_F_GETLK;
-      op_setlk = F_SETLK;
+      op_setlk = MDBX_F_SETLK;
-      op_setlkw = F_SETLKW;
+      op_setlkw = MDBX_F_SETLKW;
-      op_getlk = F_GETLK;
+      op_getlk = MDBX_F_GETLK;
       continue;
     }
 #endif /* MDBX_USE_OFDLOCKS */

src/man1/mdbx_chk.1

@@ -1,6 +1,6 @@
 .\" Copyright 2015-2022 Leonid Yuriev <leo@yuriev.ru>.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
-.TH MDBX_CHK 1 "2022-08-22" "MDBX 0.11.10"
+.TH MDBX_CHK 1 "2022-10-12" "MDBX 0.11.12"
 .SH NAME
 mdbx_chk \- MDBX checking tool
 .SH SYNOPSIS

src/man1/mdbx_copy.1

@@ -2,7 +2,7 @@
 .\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
-.TH MDBX_COPY 1 "2022-08-22" "MDBX 0.11.10"
+.TH MDBX_COPY 1 "2022-10-12" "MDBX 0.11.12"
 .SH NAME
 mdbx_copy \- MDBX environment copy tool
 .SH SYNOPSIS

src/man1/mdbx_drop.1

@@ -1,7 +1,7 @@
 .\" Copyright 2021-2022 Leonid Yuriev <leo@yuriev.ru>.
 .\" Copyright 2014-2021 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
-.TH MDBX_DROP 1 "2022-08-22" "MDBX 0.11.10"
+.TH MDBX_DROP 1 "2022-10-12" "MDBX 0.11.12"
 .SH NAME
 mdbx_drop \- MDBX database delete tool
 .SH SYNOPSIS

src/man1/mdbx_dump.1

@@ -2,7 +2,7 @@
 .\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
-.TH MDBX_DUMP 1 "2022-08-22" "MDBX 0.11.10"
+.TH MDBX_DUMP 1 "2022-10-12" "MDBX 0.11.12"
 .SH NAME
 mdbx_dump \- MDBX environment export tool
 .SH SYNOPSIS

src/man1/mdbx_load.1

@@ -2,7 +2,7 @@
 .\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
-.TH MDBX_LOAD 1 "2022-08-22" "MDBX 0.11.10"
+.TH MDBX_LOAD 1 "2022-10-12" "MDBX 0.11.12"
 .SH NAME
 mdbx_load \- MDBX environment import tool
 .SH SYNOPSIS

src/man1/mdbx_stat.1

@@ -2,7 +2,7 @@
 .\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
-.TH MDBX_STAT 1 "2022-08-22" "MDBX 0.11.10"
+.TH MDBX_STAT 1 "2022-10-12" "MDBX 0.11.12"
 .SH NAME
 mdbx_stat \- MDBX environment status tool
 .SH SYNOPSIS

src/options.h

@@ -237,7 +237,10 @@
 
 /** Advanced: Using POSIX OFD-locks (autodetection by default). */
 #ifndef MDBX_USE_OFDLOCKS
-#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) &&   \
+#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) &&                         \
+      defined(F_OFD_GETLK)) ||                                                 \
+     (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) &&                     \
+      defined(F_OFD_GETLK64))) &&                                              \
     !defined(MDBX_SAFE4QEMU) &&                                                \
     !defined(__sun) /* OFD-lock are broken on Solaris */
 #define MDBX_USE_OFDLOCKS 1

src/osal.c

@@ -527,14 +527,12 @@ static const DWORD WC_ERR_INVALID_CHARS =
         : 0;
 #endif /* WC_ERR_INVALID_CHARS */
 
-MDBX_INTERNAL_FUNC size_t mdbx_mb2w(wchar_t *dst, size_t dst_n, const char *src,
+size_t mdbx_mb2w(wchar_t *dst, size_t dst_n, const char *src, size_t src_n) {
-                                    size_t src_n) {
   return MultiByteToWideChar(CP_THREAD_ACP, MB_ERR_INVALID_CHARS, src,
                              (int)src_n, dst, (int)dst_n);
 }
 
-MDBX_INTERNAL_FUNC size_t mdbx_w2mb(char *dst, size_t dst_n, const wchar_t *src,
+size_t mdbx_w2mb(char *dst, size_t dst_n, const wchar_t *src, size_t src_n) {
-                                    size_t src_n) {
   return WideCharToMultiByte(CP_THREAD_ACP, WC_ERR_INVALID_CHARS, src,
                              (int)src_n, dst, (int)dst_n, nullptr, nullptr);
 }

src/osal.h

@@ -180,10 +180,8 @@ static inline void mdbx_free(void *ptr) { HeapFree(GetProcessHeap(), 0, ptr); }
 #define vsnprintf _vsnprintf /* ntdll */
 #endif
 
-MDBX_INTERNAL_FUNC size_t mdbx_mb2w(wchar_t *dst, size_t dst_n, const char *src,
+size_t mdbx_mb2w(wchar_t *dst, size_t dst_n, const char *src, size_t src_n);
-                                    size_t src_n);
+size_t mdbx_w2mb(char *dst, size_t dst_n, const wchar_t *src, size_t src_n);
-MDBX_INTERNAL_FUNC size_t mdbx_w2mb(char *dst, size_t dst_n, const wchar_t *src,
-                                    size_t src_n);
 
 #else /*----------------------------------------------------------------------*/
 
@@ -289,7 +287,41 @@ MDBX_MAYBE_UNUSED static __inline void mdbx_jitter4testing(bool tiny);
 #if defined(_WIN32) || defined(_WIN64)
 #define MAX_WRITE UINT32_C(0x01000000)
 #else
-#define MAX_WRITE UINT32_C(0x3fff0000)
+#define MAX_WRITE UINT32_C(0x3f000000)
+
+#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) &&         \
+    !defined(__ANDROID_API__)
+#define MDBX_F_SETLK F_SETLK64
+#define MDBX_F_SETLKW F_SETLKW64
+#define MDBX_F_GETLK F_GETLK64
+#define MDBX_FCNTL fcntl64
+#define MDBX_STRUCT_FLOCK struct flock64
+#ifndef OFF_T_MAX
+#define OFF_T_MAX UINT64_C(0x7fffFFFFfff00000)
+#endif /* OFF_T_MAX */
+#else
+#define MDBX_F_SETLK F_SETLK
+#define MDBX_F_SETLKW F_SETLKW
+#define MDBX_F_GETLK F_GETLK
+#define MDBX_FCNTL fcntl
+#define MDBX_STRUCT_FLOCK struct flock
+#endif /* MDBX_F_SETLK, MDBX_F_SETLKW, MDBX_F_GETLK */
+
+#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) &&                       \
+    defined(F_OFD_GETLK64) && !defined(__ANDROID_API__)
+#define MDBX_F_OFD_SETLK F_OFD_SETLK64
+#define MDBX_F_OFD_SETLKW F_OFD_SETLKW64
+#define MDBX_F_OFD_GETLK F_OFD_GETLK64
+#else
+#define MDBX_F_OFD_SETLK F_OFD_SETLK
+#define MDBX_F_OFD_SETLKW F_OFD_SETLKW
+#define MDBX_F_OFD_GETLK F_OFD_GETLK
+#ifndef OFF_T_MAX
+#define OFF_T_MAX                                                              \
+  (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff)
+#endif /* OFF_T_MAX */
+#endif /* MDBX_F_OFD_SETLK64, MDBX_F_OFD_SETLKW64, MDBX_F_OFD_GETLK64 */
+
 #endif
 
 #if defined(__linux__) || defined(__gnu_linux__)