mdbx: Merge branch 'master' into stable/0.1

README.md

@@ -20,6 +20,8 @@ libmdbx
    6. [Asynchronous lazy data flushing](https://sites.fas.harvard.edu/~cs265/papers/kathuria-2008.pdf) to disk(s);
    7. etc...
 
+Don't miss [Java Native Interface](https://github.com/castortech/mdbxjni) by [Castor Technologies](https://castortech.com/).
+
 -----
 
 Nowadays MDBX intended for Linux, and support Windows (since

mdbx.h

@@ -1637,7 +1637,9 @@ typedef int MDBX_pgvisitor_func(uint64_t pgno, unsigned pgnumber, void *ctx,
 LIBMDBX_API int mdbx_env_pgwalk(MDBX_txn *txn, MDBX_pgvisitor_func *visitor,
                                 void *ctx);
 
-typedef struct mdbx_canary { uint64_t x, y, z, v; } mdbx_canary;
+typedef struct mdbx_canary {
+  uint64_t x, y, z, v;
+} mdbx_canary;
 
 LIBMDBX_API int mdbx_canary_put(MDBX_txn *txn, const mdbx_canary *canary);
 LIBMDBX_API int mdbx_canary_get(MDBX_txn *txn, mdbx_canary *canary);

src/bits.h

@@ -367,19 +367,19 @@ typedef struct MDBX_page {
 #define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_data))
 
 /* The maximum size of a database page.
-*
+ *
-* It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper.
+ * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper.
-*
+ *
-* MDBX will use database pages < OS pages if needed.
+ * MDBX will use database pages < OS pages if needed.
-* That causes more I/O in write transactions: The OS must
+ * That causes more I/O in write transactions: The OS must
-* know (read) the whole page before writing a partial page.
+ * know (read) the whole page before writing a partial page.
-*
+ *
-* Note that we don't currently support Huge pages. On Linux,
+ * Note that we don't currently support Huge pages. On Linux,
-* regular data files cannot use Huge pages, and in general
+ * regular data files cannot use Huge pages, and in general
-* Huge pages aren't actually pageable. We rely on the OS
+ * Huge pages aren't actually pageable. We rely on the OS
-* demand-pager to read our data and page it out when memory
+ * demand-pager to read our data and page it out when memory
-* pressure from other processes is high. So until OSs have
+ * pressure from other processes is high. So until OSs have
-* actual paging support for Huge pages, they're not viable. */
+ * actual paging support for Huge pages, they're not viable. */
 #define MAX_PAGESIZE 0x10000u
 #define MIN_PAGESIZE 512u
 

src/lck-windows.c

@@ -127,7 +127,8 @@ int mdbx_txn_lock(MDBX_env *env, bool dontwait) {
     EnterCriticalSection(&env->me_windowsbug_lock);
   }
 
-  if (flock(env->me_fd, dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT)
+  if (flock(env->me_fd,
+            dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT)
                      : (LCK_EXCLUSIVE | LCK_WAITFOR),
             LCK_BODY))
     return MDBX_SUCCESS;

src/mdbx.c

@@ -42,8 +42,8 @@
 int __hot mdbx_e2k_memcmp_bug_workaround(const void *s1, const void *s2,
                                          size_t n) {
   if (unlikely(n > 42
-               /* LY: align followed access if reasonable possible */ &&
+               /* LY: align followed access if reasonable possible */
-               (((uintptr_t)s1) & 7) != 0 &&
+               && (((uintptr_t)s1) & 7) != 0 &&
                (((uintptr_t)s1) & 7) == (((uintptr_t)s2) & 7))) {
     if (((uintptr_t)s1) & 1) {
       const int diff = *(uint8_t *)s1 - *(uint8_t *)s2;
@@ -4733,9 +4733,8 @@ static int mdbx_sync_locked(MDBX_env *env, unsigned flags,
   mdbx_assert(env, !mdbx_meta_eq(env, pending, meta2));
 
   mdbx_assert(env, ((env->me_flags ^ flags) & MDBX_WRITEMAP) == 0);
-  mdbx_ensure(env,
+  mdbx_ensure(env, target == head || mdbx_meta_txnid_stable(env, target) <
-              target == head ||
+                                         pending->mm_txnid_a);
-                  mdbx_meta_txnid_stable(env, target) < pending->mm_txnid_a);
   if (env->me_flags & MDBX_WRITEMAP) {
     mdbx_jitter4testing(true);
     if (likely(target != head)) {
@@ -5835,8 +5834,8 @@ int __cold mdbx_env_open_ex(MDBX_env *env, const char *path, unsigned flags,
   if ((flags & MDBX_RDONLY) == 0) {
     MDBX_txn *txn;
     int tsize = sizeof(MDBX_txn),
-        size = tsize +
+        size =
-               env->me_maxdbs * (sizeof(MDBX_db) + sizeof(MDBX_cursor *) +
+            tsize + env->me_maxdbs * (sizeof(MDBX_db) + sizeof(MDBX_cursor *) +
                                       sizeof(unsigned) + 1);
     if ((env->me_pbuf = calloc(1, env->me_psize)) && (txn = calloc(1, size))) {
       txn->mt_dbs = (MDBX_db *)((char *)txn + tsize);
@@ -6315,7 +6314,7 @@ static int mdbx_page_get(MDBX_cursor *mc, pgno_t pgno, MDBX_page **ret,
 
 mapped:
   p = pgno2page(env, pgno);
-/* TODO: check p->mp_validator here */
+  /* TODO: check p->mp_validator here */
 
 done:
   *ret = p;
@@ -10038,8 +10037,7 @@ int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data,
   if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)))
     return MDBX_EINVAL;
 
-  if (unlikely(flags &
+  if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_RESERVE |
-               ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_RESERVE |
                          MDBX_APPEND | MDBX_APPENDDUP | MDBX_CURRENT)))
     return MDBX_EINVAL;
 
@@ -10680,9 +10678,9 @@ int __cold mdbx_env_info(MDBX_env *env, MDBX_envinfo *arg, size_t bytes) {
 }
 
 static MDBX_cmp_func *mdbx_default_keycmp(unsigned flags) {
-  return (flags & MDBX_REVERSEKEY) ? mdbx_cmp_memnr : (flags & MDBX_INTEGERKEY)
+  return (flags & MDBX_REVERSEKEY)
-                                                          ? mdbx_cmp_int_a2
+             ? mdbx_cmp_memnr
-                                                          : mdbx_cmp_memn;
+             : (flags & MDBX_INTEGERKEY) ? mdbx_cmp_int_a2 : mdbx_cmp_memn;
 }
 
 static MDBX_cmp_func *mdbx_default_datacmp(unsigned flags) {
@@ -11872,8 +11870,7 @@ int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *new_data,
   if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)))
     return MDBX_EINVAL;
 
-  if (unlikely(flags &
+  if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_RESERVE |
-               ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_RESERVE |
                          MDBX_APPEND | MDBX_APPENDDUP | MDBX_CURRENT)))
     return MDBX_EINVAL;
 

src/osal.c

@@ -837,13 +837,14 @@ int mdbx_mmap(int flags, mdbx_mmap_t *map, size_t size, size_t limit) {
   SectionSize.QuadPart = size;
   rc = NtCreateSection(
       &map->section,
-      /* DesiredAccess */ (flags & MDBX_WRITEMAP)
+      /* DesiredAccess */
+          (flags & MDBX_WRITEMAP)
           ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE |
                 SECTION_MAP_WRITE
           : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE,
       /* ObjectAttributes */ NULL, /* MaximumSize (InitialSize) */ &SectionSize,
-      /* SectionPageProtection */ (flags & MDBX_RDONLY) ? PAGE_READONLY
+      /* SectionPageProtection */
-                                                        : PAGE_READWRITE,
+          (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE,
       /* AllocationAttributes */ SEC_RESERVE, map->fd);
   if (!NT_SUCCESS(rc))
     return ntstatus2errcode(rc);
@@ -856,8 +857,8 @@ int mdbx_mmap(int flags, mdbx_mmap_t *map, size_t size, size_t limit) {
       /* SectionOffset */ NULL, &ViewSize,
       /* InheritDisposition */ ViewUnmap,
       /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE,
-      /* Win32Protect */ (flags & MDBX_WRITEMAP) ? PAGE_READWRITE
+      /* Win32Protect */
-                                                 : PAGE_READONLY);
+          (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY);
   if (!NT_SUCCESS(rc)) {
     NtClose(map->section);
     map->section = 0;
@@ -996,14 +997,15 @@ retry_file_and_section:
   SectionSize.QuadPart = size;
   status = NtCreateSection(
       &map->section,
-      /* DesiredAccess */ (flags & MDBX_WRITEMAP)
+      /* DesiredAccess */
+          (flags & MDBX_WRITEMAP)
           ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE |
                 SECTION_MAP_WRITE
           : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE,
       /* ObjectAttributes */ NULL,
       /* MaximumSize (InitialSize) */ &SectionSize,
-      /* SectionPageProtection */ (flags & MDBX_RDONLY) ? PAGE_READONLY
+      /* SectionPageProtection */
-                                                        : PAGE_READWRITE,
+          (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE,
       /* AllocationAttributes */ SEC_RESERVE, map->fd);
 
   if (!NT_SUCCESS(status))
@@ -1027,8 +1029,8 @@ retry_mapview:;
       /* SectionOffset */ NULL, &ViewSize,
       /* InheritDisposition */ ViewUnmap,
       /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE,
-      /* Win32Protect */ (flags & MDBX_WRITEMAP) ? PAGE_READWRITE
+      /* Win32Protect */
-                                                 : PAGE_READONLY);
+          (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY);
 
   if (!NT_SUCCESS(status)) {
     if (status == /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018 &&

src/tools/mdbx_load.c

@@ -143,8 +143,8 @@ static void readhdr(void) {
                 lineno, (char *)dbuf.iov_base + STRLENOF("mapsize="));
         exit(EXIT_FAILURE);
       }
-    } else if (!strncmp(dbuf.iov_base, "maxreaders=",
+    } else if (!strncmp(dbuf.iov_base,
-                        STRLENOF("maxreaders="))) {
+                        "maxreaders=", STRLENOF("maxreaders="))) {
       int i;
       ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
       if (ptr)

src/tools/mdbx_stat.c

@@ -152,9 +152,15 @@ int main(int argc, char *argv[]) {
     goto env_close;
   }
 
+  if (envinfo || freinfo) {
+    (void)mdbx_env_info(env, &mei, sizeof(mei));
+  } else {
+    /* LY: zap warnings from gcc */
+    memset(&mei, 0, sizeof(mei));
+  }
+
   if (envinfo) {
     (void)mdbx_env_stat(env, &mst, sizeof(mst));
-    (void)mdbx_env_info(env, &mei, sizeof(mei));
     printf("Environment Info\n");
     printf("  Pagesize: %u\n", mst.ms_psize);
     if (mei.mi_geo.lower != mei.mi_geo.upper) {
@@ -183,7 +189,6 @@ int main(int argc, char *argv[]) {
   } else {
     /* LY: zap warnings from gcc */
     memset(&mst, 0, sizeof(mst));
-    memset(&mei, 0, sizeof(mei));
   }
 
   if (rdrinfo) {

test/base.h

@@ -32,10 +32,10 @@
 #endif                          /* _MSC_VER (warnings) */
 
 /* If you wish to build your application for a previous Windows platform,
-* include WinSDKVer.h and set the _WIN32_WINNT macro to the platform you
+ * include WinSDKVer.h and set the _WIN32_WINNT macro to the platform you
-* wish to support before including SDKDDKVer.h.
+ * wish to support before including SDKDDKVer.h.
-*
+ *
-* TODO: #define _WIN32_WINNT WIN32_MUSTDIE */
+ * TODO: #define _WIN32_WINNT WIN32_MUSTDIE */
 #include <SDKDDKVer.h>
 #endif /* WINDOWS */
 

test/log.cc

@@ -206,7 +206,7 @@ void local_suffix::pop() {
 
 local_suffix::~local_suffix() { suffix.erase(trim_pos); }
 
-} /* namespace log */
+} // namespace logging
 
 void log_extra(const char *msg, ...) {
   if (logging::extra >= logging::level) {

test/log.h

@@ -70,7 +70,7 @@ public:
   ~local_suffix();
 };
 
-} /* namespace log */
+} // namespace logging
 
 void __printf_args(1, 2) log_extra(const char *msg, ...);
 void __printf_args(1, 2) log_trace(const char *msg, ...);

test/main.cc

@@ -122,8 +122,7 @@ int main(int argc, char *const argv[]) {
   if (argc < 2)
     failure("No parameters given\n");
 
-  if (argc == 2 &&
+  if (argc == 2 && strncmp(argv[1], global::thunk_param_prefix,
-      strncmp(argv[1], global::thunk_param_prefix,
                            strlen(global::thunk_param_prefix)) == 0)
     return test_execute(
                actor_config(argv[1] + strlen(global::thunk_param_prefix)))

test/osal-windows.cc

@@ -168,6 +168,90 @@ bool actor_config::osal_deserialize(const char *str, const char *end,
 typedef std::pair<HANDLE, actor_status> child;
 static std::unordered_map<mdbx_pid_t, child> childs;
 
+static void ArgvQuote(std::string &CommandLine, const std::string &Argument,
+                      bool Force = false)
+
+/*++
+
+https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/
+
+Routine Description:
+
+    This routine appends the given argument to a command line such
+    that CommandLineToArgvW will return the argument string unchanged.
+    Arguments in a command line should be separated by spaces; this
+    function does not add these spaces.
+
+Arguments:
+
+    Argument - Supplies the argument to encode.
+
+    CommandLine - Supplies the command line to which we append the encoded
+argument string.
+
+    Force - Supplies an indication of whether we should quote
+            the argument even if it does not contain any characters that would
+            ordinarily require quoting.
+
+Return Value:
+
+    None.
+
+Environment:
+
+    Arbitrary.
+
+--*/
+
+{
+  //
+  // Unless we're told otherwise, don't quote unless we actually
+  // need to do so --- hopefully avoid problems if programs won't
+  // parse quotes properly
+  //
+
+  if (Force == false && Argument.empty() == false &&
+      Argument.find_first_of(" \t\n\v\"") == Argument.npos) {
+    CommandLine.append(Argument);
+  } else {
+    CommandLine.push_back('"');
+
+    for (auto It = Argument.begin();; ++It) {
+      unsigned NumberBackslashes = 0;
+
+      while (It != Argument.end() && *It == '\\') {
+        ++It;
+        ++NumberBackslashes;
+      }
+
+      if (It == Argument.end()) {
+        //
+        // Escape all backslashes, but let the terminating
+        // double quotation mark we add below be interpreted
+        // as a metacharacter.
+        //
+        CommandLine.append(NumberBackslashes * 2, '\\');
+        break;
+      } else if (*It == L'"') {
+        //
+        // Escape all backslashes and the following
+        // double quotation mark.
+        //
+        CommandLine.append(NumberBackslashes * 2 + 1, '\\');
+        CommandLine.push_back(*It);
+      } else {
+        //
+        // Backslashes aren't special here.
+        //
+        CommandLine.append(NumberBackslashes, '\\');
+        CommandLine.push_back(*It);
+      }
+    }
+
+    CommandLine.push_back('"');
+  }
+}
+
 int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) {
   if (childs.size() == MAXIMUM_WAIT_OBJECTS)
     failure("Could't manage more that %u actors on Windows\n",
@@ -184,7 +268,8 @@ int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) {
                                   &exename_size))
     failure_perror("QueryFullProcessImageName()", GetLastError());
 
-  std::string cmdline = "test_mdbx.child " + thunk_param(config);
+  std::string cmdline = "test_mdbx.child ";
+  ArgvQuote(cmdline, thunk_param(config));
 
   PROCESS_INFORMATION ProcessInformation;
   if (!CreateProcessA(exename, const_cast<char *>(cmdline.c_str()),

test/test.cc

@@ -85,9 +85,9 @@ static void mdbx_logger(int type, const char *function, int line,
     level = logging::failure;
   }
 
-  if (logging::output(level, strncmp(function, "mdbx_", 5) == 0 ? "%s: "
+  if (logging::output(
-                                                                : "mdbx: %s: ",
+          level,
-                      function))
+          strncmp(function, "mdbx_", 5) == 0 ? "%s: " : "mdbx: %s: ", function))
     logging::feed(msg, args);
   if (type & MDBX_DBG_ASSERT)
     abort();