From ddc378793639f7f657ecb11f702954d89ddf7afd Mon Sep 17 00:00:00 2001
From: Leo Yuriev <leo@yuriev.ru>
Date: Wed, 14 Jun 2017 23:34:11 +0300
Subject: [PATCH] mdbx: fix MDBX_RESULT_TRUE handling inside
 mdbx_mutex_failed()...mdbx_oomkick().

---
 src/lck-posix.c |  2 ++
 src/mdbx.c      | 47 ++++++++++++++++++++++++++++-------------------
 2 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/src/lck-posix.c b/src/lck-posix.c
index 0d79d932..c1f23b38 100644
--- a/src/lck-posix.c
+++ b/src/lck-posix.c
@@ -295,6 +295,8 @@ static int __cold mdbx_mutex_failed(MDBX_env *env, pthread_mutex_t *mutex,
                 (rc ? "this process' env is hosed" : "recovering"));
 
     int check_rc = mdbx_reader_check0(env, rlocked, NULL);
+    check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc;
+
     int mreco_rc = pthread_mutex_consistent(mutex);
     check_rc = (mreco_rc == 0) ? check_rc : mreco_rc;
 
diff --git a/src/mdbx.c b/src/mdbx.c
index 462564db..c41eb6ae 100644
--- a/src/mdbx.c
+++ b/src/mdbx.c
@@ -9780,6 +9780,10 @@ int __cold mdbx_reader_check(MDBX_env *env, int *dead) {
   return mdbx_reader_check0(env, 0, dead);
 }
 
+/* Return:
+ *  MDBX_RESULT_TRUE - done and mutex recovered
+ *  MDBX_SUCCESS     - done
+ *  Otherwise errcode. */
 int __cold mdbx_reader_check0(MDBX_env *env, int rdt_locked, int *dead) {
   assert(rdt_locked >= 0);
 
@@ -9792,7 +9796,7 @@ int __cold mdbx_reader_check0(MDBX_env *env, int rdt_locked, int *dead) {
   mdbx_pid_t *pids = alloca((snap_nreaders + 1) * sizeof(mdbx_pid_t));
   pids[0] = 0;
 
-  int rc = MDBX_RESULT_FALSE, count = 0;
+  int rc = MDBX_SUCCESS, count = 0;
   MDBX_reader *mr = env->me_lck->mti_readers;
 
   for (unsigned i = 0; i < snap_nreaders; i++) {
@@ -9802,39 +9806,44 @@ int __cold mdbx_reader_check0(MDBX_env *env, int rdt_locked, int *dead) {
     if (pid != env->me_pid)
       continue;
     if (mdbx_pid_insert(pids, pid) != 0)
-      continue;
+      continue /* such pid already processed */;
 
-    rc = mdbx_rpid_check(env, pid);
-    if (rc == MDBX_RESULT_TRUE)
-      continue; /* reader is live */
+    int err = mdbx_rpid_check(env, pid);
+    if (err == MDBX_RESULT_TRUE)
+      continue /* reader is live */;
 
-    if (rc != MDBX_RESULT_FALSE)
-      break; /* mdbx_rpid_check() failed */
+    if (err != MDBX_SUCCESS) {
+      rc = err;
+      break /* mdbx_rpid_check() failed */;
+    }
 
     /* stale reader found */
     if (!rdt_locked) {
-      rc = mdbx_rdt_lock(env);
-      if (MDBX_IS_ERROR(rc))
+      err = mdbx_rdt_lock(env);
+      if (MDBX_IS_ERROR(rc)) {
+        rc = err;
         break;
+      }
 
       rdt_locked = -1;
-      if (rc == MDBX_RESULT_TRUE)
-        /* the above checked all readers */
+      if (err == MDBX_RESULT_TRUE) {
+        /* mutex recovered, the mdbx_mutex_failed() checked all readers */
+        rc = MDBX_RESULT_TRUE;
         break;
+      }
 
       /* a other process may have clean and reused slot, recheck */
       if (mr[i].mr_pid != pid)
         continue;
 
-      rc = mdbx_rpid_check(env, pid);
-      if (MDBX_IS_ERROR(rc))
+      err = mdbx_rpid_check(env, pid);
+      if (MDBX_IS_ERROR(rc)) {
+        rc = err;
         break;
-
-      if (rc != MDBX_RESULT_FALSE) {
-        /* the race with other process, slot reused */
-        rc = MDBX_RESULT_FALSE;
-        continue;
       }
+
+      if (err != MDBX_SUCCESS)
+        continue /* the race with other process, slot reused */;
     }
 
     /* clean it */
@@ -9896,7 +9905,7 @@ static txnid_t __cold mdbx_oomkick(MDBX_env *env, txnid_t oldest) {
   for (retry = 0; retry < INT_MAX; ++retry) {
     int reader;
 
-    if (mdbx_reader_check(env, NULL))
+    if (MDBX_IS_ERROR(mdbx_reader_check0(env, false, NULL)))
       break;
 
     txnid_t snap = mdbx_find_oldest(env->me_txn, &reader);