ISAR/libmdbx
3
0
Fork 0
mirror of https://github.com/isar/libmdbx.git synced 2025-02-01 04:38:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

mdbx: merge branch 'devel' (crutches for Wine).

Browse Source 
Change-Id: I1779bc6987449e403216584cdd2846e910f34c8a
This commit is contained in:
Leo Yuriev 2020-02-28 19:05:00 +03:00
commit 76e29c21f1
8 changed files with 92 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
GNUmakefile
View File

@ -236,10 +236,7 @@ dist/@tmp-shared_internals.inc: src/elements/version.c $(ALLOY_DEPS) $(lastword
dist/mdbx.c: dist/@tmp-shared_internals.inc $(lastword $(MAKEFILE_LIST))
mkdir -p dist && (cat dist/@tmp-shared_internals.inc \
&& cat src/elements/core.c src/elements/osal.c src/elements/version.c \
&& echo '#if defined(_WIN32) || defined(_WIN64)' \
&& cat src/elements/lck-windows.c && echo '#else /* LCK-implementation */' \
&& cat src/elements/lck-posix.c && echo '#endif /* LCK-implementation */' \
&& cat src/elements/core.c src/elements/osal.c src/elements/version.c src/elements/lck-windows.c src/elements/lck-posix.c \
) | grep -v -e '#include "' -e '#pragma once' | sed 's|@INCLUDE|#include|' > $@
define dist-tool-rule

43
mdbx.h
View File

@ -1454,9 +1454,14 @@ typedef enum MDBX_cursor_op {
/* Page has not enough space - internal error */
#define MDBX_PAGE_FULL (-30786)
/* Database contents grew beyond environment mapsize and engine was
* unable to extend mapping, e.g. since address space is unavailable or busy */
#define MDBX_MAP_RESIZED (-30785)
/* Database engine was unable to extend mapping, e.g. since address space
* is unavailable or busy. This can mean:
* - Database size extended by other process beyond to environment mapsize
* and engine was unable to extend mapping while starting read transaction.
* Environment should be reopened to continue.
* - Engine was unable to extend mapping during write transaction
* or explicit call of mdbx_env_set_geometry(). */
#define MDBX_UNABLE_EXTEND_MAPSIZE (-30785)
/* Environment or database is not compatible with the requested operation
* or the specified flags. This can mean:
@ -2028,9 +2033,9 @@ LIBMDBX_API int mdbx_env_get_fd(MDBX_env *env, mdbx_filehandle_t *fd);
* size. Besides, the upper bound defines the linear address space
* reservation in each process that opens the database. Therefore changing
* the upper bound is costly and may be required reopening environment in
* case of MDBX_MAP_RESIZED errors, and so on. Therefore, this value should
* be chosen reasonable as large as possible, to accommodate future growth
* of the database.
* case of MDBX_UNABLE_EXTEND_MAPSIZE errors, and so on. Therefore, this
* value should be chosen reasonable as large as possible, to accommodate
* future growth of the database.
* - The growth step must be greater than zero to allow the database to grow,
* but also reasonable not too small, since increasing the size by little
* steps will result a large overhead.
@ -2050,6 +2055,7 @@ LIBMDBX_API int mdbx_env_get_fd(MDBX_env *env, mdbx_filehandle_t *fd);
* - Windows does not provide the usual API to augment a memory-mapped file
* (that is, a memory-mapped partition), but only by using "Native API"
* in an undocumented way.
*
* MDBX bypasses all Windows issues, but at a cost:
* - Ability to resize database on the fly requires an additional lock
* and release SlimReadWriteLock during each read-only transaction.
@ -2070,10 +2076,10 @@ LIBMDBX_API int mdbx_env_get_fd(MDBX_env *env, mdbx_filehandle_t *fd);
*
* If the mapsize is increased by another process, MDBX silently and
* transparently adopt these changes at next transaction start. However,
* mdbx_txn_begin() will return MDBX_MAP_RESIZED if new mapping size could not
* be applied for current process (for instance if address space is busy).
* Therefore, in the case of MDBX_MAP_RESIZED error you need close and reopen
* the environment to resolve error.
* mdbx_txn_begin() will return MDBX_UNABLE_EXTEND_MAPSIZE if new mapping size
* could not be applied for current process (for instance if address space
* is busy). Therefore, in the case of MDBX_UNABLE_EXTEND_MAPSIZE error you
* need close and reopen the environment to resolve error.
*
* NOTE: Actual values may be different than your have specified because of
* rounding to specified database page size, the system page size and/or the
@ -2104,13 +2110,13 @@ LIBMDBX_API int mdbx_env_get_fd(MDBX_env *env, mdbx_filehandle_t *fd);
* database is used by other processes or threaded
* (i.e. just pass -1 in this argument except absolutely
* necessity). Otherwise you must be ready for
* MDBX_MAP_RESIZED error(s), unexpected pauses during
* remapping and/or system errors like "addtress busy",
* and so on. In other words, there is no way to handle
* a growth of the upper bound robustly because there may
* be a lack of appropriate system resources (which are
* extremely volatile in a multi-process multi-threaded
* environment).
* MDBX_UNABLE_EXTEND_MAPSIZE error(s), unexpected pauses
* during remapping and/or system errors like "addtress
* busy", and so on. In other words, there is no way to
* handle a growth of the upper bound robustly because
* there may be a lack of appropriate system resources
* (which are extremely volatile in a multi-process
* multi-threaded environment).
*
* [in] growth_step The growth step in bytes, must be greater than zero
* to allow the database to grow.
@ -2302,7 +2308,8 @@ LIBMDBX_API void *mdbx_env_get_userctx(MDBX_env *env);
* possible errors are:
* - MDBX_PANIC = a fatal error occurred earlier and the environment
* must be shut down.
* - MDBX_MAP_RESIZED = another process wrote data beyond this MDBX_env's
* - MDBX_UNABLE_EXTEND_MAPSIZE
* = another process wrote data beyond this MDBX_env's
* mapsize and this environment's map must be resized
* as well. See mdbx_env_set_mapsize().
* - MDBX_READERS_FULL = a read-only transaction was requested and the reader

7
src/CMakeLists.txt
View File

@ -19,14 +19,9 @@ message(STATUS "libmdbx version is ${MDBX_VERSION}")
if(MDBX_ALLOY_MODE)
set(LIBMDBX_SOURCES alloy.c)
else()
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
set(LIBMDBX_OSAL windows)
else()
set(LIBMDBX_OSAL posix)
endif()
set(LIBMDBX_SOURCES
elements/options.h elements/defs.h elements/internals.h elements/osal.h
elements/core.c elements/osal.c elements/lck-${LIBMDBX_OSAL}.c)
elements/core.c elements/osal.c elements/lck-posix.c elements/lck-windows.c)
endif()
list(APPEND LIBMDBX_SOURCES ../mdbx.h
"${CMAKE_CURRENT_SOURCE_DIR}/elements/version.c"

29
src/elements/core.c
View File

@ -3184,12 +3184,12 @@ static const char *__mdbx_strerr(int errnum) {
"MDBX_READERS_FULL: Too many readers (maxreaders reached)",
NULL /* MDBX_TLS_FULL (-30789): unused in MDBX */,
"MDBX_TXN_FULL: Transaction has too many dirty pages,"
" i.e transaction too big",
" i.e transaction is too big",
"MDBX_CURSOR_FULL: Internal error - Cursor stack limit reached",
"MDBX_PAGE_FULL: Internal error - Page has no more space",
"MDBX_MAP_RESIZED: Database contents grew beyond environment mapsize"
" and engine was unable to extend mapping,"
" e.g. since address space is unavailable or busy",
"MDBX_UNABLE_EXTEND_MAPSIZE: Database engine was unable to extend"
" mapping, e.g. since address space is unavailable or busy,"
" or Operation system not supported such operations"
"MDBX_INCOMPATIBLE: Environment or database is not compatible"
" with the requested operation or the specified flags",
"MDBX_BAD_RSLOT: Invalid reuse of reader locktable slot,"
@ -4662,6 +4662,8 @@ static __cold int mdbx_mapresize(MDBX_env *env, const pgno_t used_pgno,
* the local threads for safe remap.
* 2) At least on Windows 10 1803 the entire mapped section is unavailable
* for short time during NtExtendSection() or VirtualAlloc() execution.
* 3) Under Wine runtime environment on Linux a section extending is not
* supported. Therefore thread suspending is always required.
*
* THEREFORE LOCAL THREADS SUSPENDING IS ALWAYS REQUIRED! */
array_onstack.limit = ARRAY_LENGTH(array_onstack.handles);
@ -4760,7 +4762,7 @@ bailout:
}
#endif /* MDBX_USE_VALGRIND */
} else {
if (rc != MDBX_RESULT_TRUE) {
if (rc != MDBX_UNABLE_EXTEND_MAPSIZE) {
mdbx_error("failed resize datafile/mapping: "
"present %" PRIuPTR " -> %" PRIuPTR ", "
"limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d",
@ -5248,7 +5250,7 @@ skip_cache:
rc = MDBX_NOTFOUND;
if (flags & MDBX_ALLOC_NEW) {
rc = MDBX_MAP_FULL;
if (next <= txn->mt_geo.upper) {
if (next <= txn->mt_geo.upper && txn->mt_geo.grow) {
mdbx_assert(env, next > txn->mt_end_pgno);
pgno_t aligned = pgno_align2os_pgno(
env, pgno_add(next, txn->mt_geo.grow - next % txn->mt_geo.grow));
@ -5269,7 +5271,6 @@ skip_cache:
mdbx_error("unable growth datafile to %" PRIaPGNO " pages (+%" PRIaPGNO
"), errcode %d",
aligned, aligned - txn->mt_end_pgno, rc);
rc = (rc == MDBX_RESULT_TRUE) ? MDBX_MAP_FULL : rc;
} else {
mdbx_debug("gc-alloc: next %u > upper %u", next, txn->mt_geo.upper);
}
@ -6049,20 +6050,22 @@ static int mdbx_txn_renew0(MDBX_txn *txn, unsigned flags) {
if (txn->mt_geo.upper > MAX_PAGENO ||
bytes2pgno(env, pgno2bytes(env, txn->mt_geo.upper)) !=
txn->mt_geo.upper) {
rc = MDBX_MAP_RESIZED;
rc = MDBX_UNABLE_EXTEND_MAPSIZE;
goto bailout;
}
rc = mdbx_mapresize(env, txn->mt_next_pgno, txn->mt_end_pgno,
txn->mt_geo.upper);
if (rc != MDBX_SUCCESS) {
if (rc == MDBX_RESULT_TRUE)
rc = MDBX_MAP_RESIZED;
if (rc != MDBX_SUCCESS)
goto bailout;
}
}
if (txn->mt_flags & MDBX_RDONLY) {
#if defined(_WIN32) || defined(_WIN64)
if (size > env->me_dbgeo.lower && env->me_dbgeo.shrink) {
if ((size > env->me_dbgeo.lower && env->me_dbgeo.shrink) ||
(mdbx_RunningUnderWine() &&
/* under Wine acquisition of remap_guard is always required,
* since Wine don't support section extending,
* i.e. in both cases unmap+map are required. */
size < env->me_dbgeo.upper && env->me_dbgeo.grow)) {
txn->mt_flags |= MDBX_SHRINK_ALLOWED;
mdbx_srwlock_AcquireShared(&env->me_remap_guard);
}

9
src/elements/lck-posix.c
View File

@ -12,6 +12,8 @@
* <http://www.OpenLDAP.org/license.html>.
*/
#if !(defined(_WIN32) || defined(_WIN64)) /* !Windows LCK-implementation */
#include "internals.h"
#include <sys/sem.h>
@ -803,3 +805,10 @@ void mdbx_txn_unlock(MDBX_env *env) {
mdbx_panic("%s() failed: err %d\n", __func__, rc);
mdbx_jitter4testing(true);
}
#else
#ifdef _MSC_VER
#pragma warning(disable : 4206) /* nonstandard extension used: translation \
unit is empty */
#endif /* _MSC_VER (warnings) */
#endif /* !Windows LCK-implementation */

51
src/elements/lck-windows.c
View File

@ -12,17 +12,15 @@
* <http://www.OpenLDAP.org/license.html>.
*/
#include "internals.h"
#if defined(_WIN32) || defined(_WIN64) /* Windows LCK-implementation */
/* PREAMBLE FOR WINDOWS:
*
* We are not concerned for performance here.
* If you are running Windows a performance could NOT be the goal.
* Otherwise please use Linux.
*
* Regards,
* LY
*/
* Otherwise please use Linux. */
#include "internals.h"
static void mdbx_winnt_import(void);
@ -367,22 +365,24 @@ mdbx_resume_threads_after_remap(mdbx_handle_array_t *array) {
*/
static void lck_unlock(MDBX_env *env) {
int rc;
int err;
if (env->me_lfd != INVALID_HANDLE_VALUE) {
/* double `unlock` for robustly remove overlapped shared/exclusive locks */
while (funlock(env->me_lfd, LCK_LOWER))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
err = GetLastError();
assert(err == ERROR_NOT_LOCKED ||
(mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION));
(void)err;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_lfd, LCK_UPPER))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
err = GetLastError();
assert(err == ERROR_NOT_LOCKED ||
(mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION));
(void)err;
SetLastError(ERROR_SUCCESS);
}
@ -391,23 +391,26 @@ static void lck_unlock(MDBX_env *env) {
* releases such locks via deferred queues) */
while (funlock(env->me_lazy_fd, LCK_BODY))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
err = GetLastError();
assert(err == ERROR_NOT_LOCKED ||
(mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION));
(void)err;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_lazy_fd, LCK_META))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
err = GetLastError();
assert(err == ERROR_NOT_LOCKED ||
(mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION));
(void)err;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_lazy_fd, LCK_WHOLE))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
err = GetLastError();
assert(err == ERROR_NOT_LOCKED ||
(mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION));
(void)err;
SetLastError(ERROR_SUCCESS);
}
}
@ -748,11 +751,11 @@ static void mdbx_winnt_import(void) {
const HINSTANCE hKernel32dll = GetModuleHandleA("kernel32.dll");
GET_PROC_ADDR(hKernel32dll, GetFileInformationByHandleEx);
GET_PROC_ADDR(hKernel32dll, SetFileInformationByHandle);
GET_PROC_ADDR(hKernel32dll, GetTickCount64);
if (!mdbx_GetTickCount64)
mdbx_GetTickCount64 = stub_GetTickCount64;
if (!mdbx_RunningUnderWine()) {
GET_PROC_ADDR(hKernel32dll, SetFileInformationByHandle);
GET_PROC_ADDR(hKernel32dll, GetVolumeInformationByHandleW);
GET_PROC_ADDR(hKernel32dll, GetFinalPathNameByHandleW);
GET_PROC_ADDR(hKernel32dll, PrefetchVirtualMemory);
@ -794,3 +797,5 @@ static void mdbx_winnt_import(void) {
mdbx_srwlock_ReleaseExclusive = stub_srwlock_ReleaseExclusive;
}
}
#endif /* Windows LCK-implementation */

17
src/elements/osal.c
View File

@ -1300,7 +1300,8 @@ MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map,
if (!NT_SUCCESS(err))
return ntstatus2errcode(err);
SIZE_T ViewSize = (flags & MDBX_RDONLY) ? 0 : limit;
SIZE_T ViewSize =
(flags & MDBX_RDONLY) ? 0 : mdbx_RunningUnderWine() ? size : limit;
err = NtMapViewOfSection(
map->section, GetCurrentProcess(), &map->address,
/* ZeroBits */ 0,
@ -1403,7 +1404,7 @@ MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t size,
if (!(flags & MDBX_RDONLY) && limit == map->limit && size > map->current) {
/* growth rw-section */
if (!mdbx_NtExtendSection)
return ERROR_CALL_NOT_IMPLEMENTED /* workaround for Wine */;
return MDBX_UNABLE_EXTEND_MAPSIZE /* workaround for Wine */;
SectionSize.QuadPart = size;
status = mdbx_NtExtendSection(map->section, &SectionSize);
if (!NT_SUCCESS(status))
@ -1421,7 +1422,7 @@ MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t size,
status = NtAllocateVirtualMemory(GetCurrentProcess(), &BaseAddress, 0,
&RegionSize, MEM_RESERVE, PAGE_NOACCESS);
if (status == /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018)
return MDBX_RESULT_TRUE;
return MDBX_UNABLE_EXTEND_MAPSIZE;
if (!NT_SUCCESS(status))
return ntstatus2errcode(status);
@ -1539,8 +1540,8 @@ retry_mapview:;
if (map->address && (size != map->current || limit != map->limit)) {
/* try remap with previously size and limit,
* but will return MDBX_RESULT_TRUE on success */
rc = MDBX_RESULT_TRUE;
* but will return MDBX_UNABLE_EXTEND_MAPSIZE on success */
rc = MDBX_UNABLE_EXTEND_MAPSIZE;
size = map->current;
limit = map->limit;
goto retry_file_and_section;
@ -1563,7 +1564,7 @@ retry_mapview:;
if (flags & MDBX_RDONLY) {
map->current = (filesize > limit) ? limit : (size_t)filesize;
if (map->current != size)
rc = MDBX_RESULT_TRUE;
rc = MDBX_UNABLE_EXTEND_MAPSIZE;
} else if (filesize != size) {
rc = mdbx_ftruncate(map->fd, size);
if (rc != MDBX_SUCCESS)
@ -1580,7 +1581,7 @@ retry_mapview:;
case EAGAIN:
case ENOMEM:
case EFAULT /* MADV_DODUMP / MADV_DONTDUMP are mixed for mmap-range */:
rc = MDBX_RESULT_TRUE;
rc = MDBX_UNABLE_EXTEND_MAPSIZE;
}
return rc;
}
@ -1599,7 +1600,7 @@ retry_mapview:;
#else /* MREMAP_MAYMOVE */
/* TODO: Perhaps here it is worth to implement suspend/resume threads
* and perform unmap/map as like for Windows. */
rc = MDBX_RESULT_TRUE;
rc = MDBX_UNABLE_EXTEND_MAPSIZE;
#endif /* !MREMAP_MAYMOVE */
}
#endif

6
test/jitter.cc
View File

@ -53,7 +53,7 @@ bool testcase_jitter::run() {
err = mdbx_env_set_geometry(
db_guard.get(), -1, -1,
coin4size ? upper_limit * 2 / 3 : upper_limit * 3 / 2, -1, -1, -1);
if (err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE &&
if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE &&
err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE)
failure_perror("mdbx_env_set_geometry-1", err);
}
@ -62,7 +62,7 @@ bool testcase_jitter::run() {
err = mdbx_env_set_geometry(
db_guard.get(), -1, -1,
!coin4size ? upper_limit * 2 / 3 : upper_limit * 3 / 2, -1, -1, -1);
if (err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE &&
if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE &&
err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE)
failure_perror("mdbx_env_set_geometry-2", err);
@ -76,7 +76,7 @@ bool testcase_jitter::run() {
jitter_delay();
err =
mdbx_env_set_geometry(db_guard.get(), -1, -1, upper_limit, -1, -1, -1);
if (err != MDBX_SUCCESS && err != MDBX_RESULT_TRUE &&
if (err != MDBX_SUCCESS && err != MDBX_UNABLE_EXTEND_MAPSIZE &&
err != MDBX_MAP_FULL && err != MDBX_TOO_LARGE)
failure_perror("mdbx_env_set_geometry-3", err);