ISAR/libmdbx
3
0
Fork 0
mirror of https://github.com/isar/libmdbx.git synced 2024-12-30 01:54:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

mdbx: merge branch master into devel.

Browse Source
This commit is contained in:
Леонид Юрьев (Leonid Yuriev) 2022-07-23 11:56:17 +03:00
commit 5e565433f7
6 changed files with 107 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CMakeLists.txt
View File

@ -569,7 +569,10 @@ else()
"${MDBX_SOURCE_DIR}/options.h" "${MDBX_SOURCE_DIR}/base.h"
"${MDBX_SOURCE_DIR}/internals.h" "${MDBX_SOURCE_DIR}/osal.h"
"${MDBX_SOURCE_DIR}/core.c" "${MDBX_SOURCE_DIR}/osal.c"
"${MDBX_SOURCE_DIR}/lck-posix.c" "${MDBX_SOURCE_DIR}/lck-windows.c")
"${MDBX_SOURCE_DIR}/lck-posix.c")
if(NOT APPLE)
list(APPEND LIBMDBX_SOURCES "${MDBX_SOURCE_DIR}/lck-windows.c")
endif()
include_directories("${MDBX_SOURCE_DIR}")
endif()
endif(MDBX_AMALGAMATED_SOURCE)

9
ChangeLog.md
View File

@ -29,14 +29,22 @@ Not a release but preparation for changing feature set and API.
The stable bugfix release.
It is planned that this will be the last release of the v0.11 branch.
Acknowledgements:
- [Alex Sharov](https://github.com/AskAlexSharov) and Erigon team for reporting and testing.
- [Andrew Ashikhmin](https://gitflic.ru/user/yperbasis) for contributing.
New:
- Ability to customise `MDBX_LOCK_SUFFIX`, `MDBX_DATANAME`, `MDBX_LOCKNAME` just by predefine ones during build.
- Added to [`mdbx::env_managed`](https://libmdbx.dqdkfa.ru/group__cxx__api.html#classmdbx_1_1env__managed)'s methods a few overloads with `const char* pathname` parameter (C++ API).
Fixes:
- Fixed hang copy-with-compactification of a corrupted DB
or in case the volume of output pages is a multiple of `MDBX_ENVCOPY_WRITEBUF`.
- Fixed standalone non-CMake build on MacOS (`#include AvailabilityMacros.h>`).
- Fixed unexpected `MDBX_PAGE_FULL` error in rare cases with large database page sizes.
Minors:
@ -47,6 +55,7 @@ Minors:
- Minor refine/fix batch-get testcase for large page size.
- Added `--pagesize NN` option to long-stotastic test script.
- Updated Valgrind-suppressions file for modern GCC.
- Fixed `has no symbols` warning from Apple's ranlib.
-------------------------------------------------------------------------------

31
mdbx.h++
View File

@ -3228,6 +3228,8 @@ public:
#endif /* Windows */
env &copy(const ::std::string &destination, bool compactify,
bool force_dynamic_size = false);
env &copy(const char *destination, bool compactify,
bool force_dynamic_size = false);
/// \brief Copy an environment to the specified file descriptor.
env &copy(filehandle fd, bool compactify, bool force_dynamic_size = false);
@ -3252,14 +3254,16 @@ public:
/// \brief Removes the environment's files in a proper and multiprocess-safe
/// way.
#ifdef MDBX_STD_FILESYSTEM_PATH
static bool remove(const MDBX_STD_FILESYSTEM_PATH &,
static bool remove(const MDBX_STD_FILESYSTEM_PATH &pathname,
const remove_mode mode = just_remove);
#endif /* MDBX_STD_FILESYSTEM_PATH */
#if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN)
static bool remove(const ::std::wstring &,
static bool remove(const ::std::wstring &pathname,
const remove_mode mode = just_remove);
#endif /* Windows */
static bool remove(const ::std::string &,
static bool remove(const ::std::string &pathname,
const remove_mode mode = just_remove);
static bool remove(const char *pathname,
const remove_mode mode = just_remove);
/// \brief Statistics for a database in the MDBX environment.
@ -3497,15 +3501,17 @@ public:
/// \brief Open existing database.
#ifdef MDBX_STD_FILESYSTEM_PATH
env_managed(const MDBX_STD_FILESYSTEM_PATH &, const operate_parameters &,
bool accede = true);
env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname,
const operate_parameters &, bool accede = true);
#endif /* MDBX_STD_FILESYSTEM_PATH */
#if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN)
env_managed(const ::std::wstring &, const operate_parameters &,
env_managed(const ::std::wstring &pathname, const operate_parameters &,
bool accede = true);
#endif /* Windows */
env_managed(const ::std::string &, const operate_parameters &,
env_managed(const ::std::string &pathname, const operate_parameters &,
bool accede = true);
explicit env_managed(const char *pathname, const operate_parameters &,
bool accede = true);
/// \brief Additional parameters for creating a new database.
struct create_parameters {
@ -3518,15 +3524,18 @@ public:
/// \brief Create new or open existing database.
#ifdef MDBX_STD_FILESYSTEM_PATH
env_managed(const MDBX_STD_FILESYSTEM_PATH &, const create_parameters &,
const operate_parameters &, bool accede = true);
env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname,
const create_parameters &, const operate_parameters &,
bool accede = true);
#endif /* MDBX_STD_FILESYSTEM_PATH */
#if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN)
env_managed(const ::std::wstring &, const create_parameters &,
env_managed(const ::std::wstring &pathname, const create_parameters &,
const operate_parameters &, bool accede = true);
#endif /* Windows */
env_managed(const ::std::string &, const create_parameters &,
env_managed(const ::std::string &pathname, const create_parameters &,
const operate_parameters &, bool accede = true);
explicit env_managed(const char *pathname, const create_parameters &,
const operate_parameters &, bool accede = true);
/// \brief Explicitly closes the environment and release the memory map.
///

1
src/base.h
View File

@ -158,6 +158,7 @@
#endif
#ifdef __APPLE__
#include <AvailabilityMacros.h>
#ifndef MAC_OS_X_VERSION_MIN_REQUIRED
#define MAC_OS_X_VERSION_MIN_REQUIRED 1070 /* Mac OS X 10.7, 2011 */
#endif

85
src/core.c
View File

@ -660,6 +660,7 @@ page_room(const MDBX_page *mp) {
return mp->mp_upper - mp->mp_lower;
}
/* Maximum free space in an empty page */
MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned
page_space(const MDBX_env *env) {
STATIC_ASSERT(PAGEHDRSZ % 2 == 0);
@ -17630,6 +17631,7 @@ static int mdbx_node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) {
} break;
default:
assert(false);
goto bailout;
}
@ -19047,6 +19049,7 @@ static int mdbx_page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
(newindx < nkeys)
? /* split at the middle */ (nkeys + 1) / 2
: /* split at the end (i.e. like append-mode ) */ nkeys - minkeys + 1;
mdbx_assert(env, split_indx >= minkeys && split_indx <= nkeys - minkeys + 1);
mdbx_cassert(mc, !IS_BRANCH(mp) || newindx > 0);
/* It is reasonable and possible to split the page at the begin */
@ -19144,11 +19147,6 @@ static int mdbx_page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
goto done;
}
} else {
/* Maximum free space in an empty page */
const unsigned max_space = page_space(env);
const size_t new_size = IS_LEAF(mp) ? leaf_size(env, newkey, newdata)
: branch_size(env, newkey);
/* grab a page to hold a temporary copy */
tmp_ki_copy = mdbx_page_malloc(mc->mc_txn, 1);
if (unlikely(tmp_ki_copy == NULL)) {
@ -19156,6 +19154,10 @@ static int mdbx_page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
goto done;
}
const unsigned max_space = page_space(env);
const size_t new_size = IS_LEAF(mp) ? leaf_size(env, newkey, newdata)
: branch_size(env, newkey);
/* prepare to insert */
for (unsigned j = i = 0; i < nkeys; ++i, ++j) {
tmp_ki_copy->mp_ptrs[j] = 0;
@ -19168,34 +19170,35 @@ static int mdbx_page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
tmp_ki_copy->mp_lower = 0;
tmp_ki_copy->mp_upper = (indx_t)max_space;
/* When items are relatively large the split point needs
* to be checked, because being off-by-one will make the
* difference between success or failure in mdbx_node_add.
/* Добавляемый узел может не поместиться в страницу-половину вместе
* с количественной половиной узлов из исходной страницы. В худшем случае,
* в страницу-половину с добавляемым узлом могут попасть самые больше узлы
* из исходной страницы, а другую половину только узлы с самыми короткими
* ключами и с пустыми данными. Поэтому, чтобы найти подходящую границу
* разреза требуется итерировать узлы и считая их объем.
*
* It's also relevant if a page happens to be laid out
* such that one half of its nodes are all "small" and
* the other half of its nodes are "large". If the new
* item is also "large" and falls on the half with
* "large" nodes, it also may not fit.
*
* As a final tweak, if the new item goes on the last
* spot on the page (and thus, onto the new page), bias
* the split so the new page is emptier than the old page.
* This yields better packing during sequential inserts. */
* Однако, при простом количественном делении (без учета размера ключей
* и данных) на страницах-половинах будет примерно вдвое меньше узлов.
* Поэтому добавляемый узел точно поместится, если его размер не больше
* чем место "освобождающееся" от заголовков узлов, которые переедут
* в другую страницу-половину. Кроме этого, как минимум по одному байту
* будет в каждом ключе, в худшем случае кроме одного, который может быть
* нулевого размера. */
if (nkeys < 32 || new_size > max_space / 16) {
if (newindx == split_indx && split_indx + minkeys <= nkeys)
split_indx += 1;
mdbx_assert(env,
split_indx >= minkeys && split_indx <= nkeys - minkeys + 1);
const unsigned dim_nodes =
(newindx >= split_indx) ? split_indx : nkeys - split_indx;
const unsigned dim_used = (sizeof(indx_t) + NODESIZE + 1) * dim_nodes;
if (new_size >= dim_used) {
/* Find split point */
int dir;
if (newindx <= split_indx) {
i = 0;
dir = 1;
} else {
i = nkeys;
dir = -1;
}
i = (newindx < split_indx) ? 0 : nkeys;
int dir = (newindx < split_indx) ? 1 : -1;
size_t before = 0, after = new_size + page_used(env, mp);
int best = split_indx;
int best_offset = nkeys + 1;
unsigned best_split = split_indx;
unsigned best_offset = INT_MAX;
mdbx_trace("seek separator from %u, step %i, default %u, new-idx %u, "
"new-size %zu",
@ -19208,8 +19211,8 @@ static int mdbx_page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
(MDBX_node *)((char *)mp + tmp_ki_copy->mp_ptrs[i] + PAGEHDRSZ);
size = NODESIZE + node_ks(node) + sizeof(indx_t);
if (IS_LEAF(mp))
size += F_ISSET(node_flags(node), F_BIGDATA) ? sizeof(pgno_t)
: node_ds(node);
size += (node_flags(node) & F_BIGDATA) ? sizeof(pgno_t)
: node_ds(node);
size = EVEN(size);
}
@ -19219,21 +19222,23 @@ static int mdbx_page_split(MDBX_cursor *mc, const MDBX_val *const newkey,
size, before, after, max_space);
if (before <= max_space && after <= max_space) {
int offset = branchless_abs(split_indx - i);
if (offset >= best_offset)
break;
best_offset = offset;
best = i;
const unsigned split = i + (dir > 0);
if (split >= minkeys && split <= nkeys + 1 - minkeys) {
const unsigned offset = branchless_abs(split_indx - split);
if (offset >= best_offset)
break;
best_offset = offset;
best_split = split;
}
}
i += dir;
} while (i < nkeys);
split_indx = best + (dir > 0);
split_indx = (split_indx <= nkeys - minkeys + 1) ? split_indx
: nkeys - minkeys + 1;
split_indx = (split_indx >= minkeys) ? split_indx : minkeys;
split_indx = best_split;
mdbx_trace("chosen %u", split_indx);
}
mdbx_assert(env,
split_indx >= minkeys && split_indx <= nkeys - minkeys + 1);
sepkey.iov_len = newkey->iov_len;
sepkey.iov_base = newkey->iov_base;

48
src/mdbx.c++
View File

@ -214,12 +214,6 @@ MDBX_MAYBE_UNUSED PATH pchar_to_path(const char *c_str) {
return PATH(c_str);
}
template <> struct path_to_pchar<std::string> {
const char *const ptr;
path_to_pchar(const std::string &path) : ptr(path.c_str()) {}
operator const char *() const { return ptr; }
};
#if defined(_WIN32) || defined(_WIN64)
#ifndef WC_ERR_INVALID_CHARS
@ -1271,17 +1265,21 @@ env &env::copy(const ::std::wstring &destination, bool compactify,
}
#endif /* Windows */
env &env::copy(const ::std::string &destination, bool compactify,
env &env::copy(const char *destination, bool compactify,
bool force_dynamic_size) {
const path_to_pchar<::std::string> utf8(destination);
error::success_or_throw(
::mdbx_env_copy(handle_, utf8,
::mdbx_env_copy(handle_, destination,
(compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) |
(force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE
: MDBX_CP_DEFAULTS)));
return *this;
}
env &env::copy(const ::std::string &destination, bool compactify,
bool force_dynamic_size) {
return copy(destination.c_str(), compactify, force_dynamic_size);
}
env &env::copy(filehandle fd, bool compactify, bool force_dynamic_size) {
error::success_or_throw(
::mdbx_env_copy2fd(handle_, fd,
@ -1314,10 +1312,13 @@ bool env::remove(const ::std::wstring &pathname, const remove_mode mode) {
}
#endif /* Windows */
bool env::remove(const ::std::string &pathname, const remove_mode mode) {
const path_to_pchar<::std::string> utf8(pathname);
bool env::remove(const char *pathname, const remove_mode mode) {
return error::boolean_or_throw(
::mdbx_env_delete(utf8, MDBX_env_delete_mode_t(mode)));
::mdbx_env_delete(pathname, MDBX_env_delete_mode_t(mode)));
}
bool env::remove(const ::std::string &pathname, const remove_mode mode) {
return remove(pathname.c_str(), mode);
}
//------------------------------------------------------------------------------
@ -1418,35 +1419,42 @@ __cold env_managed::env_managed(const ::std::wstring &pathname,
}
#endif /* Windows */
__cold env_managed::env_managed(const ::std::string &pathname,
__cold env_managed::env_managed(const char *pathname,
const operate_parameters &op, bool accede)
: env_managed(create_env()) {
setup(op.max_maps, op.max_readers);
const path_to_pchar<::std::string> utf8(pathname);
error::success_or_throw(
::mdbx_env_open(handle_, utf8, op.make_flags(accede), 0));
::mdbx_env_open(handle_, pathname, op.make_flags(accede), 0));
if (op.options.nested_write_transactions &&
!get_options().nested_write_transactions)
MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
}
__cold env_managed::env_managed(const ::std::string &pathname,
__cold env_managed::env_managed(const char *pathname,
const env_managed::create_parameters &cp,
const env::operate_parameters &op, bool accede)
: env_managed(create_env()) {
setup(op.max_maps, op.max_readers);
const path_to_pchar<::std::string> utf8(pathname);
set_geometry(cp.geometry);
error::success_or_throw(
::mdbx_env_open(handle_, utf8, op.make_flags(accede, cp.use_subdirectory),
cp.file_mode_bits));
error::success_or_throw(::mdbx_env_open(
handle_, pathname, op.make_flags(accede, cp.use_subdirectory),
cp.file_mode_bits));
if (op.options.nested_write_transactions &&
!get_options().nested_write_transactions)
MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE);
}
__cold env_managed::env_managed(const ::std::string &pathname,
const operate_parameters &op, bool accede)
: env_managed(pathname.c_str(), op, accede) {}
__cold env_managed::env_managed(const ::std::string &pathname,
const env_managed::create_parameters &cp,
const env::operate_parameters &op, bool accede)
: env_managed(pathname.c_str(), cp, op, accede) {}
//------------------------------------------------------------------------------
txn_managed txn::start_nested() {