Compare commits

..

43 Commits

Author SHA1 Message Date
Leo Yuriev
f4a9018690 mdbx: bump internal version info to v0.1.3
Change-Id: I7db311c812d531f6628101715f6658005db3ea24
2018-04-03 20:37:07 +03:00
Leo Yuriev
f8eb423d36 mdbx-make: remote '-O3' for Elbrus (done by __hot attributes).
Change-Id: I6a3afeaf1ddf6d231941aae023dff89046bc4349
2018-04-03 20:06:45 +03:00
Leo Yuriev
797ae9d1db mdbx: minor fix debug-output for mdbx_thread_key_create().
Change-Id: I718130cf2385b6221187cbcdeedd9d48d05289bb
2018-04-03 20:04:10 +03:00
Leo Yuriev
f08c7ccac0 mdbx-make: rename mdbx_test and link with dso-library.
Change-Id: I39a04f82bc31c0865a8d02379a596048518205cd
2018-04-03 19:28:12 +03:00
Leo Yuriev
864be54f01 mdbx: fix/rework rthc counting.
Change-Id: I97fd5bd6dc1a46658138d82db10e937c9595d81f
2018-04-03 19:28:12 +03:00
Leo Yuriev
571b50622e mdbx: restore workaround for glibc's bug #21031.
Workaround for https://sourceware.org/bugzilla/show_bug.cgi?id=21031

Change-Id: I6cf9e037cc2fc298096b78ec96773f19478ed5c0
2018-04-03 17:44:18 +03:00
Leo Yuriev
36a86bbc6e mdbx: fix cursor tracking inside mdbx_rebalance(). 2018-03-26 20:20:14 +03:00
Leo Yuriev
ae708aab13 mdbx: update README.md 2018-03-26 14:17:35 +03:00
Leo Yuriev
02fc5fe158 mdbx: update internal version info. 2018-03-23 17:23:28 +03:00
Leo Yuriev
45f159ddd3 mdbx: mdbx_cursor_put(MDBX_APPEND+MDBX_NOOVERWRITE) should return MDBX_KEYEXIST instead of MDBX_EKEYMISMATCH.
When MDBX_APPEND is used the ordering of keys is important.
Therefore MDBX_EKEYMISMATCH should me returned when given key <= last.

On the other hand, if MDBX_NOOVERWRITE is also used
then the MDBX_KEYEXIST should be returned, but not MDBX_EKEYMISMATCH.
2018-03-23 17:21:27 +03:00
Leo Yuriev
6758348046 mdbx: import intrin-includes from t1ha. 2018-03-22 21:11:26 +03:00
Leo Yuriev
09ab6dd777 mdbx-make: add cross-compile targets for testing. 2018-03-22 21:11:26 +03:00
Leo Yuriev
61a2c56784 mdbx: fix/rework cache-line alignment. 2018-03-22 21:11:26 +03:00
Leo Yuriev
6a074cb85a mdbx: fix unaligned access to mp_ptrs[] on fake-page.
Based on http://www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=commitdiff;h=e77918a903d980ff789b7e32f71725481e870510
2018-03-22 21:11:26 +03:00
Leo Yuriev
dbc87a8a5e mdbx: refine MDBX_CACHE_IS_COHERENT. 2018-03-22 20:38:20 +03:00
Leo Yuriev
3a42b76fb8 mdbx-tests: fix entropy_ticks() for multiarch. 2018-03-22 20:38:18 +03:00
Leo Yuriev
c52a57dac5 mdbx: fix for MIPS cacheflush(). 2018-03-22 20:37:50 +03:00
Leo Yuriev
5ce40269b3 mdbx: fix gcc warning (minor). 2018-03-22 20:37:50 +03:00
Leo Yuriev
3d320253cd mdbx-tests: drop duplicate sysdep-includes. 2018-03-21 22:19:17 +03:00
Leo Yuriev
86274513f9 mdbx: more __has_include(). 2018-03-21 22:19:17 +03:00
Leo Yuriev
f5571e4027 mdbx: import byte-order macros from t1ha. 2018-03-21 22:19:17 +03:00
Leo Yuriev
62ae928b40 mdbx: add neutral __ia32__ and __amd64__ marcos. 2018-03-21 22:18:57 +03:00
Leo Yuriev
a282965aa0 mdbx: add check __ALIGNED__ for UNALIGNED_OK macro. 2018-03-21 21:10:57 +03:00
Leo Yuriev
3c5cc40c77 mdbx: fix README typos. 2018-03-21 14:13:54 +03:00
Leo Yuriev
475da5bcfe mdbx: make mdbx_txn_lock() and mdbx_txn_unlock() exportable for mdbx_chk. 2018-03-20 22:19:32 +03:00
Leo Yuriev
679fb56787 mdbx: lookup suitable txnid for rollback to avoid meta-pages clashes. 2018-03-20 22:16:15 +03:00
Leo Yuriev
dd6cb30820 mdbx-chk: directly use mdbx_txn_lock() for write/recovery mode. 2018-03-20 21:37:42 +03:00
Leo Yuriev
8d748a845b mdbx-tests: fix default value for inject_writefaultn. 2018-03-19 21:26:14 +03:00
Leo Yuriev
5872174db6 mdbx-tests: refine temp-dir for unix. 2018-03-19 21:01:05 +03:00
Leo Yuriev
52283d8c44 mdbx-tests: unify testdb name. 2018-03-19 20:50:30 +03:00
Leo Yuriev
838856f688 mdbx-tests: add osal_tempdir(). 2018-03-19 20:50:30 +03:00
Leo Yuriev
0aa07cc09b mdbx-tests: drop unused tableid variable (minor). 2018-03-19 20:20:47 +03:00
Leo Yuriev
e8dd15088a mdbx: add __ARM_FEATURE_UNALIGNED __e2k__ for UNALIGNED_OK. 2018-03-19 19:59:00 +03:00
Leo Yuriev
d0e1c02a8a mdbx-chk: rework/fix read-write mode. 2018-03-19 18:08:45 +03:00
Leo Yuriev
b2213c86fe mdbx-build: add check-fault target. 2018-03-19 18:08:45 +03:00
Leo Yuriev
9ccd551ebc mdbx-tests: minor refine log-levels. 2018-03-19 18:08:45 +03:00
Leo Yuriev
a2ec7f2be1 mdbx: add mdbx_txn_flags(). 2018-03-19 18:08:45 +03:00
Leo Yuriev
d63c2484fe mdbx-tests: add --inject-writefault=N option. 2018-03-19 18:08:45 +03:00
Leo Yuriev
638ed91e3e mdbx-tests: add log_flush(). 2018-03-19 16:51:10 +03:00
Leo Yuriev
64613c9061 mdbx: fix minor memleak (Coverity). 2018-03-19 16:16:31 +03:00
Leo Yuriev
c05d179035 mdbx: temporary workaround for Elbrus's libc bug.
https://bugs.mcst.ru/bugzilla/show_bug.cgi?id=2820
2018-03-19 15:05:56 +03:00
Leo Yuriev
195b53374e mdbx: fix #pragma pack(pop) balance.
Thanks to <Sergey.V.Barannikov@mcst.ru>.
2018-03-14 16:05:43 +03:00
Leo Yuriev
f521b6615e mdbx: fix #endif's comment (cosmetics). 2018-03-14 15:32:32 +03:00
28 changed files with 905 additions and 394 deletions

View File

@@ -23,20 +23,17 @@ suffix ?=
CC ?= gcc
CXX ?= g++
ifeq ($(shell (export LC_ALL=C; ($(CC) --version 2>&1; $(CC) -v 2>&1) | grep -q -i 'e2k' && echo yes)),yes)
CFLAGS ?= -O3 -g3 -Wall -Werror -Wextra -ffunction-sections -fPIC -fvisibility=hidden
else
CFLAGS ?= -O2 -g3 -Wall -Werror -Wextra -ffunction-sections -fPIC -fvisibility=hidden
endif
XCFLAGS ?= -DNDEBUG=1 -DMDBX_DEBUG=0 -DLIBMDBX_EXPORTS=1
CFLAGS += -D_GNU_SOURCE=1 -std=gnu11 -pthread $(XCFLAGS)
CXXFLAGS = -std=c++11 $(filter-out -std=gnu11,$(CFLAGS))
TESTDB ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-check.db
TESTLOG ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-check.log
TESTDB ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-test.db
TESTLOG ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-test.log
# LY: '--no-as-needed,-lrt' for ability to built with modern glibc, but then run with the old
LDFLAGS ?= -Wl,--gc-sections,-z,relro,-O,--no-as-needed,-lrt
EXE_LDFLAGS ?= -pthread -lrt
# LY: just for benchmarking
IOARENA ?= $(shell \
@@ -62,7 +59,7 @@ TEST_OBJ := $(patsubst %.cc,%.o,$(TEST_SRC))
.PHONY: mdbx all install clean check coverage
all: $(LIBRARIES) $(TOOLS) test/test example
all: $(LIBRARIES) $(TOOLS) mdbx_test example
mdbx: libmdbx.a libmdbx.so
@@ -82,10 +79,16 @@ install: $(LIBRARIES) $(TOOLS) $(HEADERS)
&& cp -t $(SANDBOX)$(mandir)/man1 $(MANPAGES)
clean:
rm -rf $(TOOLS) test/test @* *.[ao] *.[ls]o *~ tmp.db/* *.gcov *.log *.err src/*.o test/*.o
rm -rf $(TOOLS) mdbx_test @* *.[ao] *.[ls]o *~ tmp.db/* *.gcov *.log *.err src/*.o test/*.o
check: all
rm -f $(TESTDB) $(TESTLOG) && (set -o pipefail; test/test --pathname=$(TESTDB) --dont-cleanup-after basic | tee -a $(TESTLOG) | tail -n 42) && ./mdbx_chk -vvn $(TESTDB)
rm -f $(TESTDB) $(TESTLOG) && (set -o pipefail; ./mdbx_test --pathname=$(TESTDB) --dont-cleanup-after basic | tee -a $(TESTLOG) | tail -n 42) && ./mdbx_chk -vvn $(TESTDB)
check-singleprocess: all
rm -f $(TESTDB) $(TESTLOG) && (set -o pipefail; ./mdbx_test --pathname=$(TESTDB) --dont-cleanup-after --hill | tee -a $(TESTLOG) | tail -n 42) && ./mdbx_chk -vvn $(TESTDB)
check-fault: all
rm -f $(TESTDB) $(TESTLOG) && (set -o pipefail; ./mdbx_test --pathname=$(TESTDB) --inject-writefault=42 --dump-config --dont-cleanup-after basic | tee -a $(TESTLOG) | tail -n 42) && ./mdbx_chk -vvn $(TESTDB)
define core-rule
$(patsubst %.c,%.o,$(1)): $(1) $(CORE_INC) mdbx.h Makefile
@@ -108,10 +111,12 @@ libmdbx.so: $(CORE_OBJ)
$(CC) $(CFLAGS) -save-temps $^ -pthread -shared $(LDFLAGS) -o $@
mdbx_%: src/tools/mdbx_%.c libmdbx.a
$(CC) $(CFLAGS) $^ $(LDFLAGS) -o $@
$(CC) $(CFLAGS) $^ $(EXE_LDFLAGS) -o $@
test/test: $(TEST_OBJ) libmdbx.a
$(CXX) $(CXXFLAGS) $^ $(LDFLAGS) -o $@
mdbx_test: $(TEST_OBJ) libmdbx.so
$(CXX) $(CXXFLAGS) $(TEST_OBJ) -Wl,-rpath . -L . -l mdbx $(EXE_LDFLAGS) -o $@
###############################################################################
ifneq ($(wildcard $(IOARENA)),)
@@ -160,6 +165,8 @@ bench-quartet: bench-mdbx_$(NN).txt bench-lmdb_$(NN).txt bench-rocksdb_$(NN).txt
endif
###############################################################################
ci-rule = ( CC=$$(which $1); if [ -n "$$CC" ]; then \
echo -n "probe by $2 ($$(readlink -f $$(which $$CC))): " && \
$(MAKE) clean >$1.log 2>$1.err && \
@@ -175,3 +182,36 @@ ci:
@$(call ci-rule,gcc,GCC)
@$(call ci-rule,clang,clang LLVM)
@$(call ci-rule,icc,Intel C)
###############################################################################
CROSS_LIST = alpha-linux-gnu-gcc mips-linux-gnu-gcc \
powerpc64-linux-gnu-gcc powerpc-linux-gnu-gcc \
arm-linux-gnueabihf-gcc aarch64-linux-gnu-gcc
# hppa-linux-gnu-gcc - don't supported by current qemu release
# s390x-linux-gnu-gcc - qemu troubles (hang/abort)
# sh4-linux-gnu-gcc - qemu troubles (pread syscall, etc)
# mips64-linux-gnuabi64-gcc - qemu troubles (pread syscall, etc)
# sparc64-linux-gnu-gcc - qemu troubles (fcntl for F_SETLK/F_GETLK)
CROSS_LIST_NOQEMU = hppa-linux-gnu-gcc s390x-linux-gnu-gcc \
sh4-linux-gnu-gcc mips64-linux-gnuabi64-gcc sparc64-linux-gnu-gcc
cross-gcc:
@echo "CORRESPONDING CROSS-COMPILERs ARE REQUIRED."
@echo "FOR INSTANCE: apt install g++-aarch64-linux-gnu g++-alpha-linux-gnu g++-arm-linux-gnueabihf g++-hppa-linux-gnu g++-mips-linux-gnu g++-mips64-linux-gnuabi64 g++-powerpc-linux-gnu g++-powerpc64-linux-gnu g++-s390x-linux-gnu g++-sh4-linux-gnu"
@for CC in $(CROSS_LIST_NOQEMU) $(CROSS_LIST); do \
echo "===================== $$CC"; \
$(MAKE) clean && CC=$$CC CXX=$$(echo $$CC | sed 's/-gcc/-g++/') EXE_LDFLAGS=-static $(MAKE) all || exit $$?; \
done
#
# Unfortunately qemu don't provide robust support for futexes.
# Therefore it is impossible to run full multi-process tests.
cross-qemu:
@echo "CORRESPONDING CROSS-COMPILERs AND QEMUs ARE REQUIRED."
@echo "FOR INSTANCE: apt install binfmt-support qemu-user-static qemu-user qemu-system-arm qemu-system-mips qemu-system-misc qemu-system-ppc qemu-system-sparc g++-aarch64-linux-gnu g++-alpha-linux-gnu g++-arm-linux-gnueabihf g++-hppa-linux-gnu g++-mips-linux-gnu g++-mips64-linux-gnuabi64 g++-powerpc-linux-gnu g++-powerpc64-linux-gnu g++-s390x-linux-gnu g++-sh4-linux-gnu"
@for CC in $(CROSS_LIST); do \
echo "===================== $$CC + qemu"; \
$(MAKE) clean && CC=$$CC CXX=$$(echo $$CC | sed 's/-gcc/-g++/') EXE_LDFLAGS=-static $(MAKE) check-singleprocess || exit $$?; \
done

View File

@@ -12,13 +12,24 @@ and [by Yandex](https://translate.yandex.ru/translate?url=https%3A%2F%2Fgithub.c
### Project Status
**Now MDBX is under _active development_** and until 2018Q2 is expected a big
change both of API and database format. Unfortunately those update will lead to
loss of compatibility with previous versions.
The aim of this revolution in providing a clearer robust API and adding new
features, including the database properties.
**Сейчас MDBX _активно перерабатывается_** и к середине 2018
ожидается большое измените как API, так и формата базы данных.
К сожалению, обновление приведет к потере совместимости с
предыдущими версиями.
Цель этой революции в обеспечении более четкого надежного
API и добавлении новых функции, а также в наделении базы данных
новыми свойствами.
В настоящее время MDBX предназначена для Linux, а также
поддерживает Windows (начиная с Windows Server 2008) в качестве
дополнительно платформы. Поддержка других ОС может быть
обеспечена на коммерческой основе. Однако такие
усовершенствования (т. е. pull-requests) могут быть приняты в
мейнстрим только в том случае, если будет доступен
соответствующий публичный и бесплатный сервис непрерывной
интеграции (aka Countinious Integration).
## Содержание
@@ -43,7 +54,7 @@ features, including the database properties.
_libmdbx_ - это встраиваемый key-value движок хранения со специфическим
набором свойств и возможностей, ориентированный на создание уникальных
легковесных решений с предельной производительностью.
легковесных решений с предельной производительностью под Linux и Windows.
_libmdbx_ позволяет множеству процессов совместно читать и обновлять
несколько key-value таблиц с соблюдением [ACID](https://ru.wikipedia.org/wiki/ACID),

View File

@@ -9,9 +9,17 @@ libmdbx
### Project Status
**MDBX is under _active development_**, database format and API aren't stable
at least until 2018Q2. New version won't be backwards compatible.
Main focus of the rework is to provide clear and robust API and new features.
**MDBX is under _active development_**, database format and
API aren't stable at least until 2018Q3. New version won't be
backwards compatible. Main focus of the rework is to provide
clear and robust API and new features.
Nowadays MDBX intended for Linux and support Windows (since
Windows Server 2008) as complementary platform. Support for
other OS could be implemented on commercial basis. However such
enhancements (i.e. pull requests) could be accepted in
mainstream only when corresponding public and free Countinious
Integration service will be available.
## Contents
@@ -34,7 +42,7 @@ Main focus of the rework is to provide clear and robust API and new features.
## Overview
_libmdbx_ is an embedded lightweight key-value database engine oriented for performance.
_libmdbx_ is an embedded lightweight key-value database engine oriented for performance under Linux and Windows.
_libmdbx_ allows multiple processes to read and update several key-value tables concurrently,
while being [ACID](https://en.wikipedia.org/wiki/ACID)-compliant, with minimal overhead and operation cost of Olog(N).
@@ -233,7 +241,7 @@ scanning data directory.
Because of this syncing data to disk might be quite resource intensive and be main performance bottleneck
during intensive write workload.
> As compromise _libmdbx_ allows several modes of lazy and/or periodic syncing, including `MAPASYNC` mode, which modificate
> data in memory and asynchronously syncs data to disc, moment to sync is picked by OS.
> data in memory and asynchronously syncs data to disk, moment to sync is picked by OS.
>
> Although this should be used with care, synchronous transactions in a DB with transaction journal will require 2 IOPS
> minimum (probably 3-4 in practice) because of filesystem overhead, overhead depends on filesystem, not on record
@@ -339,7 +347,7 @@ Improvements over LMDB
This allows to minimize reclaim loop and make it execution time independent from total page count.
This results in OS kernel cache mechanisms working with maximum efficiency.
In case of using disc controllers or storages with
In case of using disk controllers or storages with
[BBWC](https://en.wikipedia.org/wiki/Disk_buffer#Write_acceleration) this may greatly improve
write performance.
@@ -378,7 +386,7 @@ Improvements over LMDB
9. Check if there is a row with data after current cursor position via `mdbx_cursor_eof()`.
10. Ability to explicitly request update of current record without creating new record. Implemented as `MDBX_CURRENT` flag
10. Ability to explicitly request update of present record without creating new record. Implemented as `MDBX_CURRENT` flag
for `mdbx_put()`.
11. Ability to update or delete record and get previous value via `mdbx_replace()` Also can update specific multi-value.
@@ -419,7 +427,7 @@ Improvements over LMDB
22. Additional error code `MDBX_EMULTIVAL`, which is returned by `mdbx_put()` and
`mdbx_replace()` in case is ambiguous update or delete.
23. Ability to get value by key and duplicates count by `mdbx_get_ex()`
23. Ability to get value by key and duplicates count by `mdbx_get_ex()`.
24. Functions `mdbx_cursor_on_first() and mdbx_cursor_on_last(), which allows to know if cursor is currently on first or
last position respectively.

View File

@@ -21,7 +21,7 @@
Тесты
=====
- [ ] Тестирование поддержки lockless-режима.
- [ ] Додумать имя и размещение тестовой БД по-умолчанию.
- [x] Додумать имя и размещение тестовой БД по-умолчанию.
- [ ] Реализовать cleanup в тесте.
- [ ] usage для теста.
- [ ] Логирование в файл, плюс более полный progress bar.

View File

@@ -1,4 +1,4 @@
version: 0.1.2.{build}
version: 0.1.3.{build}
environment:
matrix:

45
mdbx.h
View File

@@ -982,6 +982,15 @@ LIBMDBX_API int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, unsigned flags,
* [in] txn A transaction handle returned by mdbx_txn_begin() */
LIBMDBX_API MDBX_env *mdbx_txn_env(MDBX_txn *txn);
/* Return the transaction's flags.
*
* This returns the flags associated with this transaction.
*
* [in] txn A transaction handle returned by mdbx_txn_begin()
*
* Returns A transaction flags, valid if input is an active transaction. */
LIBMDBX_API int mdbx_txn_flags(MDBX_txn *txn);
/* Return the transaction's ID.
*
* This returns the identifier associated with this transaction. For a
@@ -1818,6 +1827,42 @@ LIBMDBX_API int mdbx_cursor_get_attr(MDBX_cursor *mc, MDBX_val *key,
LIBMDBX_API int mdbx_get_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key,
MDBX_val *data, mdbx_attr_t *attrptr);
/*----------------------------------------------------------------------------*/
/* LY: temporary workaround for Elbrus's memcmp() bug. */
#ifndef __GLIBC_PREREQ
#if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
#define __GLIBC_PREREQ(maj, min) \
((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
#else
#define __GLIBC_PREREQ(maj, min) (0)
#endif
#endif /* __GLIBC_PREREQ */
#if defined(__e2k__) && !__GLIBC_PREREQ(2, 24)
LIBMDBX_API int mdbx_e2k_memcmp_bug_workaround(const void *s1, const void *s2,
size_t n);
LIBMDBX_API int mdbx_e2k_strcmp_bug_workaround(const char *s1, const char *s2);
LIBMDBX_API int mdbx_e2k_strncmp_bug_workaround(const char *s1, const char *s2,
size_t n);
LIBMDBX_API size_t mdbx_e2k_strlen_bug_workaround(const char *s);
LIBMDBX_API size_t mdbx_e2k_strnlen_bug_workaround(const char *s,
size_t maxlen);
#include <string.h>
#include <strings.h>
#undef memcmp
#define memcmp mdbx_e2k_memcmp_bug_workaround
#undef bcmp
#define bcmp mdbx_e2k_memcmp_bug_workaround
#undef strcmp
#define strcmp mdbx_e2k_strcmp_bug_workaround
#undef strncmp
#define strncmp mdbx_e2k_strncmp_bug_workaround
#undef strlen
#define strlen mdbx_e2k_strlen_bug_workaround
#undef strnlen
#define strnlen mdbx_e2k_strnlen_bug_workaround
#endif /* Elbrus's memcmp() bug. */
#ifdef __cplusplus
}
#endif

View File

@@ -55,6 +55,7 @@
#pragma warning(disable : 4310) /* cast truncates constant value */
#pragma warning(disable : 4820) /* bytes padding added after data member for aligment */
#pragma warning(disable : 4548) /* expression before comma has no effect; expected expression with side - effect */
#pragma warning(disable : 4366) /* the result of the unary '&' operator may be unaligned */
#endif /* _MSC_VER (warnings) */
#include "../mdbx.h"
@@ -111,7 +112,7 @@
# else
# pragma warning disable "alignment-reduction-ignored"
# endif
#endif /* -Wno-constant-logical-operand */
#endif /* -Walignment-reduction-ignored */
#include "./osal.h"
@@ -252,7 +253,7 @@ typedef struct MDBX_reader {
uint8_t pad[MDBX_CACHELINE_SIZE -
(sizeof(txnid_t) + sizeof(mdbx_pid_t) + sizeof(mdbx_tid_t)) %
MDBX_CACHELINE_SIZE];
} __cache_aligned MDBX_reader;
} MDBX_reader;
/* Information about a single database in the environment. */
typedef struct MDBX_db {
@@ -410,47 +411,53 @@ typedef struct MDBX_lockinfo {
/* Flags which environment was opened. */
volatile uint32_t mti_envmode;
union {
#ifdef MDBX_OSAL_LOCK
/* Mutex protecting write access to this table. */
union {
MDBX_OSAL_LOCK mti_wmutex;
uint8_t pad_mti_wmutex[MDBX_OSAL_LOCK_SIZE % sizeof(size_t)];
};
#endif
uint64_t align_wmutex;
};
#define MDBX_lockinfo_SIZE_A \
(8 /* mti_magic_and_version */ + 4 /* mti_os_and_format */ + \
4 /* mti_envmode */ + MDBX_OSAL_LOCK_SIZE /* mti_wmutex */ + \
MDBX_OSAL_LOCK_SIZE % sizeof(size_t) /* pad_mti_wmutex */)
union {
/* The number of slots that have been used in the reader table.
* This always records the maximum count, it is not decremented
* when readers release their slots. */
volatile unsigned __cache_aligned mti_numreaders;
uint64_t align_numreaders;
};
/* cache-line alignment */
uint8_t
pad_a[MDBX_CACHELINE_SIZE - MDBX_lockinfo_SIZE_A % MDBX_CACHELINE_SIZE];
/* The number of slots that have been used in the reader table.
* This always records the maximum count, it is not decremented
* when readers release their slots. */
volatile unsigned mti_numreaders;
union {
#ifdef MDBX_OSAL_LOCK
/* Mutex protecting access to this table. */
/* Mutex protecting readers registration access to this table. */
union {
MDBX_OSAL_LOCK mti_rmutex;
uint8_t pad_mti_rmutex[MDBX_OSAL_LOCK_SIZE % sizeof(size_t)];
};
#endif
uint64_t align_rmutex;
};
union {
volatile txnid_t mti_oldest;
uint64_t align_oldest;
};
volatile txnid_t mti_oldest;
volatile uint32_t mti_readers_refresh_flag;
union {
volatile uint32_t mti_readers_refresh_flag;
uint64_t align_reader_finished_flag;
};
#define MDBX_lockinfo_SIZE_B \
(sizeof(unsigned) /* mti_numreaders */ + \
MDBX_OSAL_LOCK_SIZE /* mti_rmutex */ + sizeof(txnid_t) /* mti_oldest */ + \
sizeof(uint32_t) /* mti_readers_refresh_flag */ + \
MDBX_OSAL_LOCK_SIZE % sizeof(size_t) /* pad_mti_rmutex */)
uint8_t pad_align[MDBX_CACHELINE_SIZE - sizeof(uint64_t) * 7];
/* cache-line alignment */
uint8_t
pad_b[MDBX_CACHELINE_SIZE - MDBX_lockinfo_SIZE_B % MDBX_CACHELINE_SIZE];
MDBX_reader mti_readers[1];
MDBX_reader __cache_aligned mti_readers[1];
} MDBX_lockinfo;
#ifdef _MSC_VER
#pragma pack(pop)
#endif /* MSVC: Enable aligment */
#define MDBX_LOCKINFO_WHOLE_SIZE \
((sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) & \
@@ -941,13 +948,13 @@ static __inline void mdbx_jitter4testing(bool tiny) {
/* Internal prototypes and inlines */
int mdbx_reader_check0(MDBX_env *env, int rlocked, int *dead);
void mdbx_rthc_dtor(void *rthc);
void mdbx_rthc_lock(void);
void mdbx_rthc_unlock(void);
int mdbx_rthc_alloc(mdbx_thread_key_t *key, MDBX_reader *begin,
MDBX_reader *end);
void mdbx_rthc_remove(mdbx_thread_key_t key);
void mdbx_rthc_cleanup(void);
void mdbx_rthc_remove(const mdbx_thread_key_t key);
void mdbx_rthc_global_init(void);
void mdbx_rthc_global_dtor(void);
void mdbx_rthc_thread_dtor(void *ptr);
static __inline bool mdbx_is_power2(size_t x) { return (x & (x - 1)) == 0; }

View File

@@ -192,16 +192,6 @@
# endif
#endif /* __prefetch */
#ifndef __aligned
# if defined(__GNUC__) || __has_attribute(aligned)
# define __aligned(N) __attribute__((aligned(N)))
# elif defined(_MSC_VER)
# define __aligned(N) __declspec(align(N))
# else
# define __aligned(N)
# endif
#endif /* __aligned */
#ifndef __noreturn
# if defined(__GNUC__) || __has_attribute(noreturn)
# define __noreturn __attribute__((noreturn))

View File

@@ -32,18 +32,12 @@
/*----------------------------------------------------------------------------*/
/* rthc */
static pthread_mutex_t mdbx_rthc_mutex = PTHREAD_MUTEX_INITIALIZER;
void mdbx_rthc_lock(void) {
mdbx_ensure(NULL, pthread_mutex_lock(&mdbx_rthc_mutex) == 0);
static __cold __attribute__((constructor)) void mdbx_global_constructor(void) {
mdbx_rthc_global_init();
}
void mdbx_rthc_unlock(void) {
mdbx_ensure(NULL, pthread_mutex_unlock(&mdbx_rthc_mutex) == 0);
}
void __attribute__((destructor)) mdbx_global_destructor(void) {
mdbx_rthc_cleanup();
static __cold __attribute__((destructor)) void mdbx_global_destructor(void) {
mdbx_rthc_global_dtor();
}
/*----------------------------------------------------------------------------*/

View File

@@ -27,31 +27,24 @@
/*----------------------------------------------------------------------------*/
/* rthc */
static CRITICAL_SECTION rthc_critical_section;
static void NTAPI tls_callback(PVOID module, DWORD reason, PVOID reserved) {
(void)module;
(void)reserved;
switch (reason) {
case DLL_PROCESS_ATTACH:
InitializeCriticalSection(&rthc_critical_section);
mdbx_rthc_global_init();
break;
case DLL_PROCESS_DETACH:
DeleteCriticalSection(&rthc_critical_section);
mdbx_rthc_global_dtor();
break;
case DLL_THREAD_ATTACH:
break;
case DLL_THREAD_DETACH:
mdbx_rthc_cleanup();
mdbx_rthc_thread_dtor(module);
break;
}
}
void mdbx_rthc_lock(void) { EnterCriticalSection(&rthc_critical_section); }
void mdbx_rthc_unlock(void) { LeaveCriticalSection(&rthc_critical_section); }
/* *INDENT-OFF* */
/* clang-format off */
#if defined(_MSC_VER)

View File

@@ -37,6 +37,117 @@
#include "./bits.h"
/* LY: temporary workaround for Elbrus's memcmp() bug. */
#if defined(__e2k__) && !__GLIBC_PREREQ(2, 24)
int __hot mdbx_e2k_memcmp_bug_workaround(const void *s1, const void *s2,
size_t n) {
if (unlikely(n > 42
/* LY: align followed access if reasonable possible */ &&
(((uintptr_t)s1) & 7) != 0 &&
(((uintptr_t)s1) & 7) == (((uintptr_t)s2) & 7))) {
if (((uintptr_t)s1) & 1) {
const int diff = *(uint8_t *)s1 - *(uint8_t *)s2;
if (diff)
return diff;
s1 = (char *)s1 + 1;
s2 = (char *)s2 + 1;
n -= 1;
}
if (((uintptr_t)s1) & 2) {
const uint16_t a = *(uint16_t *)s1;
const uint16_t b = *(uint16_t *)s2;
if (likely(a != b))
return (__builtin_bswap16(a) > __builtin_bswap16(b)) ? 1 : -1;
s1 = (char *)s1 + 2;
s2 = (char *)s2 + 2;
n -= 2;
}
if (((uintptr_t)s1) & 4) {
const uint32_t a = *(uint32_t *)s1;
const uint32_t b = *(uint32_t *)s2;
if (likely(a != b))
return (__builtin_bswap32(a) > __builtin_bswap32(b)) ? 1 : -1;
s1 = (char *)s1 + 4;
s2 = (char *)s2 + 4;
n -= 4;
}
}
while (n >= 8) {
const uint64_t a = *(uint64_t *)s1;
const uint64_t b = *(uint64_t *)s2;
if (likely(a != b))
return (__builtin_bswap64(a) > __builtin_bswap64(b)) ? 1 : -1;
s1 = (char *)s1 + 8;
s2 = (char *)s2 + 8;
n -= 8;
}
if (n & 4) {
const uint32_t a = *(uint32_t *)s1;
const uint32_t b = *(uint32_t *)s2;
if (likely(a != b))
return (__builtin_bswap32(a) > __builtin_bswap32(b)) ? 1 : -1;
s1 = (char *)s1 + 4;
s2 = (char *)s2 + 4;
}
if (n & 2) {
const uint16_t a = *(uint16_t *)s1;
const uint16_t b = *(uint16_t *)s2;
if (likely(a != b))
return (__builtin_bswap16(a) > __builtin_bswap16(b)) ? 1 : -1;
s1 = (char *)s1 + 2;
s2 = (char *)s2 + 2;
}
return (n & 1) ? *(uint8_t *)s1 - *(uint8_t *)s2 : 0;
}
int __hot mdbx_e2k_strcmp_bug_workaround(const char *s1, const char *s2) {
while (true) {
int diff = *(uint8_t *)s1 - *(uint8_t *)s2;
if (likely(diff != 0) || *s1 == '\0')
return diff;
s1 += 1;
s2 += 1;
}
}
int __hot mdbx_e2k_strncmp_bug_workaround(const char *s1, const char *s2,
size_t n) {
while (n > 0) {
int diff = *(uint8_t *)s1 - *(uint8_t *)s2;
if (likely(diff != 0) || *s1 == '\0')
return diff;
s1 += 1;
s2 += 1;
n -= 1;
}
return 0;
}
size_t __hot mdbx_e2k_strlen_bug_workaround(const char *s) {
size_t n = 0;
while (*s) {
s += 1;
n += 1;
}
return n;
}
size_t __hot mdbx_e2k_strnlen_bug_workaround(const char *s, size_t maxlen) {
size_t n = 0;
while (maxlen > n && *s) {
s += 1;
n += 1;
}
return n;
}
#endif /* Elbrus's memcmp() bug. */
/*----------------------------------------------------------------------------*/
/* rthc (tls keys and destructors) */
@@ -52,43 +163,260 @@ typedef struct rthc_entry_t {
#define RTHC_INITIAL_LIMIT 16
#endif
static unsigned rthc_count;
static unsigned rthc_limit = RTHC_INITIAL_LIMIT;
#if defined(_WIN32) || defined(_WIN64)
static CRITICAL_SECTION rthc_critical_section;
#else
int __cxa_thread_atexit_impl(void (*dtor)(void *), void *obj, void *dso_symbol)
__attribute__((weak));
static pthread_mutex_t mdbx_rthc_mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t mdbx_rthc_cond = PTHREAD_COND_INITIALIZER;
static mdbx_thread_key_t mdbx_rthc_key;
static volatile uint32_t mdbx_rthc_pending;
static void __cold mdbx_workaround_glibc_bug21031(void) {
/* Workaround for https://sourceware.org/bugzilla/show_bug.cgi?id=21031
*
* Due race between pthread_key_delete() and __nptl_deallocate_tsd()
* The destructor(s) of thread-local-storate object(s) may be running
* in another thread(s) and be blocked or not finished yet.
* In such case we get a SEGFAULT after unload this library DSO.
*
* So just by yielding a few timeslices we give a chance
* to such destructor(s) for completion and avoids segfault. */
sched_yield();
sched_yield();
sched_yield();
}
#endif
static unsigned rthc_count, rthc_limit;
static rthc_entry_t *rthc_table;
static rthc_entry_t rthc_table_static[RTHC_INITIAL_LIMIT];
static rthc_entry_t *rthc_table = rthc_table_static;
__cold void mdbx_rthc_dtor(void *ptr) {
MDBX_reader *rthc = (MDBX_reader *)ptr;
mdbx_rthc_lock();
const mdbx_pid_t self_pid = mdbx_getpid();
for (unsigned i = 0; i < rthc_count; ++i) {
if (rthc >= rthc_table[i].begin && rthc < rthc_table[i].end) {
if (rthc->mr_pid == self_pid) {
rthc->mr_pid = 0;
mdbx_coherent_barrier();
}
break;
}
}
mdbx_rthc_unlock();
static __cold void mdbx_rthc_lock(void) {
#if defined(_WIN32) || defined(_WIN64)
EnterCriticalSection(&rthc_critical_section);
#else
mdbx_ensure(nullptr, pthread_mutex_lock(&mdbx_rthc_mutex) == 0);
#endif
}
__cold void mdbx_rthc_cleanup(void) {
static __cold void mdbx_rthc_unlock(void) {
#if defined(_WIN32) || defined(_WIN64)
LeaveCriticalSection(&rthc_critical_section);
#else
mdbx_ensure(nullptr, pthread_mutex_unlock(&mdbx_rthc_mutex) == 0);
#endif
}
static __inline int mdbx_thread_key_create(mdbx_thread_key_t *key) {
int rc;
#if defined(_WIN32) || defined(_WIN64)
*key = TlsAlloc();
rc = (*key != TLS_OUT_OF_INDEXES) ? MDBX_SUCCESS : GetLastError();
#else
rc = pthread_key_create(key, nullptr);
#endif
mdbx_trace("&key = %p, value 0x%x, rc %d", key, (unsigned)*key, rc);
return rc;
}
static __inline void mdbx_thread_key_delete(mdbx_thread_key_t key) {
mdbx_trace("key = 0x%x", (unsigned)key);
#if defined(_WIN32) || defined(_WIN64)
mdbx_ensure(nullptr, TlsFree(key));
#else
mdbx_ensure(nullptr, pthread_key_delete(key) == 0);
mdbx_workaround_glibc_bug21031();
#endif
}
static __inline void *mdbx_thread_rthc_get(mdbx_thread_key_t key) {
#if defined(_WIN32) || defined(_WIN64)
return TlsGetValue(key);
#else
return pthread_getspecific(key);
#endif
}
static void mdbx_thread_rthc_set(mdbx_thread_key_t key, const void *value) {
#if defined(_WIN32) || defined(_WIN64)
mdbx_ensure(nullptr, TlsSetValue(key, (void *)value));
#else
#define MDBX_THREAD_RTHC_ZERO 0
#define MDBX_THREAD_RTHC_REGISTERD 1
#define MDBX_THREAD_RTHC_COUNTED 2
static __thread uint32_t thread_registration_state;
if (value && unlikely(thread_registration_state == MDBX_THREAD_RTHC_ZERO)) {
thread_registration_state = MDBX_THREAD_RTHC_REGISTERD;
mdbx_trace("thread registered 0x%" PRIxPTR, (uintptr_t)mdbx_thread_self());
if (&__cxa_thread_atexit_impl == nullptr ||
__cxa_thread_atexit_impl(mdbx_rthc_thread_dtor,
&thread_registration_state,
(void *)&mdbx_version /* dso_anchor */)) {
mdbx_ensure(nullptr, pthread_setspecific(
mdbx_rthc_key, &thread_registration_state) == 0);
thread_registration_state = MDBX_THREAD_RTHC_COUNTED;
const unsigned count_before = mdbx_atomic_add32(&mdbx_rthc_pending, 1);
mdbx_ensure(nullptr, count_before < INT_MAX);
mdbx_trace("fallback to pthreads' tsd, key 0x%x, count %u",
(unsigned)mdbx_rthc_key, count_before);
(void)count_before;
}
}
mdbx_ensure(nullptr, pthread_setspecific(key, value) == 0);
#endif
}
__cold void mdbx_rthc_global_init(void) {
rthc_limit = RTHC_INITIAL_LIMIT;
rthc_table = rthc_table_static;
#if defined(_WIN32) || defined(_WIN64)
InitializeCriticalSection(&rthc_critical_section);
#else
mdbx_ensure(nullptr,
pthread_key_create(&mdbx_rthc_key, mdbx_rthc_thread_dtor) == 0);
mdbx_trace("pid %d, &mdbx_rthc_key = %p, value 0x%x", mdbx_getpid(),
&mdbx_rthc_key, (unsigned)mdbx_rthc_key);
#endif
}
/* dtor called for thread, i.e. for all mdbx's environment objects */
void mdbx_rthc_thread_dtor(void *ptr) {
mdbx_rthc_lock();
mdbx_trace(">> pid %d, thread 0x%" PRIxPTR ", rthc %p", mdbx_getpid(),
(uintptr_t)mdbx_thread_self(), ptr);
const mdbx_pid_t self_pid = mdbx_getpid();
for (unsigned i = 0; i < rthc_count; ++i) {
mdbx_thread_key_t key = rthc_table[i].key;
MDBX_reader *rthc = mdbx_thread_rthc_get(key);
if (rthc) {
mdbx_thread_rthc_set(key, NULL);
const mdbx_thread_key_t key = rthc_table[i].key;
MDBX_reader *const rthc = mdbx_thread_rthc_get(key);
if (rthc < rthc_table[i].begin || rthc >= rthc_table[i].end)
continue;
#if !defined(_WIN32) && !defined(_WIN64)
if (pthread_setspecific(key, nullptr) != 0) {
mdbx_trace("== thread 0x%" PRIxPTR
", rthc %p: ignore race with tsd-key deletion",
(uintptr_t)mdbx_thread_self(), ptr);
continue /* ignore race with tsd-key deletion by mdbx_env_close() */;
}
#endif
mdbx_trace("== thread 0x%" PRIxPTR
", rthc %p, [%i], %p ... %p (%+i), rtch-pid %i, "
"current-pid %i",
(uintptr_t)mdbx_thread_self(), rthc, i, rthc_table[i].begin,
rthc_table[i].end, (int)(rthc - rthc_table[i].begin),
rthc->mr_pid, self_pid);
if (rthc->mr_pid == self_pid) {
mdbx_trace("==== thread 0x%" PRIxPTR ", rthc %p, cleanup",
(uintptr_t)mdbx_thread_self(), rthc);
rthc->mr_pid = 0;
}
}
#if defined(_WIN32) || defined(_WIN64)
mdbx_trace("<< thread 0x%" PRIxPTR ", rthc %p", (uintptr_t)mdbx_thread_self(),
ptr);
mdbx_rthc_unlock();
#else
const char self_registration = *(char *)ptr;
*(char *)ptr = MDBX_THREAD_RTHC_ZERO;
mdbx_trace("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %d",
(uintptr_t)mdbx_thread_self(), ptr, mdbx_getpid(),
self_registration);
if (self_registration == MDBX_THREAD_RTHC_COUNTED)
mdbx_ensure(nullptr, mdbx_atomic_sub32(&mdbx_rthc_pending, 1) > 0);
if (mdbx_rthc_pending == 0) {
mdbx_trace("== thread 0x%" PRIxPTR ", rthc %p, pid %d, wake",
(uintptr_t)mdbx_thread_self(), ptr, mdbx_getpid());
mdbx_ensure(nullptr, pthread_cond_broadcast(&mdbx_rthc_cond) == 0);
}
mdbx_trace("<< thread 0x%" PRIxPTR ", rthc %p", (uintptr_t)mdbx_thread_self(),
ptr);
/* Allow tail call optimization, i.e. gcc should generate the jmp instruction
* instead of a call for pthread_mutex_unlock() and therefore CPU could not
* return to current DSO's code section, which may be unloaded immediately
* after the mutex got released. */
pthread_mutex_unlock(&mdbx_rthc_mutex);
#endif
}
__cold void mdbx_rthc_global_dtor(void) {
mdbx_trace(
">> pid %d, &mdbx_rthc_global_dtor %p, &mdbx_rthc_thread_dtor = %p, "
"&mdbx_rthc_remove = %p",
mdbx_getpid(), &mdbx_rthc_global_dtor, &mdbx_rthc_thread_dtor,
&mdbx_rthc_remove);
mdbx_rthc_lock();
#if !defined(_WIN32) && !defined(_WIN64)
char *rthc = (char *)pthread_getspecific(mdbx_rthc_key);
mdbx_trace("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %d",
(uintptr_t)mdbx_thread_self(), rthc, mdbx_getpid(),
rthc ? *rthc : -1);
if (rthc) {
const char self_registration = *(char *)rthc;
*rthc = MDBX_THREAD_RTHC_ZERO;
if (self_registration == MDBX_THREAD_RTHC_COUNTED)
mdbx_ensure(nullptr, mdbx_atomic_sub32(&mdbx_rthc_pending, 1) > 0);
}
struct timespec abstime;
mdbx_ensure(nullptr, clock_gettime(CLOCK_REALTIME, &abstime) == 0);
abstime.tv_nsec += 1000000000l / 10;
if (abstime.tv_nsec >= 1000000000l) {
abstime.tv_nsec -= 1000000000l;
abstime.tv_sec += 1;
}
#if MDBX_DEBUG > 0
abstime.tv_sec += 600;
#endif
for (unsigned left; (left = mdbx_rthc_pending) > 0;) {
mdbx_trace("pid %d, pending %u, wait for...", mdbx_getpid(), left);
const int rc =
pthread_cond_timedwait(&mdbx_rthc_cond, &mdbx_rthc_mutex, &abstime);
if (rc && rc != EINTR)
break;
}
mdbx_thread_key_delete(mdbx_rthc_key);
#endif
const mdbx_pid_t self_pid = mdbx_getpid();
for (unsigned i = 0; i < rthc_count; ++i) {
const mdbx_thread_key_t key = rthc_table[i].key;
mdbx_thread_key_delete(key);
for (MDBX_reader *rthc = rthc_table[i].begin; rthc < rthc_table[i].end;
++rthc) {
mdbx_trace("== [%i] = key %u, %p ... %p, rthc %p (%+i), "
"rthc-pid %i, current-pid %i",
i, key, rthc_table[i].begin, rthc_table[i].end, rthc,
(int)(rthc - rthc_table[i].begin), rthc->mr_pid, self_pid);
if (rthc->mr_pid == self_pid) {
rthc->mr_pid = 0;
mdbx_coherent_barrier();
mdbx_trace("== cleanup %p", rthc);
}
}
}
rthc_limit = rthc_count = 0;
if (rthc_table != rthc_table_static)
free(rthc_table);
rthc_table = nullptr;
mdbx_rthc_unlock();
#if defined(_WIN32) || defined(_WIN64)
DeleteCriticalSection(&rthc_critical_section);
#else
/* LY: yielding a few timeslices to give a more chance
* to racing destructor(s) for completion. */
mdbx_workaround_glibc_bug21031();
#endif
mdbx_trace("<< pid %d\n", mdbx_getpid());
}
__cold int mdbx_rthc_alloc(mdbx_thread_key_t *key, MDBX_reader *begin,
@@ -101,11 +429,13 @@ __cold int mdbx_rthc_alloc(mdbx_thread_key_t *key, MDBX_reader *begin,
return rc;
mdbx_rthc_lock();
mdbx_trace(">> key 0x%x, rthc_count %u, rthc_limit %u", *key, rthc_count,
rthc_limit);
if (rthc_count == rthc_limit) {
rthc_entry_t *new_table =
realloc((rthc_table == rthc_table_static) ? NULL : rthc_table,
realloc((rthc_table == rthc_table_static) ? nullptr : rthc_table,
sizeof(rthc_entry_t) * rthc_limit * 2);
if (new_table == NULL) {
if (new_table == nullptr) {
rc = MDBX_ENOMEM;
goto bailout;
}
@@ -114,11 +444,13 @@ __cold int mdbx_rthc_alloc(mdbx_thread_key_t *key, MDBX_reader *begin,
rthc_table = new_table;
rthc_limit *= 2;
}
mdbx_trace("== [%i] = key %u, %p ... %p", rthc_count, *key, begin, end);
rthc_table[rthc_count].key = *key;
rthc_table[rthc_count].begin = begin;
rthc_table[rthc_count].end = end;
++rthc_count;
mdbx_trace("<< key 0x%x, rthc_count %u, rthc_limit %u", *key, rthc_count,
rthc_limit);
mdbx_rthc_unlock();
return MDBX_SUCCESS;
@@ -128,18 +460,25 @@ bailout:
return rc;
}
__cold void mdbx_rthc_remove(mdbx_thread_key_t key) {
mdbx_rthc_lock();
__cold void mdbx_rthc_remove(const mdbx_thread_key_t key) {
mdbx_thread_key_delete(key);
mdbx_rthc_lock();
mdbx_trace(">> key 0x%x, rthc_count %u, rthc_limit %u", key, rthc_count,
rthc_limit);
for (unsigned i = 0; i < rthc_count; ++i) {
if (key == rthc_table[i].key) {
const mdbx_pid_t self_pid = mdbx_getpid();
mdbx_trace("== [%i], %p ...%p, current-pid %d", i, rthc_table[i].begin,
rthc_table[i].end, self_pid);
for (MDBX_reader *rthc = rthc_table[i].begin; rthc < rthc_table[i].end;
++rthc)
if (rthc->mr_pid == self_pid)
++rthc) {
if (rthc->mr_pid == self_pid) {
rthc->mr_pid = 0;
mdbx_coherent_barrier();
mdbx_trace("== cleanup %p", rthc);
}
}
if (--rthc_count > 0)
rthc_table[i] = rthc_table[rthc_count];
else if (rthc_table != rthc_table_static) {
@@ -151,6 +490,8 @@ __cold void mdbx_rthc_remove(mdbx_thread_key_t key) {
}
}
mdbx_trace("<< key 0x%x, rthc_count %u, rthc_limit %u", key, rthc_count,
rthc_limit);
mdbx_rthc_unlock();
}
@@ -806,6 +1147,7 @@ void __cold mdbx_debug_log(int type, const char *function, int line,
else if (line > 0)
fprintf(stderr, "%d: ", line);
vfprintf(stderr, fmt, args);
fflush(stderr);
}
va_end(args);
}
@@ -2507,6 +2849,12 @@ static int mdbx_txn_renew0(MDBX_txn *txn, unsigned flags) {
return MDBX_PANIC;
}
STATIC_ASSERT(sizeof(MDBX_reader) == MDBX_CACHELINE_SIZE);
STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_numreaders) % MDBX_CACHELINE_SIZE ==
0);
STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_readers) % MDBX_CACHELINE_SIZE ==
0);
pgno_t upper_pgno = 0;
if (flags & MDBX_TXN_RDONLY) {
txn->mt_flags = MDBX_TXN_RDONLY;
@@ -2562,9 +2910,6 @@ static int mdbx_txn_renew0(MDBX_txn *txn, unsigned flags) {
}
}
STATIC_ASSERT(sizeof(MDBX_reader) == MDBX_CACHELINE_SIZE);
STATIC_ASSERT(
offsetof(MDBX_lockinfo, mti_readers) % MDBX_CACHELINE_SIZE == 0);
r = &env->me_lck->mti_readers[slot];
/* Claim the reader slot, carefully since other code
* uses the reader table un-mutexed: First reset the
@@ -2876,6 +3221,13 @@ uint64_t mdbx_txn_id(MDBX_txn *txn) {
return txn->mt_txnid;
}
int mdbx_txn_flags(MDBX_txn *txn) {
if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE))
return -1;
return txn->mt_flags;
}
/* Export or close DBI handles opened in this txn. */
static void mdbx_dbis_update(MDBX_txn *txn, int keep) {
MDBX_dbi n = txn->mt_numdbs;
@@ -4042,7 +4394,7 @@ static int __cold mdbx_read_header(MDBX_env *env, MDBX_meta *meta) {
STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE);
if (mapsize_max > MAX_MAPSIZE ||
MAX_PAGENO < mdbx_roundup2((size_t)mapsize_max, env->me_os_psize) /
(uint64_t)page.mp_meta.mm_psize) {
(size_t)page.mp_meta.mm_psize) {
const uint64_t used_bytes =
page.mp_meta.mm_geo.next * (uint64_t)page.mp_meta.mm_psize;
if (page.mp_meta.mm_geo.next - 1 > MAX_PAGENO ||
@@ -4644,13 +4996,13 @@ LIBMDBX_API int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
if (pagesize < 0) {
pagesize = env->me_os_psize;
if (pagesize > MAX_PAGESIZE)
if ((uintptr_t)pagesize > MAX_PAGESIZE)
pagesize = MAX_PAGESIZE;
mdbx_assert(env, pagesize >= MIN_PAGESIZE);
mdbx_assert(env, (uintptr_t)pagesize >= MIN_PAGESIZE);
}
}
if (pagesize < MIN_PAGESIZE || pagesize > MAX_PAGESIZE ||
if (pagesize < (intptr_t)MIN_PAGESIZE || pagesize > (intptr_t)MAX_PAGESIZE ||
!mdbx_is_power2(pagesize)) {
rc = MDBX_EINVAL;
goto bailout;
@@ -4684,7 +5036,7 @@ LIBMDBX_API int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower,
size_upper = pagesize * MAX_PAGENO;
}
if (unlikely(size_lower < MIN_MAPSIZE || size_lower > size_upper)) {
if (unlikely(size_lower < (intptr_t)MIN_MAPSIZE || size_lower > size_upper)) {
rc = MDBX_EINVAL;
goto bailout;
}
@@ -4991,7 +5343,7 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, int lck_rc) {
filesize_before_mmap,
bytes2pgno(env, (size_t)filesize_before_mmap));
} else {
mdbx_notice("filesize mismatch (expect %" PRIuPTR "/%" PRIaPGNO
mdbx_notice("filesize mismatch (expect %" PRIuSIZE "/%" PRIaPGNO
", have %" PRIu64 "/%" PRIaPGNO ")",
expected_bytes, bytes2pgno(env, expected_bytes),
filesize_before_mmap,
@@ -5007,11 +5359,11 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, int lck_rc) {
if (env->me_flags & MDBX_RDONLY) {
mdbx_notice("ignore filesize mismatch in readonly-mode");
} else {
mdbx_info("resize datafile to %" PRIu64 " bytes, %" PRIaPGNO " pages",
mdbx_info("resize datafile to %" PRIuSIZE " bytes, %" PRIaPGNO " pages",
expected_bytes, bytes2pgno(env, expected_bytes));
err = mdbx_ftruncate(env->me_fd, expected_bytes);
if (unlikely(err != MDBX_SUCCESS)) {
mdbx_error("error %d, while resize datafile to %" PRIu64
mdbx_error("error %d, while resize datafile to %" PRIuSIZE
" bytes, %" PRIaPGNO " pages",
rc, expected_bytes, bytes2pgno(env, expected_bytes));
return err;
@@ -5048,15 +5400,30 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, int lck_rc) {
return MDBX_WANNA_RECOVERY /* LY: could not recovery/rollback */;
}
const MDBX_meta *const meta0 = METAPAGE(env, 0);
const MDBX_meta *const meta1 = METAPAGE(env, 1);
const MDBX_meta *const meta2 = METAPAGE(env, 2);
txnid_t undo_txnid = 0;
while (
(head != meta0 && mdbx_meta_txnid_fluid(env, meta0) == undo_txnid) ||
(head != meta1 && mdbx_meta_txnid_fluid(env, meta1) == undo_txnid) ||
(head != meta2 && mdbx_meta_txnid_fluid(env, meta2) == undo_txnid))
undo_txnid += 1;
if (unlikely(undo_txnid >= meta.mm_txnid_a)) {
mdbx_fatal("rollback failed: no suitable txnid (0,1,2) < %" PRIaTXN,
meta.mm_txnid_a);
return MDBX_PANIC /* LY: could not recovery/rollback */;
}
/* LY: rollback weak checkpoint */
mdbx_trace("rollback: from %" PRIaTXN ", to %" PRIaTXN, head_txnid,
meta.mm_txnid_a);
mdbx_trace("rollback: from %" PRIaTXN ", to %" PRIaTXN " as %" PRIaTXN,
head_txnid, meta.mm_txnid_a, undo_txnid);
mdbx_ensure(env, head_txnid == mdbx_meta_txnid_stable(env, head));
if (env->me_flags & MDBX_WRITEMAP) {
head->mm_txnid_a = 0;
head->mm_txnid_a = undo_txnid;
head->mm_datasync_sign = MDBX_DATASIGN_WEAK;
head->mm_txnid_b = 0;
head->mm_txnid_b = undo_txnid;
const size_t offset =
((uint8_t *)container_of(head, MDBX_page, mp_meta)) -
env->me_dxb_mmap.dxb;
@@ -5066,7 +5433,7 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, int lck_rc) {
err = mdbx_msync(&env->me_dxb_mmap, paged_offset, paged_length, false);
} else {
MDBX_meta rollback = *head;
mdbx_meta_set_txnid(env, &rollback, 0);
mdbx_meta_set_txnid(env, &rollback, undo_txnid);
rollback.mm_datasync_sign = MDBX_DATASIGN_WEAK;
err = mdbx_pwrite(env->me_fd, &rollback, sizeof(MDBX_meta),
(uint8_t *)head - (uint8_t *)env->me_map);
@@ -5075,7 +5442,7 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, int lck_rc) {
return err;
mdbx_invalidate_cache(env->me_map, pgno2bytes(env, NUM_METAS));
mdbx_ensure(env, 0 == mdbx_meta_txnid_fluid(env, head));
mdbx_ensure(env, undo_txnid == mdbx_meta_txnid_fluid(env, head));
mdbx_ensure(env, 0 == mdbx_meta_eq_mask(env));
continue;
}
@@ -7059,7 +7426,8 @@ int mdbx_cursor_put(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
} else {
rc = mdbx_cursor_set(mc, key, &d2, MDBX_SET, &exact);
}
if ((flags & MDBX_NOOVERWRITE) && rc == 0) {
if ((flags & MDBX_NOOVERWRITE) &&
(rc == MDBX_SUCCESS || rc == MDBX_EKEYMISMATCH)) {
mdbx_debug("duplicate key [%s]", DKEY(key));
*data = d2;
return MDBX_KEYEXIST;
@@ -7273,9 +7641,11 @@ int mdbx_cursor_put(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data,
memcpy((char *)mp + mp->mp_upper + PAGEHDRSZ,
(char *)fp + fp->mp_upper + PAGEHDRSZ,
olddata.iov_len - fp->mp_upper - PAGEHDRSZ);
memcpy((char *)(&mp->mp_ptrs), (char *)(&fp->mp_ptrs),
NUMKEYS(fp) * sizeof(mp->mp_ptrs[0]));
for (i = 0; i < NUMKEYS(fp); i++) {
mdbx_cassert(mc, fp->mp_ptrs[i] + offset <= UINT16_MAX);
mp->mp_ptrs[i] = (indx_t)(fp->mp_ptrs[i] + offset);
mdbx_cassert(mc, mp->mp_ptrs[i] + offset <= UINT16_MAX);
mp->mp_ptrs[i] += (indx_t)offset;
}
}
}
@@ -8796,27 +9166,22 @@ static int mdbx_rebalance(MDBX_cursor *mc) {
if (unlikely(rc))
return rc;
/* Adjust cursors pointing to mp */
const MDBX_dbi dbi = mc->mc_dbi;
for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[dbi]; m2;
m2 = m2->mc_next) {
MDBX_cursor *m3 =
(mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
if (!(m3->mc_flags & C_INITIALIZED) || (m3->mc_snum < mc->mc_snum))
continue;
if (m3->mc_pg[0] == mp) {
m3->mc_snum = 0;
m3->mc_top = 0;
m3->mc_flags &= ~C_INITIALIZED;
}
}
mc->mc_snum = 0;
mc->mc_top = 0;
mc->mc_flags &= ~C_INITIALIZED;
{
MDBX_cursor *m2, *m3;
MDBX_dbi dbi = mc->mc_dbi;
for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) {
if (mc->mc_flags & C_SUB)
m3 = &m2->mc_xcursor->mx_cursor;
else
m3 = m2;
if (!(m3->mc_flags & C_INITIALIZED) || (m3->mc_snum < mc->mc_snum))
continue;
if (m3->mc_pg[0] == mp) {
m3->mc_snum = 0;
m3->mc_top = 0;
m3->mc_flags &= ~C_INITIALIZED;
}
}
}
} else if (IS_BRANCH(mp) && NUMKEYS(mp) == 1) {
int i;
mdbx_debug("collapsing root page!");
@@ -10349,13 +10714,13 @@ int mdbx_dbi_open_ex(MDBX_txn *txn, const char *table_name, unsigned user_flags,
!strncmp(table_name, txn->mt_dbxs[scan].md_name.iov_base, len)) {
*dbi = scan;
rc = mdbx_dbi_bind(txn, scan, user_flags, keycmp, datacmp);
goto unlock_return_rc;
goto bailout;
}
}
if (unlikely(slot >= env->me_maxdbs)) {
rc = MDBX_DBS_FULL;
goto unlock_return_rc;
goto bailout;
}
unsigned dbflag = DB_FRESH | DB_VALID | DB_USRVALID;
@@ -10405,7 +10770,6 @@ int mdbx_dbi_open_ex(MDBX_txn *txn, const char *table_name, unsigned user_flags,
*dbi = slot;
}
unlock_return_rc:
mdbx_ensure(env, mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS);
return rc;
}

View File

@@ -700,39 +700,6 @@ int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length) {
/*----------------------------------------------------------------------------*/
int mdbx_thread_key_create(mdbx_thread_key_t *key) {
#if defined(_WIN32) || defined(_WIN64)
*key = TlsAlloc();
return (*key != TLS_OUT_OF_INDEXES) ? MDBX_SUCCESS : GetLastError();
#else
return pthread_key_create(key, mdbx_rthc_dtor);
#endif
}
void mdbx_thread_key_delete(mdbx_thread_key_t key) {
#if defined(_WIN32) || defined(_WIN64)
mdbx_ensure(NULL, TlsFree(key));
#else
mdbx_ensure(NULL, pthread_key_delete(key) == 0);
#endif
}
void *mdbx_thread_rthc_get(mdbx_thread_key_t key) {
#if defined(_WIN32) || defined(_WIN64)
return TlsGetValue(key);
#else
return pthread_getspecific(key);
#endif
}
void mdbx_thread_rthc_set(mdbx_thread_key_t key, const void *value) {
#if defined(_WIN32) || defined(_WIN64)
mdbx_ensure(NULL, TlsSetValue(key, (void *)value));
#else
mdbx_ensure(NULL, pthread_setspecific(key, value) == 0);
#endif
}
int mdbx_thread_create(mdbx_thread_t *thread,
THREAD_RESULT(THREAD_CALL *start_routine)(void *),
void *arg) {

View File

@@ -103,24 +103,47 @@ typedef struct {
typedef pthread_mutex_t mdbx_fastmutex_t;
#endif /* Platform */
/* *INDENT-OFF* */
/* clang-format off */
#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>)
#include <sys/stat.h>
#endif
#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>)
#include <sys/types.h>
#endif
#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>)
#include <sys/file.h>
#endif
/* *INDENT-ON* */
/* clang-format on */
#ifndef SSIZE_MAX
#define SSIZE_MAX INTPTR_MAX
#endif
#ifdef HAVE_SYS_STAT_H
#include <sys/stat.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_SYS_FILE_H
#include <sys/file.h>
#endif
#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
defined(i486) || defined(__i486) || defined(__i486__) || \
defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
#ifndef __ia32__
/* LY: define neutral __ia32__ for x86 and x86-64 archs */
#define __ia32__ 1
#endif /* __ia32__ */
#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
defined(__amd64) || defined(_M_X64))
/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
#define __amd64__ 1
#endif /* __amd64__ */
#endif /* all x86 */
#if !defined(UNALIGNED_OK)
#if defined(__i386) || defined(__x86_64__) || defined(_M_IX86) || \
defined(_M_X64) || defined(i386) || defined(_X86_) || defined(__i386__) || \
defined(_X86_64_)
#if (defined(__ia32__) || defined(__e2k__) || \
defined(__ARM_FEATURE_UNALIGNED)) && \
!defined(__ALIGNED__)
#define UNALIGNED_OK 1
#else
#define UNALIGNED_OK 0
@@ -135,26 +158,15 @@ typedef pthread_mutex_t mdbx_fastmutex_t;
/*----------------------------------------------------------------------------*/
/* Compiler's includes for builtins/intrinsics */
#ifdef _MSC_VER
#if _MSC_FULL_VER < 190024215
#if _MSC_FULL_VER < 180040629 && defined(_M_IX86)
#error Please use Visual Studio 2015 (MSC 19.0) or newer for 32-bit target.
#else
#pragma message( \
"It is recommended to use Visual Studio 2015 (MSC 19.0) or newer.")
#endif
#endif
#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
#include <intrin.h>
#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
#if defined(__i386__) || defined(__x86_64__)
#include <cpuid.h>
#if defined(__ia32__) || defined(__e2k__)
#include <x86intrin.h>
#endif
#elif defined(__INTEL_COMPILER)
#include <intrin.h>
#endif /* __ia32__ */
#if defined(__ia32__)
#include <cpuid.h>
#endif /* __ia32__ */
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
#include <mbarrier.h>
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
@@ -173,8 +185,10 @@ typedef pthread_mutex_t mdbx_fastmutex_t;
#pragma gcc_extensions
#elif defined(__SNC__)
/* Sony PS3 - troubles ? */
#elif defined(__hppa__) || defined(__hppa)
#include <machine/inline.h>
#else
#error Unknown C compiler, please use GNU C 5.x or newer
#error Unsupported C compiler, please use GNU C 4.4 or newer
#endif /* Compiler */
/*----------------------------------------------------------------------------*/
@@ -183,45 +197,70 @@ typedef pthread_mutex_t mdbx_fastmutex_t;
#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
!defined(__ORDER_BIG_ENDIAN__)
#if defined(HAVE_ENDIAN_H)
/* *INDENT-OFF* */
/* clang-format off */
#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID__) || \
defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
#include <endian.h>
#elif defined(HAVE_SYS_PARAM_H)
#include <sys/param.h> /* for endianness */
#elif defined(HAVE_NETINET_IN_H) && defined(HAVE_RESOLV_H)
#include <netinet/in.h>
#include <resolv.h> /* defines BYTE_ORDER on HPUX and Solaris */
#endif
#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
#include <machine/endian.h>
#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
#include <sys/isa_defs.h>
#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
(__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
#include <sys/endian.h>
#include <sys/types.h>
#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
defined(__NETBSD__) || defined(__NetBSD__) || \
defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
#include <sys/param.h>
#endif /* OS */
/* *INDENT-ON* */
/* clang-format on */
#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
#define __BYTE_ORDER__ __BYTE_ORDER
#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
#define __BYTE_ORDER__ _BYTE_ORDER
#else
#define __ORDER_LITTLE_ENDIAN__ 1234
#define __ORDER_BIG_ENDIAN__ 4321
#if defined(__LITTLE_ENDIAN__) || defined(_LITTLE_ENDIAN) || \
#if defined(__LITTLE_ENDIAN__) || \
(defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
defined(__i386) || defined(__x86_64__) || defined(_M_IX86) || \
defined(_M_X64) || defined(i386) || defined(_X86_) || defined(__i386__) || \
defined(_X86_64_) || defined(_M_ARM) || defined(_M_ARM64) || \
defined(__e2k__)
defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
defined(__WINDOWS__)
#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
#elif defined(__BIG_ENDIAN__) || defined(_BIG_ENDIAN) || defined(__ARMEB__) || \
defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || \
defined(_MIPSEB) || defined(__MIPSEB) || defined(_M_IA64)
#elif defined(__BIG_ENDIAN__) || \
(defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
#else
#error __BYTE_ORDER__ should be defined.
#endif
#endif /* Arch */
#endif
#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \
__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__
#error Unsupported byte order.
#endif
/*----------------------------------------------------------------------------*/
/* Memory/Compiler barriers, cache coherence */
@@ -286,17 +325,14 @@ static __inline void mdbx_memory_barrier(void) {
/*----------------------------------------------------------------------------*/
/* Cache coherence and invalidation */
#if defined(__i386__) || defined(__x86_64__) || defined(_M_AMD64) || \
defined(_M_IX86) || defined(__i386) || defined(__amd64) || \
defined(i386) || defined(__x86_64) || defined(_AMD64_) || defined(_M_X64)
#define MDBX_CACHE_IS_COHERENT 1
#elif defined(__hppa) || defined(__hppa__)
#define MDBX_CACHE_IS_COHERENT 1
#endif
#ifndef MDBX_CACHE_IS_COHERENT
#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
defined(__hppa__)
#define MDBX_CACHE_IS_COHERENT 1
#else
#define MDBX_CACHE_IS_COHERENT 0
#endif
#endif /* MDBX_CACHE_IS_COHERENT */
#ifndef MDBX_CACHELINE_SIZE
#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE)
@@ -308,29 +344,29 @@ static __inline void mdbx_memory_barrier(void) {
#endif
#endif /* MDBX_CACHELINE_SIZE */
#ifndef __cache_aligned
#define __cache_aligned __aligned(MDBX_CACHELINE_SIZE)
#endif
#if MDBX_CACHE_IS_COHERENT
#define mdbx_coherent_barrier() mdbx_compiler_barrier()
#else
#define mdbx_coherent_barrier() mdbx_memory_barrier()
#endif
#if defined(__mips) && defined(__linux)
#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
defined(__mips64) || defined(_M_MRX000) || defined(_MIPS_)
/* Only MIPS has explicit cache control */
#include <asm/cachectl.h>
#include <sys/cachectl.h>
#endif
static __inline void mdbx_invalidate_cache(void *addr, size_t nbytes) {
mdbx_coherent_barrier();
#if defined(__mips) && defined(__linux)
#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
defined(__mips64) || defined(_M_MRX000) || defined(_MIPS_)
#if defined(DCACHE)
/* MIPS has cache coherency issues.
* Note: for any nbytes >= on-chip cache size, entire is flushed. */
cacheflush(addr, nbytes, DCACHE);
#elif defined(_M_MRX000) || defined(_MIPS_)
#else
#error "Sorry, cacheflush() for MIPS not implemented"
#endif /* __mips__ */
#else
/* LY: assume no relevant mmap/dcache issues. */
(void)addr;
@@ -433,10 +469,6 @@ int mdbx_thread_create(mdbx_thread_t *thread,
THREAD_RESULT(THREAD_CALL *start_routine)(void *),
void *arg);
int mdbx_thread_join(mdbx_thread_t thread);
int mdbx_thread_key_create(mdbx_thread_key_t *key);
void mdbx_thread_key_delete(mdbx_thread_key_t key);
void *mdbx_thread_rthc_get(mdbx_thread_key_t key);
void mdbx_thread_rthc_set(mdbx_thread_key_t key, const void *value);
int mdbx_filesync(mdbx_filehandle_t fd, bool fullsync);
int mdbx_filesize_sync(mdbx_filehandle_t fd);
@@ -504,7 +536,13 @@ void mdbx_osal_jitter(bool tiny);
#else
#define MDBX_OSAL_LOCK pthread_mutex_t
#define MDBX_OSAL_LOCK_SIGN UINT32_C(0x8017)
#endif
#endif /* MDBX_OSAL_LOCK */
#ifdef MDBX_OSAL_LOCK
#define MDBX_OSAL_LOCK_SIZE sizeof(MDBX_OSAL_LOCK)
#else
#define MDBX_OSAL_LOCK_SIZE 0
#endif /* MDBX_OSAL_LOCK_SIZE */
int mdbx_lck_init(MDBX_env *env);
@@ -516,8 +554,8 @@ void mdbx_lck_destroy(MDBX_env *env);
int mdbx_rdt_lock(MDBX_env *env);
void mdbx_rdt_unlock(MDBX_env *env);
int mdbx_txn_lock(MDBX_env *env, bool dontwait);
void mdbx_txn_unlock(MDBX_env *env);
LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dontwait);
LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env);
int mdbx_rpid_set(MDBX_env *env);
int mdbx_rpid_clear(MDBX_env *env);

View File

@@ -77,7 +77,7 @@ int exclusive = 2;
int envflags = MDBX_RDONLY;
MDBX_env *env;
MDBX_txn *txn, *locktxn;
MDBX_txn *txn;
MDBX_envinfo envinfo;
MDBX_stat envstat;
size_t maxkeysize, userdb_count, skipped_subdb;
@@ -763,6 +763,7 @@ int main(int argc, char *argv[]) {
char *envname;
int problems_maindb = 0, problems_freedb = 0, problems_meta = 0;
int dont_traversal = 0;
bool locked = false;
double elapsed;
#if defined(_WIN32) || defined(_WIN64)
@@ -836,8 +837,8 @@ int main(int argc, char *argv[]) {
#endif /* !WINDOWS */
envname = argv[optind];
print("Running mdbx_chk for '%s' in %s mode...\n", envname,
(envflags & MDBX_RDONLY) ? "read-only" : "write-lock");
print("Running mdbx_chk for 'read-%s' in %s mode...\n", envname,
(envflags & MDBX_RDONLY) ? "read" : "write");
fflush(NULL);
rc = mdbx_env_create(&env);
@@ -862,13 +863,19 @@ int main(int argc, char *argv[]) {
if (verbose)
print(" - %s mode\n", exclusive ? "monopolistic" : "cooperative");
if (!(envflags & MDBX_RDONLY)) {
rc = mdbx_txn_begin(env, NULL, 0, &locktxn);
if (rc) {
error("mdbx_txn_begin(lock-write) failed, error %d %s\n", rc,
mdbx_strerror(rc));
if ((envflags & MDBX_RDONLY) == 0) {
rc = mdbx_txn_lock(env, false);
if (rc != MDBX_SUCCESS) {
error("mdbx_txn_lock failed, error %d %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
locked = true;
}
rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn);
if (rc) {
error("mdbx_txn_begin() failed, error %d %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_env_get_maxkeysize(env);
@@ -879,13 +886,6 @@ int main(int argc, char *argv[]) {
}
maxkeysize = rc;
rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn);
if (rc) {
error("mdbx_txn_begin(read-only) failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_env_info(env, &envinfo, sizeof(envinfo));
if (rc) {
error("mdbx_env_info failed, error %d %s\n", rc, mdbx_strerror(rc));
@@ -950,14 +950,14 @@ int main(int argc, char *argv[]) {
if (verbose)
print(" - performs full check recent-txn-id with meta-pages\n");
problems_meta += check_meta_head(true);
} else if (locktxn) {
} else if (locked) {
if (verbose)
print(" - performs lite check recent-txn-id with meta-pages (not a "
"monopolistic mode)\n");
problems_meta += check_meta_head(false);
} else if (verbose) {
print(" - skip check recent-txn-id with meta-pages (monopolistic or "
"write-lock mode only)\n");
"read-write mode only)\n");
}
if (!dont_traversal) {
@@ -1079,7 +1079,7 @@ int main(int argc, char *argv[]) {
}
if (problems_maindb == 0 && problems_freedb == 0) {
if (!dont_traversal && (exclusive || locktxn)) {
if (!dont_traversal && (exclusive || (envflags & MDBX_RDONLY) == 0)) {
if (walk.pgcount != lastpgno - freedb_pages) {
error("used pages mismatch (%" PRIu64 " != %" PRIu64 ")\n",
walk.pgcount, lastpgno - freedb_pages);
@@ -1090,7 +1090,7 @@ int main(int argc, char *argv[]) {
}
} else if (verbose) {
print(" - skip check used and gc pages (btree-traversal with "
"monopolistic or write-lock mode only)\n");
"monopolistic or read-write mode only)\n");
}
if (!process_db(MAIN_DBI, NULL, handle_maindb, true)) {
@@ -1102,8 +1102,8 @@ int main(int argc, char *argv[]) {
bailout:
if (txn)
mdbx_txn_abort(txn);
if (locktxn)
mdbx_txn_abort(locktxn);
if (locked)
mdbx_txn_unlock(env);
if (env)
mdbx_env_close(env);
fflush(NULL);

View File

@@ -18,8 +18,8 @@
#error "API version mismatch!"
#endif
#define MDBX_VERSION_RELEASE 0
#define MDBX_VERSION_REVISION 0
#define MDBX_VERSION_RELEASE 3
#define MDBX_VERSION_REVISION 1
/*LIBMDBX_EXPORTS*/ const mdbx_version_info mdbx_version = {
MDBX_VERSION_MAJOR,

View File

@@ -76,14 +76,6 @@
#include <unordered_set>
#include <vector>
#ifdef _MSC_VER
#include <intrin.h>
#endif
#if defined(__i386__) || defined(__x86_64__)
#include <x86intrin.h>
#endif
#include "../mdbx.h"
#include "../src/defs.h"
#include "../src/osal.h"

View File

@@ -278,8 +278,6 @@ void dump(const char *title) {
logging::local_suffix indent(title);
for (auto i = global::actors.begin(); i != global::actors.end(); ++i) {
const std::string tableid =
i->space_id ? "MAINDB" : ("SUB#" + std::to_string(i->space_id));
log_info("#%u, testcase %s, space_id/table %u\n", i->actor_id,
testcase2str(i->testcase), i->space_id);
indent.push();
@@ -326,6 +324,11 @@ void dump(const char *title) {
else
log_info("no-delay\n");
if (i->params.inject_writefaultn)
log_info("inject-writefault on %u ops\n", i->params.inject_writefaultn);
else
log_info("no-inject-writefault\n");
log_info("limits: readers %u, tables %u\n", i->params.max_readers,
i->params.max_tables);

View File

@@ -218,6 +218,7 @@ struct actor_params_pod {
unsigned delaystart;
unsigned waitfor_nops;
unsigned inject_writefaultn;
unsigned max_readers;
unsigned max_tables;
@@ -244,7 +245,7 @@ void dump(const char *title = "config-dump: ");
struct actor_params : public config::actor_params_pod {
std::string pathname_log;
std::string pathname_db;
void set_defaults(void);
void set_defaults(const std::string &tmpdir);
};
struct actor_config : public config::actor_config_pod {

View File

@@ -285,3 +285,5 @@ void log_trouble(const char *where, const char *what, int errnum) {
bool log_enabled(const logging::loglevel priority) {
return (priority >= logging::level);
}
void log_flush(void) { fflushall(); }

View File

@@ -81,6 +81,7 @@ void __printf_args(1, 2) log_warning(const char *msg, ...);
void __printf_args(1, 2) log_error(const char *msg, ...);
void log_trouble(const char *where, const char *what, int errnum);
void log_flush(void);
bool log_enabled(const logging::loglevel priority);
#ifdef _DEBUG

View File

@@ -22,21 +22,16 @@ void __noreturn usage(void) {
//-----------------------------------------------------------------------------
void actor_params::set_defaults(void) {
void actor_params::set_defaults(const std::string &tmpdir) {
pathname_log = "";
loglevel =
#ifdef NDEBUG
logging::notice;
logging::info;
#else
logging::trace;
#endif
pathname_db =
#ifdef __linux__
"/dev/shm/test_tmpdb.mdbx";
#else
"test_tmpdb.mdbx";
#endif
pathname_db = tmpdir + "mdbx-test.db";
mode_flags = MDBX_NOSUBDIR | MDBX_WRITEMAP | MDBX_MAPASYNC | MDBX_NORDAHEAD |
MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_LIFORECLAIM;
table_flags = MDBX_DUPSORT;
@@ -65,6 +60,7 @@ void actor_params::set_defaults(void) {
delaystart = 0;
waitfor_nops = 0;
inject_writefaultn = 0;
drop_table = false;
@@ -135,7 +131,7 @@ int main(int argc, char *const argv[]) {
: EXIT_FAILURE;
actor_params params;
params.set_defaults();
params.set_defaults(osal_tempdir());
global::config::dump_config = true;
logging::setup((logging::loglevel)params.loglevel, "main");
unsigned last_space_id = 0;
@@ -219,6 +215,9 @@ int main(int argc, char *const argv[]) {
if (config::parse_option(argc, argv, narg, "wait4ops", params.waitfor_nops,
config::decimal))
continue;
if (config::parse_option(argc, argv, narg, "inject-writefault",
params.inject_writefaultn, config::decimal))
continue;
if (config::parse_option(argc, argv, narg, "drop", params.drop_table))
continue;
if (config::parse_option(argc, argv, narg, "dump-config",

View File

@@ -272,3 +272,22 @@ void osal_udelay(unsigned us) {
}
bool osal_istty(int fd) { return isatty(fd) == 1; }
std::string osal_tempdir(void) {
const char *tempdir = getenv("TMPDIR");
if (!tempdir)
tempdir = getenv("TMP");
if (!tempdir)
tempdir = getenv("TEMPDIR");
if (!tempdir)
tempdir = getenv("TEMP");
if (tempdir) {
std::string dir(tempdir);
if (!dir.empty() && dir.at(dir.length() - 1) != '/')
dir.append("/");
return dir;
}
if (access("/dev/shm/", R_OK | W_OK | X_OK) == 0)
return "/dev/shm/";
return "";
}

View File

@@ -305,3 +305,9 @@ void osal_udelay(unsigned us) {
}
bool osal_istty(int fd) { return _isatty(fd) != 0; }
std::string osal_tempdir(void) {
char buf[MAX_PATH + 1];
DWORD len = GetTempPathA(sizeof(buf), buf);
return std::string(buf, len);
}

View File

@@ -31,6 +31,7 @@ int osal_delay(unsigned seconds);
void osal_udelay(unsigned us);
void osal_yield(void);
bool osal_istty(int fd);
std::string osal_tempdir(void);
#ifdef _MSC_VER
#ifndef STDIN_FILENO

View File

@@ -123,7 +123,7 @@ void testcase::db_prepare() {
if (config.params.loglevel <= logging::verbose)
mdbx_dbg_opts |= MDBX_DBG_PRINT;
int rc = mdbx_setup_debug(mdbx_dbg_opts, mdbx_logger);
log_info("set mdbx debug-opts: 0x%02x", rc);
log_trace("set mdbx debug-opts: 0x%02x", rc);
MDBX_env *env = nullptr;
rc = mdbx_env_create(&env);
@@ -204,6 +204,7 @@ void testcase::txn_end(bool abort) {
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_txn_abort()", rc);
} else {
txn_inject_writefault(txn);
int rc = mdbx_txn_commit(txn);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_txn_commit()", rc);
@@ -218,6 +219,27 @@ void testcase::txn_restart(bool abort, bool readonly, unsigned flags) {
txn_begin(readonly, flags);
}
void testcase::txn_inject_writefault(void) {
if (txn_guard)
txn_inject_writefault(txn_guard.get());
}
void testcase::txn_inject_writefault(MDBX_txn *txn) {
if (config.params.inject_writefaultn && txn) {
if (config.params.inject_writefaultn <= nops_completed &&
(mdbx_txn_flags(txn) & MDBX_RDONLY) == 0) {
log_info("== txn_inject_writefault(): got %u nops or more, inject FAULT",
config.params.inject_writefaultn);
log_flush();
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
TerminateProcess(GetCurrentProcess(), 42);
#else
raise(SIGKILL);
#endif
}
}
}
bool testcase::wait4start() {
if (config.wait4id) {
log_trace(">> wait4start(%u)", config.wait4id);

View File

@@ -107,6 +107,8 @@ protected:
void txn_begin(bool readonly, unsigned flags = 0);
void txn_end(bool abort);
void txn_restart(bool abort, bool readonly, unsigned flags = 0);
void txn_inject_writefault(void);
void txn_inject_writefault(MDBX_txn *txn);
void fetch_canary();
void update_canary(uint64_t increment);
void kick_progress(bool active) const;

View File

@@ -14,44 +14,10 @@
#include "test.h"
#include <float.h>
#ifdef HAVE_IEEE754_H
#if defined(HAVE_IEEE754_H) || __has_include(<ieee754.h>)
#include <ieee754.h>
#endif
/* Compiler's includes for builtins/intrinsics */
#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
#include <intrin.h>
#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
#if defined(__ia32__) || defined(__e2k__)
#include <x86intrin.h>
#endif /* __ia32__ */
#if defined(__ia32__)
#include <cpuid.h>
#endif /* __ia32__ */
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
#include <mbarrier.h>
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
#include <machine/sys/inline.h>
#elif defined(__IBMC__) && defined(__powerpc)
#include <atomic.h>
#elif defined(_AIX)
#include <builtins.h>
#include <sys/atomic_op.h>
#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
#include <c_asm.h>
#include <machine/builtins.h>
#elif defined(__MWERKS__)
/* CodeWarrior - troubles ? */
#pragma gcc_extensions
#elif defined(__SNC__)
/* Sony PS3 - troubles ? */
#elif defined(__hppa__) || defined(__hppa)
#include <machine/inline.h>
#else
#error Unsupported C compiler, please use GNU C 4.4 or newer
#endif /* Compiler */
std::string format(const char *fmt, ...) {
va_list ap, ones;
va_start(ap, fmt);
@@ -127,8 +93,22 @@ bool hex2data(const char *hex_begin, const char *hex_end, void *ptr,
//-----------------------------------------------------------------------------
/* TODO: replace my 'libmera' fomr t1ha. */
uint64_t entropy_ticks(void) {
#if defined(EMSCRIPTEN)
return (uint64_t)emscripten_get_now();
#endif /* EMSCRIPTEN */
#if defined(__APPLE__) || defined(__MACH__)
return mach_absolute_time();
#endif /* defined(__APPLE__) || defined(__MACH__) */
#if defined(__sun__) || defined(__sun)
return gethrtime();
#endif /* __sun__ */
#if defined(__GNUC__) || defined(__clang__)
#if defined(__ia64__)
uint64_t ticks;
__asm __volatile("mov %0=ar.itc" : "=r"(ticks));
@@ -145,29 +125,81 @@ uint64_t entropy_ticks(void) {
uint64_t ticks;
__asm __volatile("rpcc %0" : "=r"(ticks));
return ticks;
#elif defined(__sparc_v9__)
uint64_t ticks;
__asm __volatile("rd %%tick, %0" : "=r"(ticks));
return ticks;
#elif defined(__powerpc64__) || defined(__ppc64__)
#elif defined(__sparc__) || defined(__sparc) || defined(__sparc64__) || \
defined(__sparc64) || defined(__sparc_v8plus__) || \
defined(__sparc_v8plus) || defined(__sparc_v8plusa__) || \
defined(__sparc_v8plusa) || defined(__sparc_v9__) || defined(__sparc_v9)
union {
uint64_t u64;
struct {
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
uint32_t h, l;
#else
uint32_t l, h;
#endif
} u32;
} cycles;
#if defined(__sparc_v8plus__) || defined(__sparc_v8plusa__) || \
defined(__sparc_v9__) || defined(__sparc_v8plus) || \
defined(__sparc_v8plusa) || defined(__sparc_v9)
#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || \
defined(__sparc64__) || defined(__sparc64)
__asm __volatile("rd %%tick, %0" : "=r"(cycles.u64));
#else
__asm __volatile("rd %%tick, %1; srlx %1, 32, %0"
: "=r"(cycles.u32.h), "=r"(cycles.u32.l));
#endif /* __sparc64__ */
#else
__asm __volatile(".byte 0x83, 0x41, 0x00, 0x00; mov %%g1, %0"
: "=r"(cycles.u64)
:
: "%g1");
#endif /* __sparc8plus__ || __sparc_v9__ */
return cycles.u64;
#elif (defined(__powerpc64__) || defined(__ppc64__) || defined(__ppc64) || \
defined(__powerpc64))
uint64_t ticks;
__asm __volatile("mfspr %0, 268" : "=r"(ticks));
return ticks;
#elif defined(__ppc__) || defined(__powerpc__)
unsigned tbl, tbu;
/* LY: Here not a problem if a high-part (tbu)
* would been updated during reading. */
__asm __volatile("mftb %0" : "=r"(tbl));
__asm __volatile("mftbu %0" : "=r"(tbu));
return (((uin64_t)tbu0) << 32) | tbl;
#elif (defined(__powerpc__) || defined(__ppc__) || defined(__powerpc) || \
defined(__ppc))
#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
uint64_t ticks;
__asm __volatile("mftb %0" : "=r"(ticks));
*now = ticks;
#else
uint64_t ticks;
uint32_t low, high_before, high_after;
__asm __volatile("mftbu %0; mftb %1; mftbu %2"
: "=r"(high_before), "=r"(low), "=r"(high_after));
ticks = (uint64_t)high_after << 32;
ticks |= low & /* zeroes if high part has changed */
~(high_before - high_after);
#endif
#elif defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 7)
uint64_t virtual_timer;
__asm __volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer));
return virtual_timer;
#elif defined(__ARM_ARCH) && __ARM_ARCH > 5 && __ARM_ARCH < 8
unsigned long pmccntr;
__asm __volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(pmccntr));
return pmccntr;
#elif defined(__mips__) || defined(__mips) || defined(_R4000)
unsigned count;
__asm __volatile("rdhwr %0, $2" : "=r"(count));
return count;
#endif /* arch selector */
#endif /* __GNUC__ || __clang__ */
#if defined(__e2k__) || defined(__elbrus__) || defined(_M_IX86) || \
defined(_M_X64) || defined(__x86_64__) || defined(__i386__)
#if defined(__e2k__) || defined(__ia32__)
return __rdtsc();
#elif defined(_M_ARM)
return __rdpmccntr64();
#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
LARGE_INTEGER PerformanceCount;
if (QueryPerformanceCounter(&PerformanceCount))

View File

@@ -17,33 +17,8 @@
#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
!defined(__ORDER_BIG_ENDIAN__)
#ifndef _MSC_VER
#include <sys/param.h> /* for endianness */
#endif
#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
#define __BYTE_ORDER__ __BYTE_ORDER
#else
#define __ORDER_LITTLE_ENDIAN__ 1234
#define __ORDER_BIG_ENDIAN__ 4321
#if defined(__LITTLE_ENDIAN__) || defined(_LITTLE_ENDIAN) || \
defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
defined(__i386) || defined(__x86_64__) || defined(_M_IX86) || \
defined(_M_X64) || defined(i386) || defined(_X86_) || defined(__i386__) || \
defined(_X86_64_) || defined(_M_ARM) || defined(_M_ARM64) || \
defined(__e2k__)
#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
#elif defined(__BIG_ENDIAN__) || defined(_BIG_ENDIAN) || defined(__ARMEB__) || \
defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || \
defined(_MIPSEB) || defined(__MIPSEB) || defined(_M_IA64)
#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
#else
#error __BYTE_ORDER__ should be defined.
#endif
#endif
#endif
#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \
__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__
@@ -274,7 +249,7 @@ static __inline void memory_barrier(void) {
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
__mf();
#elif defined(__i386__) || defined(__x86_64__)
#elif defined(__ia32__)
_mm_mfence();
#else
#error "Unknown target for Intel Compiler, please report to us."
@@ -293,8 +268,7 @@ static __inline void memory_barrier(void) {
}
static __inline void cpu_relax() {
#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || \
defined(_M_X64)
#if defined(__ia32__)
_mm_pause();
#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) || \
defined(YieldProcessor)