ISAR/libmdbx
3
0
Fork 0
mirror of https://github.com/isar/libmdbx.git synced 2025-04-01 02:32:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

mdbx++: добавление inplace_storage_size_rounding в capacity_policy для буферов (backport).

Browse Source
This commit is contained in:
Леонид Юрьев (Leonid Yuriev) 2025-03-20 01:41:05 +03:00
parent 73d52c1963
commit d8890bc169
1 changed files with 66 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
mdbx.h++
View File

@ -1162,7 +1162,12 @@ template <typename T, typename A> struct swap_alloc<T, A, true> {
} // namespace allocation_aware_details } // namespace allocation_aware_details
struct default_capacity_policy { struct default_capacity_policy {
enum : size_t { extra_inplace_storage = 0, pettiness_threshold = 64, max_reserve = 65536 }; enum : size_t {
extra_inplace_storage = 0,
inplace_storage_size_rounding = 16,
pettiness_threshold = 64,
max_reserve = 65536
};
static MDBX_CXX11_CONSTEXPR size_t round(const size_t value) { static MDBX_CXX11_CONSTEXPR size_t round(const size_t value) {
static_assert((pettiness_threshold & (pettiness_threshold - 1)) == 0, "pettiness_threshold must be a power of 2"); static_assert((pettiness_threshold & (pettiness_threshold - 1)) == 0, "pettiness_threshold must be a power of 2");
@ -1486,6 +1491,10 @@ public:
max_length = MDBX_MAXDATASIZE, max_length = MDBX_MAXDATASIZE,
max_capacity = (max_length / 3u * 4u + 1023u) & ~size_t(1023), max_capacity = (max_length / 3u * 4u + 1023u) & ~size_t(1023),
extra_inplace_storage = reservation_policy::extra_inplace_storage, extra_inplace_storage = reservation_policy::extra_inplace_storage,
inplace_storage_size_rounding =
(alignof(max_align_t) * 2 > size_t(reservation_policy::inplace_storage_size_rounding))
? alignof(max_align_t) * 2
: size_t(reservation_policy::inplace_storage_size_rounding),
pettiness_threshold = reservation_policy::pettiness_threshold pettiness_threshold = reservation_policy::pettiness_threshold
}; };
@ -1529,41 +1538,51 @@ private:
#endif /* __cpp_lib_to_address */ #endif /* __cpp_lib_to_address */
} }
union bin { union alignas(max_align_t) bin {
struct allocated { struct stub_allocated_holder /* используется только для вычисления (минимального необходимого) размера,
с учетом выравнивания */
{
allocator_pointer ptr_; allocator_pointer ptr_;
size_t capacity_bytes_; size_t stub_capacity_bytes_;
constexpr allocated(allocator_pointer ptr, size_t bytes) noexcept : ptr_(ptr), capacity_bytes_(bytes) {}
constexpr allocated(const allocated &) noexcept = default;
constexpr allocated(allocated &&) noexcept = default;
MDBX_CXX17_CONSTEXPR allocated &operator=(const allocated &) noexcept = default;
MDBX_CXX17_CONSTEXPR allocated &operator=(allocated &&) noexcept = default;
}; };
allocated allocated_; enum : byte { lastbyte_poison = 0, lastbyte_inplace_signature = byte(~byte(lastbyte_poison)) };
uint64_t align_hint_;
byte inplace_[(sizeof(allocated) + extra_inplace_storage + 7u) & ~size_t(7)];
static constexpr bool is_suitable_for_inplace(size_t capacity_bytes) noexcept {
static_assert(sizeof(bin) == sizeof(inplace_), "WTF?");
return capacity_bytes < sizeof(bin);
}
enum : byte { lastbyte_inplace_signature = byte(~byte(0)) };
enum : size_t { enum : size_t {
inplace_signature_limit = size_t(lastbyte_inplace_signature) inplace_signature_limit = size_t(lastbyte_inplace_signature)
<< (sizeof(size_t /* allocated::capacity_bytes_ */) - 1) * CHAR_BIT << (sizeof(size_t /* allocated::capacity_bytes_ */) - 1) * CHAR_BIT,
inplace_size_rounding = size_t(inplace_storage_size_rounding) - 1,
inplace_size =
(sizeof(stub_allocated_holder) + extra_inplace_storage + inplace_size_rounding) & ~inplace_size_rounding
}; };
constexpr byte inplace_lastbyte() const noexcept { return inplace_[sizeof(bin) - 1]; } struct capacity_holder {
MDBX_CXX17_CONSTEXPR byte &inplace_lastbyte() noexcept { return inplace_[sizeof(bin) - 1]; } byte pad_[inplace_size - sizeof(allocator_pointer)];
size_t bytes_;
};
struct inplace_flag_holder {
byte buffer_[inplace_size - sizeof(byte)];
byte lastbyte_;
};
allocator_pointer allocated_ptr_;
capacity_holder capacity_;
inplace_flag_holder inplace_;
static constexpr bool is_suitable_for_inplace(size_t capacity_bytes) noexcept {
static_assert((size_t(reservation_policy::inplace_storage_size_rounding) &
(size_t(reservation_policy::inplace_storage_size_rounding) - 1)) == 0,
"CAPACITY_POLICY::inplace_storage_size_rounding must be power of 2");
static_assert(sizeof(bin) == sizeof(inplace_) && sizeof(bin) == sizeof(capacity_), "WTF?");
return capacity_bytes < sizeof(bin);
}
constexpr bool is_inplace() const noexcept { constexpr bool is_inplace() const noexcept {
static_assert(size_t(inplace_signature_limit) > size_t(max_capacity), "WTF?"); static_assert(size_t(inplace_signature_limit) > size_t(max_capacity), "WTF?");
static_assert(std::numeric_limits<size_t>::max() - (std::numeric_limits<size_t>::max() >> CHAR_BIT) == static_assert(std::numeric_limits<size_t>::max() - (std::numeric_limits<size_t>::max() >> CHAR_BIT) ==
inplace_signature_limit, inplace_signature_limit,
"WTF?"); "WTF?");
return inplace_lastbyte() == lastbyte_inplace_signature; return inplace_.lastbyte_ == lastbyte_inplace_signature;
} }
constexpr bool is_allocated() const noexcept { return !is_inplace(); } constexpr bool is_allocated() const noexcept { return !is_inplace(); }
@ -1571,26 +1590,27 @@ private:
if (destroy_ptr) { if (destroy_ptr) {
MDBX_CONSTEXPR_ASSERT(is_allocated()); MDBX_CONSTEXPR_ASSERT(is_allocated());
/* properly destroy allocator::pointer */ /* properly destroy allocator::pointer */
allocated_.~allocated(); allocated_ptr_.~allocator_pointer();
} }
if (::std::is_trivial<allocator_pointer>::value) if (::std::is_trivial<allocator_pointer>::value)
/* workaround for "uninitialized" warning from some compilers */ /* workaround for "uninitialized" warning from some compilers */
memset(&allocated_.ptr_, 0, sizeof(allocated_.ptr_)); memset(&allocated_ptr_, 0, sizeof(allocated_ptr_));
inplace_lastbyte() = lastbyte_inplace_signature; inplace_.lastbyte_ = lastbyte_inplace_signature;
MDBX_CONSTEXPR_ASSERT(is_inplace() && address() == inplace_ && is_suitable_for_inplace(capacity())); MDBX_CONSTEXPR_ASSERT(is_inplace() && address() == inplace_.buffer_ && is_suitable_for_inplace(capacity()));
return address(); return address();
} }
template <bool construct_ptr> template <bool construct_ptr>
MDBX_CXX17_CONSTEXPR byte *make_allocated(allocator_pointer ptr, size_t capacity_bytes) noexcept { MDBX_CXX17_CONSTEXPR byte *make_allocated(allocator_pointer ptr, size_t capacity_bytes) noexcept {
MDBX_CONSTEXPR_ASSERT(inplace_signature_limit > capacity_bytes); MDBX_CONSTEXPR_ASSERT(inplace_signature_limit > capacity_bytes);
if (construct_ptr) if (construct_ptr) {
/* properly construct allocator::pointer */ /* properly construct allocator::pointer */
new (&allocated_) allocated(ptr, capacity_bytes); new (&allocated_ptr_) allocator_pointer(ptr);
else { capacity_.bytes_ = capacity_bytes;
} else {
MDBX_CONSTEXPR_ASSERT(is_allocated()); MDBX_CONSTEXPR_ASSERT(is_allocated());
allocated_.ptr_ = ptr; allocated_ptr_ = ptr;
allocated_.capacity_bytes_ = capacity_bytes; capacity_.bytes_ = capacity_bytes;
} }
MDBX_CONSTEXPR_ASSERT(is_allocated() && address() == to_address(ptr) && capacity() == capacity_bytes); MDBX_CONSTEXPR_ASSERT(is_allocated() && address() == to_address(ptr) && capacity() == capacity_bytes);
return address(); return address();
@ -1608,16 +1628,17 @@ private:
MDBX_CXX20_CONSTEXPR ~bin() { MDBX_CXX20_CONSTEXPR ~bin() {
if (is_allocated()) if (is_allocated())
/* properly destroy allocator::pointer */ /* properly destroy allocator::pointer */
allocated_.~allocated(); allocated_ptr_.~allocator_pointer();
} }
MDBX_CXX20_CONSTEXPR bin(bin &&ditto) noexcept { MDBX_CXX20_CONSTEXPR bin(bin &&ditto) noexcept {
if (ditto.is_inplace()) { if (ditto.is_inplace()) {
// micro-optimization: don't use make_inplace<> here // micro-optimization: don't use make_inplace<> here
// since memcpy() will copy the flag. // since memcpy() will copy the flag.
memcpy(inplace_, ditto.inplace_, sizeof(inplace_)); memcpy(&inplace_, &ditto.inplace_, sizeof(inplace_));
MDBX_CONSTEXPR_ASSERT(is_inplace()); MDBX_CONSTEXPR_ASSERT(is_inplace());
} else { } else {
new (&allocated_) allocated(::std::move(ditto.allocated_)); new (&allocated_ptr_) allocator_pointer(::std::move(ditto.allocated_ptr_));
capacity_.bytes_ = ditto.capacity_.bytes_;
ditto.make_inplace<true>(); ditto.make_inplace<true>();
MDBX_CONSTEXPR_ASSERT(is_allocated()); MDBX_CONSTEXPR_ASSERT(is_allocated());
} }
@ -1629,13 +1650,13 @@ private:
// since memcpy() will copy the flag. // since memcpy() will copy the flag.
if (is_allocated()) if (is_allocated())
/* properly destroy allocator::pointer */ /* properly destroy allocator::pointer */
allocated_.~allocated(); allocated_ptr_.~allocator_pointer();
memcpy(inplace_, ditto.inplace_, sizeof(inplace_)); memcpy(&inplace_, &ditto.inplace_, sizeof(inplace_));
MDBX_CONSTEXPR_ASSERT(is_inplace()); MDBX_CONSTEXPR_ASSERT(is_inplace());
} else if (is_inplace()) } else if (is_inplace())
make_allocated<true>(ditto.allocated_.ptr_, ditto.allocated_.capacity_bytes_); make_allocated<true>(ditto.allocated_ptr_, ditto.capacity_.bytes_);
else else
make_allocated<false>(ditto.allocated_.ptr_, ditto.allocated_.capacity_bytes_); make_allocated<false>(ditto.allocated_ptr_, ditto.capacity_.bytes_);
return *this; return *this;
} }
@ -1656,12 +1677,12 @@ private:
} }
constexpr const byte *address() const noexcept { constexpr const byte *address() const noexcept {
return is_inplace() ? inplace_ : static_cast<const byte *>(to_address(allocated_.ptr_)); return is_inplace() ? inplace_.buffer_ : static_cast<const byte *>(to_address(allocated_ptr_));
} }
MDBX_CXX17_CONSTEXPR byte *address() noexcept { MDBX_CXX17_CONSTEXPR byte *address() noexcept {
return is_inplace() ? inplace_ : static_cast<byte *>(to_address(allocated_.ptr_)); return is_inplace() ? inplace_.buffer_ : static_cast<byte *>(to_address(allocated_ptr_));
} }
constexpr size_t capacity() const noexcept { return is_inplace() ? sizeof(bin) - 1 : allocated_.capacity_bytes_; } constexpr size_t capacity() const noexcept { return is_inplace() ? sizeof(bin) - 1 : capacity_.bytes_; }
} bin_; } bin_;
MDBX_CXX20_CONSTEXPR void *init(size_t capacity) { MDBX_CXX20_CONSTEXPR void *init(size_t capacity) {
@ -1678,7 +1699,7 @@ private:
MDBX_CXX20_CONSTEXPR void release() noexcept { MDBX_CXX20_CONSTEXPR void release() noexcept {
if (bin_.is_allocated()) { if (bin_.is_allocated()) {
deallocate_storage(bin_.allocated_.ptr_, bin_.allocated_.capacity_bytes_); deallocate_storage(bin_.allocated_ptr_, bin_.capacity_.bytes_);
bin_.template make_inplace<true>(); bin_.template make_inplace<true>();
} }
} }
@ -1709,7 +1730,7 @@ private:
if (bin::is_suitable_for_inplace(new_capacity)) { if (bin::is_suitable_for_inplace(new_capacity)) {
assert(bin_.is_allocated()); assert(bin_.is_allocated());
const auto old_allocated = ::std::move(bin_.allocated_.ptr_); const auto old_allocated = ::std::move(bin_.allocated_ptr_);
byte *const new_place = bin_.template make_inplace<true>() + wanna_headroom; byte *const new_place = bin_.template make_inplace<true>() + wanna_headroom;
if (MDBX_LIKELY(length)) if (MDBX_LIKELY(length))
MDBX_CXX20_LIKELY memcpy(new_place, content, length); MDBX_CXX20_LIKELY memcpy(new_place, content, length);
@ -1727,7 +1748,7 @@ private:
return new_place; return new_place;
} }
const auto old_allocated = ::std::move(bin_.allocated_.ptr_); const auto old_allocated = ::std::move(bin_.allocated_ptr_);
if (external_content) if (external_content)
deallocate_storage(old_allocated, old_capacity); deallocate_storage(old_allocated, old_capacity);
const auto pair = allocate_storage(new_capacity); const auto pair = allocate_storage(new_capacity);