mirror of
https://github.com/isar/libmdbx.git
synced 2024-12-30 01:44:13 +08:00
mdbx-test: доработка генератора ключей/значений для надежной генерации уникальных значений.
В текущем понимании коммитом этим устраняется застарелая проблема редких сбоев стохастического теста из-за вероятности ошибочной генерации повторяющихся пар key-value.
This commit is contained in:
Леонид Юрьев (Leonid Yuriev)
parent
9480599afa
commit
e29cb076d3
@ -90,7 +90,7 @@ bool testcase_hill::run() {
|
||||
assert(b_serial > a_serial);
|
||||
|
||||
// создаем первую запись из пары
|
||||
const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31);
|
||||
const keygen::serial_t age_shift = keyvalue_maker.remix_age(a_serial);
|
||||
log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift,
|
||||
a_serial);
|
||||
generate_pair(a_serial, a_key, a_data_1, age_shift);
|
||||
@ -302,7 +302,7 @@ bool testcase_hill::run() {
|
||||
assert(b_serial > a_serial);
|
||||
|
||||
// обновляем первую запись из пары
|
||||
const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31);
|
||||
const keygen::serial_t age_shift = keyvalue_maker.remix_age(a_serial);
|
||||
log_trace("downhill: update-a (age 0->%" PRIu64 ") %" PRIu64, age_shift,
|
||||
a_serial);
|
||||
generate_pair(a_serial, a_key, a_data_0, 0);
|
||||
|
||||
@ -14,6 +14,39 @@
|
||||
|
||||
#include "test.h++"
|
||||
|
||||
static const uint64_t primes[64] = {
|
||||
/* */
|
||||
0, 1, 3, 7, 13, 31, 61, 127, 251, 509, 1021, 2039, 4093, 8191, 16381,
|
||||
/* */
|
||||
UINT64_C(32749), UINT64_C(65521), UINT64_C(131071), UINT64_C(262139),
|
||||
UINT64_C(524287), UINT64_C(1048573), UINT64_C(2097143), UINT64_C(4194301),
|
||||
UINT64_C(8388593), UINT64_C(16777213), UINT64_C(33554393),
|
||||
UINT64_C(67108859), UINT64_C(134217689), UINT64_C(268435399),
|
||||
UINT64_C(536870909), UINT64_C(1073741789), UINT64_C(2147483647),
|
||||
UINT64_C(4294967291), UINT64_C(8589934583), UINT64_C(17179869143),
|
||||
UINT64_C(34359738337), UINT64_C(68719476731), UINT64_C(137438953447),
|
||||
UINT64_C(274877906899), UINT64_C(549755813881), UINT64_C(1099511627689),
|
||||
UINT64_C(2199023255531), UINT64_C(4398046511093), UINT64_C(8796093022151),
|
||||
UINT64_C(17592186044399), UINT64_C(35184372088777),
|
||||
UINT64_C(70368744177643), UINT64_C(140737488355213),
|
||||
UINT64_C(281474976710597), UINT64_C(562949953421231),
|
||||
UINT64_C(1125899906842597), UINT64_C(2251799813685119),
|
||||
UINT64_C(4503599627370449), UINT64_C(9007199254740881),
|
||||
UINT64_C(18014398509481951), UINT64_C(36028797018963913),
|
||||
UINT64_C(72057594037927931), UINT64_C(144115188075855859),
|
||||
UINT64_C(288230376151711717), UINT64_C(576460752303423433),
|
||||
UINT64_C(1152921504606846883), UINT64_C(2305843009213693951),
|
||||
UINT64_C(4611686018427387847), UINT64_C(9223372036854775783)};
|
||||
|
||||
/* static unsigned supid_log2(uint64_t v) {
|
||||
unsigned r = 0;
|
||||
while (v > 1) {
|
||||
v >>= 1;
|
||||
r += 1;
|
||||
}
|
||||
return r;
|
||||
} */
|
||||
|
||||
namespace keygen {
|
||||
|
||||
/* LY: https://en.wikipedia.org/wiki/Injective_function */
|
||||
@ -48,19 +81,19 @@ serial_t injective(const serial_t serial,
|
||||
10, 14, 22, 19, 3, 21, 18, 19, 26, 24, 2, 21, 25, 29, 24, 10, 11, 14, 20,
|
||||
19};
|
||||
|
||||
const auto mask = actor_params::serial_mask(bits);
|
||||
const auto mult = m[bits - 8];
|
||||
const auto shift = s[bits - 8];
|
||||
serial_t result = serial * mult;
|
||||
if (salt) {
|
||||
const unsigned left = bits / 2;
|
||||
const unsigned right = bits - left;
|
||||
result = (result << left) |
|
||||
((result & actor_params::serial_mask(bits)) >> right);
|
||||
result = (result << left) | ((result & mask) >> right);
|
||||
result = (result ^ salt) * mult;
|
||||
}
|
||||
|
||||
result ^= result << shift;
|
||||
result &= actor_params::serial_mask(bits);
|
||||
result ^= (result & mask) >> shift;
|
||||
result &= mask;
|
||||
log_trace("keygen-injective: serial %" PRIu64 "/%u @%" PRIx64 ",%u,%" PRIu64
|
||||
" => %" PRIu64 "/%u",
|
||||
serial, bits, mult, shift, salt, result, bits);
|
||||
@ -111,7 +144,7 @@ void __hot maker::pair(serial_t serial, const buffer &key, buffer &value,
|
||||
}
|
||||
|
||||
serial_t key_serial = serial;
|
||||
serial_t value_serial = value_age << mapping.split;
|
||||
serial_t value_serial = (value_age & value_age_mask) << mapping.split;
|
||||
if (mapping.split) {
|
||||
if (MDBX_db_flags_t(key_essentials.flags) & MDBX_DUPSORT) {
|
||||
key_serial >>= mapping.split;
|
||||
@ -200,6 +233,7 @@ void maker::setup(const config::actor_params_pod &actor,
|
||||
MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP) <
|
||||
UINT16_MAX);
|
||||
#endif
|
||||
|
||||
key_essentials.flags = uint16_t(
|
||||
actor.table_flags &
|
||||
MDBX_db_flags_t(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT));
|
||||
@ -209,6 +243,12 @@ void maker::setup(const config::actor_params_pod &actor,
|
||||
key_essentials.maxlen = std::min(
|
||||
uint32_t(actor.keylen_max),
|
||||
uint32_t(mdbx_limits_keysize_max(actor.pagesize, actor.table_flags)));
|
||||
key_essentials.bits = (key_essentials.maxlen < sizeof(serial_t))
|
||||
? key_essentials.maxlen * CHAR_BIT
|
||||
: sizeof(serial_t) * CHAR_BIT;
|
||||
key_essentials.mask = actor_params::serial_mask(key_essentials.bits);
|
||||
assert(key_essentials.bits > 63 ||
|
||||
key_essentials.mask > primes[key_essentials.bits]);
|
||||
|
||||
value_essentials.flags = uint16_t(
|
||||
actor.table_flags &
|
||||
@ -219,27 +259,44 @@ void maker::setup(const config::actor_params_pod &actor,
|
||||
value_essentials.maxlen = std::min(
|
||||
uint32_t(actor.datalen_max),
|
||||
uint32_t(mdbx_limits_valsize_max(actor.pagesize, actor.table_flags)));
|
||||
value_essentials.bits = (value_essentials.maxlen < sizeof(serial_t))
|
||||
? value_essentials.maxlen * CHAR_BIT
|
||||
: sizeof(serial_t) * CHAR_BIT;
|
||||
value_essentials.mask = actor_params::serial_mask(value_essentials.bits);
|
||||
assert(value_essentials.bits > 63 ||
|
||||
value_essentials.mask > primes[value_essentials.bits]);
|
||||
|
||||
if (!actor.keygen.zero_fill) {
|
||||
key_essentials.flags |= essentials::prng_fill_flag;
|
||||
value_essentials.flags |= essentials::prng_fill_flag;
|
||||
}
|
||||
|
||||
(void)thread_number;
|
||||
mapping = actor.keygen;
|
||||
const auto split = mapping.split;
|
||||
while (mapping.split >
|
||||
essentials::value_age_width + value_essentials.maxlen * CHAR_BIT ||
|
||||
value_essentials.bits - essentials::value_age_minwidth ||
|
||||
mapping.split >= mapping.width)
|
||||
mapping.split -= 1;
|
||||
if (split != mapping.width)
|
||||
log_notice("keygen: reduce mapping-split from %u to %u", split,
|
||||
mapping.split);
|
||||
|
||||
const auto width = mapping.width;
|
||||
while (unsigned((actor.table_flags & MDBX_DUPSORT)
|
||||
? mapping.width - mapping.split
|
||||
: mapping.width) > key_essentials.maxlen * CHAR_BIT)
|
||||
: mapping.width) > key_essentials.bits)
|
||||
mapping.width -= 1;
|
||||
if (width != mapping.width)
|
||||
log_notice("keygen: reduce mapping-width from %u to %u", width,
|
||||
mapping.width);
|
||||
|
||||
salt =
|
||||
(prng_state + uint64_t(thread_number)) * UINT64_C(14653293970879851569);
|
||||
value_age_bits = value_essentials.bits - mapping.split;
|
||||
value_age_mask = actor_params::serial_mask(value_age_bits);
|
||||
assert(value_age_bits >= essentials::value_age_minwidth);
|
||||
|
||||
salt = (prng_state ^
|
||||
(thread_number * 1575554837) * UINT64_C(59386707711075671)) *
|
||||
UINT64_C(14653293970879851569);
|
||||
base = actor.serial_base();
|
||||
}
|
||||
|
||||
@ -321,14 +378,18 @@ serial_t __hot maker::mk_begin(serial_t serial, const essentials ¶ms,
|
||||
result &out) {
|
||||
assert(out.limit >= params.maxlen);
|
||||
assert(params.maxlen >= params.minlen);
|
||||
if (params.maxlen < sizeof(serial_t)) {
|
||||
const serial_t max = actor_params::serial_mask(params.maxlen * CHAR_BIT);
|
||||
if (serial > max) {
|
||||
serial ^= (serial >> max / 2) * serial_t((sizeof(serial_t) > 4)
|
||||
? UINT64_C(40719303417517073)
|
||||
: UINT32_C(3708688457));
|
||||
serial &= max;
|
||||
}
|
||||
assert(serial <= params.mask);
|
||||
if (unlikely(serial > params.mask)) {
|
||||
#if 1
|
||||
serial %= primes[params.bits];
|
||||
assert(params.mask > primes[params.bits]);
|
||||
#else
|
||||
const serial_t maxbits = params.maxlen * CHAR_BIT;
|
||||
serial ^= (serial >> maxbits / 2) *
|
||||
serial_t((sizeof(serial_t) > 4) ? UINT64_C(40719303417517073)
|
||||
: UINT32_C(3708688457));
|
||||
serial &= params.mask;
|
||||
#endif
|
||||
assert(params.maxlen >= length(serial));
|
||||
}
|
||||
|
||||
|
||||
@ -108,10 +108,14 @@ class maker {
|
||||
|
||||
struct essentials {
|
||||
uint16_t minlen{0};
|
||||
enum { prng_fill_flag = 1, value_age_width = 8 };
|
||||
enum { prng_fill_flag = 1, value_age_minwidth = 5 };
|
||||
uint16_t flags{0};
|
||||
uint32_t maxlen{0};
|
||||
serial_t mask{0};
|
||||
unsigned bits;
|
||||
} key_essentials, value_essentials;
|
||||
unsigned value_age_bits;
|
||||
serial_t value_age_mask{0};
|
||||
|
||||
static serial_t mk_begin(serial_t serial, const essentials ¶ms,
|
||||
result &out);
|
||||
@ -136,6 +140,11 @@ public:
|
||||
}
|
||||
return increment(serial, int64_t(uint64_t(delta) << mapping.split));
|
||||
}
|
||||
|
||||
serial_t remix_age(serial_t serial) const {
|
||||
return (UINT64_C(768097847591) * (serial ^ UINT64_C(768097847591))) &
|
||||
value_age_mask;
|
||||
}
|
||||
};
|
||||
|
||||
void log_pair(logging::loglevel level, const char *prefix, const buffer &key,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user