From e29cb076d379a1cab1367884e0c62a77252276db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9B=D0=B5=D0=BE=D0=BD=D0=B8=D0=B4=20=D0=AE=D1=80=D1=8C?= =?UTF-8?q?=D0=B5=D0=B2=20=28Leonid=20Yuriev=29?= Date: Sat, 2 Mar 2024 01:08:22 +0300 Subject: [PATCH] =?UTF-8?q?mdbx-test:=20=D0=B4=D0=BE=D1=80=D0=B0=D0=B1?= =?UTF-8?q?=D0=BE=D1=82=D0=BA=D0=B0=20=D0=B3=D0=B5=D0=BD=D0=B5=D1=80=D0=B0?= =?UTF-8?q?=D1=82=D0=BE=D1=80=D0=B0=20=D0=BA=D0=BB=D1=8E=D1=87=D0=B5=D0=B9?= =?UTF-8?q?/=D0=B7=D0=BD=D0=B0=D1=87=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=B4?= =?UTF-8?q?=D0=BB=D1=8F=20=D0=BD=D0=B0=D0=B4=D0=B5=D0=B6=D0=BD=D0=BE=D0=B9?= =?UTF-8?q?=20=D0=B3=D0=B5=D0=BD=D0=B5=D1=80=D0=B0=D1=86=D0=B8=D0=B8=20?= =?UTF-8?q?=D1=83=D0=BD=D0=B8=D0=BA=D0=B0=D0=BB=D1=8C=D0=BD=D1=8B=D1=85=20?= =?UTF-8?q?=D0=B7=D0=BD=D0=B0=D1=87=D0=B5=D0=BD=D0=B8=D0=B9.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit В текущем понимании коммитом этим устраняется застарелая проблема редких сбоев стохастического теста из-за вероятности ошибочной генерации повторяющихся пар key-value. --- test/hill.c++ | 4 +- test/keygen.c++ | 97 ++++++++++++++++++++++++++++++++++++++++--------- test/keygen.h++ | 11 +++++- 3 files changed, 91 insertions(+), 21 deletions(-) diff --git a/test/hill.c++ b/test/hill.c++ index f5ca1026..bbb3b3cf 100644 --- a/test/hill.c++ +++ b/test/hill.c++ @@ -90,7 +90,7 @@ bool testcase_hill::run() { assert(b_serial > a_serial); // создаем первую запись из пары - const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31); + const keygen::serial_t age_shift = keyvalue_maker.remix_age(a_serial); log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift, a_serial); generate_pair(a_serial, a_key, a_data_1, age_shift); @@ -302,7 +302,7 @@ bool testcase_hill::run() { assert(b_serial > a_serial); // обновляем первую запись из пары - const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31); + const keygen::serial_t age_shift = keyvalue_maker.remix_age(a_serial); log_trace("downhill: update-a (age 0->%" PRIu64 ") %" PRIu64, age_shift, a_serial); generate_pair(a_serial, a_key, a_data_0, 0); diff --git a/test/keygen.c++ b/test/keygen.c++ index 46b64ecf..69b0550c 100644 --- a/test/keygen.c++ +++ b/test/keygen.c++ @@ -14,6 +14,39 @@ #include "test.h++" +static const uint64_t primes[64] = { + /* */ + 0, 1, 3, 7, 13, 31, 61, 127, 251, 509, 1021, 2039, 4093, 8191, 16381, + /* */ + UINT64_C(32749), UINT64_C(65521), UINT64_C(131071), UINT64_C(262139), + UINT64_C(524287), UINT64_C(1048573), UINT64_C(2097143), UINT64_C(4194301), + UINT64_C(8388593), UINT64_C(16777213), UINT64_C(33554393), + UINT64_C(67108859), UINT64_C(134217689), UINT64_C(268435399), + UINT64_C(536870909), UINT64_C(1073741789), UINT64_C(2147483647), + UINT64_C(4294967291), UINT64_C(8589934583), UINT64_C(17179869143), + UINT64_C(34359738337), UINT64_C(68719476731), UINT64_C(137438953447), + UINT64_C(274877906899), UINT64_C(549755813881), UINT64_C(1099511627689), + UINT64_C(2199023255531), UINT64_C(4398046511093), UINT64_C(8796093022151), + UINT64_C(17592186044399), UINT64_C(35184372088777), + UINT64_C(70368744177643), UINT64_C(140737488355213), + UINT64_C(281474976710597), UINT64_C(562949953421231), + UINT64_C(1125899906842597), UINT64_C(2251799813685119), + UINT64_C(4503599627370449), UINT64_C(9007199254740881), + UINT64_C(18014398509481951), UINT64_C(36028797018963913), + UINT64_C(72057594037927931), UINT64_C(144115188075855859), + UINT64_C(288230376151711717), UINT64_C(576460752303423433), + UINT64_C(1152921504606846883), UINT64_C(2305843009213693951), + UINT64_C(4611686018427387847), UINT64_C(9223372036854775783)}; + +/* static unsigned supid_log2(uint64_t v) { + unsigned r = 0; + while (v > 1) { + v >>= 1; + r += 1; + } + return r; +} */ + namespace keygen { /* LY: https://en.wikipedia.org/wiki/Injective_function */ @@ -48,19 +81,19 @@ serial_t injective(const serial_t serial, 10, 14, 22, 19, 3, 21, 18, 19, 26, 24, 2, 21, 25, 29, 24, 10, 11, 14, 20, 19}; + const auto mask = actor_params::serial_mask(bits); const auto mult = m[bits - 8]; const auto shift = s[bits - 8]; serial_t result = serial * mult; if (salt) { const unsigned left = bits / 2; const unsigned right = bits - left; - result = (result << left) | - ((result & actor_params::serial_mask(bits)) >> right); + result = (result << left) | ((result & mask) >> right); result = (result ^ salt) * mult; } - result ^= result << shift; - result &= actor_params::serial_mask(bits); + result ^= (result & mask) >> shift; + result &= mask; log_trace("keygen-injective: serial %" PRIu64 "/%u @%" PRIx64 ",%u,%" PRIu64 " => %" PRIu64 "/%u", serial, bits, mult, shift, salt, result, bits); @@ -111,7 +144,7 @@ void __hot maker::pair(serial_t serial, const buffer &key, buffer &value, } serial_t key_serial = serial; - serial_t value_serial = value_age << mapping.split; + serial_t value_serial = (value_age & value_age_mask) << mapping.split; if (mapping.split) { if (MDBX_db_flags_t(key_essentials.flags) & MDBX_DUPSORT) { key_serial >>= mapping.split; @@ -200,6 +233,7 @@ void maker::setup(const config::actor_params_pod &actor, MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP) < UINT16_MAX); #endif + key_essentials.flags = uint16_t( actor.table_flags & MDBX_db_flags_t(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT)); @@ -209,6 +243,12 @@ void maker::setup(const config::actor_params_pod &actor, key_essentials.maxlen = std::min( uint32_t(actor.keylen_max), uint32_t(mdbx_limits_keysize_max(actor.pagesize, actor.table_flags))); + key_essentials.bits = (key_essentials.maxlen < sizeof(serial_t)) + ? key_essentials.maxlen * CHAR_BIT + : sizeof(serial_t) * CHAR_BIT; + key_essentials.mask = actor_params::serial_mask(key_essentials.bits); + assert(key_essentials.bits > 63 || + key_essentials.mask > primes[key_essentials.bits]); value_essentials.flags = uint16_t( actor.table_flags & @@ -219,27 +259,44 @@ void maker::setup(const config::actor_params_pod &actor, value_essentials.maxlen = std::min( uint32_t(actor.datalen_max), uint32_t(mdbx_limits_valsize_max(actor.pagesize, actor.table_flags))); + value_essentials.bits = (value_essentials.maxlen < sizeof(serial_t)) + ? value_essentials.maxlen * CHAR_BIT + : sizeof(serial_t) * CHAR_BIT; + value_essentials.mask = actor_params::serial_mask(value_essentials.bits); + assert(value_essentials.bits > 63 || + value_essentials.mask > primes[value_essentials.bits]); if (!actor.keygen.zero_fill) { key_essentials.flags |= essentials::prng_fill_flag; value_essentials.flags |= essentials::prng_fill_flag; } - (void)thread_number; mapping = actor.keygen; + const auto split = mapping.split; while (mapping.split > - essentials::value_age_width + value_essentials.maxlen * CHAR_BIT || + value_essentials.bits - essentials::value_age_minwidth || mapping.split >= mapping.width) mapping.split -= 1; + if (split != mapping.width) + log_notice("keygen: reduce mapping-split from %u to %u", split, + mapping.split); + const auto width = mapping.width; while (unsigned((actor.table_flags & MDBX_DUPSORT) ? mapping.width - mapping.split - : mapping.width) > key_essentials.maxlen * CHAR_BIT) + : mapping.width) > key_essentials.bits) mapping.width -= 1; + if (width != mapping.width) + log_notice("keygen: reduce mapping-width from %u to %u", width, + mapping.width); - salt = - (prng_state + uint64_t(thread_number)) * UINT64_C(14653293970879851569); + value_age_bits = value_essentials.bits - mapping.split; + value_age_mask = actor_params::serial_mask(value_age_bits); + assert(value_age_bits >= essentials::value_age_minwidth); + salt = (prng_state ^ + (thread_number * 1575554837) * UINT64_C(59386707711075671)) * + UINT64_C(14653293970879851569); base = actor.serial_base(); } @@ -321,14 +378,18 @@ serial_t __hot maker::mk_begin(serial_t serial, const essentials ¶ms, result &out) { assert(out.limit >= params.maxlen); assert(params.maxlen >= params.minlen); - if (params.maxlen < sizeof(serial_t)) { - const serial_t max = actor_params::serial_mask(params.maxlen * CHAR_BIT); - if (serial > max) { - serial ^= (serial >> max / 2) * serial_t((sizeof(serial_t) > 4) - ? UINT64_C(40719303417517073) - : UINT32_C(3708688457)); - serial &= max; - } + assert(serial <= params.mask); + if (unlikely(serial > params.mask)) { +#if 1 + serial %= primes[params.bits]; + assert(params.mask > primes[params.bits]); +#else + const serial_t maxbits = params.maxlen * CHAR_BIT; + serial ^= (serial >> maxbits / 2) * + serial_t((sizeof(serial_t) > 4) ? UINT64_C(40719303417517073) + : UINT32_C(3708688457)); + serial &= params.mask; +#endif assert(params.maxlen >= length(serial)); } diff --git a/test/keygen.h++ b/test/keygen.h++ index 0ded8130..8eb78118 100644 --- a/test/keygen.h++ +++ b/test/keygen.h++ @@ -108,10 +108,14 @@ class maker { struct essentials { uint16_t minlen{0}; - enum { prng_fill_flag = 1, value_age_width = 8 }; + enum { prng_fill_flag = 1, value_age_minwidth = 5 }; uint16_t flags{0}; uint32_t maxlen{0}; + serial_t mask{0}; + unsigned bits; } key_essentials, value_essentials; + unsigned value_age_bits; + serial_t value_age_mask{0}; static serial_t mk_begin(serial_t serial, const essentials ¶ms, result &out); @@ -136,6 +140,11 @@ public: } return increment(serial, int64_t(uint64_t(delta) << mapping.split)); } + + serial_t remix_age(serial_t serial) const { + return (UINT64_C(768097847591) * (serial ^ UINT64_C(768097847591))) & + value_age_mask; + } }; void log_pair(logging::loglevel level, const char *prefix, const buffer &key,