mdbx-test: доработка генератора ключей/значений для надежной генерации уникальных значений.

В текущем понимании коммитом этим устраняется застарелая проблема редких
сбоев стохастического теста из-за вероятности ошибочной генерации
повторяющихся пар key-value.

test/hill.c++

@@ -90,7 +90,7 @@ bool testcase_hill::run() {
     assert(b_serial > a_serial);
 
     // создаем первую запись из пары
-    const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31);
+    const keygen::serial_t age_shift = keyvalue_maker.remix_age(a_serial);
     log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift,
               a_serial);
     generate_pair(a_serial, a_key, a_data_1, age_shift);
@@ -302,7 +302,7 @@ bool testcase_hill::run() {
     assert(b_serial > a_serial);
 
     // обновляем первую запись из пары
-    const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31);
+    const keygen::serial_t age_shift = keyvalue_maker.remix_age(a_serial);
     log_trace("downhill: update-a (age 0->%" PRIu64 ") %" PRIu64, age_shift,
               a_serial);
     generate_pair(a_serial, a_key, a_data_0, 0);

test/keygen.c++

@@ -14,6 +14,39 @@
 
 #include "test.h++"
 
+static const uint64_t primes[64] = {
+    /* */
+    0, 1, 3, 7, 13, 31, 61, 127, 251, 509, 1021, 2039, 4093, 8191, 16381,
+    /* */
+    UINT64_C(32749), UINT64_C(65521), UINT64_C(131071), UINT64_C(262139),
+    UINT64_C(524287), UINT64_C(1048573), UINT64_C(2097143), UINT64_C(4194301),
+    UINT64_C(8388593), UINT64_C(16777213), UINT64_C(33554393),
+    UINT64_C(67108859), UINT64_C(134217689), UINT64_C(268435399),
+    UINT64_C(536870909), UINT64_C(1073741789), UINT64_C(2147483647),
+    UINT64_C(4294967291), UINT64_C(8589934583), UINT64_C(17179869143),
+    UINT64_C(34359738337), UINT64_C(68719476731), UINT64_C(137438953447),
+    UINT64_C(274877906899), UINT64_C(549755813881), UINT64_C(1099511627689),
+    UINT64_C(2199023255531), UINT64_C(4398046511093), UINT64_C(8796093022151),
+    UINT64_C(17592186044399), UINT64_C(35184372088777),
+    UINT64_C(70368744177643), UINT64_C(140737488355213),
+    UINT64_C(281474976710597), UINT64_C(562949953421231),
+    UINT64_C(1125899906842597), UINT64_C(2251799813685119),
+    UINT64_C(4503599627370449), UINT64_C(9007199254740881),
+    UINT64_C(18014398509481951), UINT64_C(36028797018963913),
+    UINT64_C(72057594037927931), UINT64_C(144115188075855859),
+    UINT64_C(288230376151711717), UINT64_C(576460752303423433),
+    UINT64_C(1152921504606846883), UINT64_C(2305843009213693951),
+    UINT64_C(4611686018427387847), UINT64_C(9223372036854775783)};
+
+/* static unsigned supid_log2(uint64_t v) {
+  unsigned r = 0;
+  while (v > 1) {
+    v >>= 1;
+    r += 1;
+  }
+  return r;
+} */
+
 namespace keygen {
 
 /* LY: https://en.wikipedia.org/wiki/Injective_function */
@@ -48,19 +81,19 @@ serial_t injective(const serial_t serial,
       10, 14, 22, 19, 3, 21, 18, 19, 26, 24, 2, 21, 25, 29, 24, 10, 11, 14, 20,
       19};
 
+  const auto mask = actor_params::serial_mask(bits);
   const auto mult = m[bits - 8];
   const auto shift = s[bits - 8];
   serial_t result = serial * mult;
   if (salt) {
     const unsigned left = bits / 2;
     const unsigned right = bits - left;
-    result = (result << left) |
+    result = (result << left) | ((result & mask) >> right);
-             ((result & actor_params::serial_mask(bits)) >> right);
     result = (result ^ salt) * mult;
   }
 
-  result ^= result << shift;
+  result ^= (result & mask) >> shift;
-  result &= actor_params::serial_mask(bits);
+  result &= mask;
   log_trace("keygen-injective: serial %" PRIu64 "/%u @%" PRIx64 ",%u,%" PRIu64
             " => %" PRIu64 "/%u",
             serial, bits, mult, shift, salt, result, bits);
@@ -111,7 +144,7 @@ void __hot maker::pair(serial_t serial, const buffer &key, buffer &value,
   }
 
   serial_t key_serial = serial;
-  serial_t value_serial = value_age << mapping.split;
+  serial_t value_serial = (value_age & value_age_mask) << mapping.split;
   if (mapping.split) {
     if (MDBX_db_flags_t(key_essentials.flags) & MDBX_DUPSORT) {
       key_serial >>= mapping.split;
@@ -200,6 +233,7 @@ void maker::setup(const config::actor_params_pod &actor,
                   MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP) <
          UINT16_MAX);
 #endif
+
   key_essentials.flags = uint16_t(
       actor.table_flags &
       MDBX_db_flags_t(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT));
@@ -209,6 +243,12 @@ void maker::setup(const config::actor_params_pod &actor,
   key_essentials.maxlen = std::min(
       uint32_t(actor.keylen_max),
       uint32_t(mdbx_limits_keysize_max(actor.pagesize, actor.table_flags)));
+  key_essentials.bits = (key_essentials.maxlen < sizeof(serial_t))
+                            ? key_essentials.maxlen * CHAR_BIT
+                            : sizeof(serial_t) * CHAR_BIT;
+  key_essentials.mask = actor_params::serial_mask(key_essentials.bits);
+  assert(key_essentials.bits > 63 ||
+         key_essentials.mask > primes[key_essentials.bits]);
 
   value_essentials.flags = uint16_t(
       actor.table_flags &
@@ -219,27 +259,44 @@ void maker::setup(const config::actor_params_pod &actor,
   value_essentials.maxlen = std::min(
       uint32_t(actor.datalen_max),
       uint32_t(mdbx_limits_valsize_max(actor.pagesize, actor.table_flags)));
+  value_essentials.bits = (value_essentials.maxlen < sizeof(serial_t))
+                              ? value_essentials.maxlen * CHAR_BIT
+                              : sizeof(serial_t) * CHAR_BIT;
+  value_essentials.mask = actor_params::serial_mask(value_essentials.bits);
+  assert(value_essentials.bits > 63 ||
+         value_essentials.mask > primes[value_essentials.bits]);
 
   if (!actor.keygen.zero_fill) {
     key_essentials.flags |= essentials::prng_fill_flag;
     value_essentials.flags |= essentials::prng_fill_flag;
   }
 
-  (void)thread_number;
   mapping = actor.keygen;
+  const auto split = mapping.split;
   while (mapping.split >
-             essentials::value_age_width + value_essentials.maxlen * CHAR_BIT ||
+             value_essentials.bits - essentials::value_age_minwidth ||
          mapping.split >= mapping.width)
     mapping.split -= 1;
+  if (split != mapping.width)
+    log_notice("keygen: reduce mapping-split from %u to %u", split,
+               mapping.split);
 
+  const auto width = mapping.width;
   while (unsigned((actor.table_flags & MDBX_DUPSORT)
                       ? mapping.width - mapping.split
-                      : mapping.width) > key_essentials.maxlen * CHAR_BIT)
+                      : mapping.width) > key_essentials.bits)
     mapping.width -= 1;
+  if (width != mapping.width)
+    log_notice("keygen: reduce mapping-width from %u to %u", width,
+               mapping.width);
 
-  salt =
+  value_age_bits = value_essentials.bits - mapping.split;
-      (prng_state + uint64_t(thread_number)) * UINT64_C(14653293970879851569);
+  value_age_mask = actor_params::serial_mask(value_age_bits);
+  assert(value_age_bits >= essentials::value_age_minwidth);
 
+  salt = (prng_state ^
+          (thread_number * 1575554837) * UINT64_C(59386707711075671)) *
+         UINT64_C(14653293970879851569);
   base = actor.serial_base();
 }
 
@@ -321,14 +378,18 @@ serial_t __hot maker::mk_begin(serial_t serial, const essentials &params,
                                result &out) {
   assert(out.limit >= params.maxlen);
   assert(params.maxlen >= params.minlen);
-  if (params.maxlen < sizeof(serial_t)) {
+  assert(serial <= params.mask);
-    const serial_t max = actor_params::serial_mask(params.maxlen * CHAR_BIT);
+  if (unlikely(serial > params.mask)) {
-    if (serial > max) {
+#if 1
-      serial ^= (serial >> max / 2) * serial_t((sizeof(serial_t) > 4)
+    serial %= primes[params.bits];
-                                                   ? UINT64_C(40719303417517073)
+    assert(params.mask > primes[params.bits]);
-                                                   : UINT32_C(3708688457));
+#else
-      serial &= max;
+    const serial_t maxbits = params.maxlen * CHAR_BIT;
-    }
+    serial ^= (serial >> maxbits / 2) *
+              serial_t((sizeof(serial_t) > 4) ? UINT64_C(40719303417517073)
+                                              : UINT32_C(3708688457));
+    serial &= params.mask;
+#endif
     assert(params.maxlen >= length(serial));
   }
 

test/keygen.h++

@@ -108,10 +108,14 @@ class maker {
 
   struct essentials {
     uint16_t minlen{0};
-    enum { prng_fill_flag = 1, value_age_width = 8 };
+    enum { prng_fill_flag = 1, value_age_minwidth = 5 };
     uint16_t flags{0};
     uint32_t maxlen{0};
+    serial_t mask{0};
+    unsigned bits;
   } key_essentials, value_essentials;
+  unsigned value_age_bits;
+  serial_t value_age_mask{0};
 
   static serial_t mk_begin(serial_t serial, const essentials &params,
                            result &out);
@@ -136,6 +140,11 @@ public:
     }
     return increment(serial, int64_t(uint64_t(delta) << mapping.split));
   }
+
+  serial_t remix_age(serial_t serial) const {
+    return (UINT64_C(768097847591) * (serial ^ UINT64_C(768097847591))) &
+           value_age_mask;
+  }
 };
 
 void log_pair(logging::loglevel level, const char *prefix, const buffer &key,