mdbx-tools: поддержка не-печатных имен subDb в mdbx_chk.

This commit is contained in:
Леонид Юрьев (Leonid Yuriev) 2022-12-14 14:42:16 +03:00
parent 7011743262
commit 44493c6448
2 changed files with 159 additions and 88 deletions

View File

@ -23,6 +23,8 @@
#define xMDBX_TOOLS /* Avoid using internal eASSERT() */
#include "internals.h"
#include <ctype.h>
typedef struct flagbit {
int bit;
const char *name;
@ -71,7 +73,7 @@ static void signal_handler(int sig) {
#define EXIT_FAILURE_CHECK_MINOR EXIT_FAILURE
typedef struct {
const char *name;
MDBX_val name;
struct {
uint64_t branch, large_count, large_volume, leaf;
uint64_t subleaf_dupsort, leaf_dupfixed, subleaf_dupfixed;
@ -102,7 +104,7 @@ uint64_t total_unused_bytes, reclaimable_pages, gc_pages, alloc_pages,
unused_pages, backed_pages;
unsigned verbose;
bool ignore_wrong_order, quiet, dont_traversal;
const char *only_subdb;
MDBX_val only_subdb;
int stuck_meta = -1;
struct problem {
@ -125,6 +127,93 @@ static void MDBX_PRINTF_ARGS(1, 2) print(const char *msg, ...) {
}
}
static MDBX_val printable_buf;
static void free_printable_buf(void) { osal_free(printable_buf.iov_base); }
static const char *sdb_name(const MDBX_val *val) {
if (val == MDBX_PGWALK_MAIN)
return "@MAIN";
if (val == MDBX_PGWALK_GC)
return "@GC";
if (val == MDBX_PGWALK_META)
return "@META";
const unsigned char *const data = val->iov_base;
const size_t len = val->iov_len;
if (data == MDBX_PGWALK_MAIN)
return "@MAIN";
if (data == MDBX_PGWALK_GC)
return "@GC";
if (data == MDBX_PGWALK_META)
return "@META";
if (!len)
return "<zero-length>";
if (!data)
return "<nullptr>";
if (len > 65536) {
static char buf[64];
snprintf(buf, sizeof(buf), "<too-long-%zu>", len);
return buf;
}
bool printable = true;
bool quoting = false;
size_t xchars = 0;
for (size_t i = 0; i < val->iov_len && printable; ++i) {
quoting |= data[i] != '_' && isalnum(data[i]) == 0;
printable = isprint(data[i]) != 0 ||
(data[i] < ' ' && ++xchars < 4 && len > xchars * 4);
}
size_t need = len + 1;
if (quoting || !printable)
need += len + /* quotes */ 2 + 2 * /* max xchars */ 4;
if (need > printable_buf.iov_len) {
void *ptr = osal_realloc(printable_buf.iov_base, need);
if (!ptr)
return "<out-of-memory>";
if (!printable_buf.iov_base)
atexit(free_printable_buf);
printable_buf.iov_base = ptr;
printable_buf.iov_len = need;
}
char *out = printable_buf.iov_base;
if (!quoting) {
memcpy(out, data, len);
out += len;
} else if (printable) {
*out++ = '\'';
for (size_t i = 0; i < len; ++i) {
if (data[i] < ' ') {
assert((char *)printable_buf.iov_base + printable_buf.iov_len >
out + 4);
static const char hex[] = "0123456789abcdef";
out[0] = '\\';
out[1] = 'x';
out[2] = hex[data[i] >> 4];
out[3] = hex[data[i] & 15];
out += 4;
} else if (strchr("\"'`\\", data[i])) {
assert((char *)printable_buf.iov_base + printable_buf.iov_len >
out + 2);
out[0] = '\\';
out[1] = data[i];
out += 2;
} else {
assert((char *)printable_buf.iov_base + printable_buf.iov_len >
out + 1);
*out++ = data[i];
}
}
*out++ = '\'';
}
assert((char *)printable_buf.iov_base + printable_buf.iov_len > out);
*out = 0;
return printable_buf.iov_base;
}
static void va_log(MDBX_log_level_t level, const char *function, int line,
const char *msg, va_list args) {
static const char *const prefixes[] = {
@ -190,19 +279,17 @@ static int check_user_break(void) {
}
static void pagemap_cleanup(void) {
for (size_t i = CORE_DBS + /* account pseudo-entry for meta */ 1;
i < ARRAY_LENGTH(walk.dbi); ++i) {
if (walk.dbi[i].name) {
osal_free((void *)walk.dbi[i].name);
walk.dbi[i].name = nullptr;
}
}
osal_free(walk.pagemap);
walk.pagemap = nullptr;
}
static walk_dbi_t *pagemap_lookup_dbi(const char *dbi_name, bool silent) {
static bool eq(const MDBX_val a, const MDBX_val b) {
return a.iov_len == b.iov_len &&
(a.iov_base == b.iov_base || a.iov_len == 0 ||
!memcmp(a.iov_base, b.iov_base, a.iov_len));
}
static walk_dbi_t *pagemap_lookup_dbi(const MDBX_val *dbi_name, bool silent) {
static walk_dbi_t *last;
if (dbi_name == MDBX_PGWALK_MAIN)
@ -212,24 +299,24 @@ static walk_dbi_t *pagemap_lookup_dbi(const char *dbi_name, bool silent) {
if (dbi_name == MDBX_PGWALK_META)
return &dbi_meta;
if (last && strcmp(last->name, dbi_name) == 0)
if (last && eq(last->name, *dbi_name))
return last;
walk_dbi_t *dbi = walk.dbi + CORE_DBS + /* account pseudo-entry for meta */ 1;
for (; dbi < ARRAY_END(walk.dbi) && dbi->name; ++dbi) {
if (strcmp(dbi->name, dbi_name) == 0)
for (; dbi < ARRAY_END(walk.dbi) && dbi->name.iov_base; ++dbi) {
if (eq(dbi->name, *dbi_name))
return last = dbi;
}
if (verbose > 0 && !silent) {
print(" - found `%s` area\n", dbi_name);
print(" - found %s area\n", sdb_name(dbi_name));
fflush(nullptr);
}
if (dbi == ARRAY_END(walk.dbi))
return nullptr;
dbi->name = osal_strdup(dbi_name);
dbi->name = *dbi_name;
return last = dbi;
}
@ -304,13 +391,13 @@ static size_t problems_pop(struct problem *list) {
}
static int pgvisitor(const uint64_t pgno, const unsigned pgnumber,
void *const ctx, const int deep,
const char *const dbi_name_or_tag, const size_t page_size,
const MDBX_page_type_t pagetype, const MDBX_error_t err,
const size_t nentries, const size_t payload_bytes,
const size_t header_bytes, const size_t unused_bytes) {
void *const ctx, const int deep, const MDBX_val *dbi_name,
const size_t page_size, const MDBX_page_type_t pagetype,
const MDBX_error_t err, const size_t nentries,
const size_t payload_bytes, const size_t header_bytes,
const size_t unused_bytes) {
(void)ctx;
const bool is_gc_tree = dbi_name_or_tag == MDBX_PGWALK_GC;
const bool is_gc_tree = dbi_name == MDBX_PGWALK_GC;
if (deep > 42) {
problem_add("deep", deep, "too large", nullptr);
data_tree_problems += !is_gc_tree;
@ -318,7 +405,7 @@ static int pgvisitor(const uint64_t pgno, const unsigned pgnumber,
return MDBX_CORRUPTED /* avoid infinite loop/recursion */;
}
walk_dbi_t *dbi = pagemap_lookup_dbi(dbi_name_or_tag, false);
walk_dbi_t *dbi = pagemap_lookup_dbi(dbi_name, false);
if (!dbi) {
data_tree_problems += !is_gc_tree;
gc_tree_problems += is_gc_tree;
@ -383,14 +470,14 @@ static int pgvisitor(const uint64_t pgno, const unsigned pgnumber,
}
if (pgnumber) {
if (verbose > 3 && (!only_subdb || strcmp(only_subdb, dbi->name) == 0)) {
if (verbose > 3 && (!only_subdb.iov_base || eq(only_subdb, dbi->name))) {
if (pgnumber == 1)
print(" %s-page %" PRIu64, pagetype_caption, pgno);
else
print(" %s-span %" PRIu64 "[%u]", pagetype_caption, pgno, pgnumber);
print(" of %s: header %" PRIiPTR ", %s %" PRIiPTR ", payload %" PRIiPTR
", unused %" PRIiPTR ", deep %i\n",
dbi->name, header_bytes,
sdb_name(&dbi->name), header_bytes,
(pagetype == MDBX_page_branch) ? "keys" : "entries", nentries,
payload_bytes, unused_bytes, deep);
}
@ -408,8 +495,8 @@ static int pgvisitor(const uint64_t pgno, const unsigned pgnumber,
walk_dbi_t *coll_dbi = &walk.dbi[walk.pagemap[spanpgno] - 1];
problem_add("page", spanpgno,
(branch && coll_dbi == dbi) ? "loop" : "already used",
"%s-page: by %s, deep %i", pagetype_caption, coll_dbi->name,
deep);
"%s-page: by %s, deep %i", pagetype_caption,
sdb_name(&coll_dbi->name), deep);
already_used = true;
data_tree_problems += !is_gc_tree;
gc_tree_problems += is_gc_tree;
@ -491,7 +578,8 @@ static int pgvisitor(const uint64_t pgno, const unsigned pgnumber,
typedef int(visitor)(const uint64_t record_number, const MDBX_val *key,
const MDBX_val *data);
static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler);
static int process_db(MDBX_dbi dbi_handle, const MDBX_val *dbi_name,
visitor *handler);
static int handle_userdb(const uint64_t record_number, const MDBX_val *key,
const MDBX_val *data) {
@ -569,7 +657,7 @@ static int handle_freedb(const uint64_t record_number, const MDBX_val *key,
walk.pagemap[pgno] = -1;
else if (idx > 0)
problem_add("page", pgno, "already used", "by %s",
walk.dbi[idx - 1].name);
sdb_name(&walk.dbi[idx - 1].name));
else
problem_add("page", pgno, "already listed in GC", nullptr);
}
@ -580,7 +668,7 @@ static int handle_freedb(const uint64_t record_number, const MDBX_val *key,
: pgno_sub(pgno, span)))
++span;
}
if (verbose > 3 && !only_subdb) {
if (verbose > 3 && !only_subdb.iov_base) {
print(" transaction %" PRIaTXN ", %" PRIuPTR
" pages, maxspan %" PRIaPGNO "%s\n",
txnid, number, span, bad);
@ -607,39 +695,18 @@ static int handle_freedb(const uint64_t record_number, const MDBX_val *key,
}
static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) {
return (a->iov_len == b->iov_len &&
memcmp(a->iov_base, b->iov_base, a->iov_len) == 0)
? 0
: 1;
return eq(*a, *b) ? 0 : 1;
}
static int handle_maindb(const uint64_t record_number, const MDBX_val *key,
const MDBX_val *data) {
char *name;
int rc;
size_t i;
if (data->iov_len != sizeof(MDBX_db))
return handle_userdb(record_number, key, data);
name = key->iov_base;
for (i = 0; i < key->iov_len; ++i) {
if (name[i] < ' ')
return handle_userdb(record_number, key, data);
if (data->iov_len == sizeof(MDBX_db)) {
int rc = process_db(~0u, key, handle_userdb);
if (rc != MDBX_INCOMPATIBLE) {
userdb_count++;
return rc;
}
}
name = osal_malloc(key->iov_len + 1);
if (unlikely(!name))
return MDBX_ENOMEM;
memcpy(name, key->iov_base, key->iov_len);
name[key->iov_len] = '\0';
rc = process_db(~0u, name, handle_userdb);
osal_free(name);
if (rc != MDBX_INCOMPATIBLE) {
userdb_count++;
return rc;
}
return handle_userdb(record_number, key, data);
}
@ -693,7 +760,8 @@ static const char *db_flags2valuemode(unsigned flags) {
}
}
static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler) {
static int process_db(MDBX_dbi dbi_handle, const MDBX_val *dbi_name,
visitor *handler) {
MDBX_cursor *mc;
MDBX_stat ms;
MDBX_val key, data;
@ -708,13 +776,13 @@ static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler) {
uint64_t key_bytes = 0, data_bytes = 0;
if ((MDBX_TXN_FINISHED | MDBX_TXN_ERROR) & mdbx_txn_flags(txn)) {
print(" ! abort processing `%s` due to a previous error\n",
dbi_name ? dbi_name : "@MAIN");
print(" ! abort processing %s due to a previous error\n",
sdb_name(dbi_name));
return MDBX_BAD_TXN;
}
if (dbi_handle == ~0u) {
rc = mdbx_dbi_open_ex(
rc = mdbx_dbi_open_ex2(
txn, dbi_name, MDBX_DB_ACCEDE, &dbi_handle,
(dbi_name && ignore_wrong_order) ? equal_or_greater : nullptr,
(dbi_name && ignore_wrong_order) ? equal_or_greater : nullptr);
@ -722,17 +790,17 @@ static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler) {
if (!dbi_name ||
rc !=
MDBX_INCOMPATIBLE) /* LY: mainDB's record is not a user's DB. */ {
error("mdbx_dbi_open(`%s`) failed, error %d %s\n",
dbi_name ? dbi_name : "@MAIN", rc, mdbx_strerror(rc));
error("mdbx_dbi_open(%s) failed, error %d %s\n", sdb_name(dbi_name), rc,
mdbx_strerror(rc));
}
return rc;
}
}
if (dbi_handle >= CORE_DBS && dbi_name && only_subdb &&
strcmp(only_subdb, dbi_name) != 0) {
if (dbi_handle >= CORE_DBS && dbi_name && only_subdb.iov_base &&
!eq(only_subdb, *dbi_name)) {
if (verbose) {
print("Skip processing %s...\n", dbi_name);
print("Skip processing %s...\n", sdb_name(dbi_name));
fflush(nullptr);
}
skipped_subdb++;
@ -740,7 +808,7 @@ static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler) {
}
if (!second_pass && verbose)
print("Processing %s...\n", dbi_name ? dbi_name : "@MAIN");
print("Processing %s...\n", sdb_name(dbi_name));
fflush(nullptr);
rc = mdbx_dbi_flags(txn, dbi_handle, &flags);
@ -1110,9 +1178,9 @@ int main(int argc, char *argv[]) {
}
#endif
dbi_meta.name = "@META";
dbi_free.name = "@GC";
dbi_main.name = "@MAIN";
dbi_meta.name.iov_base = MDBX_PGWALK_META;
dbi_free.name.iov_base = MDBX_PGWALK_GC;
dbi_main.name.iov_base = MDBX_PGWALK_MAIN;
atexit(pagemap_cleanup);
if (argc < 2)
@ -1190,9 +1258,10 @@ int main(int argc, char *argv[]) {
dont_traversal = true;
break;
case 's':
if (only_subdb && strcmp(only_subdb, optarg))
if (only_subdb.iov_base && strcmp(only_subdb.iov_base, optarg))
usage(prog);
only_subdb = optarg;
only_subdb.iov_base = optarg;
only_subdb.iov_len = strlen(optarg);
break;
case 'i':
ignore_wrong_order = true;
@ -1230,7 +1299,7 @@ int main(int argc, char *argv[]) {
error("write-mode must be enabled to turn to the specified meta-page.\n");
rc = EXIT_INTERRUPTED;
}
if (only_subdb || dont_traversal) {
if (only_subdb.iov_base || dont_traversal) {
error(
"whole database checking with b-tree traversal are required to turn "
"to the specified meta-page.\n");
@ -1572,8 +1641,8 @@ int main(int argc, char *argv[]) {
unused_pages += 1;
empty_pages = lost_bytes = 0;
for (walk_dbi_t *dbi = &dbi_main; dbi < ARRAY_END(walk.dbi) && dbi->name;
++dbi) {
for (walk_dbi_t *dbi = &dbi_main;
dbi < ARRAY_END(walk.dbi) && dbi->name.iov_base; ++dbi) {
empty_pages += dbi->pages.empty;
lost_bytes += dbi->lost_bytes;
}
@ -1583,9 +1652,10 @@ int main(int argc, char *argv[]) {
print(" - pages: walked %" PRIu64 ", left/unused %" PRIu64 "\n",
walk.pgcount, unused_pages);
if (verbose > 1) {
for (walk_dbi_t *dbi = walk.dbi; dbi < ARRAY_END(walk.dbi) && dbi->name;
++dbi) {
print(" %s: subtotal %" PRIu64, dbi->name, dbi->pages.total);
for (walk_dbi_t *dbi = walk.dbi;
dbi < ARRAY_END(walk.dbi) && dbi->name.iov_base; ++dbi) {
print(" %s: subtotal %" PRIu64, sdb_name(&dbi->name),
dbi->pages.total);
if (dbi->pages.other && dbi->pages.other != dbi->pages.total)
print(", other %" PRIu64, dbi->pages.other);
if (dbi->pages.branch)
@ -1617,14 +1687,15 @@ int main(int argc, char *argv[]) {
(total_page_bytes - walk.total_payload_bytes) * 100.0 /
total_page_bytes);
if (verbose > 2) {
for (walk_dbi_t *dbi = walk.dbi; dbi < ARRAY_END(walk.dbi) && dbi->name;
++dbi)
for (walk_dbi_t *dbi = walk.dbi;
dbi < ARRAY_END(walk.dbi) && dbi->name.iov_base; ++dbi)
if (dbi->pages.total) {
uint64_t dbi_bytes = dbi->pages.total * envinfo.mi_dxb_pagesize;
print(" %s: subtotal %" PRIu64 " bytes (%.1f%%),"
" payload %" PRIu64 " (%.1f%%), unused %" PRIu64 " (%.1f%%)",
dbi->name, dbi_bytes, dbi_bytes * 100.0 / total_page_bytes,
dbi->payload_bytes, dbi->payload_bytes * 100.0 / dbi_bytes,
sdb_name(&dbi->name), dbi_bytes,
dbi_bytes * 100.0 / total_page_bytes, dbi->payload_bytes,
dbi->payload_bytes * 100.0 / dbi_bytes,
dbi_bytes - dbi->payload_bytes,
(dbi_bytes - dbi->payload_bytes) * 100.0 / dbi_bytes);
if (dbi->pages.empty)
@ -1633,7 +1704,7 @@ int main(int argc, char *argv[]) {
print(", %" PRIu64 " bytes lost", dbi->lost_bytes);
print("\n");
} else
print(" %s: empty\n", dbi->name);
print(" %s: empty\n", sdb_name(&dbi->name));
}
print(" - summary: average fill %.1f%%",
walk.total_payload_bytes * 100.0 / total_page_bytes);
@ -1653,7 +1724,7 @@ int main(int argc, char *argv[]) {
"b-tree", gc_tree_problems);
problems_freedb = gc_tree_problems;
} else
problems_freedb = process_db(FREE_DBI, "@GC", handle_freedb);
problems_freedb = process_db(FREE_DBI, MDBX_PGWALK_GC, handle_freedb);
if (verbose) {
uint64_t value = envinfo.mi_mapsize / envinfo.mi_dxb_pagesize;
@ -1719,7 +1790,7 @@ int main(int argc, char *argv[]) {
}
if (rc == 0 && total_problems == 1 && problems_meta == 1 && !dont_traversal &&
(envflags & MDBX_RDONLY) == 0 && !only_subdb && stuck_meta < 0 &&
(envflags & MDBX_RDONLY) == 0 && !only_subdb.iov_base && stuck_meta < 0 &&
get_meta_txnid(meta_recent(true)) < envinfo.mi_recent_txnid) {
print("Perform sync-to-disk for make steady checkpoint at txn-id #%" PRIi64
"\n",
@ -1738,7 +1809,7 @@ int main(int argc, char *argv[]) {
}
}
if (turn_meta && stuck_meta >= 0 && !dont_traversal && !only_subdb &&
if (turn_meta && stuck_meta >= 0 && !dont_traversal && !only_subdb.iov_base &&
(envflags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == MDBX_EXCLUSIVE) {
const bool successful_check = (rc | total_problems | problems_meta) == 0;
if (successful_check || force_turn_meta) {

View File

@ -66,7 +66,7 @@ static const char hexc[] = "0123456789abcdef";
static void dumpbyte(unsigned char c) {
putchar(hexc[c >> 4]);
putchar(hexc[c & 0xf]);
putchar(hexc[c & 15]);
}
static void text(MDBX_val *v) {