From 44493c6448d34572ddf0077664aeb06988257da6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9B=D0=B5=D0=BE=D0=BD=D0=B8=D0=B4=20=D0=AE=D1=80=D1=8C?= =?UTF-8?q?=D0=B5=D0=B2=20=28Leonid=20Yuriev=29?= Date: Wed, 14 Dec 2022 14:42:16 +0300 Subject: [PATCH] =?UTF-8?q?mdbx-tools:=20=D0=BF=D0=BE=D0=B4=D0=B4=D0=B5?= =?UTF-8?q?=D1=80=D0=B6=D0=BA=D0=B0=20=D0=BD=D0=B5-=D0=BF=D0=B5=D1=87?= =?UTF-8?q?=D0=B0=D1=82=D0=BD=D1=8B=D1=85=20=D0=B8=D0=BC=D0=B5=D0=BD=20sub?= =?UTF-8?q?Db=20=D0=B2=20`mdbx=5Fchk`.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/mdbx_chk.c | 245 +++++++++++++++++++++++++++++++----------------- src/mdbx_dump.c | 2 +- 2 files changed, 159 insertions(+), 88 deletions(-) diff --git a/src/mdbx_chk.c b/src/mdbx_chk.c index d1cb718e..3a6c59eb 100644 --- a/src/mdbx_chk.c +++ b/src/mdbx_chk.c @@ -23,6 +23,8 @@ #define xMDBX_TOOLS /* Avoid using internal eASSERT() */ #include "internals.h" +#include + typedef struct flagbit { int bit; const char *name; @@ -71,7 +73,7 @@ static void signal_handler(int sig) { #define EXIT_FAILURE_CHECK_MINOR EXIT_FAILURE typedef struct { - const char *name; + MDBX_val name; struct { uint64_t branch, large_count, large_volume, leaf; uint64_t subleaf_dupsort, leaf_dupfixed, subleaf_dupfixed; @@ -102,7 +104,7 @@ uint64_t total_unused_bytes, reclaimable_pages, gc_pages, alloc_pages, unused_pages, backed_pages; unsigned verbose; bool ignore_wrong_order, quiet, dont_traversal; -const char *only_subdb; +MDBX_val only_subdb; int stuck_meta = -1; struct problem { @@ -125,6 +127,93 @@ static void MDBX_PRINTF_ARGS(1, 2) print(const char *msg, ...) { } } +static MDBX_val printable_buf; +static void free_printable_buf(void) { osal_free(printable_buf.iov_base); } + +static const char *sdb_name(const MDBX_val *val) { + if (val == MDBX_PGWALK_MAIN) + return "@MAIN"; + if (val == MDBX_PGWALK_GC) + return "@GC"; + if (val == MDBX_PGWALK_META) + return "@META"; + + const unsigned char *const data = val->iov_base; + const size_t len = val->iov_len; + if (data == MDBX_PGWALK_MAIN) + return "@MAIN"; + if (data == MDBX_PGWALK_GC) + return "@GC"; + if (data == MDBX_PGWALK_META) + return "@META"; + + if (!len) + return ""; + if (!data) + return ""; + if (len > 65536) { + static char buf[64]; + snprintf(buf, sizeof(buf), "", len); + return buf; + } + + bool printable = true; + bool quoting = false; + size_t xchars = 0; + for (size_t i = 0; i < val->iov_len && printable; ++i) { + quoting |= data[i] != '_' && isalnum(data[i]) == 0; + printable = isprint(data[i]) != 0 || + (data[i] < ' ' && ++xchars < 4 && len > xchars * 4); + } + + size_t need = len + 1; + if (quoting || !printable) + need += len + /* quotes */ 2 + 2 * /* max xchars */ 4; + if (need > printable_buf.iov_len) { + void *ptr = osal_realloc(printable_buf.iov_base, need); + if (!ptr) + return ""; + if (!printable_buf.iov_base) + atexit(free_printable_buf); + printable_buf.iov_base = ptr; + printable_buf.iov_len = need; + } + + char *out = printable_buf.iov_base; + if (!quoting) { + memcpy(out, data, len); + out += len; + } else if (printable) { + *out++ = '\''; + for (size_t i = 0; i < len; ++i) { + if (data[i] < ' ') { + assert((char *)printable_buf.iov_base + printable_buf.iov_len > + out + 4); + static const char hex[] = "0123456789abcdef"; + out[0] = '\\'; + out[1] = 'x'; + out[2] = hex[data[i] >> 4]; + out[3] = hex[data[i] & 15]; + out += 4; + } else if (strchr("\"'`\\", data[i])) { + assert((char *)printable_buf.iov_base + printable_buf.iov_len > + out + 2); + out[0] = '\\'; + out[1] = data[i]; + out += 2; + } else { + assert((char *)printable_buf.iov_base + printable_buf.iov_len > + out + 1); + *out++ = data[i]; + } + } + *out++ = '\''; + } + assert((char *)printable_buf.iov_base + printable_buf.iov_len > out); + *out = 0; + return printable_buf.iov_base; +} + static void va_log(MDBX_log_level_t level, const char *function, int line, const char *msg, va_list args) { static const char *const prefixes[] = { @@ -190,19 +279,17 @@ static int check_user_break(void) { } static void pagemap_cleanup(void) { - for (size_t i = CORE_DBS + /* account pseudo-entry for meta */ 1; - i < ARRAY_LENGTH(walk.dbi); ++i) { - if (walk.dbi[i].name) { - osal_free((void *)walk.dbi[i].name); - walk.dbi[i].name = nullptr; - } - } - osal_free(walk.pagemap); walk.pagemap = nullptr; } -static walk_dbi_t *pagemap_lookup_dbi(const char *dbi_name, bool silent) { +static bool eq(const MDBX_val a, const MDBX_val b) { + return a.iov_len == b.iov_len && + (a.iov_base == b.iov_base || a.iov_len == 0 || + !memcmp(a.iov_base, b.iov_base, a.iov_len)); +} + +static walk_dbi_t *pagemap_lookup_dbi(const MDBX_val *dbi_name, bool silent) { static walk_dbi_t *last; if (dbi_name == MDBX_PGWALK_MAIN) @@ -212,24 +299,24 @@ static walk_dbi_t *pagemap_lookup_dbi(const char *dbi_name, bool silent) { if (dbi_name == MDBX_PGWALK_META) return &dbi_meta; - if (last && strcmp(last->name, dbi_name) == 0) + if (last && eq(last->name, *dbi_name)) return last; walk_dbi_t *dbi = walk.dbi + CORE_DBS + /* account pseudo-entry for meta */ 1; - for (; dbi < ARRAY_END(walk.dbi) && dbi->name; ++dbi) { - if (strcmp(dbi->name, dbi_name) == 0) + for (; dbi < ARRAY_END(walk.dbi) && dbi->name.iov_base; ++dbi) { + if (eq(dbi->name, *dbi_name)) return last = dbi; } if (verbose > 0 && !silent) { - print(" - found `%s` area\n", dbi_name); + print(" - found %s area\n", sdb_name(dbi_name)); fflush(nullptr); } if (dbi == ARRAY_END(walk.dbi)) return nullptr; - dbi->name = osal_strdup(dbi_name); + dbi->name = *dbi_name; return last = dbi; } @@ -304,13 +391,13 @@ static size_t problems_pop(struct problem *list) { } static int pgvisitor(const uint64_t pgno, const unsigned pgnumber, - void *const ctx, const int deep, - const char *const dbi_name_or_tag, const size_t page_size, - const MDBX_page_type_t pagetype, const MDBX_error_t err, - const size_t nentries, const size_t payload_bytes, - const size_t header_bytes, const size_t unused_bytes) { + void *const ctx, const int deep, const MDBX_val *dbi_name, + const size_t page_size, const MDBX_page_type_t pagetype, + const MDBX_error_t err, const size_t nentries, + const size_t payload_bytes, const size_t header_bytes, + const size_t unused_bytes) { (void)ctx; - const bool is_gc_tree = dbi_name_or_tag == MDBX_PGWALK_GC; + const bool is_gc_tree = dbi_name == MDBX_PGWALK_GC; if (deep > 42) { problem_add("deep", deep, "too large", nullptr); data_tree_problems += !is_gc_tree; @@ -318,7 +405,7 @@ static int pgvisitor(const uint64_t pgno, const unsigned pgnumber, return MDBX_CORRUPTED /* avoid infinite loop/recursion */; } - walk_dbi_t *dbi = pagemap_lookup_dbi(dbi_name_or_tag, false); + walk_dbi_t *dbi = pagemap_lookup_dbi(dbi_name, false); if (!dbi) { data_tree_problems += !is_gc_tree; gc_tree_problems += is_gc_tree; @@ -383,14 +470,14 @@ static int pgvisitor(const uint64_t pgno, const unsigned pgnumber, } if (pgnumber) { - if (verbose > 3 && (!only_subdb || strcmp(only_subdb, dbi->name) == 0)) { + if (verbose > 3 && (!only_subdb.iov_base || eq(only_subdb, dbi->name))) { if (pgnumber == 1) print(" %s-page %" PRIu64, pagetype_caption, pgno); else print(" %s-span %" PRIu64 "[%u]", pagetype_caption, pgno, pgnumber); print(" of %s: header %" PRIiPTR ", %s %" PRIiPTR ", payload %" PRIiPTR ", unused %" PRIiPTR ", deep %i\n", - dbi->name, header_bytes, + sdb_name(&dbi->name), header_bytes, (pagetype == MDBX_page_branch) ? "keys" : "entries", nentries, payload_bytes, unused_bytes, deep); } @@ -408,8 +495,8 @@ static int pgvisitor(const uint64_t pgno, const unsigned pgnumber, walk_dbi_t *coll_dbi = &walk.dbi[walk.pagemap[spanpgno] - 1]; problem_add("page", spanpgno, (branch && coll_dbi == dbi) ? "loop" : "already used", - "%s-page: by %s, deep %i", pagetype_caption, coll_dbi->name, - deep); + "%s-page: by %s, deep %i", pagetype_caption, + sdb_name(&coll_dbi->name), deep); already_used = true; data_tree_problems += !is_gc_tree; gc_tree_problems += is_gc_tree; @@ -491,7 +578,8 @@ static int pgvisitor(const uint64_t pgno, const unsigned pgnumber, typedef int(visitor)(const uint64_t record_number, const MDBX_val *key, const MDBX_val *data); -static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler); +static int process_db(MDBX_dbi dbi_handle, const MDBX_val *dbi_name, + visitor *handler); static int handle_userdb(const uint64_t record_number, const MDBX_val *key, const MDBX_val *data) { @@ -569,7 +657,7 @@ static int handle_freedb(const uint64_t record_number, const MDBX_val *key, walk.pagemap[pgno] = -1; else if (idx > 0) problem_add("page", pgno, "already used", "by %s", - walk.dbi[idx - 1].name); + sdb_name(&walk.dbi[idx - 1].name)); else problem_add("page", pgno, "already listed in GC", nullptr); } @@ -580,7 +668,7 @@ static int handle_freedb(const uint64_t record_number, const MDBX_val *key, : pgno_sub(pgno, span))) ++span; } - if (verbose > 3 && !only_subdb) { + if (verbose > 3 && !only_subdb.iov_base) { print(" transaction %" PRIaTXN ", %" PRIuPTR " pages, maxspan %" PRIaPGNO "%s\n", txnid, number, span, bad); @@ -607,39 +695,18 @@ static int handle_freedb(const uint64_t record_number, const MDBX_val *key, } static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) { - return (a->iov_len == b->iov_len && - memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) - ? 0 - : 1; + return eq(*a, *b) ? 0 : 1; } static int handle_maindb(const uint64_t record_number, const MDBX_val *key, const MDBX_val *data) { - char *name; - int rc; - size_t i; - - if (data->iov_len != sizeof(MDBX_db)) - return handle_userdb(record_number, key, data); - name = key->iov_base; - for (i = 0; i < key->iov_len; ++i) { - if (name[i] < ' ') - return handle_userdb(record_number, key, data); + if (data->iov_len == sizeof(MDBX_db)) { + int rc = process_db(~0u, key, handle_userdb); + if (rc != MDBX_INCOMPATIBLE) { + userdb_count++; + return rc; + } } - - name = osal_malloc(key->iov_len + 1); - if (unlikely(!name)) - return MDBX_ENOMEM; - memcpy(name, key->iov_base, key->iov_len); - name[key->iov_len] = '\0'; - - rc = process_db(~0u, name, handle_userdb); - osal_free(name); - if (rc != MDBX_INCOMPATIBLE) { - userdb_count++; - return rc; - } - return handle_userdb(record_number, key, data); } @@ -693,7 +760,8 @@ static const char *db_flags2valuemode(unsigned flags) { } } -static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler) { +static int process_db(MDBX_dbi dbi_handle, const MDBX_val *dbi_name, + visitor *handler) { MDBX_cursor *mc; MDBX_stat ms; MDBX_val key, data; @@ -708,13 +776,13 @@ static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler) { uint64_t key_bytes = 0, data_bytes = 0; if ((MDBX_TXN_FINISHED | MDBX_TXN_ERROR) & mdbx_txn_flags(txn)) { - print(" ! abort processing `%s` due to a previous error\n", - dbi_name ? dbi_name : "@MAIN"); + print(" ! abort processing %s due to a previous error\n", + sdb_name(dbi_name)); return MDBX_BAD_TXN; } if (dbi_handle == ~0u) { - rc = mdbx_dbi_open_ex( + rc = mdbx_dbi_open_ex2( txn, dbi_name, MDBX_DB_ACCEDE, &dbi_handle, (dbi_name && ignore_wrong_order) ? equal_or_greater : nullptr, (dbi_name && ignore_wrong_order) ? equal_or_greater : nullptr); @@ -722,17 +790,17 @@ static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler) { if (!dbi_name || rc != MDBX_INCOMPATIBLE) /* LY: mainDB's record is not a user's DB. */ { - error("mdbx_dbi_open(`%s`) failed, error %d %s\n", - dbi_name ? dbi_name : "@MAIN", rc, mdbx_strerror(rc)); + error("mdbx_dbi_open(%s) failed, error %d %s\n", sdb_name(dbi_name), rc, + mdbx_strerror(rc)); } return rc; } } - if (dbi_handle >= CORE_DBS && dbi_name && only_subdb && - strcmp(only_subdb, dbi_name) != 0) { + if (dbi_handle >= CORE_DBS && dbi_name && only_subdb.iov_base && + !eq(only_subdb, *dbi_name)) { if (verbose) { - print("Skip processing %s...\n", dbi_name); + print("Skip processing %s...\n", sdb_name(dbi_name)); fflush(nullptr); } skipped_subdb++; @@ -740,7 +808,7 @@ static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler) { } if (!second_pass && verbose) - print("Processing %s...\n", dbi_name ? dbi_name : "@MAIN"); + print("Processing %s...\n", sdb_name(dbi_name)); fflush(nullptr); rc = mdbx_dbi_flags(txn, dbi_handle, &flags); @@ -1110,9 +1178,9 @@ int main(int argc, char *argv[]) { } #endif - dbi_meta.name = "@META"; - dbi_free.name = "@GC"; - dbi_main.name = "@MAIN"; + dbi_meta.name.iov_base = MDBX_PGWALK_META; + dbi_free.name.iov_base = MDBX_PGWALK_GC; + dbi_main.name.iov_base = MDBX_PGWALK_MAIN; atexit(pagemap_cleanup); if (argc < 2) @@ -1190,9 +1258,10 @@ int main(int argc, char *argv[]) { dont_traversal = true; break; case 's': - if (only_subdb && strcmp(only_subdb, optarg)) + if (only_subdb.iov_base && strcmp(only_subdb.iov_base, optarg)) usage(prog); - only_subdb = optarg; + only_subdb.iov_base = optarg; + only_subdb.iov_len = strlen(optarg); break; case 'i': ignore_wrong_order = true; @@ -1230,7 +1299,7 @@ int main(int argc, char *argv[]) { error("write-mode must be enabled to turn to the specified meta-page.\n"); rc = EXIT_INTERRUPTED; } - if (only_subdb || dont_traversal) { + if (only_subdb.iov_base || dont_traversal) { error( "whole database checking with b-tree traversal are required to turn " "to the specified meta-page.\n"); @@ -1572,8 +1641,8 @@ int main(int argc, char *argv[]) { unused_pages += 1; empty_pages = lost_bytes = 0; - for (walk_dbi_t *dbi = &dbi_main; dbi < ARRAY_END(walk.dbi) && dbi->name; - ++dbi) { + for (walk_dbi_t *dbi = &dbi_main; + dbi < ARRAY_END(walk.dbi) && dbi->name.iov_base; ++dbi) { empty_pages += dbi->pages.empty; lost_bytes += dbi->lost_bytes; } @@ -1583,9 +1652,10 @@ int main(int argc, char *argv[]) { print(" - pages: walked %" PRIu64 ", left/unused %" PRIu64 "\n", walk.pgcount, unused_pages); if (verbose > 1) { - for (walk_dbi_t *dbi = walk.dbi; dbi < ARRAY_END(walk.dbi) && dbi->name; - ++dbi) { - print(" %s: subtotal %" PRIu64, dbi->name, dbi->pages.total); + for (walk_dbi_t *dbi = walk.dbi; + dbi < ARRAY_END(walk.dbi) && dbi->name.iov_base; ++dbi) { + print(" %s: subtotal %" PRIu64, sdb_name(&dbi->name), + dbi->pages.total); if (dbi->pages.other && dbi->pages.other != dbi->pages.total) print(", other %" PRIu64, dbi->pages.other); if (dbi->pages.branch) @@ -1617,14 +1687,15 @@ int main(int argc, char *argv[]) { (total_page_bytes - walk.total_payload_bytes) * 100.0 / total_page_bytes); if (verbose > 2) { - for (walk_dbi_t *dbi = walk.dbi; dbi < ARRAY_END(walk.dbi) && dbi->name; - ++dbi) + for (walk_dbi_t *dbi = walk.dbi; + dbi < ARRAY_END(walk.dbi) && dbi->name.iov_base; ++dbi) if (dbi->pages.total) { uint64_t dbi_bytes = dbi->pages.total * envinfo.mi_dxb_pagesize; print(" %s: subtotal %" PRIu64 " bytes (%.1f%%)," " payload %" PRIu64 " (%.1f%%), unused %" PRIu64 " (%.1f%%)", - dbi->name, dbi_bytes, dbi_bytes * 100.0 / total_page_bytes, - dbi->payload_bytes, dbi->payload_bytes * 100.0 / dbi_bytes, + sdb_name(&dbi->name), dbi_bytes, + dbi_bytes * 100.0 / total_page_bytes, dbi->payload_bytes, + dbi->payload_bytes * 100.0 / dbi_bytes, dbi_bytes - dbi->payload_bytes, (dbi_bytes - dbi->payload_bytes) * 100.0 / dbi_bytes); if (dbi->pages.empty) @@ -1633,7 +1704,7 @@ int main(int argc, char *argv[]) { print(", %" PRIu64 " bytes lost", dbi->lost_bytes); print("\n"); } else - print(" %s: empty\n", dbi->name); + print(" %s: empty\n", sdb_name(&dbi->name)); } print(" - summary: average fill %.1f%%", walk.total_payload_bytes * 100.0 / total_page_bytes); @@ -1653,7 +1724,7 @@ int main(int argc, char *argv[]) { "b-tree", gc_tree_problems); problems_freedb = gc_tree_problems; } else - problems_freedb = process_db(FREE_DBI, "@GC", handle_freedb); + problems_freedb = process_db(FREE_DBI, MDBX_PGWALK_GC, handle_freedb); if (verbose) { uint64_t value = envinfo.mi_mapsize / envinfo.mi_dxb_pagesize; @@ -1719,7 +1790,7 @@ int main(int argc, char *argv[]) { } if (rc == 0 && total_problems == 1 && problems_meta == 1 && !dont_traversal && - (envflags & MDBX_RDONLY) == 0 && !only_subdb && stuck_meta < 0 && + (envflags & MDBX_RDONLY) == 0 && !only_subdb.iov_base && stuck_meta < 0 && get_meta_txnid(meta_recent(true)) < envinfo.mi_recent_txnid) { print("Perform sync-to-disk for make steady checkpoint at txn-id #%" PRIi64 "\n", @@ -1738,7 +1809,7 @@ int main(int argc, char *argv[]) { } } - if (turn_meta && stuck_meta >= 0 && !dont_traversal && !only_subdb && + if (turn_meta && stuck_meta >= 0 && !dont_traversal && !only_subdb.iov_base && (envflags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == MDBX_EXCLUSIVE) { const bool successful_check = (rc | total_problems | problems_meta) == 0; if (successful_check || force_turn_meta) { diff --git a/src/mdbx_dump.c b/src/mdbx_dump.c index f710d33d..8c266aed 100644 --- a/src/mdbx_dump.c +++ b/src/mdbx_dump.c @@ -66,7 +66,7 @@ static const char hexc[] = "0123456789abcdef"; static void dumpbyte(unsigned char c) { putchar(hexc[c >> 4]); - putchar(hexc[c & 0xf]); + putchar(hexc[c & 15]); } static void text(MDBX_val *v) {