ISAR/libmdbx
3
0
Fork 0
mirror of https://github.com/isar/libmdbx.git synced 2025-01-10 11:44:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

mdbx: backport - refine mdbx_chk (squashed).

Browse Source 
- refine 'mismatch idl length' error message.
 - add/fix printf-format checking.
 - refine dbi-structure.
This commit is contained in:
Leonid Yuriev 2018-09-23 12:37:37 +03:00
parent 5a29214ad9
commit d2854e0760
1 changed files with 100 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

180
src/tools/mdbx_chk.c
View File

@ -1,4 +1,4 @@
/* mdbx_chk.c - memory-mapped database check tool */ /* mdbx_chk.c - memory-mapped database check tool */
/* /*
* Copyright 2015-2018 Leonid Yuriev <leo@yuriev.ru> * Copyright 2015-2018 Leonid Yuriev <leo@yuriev.ru>
@ -61,12 +61,18 @@ static void signal_handler(int sig) {
#define EXIT_FAILURE_CHECK_MAJOR (EXIT_FAILURE + 1) #define EXIT_FAILURE_CHECK_MAJOR (EXIT_FAILURE + 1)
#define EXIT_FAILURE_CHECK_MINOR EXIT_FAILURE #define EXIT_FAILURE_CHECK_MINOR EXIT_FAILURE
typedef struct {
const char *name;
struct {
uint64_t total;
uint64_t empty;
} pages;
uint64_t payload_bytes;
uint64_t lost_bytes;
} walk_dbi_t;
struct { struct {
const char *dbi_names[MAX_DBI]; walk_dbi_t dbi[MAX_DBI];
uint64_t dbi_pages[MAX_DBI];
uint64_t dbi_empty_pages[MAX_DBI];
uint64_t dbi_payload_bytes[MAX_DBI];
uint64_t dbi_lost_bytes[MAX_DBI];
short *pagemap; short *pagemap;
uint64_t total_payload_bytes; uint64_t total_payload_bytes;
uint64_t pgcount; uint64_t pgcount;
@ -95,7 +101,7 @@ struct problem *problems_list;
uint64_t total_problems; uint64_t total_problems;
static void static void
#ifdef __GNU__ #ifdef __GNUC__
__attribute__((format(printf, 1, 2))) __attribute__((format(printf, 1, 2)))
#endif #endif
print(const char *msg, ...) { print(const char *msg, ...) {
@ -110,7 +116,7 @@ static void
} }
static void static void
#ifdef __GNU__ #ifdef __GNUC__
__attribute__((format(printf, 1, 2))) __attribute__((format(printf, 1, 2)))
#endif #endif
error(const char *msg, ...) { error(const char *msg, ...) {
@ -131,9 +137,9 @@ static void pagemap_cleanup(void) {
int i; int i;
for (i = 1; i < MAX_DBI; ++i) { for (i = 1; i < MAX_DBI; ++i) {
if (walk.dbi_names[i]) { if (walk.dbi[i].name) {
free((void *)walk.dbi_names[i]); free((void *)walk.dbi[i].name);
walk.dbi_names[i] = NULL; walk.dbi[i].name = NULL;
} }
} }
@ -141,32 +147,35 @@ static void pagemap_cleanup(void) {
walk.pagemap = NULL; walk.pagemap = NULL;
} }
static int pagemap_lookup_dbi(const char *dbi) { static walk_dbi_t *pagemap_lookup_dbi(const char *dbi_name) {
static int last; static walk_dbi_t *last;
int i;
if (last > 0 && strcmp(walk.dbi_names[last], dbi) == 0) if (last && strcmp(last->name, dbi_name) == 0)
return last; return last;
for (i = 1; walk.dbi_names[i] && last < MAX_DBI; ++i) walk_dbi_t *dbi = walk.dbi + 1;
if (strcmp(walk.dbi_names[i], dbi) == 0) while (dbi->name) {
return last = i; if (strcmp(dbi->name, dbi_name) == 0)
return last = dbi;
if (i == MAX_DBI) if (++dbi == walk.dbi + MAX_DBI)
return -1; return NULL;
}
walk.dbi_names[i] = strdup(dbi);
dbi->name = strdup(dbi_name);
if (verbose > 1) { if (verbose > 1) {
print(" - found '%s' area\n", dbi); print(" - found '%s' area\n", dbi_name);
fflush(NULL); fflush(NULL);
} }
return last = i; return last = dbi;
} }
static void problem_add(const char *object, uint64_t entry_number, static void
const char *msg, const char *extra, ...) { #ifdef __GNUC__
__attribute__((format(printf, 4, 5)))
#endif
problem_add(const char *object, uint64_t entry_number, const char *msg,
const char *extra, ...) {
total_problems++; total_problems++;
if (!quiet) { if (!quiet) {
@ -233,7 +242,7 @@ static uint64_t problems_pop(struct problem *list) {
} }
static int pgvisitor(uint64_t pgno, unsigned pgnumber, void *ctx, static int pgvisitor(uint64_t pgno, unsigned pgnumber, void *ctx,
const char *dbi, const char *type, size_t nentries, const char *dbi_name, const char *type, size_t nentries,
size_t payload_bytes, size_t header_bytes, size_t payload_bytes, size_t header_bytes,
size_t unused_bytes) { size_t unused_bytes) {
(void)ctx; (void)ctx;
@ -241,54 +250,58 @@ static int pgvisitor(uint64_t pgno, unsigned pgnumber, void *ctx,
if (type) { if (type) {
uint64_t page_bytes = payload_bytes + header_bytes + unused_bytes; uint64_t page_bytes = payload_bytes + header_bytes + unused_bytes;
size_t page_size = (size_t)pgnumber * envstat.ms_psize; size_t page_size = (size_t)pgnumber * envstat.ms_psize;
int index = pagemap_lookup_dbi(dbi); walk_dbi_t *dbi = pagemap_lookup_dbi(dbi_name);
if (index < 0) if (!dbi)
return MDBX_ENOMEM; return MDBX_ENOMEM;
if (verbose > 2 && (!only_subdb || strcmp(only_subdb, dbi) == 0)) { if (verbose > 2 && (!only_subdb || strcmp(only_subdb, dbi_name) == 0)) {
if (pgnumber == 1) if (pgnumber == 1)
print(" %s-page %" PRIu64, type, pgno); print(" %s-page %" PRIu64, type, pgno);
else else
print(" %s-span %" PRIu64 "[%u]", type, pgno, pgnumber); print(" %s-span %" PRIu64 "[%u]", type, pgno, pgnumber);
print(" of %s: header %" PRIiPTR ", payload %" PRIiPTR print(" of %s: header %" PRIiPTR ", payload %" PRIiPTR
", unused %" PRIiPTR "\n", ", unused %" PRIiPTR "\n",
dbi, header_bytes, payload_bytes, unused_bytes); dbi_name, header_bytes, payload_bytes, unused_bytes);
} }
walk.pgcount += pgnumber; walk.pgcount += pgnumber;
if (unused_bytes > page_size) if (unused_bytes > page_size)
problem_add("page", pgno, "illegal unused-bytes", "%u < %i < %u", 0, problem_add("page", pgno, "illegal unused-bytes",
unused_bytes, envstat.ms_psize); "%u < %" PRIuPTR " < %u", 0, unused_bytes, envstat.ms_psize);
if (header_bytes < (int)sizeof(long) || if (header_bytes < (int)sizeof(long) ||
(size_t)header_bytes >= envstat.ms_psize - sizeof(long)) (size_t)header_bytes >= envstat.ms_psize - sizeof(long))
problem_add("page", pgno, "illegal header-length", problem_add("page", pgno, "illegal header-length",
"%" PRIuPTR " < %i < %" PRIuPTR "", sizeof(long), "%" PRIuPTR " < %" PRIuPTR " < %" PRIuPTR, sizeof(long),
header_bytes, envstat.ms_psize - sizeof(long)); header_bytes, envstat.ms_psize - sizeof(long));
if (payload_bytes < 1) { if (payload_bytes < 1) {
if (nentries > 1) { if (nentries > 1) {
problem_add("page", pgno, "zero size-of-entry", problem_add("page", pgno, "zero size-of-entry",
"payload %i bytes, %i entries", payload_bytes, nentries); "payload %" PRIuPTR " bytes, %" PRIuPTR " entries",
payload_bytes, nentries);
if ((size_t)header_bytes + unused_bytes < page_size) { if ((size_t)header_bytes + unused_bytes < page_size) {
/* LY: hush a misuse error */ /* LY: hush a misuse error */
page_bytes = page_size; page_bytes = page_size;
} }
} else { } else {
problem_add("page", pgno, "empty", "payload %i bytes, %i entries", problem_add("page", pgno, "empty",
"payload %" PRIuPTR " bytes, %" PRIuPTR " entries",
payload_bytes, nentries); payload_bytes, nentries);
walk.dbi_empty_pages[index] += 1; dbi->pages.empty += 1;
} }
} }
if (page_bytes != page_size) { if (page_bytes != page_size) {
problem_add("page", pgno, "misused", problem_add("page", pgno, "misused",
"%" PRIu64 " != %" PRIu64 " (%ih + %ip + %iu)", page_size, "%" PRIu64 " != %" PRIu64 " (%" PRIuPTR "h + %" PRIuPTR
page_bytes, header_bytes, payload_bytes, unused_bytes); "p + %" PRIuPTR "u)",
page_size, page_bytes, header_bytes, payload_bytes,
unused_bytes);
if (page_size > page_bytes) if (page_size > page_bytes)
walk.dbi_lost_bytes[index] += page_size - page_bytes; dbi->lost_bytes += page_size - page_bytes;
} else { } else {
walk.dbi_payload_bytes[index] += payload_bytes + header_bytes; dbi->payload_bytes += payload_bytes + header_bytes;
walk.total_payload_bytes += payload_bytes + header_bytes; walk.total_payload_bytes += payload_bytes + header_bytes;
} }
@ -299,10 +312,10 @@ static int pgvisitor(uint64_t pgno, unsigned pgnumber, void *ctx,
"%" PRIu64 " > %" PRIu64 "", pgno, lastpgno); "%" PRIu64 " > %" PRIu64 "", pgno, lastpgno);
else if (walk.pagemap[pgno]) else if (walk.pagemap[pgno])
problem_add("page", pgno, "already used", "in %s", problem_add("page", pgno, "already used", "in %s",
walk.dbi_names[walk.pagemap[pgno]]); walk.dbi[walk.pagemap[pgno]].name);
else { else {
walk.pagemap[pgno] = (short)index; walk.pagemap[pgno] = (short)(dbi - walk.dbi);
walk.dbi_pages[index] += 1; dbi->pages.total += 1;
} }
++pgno; ++pgno;
} while (--pgnumber); } while (--pgnumber);
@ -337,16 +350,22 @@ static int handle_freedb(const uint64_t record_number, const MDBX_val *key,
problem_add("entry", record_number, "wrong txn-id", "%" PRIaTXN "", txnid); problem_add("entry", record_number, "wrong txn-id", "%" PRIaTXN "", txnid);
if (data->iov_len < sizeof(pgno_t) || data->iov_len % sizeof(pgno_t)) if (data->iov_len < sizeof(pgno_t) || data->iov_len % sizeof(pgno_t))
problem_add("entry", record_number, "wrong idl size", "%" PRIuPTR "", problem_add("entry", txnid, "wrong idl size", "%" PRIuPTR "",
data->iov_len); data->iov_len);
else { else {
const pgno_t number = *iptr++; const pgno_t number = *iptr++;
if (number >= MDBX_PNL_UM_MAX) if (number < 1 || number >= INT_MAX / 2)
problem_add("entry", record_number, "wrong idl length", "%" PRIiPTR "", problem_add("entry", txnid, "wrong idl length", "%" PRIaPGNO, number);
number); else if ((number + 1) * sizeof(pgno_t) > data->iov_len)
else if ((number + 1) * sizeof(pgno_t) != data->iov_len) problem_add("entry", txnid, "trimmed idl",
problem_add("entry", record_number, "mismatch idl length", "%" PRIuSIZE " > %" PRIuSIZE " (corruption)",
"%" PRIuSIZE " != %" PRIuSIZE "", (number + 1) * sizeof(pgno_t), data->iov_len);
else if (data->iov_len - (number + 1) * sizeof(pgno_t) >=
/* LY: allow gap upto one page. it is ok
* and better than shink-and-retry inside mdbx_update_gc() */
envstat.ms_psize)
problem_add("entry", txnid, "extra idl space",
"%" PRIuSIZE " < %" PRIuSIZE " (minor, not a trouble)",
(number + 1) * sizeof(pgno_t), data->iov_len); (number + 1) * sizeof(pgno_t), data->iov_len);
else { else {
freedb_pages += number; freedb_pages += number;
@ -359,12 +378,12 @@ static int handle_freedb(const uint64_t record_number, const MDBX_val *key,
for (unsigned i = 0; i < number; ++i) { for (unsigned i = 0; i < number; ++i) {
const pgno_t pg = iptr[i]; const pgno_t pg = iptr[i];
if (pg < NUM_METAS || pg > envinfo.mi_last_pgno) if (pg < NUM_METAS || pg > envinfo.mi_last_pgno)
problem_add("entry", record_number, "wrong idl entry", problem_add("entry", txnid, "wrong idl entry",
"%u < %" PRIaPGNO " < %" PRIu64 "", NUM_METAS, pg, "%u < %" PRIaPGNO " < %" PRIu64 "", NUM_METAS, pg,
envinfo.mi_last_pgno); envinfo.mi_last_pgno);
else if (MDBX_PNL_DISORDERED(prev, pg)) { else if (MDBX_PNL_DISORDERED(prev, pg)) {
bad = " [bad sequence]"; bad = " [bad sequence]";
problem_add("entry", record_number, "bad sequence", problem_add("entry", txnid, "bad sequence",
"%" PRIaPGNO " <> %" PRIaPGNO "", prev, pg); "%" PRIaPGNO " <> %" PRIaPGNO "", prev, pg);
} }
prev = pg; prev = pg;
@ -516,7 +535,7 @@ static int process_db(MDBX_dbi dbi, char *name, visitor *handler, bool silent) {
if (key.iov_len > maxkeysize) { if (key.iov_len > maxkeysize) {
problem_add("entry", record_count, "key length exceeds max-key-size", problem_add("entry", record_count, "key length exceeds max-key-size",
"%" PRIuPTR " > %u", key.iov_len, maxkeysize); "%" PRIuPTR " > %" PRIuPTR, key.iov_len, maxkeysize);
} else if ((flags & MDBX_INTEGERKEY) && key.iov_len != sizeof(uint64_t) && } else if ((flags & MDBX_INTEGERKEY) && key.iov_len != sizeof(uint64_t) &&
key.iov_len != sizeof(uint32_t)) { key.iov_len != sizeof(uint32_t)) {
problem_add("entry", record_count, "wrong key length", problem_add("entry", record_count, "wrong key length",
@ -758,7 +777,7 @@ static void print_size(const char *prefix, const uint64_t value,
} }
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
int i, rc; int rc;
char *prog = argv[0]; char *prog = argv[0];
char *envname; char *envname;
int problems_maindb = 0, problems_freedb = 0, problems_meta = 0; int problems_maindb = 0, problems_freedb = 0, problems_meta = 0;
@ -778,14 +797,14 @@ int main(int argc, char *argv[]) {
} }
#endif #endif
walk.dbi_names[0] = "@gc"; walk.dbi[FREE_DBI].name = "@gc";
atexit(pagemap_cleanup); atexit(pagemap_cleanup);
if (argc < 2) { if (argc < 2) {
usage(prog); usage(prog);
} }
while ((i = getopt(argc, argv, "Vvqnwcds:")) != EOF) { for (int i; (i = getopt(argc, argv, "Vvqnwcds:")) != EOF;) {
switch (i) { switch (i) {
case 'V': case 'V':
printf("%s (%s, build %s)\n", mdbx_version.git.describe, printf("%s (%s, build %s)\n", mdbx_version.git.describe,
@ -988,24 +1007,25 @@ int main(int argc, char *argv[]) {
goto bailout; goto bailout;
} }
uint64_t n; for (uint64_t n = 0; n < lastpgno; ++n)
for (n = 0; n < lastpgno; ++n)
if (!walk.pagemap[n]) if (!walk.pagemap[n])
walk.dbi_pages[0] += 1; walk.dbi[FREE_DBI].pages.total += 1;
empty_pages = lost_bytes = 0; empty_pages = lost_bytes = 0;
for (i = 1; i < MAX_DBI && walk.dbi_names[i]; ++i) { for (walk_dbi_t *dbi = walk.dbi; ++dbi < walk.dbi + MAX_DBI && dbi->name;) {
empty_pages += walk.dbi_empty_pages[i]; empty_pages += dbi->pages.empty;
lost_bytes += walk.dbi_lost_bytes[i]; lost_bytes += dbi->lost_bytes;
} }
if (verbose) { if (verbose) {
uint64_t total_page_bytes = walk.pgcount * envstat.ms_psize; uint64_t total_page_bytes = walk.pgcount * envstat.ms_psize;
print(" - dbi pages: %" PRIu64 " total", walk.pgcount); print(" - dbi pages: %" PRIu64 " total", walk.pgcount);
if (verbose > 1) if (verbose > 1)
for (i = 1; i < MAX_DBI && walk.dbi_names[i]; ++i) for (walk_dbi_t *dbi = walk.dbi;
print(", %s %" PRIu64 "", walk.dbi_names[i], walk.dbi_pages[i]); ++dbi < walk.dbi + MAX_DBI && dbi->name;)
print(", %s %" PRIu64 "\n", walk.dbi_names[0], walk.dbi_pages[0]); print(", %s %" PRIu64, dbi->name, dbi->pages.total);
print(", %s %" PRIu64 "\n", walk.dbi[FREE_DBI].name,
walk.dbi[FREE_DBI].pages.total);
if (verbose > 1) { if (verbose > 1) {
print(" - space info: total %" PRIu64 " bytes, payload %" PRIu64 print(" - space info: total %" PRIu64 " bytes, payload %" PRIu64
" (%.1f%%), unused " " (%.1f%%), unused "
@ -1015,19 +1035,19 @@ int main(int argc, char *argv[]) {
total_page_bytes - walk.total_payload_bytes, total_page_bytes - walk.total_payload_bytes,
(total_page_bytes - walk.total_payload_bytes) * 100.0 / (total_page_bytes - walk.total_payload_bytes) * 100.0 /
total_page_bytes); total_page_bytes);
for (i = 1; i < MAX_DBI && walk.dbi_names[i]; ++i) { for (walk_dbi_t *dbi = walk.dbi;
uint64_t dbi_bytes = walk.dbi_pages[i] * envstat.ms_psize; ++dbi < walk.dbi + MAX_DBI && dbi->name;) {
uint64_t dbi_bytes = dbi->pages.total * envstat.ms_psize;
print(" %s: subtotal %" PRIu64 " bytes (%.1f%%)," print(" %s: subtotal %" PRIu64 " bytes (%.1f%%),"
" payload %" PRIu64 " (%.1f%%), unused %" PRIu64 " (%.1f%%)", " payload %" PRIu64 " (%.1f%%), unused %" PRIu64 " (%.1f%%)",
walk.dbi_names[i], dbi_bytes, dbi->name, dbi_bytes, dbi_bytes * 100.0 / total_page_bytes,
dbi_bytes * 100.0 / total_page_bytes, walk.dbi_payload_bytes[i], dbi->payload_bytes, dbi->payload_bytes * 100.0 / dbi_bytes,
walk.dbi_payload_bytes[i] * 100.0 / dbi_bytes, dbi_bytes - dbi->payload_bytes,
dbi_bytes - walk.dbi_payload_bytes[i], (dbi_bytes - dbi->payload_bytes) * 100.0 / dbi_bytes);
(dbi_bytes - walk.dbi_payload_bytes[i]) * 100.0 / dbi_bytes); if (dbi->pages.empty)
if (walk.dbi_empty_pages[i]) print(", %" PRIu64 " empty pages", dbi->pages.empty);
print(", %" PRIu64 " empty pages", walk.dbi_empty_pages[i]); if (dbi->lost_bytes)
if (walk.dbi_lost_bytes[i]) print(", %" PRIu64 " bytes lost", dbi->lost_bytes);
print(", %" PRIu64 " bytes lost", walk.dbi_lost_bytes[i]);
print("\n"); print("\n");
} }
} }
@ -1084,9 +1104,9 @@ int main(int argc, char *argv[]) {
error("used pages mismatch (%" PRIu64 " != %" PRIu64 ")\n", error("used pages mismatch (%" PRIu64 " != %" PRIu64 ")\n",
walk.pgcount, lastpgno - freedb_pages); walk.pgcount, lastpgno - freedb_pages);
} }
if (walk.dbi_pages[0] != freedb_pages) { if (walk.dbi[FREE_DBI].pages.total != freedb_pages) {
error("gc pages mismatch (%" PRIu64 " != %" PRIu64 ")\n", error("gc pages mismatch (%" PRIu64 " != %" PRIu64 ")\n",
walk.dbi_pages[0], freedb_pages); walk.dbi[FREE_DBI].pages.total, freedb_pages);
} }
} else if (verbose) { } else if (verbose) {
print(" - skip check used and gc pages (btree-traversal with " print(" - skip check used and gc pages (btree-traversal with "