From 1ef0106b51022098f1a3bf7b5cb21a45bfaf774f Mon Sep 17 00:00:00 2001 From: Hallvard Furuseth Date: Tue, 10 May 2016 07:11:44 +0200 Subject: [PATCH 1/6] mdbx: backport - Comment ovpage code in mdb_cursor_put(). Change-Id: I6b3bff87dc49135fdda3e9ce0a505822ab26d3d5 --- mdb.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mdb.c b/mdb.c index b2fe20cb..4dae73ba 100644 --- a/mdb.c +++ b/mdb.c @@ -6895,9 +6895,14 @@ current: /* Note - this page is already counted in parent's dirty_room */ rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2); mdb_cassert(mc, rc2 == 0); + /* Currently we make the page look as with put() in the + * parent txn, in case the user peeks at MDB_RESERVEd + * or unused parts. Some users treat ovpages specially. + */ if (1 || /* LY: Hm, why we should do this differently in dependence from MDB_RESERVE? */ !(flags & MDB_RESERVE)) { - /* Copy end of page, adjusting alignment so + /* Skip the part where LMDB will put *data. + * Copy end of page, adjusting alignment so * compiler may copy words instead of bytes. */ off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t); @@ -6905,7 +6910,7 @@ current: (size_t *)((char *)omp + off), sz - off); sz = PAGEHDRSZ; } - memcpy(np, omp, sz); /* Copy beginning of page */ + memcpy(np, omp, sz); /* Copy whole or beginning of page */ omp = np; } SETDSZ(leaf, data->mv_size); From 09d790431710f6456cb80bcfc5962da5851893ed Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Tue, 10 May 2016 13:21:38 +0300 Subject: [PATCH 2/6] mdbx: clarify ov-pages copying in cursor_put(). Change-Id: I48ae57579a7d68178ec1857785ffdd6f0c0f7e13 --- mdb.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/mdb.c b/mdb.c index 4dae73ba..6b250e6a 100644 --- a/mdb.c +++ b/mdb.c @@ -6885,7 +6885,6 @@ current: */ if (unlikely(level > 1)) { /* It is writable only in a parent txn */ - size_t sz = (size_t) env->me_psize * ovpages, off; MDB_page *np = mdb_page_malloc(mc->mc_txn, ovpages); MDB_ID2 id2; if (unlikely(!np)) @@ -6899,8 +6898,13 @@ current: * parent txn, in case the user peeks at MDB_RESERVEd * or unused parts. Some users treat ovpages specially. */ - if (1 || /* LY: Hm, why we should do this differently in dependence from MDB_RESERVE? */ - !(flags & MDB_RESERVE)) { +#if MDBX_MODE_ENABLED + /* LY: New page will contain only header from origin, + * but no any payload */ + memcpy(np, omp, PAGEHDRSZ); +#else + size_t sz = (size_t) env->me_psize * ovpages, off; + if (!(flags & MDB_RESERVE)) { /* Skip the part where LMDB will put *data. * Copy end of page, adjusting alignment so * compiler may copy words instead of bytes. @@ -6910,7 +6914,8 @@ current: (size_t *)((char *)omp + off), sz - off); sz = PAGEHDRSZ; } - memcpy(np, omp, sz); /* Copy whole or beginning of page */ + memcpy(np, omp, sz); /* Copy whole or header of page */ +#endif /* MDBX_MODE_ENABLED */ omp = np; } SETDSZ(leaf, data->mv_size); From e84b42022736a0e0a053710666f6f216cfa18880 Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Tue, 10 May 2016 16:47:17 +0300 Subject: [PATCH 3/6] mdbx: minor (cleanup non-ascii). Change-Id: I46dc4f4fb3c984210d65fa31af65f570a98f4813 --- mdbx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mdbx.h b/mdbx.h index 4de12f1a..db5bc55b 100644 --- a/mdbx.h +++ b/mdbx.h @@ -31,7 +31,7 @@ When needed drop-in replacement for liblmdb you should: - 'make lmdb' to build liblmdb.so and liblmdb.a; - #include and use mdb_* functions; - - linking with liblmdb.so оr liblmdb.a; + - linking with liblmdb.so or liblmdb.a; = This provides nearly full compatibility with original LMDB from Symas Corp. @@ -41,7 +41,7 @@ When exactly the libmdbx is needed, you should: - 'make mdbx' to build libmdbx.so and libmdbx.a; - #include and use mdbx_* functions; - - linking with libmdbx.so оr libmdbx.a; + - linking with libmdbx.so or libmdbx.a; = This allows using (linking) both MDBX and LMDB simultaneously in the one application, for instance From 856f30b52c69b189e016a8852c63e70837b7b8c5 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sun, 15 May 2016 00:44:54 +0100 Subject: [PATCH 4/6] mdbx: backport - ITS#8424 init cursor in mdb_env_cwalk. Change-Id: I979b6f492c56d1c61ce233727ff058df48f327d1 --- mdb.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mdb.c b/mdb.c index 6b250e6a..1d0f9c14 100644 --- a/mdb.c +++ b/mdb.c @@ -9187,7 +9187,7 @@ mdb_env_cthr_toggle(mdb_copy *my, int st) static int __cold mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) { - MDB_cursor mc; + MDB_cursor mc = {0}; MDB_txn *txn = my->mc_txn; MDB_node *ni; MDB_page *mo, *mp, *leaf; @@ -9200,10 +9200,9 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) return MDB_SUCCESS; mc.mc_snum = 1; - mc.mc_top = 0; mc.mc_txn = txn; - rc = mdb_page_get(my->mc_txn, *pg, &mc.mc_pg[0], NULL); + rc = mdb_page_get(txn, *pg, &mc.mc_pg[0], NULL); if (rc) return rc; rc = mdb_page_search_root(&mc, NULL, MDB_PS_FIRST); From f943bbf91ce9607172b56ab7f5eb9974aaf87fea Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Thu, 19 May 2016 22:05:46 +0300 Subject: [PATCH 5/6] mdbx: more for ASAN. Change-Id: I2bd0573cfe4c7822fa8f3cf94db60b684964a40d --- mdb.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mdb.c b/mdb.c index 1d0f9c14..f18e6742 100644 --- a/mdb.c +++ b/mdb.c @@ -1505,6 +1505,7 @@ mdb_page_malloc(MDB_txn *txn, unsigned num) size_t size = env->me_psize; MDB_page *np = env->me_dpages; if (likely(num == 1 && np)) { + ASAN_UNPOISON_MEMORY_REGION(np, size); VALGRIND_MEMPOOL_ALLOC(env, np, size); VALGRIND_MAKE_MEM_DEFINED(&np->mp_next, sizeof(np->mp_next)); env->me_dpages = np->mp_next; @@ -1583,6 +1584,7 @@ mdb_kill_page(MDB_env *env, pgno_t pgno) MDB_page *mp = (MDB_page *)(env->me_map + offs); memset(&mp->mp_pb, 0x6F /* 'o', 111 */, env->me_psize - shift); VALGRIND_MAKE_MEM_NOACCESS(&mp->mp_pb, env->me_psize - shift); + ASAN_POISON_MEMORY_REGION(&mp->mp_pb, env->me_psize - shift); } else { struct iovec iov[1]; iov[0].iov_len = env->me_psize - shift; @@ -1636,9 +1638,13 @@ mdb_page_loose(MDB_cursor *mc, MDB_page *mp) } if (loose) { mdb_debug("loosen db %d page %zu", DDBI(mc), mp->mp_pgno); - if (unlikely(txn->mt_env->me_flags & MDBX_PAGEPERTURB)) + MDB_page **link = &NEXT_LOOSE_PAGE(mp); + if (unlikely(txn->mt_env->me_flags & MDBX_PAGEPERTURB)) { mdb_kill_page(txn->mt_env, pgno); - NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs; + VALGRIND_MAKE_MEM_UNDEFINED(link, sizeof(MDB_page*)); + ASAN_UNPOISON_MEMORY_REGION(link, sizeof(MDB_page*)); + } + *link = txn->mt_loose_pgs; txn->mt_loose_pgs = mp; txn->mt_loose_count++; mp->mp_flags |= P_LOOSE; @@ -2110,6 +2116,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp, int flags) txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np); txn->mt_loose_count--; mdb_debug("db %d use loose page %zu", DDBI(mc), np->mp_pgno); + ASAN_UNPOISON_MEMORY_REGION(np, env->me_psize); *mp = np; return MDB_SUCCESS; } @@ -2361,6 +2368,7 @@ done: np = (MDB_page *)(env->me_map + env->me_psize * pgno); /* LY: reset no-access flag from mdb_kill_page() */ VALGRIND_MAKE_MEM_UNDEFINED(np, env->me_psize * num); + ASAN_UNPOISON_MEMORY_REGION(np, env->me_psize * num); } else { if (unlikely(!(np = mdb_page_malloc(txn, num)))) { rc = ENOMEM; @@ -5110,6 +5118,7 @@ mdbx_env_close_ex(MDB_env *env, int dont_sync) VALGRIND_DESTROY_MEMPOOL(env); while ((dp = env->me_dpages) != NULL) { + ASAN_UNPOISON_MEMORY_REGION(&dp->mp_next, sizeof(dp->mp_next)); VALGRIND_MAKE_MEM_DEFINED(&dp->mp_next, sizeof(dp->mp_next)); env->me_dpages = dp->mp_next; free(dp); From daba18111be8c973639908cb88c50b76f6f87bf6 Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Mon, 30 May 2016 16:27:59 +0300 Subject: [PATCH 6/6] mdbx: cleanup tools from Windows. Change-Id: I8c226a866dbc7081e57af3d6ec687b9cbea2684f --- mdb_copy.c | 8 +------- mdb_dump.c | 8 +------- mdb_load.c | 34 ++++++++++++++-------------------- 3 files changed, 16 insertions(+), 34 deletions(-) diff --git a/mdb_copy.c b/mdb_copy.c index 8de6fb5f..348bea9e 100644 --- a/mdb_copy.c +++ b/mdb_copy.c @@ -11,12 +11,6 @@ * top-level directory of the distribution or, alternatively, at * . */ -#ifdef _WIN32 -#include -#define MDB_STDOUT GetStdHandle(STD_OUTPUT_HANDLE) -#else -#define MDB_STDOUT 1 -#endif #include #include #include @@ -70,7 +64,7 @@ int main(int argc,char * argv[]) if (rc == MDB_SUCCESS) { act = "copying"; if (argc == 2) - rc = mdb_env_copyfd2(env, MDB_STDOUT, cpflags); + rc = mdb_env_copyfd2(env, STDOUT_FILENO, cpflags); else rc = mdb_env_copy2(env, argv[2], cpflags); } diff --git a/mdb_dump.c b/mdb_dump.c index e5e0c12c..aeb57801 100644 --- a/mdb_dump.c +++ b/mdb_dump.c @@ -20,12 +20,6 @@ #include #include "mdbx.h" -#ifdef _WIN32 -#define Z "I" -#else -#define Z "z" -#endif - #define PRINT 1 static int mode; @@ -116,7 +110,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name) if (name) printf("database=%s\n", name); printf("type=btree\n"); - printf("mapsize=%" Z "u\n", info.me_mapsize); + printf("mapsize=%zu\n", info.me_mapsize); if (info.me_mapaddr) printf("mapaddr=%p\n", info.me_mapaddr); printf("maxreaders=%u\n", info.me_maxreaders); diff --git a/mdb_load.c b/mdb_load.c index 3b11e541..a883e011 100644 --- a/mdb_load.c +++ b/mdb_load.c @@ -38,12 +38,6 @@ static MDB_envinfo info; static MDB_val kbuf, dbuf; -#ifdef _WIN32 -#define Z "I" -#else -#define Z "z" -#endif - #define STRLENOF(s) (sizeof(s)-1) typedef struct flagbit { @@ -78,7 +72,7 @@ static void readhdr(void) } else if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) { version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION=")); if (version > 3) { - fprintf(stderr, "%s: line %" Z "d: unsupported VERSION %d\n", + fprintf(stderr, "%s: line %zd: unsupported VERSION %d\n", prog, lineno, version); exit(EXIT_FAILURE); } @@ -88,7 +82,7 @@ static void readhdr(void) if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print"))) mode |= PRINT; else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) { - fprintf(stderr, "%s: line %" Z "d: unsupported FORMAT %s\n", + fprintf(stderr, "%s: line %zd: unsupported FORMAT %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT=")); exit(EXIT_FAILURE); } @@ -99,7 +93,7 @@ static void readhdr(void) subname = strdup((char *)dbuf.mv_data+STRLENOF("database=")); } else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) { if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree"))) { - fprintf(stderr, "%s: line %" Z "d: unsupported type %s\n", + fprintf(stderr, "%s: line %zd: unsupported type %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("type=")); exit(EXIT_FAILURE); } @@ -109,7 +103,7 @@ static void readhdr(void) if (ptr) *ptr = '\0'; i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr); if (i != 1) { - fprintf(stderr, "%s: line %" Z "d: invalid mapaddr %s\n", + fprintf(stderr, "%s: line %zd: invalid mapaddr %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr=")); exit(EXIT_FAILURE); } @@ -117,9 +111,9 @@ static void readhdr(void) int i; ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); if (ptr) *ptr = '\0'; - i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Z "u", &info.me_mapsize); + i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%zu", &info.me_mapsize); if (i != 1) { - fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n", + fprintf(stderr, "%s: line %zd: invalid mapsize %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize=")); exit(EXIT_FAILURE); } @@ -129,7 +123,7 @@ static void readhdr(void) if (ptr) *ptr = '\0'; i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders); if (i != 1) { - fprintf(stderr, "%s: line %" Z "d: invalid maxreaders %s\n", + fprintf(stderr, "%s: line %zd: invalid maxreaders %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders=")); exit(EXIT_FAILURE); } @@ -146,12 +140,12 @@ static void readhdr(void) if (!dbflags[i].bit) { ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size); if (!ptr) { - fprintf(stderr, "%s: line %" Z "d: unexpected format\n", + fprintf(stderr, "%s: line %zd: unexpected format\n", prog, lineno); exit(EXIT_FAILURE); } else { *ptr = '\0'; - fprintf(stderr, "%s: line %" Z "d: unrecognized keyword ignored: %s\n", + fprintf(stderr, "%s: line %zd: unrecognized keyword ignored: %s\n", prog, lineno, (char *)dbuf.mv_data); } } @@ -161,7 +155,7 @@ static void readhdr(void) static void badend(void) { - fprintf(stderr, "%s: line %" Z "d: unexpected end of input\n", + fprintf(stderr, "%s: line %zd: unexpected end of input\n", prog, lineno); } @@ -219,7 +213,7 @@ badend: buf->mv_data = realloc(buf->mv_data, buf->mv_size*2); if (!buf->mv_data) { Eof = 1; - fprintf(stderr, "%s: line %" Z "d: out of memory, line too long\n", + fprintf(stderr, "%s: line %zd: out of memory, line too long\n", prog, lineno); return EOF; } @@ -404,7 +398,7 @@ int main(int argc, char *argv[]) rc = readline(&data, &dbuf); if (rc) { - fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno); + fprintf(stderr, "%s: line %zd: failed to read key value\n", prog, lineno); goto txn_abort; } @@ -419,7 +413,7 @@ int main(int argc, char *argv[]) if (batch == 100) { rc = mdb_txn_commit(txn); if (rc) { - fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", + fprintf(stderr, "%s: line %zd: txn_commit: %s\n", prog, lineno, mdb_strerror(rc)); goto env_close; } @@ -439,7 +433,7 @@ int main(int argc, char *argv[]) rc = mdb_txn_commit(txn); txn = NULL; if (rc) { - fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", + fprintf(stderr, "%s: line %zd: txn_commit: %s\n", prog, lineno, mdb_strerror(rc)); goto env_close; }