From be5bbea281a89d53dfcbfb18cf7d18d1812d16e6 Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Sat, 12 Aug 2017 11:19:03 +0300 Subject: [PATCH 1/7] mdbx: more update links after move the repo. Change-Id: Ifdcfd23c78e1c843a93e9b41cd024b0ff9f866be --- Makefile | 2 +- README.md | 2 +- TODO.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8de9d316..95be9c0d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# GNU Makefile for libmdbx, https://github.com/ReOpen/libmdbx +# GNU Makefile for libmdbx, https://github.com/leo-yuriev/libmdbx ######################################################################## # Configuration. The compiler options must enable threaded compilation. diff --git a/README.md b/README.md index 8a46250c..379500c2 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ _libmdbx_ является развитием "Lightning Memory-Mapped Database" известной под аббревиатурой [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database). Изначально доработка производилась в составе проекта -[ReOpenLDAP](https://github.com/ReOpen/ReOpenLDAP). Примерно за год +[ReOpenLDAP](https://github.com/leo-yuriev/ReOpenLDAP). Примерно за год работы внесенные изменения приобрели самостоятельную ценность. Осенью 2015 доработанный движок был выделен в отдельный проект, который был [представлен на конференции Highload++ diff --git a/TODO.md b/TODO.md index 9855ba5c..8ff0b3f2 100644 --- a/TODO.md +++ b/TODO.md @@ -42,7 +42,7 @@ - [ ] Валидатор страниц по CRC32, плюс контроль номер транзакии под модулю 2^32. - [ ] Валидатор страниц по t1ha c контролем снимков/версий БД на основе Merkle Tree. - [ ] Возможность хранения ключей внутри data (libfptu). -- [ ] Асинхронная фиксация (https://github.com/ReOpen/libmdbx/issues/5). +- [ ] Асинхронная фиксация (https://github.com/leo-yuriev/libmdbx/issues/5). - [ ] (Пере)Выделять память под IDL-списки с учетом реального кол-ва страниц, т.е. max(MDB_IDL_UM_MAX/MDB_IDL_UM_MAX, npages). ----------------------------------------------------------------------- From 9b5e28e8e69b835f8ba37c0c67c56b99d2a24a01 Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Wed, 16 Aug 2017 08:44:07 +0300 Subject: [PATCH 2/7] mdbx: fix mdbx_setup_dxb() for a case when db-size not (pre)configured by user. Fix https://github.com/leo-yuriev/libmdbx/issues/11 Change-Id: I6fccd4877fed563cfb3a03f887bbba73eff5ac84 --- src/mdbx.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/mdbx.c b/src/mdbx.c index 240c115d..22af1b1d 100644 --- a/src/mdbx.c +++ b/src/mdbx.c @@ -4757,6 +4757,13 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, int lck_rc) { meta.mm_txnid_a, mdbx_durable_str(&meta)); } mdbx_ensure(env, meta.mm_geo.now >= meta.mm_geo.next); + } else { + /* geo-params not pre-configured by used, fetch present from meta. */ + env->me_dbgeo.now = pgno2bytes(env, meta.mm_geo.now); + env->me_dbgeo.lower = pgno2bytes(env, meta.mm_geo.lower); + env->me_dbgeo.upper = pgno2bytes(env, meta.mm_geo.upper); + env->me_dbgeo.grow = pgno2bytes(env, meta.mm_geo.grow); + env->me_dbgeo.shrink = pgno2bytes(env, meta.mm_geo.shrink); } uint64_t filesize_before_mmap; From b0dc98bec0281cc0987253d9e498cdb117a80100 Mon Sep 17 00:00:00 2001 From: dreamsxin Date: Wed, 16 Aug 2017 03:53:06 +0300 Subject: [PATCH 3/7] mdbx: add `tutorial/sample-mdbx.c` and drop obsolete `sample-mdb.txt`. Also this is a testcase for https://github.com/leo-yuriev/libmdbx/issues/11. Change-Id: I95c876b1120c1479b2e740b43f6c60e2ac8b33c1 --- AUTHORS | 1 + Makefile | 5 ++- libmdbx.files | 1 + tutorial/sample-mdb.txt | 66 ------------------------------ tutorial/sample-mdbx.c | 89 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 95 insertions(+), 67 deletions(-) delete mode 100644 tutorial/sample-mdb.txt create mode 100644 tutorial/sample-mdbx.c diff --git a/AUTHORS b/AUTHORS index 0c7ca23f..a2d80146 100644 --- a/AUTHORS +++ b/AUTHORS @@ -6,6 +6,7 @@ Chris Mikkelson Claude Brisson David Barbour David Wilson +dreamsxin Hallvard Furuseth , Heiko Becker Howard Chu , diff --git a/Makefile b/Makefile index 95be9c0d..dccc656a 100644 --- a/Makefile +++ b/Makefile @@ -53,10 +53,13 @@ TEST_OBJ := $(patsubst %.cc,%.o,$(TEST_SRC)) .PHONY: mdbx all install clean check coverage -all: $(LIBRARIES) $(TOOLS) test/test +all: $(LIBRARIES) $(TOOLS) test/test example mdbx: libmdbx.a libmdbx.so +example: mdbx.h tutorial/sample-mdbx.c libmdbx.so + $(CC) $(CFLAGS) -I. tutorial/sample-mdbx.c ./libmdbx.so -o example + tools: $(TOOLS) install: $(LIBRARIES) $(TOOLS) $(HEADERS) diff --git a/libmdbx.files b/libmdbx.files index 153bfe32..a07ea48d 100644 --- a/libmdbx.files +++ b/libmdbx.files @@ -1,3 +1,4 @@ +tutorial/sample-mdbx.c AUTHORS LICENSE Makefile diff --git a/tutorial/sample-mdb.txt b/tutorial/sample-mdb.txt deleted file mode 100644 index 54b56f61..00000000 --- a/tutorial/sample-mdb.txt +++ /dev/null @@ -1,66 +0,0 @@ -/* sample-mdb.txt - MDB toy/sample - * - * Do a line-by-line comparison of this and sample-bdb.txt - */ - -/* - * Copyright 2015-2017 Leonid Yuriev . - * Copyright 2012-2015 Howard Chu, Symas Corp. - * Copyright 2015,2016 Peter-Service R&D LLC. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -#include -#include "mdbx.h" - -int main(int argc,char * argv[]) -{ - int rc; - MDBX_env *env; - MDBX_dbi dbi; - MDBX_val key, data; - MDBX_txn *txn; - MDBX_cursor *cursor; - char sval[32]; - - /* Note: Most error checking omitted for simplicity */ - - rc = mdbx_env_create(&env); - rc = mdbx_env_open(env, "./testdb", 0, 0664); - rc = mdbx_txn_begin(env, NULL, 0, &txn); - rc = mdbx_dbi_open(txn, NULL, 0, &dbi); - - key.iov_len = sizeof(int); - key.iov_base = sval; - data.iov_len = sizeof(sval); - data.iov_base = sval; - - sprintf(sval, "%03x %d foo bar", 32, 3141592); - rc = mdbx_put(txn, dbi, &key, &data, 0); - rc = mdbx_txn_commit(txn); - if (rc) { - fprintf(stderr, "mdbx_txn_commit: (%d) %s\n", rc, mdbx_strerror(rc)); - goto leave; - } - rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn); - rc = mdbx_cursor_open(txn, dbi, &cursor); - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == 0) { - printf("key: %p %.*s, data: %p %.*s\n", - key.iov_base, (int) key.iov_len, (char *) key.iov_base, - data.iov_base, (int) data.iov_len, (char *) data.iov_base); - } - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); -leave: - mdbx_dbi_close(env, dbi); - mdbx_env_close(env); - return 0; -} diff --git a/tutorial/sample-mdbx.c b/tutorial/sample-mdbx.c new file mode 100644 index 00000000..07d5b7dd --- /dev/null +++ b/tutorial/sample-mdbx.c @@ -0,0 +1,89 @@ +/* sample-mdb.txt - MDB toy/sample + * + * Do a line-by-line comparison of this and sample-bdb.txt + */ + +/* + * Copyright 2017 Ilya Shipitsin . + * Copyright 2015-2017 Leonid Yuriev . + * Copyright 2012-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "mdbx.h" +#include + +int main(int argc, char *argv[]) { + (void)argc; + (void)argv; + + int rc; + MDBX_env *env; + MDBX_dbi dbi; + MDBX_val key, data; + MDBX_txn *txn; + MDBX_cursor *cursor; + char sval[32]; + + rc = mdbx_env_create(&env); + if (rc != MDBX_SUCCESS) { + fprintf(stderr, "mdbx_env_create: (%d) %s\n", rc, mdbx_strerror(rc)); + return 0; + } + rc = mdbx_env_open(env, "./example-db", + MDBX_NOSUBDIR | MDBX_COALESCE | MDBX_LIFORECLAIM, 0664); + if (rc != MDBX_SUCCESS) { + mdbx_env_close(env); + fprintf(stderr, "mdbx_env_open: (%d) %s\n", rc, mdbx_strerror(rc)); + return 0; + } + + rc = mdbx_txn_begin(env, NULL, 0, &txn); + if (rc != MDBX_SUCCESS) { + fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc)); + goto leave; + } + rc = mdbx_dbi_open(txn, NULL, 0, &dbi); + if (rc != MDBX_SUCCESS) { + fprintf(stderr, "mdbx_dbi_open: (%d) %s\n", rc, mdbx_strerror(rc)); + goto leave; + } + + key.iov_len = sizeof(int); + key.iov_base = sval; + data.iov_len = sizeof(sval); + data.iov_base = sval; + + sprintf(sval, "%03x %d foo bar", 32, 3141592); + rc = mdbx_put(txn, dbi, &key, &data, 0); + if (rc != MDBX_SUCCESS) { + fprintf(stderr, "mdbx_put: (%d) %s\n", rc, mdbx_strerror(rc)); + goto leave; + } + rc = mdbx_txn_commit(txn); + if (rc) { + fprintf(stderr, "mdbx_txn_commit: (%d) %s\n", rc, mdbx_strerror(rc)); + goto leave; + } + rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn); + rc = mdbx_cursor_open(txn, dbi, &cursor); + while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", key.iov_base, (int)key.iov_len, + (char *)key.iov_base, data.iov_base, (int)data.iov_len, + (char *)data.iov_base); + } + mdbx_cursor_close(cursor); + mdbx_txn_abort(txn); +leave: + mdbx_dbi_close(env, dbi); + mdbx_env_close(env); + return 0; +} From 276e3578da7c2fc49097db3cb5d9c4b2d05e6273 Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Wed, 16 Aug 2017 09:04:33 +0300 Subject: [PATCH 4/7] mdbx: fix `tutorial/sample-mdbx.c` Fix/refine b0dc98bec0281cc0987253d9e498cdb117a80100 Change-Id: Ie5b1cf9453261c5e1cdfc4880ce9dafdcc4545f7 --- Makefile | 2 +- tutorial/sample-mdbx.c | 55 ++++++++++++++++++++++++++++++------------ 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index dccc656a..f39db1b2 100644 --- a/Makefile +++ b/Makefile @@ -135,7 +135,7 @@ endif ci-rule = ( CC=$$(which $1); if [ -n "$$CC" ]; then \ echo -n "probe by $2 ($$(readlink -f $$(which $$CC))): " && \ $(MAKE) clean >$1.log 2>$1.err && \ - $(MAKE) CC=$$(readlink -f $$CC) XCFLAGS="-UNDEBUG -DMDBX_DEBUG=2" check 1>$1.log 2>$1.err && echo "OK" \ + $(MAKE) CC=$$(readlink -f $$CC) XCFLAGS="-UNDEBUG -DMDBX_DEBUG=2 -DLIBMDBX_EXPORTS=1" check 1>$1.log 2>$1.err && echo "OK" \ || ( echo "FAILED"; cat $1.err >&2; exit 1 ); \ else echo "no $2 ($1) for probe"; fi; ) ci: diff --git a/tutorial/sample-mdbx.c b/tutorial/sample-mdbx.c index 07d5b7dd..39adba02 100644 --- a/tutorial/sample-mdbx.c +++ b/tutorial/sample-mdbx.c @@ -20,17 +20,18 @@ #include "mdbx.h" #include +#include int main(int argc, char *argv[]) { (void)argc; (void)argv; int rc; - MDBX_env *env; - MDBX_dbi dbi; + MDBX_env *env = NULL; + MDBX_dbi dbi = 0; MDBX_val key, data; - MDBX_txn *txn; - MDBX_cursor *cursor; + MDBX_txn *txn = NULL; + MDBX_cursor *cursor = NULL; char sval[32]; rc = mdbx_env_create(&env); @@ -41,20 +42,19 @@ int main(int argc, char *argv[]) { rc = mdbx_env_open(env, "./example-db", MDBX_NOSUBDIR | MDBX_COALESCE | MDBX_LIFORECLAIM, 0664); if (rc != MDBX_SUCCESS) { - mdbx_env_close(env); fprintf(stderr, "mdbx_env_open: (%d) %s\n", rc, mdbx_strerror(rc)); - return 0; + goto bailout; } rc = mdbx_txn_begin(env, NULL, 0, &txn); if (rc != MDBX_SUCCESS) { fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc)); - goto leave; + goto bailout; } rc = mdbx_dbi_open(txn, NULL, 0, &dbi); if (rc != MDBX_SUCCESS) { fprintf(stderr, "mdbx_dbi_open: (%d) %s\n", rc, mdbx_strerror(rc)); - goto leave; + goto bailout; } key.iov_len = sizeof(int); @@ -66,24 +66,47 @@ int main(int argc, char *argv[]) { rc = mdbx_put(txn, dbi, &key, &data, 0); if (rc != MDBX_SUCCESS) { fprintf(stderr, "mdbx_put: (%d) %s\n", rc, mdbx_strerror(rc)); - goto leave; + goto bailout; } rc = mdbx_txn_commit(txn); if (rc) { fprintf(stderr, "mdbx_txn_commit: (%d) %s\n", rc, mdbx_strerror(rc)); - goto leave; + goto bailout; } + txn = NULL; + rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn); + if (rc != MDBX_SUCCESS) { + fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } rc = mdbx_cursor_open(txn, dbi, &cursor); + if (rc != MDBX_SUCCESS) { + fprintf(stderr, "mdbx_cursor_open: (%d) %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + int found = 0; while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == 0) { printf("key: %p %.*s, data: %p %.*s\n", key.iov_base, (int)key.iov_len, (char *)key.iov_base, data.iov_base, (int)data.iov_len, (char *)data.iov_base); + found += 1; } - mdbx_cursor_close(cursor); - mdbx_txn_abort(txn); -leave: - mdbx_dbi_close(env, dbi); - mdbx_env_close(env); - return 0; + if (rc != MDBX_NOTFOUND || found == 0) { + fprintf(stderr, "mdbx_cursor_get: (%d) %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } else { + rc = MDBX_SUCCESS; + } +bailout: + if (cursor) + mdbx_cursor_close(cursor); + if (txn) + mdbx_txn_abort(txn); + if (dbi) + mdbx_dbi_close(env, dbi); + if (env) + mdbx_env_close(env); + return (rc != MDBX_SUCCESS) ? EXIT_FAILURE : EXIT_SUCCESS; } From 728252058d365b87e36cc279d5a8753e747db971 Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Wed, 16 Aug 2017 09:25:35 +0300 Subject: [PATCH 5/7] mdbx: fix/refine comment. Change-Id: I8f09ca755ebcc1e58531504a8998391e7a3c8c9a --- src/mdbx.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mdbx.c b/src/mdbx.c index 22af1b1d..618721a2 100644 --- a/src/mdbx.c +++ b/src/mdbx.c @@ -4758,7 +4758,8 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, int lck_rc) { } mdbx_ensure(env, meta.mm_geo.now >= meta.mm_geo.next); } else { - /* geo-params not pre-configured by used, fetch present from meta. */ + /* geo-params not pre-configured by user, + * get current values from a meta. */ env->me_dbgeo.now = pgno2bytes(env, meta.mm_geo.now); env->me_dbgeo.lower = pgno2bytes(env, meta.mm_geo.lower); env->me_dbgeo.upper = pgno2bytes(env, meta.mm_geo.upper); From 876faab1d8443a02a504117fc405688a1c33b02e Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Wed, 16 Aug 2017 11:19:25 +0300 Subject: [PATCH 6/7] mdbx: fix mode_t typedef. Fix https://github.com/leo-yuriev/libmdbx/issues/15 Change-Id: I32bb6b0fbc710c3706a66b398711c29cb6b9a241 --- mdbx.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mdbx.h b/mdbx.h index 276c3f98..416a8571 100644 --- a/mdbx.h +++ b/mdbx.h @@ -1,4 +1,4 @@ -/* +/* * Copyright 2015-2017 Leonid Yuriev * and other libmdbx authors: please see AUTHORS file. * All rights reserved. @@ -70,7 +70,9 @@ #include #include -typedef unsigned mode_t; +#ifndef __mode_t_defined +typedef unsigned short mode_t; +#endif typedef HANDLE mdbx_filehandle_t; typedef DWORD mdbx_pid_t; typedef DWORD mdbx_tid_t; From ace062f4a73fd46b66cca8712ce2e301f59feed0 Mon Sep 17 00:00:00 2001 From: Leo Yuriev Date: Wed, 16 Aug 2017 11:28:38 +0300 Subject: [PATCH 7/7] mdbx: fix warning around `mode_t`. More for https://github.com/leo-yuriev/libmdbx/issues/15 Change-Id: I1743e01e9111400942fd2a4846a21c475101a148 --- src/mdbx.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mdbx.c b/src/mdbx.c index 618721a2..89fe3cee 100644 --- a/src/mdbx.c +++ b/src/mdbx.c @@ -4951,7 +4951,8 @@ static int __cold mdbx_setup_dxb(MDBX_env *env, int lck_rc) { /****************************************************************************/ /* Open and/or initialize the lock region for the environment. */ -static int __cold mdbx_setup_lck(MDBX_env *env, char *lck_pathname, int mode) { +static int __cold mdbx_setup_lck(MDBX_env *env, char *lck_pathname, + mode_t mode) { assert(env->me_fd != INVALID_HANDLE_VALUE); assert(env->me_lfd == INVALID_HANDLE_VALUE);