ISAR/libmdbx
3
0
Fork 0
mirror of https://github.com/isar/libmdbx.git synced 2025-01-04 18:24:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

mdbx: minor spaces/tabs.

Browse Source 
Change-Id: I9f2c42347ad25ac179bc690d403f02721a6c37ca
This commit is contained in:
Leo Yuriev 2016-04-15 22:12:46 +03:00
parent bf7f3814f9
commit 1b780162dc
2 changed files with 344 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

300
yota_test1.c
View File

@ -10,7 +10,7 @@
#include <assert.h> #include <assert.h>
#include "mdbx.h" #include "mdbx.h"
#define IP_PRINTF_ARG_HOST(addr) (int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), (int)((addr) & 0xff) #define IP_PRINTF_ARG_HOST(addr) (int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), (int)((addr) & 0xff)
char opt_db_path[PATH_MAX] = "/dev/shm/lmdb_bench1"; char opt_db_path[PATH_MAX] = "/dev/shm/lmdb_bench1";
static MDB_env *env; static MDB_env *env;
@ -23,74 +23,74 @@ int64_t lmdb_del = 0;
int64_t obj_id = 0; int64_t obj_id = 0;
static void add_id_to_pool(int64_t id) { static void add_id_to_pool(int64_t id) {
ids[ids_count] = id; ids[ids_count] = id;
ids_count++; ids_count++;
} }
static inline int64_t getTimeMicroseconds(void) { static inline int64_t getTimeMicroseconds(void) {
struct timeval val; struct timeval val;
gettimeofday(&val, NULL); gettimeofday(&val, NULL);
return val.tv_sec * ((int64_t) 1000000) + val.tv_usec; return val.tv_sec * ((int64_t) 1000000) + val.tv_usec;
} }
static int64_t get_id_from_pool() { static int64_t get_id_from_pool() {
if (ids_count == 0) { if (ids_count == 0) {
return -1; return -1;
} }
int32_t index = rand() % ids_count; int32_t index = rand() % ids_count;
int64_t id = ids[index]; int64_t id = ids[index];
ids[index] = ids[ids_count - 1]; ids[index] = ids[ids_count - 1];
ids_count--; ids_count--;
return id; return id;
} }
#define LMDB_CHECK(x) \ #define LMDB_CHECK(x) \
do {\ do {\
const int rc = (x);\ const int rc = (x);\
if ( rc != MDB_SUCCESS ) {\ if ( rc != MDB_SUCCESS ) {\
printf("Error [%d] %s in %s at %s:%d\n", rc, mdb_strerror(rc), #x, __FILE__, __LINE__); \ printf("Error [%d] %s in %s at %s:%d\n", rc, mdb_strerror(rc), #x, __FILE__, __LINE__); \
exit(EXIT_FAILURE); \ exit(EXIT_FAILURE); \
}\ }\
} while(0) } while(0)
static void db_connect() { static void db_connect() {
LMDB_CHECK(mdb_env_create(&env)); LMDB_CHECK(mdb_env_create(&env));
LMDB_CHECK(mdb_env_set_mapsize(env, 3L * 1024L * 1024L * 1024L)); LMDB_CHECK(mdb_env_set_mapsize(env, 3L * 1024L * 1024L * 1024L));
LMDB_CHECK(mdb_env_set_maxdbs(env, 30)); LMDB_CHECK(mdb_env_set_maxdbs(env, 30));
#if defined(MDB_LIFORECLAIM) #if defined(MDB_LIFORECLAIM)
LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP | MDB_LIFORECLAIM, 0664)); LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP | MDB_LIFORECLAIM, 0664));
#else #else
LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP, 0664)); LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP, 0664));
#endif #endif
printf("Connection open\n"); printf("Connection open\n");
} }
typedef struct { typedef struct {
char session_id1[100]; char session_id1[100];
char session_id2[100]; char session_id2[100];
char ip[20]; char ip[20];
uint8_t fill[100]; uint8_t fill[100];
} session_data_t; } session_data_t;
typedef struct { typedef struct {
int64_t obj_id; int64_t obj_id;
int8_t event_type; int8_t event_type;
} __attribute__((__packed__)) event_data_t; } __attribute__((__packed__)) event_data_t;
static void create_record(int64_t record_id) { static void create_record(int64_t record_id) {
MDB_dbi dbi_session; MDB_dbi dbi_session;
MDB_dbi dbi_session_id; MDB_dbi dbi_session_id;
MDB_dbi dbi_event; MDB_dbi dbi_event;
MDB_dbi dbi_ip; MDB_dbi dbi_ip;
event_data_t event; event_data_t event;
MDB_txn *txn; MDB_txn *txn;
session_data_t data; session_data_t data;
// transaction init // transaction init
snprintf(data.session_id1, sizeof (data.session_id1), "mskugw%02ld_%02ld.gx.yota.ru;3800464060;4152;%ld", record_id % 3 + 1, record_id % 9 + 1, record_id); snprintf(data.session_id1, sizeof (data.session_id1), "mskugw%02ld_%02ld.gx.yota.ru;3800464060;4152;%ld", record_id % 3 + 1, record_id % 9 + 1, record_id);
snprintf(data.session_id2, sizeof (data.session_id2), "gx_service;%ld;%ld;node@spb-jsm1", record_id, record_id % 1000000000 + 99999); snprintf(data.session_id2, sizeof (data.session_id2), "gx_service;%ld;%ld;node@spb-jsm1", record_id, record_id % 1000000000 + 99999);
snprintf(data.ip, sizeof (data.ip), "%d.%d.%d.%d", IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF)); snprintf(data.ip, sizeof (data.ip), "%d.%d.%d.%d", IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF));
event.obj_id = record_id; event.obj_id = record_id;
event.event_type = 1; event.event_type = 1;
MDB_val _session_id1_rec = {data.session_id1, strlen(data.session_id1)}; MDB_val _session_id1_rec = {data.session_id1, strlen(data.session_id1)};
MDB_val _session_id2_rec = {data.session_id2, strlen(data.session_id2)}; MDB_val _session_id2_rec = {data.session_id2, strlen(data.session_id2)};
@ -99,139 +99,139 @@ static void create_record(int64_t record_id) {
MDB_val _data_rec = {&data, offsetof(session_data_t, fill) + (rand() % sizeof (data.fill))}; MDB_val _data_rec = {&data, offsetof(session_data_t, fill) + (rand() % sizeof (data.fill))};
MDB_val _event_rec = {&event, sizeof (event)}; MDB_val _event_rec = {&event, sizeof (event)};
LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn)); LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn));
LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session)); LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session));
LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id)); LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id));
LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event)); LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event));
LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip)); LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip));
LMDB_CHECK(mdb_put(txn, dbi_session, &_obj_id_rec, &_data_rec, MDB_NOOVERWRITE | MDB_NODUPDATA)); LMDB_CHECK(mdb_put(txn, dbi_session, &_obj_id_rec, &_data_rec, MDB_NOOVERWRITE | MDB_NODUPDATA));
LMDB_CHECK(mdb_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec, MDB_NOOVERWRITE | MDB_NODUPDATA)); LMDB_CHECK(mdb_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec, MDB_NOOVERWRITE | MDB_NODUPDATA));
LMDB_CHECK(mdb_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec, MDB_NOOVERWRITE | MDB_NODUPDATA)); LMDB_CHECK(mdb_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec, MDB_NOOVERWRITE | MDB_NODUPDATA));
LMDB_CHECK(mdb_put(txn, dbi_ip, &_ip_rec, &_obj_id_rec, 0)); LMDB_CHECK(mdb_put(txn, dbi_ip, &_ip_rec, &_obj_id_rec, 0));
LMDB_CHECK(mdb_put(txn, dbi_event, &_event_rec, &_obj_id_rec, 0)); LMDB_CHECK(mdb_put(txn, dbi_event, &_event_rec, &_obj_id_rec, 0));
// transaction commit // transaction commit
LMDB_CHECK(mdb_txn_commit(txn)); LMDB_CHECK(mdb_txn_commit(txn));
lmdb_add++; lmdb_add++;
} }
static void delete_record(int64_t record_id) { static void delete_record(int64_t record_id) {
MDB_dbi dbi_session; MDB_dbi dbi_session;
MDB_dbi dbi_session_id; MDB_dbi dbi_session_id;
MDB_dbi dbi_event; MDB_dbi dbi_event;
MDB_dbi dbi_ip; MDB_dbi dbi_ip;
event_data_t event; event_data_t event;
MDB_txn *txn; MDB_txn *txn;
// transaction init // transaction init
LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn)); LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn));
// open database in read-write mode // open database in read-write mode
LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session)); LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session));
LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id)); LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id));
LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event)); LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event));
LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip)); LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip));
// put data // put data
MDB_val _obj_id_rec = {&record_id, sizeof(record_id)}; MDB_val _obj_id_rec = {&record_id, sizeof(record_id)};
MDB_val v_rec; MDB_val v_rec;
// get data // get data
LMDB_CHECK(mdb_get(txn, dbi_session, &_obj_id_rec, &v_rec)); LMDB_CHECK(mdb_get(txn, dbi_session, &_obj_id_rec, &v_rec));
session_data_t* data = (session_data_t*) v_rec.mv_data; session_data_t* data = (session_data_t*) v_rec.mv_data;
MDB_val _session_id1_rec = {data->session_id1, strlen(data->session_id1)}; MDB_val _session_id1_rec = {data->session_id1, strlen(data->session_id1)};
MDB_val _session_id2_rec = {data->session_id2, strlen(data->session_id2)}; MDB_val _session_id2_rec = {data->session_id2, strlen(data->session_id2)};
MDB_val _ip_rec = {data->ip, strlen(data->ip)}; MDB_val _ip_rec = {data->ip, strlen(data->ip)};
LMDB_CHECK(mdb_del(txn, dbi_session_id, &_session_id1_rec, NULL)); LMDB_CHECK(mdb_del(txn, dbi_session_id, &_session_id1_rec, NULL));
LMDB_CHECK(mdb_del(txn, dbi_session_id, &_session_id2_rec, NULL)); LMDB_CHECK(mdb_del(txn, dbi_session_id, &_session_id2_rec, NULL));
LMDB_CHECK(mdb_del(txn, dbi_ip, &_ip_rec, NULL)); LMDB_CHECK(mdb_del(txn, dbi_ip, &_ip_rec, NULL));
event.obj_id = record_id; event.obj_id = record_id;
event.event_type = 1; event.event_type = 1;
MDB_val _event_rec = {&event, sizeof(event)}; MDB_val _event_rec = {&event, sizeof(event)};
LMDB_CHECK(mdb_del(txn, dbi_event, &_event_rec, NULL)); LMDB_CHECK(mdb_del(txn, dbi_event, &_event_rec, NULL));
LMDB_CHECK(mdb_del(txn, dbi_session, &_obj_id_rec, NULL)); LMDB_CHECK(mdb_del(txn, dbi_session, &_obj_id_rec, NULL));
// transaction commit // transaction commit
LMDB_CHECK(mdb_txn_commit(txn)); LMDB_CHECK(mdb_txn_commit(txn));
lmdb_del++; lmdb_del++;
} }
static void db_disconnect() { static void db_disconnect() {
mdb_env_close(env); mdb_env_close(env);
printf("Connection closed\n"); printf("Connection closed\n");
} }
static void get_db_stat(const char* db, int64_t* ms_branch_pages, int64_t* ms_leaf_pages) { static void get_db_stat(const char* db, int64_t* ms_branch_pages, int64_t* ms_leaf_pages) {
MDB_txn *txn; MDB_txn *txn;
MDB_stat stat; MDB_stat stat;
MDB_dbi dbi; MDB_dbi dbi;
LMDB_CHECK(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); LMDB_CHECK(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
LMDB_CHECK(mdb_dbi_open(txn, db, MDB_CREATE, &dbi)); LMDB_CHECK(mdb_dbi_open(txn, db, MDB_CREATE, &dbi));
LMDB_CHECK(mdb_stat(txn, dbi, &stat)); LMDB_CHECK(mdb_stat(txn, dbi, &stat));
mdb_txn_abort(txn); mdb_txn_abort(txn);
printf("%15s | %15ld | %5u | %10ld | %10ld | %11ld |\n", printf("%15s | %15ld | %5u | %10ld | %10ld | %11ld |\n",
db, db,
stat.ms_branch_pages, stat.ms_branch_pages,
stat.ms_depth, stat.ms_depth,
stat.ms_entries, stat.ms_entries,
stat.ms_leaf_pages, stat.ms_leaf_pages,
stat.ms_overflow_pages); stat.ms_overflow_pages);
(*ms_branch_pages) += stat.ms_branch_pages; (*ms_branch_pages) += stat.ms_branch_pages;
(*ms_leaf_pages) += stat.ms_leaf_pages; (*ms_leaf_pages) += stat.ms_leaf_pages;
} }
static void periodic_stat(void) { static void periodic_stat(void) {
int64_t ms_branch_pages = 0; int64_t ms_branch_pages = 0;
int64_t ms_leaf_pages = 0; int64_t ms_leaf_pages = 0;
printf(" Name | ms_branch_pages | depth | entries | leaf_pages | overf_pages |\n"); printf(" Name | ms_branch_pages | depth | entries | leaf_pages | overf_pages |\n");
get_db_stat("session", &ms_branch_pages, &ms_leaf_pages); get_db_stat("session", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages); get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("event", &ms_branch_pages, &ms_leaf_pages); get_db_stat("event", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages); get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages);
printf("%15s | %15ld | %5s | %10s | %10ld | %11s |\n", "", ms_branch_pages, "", "", ms_leaf_pages, ""); printf("%15s | %15ld | %5s | %10s | %10ld | %11s |\n", "", ms_branch_pages, "", "", ms_leaf_pages, "");
static int64_t prev_add; static int64_t prev_add;
static int64_t prev_del; static int64_t prev_del;
static int64_t t = -1; static int64_t t = -1;
if (t > 0) { if (t > 0) {
int64_t delta = getTimeMicroseconds() - t; int64_t delta = getTimeMicroseconds() - t;
printf("CPS: add %ld, delete %ld, items processed - %ld\n", (lmdb_add - prev_add)*1000000 / delta, (lmdb_del - prev_del)*1000000 / delta, obj_id); printf("CPS: add %ld, delete %ld, items processed - %ld\n", (lmdb_add - prev_add)*1000000 / delta, (lmdb_del - prev_del)*1000000 / delta, obj_id);
} }
t = getTimeMicroseconds(); t = getTimeMicroseconds();
prev_add = lmdb_add; prev_add = lmdb_add;
prev_del = lmdb_del; prev_del = lmdb_del;
} }
static void periodic_add_rec() { static void periodic_add_rec() {
int i; int i;
for (i = 0; i < 10000; i++) { for (i = 0; i < 10000; i++) {
if (ids_count <= REC_COUNT) { if (ids_count <= REC_COUNT) {
int64_t id = obj_id++; int64_t id = obj_id++;
create_record(id); create_record(id);
add_id_to_pool(id); add_id_to_pool(id);
} }
if (ids_count > REC_COUNT) { if (ids_count > REC_COUNT) {
int64_t id = get_id_from_pool(); int64_t id = get_id_from_pool();
delete_record(id); delete_record(id);
} }
} }
periodic_stat(); periodic_stat();
} }
int main(int argc, char** argv) { int main(int argc, char** argv) {
char filename[PATH_MAX]; char filename[PATH_MAX];
mkdir(opt_db_path, 0775); mkdir(opt_db_path, 0775);
strcpy(filename, opt_db_path); strcpy(filename, opt_db_path);
strcat(filename, "/data.mdb"); strcat(filename, "/data.mdb");
remove(filename); remove(filename);
strcpy(filename, opt_db_path); strcpy(filename, opt_db_path);
strcat(filename, "/lock.mdb"); strcat(filename, "/lock.mdb");
remove(filename); remove(filename);
db_connect(); db_connect();
while (1) { while (1) {
periodic_add_rec(); periodic_add_rec();
} }
db_disconnect(); db_disconnect();
return 0; return 0;
} }

388
yota_test2.c
View File

@ -10,7 +10,7 @@
#include <assert.h> #include <assert.h>
#include "mdbx.h" #include "mdbx.h"
#define IP_PRINTF_ARG_HOST(addr) (int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), (int)((addr) & 0xff) #define IP_PRINTF_ARG_HOST(addr) (int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), (int)((addr) & 0xff)
char opt_db_path[PATH_MAX] = "/dev/shm/lmdb_bench2"; char opt_db_path[PATH_MAX] = "/dev/shm/lmdb_bench2";
static MDB_env *env; static MDB_env *env;
@ -25,90 +25,90 @@ int64_t lmdb_data_size = 0;
int64_t lmdb_key_size = 0; int64_t lmdb_key_size = 0;
static void add_id_to_pool(int64_t id) { static void add_id_to_pool(int64_t id) {
ids[ids_count] = id; ids[ids_count] = id;
ids_count++; ids_count++;
} }
static inline int64_t getTimeMicroseconds(void) { static inline int64_t getTimeMicroseconds(void) {
struct timeval val; struct timeval val;
gettimeofday(&val, NULL); gettimeofday(&val, NULL);
return val.tv_sec * ((int64_t) 1000000) + val.tv_usec; return val.tv_sec * ((int64_t) 1000000) + val.tv_usec;
} }
static int64_t get_id_from_pool() { static int64_t get_id_from_pool() {
if (ids_count == 0) { if (ids_count == 0) {
return -1; return -1;
} }
int32_t index = rand() % ids_count; int32_t index = rand() % ids_count;
int64_t id = ids[index]; int64_t id = ids[index];
ids[index] = ids[ids_count - 1]; ids[index] = ids[ids_count - 1];
ids_count--; ids_count--;
return id; return id;
} }
#define LMDB_CHECK(x) \ #define LMDB_CHECK(x) \
do {\ do {\
const int rc = (x);\ const int rc = (x);\
if ( rc != MDB_SUCCESS ) {\ if ( rc != MDB_SUCCESS ) {\
printf("Error [%d] %s in %s at %s:%d\n", rc, mdb_strerror(rc), #x, __FILE__, __LINE__); \ printf("Error [%d] %s in %s at %s:%d\n", rc, mdb_strerror(rc), #x, __FILE__, __LINE__); \
exit(EXIT_FAILURE); \ exit(EXIT_FAILURE); \
}\ }\
} while(0) } while(0)
static void db_connect() { static void db_connect() {
MDB_dbi dbi_session; MDB_dbi dbi_session;
MDB_dbi dbi_session_id; MDB_dbi dbi_session_id;
MDB_dbi dbi_event; MDB_dbi dbi_event;
MDB_dbi dbi_ip; MDB_dbi dbi_ip;
LMDB_CHECK(mdb_env_create(&env)); LMDB_CHECK(mdb_env_create(&env));
LMDB_CHECK(mdb_env_set_mapsize(env, 300000L * 4096L)); LMDB_CHECK(mdb_env_set_mapsize(env, 300000L * 4096L));
LMDB_CHECK(mdb_env_set_maxdbs(env, 30)); LMDB_CHECK(mdb_env_set_maxdbs(env, 30));
#if defined(MDB_LIFORECLAIM) #if defined(MDB_LIFORECLAIM)
LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP | MDB_LIFORECLAIM, 0664)); LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP | MDB_LIFORECLAIM, 0664));
#else #else
LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP, 0664)); LMDB_CHECK(mdb_env_open(env, opt_db_path, MDB_CREATE | MDB_NOSYNC | MDB_WRITEMAP, 0664));
#endif #endif
MDB_txn *txn; MDB_txn *txn;
// transaction init // transaction init
LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn)); LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn));
// open database in read-write mode // open database in read-write mode
LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session)); LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session));
LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id)); LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id));
LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event)); LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event));
LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip)); LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip));
// transaction commit // transaction commit
LMDB_CHECK(mdb_txn_commit(txn)); LMDB_CHECK(mdb_txn_commit(txn));
printf("Connection open\n"); printf("Connection open\n");
} }
typedef struct { typedef struct {
char session_id1[100]; char session_id1[100];
char session_id2[100]; char session_id2[100];
char ip[20]; char ip[20];
uint8_t fill[100]; uint8_t fill[100];
} session_data_t; } session_data_t;
typedef struct { typedef struct {
int64_t obj_id; int64_t obj_id;
int8_t event_type; int8_t event_type;
} __attribute__((__packed__)) event_data_t; } __attribute__((__packed__)) event_data_t;
static void create_record(int64_t record_id) { static void create_record(int64_t record_id) {
MDB_dbi dbi_session; MDB_dbi dbi_session;
MDB_dbi dbi_session_id; MDB_dbi dbi_session_id;
MDB_dbi dbi_event; MDB_dbi dbi_event;
MDB_dbi dbi_ip; MDB_dbi dbi_ip;
event_data_t event; event_data_t event;
MDB_txn *txn; MDB_txn *txn;
session_data_t data; session_data_t data;
// transaction init // transaction init
snprintf(data.session_id1, sizeof (data.session_id1), "mskugw%02ld_%02ld.gx.yota.ru;3800464060;4152;%ld", record_id % 3 + 1, record_id % 9 + 1, record_id); snprintf(data.session_id1, sizeof (data.session_id1), "mskugw%02ld_%02ld.gx.yota.ru;3800464060;4152;%ld", record_id % 3 + 1, record_id % 9 + 1, record_id);
snprintf(data.session_id2, sizeof (data.session_id2), "gx_service;%ld;%ld;node@spb-jsm1", record_id, record_id % 1000000000 + 99999); snprintf(data.session_id2, sizeof (data.session_id2), "gx_service;%ld;%ld;node@spb-jsm1", record_id, record_id % 1000000000 + 99999);
snprintf(data.ip, sizeof (data.ip), "%d.%d.%d.%d", IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF)); snprintf(data.ip, sizeof (data.ip), "%d.%d.%d.%d", IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF));
event.obj_id = record_id; event.obj_id = record_id;
event.event_type = 1; event.event_type = 1;
MDB_val _session_id1_rec = {data.session_id1, strlen(data.session_id1)}; MDB_val _session_id1_rec = {data.session_id1, strlen(data.session_id1)};
MDB_val _session_id2_rec = {data.session_id2, strlen(data.session_id2)}; MDB_val _session_id2_rec = {data.session_id2, strlen(data.session_id2)};
@ -117,171 +117,171 @@ static void create_record(int64_t record_id) {
MDB_val _data_rec = {&data, offsetof(session_data_t, fill) + (rand() % sizeof (data.fill))}; MDB_val _data_rec = {&data, offsetof(session_data_t, fill) + (rand() % sizeof (data.fill))};
MDB_val _event_rec = {&event, sizeof(event)}; MDB_val _event_rec = {&event, sizeof(event)};
LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn)); LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn));
LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session)); LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session));
LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id)); LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id));
LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event)); LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event));
LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip)); LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip));
LMDB_CHECK(mdb_put(txn, dbi_session, &_obj_id_rec, &_data_rec, MDB_NOOVERWRITE | MDB_NODUPDATA)); LMDB_CHECK(mdb_put(txn, dbi_session, &_obj_id_rec, &_data_rec, MDB_NOOVERWRITE | MDB_NODUPDATA));
LMDB_CHECK(mdb_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec, MDB_NOOVERWRITE | MDB_NODUPDATA)); LMDB_CHECK(mdb_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec, MDB_NOOVERWRITE | MDB_NODUPDATA));
LMDB_CHECK(mdb_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec, MDB_NOOVERWRITE | MDB_NODUPDATA)); LMDB_CHECK(mdb_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec, MDB_NOOVERWRITE | MDB_NODUPDATA));
LMDB_CHECK(mdb_put(txn, dbi_ip, &_ip_rec, &_obj_id_rec, 0)); LMDB_CHECK(mdb_put(txn, dbi_ip, &_ip_rec, &_obj_id_rec, 0));
LMDB_CHECK(mdb_put(txn, dbi_event, &_event_rec, &_obj_id_rec, 0)); LMDB_CHECK(mdb_put(txn, dbi_event, &_event_rec, &_obj_id_rec, 0));
lmdb_data_size += (_data_rec.mv_size + _obj_id_rec.mv_size * 4); lmdb_data_size += (_data_rec.mv_size + _obj_id_rec.mv_size * 4);
lmdb_key_size += (_obj_id_rec.mv_size + _session_id1_rec.mv_size + _session_id2_rec.mv_size + _ip_rec.mv_size + _event_rec.mv_size); lmdb_key_size += (_obj_id_rec.mv_size + _session_id1_rec.mv_size + _session_id2_rec.mv_size + _ip_rec.mv_size + _event_rec.mv_size);
// transaction commit // transaction commit
LMDB_CHECK(mdb_txn_commit(txn)); LMDB_CHECK(mdb_txn_commit(txn));
lmdb_add++; lmdb_add++;
} }
static void delete_record(int64_t record_id) { static void delete_record(int64_t record_id) {
MDB_dbi dbi_session; MDB_dbi dbi_session;
MDB_dbi dbi_session_id; MDB_dbi dbi_session_id;
MDB_dbi dbi_event; MDB_dbi dbi_event;
MDB_dbi dbi_ip; MDB_dbi dbi_ip;
event_data_t event; event_data_t event;
MDB_txn *txn; MDB_txn *txn;
// transaction init // transaction init
LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn)); LMDB_CHECK(mdb_txn_begin(env, NULL, 0, &txn));
// open database in read-write mode // open database in read-write mode
LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session)); LMDB_CHECK(mdb_dbi_open(txn, "session", MDB_CREATE, &dbi_session));
LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id)); LMDB_CHECK(mdb_dbi_open(txn, "session_id", MDB_CREATE, &dbi_session_id));
LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event)); LMDB_CHECK(mdb_dbi_open(txn, "event", MDB_CREATE, &dbi_event));
LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip)); LMDB_CHECK(mdb_dbi_open(txn, "ip", MDB_CREATE, &dbi_ip));
// put data // put data
MDB_val _obj_id_rec = {&record_id, sizeof(record_id)}; MDB_val _obj_id_rec = {&record_id, sizeof(record_id)};
MDB_val _data_rec; MDB_val _data_rec;
// get data // get data
LMDB_CHECK(mdb_get(txn, dbi_session, &_obj_id_rec, &_data_rec)); LMDB_CHECK(mdb_get(txn, dbi_session, &_obj_id_rec, &_data_rec));
session_data_t* data = (session_data_t*) _data_rec.mv_data; session_data_t* data = (session_data_t*) _data_rec.mv_data;
MDB_val _session_id1_rec = {data->session_id1, strlen(data->session_id1)}; MDB_val _session_id1_rec = {data->session_id1, strlen(data->session_id1)};
MDB_val _session_id2_rec = {data->session_id2, strlen(data->session_id2)}; MDB_val _session_id2_rec = {data->session_id2, strlen(data->session_id2)};
MDB_val _ip_rec = {data->ip, strlen(data->ip)}; MDB_val _ip_rec = {data->ip, strlen(data->ip)};
LMDB_CHECK(mdb_del(txn, dbi_session_id, &_session_id1_rec, NULL)); LMDB_CHECK(mdb_del(txn, dbi_session_id, &_session_id1_rec, NULL));
LMDB_CHECK(mdb_del(txn, dbi_session_id, &_session_id2_rec, NULL)); LMDB_CHECK(mdb_del(txn, dbi_session_id, &_session_id2_rec, NULL));
LMDB_CHECK(mdb_del(txn, dbi_ip, &_ip_rec, NULL)); LMDB_CHECK(mdb_del(txn, dbi_ip, &_ip_rec, NULL));
event.obj_id = record_id; event.obj_id = record_id;
event.event_type = 1; event.event_type = 1;
MDB_val _event_rec = {&event, sizeof(event)}; MDB_val _event_rec = {&event, sizeof(event)};
LMDB_CHECK(mdb_del(txn, dbi_event, &_event_rec, NULL)); LMDB_CHECK(mdb_del(txn, dbi_event, &_event_rec, NULL));
LMDB_CHECK(mdb_del(txn, dbi_session, &_obj_id_rec, NULL)); LMDB_CHECK(mdb_del(txn, dbi_session, &_obj_id_rec, NULL));
lmdb_data_size -= (_data_rec.mv_size + _obj_id_rec.mv_size * 4); lmdb_data_size -= (_data_rec.mv_size + _obj_id_rec.mv_size * 4);
lmdb_key_size -= (_obj_id_rec.mv_size + _session_id1_rec.mv_size + _session_id2_rec.mv_size + _ip_rec.mv_size + _event_rec.mv_size); lmdb_key_size -= (_obj_id_rec.mv_size + _session_id1_rec.mv_size + _session_id2_rec.mv_size + _ip_rec.mv_size + _event_rec.mv_size);
// transaction commit // transaction commit
LMDB_CHECK(mdb_txn_commit(txn)); LMDB_CHECK(mdb_txn_commit(txn));
lmdb_del++; lmdb_del++;
} }
static void db_disconnect() { static void db_disconnect() {
mdb_env_close(env); mdb_env_close(env);
printf("Connection closed\n"); printf("Connection closed\n");
} }
static void get_db_stat(const char* db, int64_t* ms_branch_pages, int64_t* ms_leaf_pages) { static void get_db_stat(const char* db, int64_t* ms_branch_pages, int64_t* ms_leaf_pages) {
MDB_txn *txn; MDB_txn *txn;
MDB_stat stat; MDB_stat stat;
MDB_dbi dbi; MDB_dbi dbi;
LMDB_CHECK(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); LMDB_CHECK(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
LMDB_CHECK(mdb_dbi_open(txn, db, MDB_CREATE, &dbi)); LMDB_CHECK(mdb_dbi_open(txn, db, MDB_CREATE, &dbi));
LMDB_CHECK(mdb_stat(txn, dbi, &stat)); LMDB_CHECK(mdb_stat(txn, dbi, &stat));
mdb_txn_abort(txn); mdb_txn_abort(txn);
printf("%15s | %15ld | %5u | %10ld | %10ld | %11ld |\n", printf("%15s | %15ld | %5u | %10ld | %10ld | %11ld |\n",
db, db,
stat.ms_branch_pages, stat.ms_branch_pages,
stat.ms_depth, stat.ms_depth,
stat.ms_entries, stat.ms_entries,
stat.ms_leaf_pages, stat.ms_leaf_pages,
stat.ms_overflow_pages); stat.ms_overflow_pages);
(*ms_branch_pages) += stat.ms_branch_pages; (*ms_branch_pages) += stat.ms_branch_pages;
(*ms_leaf_pages) += stat.ms_leaf_pages; (*ms_leaf_pages) += stat.ms_leaf_pages;
} }
static void periodic_stat(void) { static void periodic_stat(void) {
int64_t ms_branch_pages = 0; int64_t ms_branch_pages = 0;
int64_t ms_leaf_pages = 0; int64_t ms_leaf_pages = 0;
printf(" Name | ms_branch_pages | depth | entries | leaf_pages | overf_pages |\n"); printf(" Name | ms_branch_pages | depth | entries | leaf_pages | overf_pages |\n");
get_db_stat("session", &ms_branch_pages, &ms_leaf_pages); get_db_stat("session", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages); get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("event", &ms_branch_pages, &ms_leaf_pages); get_db_stat("event", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages); get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages);
printf("%15s | %15ld | %5s | %10s | %10ld | %11s |\n", "", ms_branch_pages, "", "", ms_leaf_pages, ""); printf("%15s | %15ld | %5s | %10s | %10ld | %11s |\n", "", ms_branch_pages, "", "", ms_leaf_pages, "");
static int64_t prev_add; static int64_t prev_add;
static int64_t prev_del; static int64_t prev_del;
static int64_t t = -1; static int64_t t = -1;
if (t > 0) { if (t > 0) {
int64_t delta = getTimeMicroseconds() - t; int64_t delta = getTimeMicroseconds() - t;
printf("CPS: add %ld, delete %ld, items processed - %ldK data=%ldK key=%ldK\n", (lmdb_add - prev_add)*1000000 / delta, (lmdb_del - prev_del)*1000000 / delta, obj_id / 1024, lmdb_data_size / 1024, lmdb_key_size / 1024); printf("CPS: add %ld, delete %ld, items processed - %ldK data=%ldK key=%ldK\n", (lmdb_add - prev_add)*1000000 / delta, (lmdb_del - prev_del)*1000000 / delta, obj_id / 1024, lmdb_data_size / 1024, lmdb_key_size / 1024);
printf("usage data=%ld%%\n", ((lmdb_data_size + lmdb_key_size) * 100) / ((ms_leaf_pages + ms_branch_pages)*4096)); printf("usage data=%ld%%\n", ((lmdb_data_size + lmdb_key_size) * 100) / ((ms_leaf_pages + ms_branch_pages)*4096));
} }
t = getTimeMicroseconds(); t = getTimeMicroseconds();
prev_add = lmdb_add; prev_add = lmdb_add;
prev_del = lmdb_del; prev_del = lmdb_del;
} }
//static void periodic_add_rec() { //static void periodic_add_rec() {
// for (int i = 0; i < 10240; i++) { // for (int i = 0; i < 10240; i++) {
// if (ids_count <= REC_COUNT) { // if (ids_count <= REC_COUNT) {
// int64_t id = obj_id++; // int64_t id = obj_id++;
// create_record(id); // create_record(id);
// add_id_to_pool(id); // add_id_to_pool(id);
// } // }
// if (ids_count > REC_COUNT) { // if (ids_count > REC_COUNT) {
// int64_t id = get_id_from_pool(); // int64_t id = get_id_from_pool();
// delete_record(id); // delete_record(id);
// } // }
// } // }
// periodic_stat(); // periodic_stat();
//} //}
int main(int argc, char** argv) { int main(int argc, char** argv) {
char filename[PATH_MAX]; char filename[PATH_MAX];
int i; int i;
int64_t t; int64_t t;
mkdir(opt_db_path, 0775); mkdir(opt_db_path, 0775);
strcpy(filename, opt_db_path); strcpy(filename, opt_db_path);
strcat(filename, "/data.mdb"); strcat(filename, "/data.mdb");
remove(filename); remove(filename);
strcpy(filename, opt_db_path); strcpy(filename, opt_db_path);
strcat(filename, "/lock.mdb"); strcat(filename, "/lock.mdb");
remove(filename); remove(filename);
db_connect(); db_connect();
periodic_stat(); periodic_stat();
for (i = 0; i < 1024000; i++) { for (i = 0; i < 1024000; i++) {
int64_t id = obj_id++; int64_t id = obj_id++;
create_record(id); create_record(id);
add_id_to_pool(id); add_id_to_pool(id);
} }
periodic_stat(); periodic_stat();
t = getTimeMicroseconds(); t = getTimeMicroseconds();
while (1) { while (1) {
int i; int i;
int64_t now; int64_t now;
for (i = 0; i < 100; i++) { for (i = 0; i < 100; i++) {
int64_t id = obj_id++; int64_t id = obj_id++;
create_record(id); create_record(id);
add_id_to_pool(id); add_id_to_pool(id);
id = get_id_from_pool(); id = get_id_from_pool();
delete_record(id); delete_record(id);
} }
//int64_t id = obj_id++; //int64_t id = obj_id++;
//create_record(id); //create_record(id);
//add_id_to_pool(id); //add_id_to_pool(id);
now = getTimeMicroseconds(); now = getTimeMicroseconds();
if ((now - t) > 100000) { if ((now - t) > 100000) {
periodic_stat(); periodic_stat();
t = now; t = now;
} }
} }
db_disconnect(); db_disconnect();
return 0; return 0;
} }