Commit d3759f12 authored by Petr Špaček's avatar Petr Špaček

respdiff: move DB utility functions to dbhelper module

parent dde46712
......@@ -8,3 +8,14 @@ env_open = {
db_open = {
'reverse_key': True
}
def key_stream(lenv, db):
"""
yield all keys from given db
"""
with lenv.begin(db) as txn:
with txn.cursor(db) as cur:
cont = cur.first()
while cont:
yield cur.key()
cont = cur.next()
......@@ -5,7 +5,7 @@ import sys
import lmdb
import lmdbcfg
import dbhelper
import msgdiff
......@@ -24,13 +24,13 @@ def write_blobs(blob_dict, workdir):
outfile.write(v)
config = lmdbcfg.env_open.copy()
config = dbhelper.env_open.copy()
config.update({
'path': sys.argv[1],
'readonly': True
})
lenv = lmdb.Environment(**config)
db = lenv.open_db(key=b'answers', **lmdbcfg.db_open, create=False)
db = lenv.open_db(key=b'answers', **dbhelper.db_open, create=False)
qid = str(int(sys.argv[2])).encode('ascii')
blobs = read_blobs_lmdb(lenv, db, qid)
......
......@@ -7,7 +7,7 @@ import sys
import lmdb
import lmdbcfg
import dbhelper
import makeq
......@@ -59,14 +59,14 @@ def lmdb_init(envdir):
global env
global db
config = lmdbcfg.env_open.copy()
config = dbhelper.env_open.copy()
config.update({
'path': envdir,
'sync': False, # unsafe but fast
'writemap': True # we do not care, this is a new database
})
env = lmdb.Environment(**config)
db = env.open_db(key=b'queries', **lmdbcfg.db_open)
db = env.open_db(key=b'queries', **dbhelper.db_open)
def gen_wrapper_lmdb(args):
qid, qtext = args
......
......@@ -11,16 +11,7 @@ import dns.message
import dns.exception
import lmdb
import lmdbcfg
#m1 = dns.message.from_wire(open(sys.argv[1], 'rb').read())
#print('--- m1 ---')
#print(m1)
#print('--- m1 EOM ---')
#m2 = dns.message.from_wire(open(sys.argv[2], 'rb').read())
#print('--- m2 ---')
#print(m2)
#print('--- m2 EOM ---')
import dbhelper
class DataMismatch(Exception):
......@@ -292,15 +283,15 @@ def worker_init(criteria_arg, target_arg):
global lenv
global answers_db
config = lmdbcfg.env_open.copy()
global diffs_db
config = dbhelper.env_open.copy()
config.update({
'path': sys.argv[1],
'readonly': False,
'create': False
})
lenv = lmdb.Environment(**config)
answers_db = lenv.open_db(key=b'answers', create=True, **lmdbcfg.db_open)
answers_db = lenv.open_db(key=b'answers', create=False, **dbhelper.db_open)
i = 0
#prof = cProfile.Profile()
......@@ -386,18 +377,17 @@ def main():
# print(a)
#answers_stream = itertools.islice(find_querydirs(sys.argv[1]), 300000)
config = lmdbcfg.env_open.copy()
config = dbhelper.env_open.copy()
config.update({
'path': sys.argv[1],
'readonly': True,
'create': False
})
lenv = lmdb.Environment(**config)
db = lenv.open_db(key=b'answers', create=False, **lmdbcfg.db_open)
db = lenv.open_db(key=b'answers', create=False, **dbhelper.db_open)
#qid_stream = itertools.islice(find_answer_qids(lenv, db), 300000)
#qid_stream = itertools.islice(find_answer_qids(lenv, db), 300000)
qid_stream = find_answer_qids(lenv, db)
qid_stream = dbhelper.key_stream(lenv, db)
#qid_stream = itertools.islice(find_answer_qids(lenv, db), 10000)
print('diffs = {')
......
......@@ -6,7 +6,7 @@ import threading
import lmdb
import lmdbcfg
import dbhelper
import sendrecv
timeout = 5
......@@ -31,7 +31,7 @@ def worker_init(envdir, resolvers, init_timeout):
tid = threading.current_thread().ident
selector, sockets = sendrecv.sock_init(resolvers)
config = lmdbcfg.env_open.copy()
config = dbhelper.env_open.copy()
config.update({
'path': envdir,
'writemap': True,
......@@ -40,7 +40,7 @@ def worker_init(envdir, resolvers, init_timeout):
'readonly': False
})
lenv = lmdb.Environment(**config)
adb = lenv.open_db(key=b'answers', create=True, **lmdbcfg.db_open)
adb = lenv.open_db(key=b'answers', create=True, **dbhelper.db_open)
worker_state[tid] = (lenv, adb, selector, sockets)
......@@ -68,13 +68,13 @@ def read_queries_lmdb(lenv, qdb):
# init LMDB
def reader_init(envdir):
config = lmdbcfg.env_open.copy()
config = dbhelper.env_open.copy()
config.update({
'path': envdir,
'readonly': True
})
lenv = lmdb.Environment(**config)
qdb = lenv.open_db(key=b'queries', **lmdbcfg.db_open, create=False)
qdb = lenv.open_db(key=b'queries', **dbhelper.db_open, create=False)
return (lenv, qdb)
def main():
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment