Created
March 8, 2019 20:50
-
-
Save adamharder/b278eb372c89f38a598695249a140869 to your computer and use it in GitHub Desktop.
LMDB primer
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import lmdb | |
| from pathlib import Path | |
| LMDB_FILE_NAME = Path("some_lmdb_file.lmdb").absolute() | |
| # max db size 100 GB | |
| with lmdb.open(str(LMDB_FILE_NAME.absolute()), map_size=100*1000*1000*1000, readonly=False) as env: | |
| print(env.stat()) # This forces the database into existence | |
| def get_env(readonly=False): | |
| return lmdb.open(str(LMDB_FILE_NAME.absolute()), map_size=100*1000*1000*1000, readonly=readonly) | |
| def key_exists(key: bytes)->bool: | |
| if type(key) is str: | |
| key = key.encode() | |
| assert type(key) is bytes | |
| with get_env(True) as env: | |
| with env.begin(write=False) as read_transaction: | |
| cur = read_transaction.cursor() | |
| if cur.set_range(key): | |
| return cur.key().startswith(key) | |
| return False | |
| def set_item(key: bytes, val: bytes, over_write: bool=True)->bool: | |
| if type(key) is str: | |
| key = key.encode() | |
| if type(val) is str: | |
| val = val.encode() | |
| assert type(key) is bytes | |
| assert type(val) is bytes | |
| r = True | |
| with get_env(False) as env: | |
| with env.begin(write=True) as txn: | |
| r &= txn.put(key, val, overwrite=over_write) | |
| #txn.commit() | |
| return r | |
| def delete_item(key:bytes)->bool: | |
| if key is None: | |
| return False | |
| if type(key) is str: | |
| key = key.encode() | |
| r=True | |
| with get_env(False) as env: | |
| with env.begin(write=True) as txn: | |
| r &= txn.delete(key) | |
| #txn.commit() | |
| return r | |
| def get_item(key: bytes)->bytes: | |
| if type(key) is str: | |
| key = key.encode() | |
| assert type(key) is bytes | |
| with get_env(True) as env: | |
| with env.begin(write=False) as read_transaction: | |
| cur = read_transaction.cursor() | |
| if cur.set_range(key): | |
| if not cur.key().startswith(key): | |
| return None | |
| val = cur.value() | |
| return val | |
| return None | |
| def get_all_keys_with_prefix(prefix: bytes)->list: | |
| r = [] | |
| if type(prefix) is str: | |
| prefix = prefix.encode() | |
| assert type(prefix) is bytes | |
| with get_env() as env: | |
| with env.begin(write=False) as read_transaction: | |
| cur = read_transaction.cursor() | |
| if cur.set_range(prefix): | |
| if (cur.key().startswith(prefix)): | |
| r.append(cur.key) | |
| while(cur.next()): | |
| if cur.startswith(prefix): | |
| r.append(cur.key) | |
| else: | |
| break | |
| return r |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment