json-database 0.8.0a1__py3-none-any.whl

json_database/__init__.py

@@ -0,0 +1,387 @@
+import json
+import logging
+import os
+from os import makedirs, remove
+from os.path import expanduser, isdir, dirname, exists, isfile, join
+from pprint import pprint
+from tempfile import gettempdir
+
+from combo_lock import ComboLock
+
+from json_database.crypto import decrypt_from_json, encrypt_as_json
+from json_database.exceptions import InvalidItemID, DatabaseNotCommitted, \
+    SessionError, MatchError
+from json_database.utils import DummyLock, load_commented_json, merge_dict, \
+    jsonify_recursively, get_key_recursively, get_key_recursively_fuzzy, \
+    get_value_recursively_fuzzy, get_value_recursively
+from json_database.xdg_utils import xdg_cache_home, xdg_data_home, xdg_config_home
+
+LOG = logging.getLogger("JsonDatabase")
+LOG.setLevel("INFO")
+
+
+class JsonStorage(dict):
+    """
+    persistent python dict
+    """
+
+    def __init__(self, path, disable_lock=False):
+        super().__init__()
+        lock_path = join(gettempdir(), path.split("/")[-1] + ".lock")
+        if disable_lock:
+            self.lock = DummyLock(lock_path)
+        else:
+            self.lock = ComboLock(lock_path)
+        self.path = path
+        if self.path:
+            self.load_local(self.path)
+
+    def load_local(self, path):
+        """
+            Load local json file into self.
+
+            Args:
+                path (str): file to load
+        """
+        with self.lock:
+            path = expanduser(path)
+            if exists(path) and isfile(path):
+                self.clear()
+                try:
+                    config = load_commented_json(path)
+                    for key in config:
+                        self[key] = config[key]
+                    LOG.debug("Json {} loaded".format(path))
+                except Exception as e:
+                    LOG.error("Error loading json '{}'".format(path))
+                    LOG.error(repr(e))
+            else:
+                LOG.debug("Json '{}' not defined, skipping".format(path))
+
+    def clear(self):
+        for k in dict(self):
+            self.pop(k)
+
+    def reload(self):
+        if exists(self.path) and isfile(self.path):
+            self.load_local(self.path)
+        else:
+            raise DatabaseNotCommitted
+
+    def store(self, path=None):
+        """
+            store the json db locally.
+        """
+        with self.lock:
+            path = path or self.path
+            if not path:
+                LOG.warning("json db path not set")
+                return
+            path = expanduser(path)
+            if dirname(path) and not isdir(dirname(path)):
+                makedirs(dirname(path))
+            with open(path, 'w', encoding="utf-8") as f:
+                json.dump(self, f, indent=4, ensure_ascii=False)
+
+    def remove(self):
+        with self.lock:
+            if isfile(self.path):
+                remove(self.path)
+
+    def merge(self, conf, merge_lists=True, skip_empty=True, no_dupes=True,
+              new_only=False):
+        merge_dict(self, conf, merge_lists, skip_empty, no_dupes, new_only)
+        return self
+
+    def __enter__(self):
+        """ Context handler """
+        return self
+
+    def __exit__(self, _type, value, traceback):
+        """ Commits changes and Closes the session """
+        try:
+            self.store()
+        except Exception as e:
+            LOG.error(e)
+            raise SessionError
+
+
+class EncryptedJsonStorage(JsonStorage):
+    """persistent python dict, stored AES encrypted to file"""
+
+    def __init__(self, encrypt_key: str, path: str, disable_lock=False):
+        assert len(encrypt_key) == 16
+        self.encrypt_key = encrypt_key
+        super().__init__(path, disable_lock)
+
+    def load_local(self, path):
+        """
+            Load local json file into self.
+
+            Args:
+                path (str): file to load
+        """
+        super().load_local(path)
+        # decrypt after load
+        if self:
+            decrypted = json.loads(decrypt_from_json(self.encrypt_key, dict(self)))
+            self.clear()
+            self.update(decrypted)
+
+    def store(self, path=None):
+        """
+            store the json db locally.
+        """
+        decrypted = dict(self)
+        encrypted = json.loads(encrypt_as_json(self.encrypt_key, decrypted))
+        self.clear()
+        self.merge(encrypted)  # encrypt before storage
+        super().store()
+        self.clear()
+        self.update(decrypted)  # keep it decrypted in memory
+
+
+class JsonDatabase(dict):
+    """ searchable persistent dict """
+
+    def __init__(self,
+                 name,
+                 path=None,
+                 disable_lock=False,
+                 extension="json"):
+        super().__init__()
+        self.name = name
+        self.path = path or f"{name}.{extension}"
+        self.db = JsonStorage(self.path, disable_lock=disable_lock)
+        self.db[name] = []
+        self.db.load_local(self.path)
+
+    # operator overloads
+    def __enter__(self):
+        """ Context handler """
+        return self
+
+    def __exit__(self, _type, value, traceback):
+        """ Commits changes and Closes the session """
+        try:
+            self.commit()
+        except Exception as e:
+            LOG.error(e)
+            raise SessionError
+
+    def __repr__(self):
+        return str(jsonify_recursively(self))
+
+    def __len__(self):
+        return len(self.db.get(self.name, []))
+
+    def __getitem__(self, item):
+        if not isinstance(item, int):
+            try:
+                item_id = int(item)
+            except Exception as e:
+                item_id = self.get_item_id(item)
+                if item_id < 0:
+                    raise InvalidItemID
+        else:
+            item_id = item
+        if item_id >= len(self.db[self.name]):
+            raise InvalidItemID
+        return self.db[self.name][item_id]
+
+    def __setitem__(self, item_id, value):
+        if not isinstance(item_id, int) or item_id >= len(self) or item_id < 0:
+            raise InvalidItemID
+        else:
+            self.update_item(item_id, value)
+
+    def __iter__(self):
+        for item in self.db[self.name]:
+            yield item
+
+    def __contains__(self, item):
+        item = jsonify_recursively(item)
+        return item in self.db[self.name]
+
+    # database
+    def commit(self):
+        """
+            store the json db locally.
+        """
+        self.db.store(self.path)
+
+    def reset(self):
+        self.db.reload()
+
+    def print(self):
+        pprint(jsonify_recursively(self))
+
+    # item manipulations
+    def append(self, value):
+        value = jsonify_recursively(value)
+        self.db[self.name].append(value)
+        return len(self)
+
+    def add_item(self, value, allow_duplicates=False):
+        """ add an item to database
+         if allow_duplicates is True, item is added unconditionally,
+         else only if no exact match is present
+         """
+        if allow_duplicates or value not in self:
+            self.append(value)
+            return len(self)
+        return self.get_item_id(value)
+
+    def match_item(self, value, match_strategy=None):
+        """ match value to some item in database
+        returns a list of matched items
+        """
+        value = jsonify_recursively(value)
+        matches = []
+        for idx, item in enumerate(self):
+
+            # TODO match strategy
+            # - require exact match
+            # - require list of keys to match
+            # - require at least one of key list to match
+            # - require at exactly one of key list to match
+
+            # by default check for exact matches
+            if item == value:
+                matches.append((item, idx))
+
+        return matches
+
+    def merge_item(self, value, item_id=None, match_strategy=None,
+                   merge_strategy=None):
+        """ search an item according to match criteria, merge fields"""
+        if item_id is None:
+            matches = self.match_item(value, match_strategy)
+            if not matches:
+                raise MatchError
+            match, item_id = matches[0][1]
+        else:
+            match = self[item_id]
+        # TODO merge strategy
+        # - only merge some keys
+        # - dont merge some keys
+        # - merge all keys
+        # - dont overwrite keys
+        value = jsonify_recursively(value)
+        self[item_id] = merge_dict(match, value)
+
+    def replace_item(self, value, item_id=None, match_strategy=None):
+        """ search an item according to match criteria, replace it"""
+        if item_id is None:
+            matches = self.match_item(value, match_strategy)
+            if not matches:
+                raise MatchError
+            match, item_id = matches[0][1]
+        value = jsonify_recursively(value)
+        self[item_id] = value
+
+    # item_id
+    def get_item_id(self, item):
+        """
+        item_id is simply the index of the item in the database
+        WARNING: this is not immutable across sessions
+        """
+        for match, idx in self.match_item(item):
+            return idx
+        return -1
+
+    def update_item(self, item_id, new_item):
+        """
+        item_id is simply the index of the item in the database
+        WARNING: this is not immutable across sessions
+        """
+        new_item = jsonify_recursively(new_item)
+        self.db[self.name][item_id] = new_item
+
+    def remove_item(self, item_id):
+        """
+        item_id is simply the index of the item in the database
+        WARNING: this is not immutable across sessions
+        """
+        return self.db[self.name].pop(item_id)
+
+    # search
+    def search_by_key(self, key, fuzzy=False, thresh=0.7, include_empty=False):
+        if fuzzy:
+            return get_key_recursively_fuzzy(self.db, key, thresh, not include_empty)
+        return get_key_recursively(self.db, key, not include_empty)
+
+    def search_by_value(self, key, value, fuzzy=False, thresh=0.7):
+        if fuzzy:
+            return get_value_recursively_fuzzy(self.db, key, value, thresh)
+        return get_value_recursively(self.db, key, value)
+
+
+# XDG aware classes
+
+class JsonStorageXDG(JsonStorage):
+    """ xdg respectful persistent dicts """
+
+    def __init__(self,
+                 name,
+                 xdg_folder=xdg_cache_home(),
+                 disable_lock=False, subfolder="json_database",
+                 extension="json"):
+        self.name = name
+        path = join(xdg_folder, subfolder, f"{name}.{extension}")
+        super().__init__(path, disable_lock=disable_lock)
+
+
+class EncryptedJsonStorageXDG(EncryptedJsonStorage):
+    """ xdg respectful persistent dicts """
+
+    def __init__(self,
+                 encrypt_key: str,
+                 name: str,
+                 xdg_folder=xdg_data_home(),
+                 disable_lock=False,
+                 subfolder="json_database",
+                 extension="ejson"):
+        self.name = name
+        path = join(xdg_folder, subfolder, f"{name}.{extension}")
+        super().__init__(encrypt_key=encrypt_key, path=path,
+                         disable_lock=disable_lock)
+
+
+class JsonDatabaseXDG(JsonDatabase):
+    """ xdg respectful json database """
+
+    def __init__(self, name, xdg_folder=xdg_data_home(),
+                 disable_lock=False, subfolder="json_database",
+                 extension="jsondb"):
+        path = join(xdg_folder, subfolder, f"{name}.{extension}")
+        super().__init__(name, path, disable_lock=disable_lock, extension=extension)
+
+
+class JsonConfigXDG(JsonStorageXDG):
+    """ xdg respectful config files, using json_storage.JsonStorageXDG """
+
+    def __init__(self, name, xdg_folder=xdg_config_home(),
+                 disable_lock=False, subfolder="json_database",
+                 extension="json"):
+        super().__init__(name, xdg_folder, disable_lock, subfolder, extension)
+
+
+if __name__ == "__main__":
+    # quick test
+    os.remove("/tmp/test.json")
+    db = EncryptedJsonStorage("S" * 16, "/tmp/test.json")
+    db["A"] = "42"
+    print(db)  # {'A': '42'} - not encrypted in memory
+    db.store()
+    print(db)  # {'A': '42'} - still decrypted
+    db.reload()
+    print(db)  # {'A': '42'} - still decrypted
+    db = EncryptedJsonStorage("S" * 16, "/tmp/test.json")
+    print(db)  # {'A': '42'} - still not encrypted
+
+    db = JsonStorage("/tmp/test.json")
+    print(db)  # encrypted
+    # {'ciphertext': 'ad0da72dc412d6b1240e478560354893d62caf',
+    # 'tag': '3bc39dbfad7b0d7e50f3e652ee341819',
+    # 'nonce': '3020ddafc9853e7686ee0368f9be6e25'}

json_database/crypto.py

@@ -0,0 +1,80 @@
+import json
+import zlib
+from binascii import hexlify
+from binascii import unhexlify
+
+try:
+    # pycryptodomex
+    from Cryptodome.Cipher import AES
+except ImportError:
+    # pycrypto + pycryptodome
+    try:
+        from Crypto.Cipher import AES
+    except:
+        AES = None
+
+
+def encrypt(key, text, nonce=None):
+    if AES is None:
+        raise ImportError("run pip install pycryptodomex")
+    if not isinstance(text, bytes):
+        text = bytes(text, encoding="utf-8")
+    if not isinstance(key, bytes):
+        key = bytes(key, encoding="utf-8")
+    cipher = AES.new(key, AES.MODE_GCM, nonce=nonce)
+    text = compress_payload(text)
+    ciphertext, tag = cipher.encrypt_and_digest(text)
+    return ciphertext, tag, cipher.nonce
+
+
+def decrypt(key, ciphertext, tag, nonce) -> str:
+    if AES is None:
+        raise ImportError("run pip install pycryptodomex")
+    if not isinstance(key, bytes):
+        key = bytes(key, encoding="utf-8")
+    cipher = AES.new(key, AES.MODE_GCM, nonce)
+    data = cipher.decrypt_and_verify(ciphertext, tag)
+    text = decompress_payload(data).decode(encoding="utf-8")
+    return text
+
+
+def encrypt_as_json(key, data):
+    if isinstance(data, dict):
+        data = json.dumps(data)
+    if len(key) > 16:
+        key = key[0:16]
+    ciphertext, tag, nonce = encrypt(key, data)
+    return json.dumps({"ciphertext": hexlify(ciphertext).decode('utf-8'),
+                       "tag": hexlify(tag).decode('utf-8'),
+                       "nonce": hexlify(nonce).decode('utf-8')})
+
+
+def decrypt_from_json(key, data):
+    if isinstance(data, str):
+        data = json.loads(data)
+    if len(key) > 16:
+        key = key[0:16]
+    ciphertext = unhexlify(data["ciphertext"])
+    if data.get("tag") is None:  # web crypto
+        ciphertext, tag = ciphertext[:-16], ciphertext[-16:]
+    else:
+        tag = unhexlify(data["tag"])
+    nonce = unhexlify(data["nonce"])
+    return decrypt(key, ciphertext, tag, nonce)
+
+
+def compress_payload(text):
+    # Compressing text
+    if isinstance(text, str):
+        decompressed = text.encode("utf-8")
+    else:
+        decompressed = text
+    return zlib.compress(decompressed)
+
+
+def decompress_payload(compressed):
+    # Decompressing text
+    if isinstance(compressed, str):
+        # assume hex
+        compressed = unhexlify(compressed)
+    return zlib.decompress(compressed)

json_database/exceptions.py

@@ -0,0 +1,22 @@
+class InvalidItemID(ValueError):
+    """ ItemID is invalid """
+
+
+class DatabaseNotCommitted(FileNotFoundError):
+    """ Database has not been saved in yet """
+
+
+class SessionError(RuntimeError):
+    """ Could not commit database"""
+
+
+class MatchError(ValueError):
+    """ could not match an item in db """
+
+
+class DecryptionKeyError(KeyError):
+    """ Could not decrypt payload """
+
+
+class EncryptionKeyError(KeyError):
+    """ Could not encrypt payload """

json_database/search.py

@@ -0,0 +1,169 @@
+from json_database.utils import fuzzy_match, match_one
+from json_database import JsonDatabase
+
+
+class Query:
+    def __init__(self, db):
+        if isinstance(db, JsonDatabase):
+            self.result = list(db)
+        else:
+            self.result = [db]
+
+    def contains_key(self, key, fuzzy=False, thresh=0.7, ignore_case=False):
+        if fuzzy:
+            after = []
+            for e in self.result:
+                filter = True
+                for k in e:
+                    if ignore_case:
+                        score = fuzzy_match(k.lower(), key.lower())
+                    else:
+                        score = fuzzy_match(k, key)
+                    if score < thresh:
+                        continue
+                    filter = False
+                if not filter:
+                    after.append(e)
+            self.result = after
+        elif ignore_case:
+            self.result = [a for a in self.result
+                           if a.get(key) or a.get(key.lower())]
+        else:
+            self.result = [a for a in self.result if a.get(key)]
+        return self
+
+    def contains_value(self, key, value, fuzzy=False, thresh=0.75, ignore_case=False):
+        self.contains_key(key, ignore_case=ignore_case)
+        if fuzzy:
+            after = []
+            for e in self.result:
+                if isinstance(e[key], str):
+                    if ignore_case:
+                        score = fuzzy_match(value.lower(), e[key].lower())
+                    else:
+                        score = fuzzy_match(value, e[key])
+                    if score > thresh:
+                        after.append(e)
+                elif isinstance(e[key], list):
+                    if ignore_case:
+                        v, score = match_one(value.lower(),
+                                             [_.lower() for _ in e[key]])
+                    else:
+                        v, score = match_one(value, e[key])
+                    if score < thresh:
+                        continue
+                    after.append(e)
+                elif isinstance(e[key], dict):
+                    if ignore_case:
+                        v, score = match_one(value.lower(),
+                                             [_.lower() for _ in e[key].keys()])
+                    else:
+                        v, score = match_one(value, e[key])
+                    if score < thresh:
+                        continue
+                    after.append(e)
+            self.result = after
+        elif ignore_case and isinstance(value, str):
+            after = []
+            for a in self.result:
+                if isinstance(a[key], str) and value.lower() in a[key].lower():
+                    after.append(a)
+                elif value.lower() in a[key] or value in a[key]:
+                    after.append(a)
+            self.result = after
+        else:
+            self.result = [a for a in self.result if value in a[key]]
+        return self
+
+    def value_contains(self, key, value, ignore_case=False):
+        self.contains_key(key, ignore_case=ignore_case)
+        if ignore_case:
+            after = []
+            value = str(value).lower()
+            for e in self.result:
+                if isinstance(e[key], str) and ignore_case:
+                    if value.lower() in e[key].lower():
+                        after.append(e)
+                elif isinstance(e[key], list) and ignore_case:
+                    if value.lower() in [str(_).lower() for _ in e[key]]:
+                        after.append(e)
+                elif isinstance(e[key], dict) and ignore_case:
+                    if value.lower() in [str(_).lower() for _ in e[key].keys()]:
+                        after.append(e)
+
+                elif isinstance(e[key], str):
+                    if value in e[key]:
+                        after.append(e)
+                elif isinstance(e[key], list):
+                    if value in [str(_) for _ in e[key]]:
+                        after.append(e)
+                elif isinstance(e[key], dict):
+                    if value in [str(_) for _ in e[key].keys()]:
+                        after.append(e)
+
+            self.result = after
+        else:
+            self.result = [e for e in self.result if value in e[key]]
+        return self
+
+    def value_contains_token(self, key, value, fuzzy=False, thresh=0.75, ignore_case=False):
+        self.contains_key(key, ignore_case=ignore_case)
+        after = []
+        value = str(value)
+        for e in self.result:
+            if isinstance(e[key], str):
+                if fuzzy:
+                    _, score = match_one(value.lower(),
+                                         e[key].lower().split(" "))
+                    if score > thresh:
+                        after.append(e)
+                elif ignore_case and value.lower() in e[key].lower().split(" "):
+                    after.append(e)
+                elif value in e[key].split(" "):
+                    after.append(e)
+            elif value in e[key]:
+                after.append(e)
+        self.result = after
+        return self
+
+    def equal(self, key, value, ignore_case=False):
+        self.contains_key(key, ignore_case=ignore_case)
+        if ignore_case and isinstance(value, str):
+            self.result = [a for a in self.result
+                           if a[key].lower() == value.lower()]
+        else:
+            self.result = [a for a in self.result if a[key] == value]
+        return self
+
+    def below(self, key, value, ignore_case=False):
+        self.contains_key(key, ignore_case=ignore_case)
+        self.result = [a for a in self.result if a[key] < value]
+        return self
+
+    def above(self, key, value, ignore_case=False):
+        self.contains_key(key, ignore_case=ignore_case)
+        self.result = [a for a in self.result if a[key] > value]
+        return self
+
+    def below_or_equal(self, key, value, ignore_case=False):
+        self.contains_key(key, ignore_case=ignore_case)
+        self.result = [a for a in self.result if a[key] <= value]
+        return self
+
+    def above_or_equal(self, key, value, ignore_case=False):
+        self.contains_key(key, ignore_case=ignore_case)
+        self.result = [a for a in self.result if a[key] >= value]
+        return self
+
+    def in_range(self, key, min_value, max_value, ignore_case=False):
+        self.contains_key(key, ignore_case=ignore_case)
+        self.result = [a for a in self.result if min_value < a[key] < max_value]
+        return self
+
+    def all(self):
+        return self
+
+    def build(self):
+        return self.result
+
+