json-database 0.7.0__tar.gz → 0.8.1__tar.gz

json_database-0.8.1/PKG-INFO

@@ -0,0 +1,202 @@
+Metadata-Version: 2.1
+Name: json_database
+Version: 0.8.1
+Summary: searchable json database with persistence
+Home-page: https://github.com/TigreGotico/json_database
+Author: jarbasAI
+Author-email: jarbasai@mailfence.com
+License: MIT
+Description: # Json Database
+        
+        Python dict based database with persistence and search capabilities
+        
+        For those times when you need something simple and sql is overkill
+        
+        
+        ## Features
+        
+        - pure python
+        - save and load from file
+        - search recursively by key and key/value pairs
+        - fuzzy search
+        - supports arbitrary objects
+        - supports comments in saved files
+        
+        ## Install
+        
+        ```bash
+        pip install json_database
+        ```
+        
+        ## Usage
+        
+        
+        ### JsonStorage
+        
+        Sometimes you need persistent dicts that you can save and load from file
+        
+        ```python
+        from json_database import JsonStorage
+        from os.path import exists
+        
+        save_path = "my_dict.conf"
+        
+        my_config = JsonStorage(save_path)
+        
+        my_config["lang"] = "pt"
+        my_config["secondary_lang"] = "en"
+        my_config["email"] = "jarbasai@mailfence.com"
+        
+        # my_config is a python dict
+        assert isinstance(my_config, dict)
+        
+        # save to file
+        my_config.store()
+        
+        my_config["lang"] = "pt-pt"
+        
+        # revert to previous saved file
+        my_config.reload()
+        assert my_config["lang"] == "pt"
+        
+        # clear all fields
+        my_config.clear()
+        assert my_config == {}
+        
+        # load from a specific path
+        my_config.load_local(save_path)
+        assert my_config == JsonStorage(save_path)
+        
+        # delete stored file
+        my_config.remove()
+        assert not exists(save_path)
+        
+        # keep working with dict in memory
+        print(my_config)
+        ```
+        
+        ### JsonDatabase
+        
+        Ever wanted to search a dict?
+        
+        Let's create a dummy database with users
+        
+        ```python
+        from json_database import JsonDatabase
+        
+        db_path = "users.db"
+        
+        with JsonDatabase("users", db_path) as db:
+            # add some users to the database
+        
+            for user in [
+                {"name": "bob", "age": 12},
+                {"name": "bobby"},
+                {"name": ["joe", "jony"]},
+                {"name": "john"},
+                {"name": "jones", "age": 35},
+                {"name": "joey", "birthday": "may 12"}]:
+                db.add_item(user)
+                
+            # pretty print database contents
+            db.print()
+        
+        
+        # auto saved when used with context manager
+        # db.commit()
+        
+        
+        ```
+                 
+        search entries by key
+        
+        ```python
+        from json_database import JsonDatabase
+        
+        db_path = "users.db"
+        
+        db = JsonDatabase("users", db_path) # load db created in previous example
+        
+        # search by exact key match
+        users_with_defined_age = db.search_by_key("age")
+        
+        for user in users_with_defined_age:
+            print(user["name"], user["age"])
+            
+        # fuzzy search
+        users = db.search_by_key("birth", fuzzy=True)
+        for user, conf in users:
+            print("matched with confidence", conf)
+            print(user["name"], user["birthday"])
+        ```
+        
+        search by key value pair
+        
+        ```python
+        # search by key/value pair
+        users_12years_old = db.search_by_value("age", 12)
+        
+        for user in users_12years_old:
+            assert user["age"] == 12
+        
+        # fuzzy search
+        jon_users = db.search_by_value("name", "jon", fuzzy=True)
+        for user, conf in jon_users:
+            print(user["name"])
+            print("matched with confidence", conf)
+            # NOTE that one of the users has a list instead of a string in the name, it also matches
+        ```
+        
+        updating an existing entry
+        
+        ```python
+        # get database item
+        item = {"name": "bobby"}
+        
+        item_id = db.get_item_id(item)
+        
+        if item_id >= 0:
+            new_item = {"name": "don't call me bobby"}
+            db.update_item(item_id, new_item)
+        else:
+            print("item not found in database")
+        
+        # clear changes since last commit
+        db.reset()
+        ```
+        
+        You can save arbitrary objects to the database
+        
+        ```python
+        from json_database import JsonDatabase
+        
+        db = JsonDatabase("users", "~/databases/users.json")
+        
+        
+        class User:
+            def __init__(self, email, key=None, data=None):
+                self.email = email
+                self.secret_key = key
+                self.data = data
+        
+        user1 = User("first@mail.net", data={"name": "jonas", "birthday": "12 May"})
+        user2 = User("second@mail.net", "secret", data={"name": ["joe", "jony"], "age": 12})
+        
+        # objects will be "jsonified" here, they will no longer be User objects
+        # if you need them to be a specific class use some ORM lib instead (SQLAlchemy is great)
+        db.add_item(user1)
+        db.add_item(user2)
+        
+        # search entries with non empty key
+        print(db.search_by_key("secret_key"))
+        
+        # search in user provided data
+        print(db.search_by_key("birth", fuzzy=True))
+        
+        # search entries with a certain value
+        print(db.search_by_value("age", 12))
+        print(db.search_by_value("name", "jon", fuzzy=True))
+        
+        ```
+Platform: UNKNOWN
+Description-Content-Type: text/markdown

{json_database-0.7.0 → json_database-0.8.1}/json_database/__init__.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import os
 from os import makedirs, remove
 from os.path import expanduser, isdir, dirname, exists, isfile, join
 from pprint import pprint
@@ -7,6 +8,7 @@ from tempfile import gettempdir
 
 from combo_lock import ComboLock
 
+from json_database.crypto import decrypt_from_json, encrypt_as_json
 from json_database.exceptions import InvalidItemID, DatabaseNotCommitted, \
     SessionError, MatchError
 from json_database.utils import DummyLock, load_commented_json, merge_dict, \
@@ -104,6 +106,41 @@ class JsonStorage(dict):
             raise SessionError
 
 
+class EncryptedJsonStorage(JsonStorage):
+    """persistent python dict, stored AES encrypted to file"""
+
+    def __init__(self, encrypt_key: str, path: str, disable_lock=False):
+        assert len(encrypt_key) == 16
+        self.encrypt_key = encrypt_key
+        super().__init__(path, disable_lock)
+
+    def load_local(self, path):
+        """
+            Load local json file into self.
+
+            Args:
+                path (str): file to load
+        """
+        super().load_local(path)
+        # decrypt after load
+        if self:
+            decrypted = json.loads(decrypt_from_json(self.encrypt_key, dict(self)))
+            self.clear()
+            self.update(decrypted)
+
+    def store(self, path=None):
+        """
+            store the json db locally.
+        """
+        decrypted = dict(self)
+        encrypted = json.loads(encrypt_as_json(self.encrypt_key, decrypted))
+        self.clear()
+        self.merge(encrypted)  # encrypt before storage
+        super().store()
+        self.clear()
+        self.update(decrypted)  # keep it decrypted in memory
+
+
 class JsonDatabase(dict):
     """ searchable persistent dict """
 
@@ -295,6 +332,22 @@ class JsonStorageXDG(JsonStorage):
         super().__init__(path, disable_lock=disable_lock)
 
 
+class EncryptedJsonStorageXDG(EncryptedJsonStorage):
+    """ xdg respectful persistent dicts """
+
+    def __init__(self,
+                 encrypt_key: str,
+                 name: str,
+                 xdg_folder=xdg_data_home(),
+                 disable_lock=False,
+                 subfolder="json_database",
+                 extension="ejson"):
+        self.name = name
+        path = join(xdg_folder, subfolder, f"{name}.{extension}")
+        super().__init__(encrypt_key=encrypt_key, path=path,
+                         disable_lock=disable_lock)
+
+
 class JsonDatabaseXDG(JsonDatabase):
     """ xdg respectful json database """
 
@@ -312,3 +365,23 @@ class JsonConfigXDG(JsonStorageXDG):
                  disable_lock=False, subfolder="json_database",
                  extension="json"):
         super().__init__(name, xdg_folder, disable_lock, subfolder, extension)
+
+
+if __name__ == "__main__":
+    # quick test
+    os.remove("/tmp/test.json")
+    db = EncryptedJsonStorage("S" * 16, "/tmp/test.json")
+    db["A"] = "42"
+    print(db)  # {'A': '42'} - not encrypted in memory
+    db.store()
+    print(db)  # {'A': '42'} - still decrypted
+    db.reload()
+    print(db)  # {'A': '42'} - still decrypted
+    db = EncryptedJsonStorage("S" * 16, "/tmp/test.json")
+    print(db)  # {'A': '42'} - still not encrypted
+
+    db = JsonStorage("/tmp/test.json")
+    print(db)  # encrypted
+    # {'ciphertext': 'ad0da72dc412d6b1240e478560354893d62caf',
+    # 'tag': '3bc39dbfad7b0d7e50f3e652ee341819',
+    # 'nonce': '3020ddafc9853e7686ee0368f9be6e25'}

json_database-0.8.1/json_database/crypto.py

@@ -0,0 +1,80 @@
+import json
+import zlib
+from binascii import hexlify
+from binascii import unhexlify
+
+try:
+    # pycryptodomex
+    from Cryptodome.Cipher import AES
+except ImportError:
+    # pycrypto + pycryptodome
+    try:
+        from Crypto.Cipher import AES
+    except:
+        AES = None
+
+
+def encrypt(key, text, nonce=None):
+    if AES is None:
+        raise ImportError("run pip install pycryptodomex")
+    if not isinstance(text, bytes):
+        text = bytes(text, encoding="utf-8")
+    if not isinstance(key, bytes):
+        key = bytes(key, encoding="utf-8")
+    cipher = AES.new(key, AES.MODE_GCM, nonce=nonce)
+    text = compress_payload(text)
+    ciphertext, tag = cipher.encrypt_and_digest(text)
+    return ciphertext, tag, cipher.nonce
+
+
+def decrypt(key, ciphertext, tag, nonce) -> str:
+    if AES is None:
+        raise ImportError("run pip install pycryptodomex")
+    if not isinstance(key, bytes):
+        key = bytes(key, encoding="utf-8")
+    cipher = AES.new(key, AES.MODE_GCM, nonce)
+    data = cipher.decrypt_and_verify(ciphertext, tag)
+    text = decompress_payload(data).decode(encoding="utf-8")
+    return text
+
+
+def encrypt_as_json(key, data):
+    if isinstance(data, dict):
+        data = json.dumps(data)
+    if len(key) > 16:
+        key = key[0:16]
+    ciphertext, tag, nonce = encrypt(key, data)
+    return json.dumps({"ciphertext": hexlify(ciphertext).decode('utf-8'),
+                       "tag": hexlify(tag).decode('utf-8'),
+                       "nonce": hexlify(nonce).decode('utf-8')})
+
+
+def decrypt_from_json(key, data):
+    if isinstance(data, str):
+        data = json.loads(data)
+    if len(key) > 16:
+        key = key[0:16]
+    ciphertext = unhexlify(data["ciphertext"])
+    if data.get("tag") is None:  # web crypto
+        ciphertext, tag = ciphertext[:-16], ciphertext[-16:]
+    else:
+        tag = unhexlify(data["tag"])
+    nonce = unhexlify(data["nonce"])
+    return decrypt(key, ciphertext, tag, nonce)
+
+
+def compress_payload(text):
+    # Compressing text
+    if isinstance(text, str):
+        decompressed = text.encode("utf-8")
+    else:
+        decompressed = text
+    return zlib.compress(decompressed)
+
+
+def decompress_payload(compressed):
+    # Decompressing text
+    if isinstance(compressed, str):
+        # assume hex
+        compressed = unhexlify(compressed)
+    return zlib.decompress(compressed)

{json_database-0.7.0 → json_database-0.8.1}/json_database/exceptions.py

@@ -12,3 +12,11 @@ class SessionError(RuntimeError):
 
 class MatchError(ValueError):
     """ could not match an item in db """
+
+
+class DecryptionKeyError(KeyError):
+    """ Could not decrypt payload """
+
+
+class EncryptionKeyError(KeyError):
+    """ Could not encrypt payload """

json_database-0.8.1/json_database/version.py

@@ -0,0 +1,6 @@
+# START_VERSION_BLOCK
+VERSION_MAJOR = 0
+VERSION_MINOR = 8
+VERSION_BUILD = 1
+VERSION_ALPHA = 0
+# END_VERSION_BLOCK

json_database-0.8.1/json_database.egg-info/PKG-INFO

@@ -0,0 +1,202 @@
+Metadata-Version: 2.1
+Name: json-database
+Version: 0.8.1
+Summary: searchable json database with persistence
+Home-page: https://github.com/TigreGotico/json_database
+Author: jarbasAI
+Author-email: jarbasai@mailfence.com
+License: MIT
+Description: # Json Database
+        
+        Python dict based database with persistence and search capabilities
+        
+        For those times when you need something simple and sql is overkill
+        
+        
+        ## Features
+        
+        - pure python
+        - save and load from file
+        - search recursively by key and key/value pairs
+        - fuzzy search
+        - supports arbitrary objects
+        - supports comments in saved files
+        
+        ## Install
+        
+        ```bash
+        pip install json_database
+        ```
+        
+        ## Usage
+        
+        
+        ### JsonStorage
+        
+        Sometimes you need persistent dicts that you can save and load from file
+        
+        ```python
+        from json_database import JsonStorage
+        from os.path import exists
+        
+        save_path = "my_dict.conf"
+        
+        my_config = JsonStorage(save_path)
+        
+        my_config["lang"] = "pt"
+        my_config["secondary_lang"] = "en"
+        my_config["email"] = "jarbasai@mailfence.com"
+        
+        # my_config is a python dict
+        assert isinstance(my_config, dict)
+        
+        # save to file
+        my_config.store()
+        
+        my_config["lang"] = "pt-pt"
+        
+        # revert to previous saved file
+        my_config.reload()
+        assert my_config["lang"] == "pt"
+        
+        # clear all fields
+        my_config.clear()
+        assert my_config == {}
+        
+        # load from a specific path
+        my_config.load_local(save_path)
+        assert my_config == JsonStorage(save_path)
+        
+        # delete stored file
+        my_config.remove()
+        assert not exists(save_path)
+        
+        # keep working with dict in memory
+        print(my_config)
+        ```
+        
+        ### JsonDatabase
+        
+        Ever wanted to search a dict?
+        
+        Let's create a dummy database with users
+        
+        ```python
+        from json_database import JsonDatabase
+        
+        db_path = "users.db"
+        
+        with JsonDatabase("users", db_path) as db:
+            # add some users to the database
+        
+            for user in [
+                {"name": "bob", "age": 12},
+                {"name": "bobby"},
+                {"name": ["joe", "jony"]},
+                {"name": "john"},
+                {"name": "jones", "age": 35},
+                {"name": "joey", "birthday": "may 12"}]:
+                db.add_item(user)
+                
+            # pretty print database contents
+            db.print()
+        
+        
+        # auto saved when used with context manager
+        # db.commit()
+        
+        
+        ```
+                 
+        search entries by key
+        
+        ```python
+        from json_database import JsonDatabase
+        
+        db_path = "users.db"
+        
+        db = JsonDatabase("users", db_path) # load db created in previous example
+        
+        # search by exact key match
+        users_with_defined_age = db.search_by_key("age")
+        
+        for user in users_with_defined_age:
+            print(user["name"], user["age"])
+            
+        # fuzzy search
+        users = db.search_by_key("birth", fuzzy=True)
+        for user, conf in users:
+            print("matched with confidence", conf)
+            print(user["name"], user["birthday"])
+        ```
+        
+        search by key value pair
+        
+        ```python
+        # search by key/value pair
+        users_12years_old = db.search_by_value("age", 12)
+        
+        for user in users_12years_old:
+            assert user["age"] == 12
+        
+        # fuzzy search
+        jon_users = db.search_by_value("name", "jon", fuzzy=True)
+        for user, conf in jon_users:
+            print(user["name"])
+            print("matched with confidence", conf)
+            # NOTE that one of the users has a list instead of a string in the name, it also matches
+        ```
+        
+        updating an existing entry
+        
+        ```python
+        # get database item
+        item = {"name": "bobby"}
+        
+        item_id = db.get_item_id(item)
+        
+        if item_id >= 0:
+            new_item = {"name": "don't call me bobby"}
+            db.update_item(item_id, new_item)
+        else:
+            print("item not found in database")
+        
+        # clear changes since last commit
+        db.reset()
+        ```
+        
+        You can save arbitrary objects to the database
+        
+        ```python
+        from json_database import JsonDatabase
+        
+        db = JsonDatabase("users", "~/databases/users.json")
+        
+        
+        class User:
+            def __init__(self, email, key=None, data=None):
+                self.email = email
+                self.secret_key = key
+                self.data = data
+        
+        user1 = User("first@mail.net", data={"name": "jonas", "birthday": "12 May"})
+        user2 = User("second@mail.net", "secret", data={"name": ["joe", "jony"], "age": 12})
+        
+        # objects will be "jsonified" here, they will no longer be User objects
+        # if you need them to be a specific class use some ORM lib instead (SQLAlchemy is great)
+        db.add_item(user1)
+        db.add_item(user2)
+        
+        # search entries with non empty key
+        print(db.search_by_key("secret_key"))
+        
+        # search in user provided data
+        print(db.search_by_key("birth", fuzzy=True))
+        
+        # search entries with a certain value
+        print(db.search_by_value("age", 12))
+        print(db.search_by_value("name", "jon", fuzzy=True))
+        
+        ```
+Platform: UNKNOWN
+Description-Content-Type: text/markdown

{json_database-0.7.0 → json_database-0.8.1}/json_database.egg-info/SOURCES.txt

@@ -2,12 +2,15 @@ LICENSE
 README.md
 setup.py
 json_database/__init__.py
+json_database/crypto.py
 json_database/exceptions.py
 json_database/search.py
 json_database/utils.py
+json_database/version.py
 json_database/xdg_utils.py
 json_database.egg-info/PKG-INFO
 json_database.egg-info/SOURCES.txt
 json_database.egg-info/dependency_links.txt
 json_database.egg-info/requires.txt
-json_database.egg-info/top_level.txt
+json_database.egg-info/top_level.txt
+test/test_crypto.py

json_database-0.8.1/json_database.egg-info/requires.txt

@@ -0,0 +1 @@
+combo_lock<1.0.0,>=0.2.1

json_database-0.8.1/setup.py

@@ -0,0 +1,73 @@
+import os
+import os.path
+
+from setuptools import setup
+
+BASEDIR = os.path.abspath(os.path.dirname(__file__))
+
+
+def package_files(directory):
+    paths = []
+    for (path, _, filenames) in os.walk(directory):
+        for filename in filenames:
+            paths.append(os.path.join('..', path, filename))
+    return paths
+
+
+def required(requirements_file):
+    """ Read requirements file and remove comments and empty lines. """
+    with open(os.path.join(BASEDIR, requirements_file), 'r') as f:
+        requirements = f.read().splitlines()
+        if 'MYCROFT_LOOSE_REQUIREMENTS' in os.environ:
+            print('USING LOOSE REQUIREMENTS!')
+            requirements = [r.replace('==', '>=').replace('~=', '>=') for r in requirements]
+        return [pkg for pkg in requirements
+                if pkg.strip() and not pkg.startswith("#")]
+
+
+def get_version():
+    """ Find the version of ovos-core"""
+    version = None
+    version_file = os.path.join(BASEDIR, 'json_database', 'version.py')
+    major, minor, build, alpha = (None, None, None, None)
+    with open(version_file) as f:
+        for line in f:
+            if 'VERSION_MAJOR' in line:
+                major = line.split('=')[1].strip()
+            elif 'VERSION_MINOR' in line:
+                minor = line.split('=')[1].strip()
+            elif 'VERSION_BUILD' in line:
+                build = line.split('=')[1].strip()
+            elif 'VERSION_ALPHA' in line:
+                alpha = line.split('=')[1].strip()
+
+            if ((major and minor and build and alpha) or
+                    '# END_VERSION_BLOCK' in line):
+                break
+    version = f"{major}.{minor}.{build}"
+    if int(alpha):
+        version += f"a{alpha}"
+    return version
+
+
+def get_description():
+    with open(os.path.join(BASEDIR, "README.md"), "r") as f:
+        long_description = f.read()
+    return long_description
+
+
+setup(
+    name='json_database',
+    version=get_version(),
+    packages=['json_database'],
+    package_data={'': package_files('json_database')},
+    include_package_data=True,
+    url='https://github.com/TigreGotico/json_database',
+    license='MIT',
+    author='jarbasAI',
+    author_email='jarbasai@mailfence.com',
+    install_requires=required('requirements.txt'),
+    description='searchable json database with persistence',
+    long_description=get_description(),
+    long_description_content_type="text/markdown",
+)

json_database-0.8.1/test/test_crypto.py

@@ -0,0 +1,53 @@
+import os
+import unittest
+
+from json_database import EncryptedJsonStorage, JsonStorage  # Replace with actual import
+
+
+class TestEncryptedJsonStorage(unittest.TestCase):
+    def setUp(self):
+        self.key = "S" * 16  # Replace with actual key generation if needed
+        self.file_path = "/tmp/test.json"
+        # Ensure the test file doesn't exist at the start of each test
+        if os.path.exists(self.file_path):
+            os.remove(self.file_path)
+
+    def tearDown(self):
+        # Clean up the test file after each test
+        if os.path.exists(self.file_path):
+            os.remove(self.file_path)
+
+    def test_add_and_store_data(self):
+        db = EncryptedJsonStorage(self.key, self.file_path)
+        db["A"] = "42"
+        self.assertEqual(db["A"], "42")  # Check in-memory data
+        db.store()
+        self.assertTrue(os.path.exists(self.file_path))  # File should be created
+
+    def test_encryption_in_file(self):
+        db = EncryptedJsonStorage(self.key, self.file_path)
+        db["A"] = "42"
+        db.store()
+        with open(self.file_path, "r") as file:
+            file_data = file.read()
+        self.assertNotIn("42", file_data)  # Data should be encrypted
+
+    def test_decryption_after_reload(self):
+        db = EncryptedJsonStorage(self.key, self.file_path)
+        db["A"] = "42"
+        db.store()
+        db.reload()
+        self.assertEqual(db["A"], "42")  # Data should be decrypted correctly
+
+    def test_jsonstorage_read_encrypted_data(self):
+        encrypted_db = EncryptedJsonStorage(self.key, self.file_path)
+        encrypted_db["A"] = "42"
+        encrypted_db.store()
+
+        db = JsonStorage(self.file_path)
+        self.assertIn("ciphertext", db)  # Check that it's encrypted
+        self.assertNotIn("A", db)
+
+
+if __name__ == "__main__":
+    unittest.main()

json_database-0.7.0/PKG-INFO

@@ -1,13 +0,0 @@
-Metadata-Version: 2.1
-Name: json_database
-Version: 0.7.0
-Summary: searchable json database with persistence
-Home-page: https://github.com/OpenJarbas/json_database
-Author: jarbasAI
-Author-email: jarbasai@mailfence.com
-License: MIT
-Platform: UNKNOWN
-License-File: LICENSE
-
-UNKNOWN
-

json_database-0.7.0/json_database.egg-info/PKG-INFO

@@ -1,13 +0,0 @@
-Metadata-Version: 2.1
-Name: json-database
-Version: 0.7.0
-Summary: searchable json database with persistence
-Home-page: https://github.com/OpenJarbas/json_database
-Author: jarbasAI
-Author-email: jarbasai@mailfence.com
-License: MIT
-Platform: UNKNOWN
-License-File: LICENSE
-
-UNKNOWN
-

json_database-0.7.0/json_database.egg-info/requires.txt

@@ -1 +0,0 @@
-combo_lock~=0.2.1

json_database-0.7.0/setup.py

@@ -1,13 +0,0 @@
-from setuptools import setup
-
-setup(
-    name='json_database',
-    version='0.7.0',
-    packages=['json_database'],
-    url='https://github.com/OpenJarbas/json_database',
-    license='MIT',
-    author='jarbasAI',
-    author_email='jarbasai@mailfence.com',
-    install_requires=["combo_lock~=0.2.1"],
-    description='searchable json database with persistence'
-)