dictature 0.11.1__py3-none-any.whl → 0.13.1__py3-none-any.whl

dictature/backend/mysql.py

@@ -0,0 +1,119 @@
+try:
+    import mysql.connector
+except ImportError:
+    raise ImportError('Requires: pip install mysql-connector-python') from None
+from typing import Iterable
+
+from .mock import DictatureBackendMock, DictatureTableMock, Value
+
+
+class DictatureBackendMySQL(DictatureBackendMock):
+    def __init__(self, host: str, port: int = 3306, user: str = None, password: str = None,
+                 database: str = None, prefix: str = 'tb_', **kwargs) -> None:
+        """
+        Create a new MySQL backend
+        :param host: MySQL server host
+        :param port: MySQL server port (default: 3306)
+        :param user: MySQL username
+        :param password: MySQL password
+        :param database: MySQL database name
+        :param kwargs: Additional connection parameters
+        """
+        self.__connection_params = {
+            'host': host,
+            'port': port,
+            'user': user,
+            'password': password,
+            'database': database,
+            **kwargs
+        }
+
+        # Remove None values from connection params
+        self.__connection_params = {k: v for k, v in self.__connection_params.items() if v is not None}
+
+        self.__connection = mysql.connector.connect(**self.__connection_params)
+        self.__cursor = self.__connection.cursor()
+        self.__prefix = prefix.replace('`', '').replace("'", '')
+
+    def keys(self) -> Iterable[str]:
+        # noinspection SqlResolve
+        tables = self._execute(f"SELECT table_name FROM information_schema.tables WHERE table_schema = %s AND table_name LIKE '{self.__prefix}%'", (self.__connection_params['database'],))
+        return {table[0][3:] for table in tables}
+
+    def table(self, name: str) -> 'DictatureTableMock':
+        return DictatureTableMySQL(self, name, self.__prefix)
+
+    def _execute(self, query: str, data: tuple = ()) -> list:
+        self.__cursor.execute(query, data)
+        return self.__cursor.fetchall()
+
+    def _commit(self) -> None:
+        self.__connection.commit()
+
+    def __del__(self):
+        if hasattr(self, '_DictatureBackendMySQL__connection'):
+            self.__connection.close()
+
+
+class DictatureTableMySQL(DictatureTableMock):
+    def __init__(self, parent: "DictatureBackendMySQL", name: str, prefix: str) -> None:
+        self.__parent = parent
+        # MySQL table names don't need backticks for simple names, but we'll use them for consistency
+        self.__table = f"`{prefix}{name.replace('`', '')}`"
+
+    def keys(self) -> Iterable[str]:
+        # noinspection PyProtectedMember
+        result = self.__parent._execute(f"SELECT `key` FROM {self.__table}")
+        return set(map(lambda x: x[0], result))
+
+    def drop(self) -> None:
+        # noinspection PyProtectedMember
+        self.__parent._execute(f"DROP TABLE {self.__table}")
+
+    def key_exists(self, item: str) -> bool:
+        # noinspection PyProtectedMember
+        result = self.__parent._execute(f"SELECT `value` FROM {self.__table} WHERE `key`=%s", (item,))
+        return len(result) > 0
+
+    def create(self) -> None:
+        # noinspection PyProtectedMember
+        self.__parent._execute(f"""
+        CREATE TABLE IF NOT EXISTS {self.__table} (
+        `key` VARCHAR(700) NOT NULL UNIQUE,
+        `value` TEXT,
+        `type` INT NOT NULL DEFAULT 0,
+        PRIMARY KEY (`key`)
+        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
+        """)
+
+    def set(self, item: str, value: Value) -> None:
+        if self.key_exists(item):
+            # noinspection PyProtectedMember
+            self.__parent._execute(f"""
+            UPDATE {self.__table} SET `value`=%s, `type`=%s WHERE `key`=%s
+            """, (value.value, value.mode, item))
+        else:
+            # noinspection PyProtectedMember
+            self.__parent._execute(
+                f"INSERT INTO {self.__table} (`key`, `value`, `type`) VALUES (%s, %s, %s)",
+                (item, value.value, value.mode)
+            )
+        # noinspection PyProtectedMember
+        self.__parent._commit()
+
+    def get(self, item: str) -> Value:
+        # noinspection PyProtectedMember
+        r = self.__parent._execute(f"SELECT `value`, `type` FROM {self.__table} WHERE `key`=%s", (item,))
+        if r:
+            value: str = r[0][0]
+            type_value: int = r[0][1]
+            return Value(value=value, mode=type_value)
+        raise KeyError(item)
+
+    def delete(self, item: str) -> None:
+        # noinspection PyProtectedMember
+        self.__parent._execute(
+            f"DELETE FROM {self.__table} WHERE `key`=%s", (item,)
+        )
+        # noinspection PyProtectedMember
+        self.__parent._commit()

dictature/backend/s3.py

@@ -0,0 +1,176 @@
+import io
+import posixpath
+from typing import Iterable
+
+try:
+    import boto3
+    from botocore.exceptions import ClientError
+except ImportError:
+    raise ImportError('Requires: pip install boto3')
+
+from .mock import DictatureTableMock, DictatureBackendMock, Value, ValueMode, ValueSerializer, ValueSerializerMode
+
+
+class DictatureBackendS3(DictatureBackendMock):
+    def __init__(
+            self,
+            bucket_name: str,
+            aws_access_key_id: str = None,
+            aws_secret_access_key: str = None,
+            region_name: str = None,
+            endpoint_url: str = None,
+            dir_prefix: str = 'db_',
+            item_prefix: str = 'item_'
+    ) -> None:
+        """
+        Initialize S3 backend
+
+        :param bucket_name: S3 bucket name
+        :param aws_access_key_id: AWS access key ID
+        :param aws_secret_access_key: AWS secret access key
+        :param region_name: AWS region name
+        :param endpoint_url: Custom endpoint URL for alternative S3-compatible storage (e.g., MinIO)
+        :param dir_prefix: Prefix for table directories
+        :param item_prefix: Prefix for item files
+        """
+        self.__s3 = boto3.resource(
+            's3',
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+            region_name=region_name,
+            endpoint_url=endpoint_url
+        )
+        self.__bucket = self.__s3.Bucket(bucket_name)
+        self.__bucket_name = bucket_name
+        self.__dir_prefix = dir_prefix
+        self.__item_prefix = item_prefix
+
+    def keys(self) -> Iterable[str]:
+        """
+        Get all table names in the S3 bucket
+        """
+        # Group objects by their "directory" prefix
+        prefixes = set()
+        for obj in self.__bucket.objects.filter(Prefix=self.__dir_prefix):
+            key = obj.key
+            parts = key.split('/')
+            if len(parts) > 1 and parts[0].startswith(self.__dir_prefix):
+                prefixes.add(parts[0])
+
+        for prefix in prefixes:
+            table_name_encoded = prefix[len(self.__dir_prefix):]
+            # noinspection PyProtectedMember
+            yield DictatureTableS3._filename_decode(table_name_encoded, suffix='')
+
+    def table(self, name: str) -> 'DictatureTableMock':
+        return DictatureTableS3(
+            self.__s3,
+            self.__bucket_name,
+            name,
+            self.__dir_prefix,
+            self.__item_prefix
+        )
+
+
+class DictatureTableS3(DictatureTableMock):
+    def __init__(
+            self,
+            s3_resource,
+            bucket_name: str,
+            name: str,
+            db_prefix: str,
+            prefix: str
+    ) -> None:
+        self.__s3 = s3_resource
+        self.__bucket = self.__s3.Bucket(bucket_name)
+        self.__bucket_name = bucket_name
+        self.__encoded_name = self._filename_encode(name, suffix='')
+        self.__path = f"{db_prefix}{self.__encoded_name}/"
+        self.__prefix = prefix
+        self.__name_serializer = ValueSerializer(mode=ValueSerializerMode.filename_only)
+        self.__value_serializer = ValueSerializer(mode=ValueSerializerMode.any_string)
+
+    def keys(self) -> Iterable[str]:
+        try:
+            for obj in self.__bucket.objects.filter(Prefix=self.__path):
+                key = obj.key
+                filename = posixpath.basename(key)
+                if filename.startswith(self.__prefix):
+                    item_name_encoded = filename[len(self.__prefix):]
+                    yield self._filename_decode(item_name_encoded, suffix='.txt')
+        except ClientError:
+            pass  # Return empty generator if bucket doesn't exist or other error
+
+    def drop(self) -> None:
+        """
+        Delete all objects in this table's directory
+        """
+        try:
+            self.__bucket.objects.filter(Prefix=self.__path).delete()
+        except ClientError:
+            pass  # Ignore if objects don't exist or other error
+
+    def create(self) -> None:
+        """
+        S3 doesn't need explicit directory creation, but we can create a marker file
+        to indicate the table's existence if it doesn't already have items
+        """
+        try:
+            # Check if the "directory" exists by looking for any objects with this prefix
+            objects = list(self.__bucket.objects.filter(Prefix=self.__path).limit(1))
+            if not objects:
+                # Create an empty marker object to represent the directory
+                self.__bucket.put_object(Key=f"{self.__path}.keep")
+        except ClientError:
+            # If bucket doesn't exist, this will fail, but that's expected
+            pass
+
+    def set(self, item: str, value: Value) -> None:
+        item_path = self.__item_path(item)
+        save_data: str = self.__value_serializer.serialize(value)
+        data_bytes = save_data.encode('utf-8')
+
+        with io.BytesIO(data_bytes) as buffer:
+            self.__bucket.upload_fileobj(buffer, item_path)
+
+    def get(self, item: str) -> Value:
+        item_path = self.__item_path(item)
+        buffer = io.BytesIO()
+        try:
+            self.__bucket.download_fileobj(item_path, buffer)
+            buffer.seek(0)
+            save_data = buffer.read().decode('utf-8')
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'NoSuchKey':
+                raise KeyError(item)
+            raise  # Re-raise for other errors
+        return self.__value_serializer.deserialize(save_data)
+
+    def delete(self, item: str) -> None:
+        """
+        Delete the item from S3
+        """
+        item_path = self.__item_path(item)
+        try:
+            self.__s3.Object(self.__bucket_name, item_path).delete()
+        except ClientError as e:
+            if e.response['Error']['Code'] != 'NoSuchKey':
+                raise  # Only ignore "not found" errors
+
+    def __item_path(self, item: str) -> str:
+        encoded_item_name = self.__prefix + self._filename_encode(item, suffix='.txt')
+        full_path = f"{self.__path}{encoded_item_name}"
+        return full_path
+
+    @staticmethod
+    def _filename_encode(name: str, suffix: str = '.txt') -> str:
+        return ValueSerializer(mode=ValueSerializerMode.filename_only).serialize(Value(
+            value=name,
+            mode=ValueMode.string.value
+        )) + suffix
+
+    @staticmethod
+    def _filename_decode(name: str, suffix: str = '.txt') -> str:
+        if suffix and name.endswith(suffix):
+            name = name[:-len(suffix)]
+        return ValueSerializer(mode=ValueSerializerMode.filename_only).deserialize(name).value

dictature/backend/sqlite.py

@@ -1,15 +1,16 @@
 import sqlite3
-from typing import Iterable, Union, Optional
 from pathlib import Path
+from typing import Iterable, Union, Optional
 
 from .mock import DictatureBackendMock, DictatureTableMock, Value, ValueMode
 
 
 class DictatureBackendSQLite(DictatureBackendMock):
-    def __init__(self, file: Union[str, Path]) -> None:
+    def __init__(self, file: Union[str, Path], prefix: str = 'tb_') -> None:
         """
         Create a new SQLite backend
         :param file: file to store the database
+        :param prefix: prefix for the tables (default: 'tb_')
         """
         if isinstance(file, str):
             file = Path(file)
@@ -19,13 +20,14 @@ class DictatureBackendSQLite(DictatureBackendMock):
             check_same_thread=False if sqlite3.threadsafety >= 3 else True
         )
         self.__cursor = self.__connection.cursor()
+        self.__prefix = prefix.replace("'", "").replace('`', '')
 
     def keys(self) -> Iterable[str]:
-        tables = self._execute("SELECT tbl_name FROM sqlite_master WHERE type='table' AND tbl_name LIKE 'tb_%'")
+        tables = self._execute(f"SELECT tbl_name FROM sqlite_master WHERE type='table' AND tbl_name LIKE '{self.__prefix}%'")
         return {table[0][3:] for table in tables}
 
     def table(self, name: str) -> 'DictatureTableMock':
-        return DictatureTableSQLite(self, name)
+        return DictatureTableSQLite(self, name, self.__prefix)
 
     def _execute(self, query: str, data: tuple = ()) -> list:
         return list(self.__cursor.execute(query, data))
@@ -38,10 +40,10 @@ class DictatureBackendSQLite(DictatureBackendMock):
 
 
 class DictatureTableSQLite(DictatureTableMock):
-    def __init__(self, parent: "DictatureBackendSQLite", name: str) -> None:
+    def __init__(self, parent: "DictatureBackendSQLite", name: str, prefix: str) -> None:
         self.__parent = parent
         self.__supports_jsonization: Optional[bool] = None
-        self.__table = "`tb_%s`" % name.replace('`', '``')
+        self.__table = "`%s`" % (prefix + name).replace('`', '``')
 
     def keys(self) -> Iterable[str]:
         # noinspection PyProtectedMember

{dictature-0.11.1.dist-info → dictature-0.13.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dictature
-Version: 0.11.1
+Version: 0.13.1
 Summary: dictature -- A generic wrapper around dict-like interface with mulitple backends
 Author-email: Adam Hlavacek <git@adamhlavacek.com>
 Project-URL: Homepage, https://github.com/esoadamo/dictature
@@ -60,6 +60,8 @@ Currently, the following backends are supported:
 - `DictatureBackendSQLite`: stores the data in a SQLite database
 - `DictatureBackendMISP`: stores the data in a MISP instance
 - `DictatureBackendWebdav`: stores data in a WebDav share as files
+- `DictatureBackendS3`: stores data in an S3 bucket
+- `DictatureBackendMySQL`: stores data in a MySQL database
 
 ### Transformers
 

{dictature-0.11.1.dist-info → dictature-0.13.1.dist-info}/RECORD

@@ -4,7 +4,9 @@ dictature/backend/__init__.py,sha256=d5s6QCJOUzFglVNg8Cqqx_8b61S-AOTGjEUIF6FS69U
 dictature/backend/directory.py,sha256=_-8Fx7Ef4tFbOQWHcARfKjm0kbZUpmN1f1ysyWINOr4,3600
 dictature/backend/misp.py,sha256=R9osuEJa79SoXErwByIwhViJ26cEfw2nKhWu8XxkO7Y,4860
 dictature/backend/mock.py,sha256=o4YBl6Wk-q6IHFdykGeryTKcQFsJHvymlONPbbwVkak,5891
-dictature/backend/sqlite.py,sha256=zyphYEeLY4eGuBCor16i80_-brdipMpXZ3_kONwErsE,5237
+dictature/backend/mysql.py,sha256=LdbcytXCnu2pqx3LfOH__-dFy85IwB-HGMFOxDIGBvA,4624
+dictature/backend/s3.py,sha256=9Vcf4-RagyjpV9F4m9ZQ1vjb4lbZfEcOZQBuD_SPwZg,6580
+dictature/backend/sqlite.py,sha256=mHEfRufqRcDfl0mIPl9eK0ODyUmgYD9SWjbllI87Hp0,5434
 dictature/backend/webdav.py,sha256=Y-3_WTcMyKVUnsVjiUZAxuy10FK0Yr-7Mgn13clg3po,5039
 dictature/transformer/__init__.py,sha256=JIFJpXU6iB9hIUM8L7HL2o9Nqjm_YbMEuQBQC8ZJ6b4,124
 dictature/transformer/aes.py,sha256=ZhC1dT9QpnziErkDLriWLgXDEFNGQW0KG4aqSN2AZpA,1926
@@ -13,8 +15,8 @@ dictature/transformer/hmac.py,sha256=vURsB0HlzRPn_Vkl7lGmZV9OKempQuds8AanmadDxIo
 dictature/transformer/mock.py,sha256=7zu65ZqUV_AVRaPSzNd73cVMXixXt31SeuX9OKZxaJQ,948
 dictature/transformer/passthrough.py,sha256=Pt3N6G_Qh6HJ_q75ETL5nfAwYHLB-SjkVwUwbbbMik8,344
 dictature/transformer/pipeline.py,sha256=OaQaJeJ5NpICetJe08r8ontqstsXGuW8jDbKw1zxYs4,842
-dictature-0.11.1.dist-info/LICENSE,sha256=n1U9DKr8sM5EY2QHcvxSGiKTDWUT8MyXsOC79w94MT0,1072
-dictature-0.11.1.dist-info/METADATA,sha256=b-VVWD1Tgz0GwX0OvoWHPCtx-g117pYEbyEWyNeBKbE,3283
-dictature-0.11.1.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-dictature-0.11.1.dist-info/top_level.txt,sha256=-RO39WWCF44lqiXhSUcACVqbk6SkgReZTz7ZmHKH3-U,10
-dictature-0.11.1.dist-info/RECORD,,
+dictature-0.13.1.dist-info/LICENSE,sha256=n1U9DKr8sM5EY2QHcvxSGiKTDWUT8MyXsOC79w94MT0,1072
+dictature-0.13.1.dist-info/METADATA,sha256=hJZ0-CPWRgnNPvMY2l3uhPiEO9-xuNDgSwUmVM0adfs,3394
+dictature-0.13.1.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+dictature-0.13.1.dist-info/top_level.txt,sha256=-RO39WWCF44lqiXhSUcACVqbk6SkgReZTz7ZmHKH3-U,10
+dictature-0.13.1.dist-info/RECORD,,