TestDataX 0.1.0__py3-none-any.whl

src/exporters/mysql_exporter.py

@@ -0,0 +1,184 @@
+import decimal
+from datetime import date, datetime
+from typing import Any
+from uuid import UUID
+
+from .base_exporter import BaseExporter
+from .utils.constants import DEFAULT_SCHEMA
+
+MYSQL_TYPE_MAPPING = {
+    "string": "VARCHAR(255)",
+    "text": "TEXT",
+    "integer": "INT",
+    "bigint": "BIGINT",
+    "float": "FLOAT",
+    "decimal": "DECIMAL(18,2)",
+    "boolean": "TINYINT(1)",
+    "date": "DATE",
+    "datetime": "DATETIME",
+    "blob": "LONGBLOB",
+    "uuid": "VARCHAR(36)",
+    "enum": "ENUM",
+}
+
+
+class MysqlExporter(BaseExporter):
+    """Exports data to MySQL compatible SQL file."""
+
+    def _format_value(
+        self,
+        value: (
+            None
+            | str
+            | UUID
+            | datetime
+            | date
+            | bool
+            | bytes
+            | int
+            | float
+            | decimal.Decimal
+        ),
+    ) -> str:
+        """Format a value for use in a MySQL query.
+
+        This method handles various data types and converts them to their
+        appropriate string representation for use in MySQL queries.
+        It includes proper escaping for special characters in strings.
+
+        Args:
+            value: The value to format. Can be one of the following types:
+                - None: Converted to 'NULL'
+                - str: Escaped and wrapped in single quotes
+                - UUID: Converted to string, escaped and wrapped in single quotes
+                - datetime: Converted to ISO format string and wrapped in single quotes
+                - date: Converted to ISO format string and wrapped in single quotes
+                - bool: Converted to '1' for True or '0' for False
+                - bytes: Converted to hexadecimal string with 'x' prefix
+                - int: Converted to string representation
+                - float: Converted to string representation
+                - decimal.Decimal: Converted to string representation
+
+        Returns:
+            str: The formatted value ready for use in a MySQL query.
+
+        Raises:
+            ValueError: If the input value type is not supported.
+
+        """
+        if value is None:
+            return "NULL"
+        elif isinstance(value, (str | UUID)):
+            return "'" + str(value).replace("'", "\\'").replace("\n", "\\n") + "'"
+        elif isinstance(value, (datetime | date)):
+            return f"'{value.isoformat()}'"
+        elif isinstance(value, bool):
+            return "1" if value else "0"
+        elif isinstance(value, bytes):
+            return f"x'{value.hex()}'"
+        elif isinstance(value, (int | float | decimal.Decimal)):
+            return str(value)
+        else:
+            raise ValueError(f"Unsupported type: {type(value)}")
+
+    def _get_column_type(self, field: dict) -> str:
+        """Get the MySQL column type based on the field type.
+
+        Args:
+            field (dict): A dictionary containing field information, including its type.
+                The type can be either a string or a dictionary with a 'type' key.
+
+        Returns:
+            str: The corresponding MySQL data type from MYSQL_TYPE_MAPPING.
+
+        Raises:
+            KeyError: If the field type is not found in MYSQL_TYPE_MAPPING.
+
+        """
+        field_type = field.get("type", "string")
+        if isinstance(field_type, dict):
+            field_type = field_type.get("type", "string")
+        return MYSQL_TYPE_MAPPING[field_type]
+
+    def _create_table_stmt(self, schema: dict, table_name: str = "output") -> str:
+        """Generate a MySQL CREATE TABLE statement based on provided schema.
+
+        This method constructs a CREATE TABLE SQL statement by mapping schema field
+        definitions to their corresponding MySQL column types. It supports both simple
+        type definitions and complex field definitions including ENUM types.
+
+        Args:
+            schema (dict): A dictionary defining the table schema with field names
+                as keys and type definitions as values.
+            table_name (str, optional): Name for the table. Defaults to "output".
+
+        Returns:
+            str: A complete MySQL CREATE TABLE statement as a string.
+
+        Example:
+            schema = {
+                "id": "int",
+                "status": {"type": "enum", "values": ["active", "inactive"]}
+            }
+            result = _create_table_stmt(schema, "users")
+            # Returns: CREATE TABLE users (
+            #     id INT NULL,
+            #     status ENUM('active','inactive') NULL
+            # );
+
+        """
+        columns = []
+        for field_name, field_def in schema.items():
+            field_type_dict = (
+                field_def if isinstance(field_def, dict) else {"type": field_def}
+            )
+            sql_type = self._get_column_type(field_type_dict)
+            if (
+                isinstance(field_def, dict)
+                and field_def.get("type") == "enum"
+                and "values" in field_def
+            ):
+                values = "','".join(field_def["values"])
+                sql_type = f"ENUM('{values}')"
+            columns.append(f"    {field_name} {sql_type} NULL")
+
+        return f"CREATE TABLE {table_name} (\n" + ",\n".join(columns) + "\n);\n\n"
+
+    def _create_insert_stmt(
+        self, row: dict[str, Any], table_name: str = "output"
+    ) -> str:
+        """Create a MySQL INSERT statement from a dictionary of values.
+
+        Args:
+            row (dict[str, Any]): Dictionary containing column names as keys
+                and values to insert
+            table_name (str, optional): Name of the target table. Defaults to "output"
+
+        Returns:
+            str: Formatted MySQL INSERT statement string
+
+        Example:
+            >>> row = {"id": 1, "name": "test"}
+            >>> _create_insert_stmt(row, "users")
+            'INSERT INTO users (id, name) VALUES (1, "test");'
+
+        """
+        columns = ", ".join(row.keys())
+        values = ", ".join(self._format_value(v) for v in row.values())
+        return f"INSERT INTO {table_name} ({columns}) VALUES ({values});"
+
+    def export(
+        self, data: list[dict[str, Any]], output_path: str, schema: dict | None = None
+    ) -> None:
+        """Export data to MySQL compatible SQL file."""
+        if not data:
+            return
+
+        table_name = output_path.split("/")[-1].split(".")[0]
+
+        with open(output_path, "w") as f:
+            # Always write CREATE TABLE using DEFAULT_SCHEMA if no schema provided
+            schema_to_use = schema or DEFAULT_SCHEMA
+            f.write(self._create_table_stmt(schema_to_use, table_name))
+            for row in data:
+                f.write(self._create_insert_stmt(row, table_name) + "\n")

src/exporters/oracle_exporter.py

@@ -0,0 +1,205 @@
+import decimal
+from datetime import date, datetime
+from typing import Any
+from uuid import UUID
+
+from .base_exporter import BaseExporter
+from .utils.constants import DEFAULT_SCHEMA
+
+ORACLE_TYPE_MAPPING = {
+    "string": "VARCHAR2(255)",  # Oracle's Unicode string type
+    "text": "CLOB",  # Oracle's large text type
+    "integer": "NUMBER(10)",  # Standard integer in Oracle
+    "bigint": "NUMBER(19)",  # Large integers in Oracle
+    "float": "FLOAT",  # Same in Oracle
+    "decimal": "NUMBER(18,2)",  # Oracle's decimal type
+    "boolean": "NUMBER(1)",  # Oracle has no native boolean
+    "date": "DATE",  # Same in Oracle
+    "datetime": "TIMESTAMP",  # Oracle's datetime type
+    "blob": "BLOB",  # Oracle's binary large object
+    "uuid": "VARCHAR2(36)",  # Oracle has no native UUID
+    "enum": "VARCHAR2(255)",  # Oracle has no ENUM type
+}
+
+
+class OracleExporter(BaseExporter):
+    """Exports data to ORACLE compatible SQL file."""
+
+    def _format_value(
+        self,
+        value: (
+            None
+            | str
+            | UUID
+            | datetime
+            | date
+            | bool
+            | bytes
+            | int
+            | float
+            | decimal.Decimal
+        ),
+    ) -> str:
+        """Format a value for use in a ORACLE query.
+
+        This method handles various data types and converts them to their
+        appropriate string representation for use in ORACLE queries.
+        It includes proper escaping for special characters in strings.
+
+        Args:
+            value: The value to format. Can be one of the following types:
+                - None: Converted to 'NULL'
+                - str: Escaped and wrapped in single quotes
+                - UUID: Converted to string, escaped and wrapped in single quotes
+                - datetime: Converted to ISO format string and wrapped in single quotes
+                - date: Converted to ISO format string and wrapped in single quotes
+                - bool: Converted to '1' for True or '0' for False
+                - bytes: Use 0x prefix for hexadecimal to binary conversion
+                - int: Converted to string representation
+                - float: Converted to string representation
+                - decimal.Decimal: Converted to string representation
+
+        Returns:
+            str: The formatted value ready for use in a ORACLE query.
+
+        Raises:
+            ValueError: If the input value type is not supported.
+
+        """
+        if value is None:
+            return "NULL"
+        elif isinstance(value, (str)):
+            return "'" + str(value).replace("'", "''") + "'"  # uses '' for escaping
+        elif isinstance(value, UUID):
+            return f"'{str(value)}'"
+        elif isinstance(value, datetime):
+            return (
+                f"TO_TIMESTAMP('{value.isoformat()}', "
+                f"'YYYY-MM-DD\"T\"HH24:MI:SS.FF')"
+            )
+        elif isinstance(value, date):
+            return f"DATE '{value.isoformat()}'"
+        elif isinstance(value, bool):
+            return "1" if value else "0"
+        elif isinstance(value, bytes):
+            return f"HEXTORAW('{value.hex()}')"  # Oracle BLOB format
+        elif isinstance(value, (int | float | decimal.Decimal)):
+            return str(value)
+        else:
+            raise ValueError(f"Unsupported type: {type(value)}")
+
+    def _get_column_type(self, field: dict) -> str:
+        """Get the ORACLE column type based on the field type.
+
+        Args:
+            field (dict): A dictionary containing field information, including its type.
+                The type can be either a string or a dictionary with a 'type' key.
+
+        Returns:
+            str: The corresponding ORACLE data type from ORACLE_TYPE_MAPPING.
+
+        Raises:
+            KeyError: If the field type is not found in ORACLE_TYPE_MAPPING.
+
+        """
+        field_type = field.get("type", "string")
+        if isinstance(field_type, dict):
+            field_type = field_type.get("type", "string")
+        return ORACLE_TYPE_MAPPING[field_type]
+
+    def _create_table_stmt(self, schema: dict, table_name: str = "output") -> str:
+        """Generate a ORACLE CREATE TABLE statement based on provided schema.
+
+        This method constructs a CREATE TABLE SQL statement by mapping schema field
+        definitions to their corresponding ORACLE column types. For ENUM types,
+        it creates a CHECK constraint to validate the values.
+
+        Args:
+            schema (dict): A dictionary defining the table schema with field names
+                as keys and type definitions as values.
+            table_name (str, optional): Name for the table. Defaults to "output".
+
+        Returns:
+            str: A complete ORACLE CREATE TABLE statement as a string.
+
+        Example:
+            schema = {
+                "id": "int",
+                "status": {"type": "enum", "values": ["active", "inactive"]}
+            }
+            result = _create_table_stmt(schema, "users")
+            # Returns: CREATE TABLE users (
+            #     id INT NULL,
+            #     status NVARCHAR(255) NULL CHECK (status IN ('active','inactive'))
+            # );
+
+        """
+        columns = []
+        check_constraints = []
+
+        for field_name, field_def in schema.items():
+            field_type_dict = (
+                field_def if isinstance(field_def, dict) else {"type": field_def}
+            )
+            sql_type = self._get_column_type(field_type_dict)
+
+            if (
+                isinstance(field_def, dict)
+                and field_def.get("type") == "enum"
+                and "values" in field_def
+            ):
+                values = "','".join(field_def["values"])
+                check_constraints.append(f"CHECK ({field_name} IN ('{values}'))")
+
+            columns.append(f"    {field_name} {sql_type} NULL")
+
+        # Combine columns and check constraints
+        return (
+            f"CREATE TABLE {table_name} (\n"
+            + ",\n".join(columns)
+            + (
+                (",\n    " + ",\n    ".join(check_constraints))
+                if check_constraints
+                else ""
+            )
+            + "\n);\n\n"
+        )
+
+    def _create_insert_stmt(
+        self, row: dict[str, Any], table_name: str = "output"
+    ) -> str:
+        """Create a ORACLE INSERT statement from a dictionary of values.
+
+        Args:
+            row (dict[str, Any]): Dictionary containing column names as keys
+                and values to insert
+            table_name (str, optional): Name of the target table. Defaults to "output"
+
+        Returns:
+            str: Formatted ORACLE INSERT statement string
+
+        Example:
+            >>> row = {"id": 1, "name": "test"}
+            >>> _create_insert_stmt(row, "users")
+            'INSERT INTO users (id, name) VALUES (1, "test");'
+
+        """
+        columns = ", ".join(row.keys())
+        values = ", ".join(self._format_value(v) for v in row.values())
+        return f"INSERT INTO {table_name} ({columns}) VALUES ({values});"
+
+    def export(
+        self, data: list[dict[str, Any]], output_path: str, schema: dict | None = None
+    ) -> None:
+        """Export data to ORACLE compatible SQL file."""
+        if not data:
+            return
+
+        table_name = output_path.split("/")[-1].split(".")[0]
+
+        with open(output_path, "w") as f:
+            # Always write CREATE TABLE using DEFAULT_SCHEMA if no schema provided
+            schema_to_use = schema or DEFAULT_SCHEMA
+            f.write(self._create_table_stmt(schema_to_use, table_name))
+            for row in data:
+                f.write(self._create_insert_stmt(row, table_name) + "\n")

src/exporters/orc_exporter.py

@@ -0,0 +1,100 @@
+import logging
+from typing import Any
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.orc as po
+
+from .base_exporter import BaseExporter
+from .utils.chunker import DataChunker
+from .utils.constants import (
+    CHUNK_SIZE_ORC,
+    DEFAULT_ORC_COMPRESSION,
+    VALID_ORC_COMPRESSION,
+    OrcCompression,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class OrcExporter(BaseExporter):
+    """Exports data to ORC file format using PyArrow and Pandas."""
+
+    def __init__(
+        self,
+        chunk_size: int = CHUNK_SIZE_ORC,
+        compression: OrcCompression = DEFAULT_ORC_COMPRESSION,
+    ) -> None:
+        """Initialize Exporter with specified options.
+
+        Args:
+            chunk_size: Number of rows to write at once
+            compression: Compression codec
+            compression_strategy: Compression strategy
+
+        """
+        if compression not in VALID_ORC_COMPRESSION:
+            raise ValueError(f"Invalid compression: {compression}")
+
+        self.chunker = DataChunker(chunk_size)
+        self.compression = compression
+
+    def _get_compressed_path(self, path: str) -> str:
+        """Insert compression type into filename before extension."""
+        if self.compression == "UNCOMPRESSED":
+            return path
+        # Remove any existing compression extension
+        base = path
+        for comp in VALID_ORC_COMPRESSION:
+            base = base.replace(f".{comp.lower()}", "")
+        # Split at last occurrence of .orc
+        base = base.replace(".orc", "")
+        return f"{base}.{self.compression.lower()}.orc"
+
+    def export(
+        self,
+        data: list[dict[str, Any]],
+        output_path: str,
+        schema: dict | None = None,
+    ) -> None:
+        """Export data to ORC file format.
+
+        Args:
+            data: List of dictionaries containing the data to export
+            output_path: Path where the ORC file will be written
+            schema: Schema definition for the data (not used for ORC)
+
+        """
+        if not data:
+            logger.info("No data to export")
+            raise RuntimeError("No data to export")
+
+        # Validate base path ends with .orc before adding compression
+        if not output_path.endswith(".orc"):
+            raise ValueError("Output path must have .orc extension")
+
+        output_path = self._get_compressed_path(output_path)
+        logger.info(f"Starting ORC export to {output_path} with {len(data)} rows")
+        if not isinstance(data, list):
+            raise TypeError("Data must be a list of dictionaries")
+
+        writer = None
+        try:
+            for i, chunk in enumerate(self.chunker.chunk_data(data), 1):
+                logger.debug(f"Processing chunk {i}")
+                df = pd.DataFrame(chunk)
+                if "uuid" in df.columns:
+                    df["uuid"] = df["uuid"].astype(str)
+                table = pa.Table.from_pandas(df)
+                if writer is None:
+                    writer = po.ORCWriter(
+                        str(output_path),
+                        compression=self.compression,
+                    )
+                writer.write(table)
+            logger.info(f"Successfully exported {len(data)} rows to {output_path}")
+        except Exception as e:
+            raise RuntimeError(f"Failed to write Orc file: {e}") from e
+        finally:
+            if writer:
+                writer.close()

src/exporters/parquet_exporter.py

@@ -0,0 +1,102 @@
+import logging
+from typing import Any
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from .base_exporter import BaseExporter
+from .utils.chunker import DataChunker
+from .utils.constants import (
+    CHUNK_SIZE_PARQUET,
+    DEFAULT_PARQUET_COMPRESSION,
+    VALID_PARQUET_COMPRESSION,
+    ParquetCompression,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ParquetExporter(BaseExporter):
+    """Exporter class for writing data to Parquet format files."""
+
+    def __init__(
+        self,
+        chunk_size: int = CHUNK_SIZE_PARQUET,
+        compression: ParquetCompression = DEFAULT_PARQUET_COMPRESSION,
+    ) -> None:
+        """Initialize Exporter with specified chunk size.
+
+        Args:
+            chunk_size (int, optional): Number of rows to write at once.
+            Defaults to CHUNK_SIZE_PARQUET.
+            compression (str, optional): Compression codec.
+
+        """
+        if compression not in VALID_PARQUET_COMPRESSION:
+            raise ValueError(f"Invalid compression: {compression}")
+
+        self.chunk_size = chunk_size
+        self.compression = compression
+        self.chunker = DataChunker(chunk_size)
+
+    def _get_compressed_path(self, path: str) -> str:
+        """Insert compression type into filename before extension."""
+        if self.compression == "none":
+            return path
+        base = path
+        for comp in VALID_PARQUET_COMPRESSION:
+            base = base.replace(f".{comp}", "")
+        base = base.replace(".parquet", "")
+        return f"{base}.{self.compression}.parquet"
+
+    def export(
+        self,
+        data: list[dict[str, Any]],
+        output_path: str,
+        schema: dict | None = None,
+    ) -> None:
+        """Export data to a Parquet file.
+
+        Args:
+            data: List of dictionaries containing the data to export
+            output_path: Path where the Parquet file will be written
+            schema: Optional dictionary defining the data schema
+            chunk_size: Number of rows to write at once (default: CHUNK_SIZE_PARQUET)
+
+        """
+        if not data:
+            logger.info("No data to export")
+            raise RuntimeError("No data to export")
+
+        # Validate base path ends with .parquet before adding compression
+        if not output_path.endswith(".parquet"):
+            raise ValueError("Output path must have .parquet extension")
+
+        output_path = self._get_compressed_path(output_path)
+        logger.info(f"Starting Parquet export to {output_path} with {len(data)} rows")
+        if not isinstance(data, list):
+            raise TypeError("Data must be a list of dictionaries")
+
+        writer = None
+        try:
+            for i, chunk in enumerate(self.chunker.chunk_data(data), 1):
+                logger.debug(f"Processing chunk {i}")
+                df = pd.DataFrame(chunk)
+                if "uuid" in df.columns:
+                    df["uuid"] = df["uuid"].astype(str)
+                table = pa.Table.from_pandas(df)
+                # Initialize writer with first chunk's schema
+                if writer is None:
+                    writer = pq.ParquetWriter(
+                        output_path,
+                        table.schema,
+                        compression=self.compression,
+                    )
+                writer.write_table(table)
+            logger.info(f"Successfully exported {len(data)} rows to {output_path}")
+        except Exception as e:
+            raise RuntimeError(f"Failed to write Parquet file: {e}") from e
+        finally:
+            if writer is not None:
+                writer.close()

src/exporters/utils/chunker.py

@@ -0,0 +1,27 @@
+import logging
+from collections.abc import Iterator
+from typing import TypeVar
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+class DataChunker:
+    """Handle data chunking operations."""
+
+    def __init__(self, chunk_size: int) -> None:
+        """Initialize the DataChunker with a specified chunk size."""
+        if chunk_size <= 0:
+            raise ValueError("Chunk size must be greater than zero")
+        self.chunk_size = chunk_size
+
+    def chunk_data(self, data: list[T]) -> Iterator[list[T]]:
+        """Yield chunks of data with memory cleanup."""
+        total_chunks = (len(data) + self.chunk_size - 1) // self.chunk_size
+        for chunk_num, i in enumerate(range(0, len(data), self.chunk_size), 1):
+            chunk = data[i : i + self.chunk_size]
+            logger.debug(f"Processing chunk {chunk_num}/{total_chunks}")
+            yield chunk
+            # Help garbage collection
+            del chunk

src/exporters/utils/constants.py

@@ -0,0 +1,55 @@
+from typing import Literal, cast
+
+# Export related constants
+EXPORT_FORMATS = ["csv", "json", "parquet", "orc", "mysql", "mssql", "oracle"]
+EXPORT_PATTERNS = "^(csv|json|parquet|orc|mysql|mssql|oracle)$"
+
+# Chunk size for exporting data
+CHUNK_SIZE = 10000
+CHUNK_SIZE_JSON = CHUNK_SIZE
+CHUNK_SIZE_CSV = CHUNK_SIZE
+CHUNK_SIZE_PARQUET = CHUNK_SIZE
+CHUNK_SIZE_ORC = CHUNK_SIZE
+
+# ORC compression types
+OrcCompression = Literal["UNCOMPRESSED", "SNAPPY", "ZLIB", "LZ4", "ZSTD"]
+OrcStrategy = Literal["SPEED", "COMPRESSION"]
+
+# Valid compression options
+VALID_ORC_COMPRESSION: list[OrcCompression] = cast(
+    list[OrcCompression], ["UNCOMPRESSED", "SNAPPY", "ZLIB", "LZ4", "ZSTD"]
+)
+VALID_ORC_STRATEGIES: list[OrcStrategy] = cast(
+    list[OrcStrategy], ["SPEED", "COMPRESSION"]
+)
+
+# Default settings
+DEFAULT_ORC_COMPRESSION: OrcCompression = "SNAPPY"
+DEFAULT_ORC_STRATEGY: OrcStrategy = "SPEED"
+
+# Parquet compression types - matching pyarrow's expected values
+ParquetCompression = Literal["none", "snappy", "gzip", "brotli", "lz4", "zstd"]
+
+# Valid compression options for Parquet
+VALID_PARQUET_COMPRESSION: list[ParquetCompression] = cast(
+    list[ParquetCompression], ["none", "snappy", "gzip", "brotli", "lz4", "zstd"]
+)
+
+# Default compression for Parquet
+DEFAULT_PARQUET_COMPRESSION: ParquetCompression = "snappy"
+
+# Default schema for data generation
+DEFAULT_SCHEMA = {
+    "username": {"type": "string", "faker": "name"},
+    "bio": "text",
+    "age": "integer",
+    "emp_number": "bigint",
+    "height": "float",
+    "salary": "decimal",
+    "is_active": "boolean",
+    "signup_date": "date",
+    "signup_dt": "datetime",
+    "profile_picture": "blob",
+    "uuid": "uuid",
+    "user_enum": {"type": "enum", "values": ["active", "inactive", "pending"]},
+}