easy-data-loader 0.1.0__py3-none-any.whl

easy_data_loader/database_connector.py

@@ -0,0 +1,190 @@
+import urllib.parse
+from abc import ABC, abstractmethod
+
+import pyodbc
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import Engine
+
+from .custom_exceptions import EngineTestException
+from .driver_detector import SqlServerDriverDetector
+from .log import LoggedComponent
+from .models import ConnectionSettings, ServerType
+
+pyodbc.pooling = False
+
+
+class DatabaseConnector(ABC):
+    """Abstract class that defines a database connector"""
+
+    @abstractmethod
+    def _build_connection_string(self) -> str:
+        """define the connection string to use when creating the engine - database specific"""
+
+    @abstractmethod
+    def _create_engine(self, settings: ConnectionSettings) -> Engine:
+        """Create an engine to interact with the database"""
+        pass
+
+    @abstractmethod
+    def get_engine(self) -> Engine:
+        """Get database engine by connection name"""
+        pass
+
+
+    @abstractmethod
+    def _test_engine(self) -> bool:
+        """Test a given connection by name"""
+        pass
+
+    @abstractmethod
+    def _dispose_engine(self) -> None:
+        pass
+
+
+
+class SqlServerDatabaseConnector(LoggedComponent, DatabaseConnector):
+    """Database connector to a Sql Server database"""
+
+    def __init__(self, config: ConnectionSettings):
+        super().__init__()
+        self.driver_detector = SqlServerDriverDetector()
+        self.config = config
+        self.engine = self._create_engine(self.config)
+        self._test_engine()
+
+    def _build_connection_string(self) -> str:
+        """Define the connection string from the given configuration"""
+        driver = self.driver_detector.select_preferred_driver()
+
+        if driver is not None and self.config.conn_database is not None:
+            self.logger.debug(f"Sql driver found: {driver}")
+            connection_string = (
+                f"DRIVER={driver};SERVER={self.config.conn_server},{self.config.conn_port};DATABASE={self.config.conn_database}"
+            )
+        else:
+            self.log_and_raise(ValueError, "Connection configuration is not valid")
+
+        if self.config.conn_username is not None and self.config.conn_password is not None:
+            connection_string += f";UID={self.config.conn_username};PWD={self.config.conn_password}"
+        elif self.config.conn_username is None and self.config.conn_password is None:
+            connection_string += ";Trusted_Connection=Yes"
+        else:
+            self.log_and_raise(
+                ValueError,
+                "Credentials definition is not valid. Please check username or password!",
+            )
+
+        if driver == "ODBC Driver 18 for SQL Server":
+            connection_string += ";TrustServerCertificate=Yes"
+
+        connection_string += ";APP=SqlDataLoader"
+        params = urllib.parse.quote_plus(connection_string)
+
+        return f"mssql+pyodbc:///?odbc_connect={params}"
+
+    def _create_engine(self, settings: ConnectionSettings) -> Engine:
+        """Create a sqlalchemy engine using the provided configuration"""
+        connection_string = self._build_connection_string()
+        if connection_string:
+            try:
+                engine = create_engine(
+                    connection_string,
+                    fast_executemany=True,
+                    pool_size=5,
+                    max_overflow=10,
+                    pool_timeout=30,
+                    pool_recycle=3600,
+                    echo=self.is_debug_enabled,
+                )
+                return engine
+            except Exception as e:
+                self.log_exception(e, "Could not create engine from connection string")
+                raise
+        else:
+            self.log_and_raise(
+                ValueError, "Connection string could not be created from configuration"
+            )
+
+    def get_engine(self) -> Engine:
+        """Return the engine to be used"""
+        return self.engine
+
+    def _test_engine(self) -> bool:
+        """Test if the connection works"""
+        try:
+            with self.engine.connect() as conn:
+                conn.execute(text("SELECT 1"))
+            self.logger.debug(f"Connection test successful: {self.config.conn_server} - {self.config.conn_database}")
+            return True
+        except Exception as e:
+            self.log_and_raise(
+                EngineTestException,
+                f"Connection test failed",
+                exception=str(e),
+            )
+            return False
+
+    def _dispose_engine(self) -> None:
+        self.logger.debug('Disposing of engine')
+        return self.engine.dispose()
+
+
+class SQLiteDatabaseConnector(LoggedComponent, DatabaseConnector):
+    """Database connector to a SQLite database"""
+
+    def __init__(self, config: ConnectionSettings):
+        super().__init__()
+        self.config = config
+        self.engine = self._create_engine(self.config)
+        self._test_engine()
+
+    def _build_connection_string(self) -> str:
+        """Define the connection string from the given configuration"""
+        if self.config.conn_server:
+            # For SQLite, conn_server is treated as the file path
+            return f"sqlite:///{self.config.conn_server}"
+        self.log_and_raise(ValueError, "Connection configuration is not valid. SQLite requires a file path in conn_server.")
+
+    def _create_engine(self, settings: ConnectionSettings) -> Engine:
+        """Create a sqlalchemy engine using the provided configuration"""
+        connection_string = self._build_connection_string()
+        try:
+            # SQLite specific engine creation
+            engine = create_engine(
+                connection_string,
+                echo=self.is_debug_enabled,
+            )
+            return engine
+        except Exception as e:
+            self.log_exception(e, "Could not create engine from connection string")
+            raise
+
+    def get_engine(self) -> Engine:
+        """Return the engine to be used"""
+        return self.engine
+
+    def _test_engine(self) -> bool:
+        """Test if the connection works"""
+        try:
+            with self.engine.connect() as conn:
+                conn.execute(text("SELECT 1"))
+            self.logger.info(f"Connection test successful: {self.config.conn_server} - {self.config.conn_database}")
+            return True
+        except Exception as e:
+            self.log_and_raise(
+                EngineTestException,
+                f"Connection test failed",
+                exception=str(e),
+            )
+            return False
+
+    def _dispose_engine(self) -> None:
+        self.logger.debug('Disposing of engine')
+        return self.engine.dispose()
+
+
+CONNECTOR_FACTORY = {
+    ServerType.MSSQL: SqlServerDatabaseConnector,
+    ServerType.SQLITE: SQLiteDatabaseConnector,
+    # ServerType.POSTGRESQL: PostgresDatabaseConnector
+}

easy_data_loader/database_operations.py

@@ -0,0 +1,129 @@
+from sqlalchemy import inspect, text, Table, Column, MetaData
+from sqlalchemy.engine import Engine
+from sqlalchemy.types import TypeEngine
+from pandas import DataFrame
+import pandas as pd
+from typing import Dict, Any, Optional
+from .log import LoggedComponent
+
+class DatabaseOperations(LoggedComponent):
+    """Component responsible for all SQL interactions using SQLAlchemy engines"""
+
+    def __init__(self, engine: Engine):
+        super().__init__()
+        self.engine = engine
+        self._inspector = inspect(self.engine)
+
+    def write_to_table(self, table_name: str, df: DataFrame, **kwargs) -> bool:
+        """Write a dataframe to a specified table in the database"""
+
+        self.logger.info(f"Writing {len(df)} rows to table: {table_name}")
+        try:
+            df.to_sql(table_name, con=self.engine, **kwargs)
+            return True
+        except Exception as e:
+            self.log_exception(e, f"Failed to write to table {table_name}")
+            raise
+
+    def read_data(self, sql: str, **kwargs) -> DataFrame:
+        """Read a specified table from the database into a dataframe"""
+
+        self.logger.debug(f"Reading data.")
+        try:
+            return pd.read_sql(sql, con=self.engine, **kwargs)
+        except Exception as e:
+            self.log_exception(e, f"Failed to read data")
+            raise
+
+    def inspect_table(self, table_name: str) -> dict:
+        """Read the metadata (columns and types) of a specified table"""
+
+        self.logger.debug(f"Inspecting metadata for table: {table_name}")
+        try:
+            columns = self._inspector.get_columns(table_name)
+            if not columns:
+                self.logger.warning(f"Table {table_name} not found or has no columns.")
+                return {}
+
+            return {col['name']: str(col['type']) for col in columns}
+        except Exception as e:
+            self.log_exception(e, f"Failed to inspect table {table_name}")
+            return {}
+
+    def create_table(self, table_name: str, schema: Dict[str, TypeEngine]) -> bool:
+        """
+        Create a table dynamically.
+        schema: {'col_name': sqlalchemy_type}
+        """
+        self.logger.info(f"Creating table: {table_name}")
+        try:
+            metadata = MetaData()
+            columns = [Column(name, col_type) for name, col_type in schema.items()]
+            table = Table(table_name, metadata, *columns)
+
+            metadata.create_all(self.engine)
+            return True
+        except Exception as e:
+            self.log_exception(e, f"Failed to create table {table_name}")
+            return False
+
+    def execute_stored_procedure(self, procedure_name: str, **kwargs) -> bool:
+        """Execute a stored procedure using the engine connection"""
+        self.logger.info(f"Executing stored procedure: {procedure_name}")
+        try:
+            params_str = ", ".join([f":{k}" for k in kwargs.keys()])
+            sql = text(f"EXEC {procedure_name} {params_str}")
+
+            with self.engine.begin() as conn:
+                # self.engine.begin() is a context manager for the transaction
+                # if no error occurs the transaction gets commited
+                conn.execute(sql, kwargs)
+            return True
+        except Exception as e:
+            self.log_exception(e, f"Failed to execute procedure {procedure_name}")
+            raise
+
+    def write_audit(self, table_name: str, entry: 'AuditEntry'):
+        """Write an audit entry to the database, ensuring the table exists first."""
+        self.logger.debug(f"Writing audit entry for execution: {entry.execution_id}")
+
+        try:
+            from sqlalchemy import Table, Column, String, Integer, BigInteger, DateTime, MetaData
+            metadata = MetaData()
+
+            columns = [
+                Column("execution_id", String(50), primary_key=True),
+                Column("pipeline_name", String(100)),
+                Column("status", String(50)),
+                Column("input_rows", Integer),
+                Column("output_rows", Integer),
+                Column("source_name", String(255)),
+                Column("destination_name", String(255)),
+                Column("file_name", String(255)),
+                Column("file_path", String(500)),
+                Column("file_size_bytes", BigInteger),
+                Column("file_last_modified", DateTime),
+                Column("sp_name", String(255)),
+                Column("sp_parameters", String),
+                Column("timestamp", DateTime),
+                Column("error_details", String)
+            ]
+
+            table = Table(table_name, metadata, *columns)
+            metadata.create_all(self.engine)
+
+            data = entry.model_dump()
+            # Serialize dictionary fields for storage
+            if data.get("sp_parameters"):
+                import json
+                try:
+                    data["sp_parameters"] = json.dumps(data["sp_parameters"], default=str)
+                except:
+                    data["sp_parameters"] = str(data["sp_parameters"])
+            df = pd.DataFrame([data])
+
+            df.to_sql(table_name, con=self.engine, if_exists="append", index=False)
+
+        except Exception as e:
+            self.log_exception(e, f"Failed to write audit entry to {table_name}")
+            raise

easy_data_loader/driver_detector.py

@@ -0,0 +1,46 @@
+from abc import ABC, abstractmethod
+
+import pyodbc
+
+from .custom_exceptions import DriverNotFoundException
+from .log import LoggedComponent
+
+
+class DriverDetector(ABC):
+    @abstractmethod
+    def get_available_drivers(self) -> list[str]:
+        """Get list of available drivers."""
+        pass
+
+    @abstractmethod
+    def select_preferred_driver(self) -> str:
+        """Find the best available driver."""
+        pass
+
+
+class SqlServerDriverDetector(LoggedComponent, DriverDetector):
+    def __init__(self):
+        super().__init__()
+        self._preferred_drivers = [
+            "ODBC Driver 18 for SQL Server",
+            "ODBC Driver 17 for SQL Server",
+        ]
+
+    def get_available_drivers(self) -> list[str]:
+        """Get all available ODBC drivers."""
+        return pyodbc.drivers()
+
+    def select_preferred_driver(self) -> str:
+        """Select first detected driver in order of preferrence."""
+
+        available_drivers = self.get_available_drivers()
+
+        for driver in self._preferred_drivers:
+            if driver in available_drivers:
+                return driver
+
+        self.log_and_raise(
+            DriverNotFoundException,
+            available_drivers=available_drivers,
+            preferred_drivers=self._preferred_drivers,
+        )

easy_data_loader/file_operations.py

@@ -0,0 +1,146 @@
+import pandas as pd
+from pathlib import Path
+from typing import Callable, Dict, Optional
+from .log import LoggedComponent
+from .models import FileType, FileSettings
+
+class FileOperations(LoggedComponent):
+    """Handles all file operations"""
+
+    def __init__(self, settings: FileSettings):
+        super().__init__()
+        self.settings = settings
+        self.file_type = self.settings.file_type
+        self.file_path : Optional[Path] = None
+
+        # mapping the file reader by extension
+        self._readers: Dict[FileType, Callable] = {
+            FileType.CSV : pd.read_csv,
+            FileType.EXCEL : pd.read_excel,
+            FileType.PARQUET: pd.read_parquet,
+            FileType.ORC : pd.read_orc
+        }
+
+        # mapping the writters to FileType
+        self._writers : Dict[FileType, Callable] = {
+            FileType.CSV : self._write_csv,
+            FileType.EXCEL : self._write_excel,
+            FileType.PARQUET : self._write_parquet,
+            FileType.ORC : self._write_orc,
+        }
+
+    def _find_file(self) -> Path:
+        """
+        Identifies the file based on pattern (latest) or explicit name.
+        """
+        if self.settings.file_pattern:
+            files = list(self.settings.folder_path.glob(self.settings.file_pattern))
+            if not files:
+                self.log_and_raise(ValueError, f'No files found by pattern {self.settings.file_pattern} inside folder {self.settings.folder_path}')
+            latest_file = max(files, key=lambda f: f.stat().st_mtime)
+            return latest_file
+        elif self.settings.file_name:
+            file = (self.settings.folder_path / self.settings.file_name).with_suffix(f'.{self.settings.file_type.value.lower()}')
+            if not file.exists():
+                self.log_and_raise(ValueError, 'Error finding file, please check the file definition.')
+            return file
+        self.log_and_raise(ValueError, 'Error finding file by name or pattern')
+
+    def read_file(self, **kwargs) -> pd.DataFrame:
+        """
+        Entrypoint for reading files to a pandas DataFrame.
+        Depending on the file type it will delegate to the coresponding reader.
+        """
+        if not self.file_path:
+            self.file_path = self._find_file()
+
+        try:
+            self.logger.info(f'Reading file: {self.file_path.name}')
+            return self._readers[self.file_type](self.file_path, **kwargs)
+        except Exception as e:
+            self.log_exception(e, f'Failed to read {self.file_path.name} into a dataframe')
+            raise
+
+    def write_file(self, df: pd.DataFrame, **kwargs) -> Path:
+        """
+        Ensures a valid path exists and writes the dataframe.
+        If file_name is provided, it uses it (and overwrites).
+        If file_pattern is provided, it constructs a name using the pattern and a timestamp.
+        """
+
+        output_path = self._construct_output_path()
+        try:
+            output_path.parent.mkdir(parents=True, exist_ok=True)
+            output_path.touch(exist_ok=True)
+
+            self.logger.info(f"Writing to file: {output_path.name}")
+            self._writers[self.file_type](df, output_path, **kwargs)
+
+            return output_path
+        except Exception as e:
+            self.log_exception(e, f'Failed to write to: {output_path}')
+            raise
+
+    def _construct_output_path(self) -> Path:
+        """
+        Construct the output file path based on the settings.
+        If the file name is defined the output path will be fixed,
+        if the file pattern is defined the file name will have a timestamp appended.
+        """
+        timestamp = pd.Timestamp.now().strftime('%Y%m%d_%H%M%S')
+        file_extension = f'.{self.settings.file_type.value.lower()}'
+
+        if self.settings.file_name:
+            return (self.settings.folder_path / self.settings.file_name).with_suffix(file_extension)
+        return self.settings.folder_path / f"{self.settings.file_pattern}_{timestamp}{file_extension}"
+
+    def _apply_file_preprocessor(self, preprocessor_func: Callable[[Path], Path]):
+        """
+        Executes a custom function passed from the BasePipelineDefinition
+        and returns the new file path
+        """
+
+        current_path = self._find_file()
+        self.logger.info(f'Current file path is: {current_path}')
+
+        if preprocessor_func is None:
+             self.file_path = current_path
+             return
+
+        try:
+            self.logger.info(f"Running pre-processor on: {current_path}")
+            # run pre-processor function and record the path - old or new
+            processed_path = preprocessor_func(current_path)
+
+            if isinstance(processed_path, Path):
+                self.file_path = processed_path
+                self.logger.debug(f'File path updated to: {self.file_path}')
+            else:
+                self.logger.warning(f'Pre-processor function did not return a new Path')
+                self.file_path = current_path
+
+        except Exception as e:
+            self.log_exception(e, f'Pre-processor failed for : {current_path}')
+            # In case of failure, keep the original path so the pipeline can try to proceed
+            self.file_path = current_path
+
+    def _write_csv(self, df: pd.DataFrame, file_path: Path, **kwargs):
+        """Internal CSV writer"""
+        df.to_csv(file_path, **kwargs)
+
+    def _write_excel(self, df: pd.DataFrame, file_path: Path, **kwargs):
+        """Internal Excel writer"""
+
+        df.to_excel(file_path, **kwargs)
+
+    def _write_parquet(self, df: pd.DataFrame, file_path: Path, **kwargs):
+        """Internal Parquet writer"""
+
+        df.to_parquet(file_path, **kwargs)
+
+    def _write_orc(self, df: pd.DataFrame, file_path: Path, **kwargs):
+        """Internal Orc writer"""
+
+        df.to_orc(file_path, **kwargs)
+
+

easy_data_loader/log.py

@@ -0,0 +1,90 @@
+import logging
+import logging.handlers
+from pathlib import Path
+
+
+class AppLogger:
+    _instance = None
+    _initialized = False
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self):
+        if not self._initialized:
+            self._setup_logging()
+            AppLogger._initialized = True
+
+    def _setup_logging(self):
+        Path("logs").mkdir(exist_ok=True)
+
+        formatter = logging.Formatter(
+            "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+
+        root_logger = logging.getLogger()
+        root_logger.setLevel(logging.INFO)
+        root_logger.handlers.clear()
+
+        # Console
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(formatter)
+        root_logger.addHandler(console_handler)
+
+        # File
+        file_handler = logging.handlers.RotatingFileHandler(
+            "logs/application.log", maxBytes=10 * 1024 * 1024, backupCount=5
+        )
+        file_handler.setFormatter(formatter)
+        root_logger.addHandler(file_handler)
+
+    def get_logger(self, name: str) -> logging.Logger:
+        return logging.getLogger(name)
+
+    def set_level(self, level: str):
+        """Change log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)"""
+
+        log_level = getattr(logging, level.upper())
+        logging.getLogger().setLevel(log_level)
+
+        # Update all handlers
+        for handler in logging.getLogger().handlers:
+            handler.setLevel(log_level)
+
+    @property
+    def is_debug(self) -> bool:
+        return logging.getLogger().isEnabledFor(logging.DEBUG)
+
+
+class LoggedComponent:
+    """Base class providing logging functionality to all components"""
+
+    def __init__(self):
+        self.log = AppLogger()
+        self.logger = self.log.get_logger(self.__class__.__name__)
+
+    def log_and_raise(self, exception_class, message: str = "", **context):
+        """Log error message and raise exception with optional context"""
+        if context:
+            context_str = ", ".join(f"{k}={v}" for k, v in context.items())
+            self.logger.error(f"{message} | Context: {context_str}")
+        else:
+            self.logger.error(message)
+
+        raise exception_class(message)
+
+    def log_exception(self, exc: Exception, message: str = "", **context):
+        """Log a caught exception with optional context"""
+        log_msg = message or f"Exception occurred: {str(exc)}"
+        if context:
+            context_str = ", ".join(f"{k}={v}" for k, v in context.items())
+            log_msg += f" | Context: {context_str}"
+
+        self.logger.error(log_msg, exc_info=True)
+
+    @property
+    def is_debug_enabled(self) -> bool:
+        return self.log.is_debug