easy-data-loader 0.1.8__tar.gz → 0.2.1__tar.gz

{easy_data_loader-0.1.8/src/easy_data_loader.egg-info → easy_data_loader-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easy_data_loader
-Version: 0.1.8
+Version: 0.2.1
 Summary: Data transfer utilities between files and databases
 Author-email: Bojoi Gabriel <bojoigabriel@gmail.com>
 Classifier: Development Status :: 3 - Alpha

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "easy_data_loader"
-version = "0.1.8"
+version = "0.2.1"
 description = "Data transfer utilities between files and databases"
 authors = [{ name = "Bojoi Gabriel", email = "bojoigabriel@gmail.com" }]
 readme = "README.md"

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/src/easy_data_loader/cli.py

@@ -412,6 +412,9 @@ def validate_pipelines():
             else:
                 raise ValueError(f"Unknown pipeline type: {type(definition)}")
 
+            if instance is not None and instance._init_error is not None:
+                raise ValueError(instance._init_error.message)
+
             results[name] = "OK"
         except Exception as e:
             results[name] = f"FAILED: {str(e)}"

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/src/easy_data_loader/custom_exceptions.py

@@ -26,3 +26,27 @@ class PipelineValidationError(Exception):
     def __init__(self, message: str):
         self.message = message
         super().__init__(self.message)
+
+
+class EmptyDataError(Exception):
+    def __init__(self, message: str):
+        self.message = message
+        super().__init__(self.message)
+
+
+class NoValidDestination(Exception):
+    def __init__(self, message: str = "The pipeline destination is not valid"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class NestedPipelinesError(Exception):
+    def __init__(self, message: str = "Nested pipelines are not supported"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class UnknownPipelineError(Exception):
+    def __init__(self, message: str = "Unknown pipeline type"):
+        self.message = message
+        super().__init__(self.message)

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/src/easy_data_loader/data_inferrence.py

@@ -88,7 +88,7 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                     self.logger.error(
                         f"Failed to infer dtype for column '{col}': {str(e)}. "
                         f"Column will use default inference.",
-                        exc_info=self.is_debug_enabled,
+                        exc_info=True,
                     )
                     # Skip this column - it will use default inference
                     continue
@@ -102,9 +102,8 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
 
             self.logger.info(f"Dtype inference completed for {len(dtype_dict)} columns")
 
-            # Log summary if debug enabled
-            if self.is_debug_enabled:
-                self._log_inference_summary(dtype_dict)
+            # Only visible in the file log, since it's logged at debug level
+            self._log_inference_summary(dtype_dict)
 
             return dtype_dict
 
@@ -593,7 +592,7 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                     self.logger.error(
                         f"Failed to infer dtype for column '{col_name}' from Parquet metadata: {str(e)}. "
                         f"Column will use default inference.",
-                        exc_info=self.is_debug_enabled,
+                        exc_info=True,
                     )
                     continue
 
@@ -616,7 +615,7 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                             self.logger.error(
                                 f"Failed to analyze string length for column '{col}': {str(e)}. "
                                 f"Column will use default inference.",
-                                exc_info=self.is_debug_enabled,
+                                exc_info=True,
                             )
                             # Remove from dict so it uses default
                             dtype_dict.pop(col, None)
@@ -626,7 +625,7 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                     self.logger.error(
                         f"Failed to read string columns from Parquet: {str(e)}. "
                         f"String columns will use default inference.",
-                        exc_info=self.is_debug_enabled,
+                        exc_info=True,
                     )
                     # Remove string columns from dict
                     for col in string_columns:
@@ -643,8 +642,8 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                 f"Parquet dtype inference completed for {len(dtype_dict)} columns"
             )
 
-            if self.is_debug_enabled:
-                self._log_inference_summary(dtype_dict)
+            # Only visible in the file log, since it's logged at debug level
+            self._log_inference_summary(dtype_dict)
 
             return dtype_dict
 
@@ -703,7 +702,7 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                     self.logger.error(
                         f"Failed to infer dtype for column '{col_name}' from ORC metadata: {str(e)}. "
                         f"Column will use default inference.",
-                        exc_info=self.is_debug_enabled,
+                        exc_info=True,
                     )
                     continue
 
@@ -726,7 +725,7 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                             self.logger.error(
                                 f"Failed to analyze string length for column '{col}': {str(e)}. "
                                 f"Column will use default inference.",
-                                exc_info=self.is_debug_enabled,
+                                exc_info=True,
                             )
                             dtype_dict.pop(col, None)
                             continue
@@ -735,7 +734,7 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                     self.logger.error(
                         f"Failed to read string columns from ORC: {str(e)}. "
                         f"String columns will use default inference.",
-                        exc_info=self.is_debug_enabled,
+                        exc_info=True,
                     )
                     for col in string_columns:
                         dtype_dict.pop(col, None)
@@ -751,8 +750,8 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                 f"ORC dtype inference completed for {len(dtype_dict)} columns"
             )
 
-            if self.is_debug_enabled:
-                self._log_inference_summary(dtype_dict)
+            # Only visible in the file log, since it's logged at debug level
+            self._log_inference_summary(dtype_dict)
 
             return dtype_dict
 

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/src/easy_data_loader/database_connector.py

@@ -107,7 +107,7 @@ class SqlServerDatabaseConnector(LoggedComponent, DatabaseConnector):
                     max_overflow=10,
                     pool_timeout=30,
                     pool_recycle=3600,
-                    echo=self.is_debug_enabled,
+                    echo=False,
                 )
                 return engine
             except Exception as e:
@@ -183,7 +183,7 @@ class SQLiteDatabaseConnector(LoggedComponent, DatabaseConnector):
             # SQLite specific engine creation
             engine = create_engine(
                 connection_string,
-                echo=self.is_debug_enabled,
+                echo=False,
             )
             return engine
         except Exception as e:
@@ -280,7 +280,7 @@ class PostgresDatabaseConnector(LoggedComponent, DatabaseConnector):
                 max_overflow=10,
                 pool_timeout=30,
                 pool_recycle=3600,
-                echo=self.is_debug_enabled,
+                echo=False,
             )
             return engine
         except Exception as e:

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/src/easy_data_loader/database_operations.py

@@ -19,18 +19,20 @@ class DatabaseOperations(LoggedComponent):
         self.engine = engine
         self._inspector = inspect(self.engine)
 
-    def write_to_table(
-        self, table_name: Optional[str], df: DataFrame, **kwargs
-    ) -> bool:
+    def write_to_table(self, table_name: Optional[str], df: DataFrame, **kwargs):
         """Write a dataframe to a specified table in the database"""
 
-        self.logger.info(f"Writing {len(df)} rows to table: {table_name}")
+        self.logger.debug(f"Writing {len(df)} rows to table: {table_name}")
         try:
             if table_name:
                 df.to_sql(table_name, con=self.engine, **kwargs)
-            return True
+                self.logger.debug("Finalized write to table: {table_name}")
+            return
         except Exception as e:
-            self.log_exception(e, f"Failed to write to table {table_name}")
+            self.logger.debug(
+                f"Failed to write to table {table_name} caused by the following exception: {e}",
+                exc_info=True,
+            )
             raise
 
     def read_data(self, sql: str, **kwargs) -> DataFrame:
@@ -151,7 +153,7 @@ class DatabaseOperations(LoggedComponent):
                 Column("file_last_modified", DateTime),
                 Column("sp_name", String(255)),
                 Column("sp_parameters", String),
-                Column("timestamp", DateTime),
+                Column("insert_timestamp", DateTime),
                 Column("error_details", String),
             ]
 

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/src/easy_data_loader/file_operations.py

@@ -48,7 +48,9 @@ class FileOperations(LoggedComponent):
         Identifies the file based on pattern (latest) or explicit name.
         """
         if self.settings.file_pattern:
-            files = list(self.settings.folder_path.glob('*' + self.settings.file_pattern + '*'))
+            files = list(
+                self.settings.folder_path.glob("*" + self.settings.file_pattern + "*")
+            )
             if not files:
                 self.log_and_raise(
                     ValueError,
@@ -132,7 +134,7 @@ class FileOperations(LoggedComponent):
         """
 
         current_path = self._find_file()
-        self.logger.info(f"Current file path is: {current_path}")
+        self.logger.debug(f"Current file path is: {current_path}")
 
         if preprocessor_func is None:
             self.file_path = current_path

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/src/easy_data_loader/log.py

@@ -21,25 +21,37 @@ class AppLogger:
     def _setup_logging(self):
         Path("logs").mkdir(exist_ok=True)
 
-        formatter = logging.Formatter(
+        file_formatter = logging.Formatter(
             "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
             datefmt="%Y-%m-%d %H:%M:%S",
         )
 
+        console_formatter = logging.Formatter(
+            "[%(asctime)s]-[%(levelname)s] %(message)s"
+        )
+
         root_logger = logging.getLogger()
-        root_logger.setLevel(logging.INFO)
+        root_logger.setLevel(
+            logging.DEBUG
+        )  # allow all messages and split them in the handlers
         root_logger.handlers.clear()
 
         # Console
         console_handler = logging.StreamHandler()
-        console_handler.setFormatter(formatter)
+        console_handler.setLevel(
+            logging.INFO
+        )  # the console log will have minimal friendly information
+        console_handler.setFormatter(console_formatter)
         root_logger.addHandler(console_handler)
 
         # File
         file_handler = logging.handlers.RotatingFileHandler(
             "logs/application.log", maxBytes=10 * 1024 * 1024, backupCount=5
         )
-        file_handler.setFormatter(formatter)
+        file_handler.setFormatter(file_formatter)
+        file_handler.setLevel(
+            logging.DEBUG
+        )  # the file log will have full detailed information
         root_logger.addHandler(file_handler)
 
     def get_logger(self, name: str) -> logging.Logger:
@@ -55,10 +67,6 @@ class AppLogger:
         for handler in logging.getLogger().handlers:
             handler.setLevel(log_level)
 
-    @property
-    def is_debug(self) -> bool:
-        return logging.getLogger().isEnabledFor(logging.DEBUG)
-
 
 class LoggedComponent:
     """Base class providing logging functionality to all components"""
@@ -85,7 +93,3 @@ class LoggedComponent:
             log_msg += f" | Context: {context_str}"
 
         self.logger.error(log_msg, exc_info=True)
-
-    @property
-    def is_debug_enabled(self) -> bool:
-        return self.log.is_debug

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/src/easy_data_loader/models.py

@@ -191,7 +191,7 @@ class BasePipelineDefinition(BaseModel):
     @property
     def destination_table_invalid(self) -> Optional[str]:
         if self.destination_table:
-            return f"{self.destination_table}_invalid"
+            return "edl_validation_error_log"
         return None
 
     def file_pre_process(self, file_path: Path) -> Path:
@@ -282,7 +282,7 @@ class AuditEntry(BaseModel):
     error_details: Optional[str] = None
     sp_name: Optional[str] = None
     sp_parameters: Optional[Dict[str, Any]] = None
-    timestamp: pd.Timestamp = Field(default_factory=pd.Timestamp.now)
+    insert_timestamp: pd.Timestamp = Field(default_factory=pd.Timestamp.now)
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/src/easy_data_loader/orchestrator.py

@@ -11,35 +11,55 @@ from .models import (
 )
 from .pipeline import LoadPipeline
 from .procedure_pipeline import ProcedurePipeline
+from .custom_exceptions import NestedPipelinesError, UnknownPipelineError
+from .pipeline_base import PipelineResult
 
 
 class OrchestratorPipeline(LoggedComponent):
-    """Executes a chain of pipelines defined by an OrchestratorDefinition"""
+    """Executes a chain of pipelines defined by a OrchestratorDefinition"""
 
-    def __init__(self, pipeline_name: str, orchestrator_id: Optional[str] = None):
+    def __init__(self, file_name: str, orchestrator_id: Optional[str] = None):
         super().__init__()
         self.orchestrator_id = orchestrator_id if orchestrator_id else str(uuid4())
         self.config = Configuration()
-        definition = self.config.get_pipeline(pipeline_name)
+        definition = self.config.get_pipeline(file_name)
 
         if not isinstance(definition, OrchestratorDefinition):
-            self.log_and_raise(ValueError, f"'{pipeline_name}' is not an orchestrator.")
+            raise ValueError(
+                f"'{file_name}' does not contain a OrchestratorPipeline definition."
+            )
 
         self.definition: OrchestratorDefinition = definition
 
-    def run(self) -> bool:
+    def run(self) -> PipelineResult:
+        """Entry point mirroring BasePipeline.run(), without audit/connector lifecycle."""
+        try:
+            success = self._run()
+            return PipelineResult(
+                success=success,
+                message="Orchestrator executed successfully."
+                if success
+                else "One or more pipelines failed.",
+            )
+        except Exception as e:
+            self.logger.warning(f"Orchestrator has [FAILED] {e}")
+            return PipelineResult(
+                success=False,
+                message="Orchestrator has failed",
+                error_code=e.__class__.__name__,
+                exception=e,
+            )
+
+    def _run(self) -> bool:
         self.logger.info(
             f"=== Starting Orchestrator: {self.definition.pipeline_name} (Orchestrator: {self.orchestrator_id}) ==="
         )
 
         success = True
         for pipeline_name in self.definition.pipelines:
-            self.logger.info(
-                f"[{self.definition.pipeline_name}] -> Triggering pipeline: {pipeline_name}"
-            )
+            self.logger.info(f"Triggering pipeline: {pipeline_name}")
 
             p_def: PipelineType = self.config.get_pipeline(pipeline_name)
-            p_success = False
 
             # Instantiate and run
             if isinstance(p_def, BasePipelineDefinition):
@@ -51,25 +71,17 @@ class OrchestratorPipeline(LoggedComponent):
                     pipeline_name, orchestrator_id=self.orchestrator_id
                 ).run()
             elif isinstance(p_def, OrchestratorDefinition):
-                self.logger.error(
-                    f"[{self.definition.pipeline_name}] -> Nested orchestrators are not supported."
-                )
-                p_success = False
+                # caught by run() and turned into a failed PipelineResult
+                raise NestedPipelinesError
             else:
-                self.logger.error(
-                    f"[{self.definition.pipeline_name}] -> Unknown pipeline type for '{pipeline_name}'"
-                )
-                p_success = False
+                # caught by run() and turned into a failed PipelineResult
+                raise UnknownPipelineError
 
             if not p_success:
                 success = False
-                self.logger.error(
-                    f"[{self.definition.pipeline_name}] -> Pipeline failed: {pipeline_name}"
-                )
+                self.logger.error(f"Pipeline failed: {pipeline_name}")
                 if self.definition.fail_fast:
-                    self.logger.error(
-                        f"[{self.definition.pipeline_name}] -> Fail fast enabled. Stopping orchestrator."
-                    )
+                    self.logger.error("Fail fast enabled. Stopping orchestrator.")
                     break
             else:
                 self.logger.info(

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/src/easy_data_loader/pipeline.py

@@ -14,8 +14,12 @@ from .models import (
     FileBasedConnectionSettings,
     FileType,
 )
-from .custom_exceptions import PipelineValidationError
-from .pipeline_base import BasePipeline
+from .custom_exceptions import (
+    PipelineValidationError,
+    EmptyDataError,
+    NoValidDestination,
+)
+from .pipeline_base import BasePipeline, PipelineResult
 
 
 class LoadPipeline(BasePipeline):
@@ -25,13 +29,13 @@ class LoadPipeline(BasePipeline):
     Inherits shared logic from BasePipeline.
     """
 
-    def __init__(self, pipeline_name: str, orchestrator_id: Optional[str] = None):
+    def __init__(self, file_name: str, orchestrator_id: Optional[str] = None):
 
         # Load definition from config
-        definition = Configuration().get_pipeline(pipeline_name)
+        definition = Configuration().get_pipeline(file_name)
         if not isinstance(definition, BasePipelineDefinition):
             raise ValueError(
-                f"Pipeline '{pipeline_name}' is not a LoadPipeline definition."
+                f"Pipeline '{file_name}' does not contain a LoadPipeline definition."
             )
 
         super().__init__(definition, orchestrator_id=orchestrator_id)
@@ -44,7 +48,18 @@ class LoadPipeline(BasePipeline):
         self.dst_file_ops: Optional[FileOperations] = None
         self.source_file_path: Optional[Path] = None  # For auditing
 
-        self._initialize_components()
+        if self._init_error is None:
+            try:
+                self._initialize_components()
+            except Exception as e:
+                # deferred so run() can report it as a PipelineResult instead of
+                # raising out of the constructor
+                self._init_error = PipelineResult(
+                    success=False,
+                    message=f"Pipeline setup failed: {e}",
+                    error_code=e.__class__.__name__,
+                    exception=e,
+                )
 
     def _initialize_components(self):
         """Dynamically initialize the components based on their type and definition"""
@@ -86,40 +101,37 @@ class LoadPipeline(BasePipeline):
         ):
             self.dst_db_ops = self._setup_db_operations(destination_resource)
             self.destination_name = destination_resource.conn_database
+        else:
+            raise ValueError(
+                f"Unsupported destination resource type: {type(source_resource)}"
+            )
 
-    def run(self) -> bool:
+    def _run(self):
         """Executes the entire ETL flow"""
-        self.logger.info(
-            f">>> Starting Load Pipeline: {self.definition.pipeline_name} <<<"
-        )
 
         try:
-            # 1. EXTRACT
+            # 1. READ
             df, inferred_dtypes = self._extract_step()
             if df.empty:
-                self.logger.warning("No data found! Pipeline will stop!")
-                return False
+                self.error_details = "No data found when trying to read from source!"
+                self._log_audit("FAILED")
+                raise EmptyDataError(self.error_details)
 
             self.input_rows = len(df)
+            self.logger.info(f"Rows read from source: {self.input_rows}")
 
             # 2. TRANSFORM & VALIDATE
             df, invalid_df = self._transform_step(df)
             self.output_rows = len(df)
             if df.empty:
-                self.logger.warning(
-                    "No valid data remaining after validation! Pipeline will stop gracefully."
+                self.error_details = (
+                    "After validation no data remaining - all data is invalid."
                 )
-                self._log_audit("SUCCESS")
-                return True
+                self._log_audit("FAILED")
+                raise EmptyDataError(self.error_details)
 
             # 3. LOAD
-            load_success = self._load_step(df, invalid_df, inferred_dtypes)
-            if not load_success:
-                self.logger.error(
-                    f">>> Pipeline {self.definition.pipeline_name} failed to load during the LOAD step."
-                )
-                self._log_audit("FAILED")
-                return False
+            self._load_step(df, invalid_df, inferred_dtypes)
 
             self._log_audit("SUCCESS")
             self.logger.info(
@@ -133,14 +145,14 @@ class LoadPipeline(BasePipeline):
                 f"Critical pipeline error - {self.definition.pipeline_name}: {str(e)}"
             )
             self._log_audit("FAILED")
-            return False
+            raise
         except Exception as e:
             self.error_details = str(e)
             self.log_exception(
                 e, f"Critical pipeline error - {self.definition.pipeline_name}"
             )
             self._log_audit("FAILED")
-            return False
+            raise
         finally:
             self._cleanup()
 
@@ -156,15 +168,23 @@ class LoadPipeline(BasePipeline):
         try:
             if self.src_db_ops:  # DB source
                 if self.definition.source_sql:
+                    self.logger.debug("<< reading data from database >>")
                     df = self.src_db_ops.read_data(
                         self.definition.source_sql, **self.definition.read_parameters
                     )
-                    return df, {}  # No dtype inference for DB sources
+                    dtype_map = self.definition.get_dtype_map()
+                    self.logger.debug("<< identifying columns definition >>")
+                    if not dtype_map:
+                        self.logger.debug("<< no columns definition identified >>")
+                        dtype_map = {}
+                    return df, dtype_map
 
             if self.src_file_ops:  # File source
+                self.logger.debug("<< source is a file, atempting pre-processing >>")
                 self.src_file_ops._apply_file_preprocessor(
                     self.definition.file_pre_process
                 )
+                self.logger.debug("<< reading from file source >>")
                 df = self.src_file_ops.read_file(**self.definition.read_parameters)
                 self.source_file_path = self.src_file_ops.file_path
 
@@ -212,7 +232,8 @@ class LoadPipeline(BasePipeline):
         rename_map = self.definition.get_rename_map()
         if rename_map:
             df.rename(columns=rename_map, inplace=True)
-            self.logger.info(f"Columns renamed: {list(rename_map.values())}")
+            self.logger.info("Columns renamed")
+            self.logger.debug(f"Columns renamed: {rename_map}")
 
         # 2. Pipeline hook transformation
         df = self.definition.transform(df)
@@ -307,10 +328,13 @@ class LoadPipeline(BasePipeline):
         df: pd.DataFrame,
         invalid: pd.DataFrame,
         dtype_map: Dict[str, types.TypeEngine],
-    ) -> bool:
+    ) -> None:
         """Handles loading logic based on destination type."""
         try:
             if self.dst_db_ops and self.definition.destination_table:  # DB destination
+                self.logger.info(
+                    f"Writting to table {self.definition.destination_table}"
+                )
                 self.dst_db_ops.write_to_table(
                     table_name=self.definition.destination_table,
                     df=df,
@@ -318,11 +342,12 @@ class LoadPipeline(BasePipeline):
                     **self.definition.write_parameters,
                 )
                 if not invalid.empty:
+                    self.logger.info("Invalid data found - writting to error log")
                     self.dst_db_ops.write_to_table(
                         table_name=self.definition.destination_table_invalid,
                         df=invalid,
                         index=False,
-                        if_exists="replace",
+                        if_exists="append",
                     )
             elif self.dst_file_ops:  # File destination
                 valid_path, invalid_path = self.dst_file_ops._construct_output_path()
@@ -340,18 +365,16 @@ class LoadPipeline(BasePipeline):
                         invalid, invalid_path, **self.definition.write_parameters
                     )
             else:
-                return False
+                raise NoValidDestination
 
             # file_post_process always runs against the source file, not the destination
             if self.src_file_ops and self.source_file_path:
+                self.logger.debug("Applying file post-processing.")
                 self.source_file_path = self.definition.file_post_process(
                     self.source_file_path
                 )
                 if self.definition.source_file_delete_after_load == "yes":
                     self.source_file_path.unlink()
-                    self.logger.info(f"Deleted source file: {self.source_file_path}")
-
-            return True
-        except Exception as e:
-            self.log_exception(e, "Error writing to destination")
-            return False
+                    self.logger.debug(f"Deleted source file: {self.source_file_path}")
+        except Exception:
+            raise

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/src/easy_data_loader/pipeline_base.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import List, Optional, Union
 from uuid import uuid4
+from dataclasses import dataclass
 
 import pandas as pd
 
@@ -18,6 +19,20 @@ from .models import (
 )
 
 
+@dataclass
+class PipelineResult:
+    """A definition of a pipeline execution outcome with some details"""
+
+    success: bool
+    message: str
+    error_code: Optional[str] = None
+    exception: Optional[Exception] = None
+
+    def __bool__(self):
+        """Facilitate checking the result directly in if statements: if result: ..."""
+        return self.success
+
+
 class BasePipeline(LoggedComponent, ABC):
     """
     Abstract base class for all pipeline types.
@@ -40,6 +55,7 @@ class BasePipeline(LoggedComponent, ABC):
 
         self._active_connectors: List[DatabaseConnector] = []
         self.audit_db_ops: Optional[DatabaseOperations] = None
+        self._init_error: Optional[PipelineResult] = None
 
         # Metadata for auditing
         self.input_rows = 0
@@ -49,7 +65,17 @@ class BasePipeline(LoggedComponent, ABC):
         self.file_size_bytes: Optional[int] = None
         self.file_last_modified: Optional[pd.Timestamp] = None
 
-        self._initialize_audit_resource()
+        try:
+            self._initialize_audit_resource()
+        except Exception as e:
+            # deferred so run() can report it as a PipelineResult instead of
+            # raising out of the constructor
+            self._init_error = PipelineResult(
+                success=False,
+                message=f"Pipeline setup failed: {e}",
+                error_code=e.__class__.__name__,
+                exception=e,
+            )
 
     def _initialize_audit_resource(self):
         """Initialize the database operations for auditing."""
@@ -92,9 +118,14 @@ class BasePipeline(LoggedComponent, ABC):
         return DatabaseOperations(connector.get_engine())
 
     def _cleanup(self):
-        """Dispose of all active database connectors."""
+        """Dispose of all active database connectors.
+        Never raises, so a disposal failure can't mask the pipeline's real outcome.
+        """
         for connector in self._active_connectors:
-            connector._dispose_engine()
+            try:
+                connector._dispose_engine()
+            except Exception as e:
+                self.logger.warning(f"Failed to dispose connector cleanly: {e}")
         self.logger.debug("Pipeline cleanup completed.")
 
     def _log_audit(self, status: str):
@@ -118,7 +149,7 @@ class BasePipeline(LoggedComponent, ABC):
             destination_name=getattr(self, "destination_name", None),
             sp_name=getattr(self, "sp_name", None),
             sp_parameters=getattr(self, "sp_parameters", None),
-            timestamp=pd.Timestamp.now(),
+            insert_timestamp=pd.Timestamp.now(),
             error_details=self.error_details,
             file_name=source_file_path.name if source_file_path else None,
             file_path=str(source_file_path) if source_file_path else None,
@@ -127,11 +158,49 @@ class BasePipeline(LoggedComponent, ABC):
         )
 
         try:
-            self.audit_db_ops.write_audit("execution_audit", entry)
+            self.audit_db_ops.write_audit("edl_execution_audit", entry)
         except Exception as e:
             self.logger.error(f"Failed to write audit log: {str(e)}")
 
+    def run(self) -> PipelineResult:
+        """
+        Public entry point for pipeline execution.
+        Wraps internal execution and returns a PipelineResult object.
+        """
+        if self._init_error is not None:
+            self.logger.warning(f"Pipeline has [FAILED] {self._init_error.message}")
+            return self._init_error
+
+        self.logger.info("Starting pipeline.")
+
+        try:
+            """Call the method implemented by child classes"""
+            self._run()
+            self.logger.info("Pipeline has finalized with [SUCCESS].")
+
+            return PipelineResult(
+                success=True,
+                message="Pipeline executed succesfully.",
+            )
+        except Exception as e:
+            # file log
+            self.logger.debug(
+                f"Pipeline has encountered an exception: {e}",
+                exc_info=True,
+            )
+
+            user_msg = getattr(e, "user_message", str(e))
+            # console log
+            self.logger.warning(f"Pipeline has [FAILED] {user_msg}")
+
+            return PipelineResult(
+                success=False,
+                message="Pipeline has failed",
+                error_code=e.__class__.__name__,
+                exception=e,
+            )
+
     @abstractmethod
-    def run(self) -> bool:
+    def _run(self) -> bool:
         """Main execution logic to be implemented by child classes."""
         pass

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/src/easy_data_loader/procedure_pipeline.py

@@ -6,7 +6,7 @@ from .models import (
     ServerBasedConnectionSettings,
     FileBasedConnectionSettings,
 )
-from .pipeline_base import BasePipeline
+from .pipeline_base import BasePipeline, PipelineResult
 
 
 class ProcedurePipeline(BasePipeline):
@@ -14,30 +14,44 @@ class ProcedurePipeline(BasePipeline):
     Pipeline specialized in executing database stored procedures.
     """
 
-    def __init__(self, pipeline_name: str, orchestrator_id: Optional[str] = None):
+    def __init__(self, file_name: str, orchestrator_id: Optional[str] = None):
         # Load definition from config
-        definition = Configuration().get_pipeline(pipeline_name)
+        definition = Configuration().get_pipeline(file_name)
         if not isinstance(definition, ProcedureDefinition):
             raise ValueError(
-                f"Pipeline '{pipeline_name}' is not a ProcedureDefinition."
+                f"Pipeline '{file_name}' does not contain a ProcedureDefinition."
             )
 
         super().__init__(definition, orchestrator_id=orchestrator_id)
         self.definition: ProcedureDefinition = definition
 
-        # Set up primary database connection
-        resource = self.config.get_resource(self.definition.resource)
-        if not isinstance(
-            resource, (ServerBasedConnectionSettings, FileBasedConnectionSettings)
-        ):
-            self.log_and_raise(
-                ValueError,
-                f"Resource '{self.definition.resource}' must be a database connection.",
-            )
+        if self._init_error is None:
+            try:
+                # Set up primary database connection
+                resource = self.config.get_resource(self.definition.resource)
+                if not isinstance(
+                    resource,
+                    (ServerBasedConnectionSettings, FileBasedConnectionSettings),
+                ):
+                    raise ValueError(
+                        f"Resource '{self.definition.resource}' must be a database connection."
+                    )
 
-        self.db_ops = self._setup_db_operations(resource)
+                self.db_ops = self._setup_db_operations(resource)
+            except Exception as e:
+                self.error_details = str(e)
+                self._log_audit("FAILED")
+                self.logger.error(self.error_details)
+                # deferred so run() can report it as a PipelineResult instead of
+                # raising out of the constructor
+                self._init_error = PipelineResult(
+                    success=False,
+                    message=f"Pipeline setup failed: {e}",
+                    error_code=e.__class__.__name__,
+                    exception=e,
+                )
 
-    def run(self) -> bool:
+    def _run(self):
         """Execute all procedures in the definition."""
         self.logger.info(
             f">>> Starting Procedure Pipeline: {self.definition.pipeline_name} <<<"
@@ -65,7 +79,7 @@ class ProcedurePipeline(BasePipeline):
             self.logger.info(
                 f">>> Procedure Pipeline {self.definition.pipeline_name} finished successfully <<<"
             )
-            return True
+            return
 
         except Exception as e:
             self.error_details = str(e)
@@ -78,6 +92,6 @@ class ProcedurePipeline(BasePipeline):
             self.sp_parameters = {p[0]: p[1] for p in self.definition.procedures}
 
             self._log_audit("FAILED")
-            return False
+            raise
         finally:
             self._cleanup()

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1/src/easy_data_loader.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easy_data_loader
-Version: 0.1.8
+Version: 0.2.1
 Summary: Data transfer utilities between files and databases
 Author-email: Bojoi Gabriel <bojoigabriel@gmail.com>
 Classifier: Development Status :: 3 - Alpha

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/tests/test_cli.py

@@ -1,4 +1,5 @@
 import os
+import re
 import sqlite3
 
 import pytest
@@ -138,7 +139,9 @@ def test_validate_resources_reports_ok_and_failed(runner, tmp_path):
     result = runner.invoke(main, ["validate-resources"])
     assert result.exit_code == 0
     assert "Resource: good_folder ... OK (Path Exists)" in result.output
-    assert "Resource: good_db ... OK (Connected)" in result.output
+    # SQLAlchemy's echo logging can interleave stderr noise between the label
+    # and status, so match across lines instead of requiring adjacency.
+    assert re.search(r"Resource: good_db \.\.\.[\s\S]*?OK \(Connected\)", result.output)
     assert "Resource: bad_db ... FAILED" in result.output
 
 
@@ -179,9 +182,11 @@ def test_validate_pipelines_reports_ok_for_load_procedure_and_orchestrator(
 
     result = runner.invoke(main, ["validate-pipelines"])
     assert result.exit_code == 0
-    assert "Pipeline: load_pipeline ... OK" in result.output
-    assert "Pipeline: procedure_pipeline ... OK" in result.output
-    assert "Pipeline: my_orchestrator ... OK" in result.output
+    # SQLAlchemy's echo logging can interleave stderr noise between the label
+    # and status, so match across lines instead of requiring adjacency.
+    assert re.search(r"Pipeline: load_pipeline \.\.\.[\s\S]*?OK\b", result.output)
+    assert re.search(r"Pipeline: procedure_pipeline \.\.\.[\s\S]*?OK\b", result.output)
+    assert re.search(r"Pipeline: my_orchestrator \.\.\.[\s\S]*?OK\b", result.output)
 
 
 def test_validate_pipelines_reports_failed_for_unresolvable_resource(runner, tmp_path):
@@ -193,4 +198,4 @@ def test_validate_pipelines_reports_failed_for_unresolvable_resource(runner, tmp
 
     result = runner.invoke(main, ["validate-pipelines"])
     assert result.exit_code == 0
-    assert "Pipeline: broken_pipeline ... FAILED" in result.output
+    assert re.search(r"Pipeline: broken_pipeline \.\.\.[\s\S]*?FAILED", result.output)

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/tests/test_database_connector.py

@@ -24,7 +24,7 @@ def test_connector_factory_maps_expected_connector_classes():
 
 
 def test_sqlite_connector_builds_expected_connection_string():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         db_path = f"{tmpdir}/test.db"
         settings = FileBasedConnectionSettings(
             conn_server_type=ServerType.SQLITE, file_path=db_path

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/tests/test_orchestrator.py

@@ -25,7 +25,7 @@ def reset_configuration_singleton():
 
 def test_orchestrator_passes_orchestrator_id_and_writes_audit():
     # Create temporary file paths for our sqlite databases
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         src_db_path = os.path.join(tmpdir, "src.db")
         dst_db_path = os.path.join(tmpdir, "dst.db")
         audit_db_path = os.path.join(tmpdir, "audit.db")
@@ -93,7 +93,7 @@ def test_orchestrator_passes_orchestrator_id_and_writes_audit():
         assert orchestrator.orchestrator_id is not None
 
         success = orchestrator.run()
-        assert success is True
+        assert success.success is True
 
         # Verify target tables were created and populated in destination
         conn_dst = sqlite3.connect(dst_db_path)
@@ -111,9 +111,9 @@ def test_orchestrator_passes_orchestrator_id_and_writes_audit():
         conn_audit = sqlite3.connect(audit_db_path)
         cursor_audit = conn_audit.cursor()
 
-        # Read the execution_audit table columns and rows
+        # Read the edl_execution_audit table columns and rows
         cursor_audit.execute(
-            "SELECT pipeline_id, orchestrator_id, pipeline_name, status FROM execution_audit"
+            "SELECT pipeline_id, orchestrator_id, pipeline_name, status FROM edl_execution_audit"
         )
         rows = cursor_audit.fetchall()
         assert len(rows) == 2

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/tests/test_pipeline_transform_order.py

@@ -31,7 +31,7 @@ def _transform_requires_renamed_column(df):
 
 
 def test_transform_hook_sees_renamed_columns():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         src_db_path = os.path.join(tmpdir, "src.db")
         dst_db_path = os.path.join(tmpdir, "dst.db")
         audit_db_path = os.path.join(tmpdir, "audit.db")
@@ -72,7 +72,7 @@ def test_transform_hook_sees_renamed_columns():
         pipeline = LoadPipeline("test_pipeline")
         success = pipeline.run()
 
-        assert success is True
+        assert success.success is True
 
         conn_dst = sqlite3.connect(dst_db_path)
         cursor_dst = conn_dst.cursor()

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/tests/test_procedure_execution_mode.py

@@ -60,7 +60,7 @@ def test_execution_mode_defaults_to_sequential():
 
 
 def test_sequential_mode_runs_all_procedures_when_each_succeeds():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         pipeline = _build_pipeline(tmpdir, execution_mode="sequential")
 
         recorded_calls = []
@@ -72,12 +72,12 @@ def test_sequential_mode_runs_all_procedures_when_each_succeeds():
         pipeline.db_ops.execute_stored_procedure = _record_call  # type: ignore[method-assign]
 
         success = pipeline.run()
-        assert success is True
+        assert success.success is True
         assert [c[0] for c in recorded_calls] == ["proc_a", "proc_b"]
 
 
 def test_loop_stops_after_first_failed_procedure():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         pipeline = _build_pipeline(tmpdir)
 
         recorded_calls = []

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/tests/test_resource_access.py

@@ -29,7 +29,7 @@ def reset_configuration_singleton():
 
 
 def test_read_data_returns_dataframe_for_multiple_rows():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         db_path = os.path.join(tmpdir, "db.sqlite")
         conn = sqlite3.connect(db_path)
         conn.execute("CREATE TABLE t (id INTEGER)")
@@ -48,7 +48,7 @@ def test_read_data_returns_dataframe_for_multiple_rows():
 
 
 def test_read_data_returns_dict_for_single_row():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         db_path = os.path.join(tmpdir, "db.sqlite")
         conn = sqlite3.connect(db_path)
         conn.execute("CREATE TABLE t (id INTEGER, name TEXT)")
@@ -66,7 +66,7 @@ def test_read_data_returns_dict_for_single_row():
 
 
 def test_read_data_returns_empty_dataframe_for_no_rows():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         db_path = os.path.join(tmpdir, "db.sqlite")
         conn = sqlite3.connect(db_path)
         conn.execute("CREATE TABLE t (id INTEGER)")
@@ -138,7 +138,7 @@ def test_procedure_definition_default_build_parameters_is_empty():
 
 
 def test_procedure_pipeline_merges_dynamic_parameters():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         db_path = os.path.join(tmpdir, "db.sqlite")
 
         config = Configuration()
@@ -173,7 +173,7 @@ def test_procedure_pipeline_merges_dynamic_parameters():
         pipeline.db_ops.execute_stored_procedure = _record_call  # type: ignore[method-assign]
 
         success = pipeline.run()
-        assert success is True
+        assert success.success is True
         assert recorded_calls == [
             ("insert_call", {"name": "static_name", "value": "dynamic_value"})
         ]

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/tests/test_source_file_delete.py

@@ -27,7 +27,7 @@ def reset_configuration_singleton():
 
 
 def test_source_file_deleted_after_successful_load_to_db():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         (Path(tmpdir) / "data.csv").write_text("a\n1\n2\n")
         dst_db_path = os.path.join(tmpdir, "dst.db")
 
@@ -52,12 +52,12 @@ def test_source_file_deleted_after_successful_load_to_db():
         pipeline = LoadPipeline("test_pipeline")
         success = pipeline.run()
 
-        assert success is True
+        assert success.success is True
         assert not (Path(tmpdir) / "data.csv").exists()
 
 
 def test_file_post_process_runs_on_source_when_destination_is_db():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         (Path(tmpdir) / "data.csv").write_text("a\n1\n2\n")
         dst_db_path = os.path.join(tmpdir, "dst.db")
 
@@ -86,12 +86,12 @@ def test_file_post_process_runs_on_source_when_destination_is_db():
         pipeline = LoadPipeline("test_pipeline")
         success = pipeline.run()
 
-        assert success is True
+        assert success.success is True
         assert post_processed_paths == [Path(tmpdir) / "data.csv"]
 
 
 def test_source_file_delete_after_load_raises_when_source_is_not_a_file():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         src_db_path = os.path.join(tmpdir, "src.db")
         conn = sqlite3.connect(src_db_path)
         conn.execute("CREATE TABLE source_table (a INTEGER)")
@@ -118,5 +118,10 @@ def test_source_file_delete_after_load_raises_when_source_is_not_a_file():
         )
         config.pipelines["test_pipeline"] = pipeline_def
 
-        with pytest.raises(ValueError, match="source_file_delete_after_load"):
-            LoadPipeline("test_pipeline")
+        # Setup failures are deferred to run() rather than raised from the
+        # constructor, so they can be reported as a PipelineResult.
+        pipeline = LoadPipeline("test_pipeline")
+        result = pipeline.run()
+
+        assert result.success is False
+        assert "source_file_delete_after_load" in result.message

{easy_data_loader-0.1.8 → easy_data_loader-0.2.1}/tests/test_validation.py

@@ -34,7 +34,7 @@ def reset_configuration_singleton():
 
 
 def test_validation_fail_false_keeps_valid_rows():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         src_db_path = os.path.join(tmpdir, "src.db")
         dst_db_path = os.path.join(tmpdir, "dst.db")
         audit_db_path = os.path.join(tmpdir, "audit.db")
@@ -87,7 +87,7 @@ def test_validation_fail_false_keeps_valid_rows():
         success = pipeline.run()
 
         # Should be successful because validation_fail=False
-        assert success is True
+        assert success.success is True
 
         # Verify destination table only has valid rows (1 and 3)
         conn_dst = sqlite3.connect(dst_db_path)
@@ -101,7 +101,7 @@ def test_validation_fail_false_keeps_valid_rows():
         assert rows[1] == (3, "Charlie")
 
         cursor_dst = sqlite3.connect(dst_db_path).cursor()
-        cursor_dst.execute("SELECT error FROM target_table_invalid")
+        cursor_dst.execute("SELECT error FROM edl_validation_error_log")
         invalid_rows = cursor_dst.fetchall()
         cursor_dst.connection.close()
 
@@ -111,7 +111,7 @@ def test_validation_fail_false_keeps_valid_rows():
 
 
 def test_validation_writes_valid_and_invalid_file_outputs():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         src_db_path = os.path.join(tmpdir, "src.db")
         audit_db_path = os.path.join(tmpdir, "audit.db")
 
@@ -146,7 +146,7 @@ def test_validation_writes_valid_and_invalid_file_outputs():
 
         success = LoadPipeline("test_pipeline").run()
 
-        assert success is True
+        assert success.success is True
 
         valid_df = pd.read_csv(Path(tmpdir) / "output.csv")
         invalid_df = pd.read_csv(Path(tmpdir) / "output_invalid.csv")
@@ -157,7 +157,7 @@ def test_validation_writes_valid_and_invalid_file_outputs():
 
 
 def test_validation_fail_true_fails_pipeline():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         src_db_path = os.path.join(tmpdir, "src.db")
         dst_db_path = os.path.join(tmpdir, "dst.db")
         audit_db_path = os.path.join(tmpdir, "audit.db")
@@ -206,14 +206,14 @@ def test_validation_fail_true_fails_pipeline():
         success = pipeline.run()
 
         # Should fail because validation_fail=True
-        assert success is False
+        assert success.success is False
         assert pipeline.error_details is not None
         assert "Validation failed" in pipeline.error_details
         assert "Row 1:" in pipeline.error_details  # Row 2 (0-indexed row 1) failed
 
 
 def test_validation_fail_false_graceful_stop_when_all_rows_invalid():
-    with tempfile.TemporaryDirectory() as tmpdir:
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
         src_db_path = os.path.join(tmpdir, "src.db")
         dst_db_path = os.path.join(tmpdir, "dst.db")
         audit_db_path = os.path.join(tmpdir, "audit.db")
@@ -261,8 +261,10 @@ def test_validation_fail_false_graceful_stop_when_all_rows_invalid():
         pipeline = LoadPipeline("test_pipeline")
         success = pipeline.run()
 
-        # Should stop gracefully and return True because validation_fail=False
-        assert success is True
+        # validation_fail=False keeps invalid rows out of the load, but if that
+        # leaves no data at all the pipeline still fails with an EmptyDataError.
+        assert success.success is False
+        assert success.error_code == "EmptyDataError"
         assert pipeline.output_rows == 0
 
         # Destination table should NOT have been loaded or have any data