easy-data-loader 0.1.9__tar.gz → 0.2.2__tar.gz

{easy_data_loader-0.1.9/src/easy_data_loader.egg-info → easy_data_loader-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easy_data_loader
-Version: 0.1.9
+Version: 0.2.2
 Summary: Data transfer utilities between files and databases
 Author-email: Bojoi Gabriel <bojoigabriel@gmail.com>
 Classifier: Development Status :: 3 - Alpha
@@ -36,6 +36,8 @@ Dynamic: license-file
 
 **Easy Data Loader** is a flexible, modular Python library designed to streamline ETL (Extract, Transform, Load) processes between various file data sources (csv, xlsx, parquet, orc) and databases (MSSQL, PostgreSQL and others).
 
+The [documentation guide](docs/README.md) is the canonical onboarding path. It is also the source for the self-paced presentation outline in [docs/presentation-outline.md](docs/presentation-outline.md).
+
 ## ✨ Key Features
 - **Declarative Configuration**: Manage connections and pipelines through simple python files and `.env` resources.
 - **Integrated CLI**: Initialize a standardized project structure with a single command.

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/README.md

@@ -7,6 +7,8 @@
 
 **Easy Data Loader** is a flexible, modular Python library designed to streamline ETL (Extract, Transform, Load) processes between various file data sources (csv, xlsx, parquet, orc) and databases (MSSQL, PostgreSQL and others).
 
+The [documentation guide](docs/README.md) is the canonical onboarding path. It is also the source for the self-paced presentation outline in [docs/presentation-outline.md](docs/presentation-outline.md).
+
 ## ✨ Key Features
 - **Declarative Configuration**: Manage connections and pipelines through simple python files and `.env` resources.
 - **Integrated CLI**: Initialize a standardized project structure with a single command.

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "easy_data_loader"
-version = "0.1.9"
+version = "0.2.2"
 description = "Data transfer utilities between files and databases"
 authors = [{ name = "Bojoi Gabriel", email = "bojoigabriel@gmail.com" }]
 readme = "README.md"
@@ -30,7 +30,14 @@ classifiers = [
 ]
 
 [dependency-groups]
-dev = ["ipykernel>=7.1.0", "pytest>=8.4.2", "ruff", "mypy", "pre-commit"]
+dev = [
+    "ipykernel>=7.1.0",
+    "mypy",
+    "pre-commit",
+    "pytest>=8.4.2",
+    "python-pptx>=1.0.2",
+    "ruff",
+]
 
 [project.scripts]
 easy-data-loader = "easy_data_loader.cli:main"
@@ -38,6 +45,9 @@ easy-data-loader = "easy_data_loader.cli:main"
 [tool.setuptools.packages.find]
 where = ["src"]
 
+[tool.setuptools.package-data]
+"*" = ["*"]
+
 [tool.pytest.ini_options]
 pythonpath = "src"
 

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/src/easy_data_loader/cli.py

@@ -6,6 +6,7 @@ from .utils import to_snake_case
 # Integrated templates
 PIPELINE_TEMPLATE = """from easy_data_loader.models import BasePipelineDefinition, ColumnDefinition
 import pandas as pd
+from sqlalchemy import DateTime, Integer, Numeric, String
 
 
 example_pipeline = BasePipelineDefinition(
@@ -28,13 +29,13 @@ example_pipeline = BasePipelineDefinition(
 
     # columns definition if we are sending data to a database table
     columns={
-        "transaction_id": ColumnDefinition(target_name="new_transaction_id", data_type=INT()),
-        "date": ColumnDefinition(target_name="sales_date", data_type=DATETIME()),
-        "customer_id": ColumnDefinition(target_name="id_customer", data_type=INT()),
-        "product_category" : ColumnDefinition(target_name="category_of_product", data_type=NVARCHAR(100)),
-        "units_sold" : ColumnDefinition(target_name="units", data_type=INT()),
-        "unit_price" : ColumnDefinition(target_name="price", data_type=DECIMAL(6,2)),
-        "raw_notes" : ColumnDefinition(target_name="notes", data_type=NVARCHAR(100))
+        "transaction_id": ColumnDefinition(target_name="new_transaction_id", data_type=Integer()),
+        "date": ColumnDefinition(target_name="sales_date", data_type=DateTime()),
+        "customer_id": ColumnDefinition(target_name="id_customer", data_type=Integer()),
+        "product_category" : ColumnDefinition(target_name="category_of_product", data_type=String(100)),
+        "units_sold" : ColumnDefinition(target_name="units", data_type=Integer()),
+        "unit_price" : ColumnDefinition(target_name="price", data_type=Numeric(6,2)),
+        "raw_notes" : ColumnDefinition(target_name="notes", data_type=String(100))
     },
 
     # different parameters passed to the write functions to be forwarded to the pandas function that will write to destination (e.g. to_sql, to_csv, to_excel, etc.)
@@ -118,14 +119,14 @@ MAIN = """from easy_data_loader import LoadPipeline, OrchestratorPipeline, Proce
 
 def main():
     # Run an ETL pipeline
-    first_step = LoadPipeline(pipeline_name="example_pipeline").run()
+    first_step = LoadPipeline(file_name="example_pipeline").run()
 
     # Run a procedure pipeline
     if first_step:
-        ProcedurePipeline(pipeline_name="example_procedure").run()
+        ProcedurePipeline(file_name="example_procedure").run()
 
     # Run an orchestrator pipeline
-    OrchestratorPipeline(pipeline_name="example_orchestrator").run()
+    OrchestratorPipeline(file_name="example_orchestrator").run()
 
 if __name__ == "__main__":
     main()
@@ -412,6 +413,9 @@ def validate_pipelines():
             else:
                 raise ValueError(f"Unknown pipeline type: {type(definition)}")
 
+            if instance is not None and instance._init_error is not None:
+                raise ValueError(instance._init_error.message)
+
             results[name] = "OK"
         except Exception as e:
             results[name] = f"FAILED: {str(e)}"

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/src/easy_data_loader/custom_exceptions.py

@@ -26,3 +26,27 @@ class PipelineValidationError(Exception):
     def __init__(self, message: str):
         self.message = message
         super().__init__(self.message)
+
+
+class EmptyDataError(Exception):
+    def __init__(self, message: str):
+        self.message = message
+        super().__init__(self.message)
+
+
+class NoValidDestination(Exception):
+    def __init__(self, message: str = "The pipeline destination is not valid"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class NestedPipelinesError(Exception):
+    def __init__(self, message: str = "Nested pipelines are not supported"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class UnknownPipelineError(Exception):
+    def __init__(self, message: str = "Unknown pipeline type"):
+        self.message = message
+        super().__init__(self.message)

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/src/easy_data_loader/data_inferrence.py

@@ -88,7 +88,7 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                     self.logger.error(
                         f"Failed to infer dtype for column '{col}': {str(e)}. "
                         f"Column will use default inference.",
-                        exc_info=self.is_debug_enabled,
+                        exc_info=True,
                     )
                     # Skip this column - it will use default inference
                     continue
@@ -100,18 +100,17 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                 )
                 return {}
 
-            self.logger.info(f"Dtype inference completed for {len(dtype_dict)} columns")
+            self.logger.info(f"Type inference completed for {len(dtype_dict)} columns")
 
-            # Log summary if debug enabled
-            if self.is_debug_enabled:
-                self._log_inference_summary(dtype_dict)
+            # Only visible in the file log, since it's logged at debug level
+            self._log_inference_summary(dtype_dict)
 
             return dtype_dict
 
         except Exception as e:
             # Catch-all for any unexpected errors
             self.logger.error(
-                f"Dtype inference failed completely: {str(e)}. "
+                f"Type inference failed completely: {str(e)}. "
                 f"Returning empty dict - will use default inference.",
                 exc_info=True,
             )
@@ -593,7 +592,7 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                     self.logger.error(
                         f"Failed to infer dtype for column '{col_name}' from Parquet metadata: {str(e)}. "
                         f"Column will use default inference.",
-                        exc_info=self.is_debug_enabled,
+                        exc_info=True,
                     )
                     continue
 
@@ -616,7 +615,7 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                             self.logger.error(
                                 f"Failed to analyze string length for column '{col}': {str(e)}. "
                                 f"Column will use default inference.",
-                                exc_info=self.is_debug_enabled,
+                                exc_info=True,
                             )
                             # Remove from dict so it uses default
                             dtype_dict.pop(col, None)
@@ -626,7 +625,7 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                     self.logger.error(
                         f"Failed to read string columns from Parquet: {str(e)}. "
                         f"String columns will use default inference.",
-                        exc_info=self.is_debug_enabled,
+                        exc_info=True,
                     )
                     # Remove string columns from dict
                     for col in string_columns:
@@ -643,8 +642,8 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                 f"Parquet dtype inference completed for {len(dtype_dict)} columns"
             )
 
-            if self.is_debug_enabled:
-                self._log_inference_summary(dtype_dict)
+            # Only visible in the file log, since it's logged at debug level
+            self._log_inference_summary(dtype_dict)
 
             return dtype_dict
 
@@ -703,7 +702,7 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                     self.logger.error(
                         f"Failed to infer dtype for column '{col_name}' from ORC metadata: {str(e)}. "
                         f"Column will use default inference.",
-                        exc_info=self.is_debug_enabled,
+                        exc_info=True,
                     )
                     continue
 
@@ -726,7 +725,7 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                             self.logger.error(
                                 f"Failed to analyze string length for column '{col}': {str(e)}. "
                                 f"Column will use default inference.",
-                                exc_info=self.is_debug_enabled,
+                                exc_info=True,
                             )
                             dtype_dict.pop(col, None)
                             continue
@@ -735,7 +734,7 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                     self.logger.error(
                         f"Failed to read string columns from ORC: {str(e)}. "
                         f"String columns will use default inference.",
-                        exc_info=self.is_debug_enabled,
+                        exc_info=True,
                     )
                     for col in string_columns:
                         dtype_dict.pop(col, None)
@@ -751,8 +750,8 @@ class SQLAlchemyDTypeInferrer(LoggedComponent):
                 f"ORC dtype inference completed for {len(dtype_dict)} columns"
             )
 
-            if self.is_debug_enabled:
-                self._log_inference_summary(dtype_dict)
+            # Only visible in the file log, since it's logged at debug level
+            self._log_inference_summary(dtype_dict)
 
             return dtype_dict
 

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/src/easy_data_loader/database_connector.py

@@ -107,7 +107,7 @@ class SqlServerDatabaseConnector(LoggedComponent, DatabaseConnector):
                     max_overflow=10,
                     pool_timeout=30,
                     pool_recycle=3600,
-                    echo=self.is_debug_enabled,
+                    echo=False,
                 )
                 return engine
             except Exception as e:
@@ -183,7 +183,7 @@ class SQLiteDatabaseConnector(LoggedComponent, DatabaseConnector):
             # SQLite specific engine creation
             engine = create_engine(
                 connection_string,
-                echo=self.is_debug_enabled,
+                echo=False,
             )
             return engine
         except Exception as e:
@@ -280,7 +280,7 @@ class PostgresDatabaseConnector(LoggedComponent, DatabaseConnector):
                 max_overflow=10,
                 pool_timeout=30,
                 pool_recycle=3600,
-                echo=self.is_debug_enabled,
+                echo=False,
             )
             return engine
         except Exception as e:

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/src/easy_data_loader/database_operations.py

@@ -19,18 +19,20 @@ class DatabaseOperations(LoggedComponent):
         self.engine = engine
         self._inspector = inspect(self.engine)
 
-    def write_to_table(
-        self, table_name: Optional[str], df: DataFrame, **kwargs
-    ) -> bool:
+    def write_to_table(self, table_name: Optional[str], df: DataFrame, **kwargs):
         """Write a dataframe to a specified table in the database"""
 
-        self.logger.info(f"Writing {len(df)} rows to table: {table_name}")
+        self.logger.debug(f"Writing {len(df)} rows to table: {table_name}")
         try:
             if table_name:
                 df.to_sql(table_name, con=self.engine, **kwargs)
-            return True
+                self.logger.debug("Finalized write to table: {table_name}")
+            return
         except Exception as e:
-            self.log_exception(e, f"Failed to write to table {table_name}")
+            self.logger.debug(
+                f"Failed to write to table {table_name} caused by the following exception: {e}",
+                exc_info=True,
+            )
             raise
 
     def read_data(self, sql: str, **kwargs) -> DataFrame:

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/src/easy_data_loader/file_operations.py

@@ -134,7 +134,7 @@ class FileOperations(LoggedComponent):
         """
 
         current_path = self._find_file()
-        self.logger.info(f"Current file path is: {current_path}")
+        self.logger.debug(f"Current file path is: {current_path}")
 
         if preprocessor_func is None:
             self.file_path = current_path

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/src/easy_data_loader/log.py

@@ -21,25 +21,38 @@ class AppLogger:
     def _setup_logging(self):
         Path("logs").mkdir(exist_ok=True)
 
-        formatter = logging.Formatter(
+        file_formatter = logging.Formatter(
             "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
             datefmt="%Y-%m-%d %H:%M:%S",
         )
 
+        console_formatter = logging.Formatter(
+            "[%(asctime)s]-[%(levelname)s] %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+
         root_logger = logging.getLogger()
-        root_logger.setLevel(logging.INFO)
+        root_logger.setLevel(
+            logging.DEBUG
+        )  # allow all messages and split them in the handlers
         root_logger.handlers.clear()
 
         # Console
         console_handler = logging.StreamHandler()
-        console_handler.setFormatter(formatter)
+        console_handler.setLevel(
+            logging.INFO
+        )  # the console log will have minimal friendly information
+        console_handler.setFormatter(console_formatter)
         root_logger.addHandler(console_handler)
 
         # File
         file_handler = logging.handlers.RotatingFileHandler(
             "logs/application.log", maxBytes=10 * 1024 * 1024, backupCount=5
         )
-        file_handler.setFormatter(formatter)
+        file_handler.setFormatter(file_formatter)
+        file_handler.setLevel(
+            logging.DEBUG
+        )  # the file log will have full detailed information
         root_logger.addHandler(file_handler)
 
     def get_logger(self, name: str) -> logging.Logger:
@@ -55,10 +68,6 @@ class AppLogger:
         for handler in logging.getLogger().handlers:
             handler.setLevel(log_level)
 
-    @property
-    def is_debug(self) -> bool:
-        return logging.getLogger().isEnabledFor(logging.DEBUG)
-
 
 class LoggedComponent:
     """Base class providing logging functionality to all components"""
@@ -85,7 +94,3 @@ class LoggedComponent:
             log_msg += f" | Context: {context_str}"
 
         self.logger.error(log_msg, exc_info=True)
-
-    @property
-    def is_debug_enabled(self) -> bool:
-        return self.log.is_debug

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/src/easy_data_loader/models.py

@@ -191,7 +191,7 @@ class BasePipelineDefinition(BaseModel):
     @property
     def destination_table_invalid(self) -> Optional[str]:
         if self.destination_table:
-            return f"{self.destination_table}_invalid"
+            return "edl_validation_error_log"
         return None
 
     def file_pre_process(self, file_path: Path) -> Path:

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/src/easy_data_loader/orchestrator.py

@@ -11,6 +11,8 @@ from .models import (
 )
 from .pipeline import LoadPipeline
 from .procedure_pipeline import ProcedurePipeline
+from .custom_exceptions import NestedPipelinesError, UnknownPipelineError
+from .pipeline_base import PipelineResult
 
 
 class OrchestratorPipeline(LoggedComponent):
@@ -23,25 +25,41 @@ class OrchestratorPipeline(LoggedComponent):
         definition = self.config.get_pipeline(file_name)
 
         if not isinstance(definition, OrchestratorDefinition):
-            self.log_and_raise(
-                ValueError, f"'{file_name}' does not contain a OrchestratorDefinition."
+            raise ValueError(
+                f"'{file_name}' does not contain a OrchestratorPipeline definition."
             )
 
         self.definition: OrchestratorDefinition = definition
 
-    def run(self) -> bool:
+    def run(self) -> PipelineResult:
+        """Entry point mirroring BasePipeline.run(), without audit/connector lifecycle."""
+        try:
+            success = self._run()
+            return PipelineResult(
+                success=success,
+                message="Orchestrator executed successfully."
+                if success
+                else "One or more pipelines failed.",
+            )
+        except Exception as e:
+            self.logger.warning(f"Orchestrator has [FAILED] {e}")
+            return PipelineResult(
+                success=False,
+                message="Orchestrator has failed",
+                error_code=e.__class__.__name__,
+                exception=e,
+            )
+
+    def _run(self) -> bool:
         self.logger.info(
             f"=== Starting Orchestrator: {self.definition.pipeline_name} (Orchestrator: {self.orchestrator_id}) ==="
         )
 
         success = True
         for pipeline_name in self.definition.pipelines:
-            self.logger.info(
-                f"[{self.definition.pipeline_name}] -> Triggering pipeline: {pipeline_name}"
-            )
+            self.logger.info(f"Triggering pipeline: {pipeline_name}")
 
             p_def: PipelineType = self.config.get_pipeline(pipeline_name)
-            p_success = False
 
             # Instantiate and run
             if isinstance(p_def, BasePipelineDefinition):
@@ -53,25 +71,17 @@ class OrchestratorPipeline(LoggedComponent):
                     pipeline_name, orchestrator_id=self.orchestrator_id
                 ).run()
             elif isinstance(p_def, OrchestratorDefinition):
-                self.logger.error(
-                    f"[{self.definition.pipeline_name}] -> Nested orchestrators are not supported."
-                )
-                p_success = False
+                # caught by run() and turned into a failed PipelineResult
+                raise NestedPipelinesError
             else:
-                self.logger.error(
-                    f"[{self.definition.pipeline_name}] -> Unknown pipeline type for '{pipeline_name}'"
-                )
-                p_success = False
+                # caught by run() and turned into a failed PipelineResult
+                raise UnknownPipelineError
 
             if not p_success:
                 success = False
-                self.logger.error(
-                    f"[{self.definition.pipeline_name}] -> Pipeline failed: {pipeline_name}"
-                )
+                self.logger.error(f"Pipeline failed: {pipeline_name}")
                 if self.definition.fail_fast:
-                    self.logger.error(
-                        f"[{self.definition.pipeline_name}] -> Fail fast enabled. Stopping orchestrator."
-                    )
+                    self.logger.error("Fail fast enabled. Stopping orchestrator.")
                     break
             else:
                 self.logger.info(

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/src/easy_data_loader/pipeline.py

@@ -14,8 +14,12 @@ from .models import (
     FileBasedConnectionSettings,
     FileType,
 )
-from .custom_exceptions import PipelineValidationError
-from .pipeline_base import BasePipeline
+from .custom_exceptions import (
+    PipelineValidationError,
+    EmptyDataError,
+    NoValidDestination,
+)
+from .pipeline_base import BasePipeline, PipelineResult
 
 
 class LoadPipeline(BasePipeline):
@@ -44,7 +48,18 @@ class LoadPipeline(BasePipeline):
         self.dst_file_ops: Optional[FileOperations] = None
         self.source_file_path: Optional[Path] = None  # For auditing
 
-        self._initialize_components()
+        if self._init_error is None:
+            try:
+                self._initialize_components()
+            except Exception as e:
+                # deferred so run() can report it as a PipelineResult instead of
+                # raising out of the constructor
+                self._init_error = PipelineResult(
+                    success=False,
+                    message=f"Pipeline setup failed: {e}",
+                    error_code=e.__class__.__name__,
+                    exception=e,
+                )
 
     def _initialize_components(self):
         """Dynamically initialize the components based on their type and definition"""
@@ -86,40 +101,37 @@ class LoadPipeline(BasePipeline):
         ):
             self.dst_db_ops = self._setup_db_operations(destination_resource)
             self.destination_name = destination_resource.conn_database
+        else:
+            raise ValueError(
+                f"Unsupported destination resource type: {type(source_resource)}"
+            )
 
-    def run(self) -> bool:
+    def _run(self):
         """Executes the entire ETL flow"""
-        self.logger.info(
-            f">>> Starting Load Pipeline: {self.definition.pipeline_name} <<<"
-        )
 
         try:
-            # 1. EXTRACT
+            # 1. READ
             df, inferred_dtypes = self._extract_step()
             if df.empty:
-                self.logger.warning("No data found! Pipeline will stop!")
-                return False
+                self.error_details = "No data found when trying to read from source!"
+                self._log_audit("FAILED")
+                raise EmptyDataError(self.error_details)
 
             self.input_rows = len(df)
+            self.logger.info(f"Rows read from source: {self.input_rows}")
 
             # 2. TRANSFORM & VALIDATE
             df, invalid_df = self._transform_step(df)
             self.output_rows = len(df)
             if df.empty:
-                self.logger.warning(
-                    "No valid data remaining after validation! Pipeline will stop gracefully."
+                self.error_details = (
+                    "After validation no data remaining - all data is invalid."
                 )
-                self._log_audit("SUCCESS")
-                return True
+                self._log_audit("FAILED")
+                raise EmptyDataError(self.error_details)
 
             # 3. LOAD
-            load_success = self._load_step(df, invalid_df, inferred_dtypes)
-            if not load_success:
-                self.logger.error(
-                    f">>> Pipeline {self.definition.pipeline_name} failed to load during the LOAD step."
-                )
-                self._log_audit("FAILED")
-                return False
+            self._load_step(df, invalid_df, inferred_dtypes)
 
             self._log_audit("SUCCESS")
             self.logger.info(
@@ -130,17 +142,19 @@ class LoadPipeline(BasePipeline):
         except PipelineValidationError as e:
             self.error_details = str(e)
             self.logger.error(
-                f"Critical pipeline error - {self.definition.pipeline_name}: {str(e)}"
+                f"Critical pipeline error - {self.definition.pipeline_name}"
             )
+            self.logger.debug(str(e))
             self._log_audit("FAILED")
-            return False
+            raise
         except Exception as e:
             self.error_details = str(e)
-            self.log_exception(
-                e, f"Critical pipeline error - {self.definition.pipeline_name}"
+            self.logger.error(
+                f"Critical pipeline error - {self.definition.pipeline_name}"
             )
+            self.logger.debug(str(e))
             self._log_audit("FAILED")
-            return False
+            raise
         finally:
             self._cleanup()
 
@@ -156,18 +170,23 @@ class LoadPipeline(BasePipeline):
         try:
             if self.src_db_ops:  # DB source
                 if self.definition.source_sql:
+                    self.logger.debug("<< reading data from database >>")
                     df = self.src_db_ops.read_data(
                         self.definition.source_sql, **self.definition.read_parameters
                     )
                     dtype_map = self.definition.get_dtype_map()
+                    self.logger.debug("<< identifying columns definition >>")
                     if not dtype_map:
+                        self.logger.debug("<< no columns definition identified >>")
                         dtype_map = {}
                     return df, dtype_map
 
             if self.src_file_ops:  # File source
+                self.logger.debug("<< source is a file, atempting pre-processing >>")
                 self.src_file_ops._apply_file_preprocessor(
                     self.definition.file_pre_process
                 )
+                self.logger.debug("<< reading from file source >>")
                 df = self.src_file_ops.read_file(**self.definition.read_parameters)
                 self.source_file_path = self.src_file_ops.file_path
 
@@ -201,8 +220,10 @@ class LoadPipeline(BasePipeline):
 
             self.logger.error("No valid source configured for the Pipeline")
 
-        except Exception as e:
-            self.log_exception(e, "Extraction step failed")
+        except Exception:
+            self.error_details = "Extraction step failed"
+            self.logger.error(self.error_details)
+            self._log_audit("FAILED")
             raise
 
         return pd.DataFrame(), {}
@@ -215,7 +236,8 @@ class LoadPipeline(BasePipeline):
         rename_map = self.definition.get_rename_map()
         if rename_map:
             df.rename(columns=rename_map, inplace=True)
-            self.logger.info(f"Columns renamed: {list(rename_map.values())}")
+            self.logger.info("Columns renamed")
+            self.logger.debug(f"Columns renamed: {rename_map}")
 
         # 2. Pipeline hook transformation
         df = self.definition.transform(df)
@@ -310,10 +332,13 @@ class LoadPipeline(BasePipeline):
         df: pd.DataFrame,
         invalid: pd.DataFrame,
         dtype_map: Dict[str, types.TypeEngine],
-    ) -> bool:
+    ) -> None:
         """Handles loading logic based on destination type."""
         try:
             if self.dst_db_ops and self.definition.destination_table:  # DB destination
+                self.logger.info(
+                    f"Writting to table {self.definition.destination_table}"
+                )
                 self.dst_db_ops.write_to_table(
                     table_name=self.definition.destination_table,
                     df=df,
@@ -321,11 +346,12 @@ class LoadPipeline(BasePipeline):
                     **self.definition.write_parameters,
                 )
                 if not invalid.empty:
+                    self.logger.info("Invalid data found - writting to error log")
                     self.dst_db_ops.write_to_table(
                         table_name=self.definition.destination_table_invalid,
                         df=invalid,
                         index=False,
-                        if_exists="replace",
+                        if_exists="append",
                     )
             elif self.dst_file_ops:  # File destination
                 valid_path, invalid_path = self.dst_file_ops._construct_output_path()
@@ -343,18 +369,16 @@ class LoadPipeline(BasePipeline):
                         invalid, invalid_path, **self.definition.write_parameters
                     )
             else:
-                return False
+                raise NoValidDestination
 
             # file_post_process always runs against the source file, not the destination
             if self.src_file_ops and self.source_file_path:
+                self.logger.debug("Applying file post-processing.")
                 self.source_file_path = self.definition.file_post_process(
                     self.source_file_path
                 )
                 if self.definition.source_file_delete_after_load == "yes":
                     self.source_file_path.unlink()
-                    self.logger.info(f"Deleted source file: {self.source_file_path}")
-
-            return True
-        except Exception as e:
-            self.log_exception(e, "Error writing to destination")
-            return False
+                    self.logger.debug(f"Deleted source file: {self.source_file_path}")
+        except Exception:
+            raise

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/src/easy_data_loader/pipeline_base.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import List, Optional, Union
 from uuid import uuid4
+from dataclasses import dataclass
 
 import pandas as pd
 
@@ -18,6 +19,20 @@ from .models import (
 )
 
 
+@dataclass
+class PipelineResult:
+    """A definition of a pipeline execution outcome with some details"""
+
+    success: bool
+    message: str
+    error_code: Optional[str] = None
+    exception: Optional[Exception] = None
+
+    def __bool__(self):
+        """Facilitate checking the result directly in if statements: if result: ..."""
+        return self.success
+
+
 class BasePipeline(LoggedComponent, ABC):
     """
     Abstract base class for all pipeline types.
@@ -40,6 +55,7 @@ class BasePipeline(LoggedComponent, ABC):
 
         self._active_connectors: List[DatabaseConnector] = []
         self.audit_db_ops: Optional[DatabaseOperations] = None
+        self._init_error: Optional[PipelineResult] = None
 
         # Metadata for auditing
         self.input_rows = 0
@@ -49,7 +65,17 @@ class BasePipeline(LoggedComponent, ABC):
         self.file_size_bytes: Optional[int] = None
         self.file_last_modified: Optional[pd.Timestamp] = None
 
-        self._initialize_audit_resource()
+        try:
+            self._initialize_audit_resource()
+        except Exception as e:
+            # deferred so run() can report it as a PipelineResult instead of
+            # raising out of the constructor
+            self._init_error = PipelineResult(
+                success=False,
+                message=f"Pipeline setup failed: {e}",
+                error_code=e.__class__.__name__,
+                exception=e,
+            )
 
     def _initialize_audit_resource(self):
         """Initialize the database operations for auditing."""
@@ -92,9 +118,14 @@ class BasePipeline(LoggedComponent, ABC):
         return DatabaseOperations(connector.get_engine())
 
     def _cleanup(self):
-        """Dispose of all active database connectors."""
+        """Dispose of all active database connectors.
+        Never raises, so a disposal failure can't mask the pipeline's real outcome.
+        """
         for connector in self._active_connectors:
-            connector._dispose_engine()
+            try:
+                connector._dispose_engine()
+            except Exception as e:
+                self.logger.warning(f"Failed to dispose connector cleanly: {e}")
         self.logger.debug("Pipeline cleanup completed.")
 
     def _log_audit(self, status: str):
@@ -131,7 +162,45 @@ class BasePipeline(LoggedComponent, ABC):
         except Exception as e:
             self.logger.error(f"Failed to write audit log: {str(e)}")
 
+    def run(self) -> PipelineResult:
+        """
+        Public entry point for pipeline execution.
+        Wraps internal execution and returns a PipelineResult object.
+        """
+        if self._init_error is not None:
+            self.logger.warning(f"Pipeline has [FAILED] {self._init_error.message}")
+            return self._init_error
+
+        self.logger.info("Starting pipeline.")
+
+        try:
+            """Call the method implemented by child classes"""
+            self._run()
+            self.logger.info("Pipeline has finalized with [SUCCESS].")
+
+            return PipelineResult(
+                success=True,
+                message="Pipeline executed succesfully.",
+            )
+        except Exception as e:
+            # file log
+            self.logger.debug(
+                f"Pipeline has encountered an exception: {e}",
+                exc_info=True,
+            )
+
+            user_msg = getattr(e, "user_message", str(e))
+            # console log
+            self.logger.warning(f"Pipeline has [FAILED] {user_msg}")
+
+            return PipelineResult(
+                success=False,
+                message="Pipeline has failed",
+                error_code=e.__class__.__name__,
+                exception=e,
+            )
+
     @abstractmethod
-    def run(self) -> bool:
+    def _run(self) -> bool:
         """Main execution logic to be implemented by child classes."""
         pass

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/src/easy_data_loader/procedure_pipeline.py

@@ -6,7 +6,7 @@ from .models import (
     ServerBasedConnectionSettings,
     FileBasedConnectionSettings,
 )
-from .pipeline_base import BasePipeline
+from .pipeline_base import BasePipeline, PipelineResult
 
 
 class ProcedurePipeline(BasePipeline):
@@ -25,19 +25,33 @@ class ProcedurePipeline(BasePipeline):
         super().__init__(definition, orchestrator_id=orchestrator_id)
         self.definition: ProcedureDefinition = definition
 
-        # Set up primary database connection
-        resource = self.config.get_resource(self.definition.resource)
-        if not isinstance(
-            resource, (ServerBasedConnectionSettings, FileBasedConnectionSettings)
-        ):
-            self.log_and_raise(
-                ValueError,
-                f"Resource '{self.definition.resource}' must be a database connection.",
-            )
+        if self._init_error is None:
+            try:
+                # Set up primary database connection
+                resource = self.config.get_resource(self.definition.resource)
+                if not isinstance(
+                    resource,
+                    (ServerBasedConnectionSettings, FileBasedConnectionSettings),
+                ):
+                    raise ValueError(
+                        f"Resource '{self.definition.resource}' must be a database connection."
+                    )
 
-        self.db_ops = self._setup_db_operations(resource)
+                self.db_ops = self._setup_db_operations(resource)
+            except Exception as e:
+                self.error_details = str(e)
+                self._log_audit("FAILED")
+                self.logger.error(self.error_details)
+                # deferred so run() can report it as a PipelineResult instead of
+                # raising out of the constructor
+                self._init_error = PipelineResult(
+                    success=False,
+                    message=f"Pipeline setup failed: {e}",
+                    error_code=e.__class__.__name__,
+                    exception=e,
+                )
 
-    def run(self) -> bool:
+    def _run(self):
         """Execute all procedures in the definition."""
         self.logger.info(
             f">>> Starting Procedure Pipeline: {self.definition.pipeline_name} <<<"
@@ -65,7 +79,7 @@ class ProcedurePipeline(BasePipeline):
             self.logger.info(
                 f">>> Procedure Pipeline {self.definition.pipeline_name} finished successfully <<<"
             )
-            return True
+            return
 
         except Exception as e:
             self.error_details = str(e)
@@ -78,6 +92,6 @@ class ProcedurePipeline(BasePipeline):
             self.sp_parameters = {p[0]: p[1] for p in self.definition.procedures}
 
             self._log_audit("FAILED")
-            return False
+            raise
         finally:
             self._cleanup()

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2/src/easy_data_loader.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easy_data_loader
-Version: 0.1.9
+Version: 0.2.2
 Summary: Data transfer utilities between files and databases
 Author-email: Bojoi Gabriel <bojoigabriel@gmail.com>
 Classifier: Development Status :: 3 - Alpha
@@ -36,6 +36,8 @@ Dynamic: license-file
 
 **Easy Data Loader** is a flexible, modular Python library designed to streamline ETL (Extract, Transform, Load) processes between various file data sources (csv, xlsx, parquet, orc) and databases (MSSQL, PostgreSQL and others).
 
+The [documentation guide](docs/README.md) is the canonical onboarding path. It is also the source for the self-paced presentation outline in [docs/presentation-outline.md](docs/presentation-outline.md).
+
 ## ✨ Key Features
 - **Declarative Configuration**: Manage connections and pipelines through simple python files and `.env` resources.
 - **Integrated CLI**: Initialize a standardized project structure with a single command.

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/tests/test_cli.py

@@ -1,4 +1,5 @@
 import os
+import re
 import sqlite3
 
 import pytest
@@ -138,7 +139,9 @@ def test_validate_resources_reports_ok_and_failed(runner, tmp_path):
     result = runner.invoke(main, ["validate-resources"])
     assert result.exit_code == 0
     assert "Resource: good_folder ... OK (Path Exists)" in result.output
-    assert "Resource: good_db ... OK (Connected)" in result.output
+    # SQLAlchemy's echo logging can interleave stderr noise between the label
+    # and status, so match across lines instead of requiring adjacency.
+    assert re.search(r"Resource: good_db \.\.\.[\s\S]*?OK \(Connected\)", result.output)
     assert "Resource: bad_db ... FAILED" in result.output
 
 
@@ -179,9 +182,11 @@ def test_validate_pipelines_reports_ok_for_load_procedure_and_orchestrator(
 
     result = runner.invoke(main, ["validate-pipelines"])
     assert result.exit_code == 0
-    assert "Pipeline: load_pipeline ... OK" in result.output
-    assert "Pipeline: procedure_pipeline ... OK" in result.output
-    assert "Pipeline: my_orchestrator ... OK" in result.output
+    # SQLAlchemy's echo logging can interleave stderr noise between the label
+    # and status, so match across lines instead of requiring adjacency.
+    assert re.search(r"Pipeline: load_pipeline \.\.\.[\s\S]*?OK\b", result.output)
+    assert re.search(r"Pipeline: procedure_pipeline \.\.\.[\s\S]*?OK\b", result.output)
+    assert re.search(r"Pipeline: my_orchestrator \.\.\.[\s\S]*?OK\b", result.output)
 
 
 def test_validate_pipelines_reports_failed_for_unresolvable_resource(runner, tmp_path):
@@ -193,4 +198,4 @@ def test_validate_pipelines_reports_failed_for_unresolvable_resource(runner, tmp
 
     result = runner.invoke(main, ["validate-pipelines"])
     assert result.exit_code == 0
-    assert "Pipeline: broken_pipeline ... FAILED" in result.output
+    assert re.search(r"Pipeline: broken_pipeline \.\.\.[\s\S]*?FAILED", result.output)

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/tests/test_orchestrator.py

@@ -93,7 +93,7 @@ def test_orchestrator_passes_orchestrator_id_and_writes_audit():
         assert orchestrator.orchestrator_id is not None
 
         success = orchestrator.run()
-        assert success is True
+        assert success.success is True
 
         # Verify target tables were created and populated in destination
         conn_dst = sqlite3.connect(dst_db_path)

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/tests/test_pipeline_transform_order.py

@@ -72,7 +72,7 @@ def test_transform_hook_sees_renamed_columns():
         pipeline = LoadPipeline("test_pipeline")
         success = pipeline.run()
 
-        assert success is True
+        assert success.success is True
 
         conn_dst = sqlite3.connect(dst_db_path)
         cursor_dst = conn_dst.cursor()

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/tests/test_procedure_execution_mode.py

@@ -72,7 +72,7 @@ def test_sequential_mode_runs_all_procedures_when_each_succeeds():
         pipeline.db_ops.execute_stored_procedure = _record_call  # type: ignore[method-assign]
 
         success = pipeline.run()
-        assert success is True
+        assert success.success is True
         assert [c[0] for c in recorded_calls] == ["proc_a", "proc_b"]
 
 

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/tests/test_resource_access.py

@@ -173,7 +173,7 @@ def test_procedure_pipeline_merges_dynamic_parameters():
         pipeline.db_ops.execute_stored_procedure = _record_call  # type: ignore[method-assign]
 
         success = pipeline.run()
-        assert success is True
+        assert success.success is True
         assert recorded_calls == [
             ("insert_call", {"name": "static_name", "value": "dynamic_value"})
         ]

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/tests/test_source_file_delete.py

@@ -52,7 +52,7 @@ def test_source_file_deleted_after_successful_load_to_db():
         pipeline = LoadPipeline("test_pipeline")
         success = pipeline.run()
 
-        assert success is True
+        assert success.success is True
         assert not (Path(tmpdir) / "data.csv").exists()
 
 
@@ -86,7 +86,7 @@ def test_file_post_process_runs_on_source_when_destination_is_db():
         pipeline = LoadPipeline("test_pipeline")
         success = pipeline.run()
 
-        assert success is True
+        assert success.success is True
         assert post_processed_paths == [Path(tmpdir) / "data.csv"]
 
 
@@ -118,5 +118,10 @@ def test_source_file_delete_after_load_raises_when_source_is_not_a_file():
         )
         config.pipelines["test_pipeline"] = pipeline_def
 
-        with pytest.raises(ValueError, match="source_file_delete_after_load"):
-            LoadPipeline("test_pipeline")
+        # Setup failures are deferred to run() rather than raised from the
+        # constructor, so they can be reported as a PipelineResult.
+        pipeline = LoadPipeline("test_pipeline")
+        result = pipeline.run()
+
+        assert result.success is False
+        assert "source_file_delete_after_load" in result.message

{easy_data_loader-0.1.9 → easy_data_loader-0.2.2}/tests/test_validation.py

@@ -87,7 +87,7 @@ def test_validation_fail_false_keeps_valid_rows():
         success = pipeline.run()
 
         # Should be successful because validation_fail=False
-        assert success is True
+        assert success.success is True
 
         # Verify destination table only has valid rows (1 and 3)
         conn_dst = sqlite3.connect(dst_db_path)
@@ -101,7 +101,7 @@ def test_validation_fail_false_keeps_valid_rows():
         assert rows[1] == (3, "Charlie")
 
         cursor_dst = sqlite3.connect(dst_db_path).cursor()
-        cursor_dst.execute("SELECT error FROM target_table_invalid")
+        cursor_dst.execute("SELECT error FROM edl_validation_error_log")
         invalid_rows = cursor_dst.fetchall()
         cursor_dst.connection.close()
 
@@ -146,7 +146,7 @@ def test_validation_writes_valid_and_invalid_file_outputs():
 
         success = LoadPipeline("test_pipeline").run()
 
-        assert success is True
+        assert success.success is True
 
         valid_df = pd.read_csv(Path(tmpdir) / "output.csv")
         invalid_df = pd.read_csv(Path(tmpdir) / "output_invalid.csv")
@@ -206,7 +206,7 @@ def test_validation_fail_true_fails_pipeline():
         success = pipeline.run()
 
         # Should fail because validation_fail=True
-        assert success is False
+        assert success.success is False
         assert pipeline.error_details is not None
         assert "Validation failed" in pipeline.error_details
         assert "Row 1:" in pipeline.error_details  # Row 2 (0-indexed row 1) failed
@@ -261,8 +261,10 @@ def test_validation_fail_false_graceful_stop_when_all_rows_invalid():
         pipeline = LoadPipeline("test_pipeline")
         success = pipeline.run()
 
-        # Should stop gracefully and return True because validation_fail=False
-        assert success is True
+        # validation_fail=False keeps invalid rows out of the load, but if that
+        # leaves no data at all the pipeline still fails with an EmptyDataError.
+        assert success.success is False
+        assert success.error_code == "EmptyDataError"
         assert pipeline.output_rows == 0
 
         # Destination table should NOT have been loaded or have any data