easy-data-loader 0.1.7__tar.gz → 0.1.8__tar.gz

{easy_data_loader-0.1.7/src/easy_data_loader.egg-info → easy_data_loader-0.1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easy_data_loader
-Version: 0.1.7
+Version: 0.1.8
 Summary: Data transfer utilities between files and databases
 Author-email: Bojoi Gabriel <bojoigabriel@gmail.com>
 Classifier: Development Status :: 3 - Alpha

{easy_data_loader-0.1.7 → easy_data_loader-0.1.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "easy_data_loader"
-version = "0.1.7"
+version = "0.1.8"
 description = "Data transfer utilities between files and databases"
 authors = [{ name = "Bojoi Gabriel", email = "bojoigabriel@gmail.com" }]
 readme = "README.md"

{easy_data_loader-0.1.7 → easy_data_loader-0.1.8}/src/easy_data_loader/file_operations.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Callable, Dict, Optional, Union
+from typing import Callable, Dict, Optional, Union, Tuple
 import pandas as pd
 from sqlalchemy import types
 
@@ -16,6 +16,8 @@ class FileOperations(LoggedComponent):
         self.settings = settings
         self.file_type = self.settings.file_type
         self.file_path: Optional[Path] = None
+        if self.settings.file_name:
+            self.file_name_invalid = self.settings.file_name + "_invalid"
 
         # mapping the file reader by extension
         self._readers: Dict[FileType, Callable] = {
@@ -46,7 +48,7 @@ class FileOperations(LoggedComponent):
         Identifies the file based on pattern (latest) or explicit name.
         """
         if self.settings.file_pattern:
-            files = list(self.settings.folder_path.glob(self.settings.file_pattern))
+            files = list(self.settings.folder_path.glob('*' + self.settings.file_pattern + '*'))
             if not files:
                 self.log_and_raise(
                     ValueError,
@@ -82,26 +84,21 @@ class FileOperations(LoggedComponent):
             )
             raise
 
-    def write_file(self, df: pd.DataFrame, **kwargs) -> Path:
+    def write_file(self, df: pd.DataFrame, path: Path, **kwargs) -> Path:
         """
-        Ensures a valid path exists and writes the dataframe.
         If file_name is provided, it uses it (and overwrites).
         If file_pattern is provided, it constructs a name using the pattern and a timestamp.
         """
 
-        output_path = self._construct_output_path()
         try:
-            output_path.parent.mkdir(parents=True, exist_ok=True)
-
-            self.logger.info(f"Writing to file: {output_path.name}")
-            self._writers[self.file_type](df, output_path, **kwargs)
+            self._writers[self.file_type](df, path, **kwargs)
 
-            return output_path
+            return path
         except Exception as e:
-            self.log_exception(e, f"Failed to write to: {output_path}")
+            self.log_exception(e, f"Failed to write to: {path}")
             raise
 
-    def _construct_output_path(self) -> Path:
+    def _construct_output_path(self) -> Tuple[Path, Path]:
         """
         Construct the output file path based on the settings.
         If the file name is defined the output path will be fixed,
@@ -112,11 +109,21 @@ class FileOperations(LoggedComponent):
 
         if self.settings.file_name:
             base_path: Path = self.settings.folder_path / self.settings.file_name
-            return base_path.with_suffix(file_extension)
-        return (
+            invalid_base_path: Path = self.settings.folder_path / self.file_name_invalid
+
+            return base_path.with_suffix(file_extension), invalid_base_path.with_suffix(
+                file_extension
+            )
+
+        valid_pattern = (
             self.settings.folder_path
             / f"{self.settings.file_pattern}_{timestamp}{file_extension}"
         )
+        invalid_pattern = (
+            self.settings.folder_path
+            / f"{self.settings.file_pattern}_invalid_{timestamp}{file_extension}"
+        )
+        return valid_pattern, invalid_pattern
 
     def _apply_file_preprocessor(self, preprocessor_func: Callable[[Path], Path]):
         """

{easy_data_loader-0.1.7 → easy_data_loader-0.1.8}/src/easy_data_loader/pipeline.py

@@ -290,12 +290,15 @@ class LoadPipeline(BasePipeline):
 
             current_timestamp = pd.Timestamp.now()
 
-            invalid = pd.DataFrame({
-                "orchestrator_id" : str(self.orchestrator_id),
-                "pipeline_id" : str(self.pipeline_id),
-                "pipeline_name" : self.definition.pipeline_name,
-                "error": invalid_error_messages,
-                "insert_timestamp" : current_timestamp})
+            invalid = pd.DataFrame(
+                {
+                    "orchestrator_id": str(self.orchestrator_id),
+                    "pipeline_id": str(self.pipeline_id),
+                    "pipeline_name": self.definition.pipeline_name,
+                    "error": invalid_error_messages,
+                    "insert_timestamp": current_timestamp,
+                }
+            )
 
         return valid, invalid
 
@@ -319,13 +322,22 @@ class LoadPipeline(BasePipeline):
                         table_name=self.definition.destination_table_invalid,
                         df=invalid,
                         index=False,
-                        if_exists="replace"
+                        if_exists="replace",
                     )
             elif self.dst_file_ops:  # File destination
-                self.dst_file_ops.write_file(df, **self.definition.write_parameters)
+                valid_path, invalid_path = self.dst_file_ops._construct_output_path()
+
+                valid_path.parent.mkdir(parents=True, exist_ok=True)
+
+                self.logger.info(f"Writing to file: {valid_path.name}")
+                self.dst_file_ops.write_file(
+                    df, valid_path, **self.definition.write_parameters
+                )
                 if not invalid.empty:
+                    invalid_path.parent.mkdir(parents=True, exist_ok=True)
+                    self.logger.info(f"Writing to file: {invalid_path.name}")
                     self.dst_file_ops.write_file(
-                        invalid, **self.definition.write_parameters
+                        invalid, invalid_path, **self.definition.write_parameters
                     )
             else:
                 return False

{easy_data_loader-0.1.7 → easy_data_loader-0.1.8/src/easy_data_loader.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easy_data_loader
-Version: 0.1.7
+Version: 0.1.8
 Summary: Data transfer utilities between files and databases
 Author-email: Bojoi Gabriel <bojoigabriel@gmail.com>
 Classifier: Development Status :: 3 - Alpha

{easy_data_loader-0.1.7 → easy_data_loader-0.1.8}/tests/test_file_operations.py

@@ -54,7 +54,8 @@ def test_read_write_csv_round_trip():
         ops = FileOperations(settings)
 
         df = pd.DataFrame({"a": [1, 2], "b": ["x", "y"]})
-        output_path = ops.write_file(df)
+        output_path, _ = ops._construct_output_path()
+        ops.write_file(df, output_path)
 
         assert output_path.exists()
 
@@ -73,8 +74,9 @@ def test_construct_output_path_uses_file_name_when_defined():
             file_type=FileType.CSV, folder_path=tmpdir, file_name="fixed_output"
         )
         ops = FileOperations(settings)
-        path = ops._construct_output_path()
-        assert path == Path(tmpdir) / "fixed_output.csv"
+        valid_path, invalid_path = ops._construct_output_path()
+        assert valid_path == Path(tmpdir) / "fixed_output.csv"
+        assert invalid_path == Path(tmpdir) / "fixed_output_invalid.csv"
 
 
 def test_construct_output_path_uses_pattern_with_timestamp():
@@ -83,7 +85,10 @@ def test_construct_output_path_uses_pattern_with_timestamp():
             file_type=FileType.CSV, folder_path=tmpdir, file_pattern="export"
         )
         ops = FileOperations(settings)
-        path = ops._construct_output_path()
-        assert path.parent == Path(tmpdir)
-        assert path.name.startswith("export_")
-        assert path.suffix == ".csv"
+        valid_path, invalid_path = ops._construct_output_path()
+        assert valid_path.parent == Path(tmpdir)
+        assert valid_path.name.startswith("export_")
+        assert valid_path.suffix == ".csv"
+        assert invalid_path.parent == Path(tmpdir)
+        assert invalid_path.name.startswith("export_invalid_")
+        assert invalid_path.suffix == ".csv"

{easy_data_loader-0.1.7 → easy_data_loader-0.1.8}/tests/test_validation.py

@@ -1,6 +1,9 @@
 import os
 import tempfile
 import sqlite3
+from pathlib import Path
+
+import pandas as pd
 import pytest
 from pydantic import BaseModel, Field
 
@@ -8,6 +11,8 @@ from easy_data_loader.config_loader import Configuration
 from easy_data_loader.models import (
     BasePipelineDefinition,
     FileBasedConnectionSettings,
+    FileSettings,
+    FileType,
     ServerType,
 )
 from easy_data_loader.pipeline import LoadPipeline
@@ -95,6 +100,61 @@ def test_validation_fail_false_keeps_valid_rows():
         assert rows[0] == (1, "Alice")
         assert rows[1] == (3, "Charlie")
 
+        cursor_dst = sqlite3.connect(dst_db_path).cursor()
+        cursor_dst.execute("SELECT error FROM target_table_invalid")
+        invalid_rows = cursor_dst.fetchall()
+        cursor_dst.connection.close()
+
+        assert len(invalid_rows) == 1
+        assert "name=Bo" in invalid_rows[0][0]
+        assert "at least 3 characters" in invalid_rows[0][0]
+
+
+def test_validation_writes_valid_and_invalid_file_outputs():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        src_db_path = os.path.join(tmpdir, "src.db")
+        audit_db_path = os.path.join(tmpdir, "audit.db")
+
+        conn_src = sqlite3.connect(src_db_path)
+        cursor_src = conn_src.cursor()
+        cursor_src.execute("CREATE TABLE source_table (id INTEGER, name TEXT)")
+        cursor_src.execute("INSERT INTO source_table VALUES (1, 'Alice')")
+        cursor_src.execute("INSERT INTO source_table VALUES (2, 'Bo')")
+        conn_src.commit()
+        conn_src.close()
+
+        config = Configuration()
+        config.resources["src_db"] = FileBasedConnectionSettings(
+            conn_server_type=ServerType.SQLITE, file_path=src_db_path
+        )
+        config.resources["output_file"] = FileSettings(
+            file_type=FileType.CSV, folder_path=tmpdir, file_name="output"
+        )
+        config.resources["audit_db"] = FileBasedConnectionSettings(
+            conn_server_type=ServerType.SQLITE, file_path=audit_db_path
+        )
+
+        config.pipelines["test_pipeline"] = BasePipelineDefinition(
+            pipeline_name="test_pipeline",
+            source="src_db",
+            source_sql="SELECT * FROM source_table",
+            destination="output_file",
+            audit="audit_db",
+            validator=ValidationTestModel,
+            validation_fail=False,
+        )
+
+        success = LoadPipeline("test_pipeline").run()
+
+        assert success is True
+
+        valid_df = pd.read_csv(Path(tmpdir) / "output.csv")
+        invalid_df = pd.read_csv(Path(tmpdir) / "output_invalid.csv")
+
+        assert valid_df.to_dict(orient="records") == [{"id": 1, "name": "Alice"}]
+        assert invalid_df["error"].str.contains("name=Bo").all()
+        assert invalid_df["error"].str.contains("at least 3 characters").all()
+
 
 def test_validation_fail_true_fails_pipeline():
     with tempfile.TemporaryDirectory() as tmpdir: