odibi 2.5.0__py3-none-any.whl

odibi/patterns/merge.py

@@ -0,0 +1,128 @@
+import time
+from typing import Any
+
+from odibi.context import EngineContext
+from odibi.patterns.base import Pattern
+from odibi.transformers.merge_transformer import MergeParams, merge
+from odibi.utils.logging_context import get_logging_context
+
+
+class MergePattern(Pattern):
+    """
+    Merge Pattern: Upsert/Merge logic.
+
+    Configuration Options (via params dict):
+        - **target** (str): Target table/path.
+        - **keys** (list): Join keys.
+        - **strategy** (str): 'upsert', 'append_only', 'delete_match'.
+    """
+
+    def validate(self) -> None:
+        ctx = get_logging_context()
+
+        # Support both 'target' and 'path' for compatibility with merge transformer
+        target = self.params.get("target") or self.params.get("path")
+
+        ctx.debug(
+            "MergePattern validation starting",
+            pattern="MergePattern",
+            target=target,
+            keys=self.params.get("keys"),
+            strategy=self.params.get("strategy"),
+        )
+
+        if not target:
+            ctx.error(
+                "MergePattern validation failed: 'target' or 'path' is required",
+                pattern="MergePattern",
+            )
+            provided_params = {k: v for k, v in self.params.items() if v is not None}
+            raise ValueError(
+                f"MergePattern: 'target' or 'path' is required. "
+                f"Expected: A target table path string. "
+                f"Provided params: {list(provided_params.keys())}. "
+                f"Fix: Add 'target' or 'path' to your pattern configuration."
+            )
+        if not self.params.get("keys"):
+            ctx.error(
+                "MergePattern validation failed: 'keys' is required",
+                pattern="MergePattern",
+            )
+            source_columns = list(self.source.columns) if hasattr(self.source, "columns") else []
+            raise ValueError(
+                f"MergePattern: 'keys' is required. "
+                f"Expected: A list of column names to match source and target rows for merge. "
+                f"Available source columns: {source_columns}. "
+                f"Fix: Add 'keys' with columns that uniquely identify rows (e.g., keys=['id'])."
+            )
+
+        ctx.debug(
+            "MergePattern validation passed",
+            pattern="MergePattern",
+            target=self.params.get("target"),
+            keys=self.params.get("keys"),
+            strategy=self.params.get("strategy", "upsert"),
+        )
+
+    def execute(self, context: EngineContext) -> Any:
+        ctx = get_logging_context()
+        start_time = time.time()
+
+        # Support both 'target' and 'path' for compatibility
+        target = self.params.get("target") or self.params.get("path")
+        keys = self.params.get("keys")
+        strategy = self.params.get("strategy", "upsert")
+
+        ctx.debug(
+            "Merge pattern starting",
+            pattern="MergePattern",
+            target=target,
+            keys=keys,
+            strategy=strategy,
+        )
+
+        source_count = None
+        try:
+            if context.engine_type == "spark":
+                source_count = context.df.count()
+            else:
+                source_count = len(context.df)
+            ctx.debug(
+                "Merge source data loaded",
+                pattern="MergePattern",
+                source_rows=source_count,
+            )
+        except Exception:
+            ctx.debug("Merge could not determine source row count", pattern="MergePattern")
+
+        valid_keys = MergeParams.model_fields.keys()
+        filtered_params = {k: v for k, v in self.params.items() if k in valid_keys}
+
+        try:
+            merge(context, context.df, **filtered_params)
+        except Exception as e:
+            elapsed_ms = (time.time() - start_time) * 1000
+            ctx.error(
+                f"Merge pattern execution failed: {e}",
+                pattern="MergePattern",
+                error_type=type(e).__name__,
+                elapsed_ms=round(elapsed_ms, 2),
+                target=target,
+                keys=keys,
+                strategy=strategy,
+            )
+            raise
+
+        elapsed_ms = (time.time() - start_time) * 1000
+
+        ctx.info(
+            "Merge pattern completed",
+            pattern="MergePattern",
+            elapsed_ms=round(elapsed_ms, 2),
+            source_rows=source_count,
+            target=target,
+            keys=keys,
+            strategy=strategy,
+        )
+
+        return context.df

odibi/patterns/scd2.py

@@ -0,0 +1,148 @@
+import time
+from typing import Any
+
+from odibi.context import EngineContext
+from odibi.patterns.base import Pattern
+from odibi.transformers.scd import SCD2Params, scd2
+from odibi.utils.logging_context import get_logging_context
+
+
+class SCD2Pattern(Pattern):
+    """
+    SCD2 Pattern: Slowly Changing Dimension Type 2.
+
+    Tracks history by creating new rows for updates.
+
+    Configuration Options (via params dict):
+        - **keys** (list): Business keys.
+        - **time_col** (str): Timestamp column for versioning (default: current time).
+        - **valid_from_col** (str): Name of start date column (default: valid_from).
+        - **valid_to_col** (str): Name of end date column (default: valid_to).
+        - **is_current_col** (str): Name of current flag column (default: is_current).
+    """
+
+    def validate(self) -> None:
+        ctx = get_logging_context()
+        ctx.debug(
+            "SCD2Pattern validation starting",
+            pattern="SCD2Pattern",
+            keys=self.params.get("keys"),
+            target=self.params.get("target"),
+        )
+
+        if not self.params.get("keys"):
+            ctx.error(
+                "SCD2Pattern validation failed: 'keys' parameter is required",
+                pattern="SCD2Pattern",
+            )
+            raise ValueError(
+                "SCD2Pattern: 'keys' parameter is required. "
+                f"Expected a list of business key column names, but got: {self.params.get('keys')!r}. "
+                f"Available params: {list(self.params.keys())}. "
+                "Fix: Provide 'keys' as a list, e.g., keys=['customer_id']."
+            )
+        if not self.params.get("target"):
+            ctx.error(
+                "SCD2Pattern validation failed: 'target' parameter is required",
+                pattern="SCD2Pattern",
+            )
+            raise ValueError(
+                "SCD2Pattern: 'target' parameter is required. "
+                f"Expected a table name or path string, but got: {self.params.get('target')!r}. "
+                "Fix: Provide 'target' as a string, e.g., target='dim_customer'."
+            )
+
+        ctx.debug(
+            "SCD2Pattern validation passed",
+            pattern="SCD2Pattern",
+            keys=self.params.get("keys"),
+            target=self.params.get("target"),
+        )
+
+    def execute(self, context: EngineContext) -> Any:
+        ctx = get_logging_context()
+        start_time = time.time()
+
+        keys = self.params.get("keys")
+        target = self.params.get("target")
+        valid_from_col = self.params.get("valid_from_col", "valid_from")
+        valid_to_col = self.params.get("valid_to_col", "valid_to")
+        is_current_col = self.params.get("is_current_col", "is_current")
+        track_cols = self.params.get("track_cols")
+
+        ctx.debug(
+            "SCD2 pattern starting",
+            pattern="SCD2Pattern",
+            keys=keys,
+            target=target,
+            valid_from_col=valid_from_col,
+            valid_to_col=valid_to_col,
+            is_current_col=is_current_col,
+            track_cols=track_cols,
+        )
+
+        source_count = None
+        try:
+            if context.engine_type == "spark":
+                source_count = context.df.count()
+            else:
+                source_count = len(context.df)
+            ctx.debug("SCD2 source data loaded", pattern="SCD2Pattern", source_rows=source_count)
+        except Exception:
+            ctx.debug("SCD2 could not determine source row count", pattern="SCD2Pattern")
+
+        valid_keys = SCD2Params.model_fields.keys()
+        filtered_params = {k: v for k, v in self.params.items() if k in valid_keys}
+
+        try:
+            scd_params = SCD2Params(**filtered_params)
+        except Exception as e:
+            ctx.error(
+                f"SCD2 invalid parameters: {e}",
+                pattern="SCD2Pattern",
+                error_type=type(e).__name__,
+                params=filtered_params,
+            )
+            raise ValueError(
+                f"Invalid SCD2 parameters: {e}. "
+                f"Provided params: {filtered_params}. "
+                f"Valid param names: {list(valid_keys)}."
+            )
+
+        try:
+            result_ctx = scd2(context, scd_params)
+        except Exception as e:
+            elapsed_ms = (time.time() - start_time) * 1000
+            ctx.error(
+                f"SCD2 pattern execution failed: {e}",
+                pattern="SCD2Pattern",
+                error_type=type(e).__name__,
+                elapsed_ms=round(elapsed_ms, 2),
+            )
+            raise
+
+        result_df = result_ctx.df
+        elapsed_ms = (time.time() - start_time) * 1000
+
+        result_count = None
+        try:
+            if context.engine_type == "spark":
+                result_count = result_df.count()
+            else:
+                result_count = len(result_df)
+        except Exception:
+            pass
+
+        ctx.info(
+            "SCD2 pattern completed",
+            pattern="SCD2Pattern",
+            elapsed_ms=round(elapsed_ms, 2),
+            source_rows=source_count,
+            result_rows=result_count,
+            keys=keys,
+            target=target,
+            valid_from_col=valid_from_col,
+            valid_to_col=valid_to_col,
+        )
+
+        return result_df