churnkit 0.75.0a1__py3-none-any.whl

customer_retention/generators/orchestration/context.py

@@ -0,0 +1,147 @@
+import json
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+from customer_retention.analysis.auto_explorer import ExplorationFindings
+from customer_retention.core.config.column_config import ColumnConfig, ColumnType
+
+
+@dataclass
+class PipelineContext:
+    run_id: str = field(default_factory=lambda: str(uuid.uuid4())[:8])
+    project_name: str = "customer_retention"
+    started_at: str = field(default_factory=lambda: datetime.now().isoformat())
+    raw_data_path: Optional[str] = None
+    bronze_path: Optional[str] = None
+    silver_path: Optional[str] = None
+    gold_path: Optional[str] = None
+    exploration_findings: Optional[ExplorationFindings] = None
+    profiling_results: Optional[Dict] = None
+    cleaning_report: Optional[Dict] = None
+    transformation_pipeline: Optional[Any] = None
+    feature_manifest: Optional[Any] = None
+    selected_features: Optional[List[str]] = None
+    model_results: Optional[Dict] = None
+    validation_results: Optional[Dict] = None
+    current_df: Optional[Any] = None
+    current_stage: str = "raw"
+    artifacts: Dict[str, str] = field(default_factory=dict)
+    delta_versions: Dict[str, int] = field(default_factory=dict)
+    run_type: str = "exploration"
+    _context_dir: str = "./runs"
+
+    @property
+    def column_types(self) -> Dict[str, ColumnType]:
+        return self.exploration_findings.column_types if self.exploration_findings else {}
+
+    @property
+    def column_configs(self) -> Dict[str, ColumnConfig]:
+        return self.exploration_findings.column_configs if self.exploration_findings else {}
+
+    @property
+    def target_column(self) -> Optional[str]:
+        return self.exploration_findings.target_column if self.exploration_findings else None
+
+    def build_commit_metadata(self) -> Dict[str, str]:
+        return {
+            "run_id": self.run_id,
+            "run_type": self.run_type,
+            "pipeline_stage": self.current_stage,
+            "timestamp": self.started_at,
+        }
+
+    def save(self, path: str = None):
+        Path(self._context_dir).mkdir(parents=True, exist_ok=True)
+        if path is None:
+            path = f"{self._context_dir}/{self.run_id}_context.json"
+        data = {
+            "run_id": self.run_id,
+            "project_name": self.project_name,
+            "started_at": self.started_at,
+            "raw_data_path": self.raw_data_path,
+            "bronze_path": self.bronze_path,
+            "silver_path": self.silver_path,
+            "gold_path": self.gold_path,
+            "current_stage": self.current_stage,
+            "artifacts": self.artifacts,
+            "delta_versions": self.delta_versions,
+            "run_type": self.run_type,
+            "exploration_findings_path": None
+        }
+        if self.exploration_findings:
+            findings_path = f"{self._context_dir}/{self.run_id}_findings.yaml"
+            self.exploration_findings.save(findings_path)
+            data["exploration_findings_path"] = findings_path
+        with open(path, "w") as f:
+            json.dump(data, f, indent=2)
+        print(f"Context saved: {path}")
+
+    @classmethod
+    def load(cls, path: str) -> "PipelineContext":
+        with open(path, "r") as f:
+            data = json.load(f)
+        exploration_findings = None
+        if data.get("exploration_findings_path"):
+            exploration_findings = ExplorationFindings.load(data["exploration_findings_path"])
+        ctx = cls(
+            run_id=data["run_id"],
+            project_name=data["project_name"],
+            started_at=data["started_at"],
+            raw_data_path=data.get("raw_data_path"),
+            bronze_path=data.get("bronze_path"),
+            silver_path=data.get("silver_path"),
+            gold_path=data.get("gold_path"),
+            current_stage=data.get("current_stage", "raw"),
+            artifacts=data.get("artifacts", {}),
+            delta_versions=data.get("delta_versions", {}),
+            run_type=data.get("run_type", "exploration"),
+            exploration_findings=exploration_findings
+        )
+        ctx._context_dir = str(Path(path).parent)
+        return ctx
+
+
+class ContextManager:
+    def __init__(self, context: PipelineContext, auto_save: bool = True):
+        self.context = context
+        self.auto_save = auto_save
+
+    def update(self, **kwargs):
+        for key, value in kwargs.items():
+            if hasattr(self.context, key):
+                setattr(self.context, key, value)
+        if self.auto_save:
+            self.context.save()
+
+    def add_artifact(self, name: str, path: str):
+        self.context.artifacts[name] = path
+        if self.auto_save:
+            self.context.save()
+
+
+def setup_notebook_context(
+    project_name: str = None,
+    exploration_findings: Any = None,
+    resume_run: str = None,
+    output_dir: str = "./runs"
+) -> Tuple[PipelineContext, ContextManager]:
+    if resume_run:
+        context_path = f"{output_dir}/{resume_run}_context.json"
+        context = PipelineContext.load(context_path)
+        print(f"Resumed run: {context.run_id}")
+    else:
+        context = PipelineContext(project_name=project_name or "customer_retention")
+        context._context_dir = output_dir
+        if exploration_findings:
+            if isinstance(exploration_findings, str):
+                context.exploration_findings = ExplorationFindings.load(exploration_findings)
+            else:
+                context.exploration_findings = exploration_findings
+            context.raw_data_path = context.exploration_findings.source_path
+            print(f"New run: {context.run_id}")
+            print(f"Loaded findings: {len(context.column_types)} columns")
+            print(f"Target: {context.target_column}")
+    return context, ContextManager(context)

customer_retention/generators/orchestration/data_materializer.py

@@ -0,0 +1,188 @@
+from pathlib import Path
+from typing import Optional, Tuple
+
+import numpy as np
+import pandas as pd
+from sklearn.preprocessing import RobustScaler, StandardScaler
+
+from customer_retention.analysis.auto_explorer.layered_recommendations import (
+    LayeredRecommendation,
+    RecommendationRegistry,
+)
+
+
+class DataMaterializer:
+    def __init__(self, registry: RecommendationRegistry, output_dir: Optional[str] = None,
+                 storage=None, context=None):
+        self.registry = registry
+        self.output_dir = output_dir or "./prepared_data"
+        self.storage = storage or _get_storage()
+        self.context = context
+
+    def transform(self, df: pd.DataFrame) -> pd.DataFrame:
+        df = self.apply_bronze(df)
+        df = self.apply_silver(df)
+        df = self.apply_gold(df)
+        return df
+
+    def materialize(self, df: pd.DataFrame, output_name: str) -> Tuple[pd.DataFrame, str]:
+        result_df = self.transform(df)
+        output_path = self._save(result_df, output_name)
+        return result_df, output_path
+
+    def apply_bronze(self, df: pd.DataFrame) -> pd.DataFrame:
+        if not self.registry.bronze:
+            return df
+        df = df.copy()
+        for rec in self.registry.bronze.null_handling:
+            df = self._apply_null_handling(df, rec)
+        for rec in self.registry.bronze.outlier_handling:
+            df = self._apply_outlier_handling(df, rec)
+        for rec in self.registry.bronze.type_conversions:
+            df = self._apply_type_conversion(df, rec)
+        for rec in self.registry.bronze.filtering:
+            df = self._apply_filtering(df, rec)
+        return df
+
+    def apply_silver(self, df: pd.DataFrame) -> pd.DataFrame:
+        if not self.registry.silver:
+            return df
+        df = df.copy()
+        for rec in self.registry.silver.aggregations:
+            df = self._apply_aggregation(df, rec)
+        for rec in self.registry.silver.derived_columns:
+            df = self._apply_derived(df, rec)
+        return df
+
+    def apply_gold(self, df: pd.DataFrame) -> pd.DataFrame:
+        if not self.registry.gold:
+            return df
+        df = df.copy()
+        for rec in self.registry.gold.transformations:
+            df = self._apply_transformation(df, rec)
+        for rec in self.registry.gold.encoding:
+            df = self._apply_encoding(df, rec)
+        for rec in self.registry.gold.scaling:
+            df = self._apply_scaling(df, rec)
+        return df
+
+    def _apply_null_handling(self, df: pd.DataFrame, rec: LayeredRecommendation) -> pd.DataFrame:
+        col = rec.target_column
+        if col not in df.columns:
+            return df
+        strategy = rec.parameters.get("strategy", "median")
+        if strategy == "median":
+            df[col] = df[col].fillna(df[col].median())
+        elif strategy == "mean":
+            df[col] = df[col].fillna(df[col].mean())
+        elif strategy == "mode":
+            df[col] = df[col].fillna(df[col].mode().iloc[0] if len(df[col].mode()) > 0 else df[col])
+        elif strategy == "zero":
+            df[col] = df[col].fillna(0)
+        return df
+
+    def _apply_outlier_handling(self, df: pd.DataFrame, rec: LayeredRecommendation) -> pd.DataFrame:
+        col = rec.target_column
+        if col not in df.columns:
+            return df
+        method = rec.parameters.get("method", "iqr")
+        if method == "iqr":
+            factor = rec.parameters.get("factor", 1.5)
+            q1, q3 = df[col].quantile([0.25, 0.75])
+            iqr = q3 - q1
+            lower, upper = q1 - factor * iqr, q3 + factor * iqr
+            df[col] = df[col].clip(lower, upper)
+        return df
+
+    def _apply_type_conversion(self, df: pd.DataFrame, rec: LayeredRecommendation) -> pd.DataFrame:
+        col = rec.target_column
+        if col not in df.columns:
+            return df
+        target_type = rec.parameters.get("target_type", "str")
+        if target_type == "datetime":
+            df[col] = pd.to_datetime(df[col])
+        else:
+            df[col] = df[col].astype(target_type)
+        return df
+
+    def _apply_filtering(self, df: pd.DataFrame, rec: LayeredRecommendation) -> pd.DataFrame:
+        col = rec.target_column
+        if rec.action == "drop" and col in df.columns:
+            df = df.drop(columns=[col])
+        return df
+
+    def _apply_aggregation(self, df: pd.DataFrame, rec: LayeredRecommendation) -> pd.DataFrame:
+        col = rec.target_column
+        if col not in df.columns or not self.registry.silver:
+            return df
+        entity_col = self.registry.silver.entity_column
+        agg_func = rec.parameters.get("aggregation", "sum")
+        feature_name = f"{col}_{agg_func}"
+        df[feature_name] = df.groupby(entity_col)[col].transform(agg_func)
+        return df
+
+    def _apply_derived(self, df: pd.DataFrame, rec: LayeredRecommendation) -> pd.DataFrame:
+        return df
+
+    def _apply_transformation(self, df: pd.DataFrame, rec: LayeredRecommendation) -> pd.DataFrame:
+        col = rec.target_column
+        if col not in df.columns:
+            return df
+        method = rec.parameters.get("method", "log")
+        if method == "log":
+            df[col] = np.log1p(df[col])
+        elif method == "sqrt":
+            df[col] = np.sqrt(df[col])
+        return df
+
+    def _apply_encoding(self, df: pd.DataFrame, rec: LayeredRecommendation) -> pd.DataFrame:
+        col = rec.target_column
+        if col not in df.columns:
+            return df
+        method = rec.parameters.get("method", "one_hot")
+        if method == "one_hot":
+            dummies = pd.get_dummies(df[col], prefix=col, drop_first=False)
+            df = pd.concat([df.drop(columns=[col]), dummies], axis=1)
+        return df
+
+    def _apply_scaling(self, df: pd.DataFrame, rec: LayeredRecommendation) -> pd.DataFrame:
+        col = rec.target_column
+        if col not in df.columns:
+            return df
+        method = rec.parameters.get("method", "standard")
+        scaler = StandardScaler() if method == "standard" else RobustScaler()
+        df[col] = scaler.fit_transform(df[[col]])
+        return df
+
+    def _save(self, df: pd.DataFrame, output_name: str) -> str:
+        Path(self.output_dir).mkdir(parents=True, exist_ok=True)
+        output_path = f"{self.output_dir}/{output_name}"
+        if self.storage:
+            metadata = self.context.build_commit_metadata() if self.context else None
+            self.storage.write(df, output_path, metadata=metadata)
+            if self.context:
+                version = len(self.storage.history(output_path)) - 1
+                self.context.delta_versions[output_path] = version
+        else:
+            output_path = f"{output_path}.parquet"
+            df.to_parquet(output_path, index=False)
+        return output_path
+
+    def compare_runs(self, path: str, version_a: int, version_b: int) -> pd.DataFrame:
+        if not self.storage:
+            raise RuntimeError("DeltaStorage required for version comparison")
+        df_a = self.storage.read(path, version=version_a)
+        df_b = self.storage.read(path, version=version_b)
+        return pd.DataFrame({
+            "metric": ["row_count", "column_count"],
+            "version_a": [len(df_a), len(df_a.columns)],
+            "version_b": [len(df_b), len(df_b.columns)],
+        })
+
+
+def _get_storage():
+    try:
+        from customer_retention.integrations.adapters.factory import get_delta
+        return get_delta(force_local=True)
+    except ImportError:
+        return None