churnkit 0.75.0a1__py3-none-any.whl

customer_retention/core/compat/__init__.py

@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Union
+
+import numpy as np
+import pandas as _pandas
+
+from .detection import (
+    configure_spark_pandas,
+    enable_arrow_optimization,
+    get_dbutils,
+    get_display_function,
+    get_spark_session,
+    is_databricks,
+    is_notebook,
+    is_pandas_api_on_spark,
+    is_spark_available,
+    set_spark_config,
+)
+from .ops import DataOps, ops
+
+_SPARK_PANDAS_AVAILABLE = is_spark_available()
+
+if _SPARK_PANDAS_AVAILABLE:
+    try:
+        import pyspark.pandas as ps
+        pd = ps
+        DataFrame = Union[ps.DataFrame, _pandas.DataFrame]
+        Series = Union[ps.Series, _pandas.Series]
+    except ImportError:
+        import databricks.koalas as ps
+        pd = ps
+        DataFrame = Union[ps.DataFrame, _pandas.DataFrame]
+        Series = Union[ps.Series, _pandas.Series]
+else:
+    pd = _pandas
+    DataFrame = _pandas.DataFrame
+    Series = _pandas.Series
+
+
+def get_pandas() -> Any:
+    return _pandas
+
+
+def to_pandas(df: Any) -> _pandas.DataFrame:
+    if isinstance(df, _pandas.DataFrame):
+        return df
+    if _SPARK_PANDAS_AVAILABLE:
+        try:
+            import pyspark.pandas as ps
+            if isinstance(df, ps.DataFrame):
+                return df.to_pandas()
+        except ImportError:
+            pass
+    try:
+        from pyspark.sql import DataFrame as NativeSparkDF
+        if isinstance(df, NativeSparkDF):
+            return df.toPandas()
+    except ImportError:
+        pass
+    return _pandas.DataFrame(df)
+
+
+def to_spark_pandas(df: Any) -> Any:
+    if not _SPARK_PANDAS_AVAILABLE:
+        return df if isinstance(df, _pandas.DataFrame) else _pandas.DataFrame(df)
+    try:
+        import pyspark.pandas as ps
+        if isinstance(df, ps.DataFrame):
+            return df
+        if isinstance(df, _pandas.DataFrame):
+            return ps.from_pandas(df)
+        return ps.DataFrame(df)
+    except ImportError:
+        return df
+
+
+def ensure_pandas_series(series: Any) -> _pandas.Series:
+    if isinstance(series, _pandas.Series):
+        return series
+    if _SPARK_PANDAS_AVAILABLE:
+        try:
+            import pyspark.pandas as ps
+            if isinstance(series, ps.Series):
+                return series.to_pandas()
+        except ImportError:
+            pass
+    return _pandas.Series(series)
+
+
+def concat(objs: list, axis: int = 0, ignore_index: bool = False, **kwargs: Any) -> Any:
+    if not objs:
+        return pd.DataFrame()
+    return pd.concat(objs, axis=axis, ignore_index=ignore_index, **kwargs)
+
+
+def merge(left: Any, right: Any, how: str = "inner", on: Any = None, **kwargs: Any) -> Any:
+    return pd.merge(left, right, how=how, on=on, **kwargs)
+
+
+Timestamp = _pandas.Timestamp
+Timedelta = _pandas.Timedelta
+DatetimeIndex = _pandas.DatetimeIndex
+CategoricalDtype = _pandas.CategoricalDtype
+NA = _pandas.NA
+NaT = _pandas.NaT
+
+api_types = _pandas.api.types
+
+
+def is_numeric_dtype(arr_or_dtype: Any) -> bool:
+    return _pandas.api.types.is_numeric_dtype(arr_or_dtype)
+
+
+def is_string_dtype(arr_or_dtype: Any) -> bool:
+    return _pandas.api.types.is_string_dtype(arr_or_dtype)
+
+
+def is_datetime64_any_dtype(arr_or_dtype: Any) -> bool:
+    return _pandas.api.types.is_datetime64_any_dtype(arr_or_dtype)
+
+
+def is_bool_dtype(arr_or_dtype: Any) -> bool:
+    return _pandas.api.types.is_bool_dtype(arr_or_dtype)
+
+
+def is_categorical_dtype(arr_or_dtype: Any) -> bool:
+    return _pandas.api.types.is_categorical_dtype(arr_or_dtype)
+
+
+def is_integer_dtype(arr_or_dtype: Any) -> bool:
+    return _pandas.api.types.is_integer_dtype(arr_or_dtype)
+
+
+def is_float_dtype(arr_or_dtype: Any) -> bool:
+    return _pandas.api.types.is_float_dtype(arr_or_dtype)
+
+
+class PandasCompat:
+    @staticmethod
+    def value_counts_normalize(series: Any, normalize: bool = False) -> Any:
+        return series.value_counts(normalize=normalize)
+
+    @staticmethod
+    def apply_with_meta(df: Any, func: Any, meta: Any = None, **kwargs: Any) -> Any:
+        return df.apply(func, **kwargs)
+
+    @staticmethod
+    def groupby_apply(grouped: Any, func: Any, **kwargs: Any) -> Any:
+        return grouped.apply(func, **kwargs)
+
+
+compat = PandasCompat()
+
+__all__ = [
+    "pd",
+    "DataFrame",
+    "Series",
+    "Timestamp",
+    "Timedelta",
+    "DatetimeIndex",
+    "CategoricalDtype",
+    "NA",
+    "NaT",
+    "is_spark_available",
+    "is_pandas_api_on_spark",
+    "get_pandas",
+    "to_pandas",
+    "to_spark_pandas",
+    "ensure_pandas_series",
+    "concat",
+    "merge",
+    "api_types",
+    "is_numeric_dtype",
+    "is_string_dtype",
+    "is_datetime64_any_dtype",
+    "is_bool_dtype",
+    "is_categorical_dtype",
+    "is_integer_dtype",
+    "is_float_dtype",
+    "get_spark_session",
+    "set_spark_config",
+    "enable_arrow_optimization",
+    "configure_spark_pandas",
+    "compat",
+    "PandasCompat",
+    "is_databricks",
+    "is_notebook",
+    "get_display_function",
+    "get_dbutils",
+    "ops",
+    "DataOps",
+]

customer_retention/core/compat/detection.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import os
+from typing import Any, Callable, Optional
+
+_SPARK_PANDAS_AVAILABLE = False
+_PANDAS_API_ON_SPARK = False
+
+try:
+    import pyspark.pandas as ps
+    _SPARK_PANDAS_AVAILABLE = True
+    _PANDAS_API_ON_SPARK = True
+except ImportError:
+    pass
+
+if not _SPARK_PANDAS_AVAILABLE:
+    try:
+        import databricks.koalas as ps
+        _SPARK_PANDAS_AVAILABLE = True
+    except ImportError:
+        pass
+
+
+def is_spark_available() -> bool:
+    return _SPARK_PANDAS_AVAILABLE
+
+
+def is_pandas_api_on_spark() -> bool:
+    return _PANDAS_API_ON_SPARK
+
+
+def is_databricks() -> bool:
+    return bool(os.environ.get("DATABRICKS_RUNTIME_VERSION"))
+
+
+def is_notebook() -> bool:
+    try:
+        shell = get_ipython().__class__.__name__  # type: ignore[name-defined]
+        return shell in ("ZMQInteractiveShell", "DatabricksShell", "Shell")
+    except NameError:
+        return False
+
+
+def get_spark_session() -> Optional[Any]:
+    if not _SPARK_PANDAS_AVAILABLE:
+        return None
+    try:
+        from pyspark.sql import SparkSession
+        return SparkSession.getActiveSession()
+    except Exception:
+        return None
+
+
+def get_display_function() -> Callable[[str], None]:
+    if is_databricks():
+        try:
+            return displayHTML  # type: ignore[name-defined]
+        except NameError:
+            pass
+    if is_notebook():
+        from IPython.display import HTML, display
+        return lambda html: display(HTML(html))
+    return print
+
+
+def get_dbutils() -> Optional[Any]:
+    if not is_databricks():
+        return None
+    try:
+        return dbutils  # type: ignore[name-defined]
+    except NameError:
+        spark = get_spark_session()
+        if spark:
+            try:
+                from pyspark.dbutils import DBUtils
+                return DBUtils(spark)
+            except ImportError:
+                pass
+    return None
+
+
+def set_spark_config(key: str, value: Any) -> None:
+    spark = get_spark_session()
+    if spark:
+        spark.conf.set(key, value)
+
+
+def enable_arrow_optimization() -> None:
+    set_spark_config("spark.sql.execution.arrow.pyspark.enabled", "true")
+
+
+def configure_spark_pandas(compute_max_rows: int = 1000, display_max_rows: int = 100) -> None:
+    if _PANDAS_API_ON_SPARK:
+        try:
+            import pyspark.pandas as ps
+            ps.set_option("compute.max_rows", compute_max_rows)
+            ps.set_option("display.max_rows", display_max_rows)
+        except Exception:
+            pass

customer_retention/core/compat/ops.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Union
+
+import pandas as pd
+
+from . import pandas_backend
+from .detection import is_spark_available
+
+
+class DataOps:
+    def __init__(self):
+        self._use_spark = is_spark_available()
+
+    def _get_backend(self) -> Any:
+        if self._use_spark:
+            from . import spark_backend
+            return spark_backend
+        return pandas_backend
+
+    def read_csv(self, path: str, **kwargs: Any) -> pd.DataFrame:
+        return self._get_backend().read_csv(path, **kwargs)
+
+    def read_delta(self, path: str, version: Optional[int] = None) -> pd.DataFrame:
+        return self._get_backend().read_delta(path, version=version)
+
+    def write_delta(self, df: Union[pd.DataFrame, Any], path: str, mode: str = "overwrite",
+                    partition_by: Optional[List[str]] = None) -> None:
+        self._get_backend().write_delta(df, path, mode=mode, partition_by=partition_by)
+
+    def get_missing_stats(self, df: Union[pd.DataFrame, Any]) -> Dict[str, float]:
+        return self._get_backend().get_missing_stats(df)
+
+    def correlation_matrix(self, df: Union[pd.DataFrame, Any],
+                           columns: Optional[List[str]] = None) -> pd.DataFrame:
+        return self._get_backend().correlation_matrix(df, columns=columns)
+
+    def get_dtype_info(self, df: Union[pd.DataFrame, Any]) -> Dict[str, str]:
+        return self._get_backend().get_dtype_info(df)
+
+    def sample(self, df: Union[pd.DataFrame, Any], n: int, random_state: int = 42) -> pd.DataFrame:
+        return self._get_backend().sample(df, n=n, random_state=random_state)
+
+    def concat(self, dfs: List[Union[pd.DataFrame, Any]], axis: int = 0) -> pd.DataFrame:
+        return self._get_backend().concat(dfs, axis=axis)
+
+
+ops = DataOps()

customer_retention/core/compat/pandas_backend.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+import pandas as pd
+
+try:
+    import deltalake
+    DELTA_RS_AVAILABLE = True
+except ImportError:
+    DELTA_RS_AVAILABLE = False
+
+
+def read_csv(path: str, **kwargs: Any) -> pd.DataFrame:
+    return pd.read_csv(path, **kwargs)
+
+
+def read_delta(path: str, version: Optional[int] = None) -> pd.DataFrame:
+    if not DELTA_RS_AVAILABLE:
+        raise ImportError("deltalake package required: pip install deltalake")
+    if version is not None:
+        dt = deltalake.DeltaTable(path, version=version)
+    else:
+        dt = deltalake.DeltaTable(path)
+    return dt.to_pandas()
+
+
+def write_delta(df: pd.DataFrame, path: str, mode: str = "overwrite",
+                partition_by: Optional[List[str]] = None) -> None:
+    if not DELTA_RS_AVAILABLE:
+        raise ImportError("deltalake package required: pip install deltalake")
+    from deltalake import write_deltalake
+    write_deltalake(path, df, mode=mode, partition_by=partition_by)
+
+
+def get_missing_stats(df: pd.DataFrame) -> Dict[str, float]:
+    return (df.isnull().sum() / len(df)).to_dict()
+
+
+def correlation_matrix(df: pd.DataFrame, columns: Optional[List[str]] = None) -> pd.DataFrame:
+    if columns:
+        return df[columns].corr()
+    return df.select_dtypes(include=["number"]).corr()
+
+
+def get_dtype_info(df: pd.DataFrame) -> Dict[str, str]:
+    return {col: str(dtype) for col, dtype in df.dtypes.items()}
+
+
+def sample(df: pd.DataFrame, n: int, random_state: int = 42) -> pd.DataFrame:
+    return df.sample(n=min(n, len(df)), random_state=random_state)
+
+
+def concat(dfs: List[pd.DataFrame], axis: int = 0, ignore_index: bool = True) -> pd.DataFrame:
+    if axis == 1:
+        return pd.concat(dfs, axis=axis, ignore_index=False)
+    return pd.concat(dfs, axis=axis, ignore_index=ignore_index)

customer_retention/core/compat/spark_backend.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+try:
+    import pyspark.pandas as ps
+    from pyspark.sql import SparkSession
+    SPARK_AVAILABLE = True
+except ImportError:
+    SPARK_AVAILABLE = False
+
+
+def _get_spark() -> Any:
+    if not SPARK_AVAILABLE:
+        raise ImportError("pyspark required for Spark backend")
+    return SparkSession.getActiveSession() or SparkSession.builder.getOrCreate()
+
+
+def read_csv(path: str, **kwargs: Any) -> Any:
+    if not SPARK_AVAILABLE:
+        raise ImportError("pyspark required")
+    return ps.read_csv(path, **kwargs)
+
+
+def read_delta(path: str, version: Optional[int] = None) -> Any:
+    if not SPARK_AVAILABLE:
+        raise ImportError("pyspark required")
+    spark = _get_spark()
+    reader = spark.read.format("delta")
+    if version is not None:
+        reader = reader.option("versionAsOf", version)
+    return reader.load(path).to_pandas_on_spark()
+
+
+def write_delta(df: Any, path: str, mode: str = "overwrite",
+                partition_by: Optional[List[str]] = None) -> None:
+    if not SPARK_AVAILABLE:
+        raise ImportError("pyspark required")
+    spark_df = df.to_spark() if hasattr(df, "to_spark") else df
+    writer = spark_df.write.format("delta").mode(mode)
+    if partition_by:
+        writer = writer.partitionBy(*partition_by)
+    writer.save(path)
+
+
+def get_missing_stats(df: Any) -> Dict[str, float]:
+    if not SPARK_AVAILABLE:
+        raise ImportError("pyspark required")
+    pdf = df.to_pandas() if hasattr(df, "to_pandas") else df
+    return (pdf.isnull().sum() / len(pdf)).to_dict()
+
+
+def correlation_matrix(df: Any, columns: Optional[List[str]] = None) -> Any:
+    if not SPARK_AVAILABLE:
+        raise ImportError("pyspark required")
+    if columns:
+        return df[columns].to_pandas().corr()
+    return df.select_dtypes(include=["number"]).to_pandas().corr()
+
+
+def get_dtype_info(df: Any) -> Dict[str, str]:
+    return {col: str(dtype) for col, dtype in df.dtypes.items()}
+
+
+def sample(df: Any, n: int, random_state: int = 42) -> Any:
+    if not SPARK_AVAILABLE:
+        raise ImportError("pyspark required")
+    fraction = min(1.0, n / len(df))
+    return df.sample(frac=fraction, random_state=random_state).head(n)
+
+
+def concat(dfs: List[Any], axis: int = 0, ignore_index: bool = True) -> Any:
+    if not SPARK_AVAILABLE:
+        raise ImportError("pyspark required")
+    return ps.concat(dfs, axis=axis, ignore_index=ignore_index)

customer_retention/core/components/__init__.py

@@ -0,0 +1,11 @@
+from .base import Component, ComponentResult, ComponentStatus
+from .enums import ModelType, Severity
+from .orchestrator import Orchestrator, OrchestratorResult
+from .registry import ComponentRegistration, ComponentRegistry, get_default_registry
+
+__all__ = [
+    "Component", "ComponentResult", "ComponentStatus",
+    "ComponentRegistry", "ComponentRegistration", "get_default_registry",
+    "Orchestrator", "OrchestratorResult",
+    "Severity", "ModelType"
+]

customer_retention/core/components/base.py

@@ -0,0 +1,79 @@
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+if TYPE_CHECKING:
+    from customer_retention.generators.orchestration.context import PipelineContext
+
+
+class ComponentStatus(str, Enum):
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    SKIPPED = "skipped"
+
+
+@dataclass
+class ComponentResult:
+    success: bool
+    status: ComponentStatus
+    artifacts: Dict[str, str] = field(default_factory=dict)
+    metrics: Dict[str, float] = field(default_factory=dict)
+    errors: List[str] = field(default_factory=list)
+    warnings: List[str] = field(default_factory=list)
+    duration_seconds: float = 0.0
+    output_data: Optional[Any] = None
+
+    def get_summary(self) -> str:
+        return f"{self.status.value.upper()} in {self.duration_seconds:.1f}s"
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "success": self.success,
+            "status": self.status.value,
+            "artifacts": self.artifacts,
+            "metrics": self.metrics,
+            "errors": self.errors,
+            "warnings": self.warnings,
+            "duration_seconds": self.duration_seconds,
+        }
+
+
+class Component(ABC):
+    def __init__(self, name: str, chapters: List[int]):
+        self.name = name
+        self.chapters = chapters
+        self._start_time: Optional[float] = None
+
+    @abstractmethod
+    def validate_inputs(self, context: "PipelineContext") -> List[str]:
+        pass
+
+    @abstractmethod
+    def run(self, context: "PipelineContext") -> ComponentResult:
+        pass
+
+    def should_skip(self, context: "PipelineContext") -> bool:
+        return False
+
+    def create_result(self, success: bool, artifacts: Optional[Dict[str, str]] = None,
+                      metrics: Optional[Dict[str, float]] = None, errors: Optional[List[str]] = None,
+                      warnings: Optional[List[str]] = None, output_data: Optional[Any] = None) -> ComponentResult:
+        duration = time.time() - self._start_time if self._start_time else 0.0
+        status = ComponentStatus.COMPLETED if success else ComponentStatus.FAILED
+        return ComponentResult(
+            success=success,
+            status=status,
+            artifacts=artifacts or {},
+            metrics=metrics or {},
+            errors=errors or [],
+            warnings=warnings or [],
+            duration_seconds=duration,
+            output_data=output_data
+        )
+
+    def _start_timer(self) -> None:
+        self._start_time = time.time()

customer_retention/core/components/components/__init__.py

@@ -0,0 +1,13 @@
+from .deployer import Deployer
+from .explainer import Explainer
+from .feature_eng import FeatureEngineer
+from .ingester import Ingester
+from .profiler import Profiler
+from .trainer import Trainer
+from .transformer import Transformer
+from .validator import Validator
+
+__all__ = [
+    "Ingester", "Profiler", "Transformer", "FeatureEngineer",
+    "Trainer", "Validator", "Explainer", "Deployer"
+]

customer_retention/core/components/components/deployer.py

@@ -0,0 +1,26 @@
+from typing import List
+
+from customer_retention.generators.orchestration.context import PipelineContext
+
+from ..base import Component, ComponentResult
+
+
+class Deployer(Component):
+    def __init__(self):
+        super().__init__(name="Deployer", chapters=[8])
+
+    def validate_inputs(self, context: PipelineContext) -> List[str]:
+        errors = []
+        if not context.model_results:
+            errors.append("No model results available for deployment")
+        return errors
+
+    def run(self, context: PipelineContext) -> ComponentResult:
+        self._start_timer()
+        try:
+            return self.create_result(
+                success=True,
+                metrics={"models_registered": 1}
+            )
+        except Exception as e:
+            return self.create_result(success=False, errors=[str(e)])

customer_retention/core/components/components/explainer.py

@@ -0,0 +1,26 @@
+from typing import List
+
+from customer_retention.generators.orchestration.context import PipelineContext
+
+from ..base import Component, ComponentResult
+
+
+class Explainer(Component):
+    def __init__(self):
+        super().__init__(name="Explainer", chapters=[7])
+
+    def validate_inputs(self, context: PipelineContext) -> List[str]:
+        errors = []
+        if not context.model_results:
+            errors.append("No model results available for explanation")
+        return errors
+
+    def run(self, context: PipelineContext) -> ComponentResult:
+        self._start_timer()
+        try:
+            return self.create_result(
+                success=True,
+                metrics={"explanations_generated": 1}
+            )
+        except Exception as e:
+            return self.create_result(success=False, errors=[str(e)])

customer_retention/core/components/components/feature_eng.py

@@ -0,0 +1,33 @@
+from typing import List
+
+from customer_retention.generators.orchestration.context import PipelineContext
+
+from ..base import Component, ComponentResult
+
+
+class FeatureEngineer(Component):
+    def __init__(self):
+        super().__init__(name="FeatureEngineer", chapters=[4])
+
+    def validate_inputs(self, context: PipelineContext) -> List[str]:
+        errors = []
+        if context.current_df is None:
+            errors.append("No DataFrame available for feature engineering")
+        return errors
+
+    def run(self, context: PipelineContext) -> ComponentResult:
+        self._start_timer()
+        try:
+            from customer_retention.stages.features.feature_engineer import FeatureEngineer as FE
+            df = context.current_df
+            fe = FE()
+            df = fe.engineer_all(df, context.column_configs)
+            context.current_df = df
+            context.current_stage = "gold"
+            return self.create_result(
+                success=True,
+                artifacts={"gold_data": context.gold_path} if context.gold_path else {},
+                metrics={"feature_count": len(df.columns)}
+            )
+        except Exception as e:
+            return self.create_result(success=False, errors=[str(e)])