churnkit 0.75.0a1__py3-none-any.whl

customer_retention/stages/transformation/datetime_transformer.py

@@ -0,0 +1,97 @@
+from dataclasses import dataclass, field
+from typing import Optional, Union
+
+import numpy as np
+
+from customer_retention.core.compat import DataFrame, Series, Timestamp, is_datetime64_any_dtype, pd
+
+
+@dataclass
+class DatetimeTransformResult:
+    df: DataFrame
+    extracted_features: list = field(default_factory=list)
+    cyclical_features: list = field(default_factory=list)
+    drop_original: bool = True
+
+
+class DatetimeTransformer:
+    FEATURE_EXTRACTORS = {
+        "year": lambda s: s.dt.year,
+        "month": lambda s: s.dt.month,
+        "day": lambda s: s.dt.day,
+        "day_of_week": lambda s: s.dt.dayofweek,
+        "day_of_year": lambda s: s.dt.dayofyear,
+        "week_of_year": lambda s: s.dt.isocalendar().week.astype(int),
+        "quarter": lambda s: s.dt.quarter,
+        "hour": lambda s: s.dt.hour,
+        "minute": lambda s: s.dt.minute,
+        "is_weekend": lambda s: s.dt.dayofweek.isin([5, 6]).astype(int),
+        "is_month_start": lambda s: s.dt.is_month_start.astype(int),
+        "is_month_end": lambda s: s.dt.is_month_end.astype(int),
+        "is_quarter_start": lambda s: s.dt.is_quarter_start.astype(int),
+        "is_quarter_end": lambda s: s.dt.is_quarter_end.astype(int),
+    }
+
+    CYCLICAL_PERIODS = {
+        "month": 12,
+        "day_of_week": 7,
+        "day_of_year": 365,
+        "quarter": 4,
+        "hour": 24,
+        "minute": 60,
+    }
+
+    def __init__(
+        self,
+        extract_features: Optional[list[str]] = None,
+        cyclical_features: Optional[list[str]] = None,
+        reference_date: Optional[Union[str, Timestamp]] = None,
+        drop_original: bool = True
+    ):
+        self.extract_features = extract_features or ["year", "month", "day_of_week"]
+        self.cyclical_features = cyclical_features or []
+        self.reference_date = Timestamp(reference_date) if reference_date else None
+        self.drop_original = drop_original
+
+    def fit(self, series: Series) -> "DatetimeTransformer":
+        return self
+
+    def transform(self, series: Series) -> DatetimeTransformResult:
+        return self._apply_transform(series)
+
+    def fit_transform(self, series: Series) -> DatetimeTransformResult:
+        return self._apply_transform(series)
+
+    def _apply_transform(self, series: Series) -> DatetimeTransformResult:
+        dt_series = self._ensure_datetime(series)
+        result_dict = {}
+        extracted = []
+
+        for feature in self.extract_features:
+            if feature in self.FEATURE_EXTRACTORS:
+                values = self.FEATURE_EXTRACTORS[feature](dt_series)
+                result_dict[feature] = values
+                extracted.append(feature)
+
+                if feature in self.cyclical_features:
+                    period = self.CYCLICAL_PERIODS.get(feature)
+                    if period:
+                        sin_col = f"{feature}_sin"
+                        cos_col = f"{feature}_cos"
+                        result_dict[sin_col] = np.sin(2 * np.pi * values / period)
+                        result_dict[cos_col] = np.cos(2 * np.pi * values / period)
+
+        if self.reference_date is not None:
+            result_dict["days_since"] = (self.reference_date - dt_series).dt.days
+
+        df = DataFrame(result_dict)
+
+        return DatetimeTransformResult(
+            df=df, extracted_features=extracted,
+            cyclical_features=self.cyclical_features, drop_original=self.drop_original
+        )
+
+    def _ensure_datetime(self, series: Series) -> Series:
+        if is_datetime64_any_dtype(series):
+            return series
+        return pd.to_datetime(series, errors='coerce', format='mixed')

customer_retention/stages/transformation/numeric_transformer.py

@@ -0,0 +1,181 @@
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Optional
+
+import numpy as np
+
+from customer_retention.core.compat import Series, pd
+
+
+class ScalingStrategy(str, Enum):
+    STANDARD = "standard"
+    MINMAX = "minmax"
+    ROBUST = "robust"
+    MAXABS = "maxabs"
+    NONE = "none"
+
+
+class PowerTransform(str, Enum):
+    LOG = "log"
+    LOG1P = "log1p"
+    SQRT = "sqrt"
+    BOXCOX = "boxcox"
+    YEOJOHNSON = "yeojohnson"
+    NONE = "none"
+
+
+@dataclass
+class NumericTransformResult:
+    series: Series
+    original_mean: float
+    original_std: float
+    original_min: float
+    original_max: float
+    transformed_mean: float
+    transformed_std: float
+    transformations_applied: list = field(default_factory=list)
+    scaler_params: dict = field(default_factory=dict)
+
+
+class NumericTransformer:
+    def __init__(
+        self,
+        scaling: ScalingStrategy = ScalingStrategy.NONE,
+        power_transform: PowerTransform = PowerTransform.NONE
+    ):
+        self.scaling = scaling
+        self.power_transform = power_transform
+        self._mean: Optional[float] = None
+        self._std: Optional[float] = None
+        self._min: Optional[float] = None
+        self._max: Optional[float] = None
+        self._median: Optional[float] = None
+        self._iqr: Optional[float] = None
+        self._max_abs: Optional[float] = None
+        self._is_fitted = False
+
+    def fit(self, series: Series) -> "NumericTransformer":
+        clean = series.dropna()
+        transformed = self._apply_power_transform(clean)
+
+        self._mean = float(transformed.mean())
+        self._std = float(transformed.std(ddof=0))
+        self._min = float(transformed.min())
+        self._max = float(transformed.max())
+        self._median = float(transformed.median())
+        q1, q3 = transformed.quantile(0.25), transformed.quantile(0.75)
+        self._iqr = float(q3 - q1)
+        self._max_abs = float(transformed.abs().max())
+        self._is_fitted = True
+        return self
+
+    def transform(self, series: Series) -> NumericTransformResult:
+        if not self._is_fitted:
+            raise ValueError("Transformer not fitted. Call fit() or fit_transform() first.")
+        return self._apply_transformations(series)
+
+    def fit_transform(self, series: Series) -> NumericTransformResult:
+        self.fit(series)
+        return self._apply_transformations(series)
+
+    def inverse_transform(self, series: Series) -> Series:
+        result = series.copy()
+
+        if self.scaling == ScalingStrategy.STANDARD:
+            result = result * self._std + self._mean
+        elif self.scaling == ScalingStrategy.MINMAX:
+            result = result * (self._max - self._min) + self._min
+        elif self.scaling == ScalingStrategy.ROBUST:
+            result = result * self._iqr + self._median
+        elif self.scaling == ScalingStrategy.MAXABS:
+            result = result * self._max_abs
+
+        if self.power_transform == PowerTransform.LOG:
+            result = np.exp(result)
+        elif self.power_transform == PowerTransform.LOG1P:
+            result = np.expm1(result)
+        elif self.power_transform == PowerTransform.SQRT:
+            result = result ** 2
+
+        return result
+
+    def _apply_power_transform(self, series: Series) -> Series:
+        if self.power_transform == PowerTransform.NONE:
+            return series
+
+        if self.power_transform == PowerTransform.LOG:
+            if (series <= 0).any():
+                raise ValueError("Log transform requires positive values")
+            return np.log(series)
+
+        if self.power_transform == PowerTransform.LOG1P:
+            if (series < 0).any():
+                raise ValueError("Log1p transform requires non-negative values")
+            return np.log1p(series)
+
+        if self.power_transform == PowerTransform.SQRT:
+            if (series < 0).any():
+                raise ValueError("Sqrt transform requires non-negative values")
+            return np.sqrt(series)
+
+        return series
+
+    def _apply_scaling(self, series: Series) -> Series:
+        if self.scaling == ScalingStrategy.NONE:
+            return series
+
+        if self.scaling == ScalingStrategy.STANDARD:
+            if self._std == 0:
+                return series - self._mean
+            return (series - self._mean) / self._std
+
+        if self.scaling == ScalingStrategy.MINMAX:
+            range_val = self._max - self._min
+            if range_val == 0:
+                return pd.Series(0.0, index=series.index)
+            return (series - self._min) / range_val
+
+        if self.scaling == ScalingStrategy.ROBUST:
+            if self._iqr == 0:
+                return series - self._median
+            return (series - self._median) / self._iqr
+
+        if self.scaling == ScalingStrategy.MAXABS:
+            if self._max_abs == 0:
+                return series
+            return series / self._max_abs
+
+        return series
+
+    def _apply_transformations(self, series: Series) -> NumericTransformResult:
+        original_clean = series.dropna()
+        original_mean = float(original_clean.mean())
+        original_std = float(original_clean.std(ddof=0))
+        original_min = float(original_clean.min())
+        original_max = float(original_clean.max())
+
+        transformations = []
+
+        mask = series.notna()
+        result = series.copy()
+
+        if self.power_transform != PowerTransform.NONE:
+            result.loc[mask] = self._apply_power_transform(series[mask])
+            transformations.append(self.power_transform)
+
+        if self.scaling != ScalingStrategy.NONE:
+            result.loc[mask] = self._apply_scaling(result[mask])
+            transformations.append(self.scaling)
+
+        result_clean = result.dropna()
+        transformed_mean = float(result_clean.mean()) if len(result_clean) > 0 else 0.0
+        transformed_std = float(result_clean.std(ddof=0)) if len(result_clean) > 0 else 0.0
+
+        return NumericTransformResult(
+            series=result,
+            original_mean=original_mean, original_std=original_std,
+            original_min=original_min, original_max=original_max,
+            transformed_mean=transformed_mean, transformed_std=transformed_std,
+            transformations_applied=transformations,
+            scaler_params={"mean": self._mean, "std": self._std, "min": self._min, "max": self._max}
+        )

customer_retention/stages/transformation/pipeline.py

@@ -0,0 +1,257 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Optional
+
+import numpy as np
+
+from customer_retention.core.compat import DataFrame, pd
+from customer_retention.core.config import ColumnType
+from customer_retention.stages.cleaning import MissingValueHandler, OutlierHandler, OutlierTreatmentStrategy
+
+from .binary_handler import BinaryHandler
+from .categorical_encoder import CategoricalEncoder, EncodingStrategy
+from .datetime_transformer import DatetimeTransformer
+from .numeric_transformer import NumericTransformer, ScalingStrategy
+
+
+@dataclass
+class TransformationManifest:
+    timestamp: str = ""
+    version: str = "1.0"
+    input_rows: int = 0
+    input_columns: int = 0
+    output_rows: int = 0
+    output_columns: int = 0
+    columns_dropped: dict = field(default_factory=dict)
+    missing_value_handling: dict = field(default_factory=dict)
+    outlier_treatment: dict = field(default_factory=dict)
+    numeric_transformations: dict = field(default_factory=dict)
+    categorical_encodings: dict = field(default_factory=dict)
+    datetime_transformations: dict = field(default_factory=dict)
+    binary_mappings: dict = field(default_factory=dict)
+    column_mapping: dict = field(default_factory=dict)
+    final_schema: dict = field(default_factory=dict)
+    execution_order: list = field(default_factory=list)
+
+
+@dataclass
+class PipelineResult:
+    df: DataFrame
+    manifest: TransformationManifest
+    validation_passed: bool = True
+    validation_errors: list = field(default_factory=list)
+
+
+class TransformationPipeline:
+    EXECUTION_ORDER = [
+        "drop_columns", "handle_missing", "treat_outliers",
+        "transform_datetime", "transform_numeric",
+        "encode_categorical", "standardize_binary", "validate"
+    ]
+
+    def __init__(
+        self,
+        column_types: Optional[dict[str, ColumnType]] = None,
+        auto_from_profile: bool = True,
+        column_configs: Optional[dict] = None,
+        drop_constant_columns: bool = False,
+        drop_high_missing: bool = True,
+        create_missing_indicators: bool = False,
+        validate_output: bool = True
+    ):
+        self.column_types = column_types or {}
+        self.auto_from_profile = auto_from_profile
+        self.column_configs = column_configs or {}
+        self.drop_constant_columns = drop_constant_columns
+        self.drop_high_missing = drop_high_missing
+        self.create_missing_indicators = create_missing_indicators
+        self.validate_output = validate_output
+
+        self._missing_handlers: dict[str, MissingValueHandler] = {}
+        self._outlier_handlers: dict[str, OutlierHandler] = {}
+        self._numeric_transformers: dict[str, NumericTransformer] = {}
+        self._categorical_encoders: dict[str, CategoricalEncoder] = {}
+        self._datetime_transformers: dict[str, DatetimeTransformer] = {}
+        self._binary_handlers: dict[str, BinaryHandler] = {}
+        self._columns_to_drop: list[str] = []
+        self._is_fitted = False
+
+    def fit(self, df: DataFrame) -> "TransformationPipeline":
+        self._identify_columns_to_drop(df)
+        working_df = df.drop(columns=self._columns_to_drop, errors='ignore')
+
+        for col, col_type in self.column_types.items():
+            if col in self._columns_to_drop or col not in working_df.columns:
+                continue
+            self._fit_column(working_df, col, col_type)
+
+        self._is_fitted = True
+        return self
+
+    def transform(self, df: DataFrame) -> PipelineResult:
+        if not self._is_fitted:
+            raise ValueError("Pipeline not fitted. Call fit() or fit_transform() first.")
+        return self._apply_transformations(df)
+
+    def fit_transform(self, df: DataFrame) -> PipelineResult:
+        self.fit(df)
+        return self._apply_transformations(df)
+
+    def _identify_columns_to_drop(self, df: DataFrame):
+        self._columns_to_drop = []
+        for col, col_type in self.column_types.items():
+            if col not in df.columns:
+                continue
+            if col_type == ColumnType.IDENTIFIER:
+                self._columns_to_drop.append(col)
+            if self.drop_high_missing and df[col].isna().mean() > 0.95:
+                self._columns_to_drop.append(col)
+            if self.drop_constant_columns and df[col].nunique() <= 1:
+                self._columns_to_drop.append(col)
+
+    def _fit_column(self, df: DataFrame, col: str, col_type: ColumnType):
+        if col_type == ColumnType.TARGET:
+            return
+
+        series = df[col]
+        config = self.column_configs.get(col, {})
+
+        if series.isna().any():
+            handler = MissingValueHandler.from_column_type(col_type)
+            if "missing_strategy" in config:
+                from customer_retention.stages.cleaning import ImputationStrategy
+                handler.strategy = ImputationStrategy(config["missing_strategy"])
+            handler.fit(series)
+            self._missing_handlers[col] = handler
+
+        if col_type in [ColumnType.NUMERIC_CONTINUOUS, ColumnType.NUMERIC_DISCRETE]:
+            self._outlier_handlers[col] = OutlierHandler(
+                treatment_strategy=OutlierTreatmentStrategy.CAP_IQR
+            )
+            self._outlier_handlers[col].fit(series.dropna())
+
+            # Fit numeric transformer on CAPPED data to ensure proper scaling
+            outlier_result = self._outlier_handlers[col].transform(series.dropna())
+            self._numeric_transformers[col] = NumericTransformer(scaling=ScalingStrategy.STANDARD)
+            self._numeric_transformers[col].fit(outlier_result.series)
+
+        elif col_type in [ColumnType.CATEGORICAL_NOMINAL, ColumnType.CATEGORICAL_ORDINAL]:
+            self._categorical_encoders[col] = CategoricalEncoder(
+                strategy=EncodingStrategy.ONE_HOT, drop_first=True
+            )
+            self._categorical_encoders[col].fit(series)
+
+        elif col_type == ColumnType.DATETIME:
+            self._datetime_transformers[col] = DatetimeTransformer()
+            self._datetime_transformers[col].fit(series)
+
+        elif col_type == ColumnType.BINARY:
+            self._binary_handlers[col] = BinaryHandler()
+            self._binary_handlers[col].fit(series)
+
+    def _apply_transformations(self, df: DataFrame) -> PipelineResult:
+        manifest = TransformationManifest(
+            timestamp=datetime.now().isoformat(),
+            input_rows=len(df), input_columns=len(df.columns),
+            execution_order=self.EXECUTION_ORDER
+        )
+
+        working_df = df.copy()
+
+        manifest.columns_dropped = {col: "identifier/high_missing/constant" for col in self._columns_to_drop}
+        working_df = working_df.drop(columns=self._columns_to_drop, errors='ignore')
+
+        for col, handler in self._missing_handlers.items():
+            if col in working_df.columns:
+                result = handler.transform(working_df[col])
+                working_df[col] = result.series
+                manifest.missing_value_handling[col] = {
+                    "strategy": str(result.strategy_used), "values_imputed": result.values_imputed
+                }
+
+        for col, handler in self._outlier_handlers.items():
+            if col in working_df.columns:
+                result = handler.transform(working_df[col])
+                working_df[col] = result.series
+                manifest.outlier_treatment[col] = {
+                    "method": str(result.method_used),
+                    "outliers_detected": result.outliers_detected
+                }
+
+        datetime_cols_to_drop = []
+        datetime_extracted_cols = []
+        for col, transformer in self._datetime_transformers.items():
+            if col in working_df.columns:
+                result = transformer.transform(working_df[col])
+                for new_col in result.df.columns:
+                    working_df[new_col] = result.df[new_col].values
+                    datetime_extracted_cols.append(new_col)
+                datetime_cols_to_drop.append(col)
+                manifest.datetime_transformations[col] = {
+                    "extracted": result.extracted_features
+                }
+                manifest.column_mapping[col] = list(result.df.columns)
+        working_df = working_df.drop(columns=datetime_cols_to_drop, errors='ignore')
+
+        # Handle NaN values from invalid datetime parsing (e.g., '1/0/00')
+        for col in datetime_extracted_cols:
+            if col in working_df.columns and working_df[col].isna().any():
+                # Fill with median for extracted datetime features
+                median_val = working_df[col].median()
+                if pd.notna(median_val):
+                    working_df[col] = working_df[col].fillna(median_val)
+
+        for col, transformer in self._numeric_transformers.items():
+            if col in working_df.columns:
+                result = transformer.transform(working_df[col])
+                working_df[col] = result.series
+                manifest.numeric_transformations[col] = {
+                    "transformations": [str(t) for t in result.transformations_applied]
+                }
+
+        categorical_cols_to_drop = []
+        for col, encoder in self._categorical_encoders.items():
+            if col in working_df.columns:
+                result = encoder.transform(working_df[col])
+                if result.df is not None:
+                    for new_col in result.df.columns:
+                        working_df[new_col] = result.df[new_col].values
+                    categorical_cols_to_drop.append(col)
+                    manifest.column_mapping[col] = list(result.df.columns)
+                manifest.categorical_encodings[col] = {
+                    "strategy": str(result.strategy), "columns_created": result.columns_created
+                }
+        working_df = working_df.drop(columns=categorical_cols_to_drop, errors='ignore')
+
+        for col, handler in self._binary_handlers.items():
+            if col in working_df.columns:
+                result = handler.transform(working_df[col])
+                working_df[col] = result.series
+                manifest.binary_mappings[col] = {"mapping": result.mapping}
+
+        validation_passed, validation_errors = self._validate_output(working_df)
+
+        manifest.output_rows = len(working_df)
+        manifest.output_columns = len(working_df.columns)
+        manifest.final_schema = {col: str(working_df[col].dtype) for col in working_df.columns}
+
+        return PipelineResult(
+            df=working_df, manifest=manifest,
+            validation_passed=validation_passed, validation_errors=validation_errors
+        )
+
+    def _validate_output(self, df: DataFrame) -> tuple[bool, list[str]]:
+        errors = []
+
+        target_cols = [c for c, t in self.column_types.items() if t == ColumnType.TARGET and c in df.columns]
+        non_target = df.drop(columns=target_cols, errors='ignore')
+
+        if non_target.isna().any().any():
+            null_cols = non_target.columns[non_target.isna().any()].tolist()
+            errors.append(f"TQ001: Null values in columns: {null_cols}")
+
+        numeric_df = non_target.select_dtypes(include=[np.number])
+        if np.isinf(numeric_df.values).any():
+            errors.append("TQ002: Infinite values found")
+
+        return len(errors) == 0, errors

customer_retention/stages/validation/__init__.py

@@ -0,0 +1,60 @@
+from customer_retention.core.components.enums import Severity
+
+from .adversarial_scoring_validator import (
+    AdversarialScoringValidator,
+    AdversarialValidationResult,
+    DriftSeverity,
+    FeatureDrift,
+)
+from .business_sense_gate import BusinessCheck, BusinessSenseGate, BusinessSenseResult
+from .data_quality_gate import DataQualityGate
+from .data_validators import DataValidator, DateLogicResult, DuplicateResult, RangeValidationResult
+from .feature_quality_gate import FeatureQualityGate
+from .gates import GateResult, ValidationGate, ValidationIssue
+from .leakage_gate import LeakageCheckResult, LeakageGate
+from .model_validity_gate import ModelValidityGate, ModelValidityResult
+from .pipeline_validation_runner import (
+    PipelineValidationConfig,
+    PipelineValidationRunner,
+    compare_pipeline_outputs,
+    run_pipeline_validation,
+    validate_feature_transformation,
+)
+from .quality_scorer import QualityLevel, QualityScorer, QualityScoreResult
+from .rule_generator import RuleGenerator
+from .scoring_pipeline_validator import (
+    FeatureMismatch,
+    MismatchSeverity,
+    PredictionMismatch,
+    ScoringPipelineValidator,
+    ValidationConfig,
+    ValidationReport,
+)
+from .timeseries_detector import (
+    DatasetType,
+    TimeSeriesCharacteristics,
+    TimeSeriesDetector,
+    TimeSeriesFrequency,
+    TimeSeriesValidationResult,
+    TimeSeriesValidator,
+)
+
+__all__ = [
+    "Severity", "ValidationIssue", "GateResult", "ValidationGate",
+    "DataQualityGate", "FeatureQualityGate",
+    "LeakageGate", "LeakageCheckResult",
+    "ModelValidityGate", "ModelValidityResult",
+    "BusinessSenseGate", "BusinessSenseResult", "BusinessCheck",
+    "DataValidator", "DuplicateResult", "DateLogicResult", "RangeValidationResult",
+    "QualityScorer", "QualityScoreResult", "QualityLevel",
+    "RuleGenerator",
+    "ScoringPipelineValidator", "ValidationReport", "ValidationConfig",
+    "FeatureMismatch", "PredictionMismatch", "MismatchSeverity",
+    "PipelineValidationRunner", "PipelineValidationConfig",
+    "run_pipeline_validation", "validate_feature_transformation", "compare_pipeline_outputs",
+    "TimeSeriesDetector", "TimeSeriesValidator",
+    "TimeSeriesCharacteristics", "TimeSeriesValidationResult",
+    "DatasetType", "TimeSeriesFrequency",
+    "AdversarialScoringValidator", "AdversarialValidationResult",
+    "FeatureDrift", "DriftSeverity",
+]