churnkit 0.75.0a1__py3-none-any.whl

customer_retention/generators/pipeline_generator/generator.py

@@ -0,0 +1,142 @@
+from pathlib import Path
+from typing import List
+
+from .findings_parser import FindingsParser
+from .models import PipelineConfig
+from .renderer import CodeRenderer
+
+
+class PipelineGenerator:
+    def __init__(self, findings_dir: str, output_dir: str, pipeline_name: str, experiments_dir: str = None, production_dir: str = None):
+        self._findings_dir = Path(findings_dir)
+        self._output_dir = Path(output_dir)
+        self._pipeline_name = pipeline_name
+        self._experiments_dir = experiments_dir
+        self._production_dir = production_dir
+        self._parser = FindingsParser(findings_dir)
+        self._renderer = CodeRenderer()
+
+    def generate(self) -> List[Path]:
+        config = self._parser.parse()
+        config.name = self._pipeline_name
+        config.output_dir = "."
+        config.experiments_dir = self._experiments_dir
+        config.production_dir = self._production_dir
+        self._renderer.set_docs_base(self._experiments_dir)
+        generated_files = [
+            self._write_run_all(config),
+            self._write_config(config),
+            *self._write_landing(config),
+            *self._write_bronze_files(config),
+            self._write_silver(config),
+            self._write_gold(config),
+            self._write_training(config),
+            self._write_runner(config),
+            self._write_workflow(config),
+            *self._write_feast_repo(config),
+            *self._write_validation(config),
+            self._write_exploration_report(config),
+        ]
+        return generated_files
+
+    def _write_run_all(self, config: PipelineConfig) -> Path:
+        path = self._output_dir / "run_all.py"
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(self._renderer.render_run_all(config))
+        return path
+
+    def _write_config(self, config: PipelineConfig) -> Path:
+        path = self._output_dir / "config.py"
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(self._renderer.render_config(config))
+        return path
+
+    def _write_landing(self, config: PipelineConfig) -> List[Path]:
+        if not config.landing:
+            return []
+        landing_dir = self._output_dir / "landing"
+        landing_dir.mkdir(parents=True, exist_ok=True)
+        paths = []
+        for landing_name, landing_config in config.landing.items():
+            path = landing_dir / f"landing_{landing_name}.py"
+            path.write_text(self._renderer.render_landing(landing_name, landing_config))
+            paths.append(path)
+        return paths
+
+    def _write_bronze_files(self, config: PipelineConfig) -> List[Path]:
+        bronze_dir = self._output_dir / "bronze"
+        bronze_dir.mkdir(parents=True, exist_ok=True)
+        paths = []
+        for source_name, bronze_config in config.bronze.items():
+            path = bronze_dir / f"bronze_{source_name}.py"
+            path.write_text(self._renderer.render_bronze(source_name, bronze_config))
+            paths.append(path)
+        for source_name, event_config in config.bronze_event.items():
+            path = bronze_dir / f"bronze_{source_name}.py"
+            path.write_text(self._renderer.render_bronze_event(source_name, event_config))
+            paths.append(path)
+        return paths
+
+    def _write_silver(self, config: PipelineConfig) -> Path:
+        path = self._output_dir / "silver" / "silver_merge.py"
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(self._renderer.render_silver(config))
+        return path
+
+    def _write_gold(self, config: PipelineConfig) -> Path:
+        path = self._output_dir / "gold" / "gold_features.py"
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(self._renderer.render_gold(config))
+        return path
+
+    def _write_training(self, config: PipelineConfig) -> Path:
+        path = self._output_dir / "training" / "ml_experiment.py"
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(self._renderer.render_training(config))
+        return path
+
+    def _write_runner(self, config: PipelineConfig) -> Path:
+        path = self._output_dir / "pipeline_runner.py"
+        path.write_text(self._renderer.render_runner(config))
+        return path
+
+    def _write_workflow(self, config: PipelineConfig) -> Path:
+        path = self._output_dir / "workflow.json"
+        path.write_text(self._renderer.render_workflow(config))
+        return path
+
+    def _write_feast_repo(self, config: PipelineConfig) -> List[Path]:
+        feast_dir = self._output_dir / "feature_repo"
+        feast_dir.mkdir(parents=True, exist_ok=True)
+        (feast_dir / "data").mkdir(parents=True, exist_ok=True)
+        paths = []
+        config_path = feast_dir / "feature_store.yaml"
+        config_path.write_text(self._renderer.render_feast_config(config))
+        paths.append(config_path)
+        features_path = feast_dir / "features.py"
+        features_path.write_text(self._renderer.render_feast_features(config))
+        paths.append(features_path)
+        return paths
+
+
+    def _write_exploration_report(self, config: PipelineConfig) -> Path:
+        docs_dir = self._output_dir / "docs"
+        docs_dir.mkdir(parents=True, exist_ok=True)
+        path = docs_dir / "exploration_report.py"
+        path.write_text(self._renderer.render_exploration_report(config))
+        return path
+
+    def _write_validation(self, config: PipelineConfig) -> List[Path]:
+        validation_dir = self._output_dir / "validation"
+        validation_dir.mkdir(parents=True, exist_ok=True)
+        paths = []
+        init_path = validation_dir / "__init__.py"
+        init_path.write_text("")
+        paths.append(init_path)
+        validate_path = validation_dir / "validate_pipeline.py"
+        validate_path.write_text(self._renderer.render_validation(config))
+        paths.append(validate_path)
+        run_validation_path = validation_dir / "run_validation.py"
+        run_validation_path.write_text(self._renderer.render_run_validation(config))
+        paths.append(run_validation_path)
+        return paths

customer_retention/generators/pipeline_generator/models.py

@@ -0,0 +1,166 @@
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+
+class PipelineTransformationType(Enum):
+    IMPUTE_NULL = "impute_null"
+    CAP_OUTLIER = "cap_outlier"
+    TYPE_CAST = "type_cast"
+    ENCODE = "encode"
+    SCALE = "scale"
+    AGGREGATE = "aggregate"
+    JOIN = "join"
+    DROP_COLUMN = "drop_column"
+    WINSORIZE = "winsorize"
+    SEGMENT_AWARE_CAP = "segment_aware_cap"
+    LOG_TRANSFORM = "log_transform"
+    SQRT_TRANSFORM = "sqrt_transform"
+    YEO_JOHNSON = "yeo_johnson"
+    ZERO_INFLATION_HANDLING = "zero_inflation_handling"
+    CAP_THEN_LOG = "cap_then_log"
+    FEATURE_SELECT = "feature_select"
+    DERIVED_COLUMN = "derived_column"
+
+
+@dataclass
+class SourceConfig:
+    name: str
+    path: str
+    format: str
+    entity_key: str
+    raw_source_path: str = ""
+    time_column: Optional[str] = None
+    is_event_level: bool = False
+    excluded: bool = False
+
+
+@dataclass
+class TransformationStep:
+    type: PipelineTransformationType
+    column: str
+    parameters: Dict[str, Any]
+    rationale: str
+    source_notebook: Optional[str] = None
+
+
+@dataclass
+class BronzeLayerConfig:
+    source: SourceConfig
+    transformations: List[TransformationStep] = field(default_factory=list)
+    lifecycle: Optional["LifecycleConfig"] = None
+    entity_column: Optional[str] = None
+    time_column: Optional[str] = None
+    raw_time_column: Optional[str] = None
+
+
+@dataclass
+class SilverLayerConfig:
+    joins: List[Dict[str, str]] = field(default_factory=list)
+    aggregations: List[Dict[str, Any]] = field(default_factory=list)
+    derived_columns: List[TransformationStep] = field(default_factory=list)
+
+
+@dataclass
+class GoldLayerConfig:
+    encodings: List[TransformationStep] = field(default_factory=list)
+    scalings: List[TransformationStep] = field(default_factory=list)
+    feature_selections: List[str] = field(default_factory=list)
+    transformations: List[TransformationStep] = field(default_factory=list)
+
+
+@dataclass
+class FeastConfig:
+    repo_path: str = "./feature_repo"
+    feature_view_name: str = "customer_features"
+    entity_name: str = "customer"
+    entity_key: str = "customer_id"
+    timestamp_column: str = "event_timestamp"
+    ttl_days: int = 365
+    exclude_prefixes: List[str] = field(default_factory=lambda: ["original_"])
+
+
+@dataclass
+class ScoringConfig:
+    holdout_manifest_path: Optional[str] = None
+    original_column: Optional[str] = None
+    model_uri: Optional[str] = None
+    output_predictions_path: Optional[str] = None
+
+
+@dataclass
+class AggregationWindowConfig:
+    windows: List[str] = field(default_factory=list)
+    value_columns: List[str] = field(default_factory=list)
+    agg_funcs: List[str] = field(default_factory=list)
+    reference_date: Optional[str] = None
+
+
+@dataclass
+class LifecycleConfig:
+    include_lifecycle_quadrant: bool = False
+    include_cyclical_features: bool = False
+    include_recency_bucket: bool = False
+    momentum_pairs: List[Dict[str, Any]] = field(default_factory=list)
+
+
+@dataclass
+class TimestampCoalesceConfig:
+    datetime_columns_ordered: List[str]
+    output_column: str = "feature_timestamp"
+
+
+@dataclass
+class LabelTimestampConfig:
+    label_column: Optional[str] = None
+    fallback_window_days: int = 180
+    output_column: str = "label_timestamp"
+
+
+@dataclass
+class BronzeEventConfig:
+    source: SourceConfig
+    entity_column: str
+    time_column: str
+    deduplicate: bool = False
+    pre_shaping: List[TransformationStep] = field(default_factory=list)
+    aggregation: Optional[AggregationWindowConfig] = None
+    lifecycle: Optional[LifecycleConfig] = None
+    post_shaping: List[TransformationStep] = field(default_factory=list)
+    raw_time_column: Optional[str] = None
+
+
+@dataclass
+class LandingLayerConfig:
+    source: SourceConfig
+    raw_source_path: str
+    raw_source_format: str
+    entity_column: str
+    time_column: str
+    target_column: str
+    original_target_column: Optional[str] = None
+    raw_time_column: Optional[str] = None
+    timestamp_coalesce: Optional[TimestampCoalesceConfig] = None
+    label_timestamp: Optional[LabelTimestampConfig] = None
+
+
+@dataclass
+class PipelineConfig:
+    name: str
+    target_column: str
+    sources: List[SourceConfig]
+    bronze: Dict[str, BronzeLayerConfig]
+    silver: SilverLayerConfig
+    gold: GoldLayerConfig
+    output_dir: str
+    iteration_id: Optional[str] = None
+    parent_iteration_id: Optional[str] = None
+    recommendations_hash: Optional[str] = None
+    feast: Optional[FeastConfig] = None
+    scoring: Optional[ScoringConfig] = None
+    experiments_dir: Optional[str] = None
+    production_dir: Optional[str] = None
+    fit_mode: bool = True
+    artifacts_path: Optional[str] = None
+    landing: Dict[str, LandingLayerConfig] = field(default_factory=dict)
+    bronze_event: Dict[str, BronzeEventConfig] = field(default_factory=dict)