churnkit 0.75.0a1__py3-none-any.whl

customer_retention/generators/pipeline_generator/renderer.py

@@ -0,0 +1,2125 @@
+from collections import OrderedDict, namedtuple
+from pathlib import Path
+from typing import List, Tuple
+
+from jinja2 import BaseLoader, Environment
+
+from .models import (
+    BronzeEventConfig,
+    BronzeLayerConfig,
+    LandingLayerConfig,
+    PipelineConfig,
+    PipelineTransformationType,
+    TransformationStep,
+)
+
+SECTION_MAP = {
+    PipelineTransformationType.IMPUTE_NULL: "Missing Value Analysis",
+    PipelineTransformationType.DROP_COLUMN: "Missing Value Analysis",
+    PipelineTransformationType.CAP_OUTLIER: "Global Outlier Detection",
+    PipelineTransformationType.WINSORIZE: "Global Outlier Detection",
+    PipelineTransformationType.SEGMENT_AWARE_CAP: "Segment-Aware Outlier Analysis",
+    PipelineTransformationType.LOG_TRANSFORM: "Feature Distributions",
+    PipelineTransformationType.SQRT_TRANSFORM: "Feature Distributions",
+    PipelineTransformationType.YEO_JOHNSON: "Feature Distributions",
+    PipelineTransformationType.CAP_THEN_LOG: "Feature Distributions",
+    PipelineTransformationType.ZERO_INFLATION_HANDLING: "Feature Distributions",
+    PipelineTransformationType.ENCODE: "Categorical Feature Analysis",
+    PipelineTransformationType.SCALE: "Feature-Target Correlations",
+    PipelineTransformationType.FEATURE_SELECT: "Feature Selection Recommendations",
+    PipelineTransformationType.DERIVED_COLUMN: "Feature Engineering Recommendations",
+    PipelineTransformationType.TYPE_CAST: "Data Consistency Checks",
+}
+
+ANCHOR_MAP = {
+    PipelineTransformationType.IMPUTE_NULL: "3.5-Missing-Value-Analysis",
+    PipelineTransformationType.DROP_COLUMN: "3.5-Missing-Value-Analysis",
+    PipelineTransformationType.CAP_OUTLIER: "3.8-Global-Outlier-Detection",
+    PipelineTransformationType.WINSORIZE: "3.8-Global-Outlier-Detection",
+    PipelineTransformationType.SEGMENT_AWARE_CAP: "3.7-Segment-Aware-Outlier-Analysis",
+    PipelineTransformationType.LOG_TRANSFORM: "4.4-Feature-Distributions-by-Retention-Status",
+    PipelineTransformationType.SQRT_TRANSFORM: "4.4-Feature-Distributions-by-Retention-Status",
+    PipelineTransformationType.YEO_JOHNSON: "4.4-Feature-Distributions-by-Retention-Status",
+    PipelineTransformationType.CAP_THEN_LOG: "4.4-Feature-Distributions-by-Retention-Status",
+    PipelineTransformationType.ZERO_INFLATION_HANDLING: "4.4-Feature-Distributions-by-Retention-Status",
+    PipelineTransformationType.ENCODE: "4.6-Categorical-Feature-Analysis",
+    PipelineTransformationType.SCALE: "4.5-Feature-Target-Correlations",
+    PipelineTransformationType.FEATURE_SELECT: "4.9.1-Feature-Selection-Recommendations",
+    PipelineTransformationType.DERIVED_COLUMN: "4.9.4-Feature-Engineering-Recommendations",
+    PipelineTransformationType.TYPE_CAST: "3.11-Data-Consistency-Checks",
+}
+
+DEFAULT_NOTEBOOK_MAP = {
+    PipelineTransformationType.IMPUTE_NULL: "03_quality_assessment",
+    PipelineTransformationType.DROP_COLUMN: "03_quality_assessment",
+    PipelineTransformationType.CAP_OUTLIER: "03_quality_assessment",
+    PipelineTransformationType.WINSORIZE: "03_quality_assessment",
+    PipelineTransformationType.SEGMENT_AWARE_CAP: "03_quality_assessment",
+    PipelineTransformationType.TYPE_CAST: "03_quality_assessment",
+    PipelineTransformationType.LOG_TRANSFORM: "04_relationship_analysis",
+    PipelineTransformationType.SQRT_TRANSFORM: "04_relationship_analysis",
+    PipelineTransformationType.YEO_JOHNSON: "04_relationship_analysis",
+    PipelineTransformationType.CAP_THEN_LOG: "04_relationship_analysis",
+    PipelineTransformationType.ZERO_INFLATION_HANDLING: "04_relationship_analysis",
+    PipelineTransformationType.ENCODE: "04_relationship_analysis",
+    PipelineTransformationType.SCALE: "04_relationship_analysis",
+    PipelineTransformationType.FEATURE_SELECT: "04_relationship_analysis",
+    PipelineTransformationType.DERIVED_COLUMN: "04_relationship_analysis",
+}
+
+
+_docs_base: str = "docs"
+
+
+def _notebook_title(notebook: str) -> str:
+    name = notebook.split("_", 1)[1] if "_" in notebook else notebook
+    return name.replace("_", " ").title()
+
+
+def provenance_docstring(step: TransformationStep) -> str:
+    notebook = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
+    if not notebook:
+        return ""
+    title = _notebook_title(notebook)
+    anchor = ANCHOR_MAP.get(step.type)
+    section = SECTION_MAP.get(step.type)
+    base = _docs_base
+    if anchor:
+        return f"{title} {section}\n    {base}/{notebook}.html#{anchor}"
+    return f"{title}\n    {base}/{notebook}.html"
+
+
+def provenance_docstring_block(steps) -> str:
+    seen = set()
+    entries = []
+    for step in steps:
+        key = provenance_key(step)
+        if not key or key in seen:
+            continue
+        seen.add(key)
+        entry = provenance_docstring(step)
+        if entry:
+            entries.append(entry)
+    if not entries:
+        return ""
+    body = "\n    ".join(entries)
+    return f'    """\n    {body}\n    """'
+
+
+def provenance_key(step: TransformationStep) -> str:
+    notebook = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
+    section = SECTION_MAP.get(step.type, "")
+    return f"{notebook}:{section}" if notebook else ""
+
+
+class StepGrouper:
+
+    _TYPE_TO_FUNC = {
+        PipelineTransformationType.DROP_COLUMN: "drop_unusable_columns",
+        PipelineTransformationType.IMPUTE_NULL: "impute_remaining_nulls",
+        PipelineTransformationType.CAP_OUTLIER: "cap_outliers",
+        PipelineTransformationType.TYPE_CAST: "apply_type_casts",
+        PipelineTransformationType.WINSORIZE: "winsorize_outliers",
+        PipelineTransformationType.SEGMENT_AWARE_CAP: "cap_segment_aware_outliers",
+        PipelineTransformationType.LOG_TRANSFORM: "apply_log_transforms",
+        PipelineTransformationType.SQRT_TRANSFORM: "apply_sqrt_transforms",
+        PipelineTransformationType.ZERO_INFLATION_HANDLING: "handle_zero_inflation",
+        PipelineTransformationType.CAP_THEN_LOG: "apply_cap_then_log_transforms",
+        PipelineTransformationType.YEO_JOHNSON: "apply_power_transforms",
+        PipelineTransformationType.FEATURE_SELECT: "apply_feature_selection",
+    }
+
+    _DERIVED_ACTION_TO_FUNC = {
+        "ratio": "create_ratio_features",
+        "interaction": "create_interaction_features",
+        "composite": "create_composite_features",
+    }
+
+    @classmethod
+    def group(cls, steps: List[TransformationStep]) -> List[Tuple[str, List[TransformationStep]]]:
+        if not steps:
+            return []
+        groups: OrderedDict[str, List[TransformationStep]] = OrderedDict()
+        for step in steps:
+            groups.setdefault(cls._func_name(step), []).append(step)
+        return list(groups.items())
+
+    @classmethod
+    def _func_name(cls, step: TransformationStep) -> str:
+        if step.type == PipelineTransformationType.DERIVED_COLUMN:
+            action = step.parameters.get("action", "ratio")
+            return cls._DERIVED_ACTION_TO_FUNC.get(action, f"create_{action}_features")
+        return cls._TYPE_TO_FUNC.get(step.type, f"apply_{step.type.value}")
+
+
+group_steps = StepGrouper.group
+
+
+class InlineLoader(BaseLoader):
+    def __init__(self, templates: dict):
+        self._templates = templates
+
+    def get_source(self, environment, template):
+        if template in self._templates:
+            return self._templates[template], template, lambda: True
+        raise Exception(f"Template {template} not found")
+
+
+TEMPLATES = {
+    "config.py.j2": """import os
+from pathlib import Path
+
+PIPELINE_NAME = "{{ config.name }}"
+TARGET_COLUMN = "{{ config.target_column }}"
+OUTPUT_DIR = Path("{{ config.output_dir }}")
+
+# Iteration tracking
+ITERATION_ID = {{ '"%s"' % config.iteration_id if config.iteration_id else 'None' }}
+PARENT_ITERATION_ID = {{ '"%s"' % config.parent_iteration_id if config.parent_iteration_id else 'None' }}
+
+# Recommendations hash for experiment tracking
+RECOMMENDATIONS_HASH = {{ '"%s"' % config.recommendations_hash if config.recommendations_hash else 'None' }}
+
+
+def _find_project_root():
+    path = Path(__file__).parent
+    for _ in range(10):
+        if (path / "pyproject.toml").exists() or (path / ".git").exists():
+            return path
+        path = path.parent
+    return Path(__file__).parent
+
+
+PROJECT_ROOT = _find_project_root()
+
+# Experiments directory - all artifacts (data, mlruns, feast) go here
+# Override with CR_EXPERIMENTS_DIR environment variable for Databricks/custom locations
+_default_experiments = {{ '"%s"' % config.experiments_dir if config.experiments_dir else '"experiments"' }}
+EXPERIMENTS_DIR = Path(os.environ.get("CR_EXPERIMENTS_DIR", str(PROJECT_ROOT / _default_experiments)))
+
+# Documentation base URL for provenance links in generated code
+# Local: file:// URI to HTML docs (from export_tutorial_html.py)
+# Databricks: set to workspace notebook path for exploration report
+DOCS_BASE_URL = os.environ.get("CR_DOCS_BASE_URL", str(EXPERIMENTS_DIR / "docs"))
+
+# Production output directory - all pipeline writes go here
+# Override with CR_PRODUCTION_DIR environment variable
+_default_production = {{ '"%s"' % config.production_dir if config.production_dir else 'str(EXPERIMENTS_DIR)' }}
+PRODUCTION_DIR = Path(os.environ.get("CR_PRODUCTION_DIR", _default_production))
+
+# MLflow tracking - using SQLite backend (recommended over deprecated file-based backend)
+MLFLOW_TRACKING_URI = os.environ.get("MLFLOW_TRACKING_URI", f"sqlite:///{EXPERIMENTS_DIR / 'mlruns.db'}")
+MLFLOW_ARTIFACT_ROOT = str(EXPERIMENTS_DIR / "mlruns" / "artifacts")
+
+# Feast feature store configuration - stored in experiments directory
+FEAST_REPO_PATH = str(PRODUCTION_DIR / "feature_repo")
+FEAST_FEATURE_VIEW = "{{ config.feast.feature_view_name if config.feast else config.name + '_features' }}"
+FEAST_ENTITY_NAME = "{{ config.feast.entity_name if config.feast else 'customer' }}"
+FEAST_ENTITY_KEY = "{{ config.feast.entity_key if config.feast else config.sources[0].entity_key }}"
+FEAST_TIMESTAMP_COL = "{{ config.feast.timestamp_column if config.feast else 'event_timestamp' }}"
+FEAST_TTL_DAYS = {{ config.feast.ttl_days if config.feast else 365 }}
+
+# Source paths - findings directory is a subfolder of experiments
+FINDINGS_DIR = EXPERIMENTS_DIR / "findings"
+
+SOURCES = {
+{% for source in config.sources %}
+    "{{ source.name }}": {
+        "path": str(FINDINGS_DIR / "{{ source.path }}"),
+        "format": "{{ source.format }}",
+        "entity_key": "{{ source.entity_key }}",
+{% if source.time_column %}
+        "time_column": "{{ source.time_column }}",
+{% endif %}
+        "is_event_level": {{ source.is_event_level }},
+    },
+{% endfor %}
+}
+
+
+def get_bronze_path(source_name: str) -> Path:
+    return PRODUCTION_DIR / "data" / "bronze" / f"{source_name}.parquet"
+
+
+def get_silver_path() -> Path:
+    return PRODUCTION_DIR / "data" / "silver" / "merged.parquet"
+
+
+def get_gold_path() -> Path:
+    return PRODUCTION_DIR / "data" / "gold" / "features.parquet"
+
+
+def get_feast_data_path() -> Path:
+    return Path(FEAST_REPO_PATH) / "data" / f"{FEAST_FEATURE_VIEW}.parquet"
+
+
+# Fit mode configuration for training vs scoring separation
+FIT_MODE = {{ 'True' if config.fit_mode else 'False' }}
+ARTIFACTS_PATH = {{ '"%s"' % config.artifacts_path if config.artifacts_path else 'str(PRODUCTION_DIR / "artifacts" / (RECOMMENDATIONS_HASH or "default"))' }}
+
+RAW_SOURCES = {
+{% for name, landing in config.landing.items() %}
+    "{{ name }}": {
+        "path": "{{ landing.raw_source_path }}",
+        "format": "{{ landing.raw_source_format }}",
+        "entity_key": "{{ landing.entity_column }}",
+        "time_column": "{{ landing.time_column }}",
+    },
+{% endfor %}
+}
+
+EXCLUDED_SOURCES = [
+{% for source in config.sources %}
+{% if source.excluded %}
+    "{{ source.name }}",
+{% endif %}
+{% endfor %}
+]
+
+EXPLORATION_ARTIFACTS = {
+    "bronze": {name: str(EXPERIMENTS_DIR / "data" / "bronze" / f"{name}.parquet") for name in SOURCES},
+    "silver": str(EXPERIMENTS_DIR / "data" / "silver" / "merged.parquet"),
+    "gold": str(EXPERIMENTS_DIR / "data" / "gold" / "features.parquet"),
+    "scoring": str(EXPERIMENTS_DIR / "data" / "scoring" / "predictions.parquet"),
+}
+""",
+    "bronze.py.j2": """import pandas as pd
+import numpy as np
+from pathlib import Path
+{% set ops, fitted = collect_imports(config.transformations, False) %}
+{% if ops %}
+from customer_retention.transforms import {{ ops | sort | join(', ') }}
+{% endif %}
+from config import SOURCES, get_bronze_path{{ ', RAW_SOURCES' if config.lifecycle else '' }}
+
+SOURCE_NAME = "{{ source }}"
+
+
+def load_{{ source }}():
+    source_config = SOURCES[SOURCE_NAME]
+    path = Path(source_config["path"])
+    if path.is_dir() and (path / "_delta_log").is_dir():
+        from customer_retention.integrations.adapters.factory import get_delta
+        return get_delta(force_local=True).read(str(path))
+    if not path.exists():
+        raise FileNotFoundError(f"Source file not found: {path}")
+    if source_config["format"] == "csv":
+        return pd.read_csv(path)
+    return pd.read_parquet(path)
+
+
+{% set groups = group_steps(config.transformations) %}
+
+def apply_transformations(df: pd.DataFrame) -> pd.DataFrame:
+{%- if groups %}
+{%- for func_name, steps in groups %}
+    df = {{ func_name }}(df)
+{%- endfor %}
+{%- endif %}
+    return df
+
+{% for func_name, steps in groups %}
+
+def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
+{%- set _prov = provenance_docstring_block(steps) %}
+{%- if _prov %}
+{{ _prov }}
+{%- endif %}
+{%- for t in steps %}
+    # {{ t.rationale }}
+    # {{ action_description(t) }}
+    df = {{ render_step_call(t) }}
+{%- endfor %}
+    return df
+{% endfor %}
+
+{% if config.lifecycle %}
+
+# --- Lifecycle enrichment (computed on cleaned data) ---
+
+ENTITY_COLUMN = "{{ config.entity_column or config.source.entity_key }}"
+TIME_COLUMN = "{{ config.time_column or config.source.time_column }}"
+
+
+def _load_raw_events():
+    source = RAW_SOURCES[SOURCE_NAME]
+    path = Path(source["path"])
+    if path.is_dir() and (path / "_delta_log").is_dir():
+        from customer_retention.integrations.adapters.factory import get_delta
+        return get_delta(force_local=True).read(str(path))
+    if not path.exists():
+        raise FileNotFoundError(f"Raw source not found: {path}")
+    if source["format"] == "csv":
+        return pd.read_csv(path)
+    return pd.read_parquet(path)
+
+{% if config.lifecycle.include_recency_bucket %}
+
+def add_recency_tenure(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
+    raw_df[TIME_COLUMN] = pd.to_datetime(raw_df[TIME_COLUMN])
+    reference_date = raw_df[TIME_COLUMN].max()
+    entity_stats = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].agg(["min", "max"])
+    entity_stats["days_since_last"] = (reference_date - entity_stats["max"]).dt.days
+    entity_stats["days_since_first"] = (reference_date - entity_stats["min"]).dt.days
+    df = df.merge(entity_stats[["days_since_last", "days_since_first"]], left_on=ENTITY_COLUMN, right_index=True, how="left")
+    return df
+
+
+def add_recency_buckets(df: pd.DataFrame) -> pd.DataFrame:
+    if "days_since_last" in df.columns:
+        df["recency_bucket"] = pd.cut(df["days_since_last"], bins=[0, 7, 30, 90, 180, 365, float("inf")],
+                                       labels=["0-7d", "7-30d", "30-90d", "90-180d", "180-365d", "365d+"])
+    return df
+
+{% endif %}
+{% if config.lifecycle.include_lifecycle_quadrant %}
+
+def add_lifecycle_quadrant(df: pd.DataFrame) -> pd.DataFrame:
+    if "days_since_first" not in df.columns:
+        return df
+    tenure = df["days_since_first"]
+    intensity_col = [c for c in df.columns if c.startswith("event_count_")]
+    if not intensity_col:
+        return df
+    intensity = df[intensity_col[0]]
+    tenure_med = tenure.median()
+    intensity_med = intensity.median()
+    conditions = [
+        (tenure >= tenure_med) & (intensity >= intensity_med),
+        (tenure >= tenure_med) & (intensity < intensity_med),
+        (tenure < tenure_med) & (intensity >= intensity_med),
+        (tenure < tenure_med) & (intensity < intensity_med),
+    ]
+    labels = ["loyal", "at_risk", "new_active", "new_inactive"]
+    df["lifecycle_quadrant"] = np.select(conditions, labels, default="unknown")
+    return df
+
+{% endif %}
+{% if config.lifecycle.include_cyclical_features %}
+
+def add_cyclical_features(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
+    raw_df[TIME_COLUMN] = pd.to_datetime(raw_df[TIME_COLUMN])
+    mean_dow = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].apply(lambda x: x.dt.dayofweek.mean())
+    df = df.merge(mean_dow.rename("mean_dow"), left_on=ENTITY_COLUMN, right_index=True, how="left")
+    df["dow_sin"] = np.sin(2 * np.pi * df["mean_dow"] / 7)
+    df["dow_cos"] = np.cos(2 * np.pi * df["mean_dow"] / 7)
+    df = df.drop(columns=["mean_dow"], errors="ignore")
+    return df
+
+{% endif %}
+{% if config.lifecycle.momentum_pairs %}
+
+def add_momentum_ratios(df: pd.DataFrame) -> pd.DataFrame:
+{% for pair in config.lifecycle.momentum_pairs %}
+    short_col = "event_count_{{ pair.short_window }}"
+    long_col = "event_count_{{ pair.long_window }}"
+    if short_col in df.columns and long_col in df.columns:
+        df["momentum_{{ pair.short_window }}_{{ pair.long_window }}"] = df[short_col] / df[long_col].replace(0, float("nan"))
+{% endfor %}
+    return df
+
+{% endif %}
+
+def enrich_lifecycle(df: pd.DataFrame) -> pd.DataFrame:
+    raw_df = _load_raw_events()
+{% if config.raw_time_column %}
+    raw_df = raw_df.rename(columns={"{{ config.raw_time_column }}": TIME_COLUMN})
+{% endif %}
+{% if config.lifecycle.include_recency_bucket %}
+    df = add_recency_tenure(df, raw_df)
+    df = add_recency_buckets(df)
+{% endif %}
+{% if config.lifecycle.include_lifecycle_quadrant %}
+    df = add_lifecycle_quadrant(df)
+{% endif %}
+{% if config.lifecycle.include_cyclical_features %}
+    df = add_cyclical_features(df, raw_df)
+{% endif %}
+{% if config.lifecycle.momentum_pairs %}
+    df = add_momentum_ratios(df)
+{% endif %}
+    return df
+{% endif %}
+
+
+def run_bronze_{{ source }}():
+    df = load_{{ source }}()
+    df = apply_transformations(df)
+{% if config.lifecycle %}
+    df = enrich_lifecycle(df)
+{% endif %}
+    output_path = get_bronze_path(SOURCE_NAME)
+    output_dir = output_path.parent
+    output_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        from customer_retention.integrations.adapters.factory import get_delta
+        storage = get_delta(force_local=True)
+        storage.write(df, str(output_dir / SOURCE_NAME))
+    except ImportError:
+        df.to_parquet(output_path, index=False)
+    return df
+
+
+if __name__ == "__main__":
+    run_bronze_{{ source }}()
+""",
+    "silver.py.j2": '''import pandas as pd
+{% set ops, fitted = collect_imports(config.silver.derived_columns, False) %}
+{% if ops %}
+from customer_retention.transforms import {{ ops | sort | join(', ') }}
+{% endif %}
+from config import SOURCES, get_bronze_path, get_silver_path, TARGET_COLUMN
+
+
+def _load_artifact(path):
+    from pathlib import Path as _P
+    p = _P(path)
+    if p.parent.is_dir() and (p.parent / p.stem / "_delta_log").is_dir():
+        from customer_retention.integrations.adapters.factory import get_delta
+        return get_delta(force_local=True).read(str(p.parent / p.stem))
+    return pd.read_parquet(path)
+
+
+def load_bronze_outputs() -> dict:
+    return {name: _load_artifact(get_bronze_path(name))
+            for name in SOURCES.keys() if not SOURCES[name].get("excluded")}
+
+
+def merge_sources(bronze_outputs: dict) -> pd.DataFrame:
+    base_source = "{{ config.sources[0].name }}"
+    merged = bronze_outputs[base_source]
+{% for join in config.silver.joins %}
+    merged = merged.merge(
+        bronze_outputs["{{ join.right_source }}"],
+        left_on="{{ join.left_key }}",
+        right_on="{{ join.right_key }}",
+        how="{{ join.how }}"
+    )
+{% endfor %}
+    return merged
+
+
+def create_holdout_mask(df: pd.DataFrame, holdout_fraction: float = 0.1, random_state: int = 42) -> pd.DataFrame:
+    """Create holdout set by masking target for a fraction of records.
+
+    IMPORTANT: This must happen in the silver layer (BEFORE gold layer feature computation)
+    to prevent temporal leakage. If holdout is created after features are computed,
+    the features may contain information derived from the target values that will be masked.
+
+    Args:
+        df: DataFrame with TARGET_COLUMN
+        holdout_fraction: Fraction of records to use for holdout (default 10%)
+        random_state: Random seed for reproducibility
+
+    Returns:
+        DataFrame with holdout mask applied (original values stored in original_{TARGET_COLUMN})
+    """
+    ORIGINAL_COLUMN = f"original_{TARGET_COLUMN}"
+
+    # Skip if holdout already exists
+    if ORIGINAL_COLUMN in df.columns:
+        print(f"  Holdout already exists ({ORIGINAL_COLUMN}), skipping creation")
+        return df
+
+    if TARGET_COLUMN not in df.columns:
+        print(f"  Warning: TARGET_COLUMN \\'{TARGET_COLUMN}\\' not found, skipping holdout creation")
+        return df
+
+    print(f"Creating holdout set ({holdout_fraction:.0%} of data)...")
+    df = df.copy()
+
+    n_holdout = int(len(df) * holdout_fraction)
+    holdout_idx = df.sample(n=n_holdout, random_state=random_state).index
+
+    # Store original values for holdout records only
+    df[ORIGINAL_COLUMN] = pd.NA
+    df.loc[holdout_idx, ORIGINAL_COLUMN] = df.loc[holdout_idx, TARGET_COLUMN]
+
+    # Mask target values for holdout records
+    df.loc[holdout_idx, TARGET_COLUMN] = pd.NA
+
+    print(f"  Holdout records: {n_holdout:,} ({holdout_fraction:.0%})")
+    print(f"  Training records: {len(df) - n_holdout:,} ({1-holdout_fraction:.0%})")
+
+    return df
+
+
+{% set derived_groups = group_steps(config.silver.derived_columns) %}
+
+def create_derived_columns(df: pd.DataFrame) -> pd.DataFrame:
+{%- if derived_groups %}
+{%- for func_name, steps in derived_groups %}
+    df = {{ func_name }}(df)
+{%- endfor %}
+{%- endif %}
+    return df
+
+{% for func_name, steps in derived_groups %}
+
+def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
+{%- set _prov = provenance_docstring_block(steps) %}
+{%- if _prov %}
+{{ _prov }}
+{%- endif %}
+{%- for dc in steps %}
+    # {{ dc.rationale }}
+    # {{ action_description(dc) }}
+    df = {{ render_step_call(dc) }}
+{%- endfor %}
+    return df
+{% endfor %}
+
+
+def run_silver_merge(create_holdout: bool = True, holdout_fraction: float = 0.1):
+    bronze_outputs = load_bronze_outputs()
+    silver = merge_sources(bronze_outputs)
+    silver = create_derived_columns(silver)
+
+    if create_holdout:
+        silver = create_holdout_mask(silver, holdout_fraction=holdout_fraction)
+
+    output_path = get_silver_path()
+    output_dir = output_path.parent
+    output_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        from customer_retention.integrations.adapters.factory import get_delta
+        storage = get_delta(force_local=True)
+        storage.write(silver, str(output_dir / "silver"))
+    except ImportError:
+        silver.to_parquet(output_path, index=False)
+    return silver
+
+
+if __name__ == "__main__":
+    run_silver_merge()
+''',
+    "gold.py.j2": '''import pandas as pd
+import warnings
+from datetime import datetime
+from pathlib import Path
+{% set all_gold_steps = config.gold.transformations + config.gold.encodings + config.gold.scalings %}
+{% set ops, fitted = collect_imports(all_gold_steps, True) %}
+{% set fs_ops = ['apply_feature_select'] if config.gold.feature_selections else [] %}
+from customer_retention.transforms import ArtifactStore{{ (', ' + (ops | sort | join(', '))) if ops }}{{ (', ' + (fs_ops | join(', '))) if fs_ops and 'apply_feature_select' not in ops }}
+{% if fitted %}
+from customer_retention.transforms.fitted import {{ fitted | sort | join(', ') }}
+{% endif %}
+from config import (get_silver_path, get_gold_path, get_feast_data_path,
+                    TARGET_COLUMN, RECOMMENDATIONS_HASH, FEAST_REPO_PATH,
+                    FEAST_FEATURE_VIEW, FEAST_ENTITY_KEY, FEAST_TIMESTAMP_COL, EXPERIMENTS_DIR,
+                    ARTIFACTS_PATH, FIT_MODE)
+
+{% if config.fit_mode %}
+_store = ArtifactStore(Path(ARTIFACTS_PATH))
+{% else %}
+_store = ArtifactStore.from_manifest(Path(ARTIFACTS_PATH) / "manifest.yaml")
+{% endif %}
+
+from customer_retention.generators.pipeline_generator.models import (
+    PipelineTransformationType,
+    TransformationStep,
+)
+
+ENCODINGS = [
+{% for enc in config.gold.encodings %}
+    TransformationStep(type=PipelineTransformationType.ENCODE, column="{{ enc.column }}", parameters={{ enc.parameters }}, rationale="{{ enc.rationale }}"),
+{% endfor %}
+]
+
+SCALINGS = [
+{% for scale in config.gold.scalings %}
+    TransformationStep(type=PipelineTransformationType.SCALE, column="{{ scale.column }}", parameters={{ scale.parameters }}, rationale="{{ scale.rationale }}"),
+{% endfor %}
+]
+
+
+def load_silver() -> pd.DataFrame:
+    path = get_silver_path()
+    parent = path.parent
+    delta_path = parent / "silver"
+    if delta_path.is_dir() and (delta_path / "_delta_log").is_dir():
+        from customer_retention.integrations.adapters.factory import get_delta
+        return get_delta(force_local=True).read(str(delta_path))
+    return pd.read_parquet(path)
+
+
+def load_gold() -> pd.DataFrame:
+    path = get_gold_path()
+    delta_path = path.parent / "gold"
+    if delta_path.is_dir() and (delta_path / "_delta_log").is_dir():
+        from customer_retention.integrations.adapters.factory import get_delta
+        return get_delta(force_local=True).read(str(delta_path))
+    return pd.read_parquet(path)
+
+
+{% set transform_groups = group_steps(config.gold.transformations) %}
+
+def apply_gold_transformations(df: pd.DataFrame) -> pd.DataFrame:
+{%- if transform_groups %}
+{%- for func_name, steps in transform_groups %}
+    df = {{ func_name }}(df)
+{%- endfor %}
+{%- endif %}
+    return df
+
+{% for func_name, steps in transform_groups %}
+
+def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
+{%- set _prov = provenance_docstring_block(steps) %}
+{%- if _prov %}
+{{ _prov }}
+{%- endif %}
+{%- for t in steps %}
+    # {{ t.rationale }}
+    # {{ action_description(t) }}
+    df = {{ render_step_call(t, config.fit_mode) }}
+{%- endfor %}
+    return df
+{% endfor %}
+
+
+def apply_encodings(df: pd.DataFrame) -> pd.DataFrame:
+{%- set _prov = provenance_docstring_block(config.gold.encodings) %}
+{%- if _prov %}
+{{ _prov }}
+{%- endif %}
+{%- if config.gold.encodings %}
+{%- for enc in config.gold.encodings %}
+    # {{ enc.rationale }}
+    # {{ action_description(enc) }}
+    df = {{ render_step_call(enc, config.fit_mode) }}
+{%- endfor %}
+{%- endif %}
+    return df
+
+
+def apply_scaling(df: pd.DataFrame) -> pd.DataFrame:
+{%- set _prov = provenance_docstring_block(config.gold.scalings) %}
+{%- if _prov %}
+{{ _prov }}
+{%- endif %}
+{%- if config.gold.scalings %}
+{%- for scale in config.gold.scalings %}
+    # {{ scale.rationale }}
+    # {{ action_description(scale) }}
+    df = {{ render_step_call(scale, config.fit_mode) }}
+{%- endfor %}
+{%- endif %}
+    return df
+
+
+def apply_feature_selection(df: pd.DataFrame) -> pd.DataFrame:
+{% if config.gold.feature_selections %}
+{% for fs in config.gold.feature_selections %}
+    # Feature selection
+    # drop {{ fs }} (feature selection)
+    df = apply_feature_select(df, '{{ fs }}')
+{% endfor %}
+{% endif %}
+    return df
+
+
+def get_feature_version_tag() -> str:
+    if RECOMMENDATIONS_HASH:
+        return f"v1.0.0_{RECOMMENDATIONS_HASH}"
+    return "v1.0.0"
+
+
+def add_feast_timestamp(df: pd.DataFrame, reference_date=None) -> pd.DataFrame:
+    if FEAST_TIMESTAMP_COL not in df.columns:
+        if "aggregation_reference_date" in df.attrs:
+            timestamp = df.attrs["aggregation_reference_date"]
+            print(f"  Using aggregation reference_date for Feast timestamp: {timestamp}")
+        elif reference_date is not None:
+            timestamp = reference_date
+            print(f"  Using provided reference_date for Feast timestamp: {timestamp}")
+        else:
+            timestamp = datetime.now()
+            warnings.warn(
+                f"No reference_date available for Feast timestamp. Using datetime.now() ({timestamp}). "
+                "This may cause temporal leakage - features should use actual aggregation dates. "
+                "Set aggregation_reference_date in DataFrame.attrs during aggregation.",
+                UserWarning
+            )
+        df[FEAST_TIMESTAMP_COL] = timestamp
+    return df
+
+
+def materialize_to_feast(df: pd.DataFrame) -> None:
+    feast_path = get_feast_data_path()
+    feast_path.parent.mkdir(parents=True, exist_ok=True)
+    df_feast = df.copy()
+    df_feast = add_feast_timestamp(df_feast)
+    original_cols = [c for c in df_feast.columns if c.startswith("original_")]
+    if original_cols:
+        print(f"  Excluding holdout columns from Feast: {original_cols}")
+        df_feast = df_feast.drop(columns=original_cols, errors="ignore")
+    try:
+        from customer_retention.integrations.adapters.factory import get_delta
+        storage = get_delta(force_local=True)
+        storage.write(df_feast, str(feast_path.parent / feast_path.stem))
+    except ImportError:
+        df_feast.to_parquet(feast_path, index=False)
+    print(f"Features materialized to Feast: {feast_path}")
+    print(f"  Entity key: {FEAST_ENTITY_KEY}")
+    print(f"  Feature view: {FEAST_FEATURE_VIEW}")
+    print(f"  Rows: {len(df_feast):,}")
+
+
+def run_gold_features():
+    silver = load_silver()
+    gold = apply_gold_transformations(silver)
+    gold = apply_encodings(gold)
+    gold = apply_scaling(gold)
+    gold = apply_feature_selection(gold)
+{% if config.fit_mode %}
+    _store.save_manifest()
+    print(f"Fit artifacts saved to: {ARTIFACTS_PATH}")
+{% endif %}
+    output_path = get_gold_path()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    gold.attrs["recommendations_hash"] = RECOMMENDATIONS_HASH
+    gold.attrs["feature_version"] = get_feature_version_tag()
+    try:
+        from customer_retention.integrations.adapters.factory import get_delta
+        storage = get_delta(force_local=True)
+        storage.write(gold, str(output_path.parent / "gold"))
+    except ImportError:
+        gold.to_parquet(output_path, index=False)
+    print(f"Gold features saved with version: {get_feature_version_tag()}")
+    materialize_to_feast(gold)
+    return gold
+
+
+if __name__ == "__main__":
+    run_gold_features()
+''',
+    "training.py.j2": '''import pandas as pd
+import mlflow
+import mlflow.sklearn
+import mlflow.xgboost
+import xgboost as xgb
+from pathlib import Path
+from feast import FeatureStore
+from sklearn.model_selection import train_test_split, cross_val_score
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.preprocessing import LabelEncoder
+from sklearn.metrics import (roc_auc_score, average_precision_score, f1_score,
+                             precision_score, recall_score, accuracy_score)
+from config import (TARGET_COLUMN, PIPELINE_NAME, RECOMMENDATIONS_HASH, MLFLOW_TRACKING_URI, MLFLOW_ARTIFACT_ROOT,
+                    FEAST_REPO_PATH, FEAST_FEATURE_VIEW, FEAST_ENTITY_KEY, FEAST_TIMESTAMP_COL,
+                    get_feast_data_path)
+
+# Set tracking URI immediately to prevent default mlruns directory creation
+mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)
+
+
+def _load_feast_data():
+    feast_path = get_feast_data_path()
+    delta_path = feast_path.parent / feast_path.stem
+    if delta_path.is_dir() and (delta_path / "_delta_log").is_dir():
+        from customer_retention.integrations.adapters.factory import get_delta
+        return get_delta(force_local=True).read(str(delta_path))
+    return pd.read_parquet(feast_path)
+
+
+def get_training_data_from_feast() -> pd.DataFrame:
+    """Retrieve training data from Feast for training/serving consistency.
+
+    Uses get_historical_features for point-in-time correct feature retrieval.
+    This ensures training uses the exact same feature retrieval path as inference.
+    """
+    feast_path = Path(FEAST_REPO_PATH)
+
+    # Check if Feast repo is initialized
+    if not (feast_path / "feature_store.yaml").exists():
+        print("Feast repo not initialized, falling back to data file")
+        return _load_feast_data()
+
+    try:
+        store = FeatureStore(repo_path=str(feast_path))
+
+        # Read the materialized features to get entity keys and timestamps
+        features_df = _load_feast_data()
+
+        # Create entity dataframe for historical feature retrieval
+        entity_df = features_df[[FEAST_ENTITY_KEY, FEAST_TIMESTAMP_COL]].copy()
+
+        # Get all feature names (excluding entity key, timestamp, target, and holdout ground truth)
+        exclude_cols = {FEAST_ENTITY_KEY, FEAST_TIMESTAMP_COL, TARGET_COLUMN}
+        feature_cols = [c for c in features_df.columns
+                        if c not in exclude_cols and not c.startswith("original_")]
+
+        # Build feature references
+        feature_refs = [f"{FEAST_FEATURE_VIEW}:{col}" for col in feature_cols]
+
+        print(f"Retrieving {len(feature_refs)} features from Feast...")
+        print(f"  Feature view: {FEAST_FEATURE_VIEW}")
+        print(f"  Entity key: {FEAST_ENTITY_KEY}")
+
+        # Get historical features with point-in-time correctness
+        training_df = store.get_historical_features(
+            entity_df=entity_df,
+            features=feature_refs
+        ).to_df()
+
+        # Add target column back
+        training_df = training_df.merge(
+            features_df[[FEAST_ENTITY_KEY, TARGET_COLUMN]],
+            on=FEAST_ENTITY_KEY,
+            how="left"
+        )
+
+        print(f"  Retrieved {len(training_df):,} rows, {len(training_df.columns)} columns")
+        return training_df
+
+    except Exception as e:
+        print(f"Feast retrieval failed ({e}), falling back to data file")
+        return _load_feast_data()
+
+
+def prepare_features(df: pd.DataFrame) -> pd.DataFrame:
+    """Prepare features for model training.
+
+    Explicitly excludes original_* columns which contain holdout ground truth.
+    These columns are reserved for scoring validation and must never be used in training.
+    """
+    df = df.copy()
+
+    # Drop Feast metadata columns
+    drop_cols = [FEAST_ENTITY_KEY, FEAST_TIMESTAMP_COL]
+    df = df.drop(columns=[c for c in drop_cols if c in df.columns], errors="ignore")
+
+    # Exclude original_* columns (holdout ground truth - prevents data leakage)
+    original_cols = [c for c in df.columns if c.startswith("original_")]
+    df = df.drop(columns=original_cols, errors="ignore")
+
+    # Encode categorical columns
+    for col in df.select_dtypes(include=["object", "category"]).columns:
+        df[col] = LabelEncoder().fit_transform(df[col].astype(str))
+
+    return df.select_dtypes(include=["int64", "float64", "int32", "float32"]).fillna(0)
+
+
+def compute_metrics(y_true, y_proba, y_pred) -> dict:
+    return {
+        "roc_auc": roc_auc_score(y_true, y_proba),
+        "pr_auc": average_precision_score(y_true, y_proba),
+        "f1": f1_score(y_true, y_pred),
+        "precision": precision_score(y_true, y_pred),
+        "recall": recall_score(y_true, y_pred),
+        "accuracy": accuracy_score(y_true, y_pred),
+    }
+
+
+def get_feature_importance(model, feature_names) -> pd.DataFrame:
+    if hasattr(model, "feature_importances_"):
+        importance = model.feature_importances_
+    elif hasattr(model, "coef_"):
+        importance = abs(model.coef_[0])
+    else:
+        return None
+    df = pd.DataFrame({"feature": feature_names, "importance": importance})
+    return df.sort_values("importance", ascending=False).reset_index(drop=True)
+
+
+def log_feature_importance(model, feature_names):
+    fi = get_feature_importance(model, feature_names)
+    if fi is None:
+        return
+    fi.to_csv("feature_importance.csv", index=False)
+    mlflow.log_artifact("feature_importance.csv")
+
+
+def train_xgboost(X_train, y_train, X_test, y_test, feature_names):
+    mlflow.xgboost.autolog(log_datasets=False, log_models=False)
+    dtrain = xgb.DMatrix(X_train, label=y_train, feature_names=feature_names)
+    dtest = xgb.DMatrix(X_test, label=y_test, feature_names=feature_names)
+    params = {"objective": "binary:logistic", "eval_metric": ["auc", "logloss"],
+              "max_depth": 6, "learning_rate": 0.1, "seed": 42}
+    model = xgb.train(params, dtrain, num_boost_round=100,
+                      evals=[(dtrain, "train"), (dtest, "eval")], verbose_eval=False)
+    return model
+
+
+def get_model_name_with_hash(base_name: str) -> str:
+    if RECOMMENDATIONS_HASH:
+        return f"{base_name}_{RECOMMENDATIONS_HASH}"
+    return base_name
+
+
+def run_experiment():
+    mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)
+    experiment = mlflow.get_experiment_by_name(PIPELINE_NAME)
+    if experiment is None:
+        mlflow.create_experiment(PIPELINE_NAME, artifact_location=MLFLOW_ARTIFACT_ROOT)
+    mlflow.set_experiment(PIPELINE_NAME)
+    print(f"MLflow tracking: {MLFLOW_TRACKING_URI}")
+    print(f"Artifacts: {MLFLOW_ARTIFACT_ROOT}")
+
+    # Load training data from Feast (ensures training/serving consistency)
+    print("\\nLoading training data from Feast...")
+    training_data = get_training_data_from_feast()
+
+    y = training_data[TARGET_COLUMN]
+    X = prepare_features(training_data.drop(columns=[TARGET_COLUMN]))
+    feature_names = list(X.columns)
+    train_mask = y.notna()
+    X, y = X.loc[train_mask], y.loc[train_mask]
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
+
+    sklearn_models = {
+        "logistic_regression": LogisticRegression(max_iter=5000, random_state=42),
+        "random_forest": RandomForestClassifier(n_estimators=100, random_state=42),
+    }
+
+    run_name = get_model_name_with_hash("pipeline_run")
+    with mlflow.start_run(run_name=run_name):
+        mlflow.log_params({"train_samples": len(X_train), "test_samples": len(X_test), "n_features": X.shape[1]})
+        mlflow.set_tag("feature_source", "feast")
+        mlflow.set_tag("feast_feature_view", FEAST_FEATURE_VIEW)
+        if RECOMMENDATIONS_HASH:
+            mlflow.set_tag("recommendations_hash", RECOMMENDATIONS_HASH)
+        best_model, best_auc = None, 0
+
+        for name, model in sklearn_models.items():
+            with mlflow.start_run(run_name=name, nested=True):
+                if RECOMMENDATIONS_HASH:
+                    mlflow.set_tag("recommendations_hash", RECOMMENDATIONS_HASH)
+                mlflow.set_tag("feature_source", "feast")
+                model.fit(X_train, y_train)
+                y_proba = model.predict_proba(X_test)[:, 1]
+                y_pred = model.predict(X_test)
+                metrics = compute_metrics(y_test, y_proba, y_pred)
+                cv = cross_val_score(model, X_train, y_train, cv=5, scoring="roc_auc")
+                mlflow.log_metrics({**metrics, "cv_mean": cv.mean(), "cv_std": cv.std()})
+                log_feature_importance(model, feature_names)
+                model_artifact_name = get_model_name_with_hash(f"model_{name}")
+                mlflow.sklearn.log_model(model, name=model_artifact_name)
+                print(f"{name}: ROC-AUC={metrics['roc_auc']:.4f}, PR-AUC={metrics['pr_auc']:.4f}, F1={metrics['f1']:.4f}")
+                if metrics["roc_auc"] > best_auc:
+                    best_auc, best_model = metrics["roc_auc"], name
+
+        with mlflow.start_run(run_name="xgboost", nested=True):
+            if RECOMMENDATIONS_HASH:
+                mlflow.set_tag("recommendations_hash", RECOMMENDATIONS_HASH)
+            mlflow.set_tag("feature_source", "feast")
+            xgb_model = train_xgboost(X_train, y_train, X_test, y_test, feature_names)
+            dtest = xgb.DMatrix(X_test, feature_names=feature_names)
+            y_proba = xgb_model.predict(dtest)
+            y_pred = (y_proba > 0.5).astype(int)
+            metrics = compute_metrics(y_test, y_proba, y_pred)
+            mlflow.log_metrics(metrics)
+            xgb_model_name = get_model_name_with_hash("model_xgboost")
+            mlflow.xgboost.log_model(xgb_model, name=xgb_model_name)
+            importance = xgb_model.get_score(importance_type="gain")
+            fi = pd.DataFrame({"feature": importance.keys(), "importance": importance.values()})
+            fi = fi.sort_values("importance", ascending=False).reset_index(drop=True)
+            fi.to_csv("feature_importance.csv", index=False)
+            mlflow.log_artifact("feature_importance.csv")
+            print(f"xgboost: ROC-AUC={metrics['roc_auc']:.4f}, PR-AUC={metrics['pr_auc']:.4f}, F1={metrics['f1']:.4f}")
+            if metrics["roc_auc"] > best_auc:
+                best_auc, best_model = metrics["roc_auc"], "xgboost"
+
+        mlflow.set_tag("best_model", best_model)
+        mlflow.log_metric("best_roc_auc", best_auc)
+        print(f"Best: {best_model} (ROC-AUC={best_auc:.4f})")
+
+
+if __name__ == "__main__":
+    run_experiment()
+''',
+    "runner.py.j2": '''import argparse
+from concurrent.futures import ThreadPoolExecutor
+from config import PIPELINE_NAME, EXPERIMENTS_DIR, PRODUCTION_DIR
+{% for name in config.landing %}
+from landing.landing_{{ name }} import run_landing_{{ name }}
+{% endfor %}
+{% for name, _ in config.bronze.items() %}
+from bronze.bronze_{{ name }} import run_bronze_{{ name }}
+{% endfor %}
+{% for name, _ in config.bronze_event.items() %}
+from bronze.bronze_{{ name }} import run_bronze_{{ name }}
+{% endfor %}
+from silver.silver_merge import run_silver_merge
+from gold.gold_features import run_gold_features
+from training.ml_experiment import run_experiment
+
+
+def setup_experiments_dir():
+    EXPERIMENTS_DIR.mkdir(parents=True, exist_ok=True)
+    (EXPERIMENTS_DIR / "mlruns").mkdir(parents=True, exist_ok=True)
+    PRODUCTION_DIR.mkdir(parents=True, exist_ok=True)
+    (PRODUCTION_DIR / "data" / "bronze").mkdir(parents=True, exist_ok=True)
+    (PRODUCTION_DIR / "data" / "silver").mkdir(parents=True, exist_ok=True)
+    (PRODUCTION_DIR / "data" / "gold").mkdir(parents=True, exist_ok=True)
+
+
+def run_pipeline(validate=False):
+    print(f"Starting pipeline: {PIPELINE_NAME}")
+    setup_experiments_dir()
+{% if config.landing %}
+
+    print("\\n[1/6] Landing (event sources)...")
+{% for name in config.landing %}
+    run_landing_{{ name }}()
+{% endfor %}
+    print("Landing complete")
+    if validate:
+        from validation.validate_pipeline import validate_landing
+        validate_landing()
+{% endif %}
+
+    print("\\n[{{ '2/6' if config.landing else '1/4' }}] Bronze (parallel)...")
+    with ThreadPoolExecutor(max_workers={{ (config.bronze | length) + (config.bronze_event | length) }}) as executor:
+        bronze_futures = [
+{% for name in config.bronze %}
+            executor.submit(run_bronze_{{ name }}),
+{% endfor %}
+{% for name in config.bronze_event %}
+            executor.submit(run_bronze_{{ name }}),
+{% endfor %}
+        ]
+        for f in bronze_futures:
+            f.result()
+    print("Bronze complete")
+    if validate:
+        from validation.validate_pipeline import validate_bronze
+        validate_bronze()
+
+    print("\\n[{{ '3/6' if config.landing else '2/4' }}] Silver...")
+    run_silver_merge()
+    print("Silver complete")
+    if validate:
+        from validation.validate_pipeline import validate_silver
+        validate_silver()
+
+    print("\\n[{{ '4/6' if config.landing else '3/4' }}] Gold...")
+    run_gold_features()
+    print("Gold complete")
+    if validate:
+        from validation.validate_pipeline import validate_gold
+        validate_gold()
+
+    print("\\n[{{ '5/6' if config.landing else '4/4' }}] Training...")
+    run_experiment()
+    print("Training complete")
+    if validate:
+        from validation.validate_pipeline import validate_training
+        validate_training()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--validate", action="store_true")
+    args = parser.parse_args()
+    run_pipeline(validate=args.validate)
+''',
+    "run_all.py.j2": '''"""{{ config.name }} - Pipeline Runner with MLflow UI
+
+All artifacts (data, mlruns, feast) are stored in the experiments directory.
+Override location with CR_EXPERIMENTS_DIR environment variable.
+"""
+import os
+import sys
+import webbrowser
+import subprocess
+import time
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
+
+sys.path.insert(0, str(Path(__file__).parent))
+
+from config import PIPELINE_NAME, SOURCES, MLFLOW_TRACKING_URI, EXPERIMENTS_DIR, PRODUCTION_DIR, FINDINGS_DIR
+{% for name in config.landing %}
+from landing.landing_{{ name }} import run_landing_{{ name }}
+{% endfor %}
+{% for name in config.bronze %}
+from bronze.bronze_{{ name }} import run_bronze_{{ name }}
+{% endfor %}
+{% for name in config.bronze_event %}
+from bronze.bronze_{{ name }} import run_bronze_{{ name }}
+{% endfor %}
+from silver.silver_merge import run_silver_merge
+from gold.gold_features import run_gold_features
+from training.ml_experiment import run_experiment
+
+
+def setup_experiments_dir():
+    EXPERIMENTS_DIR.mkdir(parents=True, exist_ok=True)
+    (EXPERIMENTS_DIR / "mlruns").mkdir(parents=True, exist_ok=True)
+    PRODUCTION_DIR.mkdir(parents=True, exist_ok=True)
+    (PRODUCTION_DIR / "data" / "bronze").mkdir(parents=True, exist_ok=True)
+    (PRODUCTION_DIR / "data" / "silver").mkdir(parents=True, exist_ok=True)
+    (PRODUCTION_DIR / "data" / "gold").mkdir(parents=True, exist_ok=True)
+    print(f"Experiments directory: {EXPERIMENTS_DIR}")
+    print(f"Production directory: {PRODUCTION_DIR}")
+    print(f"MLflow tracking: {MLFLOW_TRACKING_URI}")
+    print(f"Findings directory: {FINDINGS_DIR}")
+
+
+def run_landing():
+{% for name in config.landing %}
+    run_landing_{{ name }}()
+{% endfor %}
+    pass
+
+
+def run_bronze_parallel():
+    bronze_funcs = [
+{% for name in config.bronze %}
+        run_bronze_{{ name }},
+{% endfor %}
+{% for name in config.bronze_event %}
+        run_bronze_{{ name }},
+{% endfor %}
+    ]
+    with ThreadPoolExecutor(max_workers={{ (config.bronze | length) + (config.bronze_event | length) }}) as ex:
+        list(ex.map(lambda f: f(), bronze_funcs))
+
+
+def is_port_in_use(port):
+    import socket
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        return s.connect_ex(('localhost', port)) == 0
+
+
+def start_mlflow_ui():
+    port = 5050
+    if is_port_in_use(port):
+        print(f"\\n⚠ Port {port} is already in use.")
+        print(f"  Either mlflow is already running, or kill the old process:")
+        print(f"  pkill -f 'mlflow ui'")
+        print(f"\\n  Opening browser to existing server...")
+        webbrowser.open(f"http://localhost:{port}")
+        return None
+
+    print(f"\\nStarting MLflow UI (tracking: {MLFLOW_TRACKING_URI})...")
+    process = subprocess.Popen(
+        ["mlflow", "ui", "--backend-store-uri", MLFLOW_TRACKING_URI, "--port", str(port)],
+        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+    )
+    time.sleep(2)
+    webbrowser.open(f"http://localhost:{port}")
+    print(f"MLflow UI running at http://localhost:{port}")
+    print("Press Ctrl+C to stop")
+    return process
+
+
+def run_pipeline():
+    print(f"Running {PIPELINE_NAME}")
+    print("=" * 50)
+
+    setup_experiments_dir()
+{% if config.landing %}
+
+    print("\\n[1/6] Landing (event sources)...")
+    run_landing()
+    print("Landing complete")
+{% endif %}
+
+    print("\\n[{{ '2/6' if config.landing else '1/4' }}] Bronze (parallel)...")
+    run_bronze_parallel()
+    print("Bronze complete")
+
+    print("\\n[{{ '3/6' if config.landing else '2/4' }}] Silver...")
+    run_silver_merge()
+    print("Silver complete")
+
+    print("\\n[{{ '4/6' if config.landing else '3/4' }}] Gold...")
+    run_gold_features()
+    print("Gold complete")
+
+    print("\\n[{{ '5/6' if config.landing else '4/4' }}] Training...")
+    run_experiment()
+    print("Training complete")
+
+    print("\\n" + "=" * 50)
+    print("Pipeline finished!")
+
+    mlflow_process = start_mlflow_ui()
+    if mlflow_process:
+        try:
+            mlflow_process.wait()
+        except KeyboardInterrupt:
+            mlflow_process.terminate()
+            print("\\nMLflow UI stopped")
+
+
+if __name__ == "__main__":
+    run_pipeline()
+''',
+    "workflow.json.j2": """{
+  "name": "{{ config.name }}_pipeline",
+  "tasks": [
+{% for name in config.landing %}
+    {
+      "task_key": "landing_{{ name }}",
+      "notebook_task": {
+        "notebook_path": "/Workspace/orchestration/{{ config.name }}/landing/landing_{{ name }}"
+      }
+    },
+{% endfor %}
+{% for source in config.sources %}
+    {
+      "task_key": "bronze_{{ source.name }}",
+{% if config.landing %}
+      "depends_on": [
+{% for name in config.landing %}
+        {"task_key": "landing_{{ name }}"}{{ "," if not loop.last else "" }}
+{% endfor %}
+      ],
+{% endif %}
+      "notebook_task": {
+        "notebook_path": "/Workspace/orchestration/{{ config.name }}/bronze/bronze_{{ source.name }}"
+      }
+    },
+{% endfor %}
+    {
+      "task_key": "silver_merge",
+      "depends_on": [
+{% for source in config.sources %}
+        {"task_key": "bronze_{{ source.name }}"}{{ "," if not loop.last else "" }}
+{% endfor %}
+      ],
+      "notebook_task": {
+        "notebook_path": "/Workspace/orchestration/{{ config.name }}/silver/silver_merge"
+      }
+    },
+    {
+      "task_key": "gold_features",
+      "depends_on": [{"task_key": "silver_merge"}],
+      "notebook_task": {
+        "notebook_path": "/Workspace/orchestration/{{ config.name }}/gold/gold_features"
+      }
+    },
+    {
+      "task_key": "ml_experiment",
+      "depends_on": [{"task_key": "gold_features"}],
+      "notebook_task": {
+        "notebook_path": "/Workspace/orchestration/{{ config.name }}/training/ml_experiment"
+      }
+    }
+  ]
+}
+""",
+    "feature_store.yaml.j2": """project: {{ config.name }}
+registry: data/registry.db
+provider: local
+online_store:
+  type: sqlite
+  path: data/online_store.db
+offline_store:
+  type: file
+entity_key_serialization_version: 2
+""",
+    "features.py.j2": '''"""Feast Feature Definitions for {{ config.name }}
+
+Auto-generated feature view definitions for training/serving consistency.
+Feature version: {{ config.recommendations_hash or "unversioned" }}
+"""
+from datetime import timedelta
+from feast import Entity, FeatureView, Field, FileSource
+from feast.types import Float32, Float64, Int64, String
+
+# Entity definition
+{{ config.feast.entity_name if config.feast else "customer" }} = Entity(
+    name="{{ config.feast.entity_name if config.feast else 'customer' }}",
+    join_keys=["{{ config.feast.entity_key if config.feast else config.sources[0].entity_key }}"],
+    description="Primary entity for {{ config.name }} pipeline"
+)
+
+# File source pointing to materialized features
+{{ config.feast.feature_view_name if config.feast else config.name + '_features' }}_source = FileSource(
+    path="data/{{ config.feast.feature_view_name if config.feast else config.name + '_features' }}.parquet",
+    timestamp_field="{{ config.feast.timestamp_column if config.feast else 'event_timestamp' }}"
+)
+
+# Feature view definition
+# Note: Features are dynamically determined from the parquet file schema
+# This is a placeholder that gets populated when feast apply is run
+{{ config.feast.feature_view_name if config.feast else config.name + '_features' }} = FeatureView(
+    name="{{ config.feast.feature_view_name if config.feast else config.name + '_features' }}",
+    entities=[{{ config.feast.entity_name if config.feast else "customer" }}],
+    ttl=timedelta(days={{ config.feast.ttl_days if config.feast else 365 }}),
+    source={{ config.feast.feature_view_name if config.feast else config.name + '_features' }}_source,
+    tags={
+        "pipeline": "{{ config.name }}",
+        "recommendations_hash": "{{ config.recommendations_hash or 'none' }}",
+        "version": "v1.0.0_{{ config.recommendations_hash or 'unversioned' }}"
+    }
+)
+''',
+    "landing.py.j2": '''import pandas as pd
+import numpy as np
+from pathlib import Path
+from config import RAW_SOURCES, PRODUCTION_DIR
+
+SOURCE_NAME = "{{ name }}"
+ENTITY_COLUMN = "{{ config.entity_column }}"
+TIME_COLUMN = "{{ config.time_column }}"
+TARGET_COLUMN = "{{ config.target_column }}"
+
+
+def load_raw_data() -> pd.DataFrame:
+    source = RAW_SOURCES[SOURCE_NAME]
+    path = Path(source["path"])
+    if path.is_dir() and (path / "_delta_log").is_dir():
+        from customer_retention.integrations.adapters.factory import get_delta
+        return get_delta(force_local=True).read(str(path))
+    if not path.exists():
+        raise FileNotFoundError(f"Raw source not found: {path}")
+    if source["format"] == "csv":
+        return pd.read_csv(path)
+    return pd.read_parquet(path)
+
+{% if config.timestamp_coalesce %}
+
+def coalesce_timestamps(df: pd.DataFrame) -> pd.DataFrame:
+{% set cols = config.timestamp_coalesce.datetime_columns_ordered %}
+{% set out = config.timestamp_coalesce.output_column %}
+    df["{{ out }}"] = pd.to_datetime(df["{{ cols[-1] }}"], errors="coerce")
+{% for col in cols[:-1] | reverse %}
+    df["{{ out }}"] = df["{{ out }}"].fillna(pd.to_datetime(df["{{ col }}"], errors="coerce"))
+{% endfor %}
+    return df
+{% endif %}
+
+{% if config.label_timestamp %}
+
+def derive_label_timestamp(df: pd.DataFrame) -> pd.DataFrame:
+{% set lt = config.label_timestamp %}
+{% set feature_ts = config.timestamp_coalesce.output_column if config.timestamp_coalesce else config.time_column %}
+{% if lt.label_column %}
+    df["{{ lt.output_column }}"] = pd.to_datetime(df["{{ lt.label_column }}"], errors="coerce")
+    df["{{ lt.output_column }}"] = df["{{ lt.output_column }}"].fillna(
+        pd.to_datetime(df["{{ feature_ts }}"], errors="coerce") + pd.Timedelta(days={{ lt.fallback_window_days }})
+    )
+{% else %}
+    df["{{ lt.output_column }}"] = pd.to_datetime(df["{{ feature_ts }}"], errors="coerce") + pd.Timedelta(days={{ lt.fallback_window_days }})
+{% endif %}
+    return df
+{% endif %}
+
+
+def get_landing_output_path() -> Path:
+    return PRODUCTION_DIR / "data" / "landing" / f"{SOURCE_NAME}.parquet"
+
+
+def run_landing_{{ name }}():
+    print(f"Landing: {SOURCE_NAME}")
+    df = load_raw_data()
+    print(f"  Raw records: {len(df):,}")
+{% if config.raw_time_column %}
+    df = df.rename(columns={"{{ config.raw_time_column }}": TIME_COLUMN})
+{% endif %}
+{% if config.original_target_column %}
+    df = df.rename(columns={"{{ config.original_target_column }}": TARGET_COLUMN})
+{% endif %}
+{% if config.timestamp_coalesce %}
+    df = coalesce_timestamps(df)
+{% endif %}
+{% if config.label_timestamp %}
+    df = derive_label_timestamp(df)
+{% endif %}
+    output_path = get_landing_output_path()
+    output_dir = output_path.parent
+    output_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        from customer_retention.integrations.adapters.factory import get_delta
+        storage = get_delta(force_local=True)
+        storage.write(df, str(output_dir / SOURCE_NAME))
+    except ImportError:
+        df.to_parquet(output_path, index=False)
+    print(f"  Records: {len(df):,}")
+    print(f"  Output: {output_path}")
+    return df
+
+
+if __name__ == "__main__":
+    run_landing_{{ name }}()
+''',
+    "bronze_event.py.j2": '''import pandas as pd
+import numpy as np
+from pathlib import Path
+{% set ops, fitted = collect_imports(config.pre_shaping + config.post_shaping, False) %}
+{% if ops %}
+from customer_retention.transforms import {{ ops | sort | join(', ') }}
+{% endif %}
+from config import PRODUCTION_DIR, RAW_SOURCES, TARGET_COLUMN
+
+SOURCE_NAME = "{{ source }}"
+ENTITY_COLUMN = "{{ config.entity_column }}"
+TIME_COLUMN = "{{ config.time_column }}"
+
+{% set pre_groups = group_steps(config.pre_shaping) %}
+
+def apply_pre_shaping(df: pd.DataFrame) -> pd.DataFrame:
+{% if config.deduplicate %}
+    df = df.drop_duplicates(subset=[ENTITY_COLUMN, TIME_COLUMN], keep="first")
+{% endif %}
+{%- if pre_groups %}
+{%- for func_name, steps in pre_groups %}
+    df = {{ func_name }}(df)
+{%- endfor %}
+{%- endif %}
+    return df
+
+{% for func_name, steps in pre_groups %}
+
+def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
+{%- set _prov = provenance_docstring_block(steps) %}
+{%- if _prov %}
+{{ _prov }}
+{%- endif %}
+{%- for t in steps %}
+    # {{ t.rationale }}
+    # {{ action_description(t) }}
+    df = {{ render_step_call(t) }}
+{%- endfor %}
+    return df
+{% endfor %}
+
+{% if config.aggregation %}
+def _parse_window(window_str):
+    if window_str == "all_time":
+        return None
+    if window_str.endswith("d"):
+        return pd.Timedelta(days=int(window_str[:-1]))
+    if window_str.endswith("h"):
+        return pd.Timedelta(hours=int(window_str[:-1]))
+    if window_str.endswith("w"):
+        return pd.Timedelta(weeks=int(window_str[:-1]))
+    return pd.Timedelta(days=int(window_str))
+
+
+AGGREGATION_WINDOWS = {{ config.aggregation.windows }}
+VALUE_COLUMNS = {{ config.aggregation.value_columns }}
+AGG_FUNCS = {{ config.aggregation.agg_funcs }}
+{% endif %}
+
+
+def apply_reshaping(df: pd.DataFrame) -> pd.DataFrame:
+{% if config.aggregation %}
+    df[TIME_COLUMN] = pd.to_datetime(df[TIME_COLUMN])
+    reference_date = df[TIME_COLUMN].max()
+    result = df.groupby(ENTITY_COLUMN).agg("first")[[]]
+    if TARGET_COLUMN in df.columns:
+        result[TARGET_COLUMN] = df.groupby(ENTITY_COLUMN)[TARGET_COLUMN].first()
+    for window in AGGREGATION_WINDOWS:
+        td = _parse_window(window)
+        window_df = df if td is None else df[df[TIME_COLUMN] >= (reference_date - td)]
+        for col in VALUE_COLUMNS:
+            for func in AGG_FUNCS:
+                result[f"{col}_{func}_{window}"] = window_df.groupby(ENTITY_COLUMN)[col].agg(func)
+        result[f"event_count_{window}"] = window_df.groupby(ENTITY_COLUMN).size()
+    df = result.reset_index()
+{% endif %}
+    return df
+
+{% if config.lifecycle %}
+
+def _load_raw_events():
+    source = RAW_SOURCES[SOURCE_NAME]
+    path = Path(source["path"])
+    if path.is_dir() and (path / "_delta_log").is_dir():
+        from customer_retention.integrations.adapters.factory import get_delta
+        return get_delta(force_local=True).read(str(path))
+    if not path.exists():
+        raise FileNotFoundError(f"Raw source not found: {path}")
+    if source["format"] == "csv":
+        return pd.read_csv(path)
+    return pd.read_parquet(path)
+
+{% if config.lifecycle.include_recency_bucket %}
+
+def add_recency_tenure(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
+    raw_df[TIME_COLUMN] = pd.to_datetime(raw_df[TIME_COLUMN])
+    reference_date = raw_df[TIME_COLUMN].max()
+    entity_stats = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].agg(["min", "max"])
+    entity_stats["days_since_last"] = (reference_date - entity_stats["max"]).dt.days
+    entity_stats["days_since_first"] = (reference_date - entity_stats["min"]).dt.days
+    df = df.merge(entity_stats[["days_since_last", "days_since_first"]], left_on=ENTITY_COLUMN, right_index=True, how="left")
+    return df
+
+
+def add_recency_buckets(df: pd.DataFrame) -> pd.DataFrame:
+    if "days_since_last" in df.columns:
+        df["recency_bucket"] = pd.cut(df["days_since_last"], bins=[0, 7, 30, 90, 180, 365, float("inf")],
+                                       labels=["0-7d", "7-30d", "30-90d", "90-180d", "180-365d", "365d+"])
+    return df
+
+{% endif %}
+{% if config.lifecycle.include_lifecycle_quadrant %}
+
+def add_lifecycle_quadrant(df: pd.DataFrame) -> pd.DataFrame:
+    if "days_since_first" not in df.columns:
+        return df
+    tenure = df["days_since_first"]
+    intensity_col = [c for c in df.columns if c.startswith("event_count_")]
+    if not intensity_col:
+        return df
+    intensity = df[intensity_col[0]]
+    tenure_med = tenure.median()
+    intensity_med = intensity.median()
+    conditions = [
+        (tenure >= tenure_med) & (intensity >= intensity_med),
+        (tenure >= tenure_med) & (intensity < intensity_med),
+        (tenure < tenure_med) & (intensity >= intensity_med),
+        (tenure < tenure_med) & (intensity < intensity_med),
+    ]
+    labels = ["loyal", "at_risk", "new_active", "new_inactive"]
+    df["lifecycle_quadrant"] = np.select(conditions, labels, default="unknown")
+    return df
+
+{% endif %}
+{% if config.lifecycle.include_cyclical_features %}
+
+def add_cyclical_features(df: pd.DataFrame, raw_df: pd.DataFrame) -> pd.DataFrame:
+    raw_df[TIME_COLUMN] = pd.to_datetime(raw_df[TIME_COLUMN])
+    mean_dow = raw_df.groupby(ENTITY_COLUMN)[TIME_COLUMN].apply(lambda x: x.dt.dayofweek.mean())
+    df = df.merge(mean_dow.rename("mean_dow"), left_on=ENTITY_COLUMN, right_index=True, how="left")
+    df["dow_sin"] = np.sin(2 * np.pi * df["mean_dow"] / 7)
+    df["dow_cos"] = np.cos(2 * np.pi * df["mean_dow"] / 7)
+    df = df.drop(columns=["mean_dow"], errors="ignore")
+    return df
+
+{% endif %}
+{% if config.lifecycle.momentum_pairs %}
+
+def add_momentum_ratios(df: pd.DataFrame) -> pd.DataFrame:
+{% for pair in config.lifecycle.momentum_pairs %}
+    short_col = "event_count_{{ pair.short_window }}"
+    long_col = "event_count_{{ pair.long_window }}"
+    if short_col in df.columns and long_col in df.columns:
+        df["momentum_{{ pair.short_window }}_{{ pair.long_window }}"] = df[short_col] / df[long_col].replace(0, float("nan"))
+{% endfor %}
+    return df
+
+{% endif %}
+
+def enrich_lifecycle(df: pd.DataFrame) -> pd.DataFrame:
+    raw_df = _load_raw_events()
+{% if config.raw_time_column %}
+    raw_df = raw_df.rename(columns={"{{ config.raw_time_column }}": TIME_COLUMN})
+{% endif %}
+{% if config.lifecycle.include_recency_bucket %}
+    df = add_recency_tenure(df, raw_df)
+    df = add_recency_buckets(df)
+{% endif %}
+{% if config.lifecycle.include_lifecycle_quadrant %}
+    df = add_lifecycle_quadrant(df)
+{% endif %}
+{% if config.lifecycle.include_cyclical_features %}
+    df = add_cyclical_features(df, raw_df)
+{% endif %}
+{% if config.lifecycle.momentum_pairs %}
+    df = add_momentum_ratios(df)
+{% endif %}
+    return df
+{% endif %}
+
+{% set post_groups = group_steps(config.post_shaping) %}
+
+def apply_post_shaping(df: pd.DataFrame) -> pd.DataFrame:
+{% if config.lifecycle %}
+    df = enrich_lifecycle(df)
+{% endif %}
+{%- if post_groups %}
+{%- for func_name, steps in post_groups %}
+    df = {{ func_name }}(df)
+{%- endfor %}
+{%- endif %}
+    return df
+
+{% for func_name, steps in post_groups %}
+
+def {{ func_name }}(df: pd.DataFrame) -> pd.DataFrame:
+{%- set _prov = provenance_docstring_block(steps) %}
+{%- if _prov %}
+{{ _prov }}
+{%- endif %}
+{%- for t in steps %}
+    # {{ t.rationale }}
+    # {{ action_description(t) }}
+    df = {{ render_step_call(t) }}
+{%- endfor %}
+    return df
+{% endfor %}
+
+
+def run_bronze_{{ source }}():
+    landing_dir = PRODUCTION_DIR / "data" / "landing" / SOURCE_NAME
+    landing_parquet = PRODUCTION_DIR / "data" / "landing" / f"{SOURCE_NAME}.parquet"
+    if landing_dir.is_dir() and (landing_dir / "_delta_log").is_dir():
+        from customer_retention.integrations.adapters.factory import get_delta
+        df = get_delta(force_local=True).read(str(landing_dir))
+    elif landing_parquet.exists():
+        df = pd.read_parquet(landing_parquet)
+    else:
+        raise FileNotFoundError(f"Landing output not found: {landing_parquet}")
+    df = apply_pre_shaping(df)
+    df = apply_reshaping(df)
+    df = apply_post_shaping(df)
+    bronze_dir = PRODUCTION_DIR / "data" / "bronze"
+    bronze_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        from customer_retention.integrations.adapters.factory import get_delta
+        storage = get_delta(force_local=True)
+        storage.write(df, str(bronze_dir / SOURCE_NAME))
+    except ImportError:
+        output_path = bronze_dir / f"{SOURCE_NAME}.parquet"
+        df.to_parquet(output_path, index=False)
+    return df
+
+
+if __name__ == "__main__":
+    run_bronze_{{ source }}()
+''',
+    "validate.py.j2": '''import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+import pandas as pd
+import numpy as np
+from config import SOURCES, EXPLORATION_ARTIFACTS, EXPERIMENTS_DIR, PRODUCTION_DIR, TARGET_COLUMN
+
+
+def _load_artifact(path):
+    path = Path(path)
+    if path.is_dir() and (path / "_delta_log").is_dir():
+        from customer_retention.integrations.adapters.factory import get_delta
+        return get_delta(force_local=True).read(str(path))
+    return pd.read_parquet(path)
+
+
+def _compare_dataframes(stage, production_path, exploration_path, entity_key=None, tolerance=1e-5):
+    if not Path(production_path).exists() and not (Path(production_path).is_dir() and (Path(production_path) / "_delta_log").is_dir()):
+        raise FileNotFoundError(f"[{stage}] Production output not found: {production_path}")
+    if not Path(exploration_path).exists() and not (Path(exploration_path).is_dir() and (Path(exploration_path) / "_delta_log").is_dir()):
+        print(f"[{stage}] SKIP - exploration artifact not found: {exploration_path}")
+        return True
+
+    prod = _load_artifact(production_path)
+    expl = _load_artifact(exploration_path)
+
+    if entity_key and entity_key in prod.columns and entity_key in expl.columns:
+        prod = prod.sort_values(entity_key).reset_index(drop=True)
+        expl = expl.sort_values(entity_key).reset_index(drop=True)
+
+    if prod.shape[0] != expl.shape[0]:
+        raise AssertionError(f"[{stage}] Row count: production={prod.shape[0]} vs exploration={expl.shape[0]}")
+
+    prod_cols = set(prod.columns)
+    expl_cols = set(expl.columns)
+    missing = expl_cols - prod_cols
+    extra = prod_cols - expl_cols
+    if missing:
+        print(f"[{stage}] WARNING: missing columns: {missing}")
+    if extra:
+        print(f"[{stage}] INFO: extra columns: {extra}")
+
+    common = sorted(prod_cols & expl_cols)
+    for col in common:
+        if pd.api.types.is_numeric_dtype(prod[col]) and pd.api.types.is_numeric_dtype(expl[col]):
+            try:
+                pd.testing.assert_series_equal(prod[col], expl[col], check_exact=False, rtol=tolerance, check_names=False)
+            except AssertionError as e:
+                delta = (prod[col].astype(float) - expl[col].astype(float)).abs()
+                max_idx = delta.idxmax()
+                raise AssertionError(
+                    f"[{stage}] Column '{col}' diverges at row {max_idx}: "
+                    f"production={prod[col].iloc[max_idx]} vs exploration={expl[col].iloc[max_idx]} "
+                    f"(max delta={delta.max():.2e})"
+                ) from None
+
+    print(f"[{stage}] PASS - {prod.shape[0]} rows, {len(common)} common cols, tolerance={tolerance}")
+    return True
+
+
+def validate_landing(tolerance=1e-5):
+    landing_dir = PRODUCTION_DIR / "data" / "landing"
+    if not landing_dir.exists():
+        print("[Landing] SKIP - no landing directory")
+        return True
+    for path in landing_dir.glob("*.parquet"):
+        name = path.stem
+        expl_key = f"landing_{name}" if f"landing_{name}" in EXPLORATION_ARTIFACTS else "landing"
+        if expl_key in EXPLORATION_ARTIFACTS:
+            _compare_dataframes(f"Landing/{name}", str(path), EXPLORATION_ARTIFACTS[expl_key])
+    return True
+
+
+def validate_bronze(tolerance=1e-5):
+    bronze_artifacts = EXPLORATION_ARTIFACTS.get("bronze", {})
+    for name, expl_path in bronze_artifacts.items():
+        prod_path = PRODUCTION_DIR / "data" / "bronze" / f"{name}.parquet"
+        _compare_dataframes(f"Bronze/{name}", str(prod_path), expl_path, tolerance=tolerance)
+    return True
+
+
+def validate_silver(tolerance=1e-5):
+    prod_path = PRODUCTION_DIR / "data" / "silver" / "merged.parquet"
+    expl_path = EXPLORATION_ARTIFACTS.get("silver", "")
+    entity_key = list(SOURCES.values())[0]["entity_key"] if SOURCES else None
+    _compare_dataframes("Silver", str(prod_path), expl_path, entity_key=entity_key, tolerance=tolerance)
+    return True
+
+
+def validate_gold(tolerance=1e-5):
+    prod_path = PRODUCTION_DIR / "data" / "gold" / "features.parquet"
+    expl_path = EXPLORATION_ARTIFACTS.get("gold", "")
+    entity_key = list(SOURCES.values())[0]["entity_key"] if SOURCES else None
+    _compare_dataframes("Gold", str(prod_path), expl_path, entity_key=entity_key, tolerance=tolerance)
+    return True
+
+
+def validate_training():
+    print("[Training] PASS - training validation requires MLflow comparison (not yet implemented)")
+    return True
+
+
+def validate_scoring(tolerance=1e-5):
+    prod_path = PRODUCTION_DIR / "data" / "scoring" / "predictions.parquet"
+    expl_path = EXPLORATION_ARTIFACTS.get("scoring", "")
+    _compare_dataframes("Scoring", str(prod_path), expl_path, tolerance=tolerance)
+    return True
+
+
+def run_all_validations(tolerance=1e-5):
+    stages = [
+        ("Landing", lambda: validate_landing(tolerance)),
+        ("Bronze", lambda: validate_bronze(tolerance)),
+        ("Silver", lambda: validate_silver(tolerance)),
+        ("Gold", lambda: validate_gold(tolerance)),
+        ("Training", validate_training),
+        ("Scoring", lambda: validate_scoring(tolerance)),
+    ]
+    results = []
+    for name, fn in stages:
+        try:
+            fn()
+            results.append((name, "PASS"))
+        except Exception as e:
+            results.append((name, f"FAIL: {e}"))
+            break
+
+    print("\\nStage Validation Report")
+    print("=" * 50)
+    for name, status in results:
+        print(f"[{status.split(':')[0]:4s}] {name}")
+    return results
+''',
+    "run_validation.py.j2": '''"""{{ config.name }} - Standalone Validation Runner
+
+Compares pipeline outputs against exploration artifacts.
+Run after pipeline completes to verify correctness.
+"""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from validation.validate_pipeline import run_all_validations
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description="Validate pipeline outputs")
+    parser.add_argument("--tolerance", type=float, default=1e-5)
+    args = parser.parse_args()
+
+    results = run_all_validations(tolerance=args.tolerance)
+    failures = [r for r in results if not r[1].startswith("PASS")]
+    sys.exit(1 if failures else 0)
+''',
+    "exploration_report.py.j2": '''"""Exploration Report Viewer
+
+Opens HTML documentation for the exploration notebooks that informed
+the pipeline transformations. Works both locally (file:// URI) and
+on Databricks (displayHTML with scroll-to-anchor injection).
+"""
+import os
+import webbrowser
+from pathlib import Path
+
+# Known notebooks referenced by pipeline provenance comments
+KNOWN_NOTEBOOKS = [
+{% for nb in notebooks %}
+    "{{ nb }}",
+{% endfor %}
+]
+
+DOCS_DIR = Path(os.environ.get("CR_DOCS_BASE_URL", str(Path(__file__).parent)))
+
+
+def _is_databricks():
+    return "DATABRICKS_RUNTIME_VERSION" in os.environ
+
+
+def list_reports():
+    for nb in KNOWN_NOTEBOOKS:
+        html_path = DOCS_DIR / f"{nb}.html"
+        status = "available" if html_path.exists() else "missing"
+        print(f"  {nb}: {status}")
+
+
+if __name__ == "__main__":
+    print("Available exploration reports:")
+    list_reports()
+''',
+}
+
+
+class CodeRenderer:
+    _TEMPLATE_MAP = {
+        "config": "config.py.j2",
+        "silver": "silver.py.j2",
+        "gold": "gold.py.j2",
+        "training": "training.py.j2",
+        "runner": "runner.py.j2",
+        "workflow": "workflow.json.j2",
+        "run_all": "run_all.py.j2",
+        "feast_config": "feature_store.yaml.j2",
+        "feast_features": "features.py.j2",
+
+        "landing": "landing.py.j2",
+        "bronze_event": "bronze_event.py.j2",
+        "validation": "validate.py.j2",
+        "run_validation": "run_validation.py.j2",
+        "exploration_report": "exploration_report.py.j2",
+    }
+
+    def __init__(self):
+        self._env = Environment(loader=InlineLoader(TEMPLATES))
+        self._env.globals["action_description"] = action_description
+        self._env.globals["render_step_call"] = render_step_call
+        self._env.globals["collect_imports"] = collect_imports
+        self._env.globals["group_steps"] = group_steps
+        self._env.globals["provenance_docstring_block"] = provenance_docstring_block
+        self._env.globals["provenance_key"] = provenance_key
+
+    def set_docs_base(self, experiments_dir: str | None) -> None:
+        global _docs_base
+        if experiments_dir:
+            _docs_base = f"file://{Path(experiments_dir).resolve() / 'docs'}"
+        else:
+            _docs_base = "docs"
+
+    def _render(self, template_key: str, **context) -> str:
+        return self._env.get_template(self._TEMPLATE_MAP[template_key]).render(**context)
+
+    def render_config(self, config: PipelineConfig) -> str:
+        return self._render("config", config=config)
+
+    def render_bronze(self, source_name: str, bronze_config: BronzeLayerConfig) -> str:
+        return self._env.get_template("bronze.py.j2").render(source=source_name, config=bronze_config)
+
+    def render_silver(self, config: PipelineConfig) -> str:
+        return self._render("silver", config=config)
+
+    def render_gold(self, config: PipelineConfig) -> str:
+        return self._render("gold", config=config)
+
+    def render_training(self, config: PipelineConfig) -> str:
+        return self._render("training", config=config)
+
+    def render_runner(self, config: PipelineConfig) -> str:
+        return self._render("runner", config=config)
+
+    def render_workflow(self, config: PipelineConfig) -> str:
+        return self._render("workflow", config=config)
+
+    def render_run_all(self, config: PipelineConfig) -> str:
+        return self._render("run_all", config=config)
+
+    def render_feast_config(self, config: PipelineConfig) -> str:
+        return self._render("feast_config", config=config)
+
+    def render_feast_features(self, config: PipelineConfig) -> str:
+        return self._render("feast_features", config=config)
+
+
+    def render_landing(self, name: str, config: LandingLayerConfig) -> str:
+        return self._env.get_template("landing.py.j2").render(name=name, config=config)
+
+    def render_bronze_event(self, source_name: str, config: BronzeEventConfig) -> str:
+        return self._env.get_template("bronze_event.py.j2").render(source=source_name, config=config)
+
+    def render_validation(self, config: PipelineConfig) -> str:
+        return self._render("validation", config=config)
+
+    def render_run_validation(self, config: PipelineConfig) -> str:
+        return self._render("run_validation", config=config)
+
+    def render_exploration_report(self, config: PipelineConfig) -> str:
+        notebooks = set()
+        for bronze in config.bronze.values():
+            for step in bronze.transformations:
+                nb = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
+                if nb:
+                    notebooks.add(nb)
+        for step in config.gold.transformations + config.gold.encodings + config.gold.scalings:
+            nb = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
+            if nb:
+                notebooks.add(nb)
+        for step in config.silver.derived_columns:
+            nb = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
+            if nb:
+                notebooks.add(nb)
+        for be in config.bronze_event.values():
+            for step in be.pre_shaping + be.post_shaping:
+                nb = step.source_notebook or DEFAULT_NOTEBOOK_MAP.get(step.type)
+                if nb:
+                    notebooks.add(nb)
+        return self._render("exploration_report", notebooks=sorted(notebooks))
+
+
+_StepMeta = namedtuple("_StepMeta", ["desc_tpl", "call_tpl", "import_name", "param_defaults"])
+
+_STATELESS_REGISTRY = {
+    PipelineTransformationType.IMPUTE_NULL: _StepMeta(
+        "impute nulls in {col} with {value}",
+        "apply_impute_null(df, '{col}', value='{value}')",
+        "apply_impute_null", {"value": 0}),
+    PipelineTransformationType.CAP_OUTLIER: _StepMeta(
+        "cap outliers in {col} to [{lower}, {upper}]",
+        "apply_cap_outlier(df, '{col}', lower={lower}, upper={upper})",
+        "apply_cap_outlier", {"lower": 0, "upper": 1000000}),
+    PipelineTransformationType.TYPE_CAST: _StepMeta(
+        "cast {col} to {dtype}",
+        "apply_type_cast(df, '{col}', dtype='{dtype}')",
+        "apply_type_cast", {"dtype": "float"}),
+    PipelineTransformationType.DROP_COLUMN: _StepMeta(
+        "drop column {col}",
+        "apply_drop_column(df, '{col}')",
+        "apply_drop_column", {}),
+    PipelineTransformationType.WINSORIZE: _StepMeta(
+        "winsorize {col} to [{lower_bound}, {upper_bound}]",
+        "apply_winsorize(df, '{col}', lower_bound={lower_bound}, upper_bound={upper_bound})",
+        "apply_winsorize", {"lower_bound": 0, "upper_bound": 1000000}),
+    PipelineTransformationType.SEGMENT_AWARE_CAP: _StepMeta(
+        "segment-aware outlier cap on {col} ({n_segments} segments)",
+        "apply_segment_aware_cap(df, '{col}', n_segments={n_segments})",
+        "apply_segment_aware_cap", {"n_segments": 2}),
+    PipelineTransformationType.LOG_TRANSFORM: _StepMeta(
+        "log-transform {col}",
+        "apply_log_transform(df, '{col}')",
+        "apply_log_transform", {}),
+    PipelineTransformationType.SQRT_TRANSFORM: _StepMeta(
+        "sqrt-transform {col}",
+        "apply_sqrt_transform(df, '{col}')",
+        "apply_sqrt_transform", {}),
+    PipelineTransformationType.ZERO_INFLATION_HANDLING: _StepMeta(
+        "handle zero-inflation in {col}",
+        "apply_zero_inflation_handling(df, '{col}')",
+        "apply_zero_inflation_handling", {}),
+    PipelineTransformationType.CAP_THEN_LOG: _StepMeta(
+        "cap at p99 then log-transform {col}",
+        "apply_cap_then_log(df, '{col}')",
+        "apply_cap_then_log", {}),
+    PipelineTransformationType.FEATURE_SELECT: _StepMeta(
+        "drop {col} (feature selection)",
+        "apply_feature_select(df, '{col}')",
+        "apply_feature_select", {}),
+}
+
+
+def _extract_params(step, meta):
+    return {k: step.parameters.get(k, v) for k, v in meta.param_defaults.items()}
+
+
+def action_description(step: TransformationStep) -> str:
+    t, col, p = step.type, step.column, step.parameters
+    meta = _STATELESS_REGISTRY.get(t)
+    if meta is not None:
+        return meta.desc_tpl.format(col=col, **_extract_params(step, meta))
+    if t == PipelineTransformationType.YEO_JOHNSON:
+        return f"yeo-johnson transform {col}"
+    if t == PipelineTransformationType.ENCODE:
+        method = p.get("method", "one_hot")
+        if method in ("one_hot", "onehot"):
+            return f"one-hot encode {col}"
+        return f"label-encode {col}"
+    if t == PipelineTransformationType.SCALE:
+        method = p.get("method", "standard")
+        if method == "minmax":
+            return f"min-max scale {col}"
+        return f"standard-scale {col}"
+    if t == PipelineTransformationType.DERIVED_COLUMN:
+        action = p.get("action", "ratio")
+        if action == "ratio":
+            return f"create {col} = {p.get('numerator', '?')} / {p.get('denominator', '?')}"
+        if action == "interaction":
+            features = p.get("features", [])
+            col_a = features[0] if len(features) > 0 else p.get("col_a", "?")
+            col_b = features[1] if len(features) > 1 else p.get("col_b", "?")
+            return f"create {col} = {col_a} * {col_b}"
+        if action == "composite":
+            return f"create {col} = mean({', '.join(p.get('columns', []))})"
+    return f"transform {col}"
+
+
+def render_step_call(step: TransformationStep, fit_mode: bool = True) -> str:
+    t, col, p = step.type, step.column, step.parameters
+    meta = _STATELESS_REGISTRY.get(t)
+    if meta is not None:
+        return meta.call_tpl.format(col=col, **_extract_params(step, meta))
+    if t == PipelineTransformationType.YEO_JOHNSON:
+        method = "fit_transform" if fit_mode else "transform"
+        return f"FittedPowerTransform().{method}(df, '{col}', _store)"
+    if t == PipelineTransformationType.ENCODE:
+        method = p.get("method", "one_hot")
+        if method in ("one_hot", "onehot"):
+            return f"apply_one_hot_encode(df, '{col}')"
+        fit_method = "fit_transform" if fit_mode else "transform"
+        return f"FittedEncoder().{fit_method}(df, '{col}', _store)"
+    if t == PipelineTransformationType.SCALE:
+        method = p.get("method", "standard")
+        fit_method = "fit_transform" if fit_mode else "transform"
+        return f"FittedScaler('{method}').{fit_method}(df, '{col}', _store)"
+    if t == PipelineTransformationType.DERIVED_COLUMN:
+        action = p.get("action", "ratio")
+        if action == "ratio":
+            return f"apply_derived_ratio(df, '{col}', numerator='{p.get('numerator', '')}', denominator='{p.get('denominator', '')}')"
+        if action == "interaction":
+            features = p.get("features", [])
+            col_a = features[0] if len(features) > 0 else p.get("col_a", "")
+            col_b = features[1] if len(features) > 1 else p.get("col_b", "")
+            return f"apply_derived_interaction(df, '{col}', col_a='{col_a}', col_b='{col_b}')"
+        if action == "composite":
+            return f"apply_derived_composite(df, '{col}', columns={p.get('columns', [])})"
+    raise ValueError(f"Unknown transformation type: {step.type}")
+
+
+def collect_imports(steps, include_fitted):
+    ops = set()
+    fitted = set()
+    _OPS_MAP = {k: v.import_name for k, v in _STATELESS_REGISTRY.items()}
+    for step in steps:
+        t, p = step.type, step.parameters
+        if t in _OPS_MAP:
+            ops.add(_OPS_MAP[t])
+        elif t == PipelineTransformationType.ENCODE:
+            method = p.get("method", "one_hot")
+            if method in ("one_hot", "onehot"):
+                ops.add("apply_one_hot_encode")
+            elif include_fitted:
+                fitted.add("FittedEncoder")
+        elif t == PipelineTransformationType.SCALE:
+            if include_fitted:
+                fitted.add("FittedScaler")
+        elif t == PipelineTransformationType.YEO_JOHNSON:
+            if include_fitted:
+                fitted.add("FittedPowerTransform")
+        elif t == PipelineTransformationType.DERIVED_COLUMN:
+            action = p.get("action", "ratio")
+            if action == "ratio":
+                ops.add("apply_derived_ratio")
+            elif action == "interaction":
+                ops.add("apply_derived_interaction")
+            elif action == "composite":
+                ops.add("apply_derived_composite")
+    return ops, fitted