churnkit 0.75.0a1__py3-none-any.whl

customer_retention/generators/notebook_generator/stages/base_stage.py

@@ -0,0 +1,86 @@
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, List, Optional
+
+import nbformat
+
+from ..base import NotebookStage
+
+if TYPE_CHECKING:
+    from customer_retention.analysis.auto_explorer import ExplorationFindings
+from ..cell_builder import CellBuilder
+from ..config import NotebookConfig, Platform
+
+
+class StageGenerator(ABC):
+    def __init__(self, config: NotebookConfig, findings: Optional["ExplorationFindings"]):
+        self.config = config
+        self.findings = findings
+        self.cb = CellBuilder
+
+    @property
+    @abstractmethod
+    def stage(self) -> NotebookStage:
+        pass
+
+    @property
+    @abstractmethod
+    def title(self) -> str:
+        pass
+
+    @property
+    def description(self) -> str:
+        return ""
+
+    def generate(self, platform: Platform) -> List[nbformat.NotebookNode]:
+        if platform == Platform.LOCAL:
+            return self.generate_local_cells()
+        return self.generate_databricks_cells()
+
+    @abstractmethod
+    def generate_local_cells(self) -> List[nbformat.NotebookNode]:
+        pass
+
+    @abstractmethod
+    def generate_databricks_cells(self) -> List[nbformat.NotebookNode]:
+        pass
+
+    def header_cells(self) -> List[nbformat.NotebookNode]:
+        cells = [self.cb.header(self.title)]
+        if self.description:
+            cells.append(self.cb.markdown(self.description))
+        return cells
+
+    def get_target_column(self) -> str:
+        if self.findings and hasattr(self.findings, "target_column") and self.findings.target_column:
+            return self.findings.target_column
+        return "target"
+
+    def get_identifier_columns(self) -> List[str]:
+        if self.findings and hasattr(self.findings, "identifier_columns") and self.findings.identifier_columns:
+            return self.findings.identifier_columns
+        return ["customer_id"]
+
+    def get_feature_columns(self) -> List[str]:
+        if not self.findings or not hasattr(self.findings, "columns"):
+            return []
+        from customer_retention.core.config import ColumnType
+        feature_types = {ColumnType.NUMERIC_CONTINUOUS, ColumnType.NUMERIC_DISCRETE,
+                        ColumnType.CATEGORICAL_NOMINAL, ColumnType.CATEGORICAL_ORDINAL, ColumnType.BINARY}
+        return [name for name, col in self.findings.columns.items()
+                if hasattr(col, "inferred_type") and col.inferred_type in feature_types]
+
+    def get_numeric_columns(self) -> List[str]:
+        if not self.findings or not hasattr(self.findings, "columns"):
+            return []
+        from customer_retention.core.config import ColumnType
+        return [name for name, col in self.findings.columns.items()
+                if hasattr(col, "inferred_type") and col.inferred_type in
+                {ColumnType.NUMERIC_CONTINUOUS, ColumnType.NUMERIC_DISCRETE}]
+
+    def get_categorical_columns(self) -> List[str]:
+        if not self.findings or not hasattr(self.findings, "columns"):
+            return []
+        from customer_retention.core.config import ColumnType
+        return [name for name, col in self.findings.columns.items()
+                if hasattr(col, "inferred_type") and col.inferred_type in
+                {ColumnType.CATEGORICAL_NOMINAL, ColumnType.CATEGORICAL_ORDINAL}]

customer_retention/generators/notebook_generator/stages/s01_ingestion.py

@@ -0,0 +1,100 @@
+from typing import List
+
+import nbformat
+
+from ..base import NotebookStage
+from .base_stage import StageGenerator
+
+
+class IngestionStage(StageGenerator):
+    @property
+    def stage(self) -> NotebookStage:
+        return NotebookStage.INGESTION
+
+    @property
+    def title(self) -> str:
+        return "01 - Configuration & Data Ingestion"
+
+    @property
+    def description(self) -> str:
+        return "Load raw data, configure pipeline context, and save to bronze layer."
+
+    def generate_local_cells(self) -> List[nbformat.NotebookNode]:
+        findings_path = self.findings.source_path if self.findings else "./data/customers.csv"
+        source_format = getattr(self.findings, "source_format", "csv") if self.findings else "csv"
+        return self.header_cells() + [
+            self.cb.section("Imports"),
+            self.cb.from_imports_cell({
+                "customer_retention.generators.orchestration": ["setup_notebook_context", "PipelineContext"],
+                "customer_retention.stages.ingestion": ["DataSourceRegistry"],
+                "customer_retention.analysis.auto_explorer": ["ExplorationFindings"],
+                "customer_retention.stages.temporal": ["ScenarioDetector", "UnifiedDataPreparer"],
+                "datetime": ["datetime"],
+                "pathlib": ["Path"],
+            }),
+            self.cb.section("Configuration"),
+            self.cb.code(f'''FINDINGS_PATH = "{findings_path}"
+DATA_FORMAT = "{source_format}"
+OUTPUT_DIR = Path("./experiments/data")'''),
+            self.cb.section("Load Exploration Findings"),
+            self.cb.code('''findings = ExplorationFindings.load(FINDINGS_PATH)
+print(f"Loaded findings: {findings.row_count} rows, {findings.column_count} columns")
+print(f"Target column: {findings.target_column}")'''),
+            self.cb.section("Setup Pipeline Context"),
+            self.cb.code('''ctx, manager = setup_notebook_context(exploration_findings=findings)
+print(f"Pipeline context initialized for: {ctx.config.project_name}")'''),
+            self.cb.section("Load Raw Data"),
+            self.cb.code('''registry = DataSourceRegistry()
+df = registry.load(findings.source_path, format=DATA_FORMAT)
+print(f"Loaded {len(df)} rows")
+df.head()'''),
+            self.cb.section("Detect Timestamp Scenario"),
+            self.cb.code('''detector = ScenarioDetector()
+scenario, ts_config, discovery_result = detector.detect(df, findings.target_column)
+print(f"Detected scenario: {scenario}")
+print(f"Strategy: {ts_config.strategy.value}")
+print(f"Recommendation: {discovery_result.recommendation}")'''),
+            self.cb.section("Prepare Data with Timestamps"),
+            self.cb.code('''preparer = UnifiedDataPreparer(OUTPUT_DIR, ts_config)
+unified_df = preparer.prepare_from_raw(
+    df,
+    target_column=findings.target_column,
+    entity_column=findings.entity_id_column or "custid"
+)
+print(f"Prepared {len(unified_df)} rows with timestamps")
+print(f"Timestamp columns: feature_timestamp, label_timestamp, label_available_flag")'''),
+            self.cb.section("Create Training Snapshot"),
+            self.cb.code('''cutoff_date = datetime.now()
+snapshot_df, metadata = preparer.create_training_snapshot(unified_df, cutoff_date)
+print(f"Created snapshot: {metadata['snapshot_id']}")
+print(f"Rows: {metadata['row_count']}")
+print(f"Features: {len(metadata['feature_columns'])}")'''),
+            self.cb.section("Save Processed Data"),
+            self.cb.code('''manager.update(current_df=snapshot_df, current_stage="bronze")
+print(f"Pipeline context updated. Use snapshot '{metadata['snapshot_id']}' for training.")'''),
+        ]
+
+    def generate_databricks_cells(self) -> List[nbformat.NotebookNode]:
+        catalog = self.config.feature_store.catalog
+        schema = self.config.feature_store.schema
+        data_path = self.findings.source_path if self.findings else "/mnt/landing/customers"
+        source_format = getattr(self.findings, "source_format", "csv") if self.findings else "csv"
+        return self.header_cells() + [
+            self.cb.section("Configuration"),
+            self.cb.code(f'''CATALOG = "{catalog}"
+SCHEMA = "{schema}"
+DATA_PATH = "{data_path}"
+spark.sql(f"USE CATALOG {{CATALOG}}")
+spark.sql(f"USE SCHEMA {{SCHEMA}}")'''),
+            self.cb.section("Load Raw Data"),
+            self.cb.code(f'''df_raw = (spark.read
+    .format("{source_format}")
+    .option("header", "true")
+    .option("inferSchema", "true")
+    .load(DATA_PATH))
+print(f"Loaded {{df_raw.count()}} rows")
+display(df_raw.limit(10))'''),
+            self.cb.section("Save to Bronze Table"),
+            self.cb.code('''df_raw.write.format("delta").mode("overwrite").saveAsTable(f"{CATALOG}.{SCHEMA}.bronze_customers")
+print("Bronze table created")'''),
+        ]

customer_retention/generators/notebook_generator/stages/s02_profiling.py

@@ -0,0 +1,95 @@
+from typing import List
+
+import nbformat
+
+from ..base import NotebookStage
+from .base_stage import StageGenerator
+
+
+class ProfilingStage(StageGenerator):
+    @property
+    def stage(self) -> NotebookStage:
+        return NotebookStage.PROFILING
+
+    @property
+    def title(self) -> str:
+        return "02 - Data Profiling"
+
+    @property
+    def description(self) -> str:
+        return "Generate column statistics, type detection, and quality metrics."
+
+    def generate_local_cells(self) -> List[nbformat.NotebookNode]:
+        return self.header_cells() + [
+            self.cb.section("Imports"),
+            self.cb.from_imports_cell({
+                "customer_retention.stages.profiling": ["TypeDetector", "ProfilerFactory", "QualityCheckRegistry"],
+                "customer_retention.analysis.visualization": ["ChartBuilder"],
+                "pandas": ["pd"],
+            }),
+            self.cb.section("Load Bronze Data"),
+            self.cb.code('''from customer_retention.integrations.adapters.factory import get_delta
+storage = get_delta(force_local=True)
+df = storage.read("./experiments/data/bronze/customers")
+print(f"Loaded {len(df)} rows, {len(df.columns)} columns")'''),
+            self.cb.section("Type Detection"),
+            self.cb.code('''detector = TypeDetector()
+type_results = {col: detector.detect(df[col]) for col in df.columns}
+for col, result in type_results.items():
+    print(f"{col}: {result.column_type.value} (confidence: {result.confidence:.2f})")'''),
+            self.cb.section("Column Profiling"),
+            self.cb.code('''factory = ProfilerFactory()
+profiles = {}
+for col in df.columns:
+    profiler = factory.get_profiler(type_results[col].column_type)
+    profiles[col] = profiler.profile(df[col])'''),
+            self.cb.section("Quality Checks"),
+            self.cb.code('''registry = QualityCheckRegistry()
+checks = registry.get_all_checks()
+results = []
+for check in checks:
+    for col in df.columns:
+        result = check.check(df[col], profiles.get(col))
+        if result.passed is False:
+            results.append({"column": col, "check": check.name, "severity": result.severity.value, "message": result.message})
+quality_df = pd.DataFrame(results)
+quality_df'''),
+            self.cb.section("Visualize Quality"),
+            self.cb.code('''charts = ChartBuilder()
+if len(quality_df) > 0:
+    fig = charts.quality_heatmap(quality_df)
+    fig.show()'''),
+        ]
+
+    def generate_databricks_cells(self) -> List[nbformat.NotebookNode]:
+        catalog = self.config.feature_store.catalog
+        schema = self.config.feature_store.schema
+        return self.header_cells() + [
+            self.cb.section("Load Bronze Data"),
+            self.cb.code(f'''df = spark.table("{catalog}.{schema}.bronze_customers")
+print(f"Loaded {{df.count()}} rows")'''),
+            self.cb.section("Basic Statistics"),
+            self.cb.code('''summary = df.describe()
+display(summary)'''),
+            self.cb.section("Column Types and Nulls"),
+            self.cb.code('''from pyspark.sql.functions import col, count, when, isnan
+
+null_counts = df.select([
+    count(when(col(c).isNull() | isnan(col(c)), c)).alias(c)
+    for c in df.columns
+])
+display(null_counts)'''),
+            self.cb.section("Distinct Values"),
+            self.cb.code('''from pyspark.sql.functions import countDistinct
+
+distinct_counts = df.select([countDistinct(col(c)).alias(c) for c in df.columns])
+display(distinct_counts)'''),
+            self.cb.section("Save Profiling Results"),
+            self.cb.code('''profile_data = {
+    "columns": df.columns,
+    "dtypes": [str(f.dataType) for f in df.schema.fields],
+    "row_count": df.count()
+}
+import json
+dbutils.fs.put("/tmp/profile_results.json", json.dumps(profile_data), overwrite=True)'''),
+        ]

customer_retention/generators/notebook_generator/stages/s03_cleaning.py

@@ -0,0 +1,180 @@
+from typing import List
+
+import nbformat
+
+from ..base import NotebookStage
+from .base_stage import StageGenerator
+
+
+class CleaningStage(StageGenerator):
+    @property
+    def stage(self) -> NotebookStage:
+        return NotebookStage.CLEANING
+
+    @property
+    def title(self) -> str:
+        return "03 - Data Cleaning"
+
+    @property
+    def description(self) -> str:
+        return "Handle missing values and outliers based on column types with MLflow tracking."
+
+    def _get_cleaning_recommendations(self) -> dict:
+        recommendations = {}
+        if not self.findings or not hasattr(self.findings, "columns"):
+            return recommendations
+        for col_name, col_finding in self.findings.columns.items():
+            if hasattr(col_finding, "cleaning_recommendations") and col_finding.cleaning_recommendations:
+                recommendations[col_name] = col_finding.cleaning_recommendations
+        return recommendations
+
+    def generate_local_cells(self) -> List[nbformat.NotebookNode]:
+        numeric_cols = self.get_numeric_columns()
+        categorical_cols = self.get_categorical_columns()
+        tracking_uri = self.config.mlflow.tracking_uri
+        exp_name = self.config.mlflow.experiment_name
+        cleaning_recs = self._get_cleaning_recommendations()
+
+        cells = self.header_cells() + [
+            self.cb.section("Imports"),
+            self.cb.from_imports_cell({
+                "customer_retention.stages.cleaning": ["MissingValueHandler", "OutlierHandler"],
+                "customer_retention.integrations.adapters": ["get_mlflow"],
+                "pandas": ["pd"],
+            }),
+            self.cb.section("Setup MLflow Tracking"),
+            self.cb.code(f'''mlflow_adapter = get_mlflow(tracking_uri="{tracking_uri}", force_local=True)
+mlflow_adapter.start_run("{exp_name}", run_name="03_data_cleaning")
+cleaning_stats = {{}}'''),
+            self.cb.section("Load Bronze Data"),
+            self.cb.code('''from customer_retention.integrations.adapters.factory import get_delta
+storage = get_delta(force_local=True)
+df = storage.read("./experiments/data/bronze/customers")
+initial_shape = df.shape
+initial_nulls = df.isnull().sum().sum()
+print(f"Initial shape: {df.shape}")
+print(f"Total missing values: {initial_nulls}")
+
+mlflow_adapter.log_metrics({
+    "bronze_rows": initial_shape[0],
+    "bronze_columns": initial_shape[1],
+    "bronze_total_nulls": initial_nulls,
+})'''),
+        ]
+
+        if cleaning_recs:
+            cells.append(self.cb.section("Apply Cleaning from Exploration Findings"))
+            cells.append(self.cb.code(f'''cleaning_recommendations = {cleaning_recs}
+print(f"Found cleaning recommendations for {{len(cleaning_recommendations)}} columns")'''))
+
+        cells.extend([
+            self.cb.section("Handle Missing Values - Numeric Columns"),
+            self.cb.code(f'''numeric_cols = {numeric_cols}
+missing_handler = MissingValueHandler(strategy="median")
+for col in numeric_cols:
+    if col in df.columns and df[col].isnull().any():
+        nulls_before = df[col].isnull().sum()
+        df[col] = missing_handler.fit_transform(df[col])
+        cleaning_stats[f"{{col}}_nulls_imputed"] = nulls_before
+        print(f"Imputed {{col}}: {{nulls_before}} missing values")'''),
+            self.cb.section("Handle Missing Values - Categorical Columns"),
+            self.cb.code(f'''categorical_cols = {categorical_cols}
+missing_handler_cat = MissingValueHandler(strategy="mode")
+for col in categorical_cols:
+    if col in df.columns and df[col].isnull().any():
+        nulls_before = df[col].isnull().sum()
+        df[col] = missing_handler_cat.fit_transform(df[col])
+        cleaning_stats[f"{{col}}_nulls_imputed"] = nulls_before
+        print(f"Imputed {{col}}: {{nulls_before}} missing values")'''),
+            self.cb.section("Handle Outliers"),
+            self.cb.code('''outlier_handler = OutlierHandler(method="iqr", treatment="cap")
+for col in numeric_cols:
+    if col in df.columns:
+        q1, q3 = df[col].quantile([0.25, 0.75])
+        iqr = q3 - q1
+        outliers = ((df[col] < q1 - 1.5*iqr) | (df[col] > q3 + 1.5*iqr)).sum()
+        cleaning_stats[f"{col}_outliers_capped"] = outliers
+        df[col] = outlier_handler.fit_transform(df[col])
+print("Outliers capped using IQR method")'''),
+            self.cb.section("Log Cleaning Statistics to MLflow"),
+            self.cb.code('''final_nulls = df.isnull().sum().sum()
+mlflow_adapter.log_params({
+    "numeric_strategy": "median",
+    "categorical_strategy": "mode",
+    "outlier_method": "iqr",
+    "outlier_treatment": "cap",
+})
+mlflow_adapter.log_metrics({
+    "silver_rows": df.shape[0],
+    "silver_columns": df.shape[1],
+    "silver_total_nulls": final_nulls,
+    "nulls_removed": initial_nulls - final_nulls,
+    **{k: v for k, v in cleaning_stats.items() if isinstance(v, (int, float))}
+})
+print(f"Logged {len(cleaning_stats)} cleaning statistics to MLflow")'''),
+            self.cb.section("Save to Silver Layer"),
+            self.cb.code('''storage.write(df, "./experiments/data/silver/customers_cleaned")
+mlflow_adapter.end_run()
+print(f"Silver layer saved: {df.shape}")'''),
+        ])
+        return cells
+
+    def generate_databricks_cells(self) -> List[nbformat.NotebookNode]:
+        catalog = self.config.feature_store.catalog
+        schema = self.config.feature_store.schema
+        exp_name = self.config.mlflow.experiment_name
+        numeric_cols = self.get_numeric_columns()
+        categorical_cols = self.get_categorical_columns()
+        return self.header_cells() + [
+            self.cb.section("Setup MLflow Tracking"),
+            self.cb.code(f'''import mlflow
+
+mlflow.set_experiment("/Users/{{spark.conf.get('spark.databricks.notebook.username', 'default')}}/{exp_name}")
+mlflow.start_run(run_name="03_data_cleaning")
+cleaning_stats = {{}}'''),
+            self.cb.section("Load Bronze Data"),
+            self.cb.code(f'''df = spark.table("{catalog}.{schema}.bronze_customers")
+initial_count = df.count()
+print(f"Initial count: {{initial_count}}")
+mlflow.log_metric("bronze_rows", initial_count)'''),
+            self.cb.section("Handle Missing Values - Numeric Columns"),
+            self.cb.code(f'''from pyspark.sql.functions import col, when, lit, sum as spark_sum
+from pyspark.ml.feature import Imputer
+
+numeric_cols = {numeric_cols}
+imputer = Imputer(inputCols=numeric_cols, outputCols=numeric_cols, strategy="median")
+df = imputer.fit(df).transform(df)
+mlflow.log_param("numeric_strategy", "median")
+print("Numeric columns imputed with median")'''),
+            self.cb.section("Handle Missing Values - Categorical Columns"),
+            self.cb.code(f'''categorical_cols = {categorical_cols}
+for col_name in categorical_cols:
+    mode_val = df.groupBy(col_name).count().orderBy("count", ascending=False).first()[0]
+    df = df.fillna({{col_name: mode_val}})
+mlflow.log_param("categorical_strategy", "mode")
+print("Categorical columns imputed with mode")'''),
+            self.cb.section("Handle Outliers with IQR"),
+            self.cb.code('''for col_name in numeric_cols:
+    quantiles = df.approxQuantile(col_name, [0.25, 0.75], 0.05)
+    if len(quantiles) == 2:
+        q1, q3 = quantiles
+        iqr = q3 - q1
+        lower = q1 - 1.5 * iqr
+        upper = q3 + 1.5 * iqr
+        df = df.withColumn(col_name, when(col(col_name) < lower, lower)
+                          .when(col(col_name) > upper, upper)
+                          .otherwise(col(col_name)))
+mlflow.log_params({"outlier_method": "iqr", "outlier_treatment": "cap"})
+print("Outliers capped using IQR")'''),
+            self.cb.section("Log Cleaning Statistics"),
+            self.cb.code('''final_count = df.count()
+mlflow.log_metrics({
+    "silver_rows": final_count,
+    "rows_preserved_pct": final_count / initial_count * 100,
+})
+print(f"Final count: {final_count}")'''),
+            self.cb.section("Save to Silver Table"),
+            self.cb.code(f'''df.write.format("delta").mode("overwrite").saveAsTable("{catalog}.{schema}.silver_customers")
+mlflow.end_run()
+print("Silver table created")'''),
+        ]

customer_retention/generators/notebook_generator/stages/s04_transformation.py

@@ -0,0 +1,165 @@
+from typing import List
+
+import nbformat
+
+from ..base import NotebookStage
+from .base_stage import StageGenerator
+
+
+class TransformationStage(StageGenerator):
+    @property
+    def stage(self) -> NotebookStage:
+        return NotebookStage.TRANSFORMATION
+
+    @property
+    def title(self) -> str:
+        return "04 - Data Transformation"
+
+    @property
+    def description(self) -> str:
+        return "Apply scaling, encoding, and type transformations with MLflow tracking."
+
+    def _get_transform_recommendations(self) -> dict:
+        recommendations = {}
+        if not self.findings or not hasattr(self.findings, "columns"):
+            return recommendations
+        for col_name, col_finding in self.findings.columns.items():
+            if hasattr(col_finding, "transformation_recommendations") and col_finding.transformation_recommendations:
+                recommendations[col_name] = col_finding.transformation_recommendations
+        return recommendations
+
+    def generate_local_cells(self) -> List[nbformat.NotebookNode]:
+        numeric_cols = self.get_numeric_columns()
+        categorical_cols = self.get_categorical_columns()
+        tracking_uri = self.config.mlflow.tracking_uri
+        exp_name = self.config.mlflow.experiment_name
+        transform_recs = self._get_transform_recommendations()
+
+        cells = self.header_cells() + [
+            self.cb.section("Imports"),
+            self.cb.from_imports_cell({
+                "customer_retention.stages.transformation": ["NumericTransformer", "CategoricalEncoder"],
+                "customer_retention.stages.preprocessing": ["TransformerManager"],
+                "customer_retention.integrations.adapters": ["get_mlflow"],
+                "pandas": ["pd"],
+            }),
+            self.cb.section("Setup MLflow Tracking"),
+            self.cb.code(f'''mlflow_adapter = get_mlflow(tracking_uri="{tracking_uri}", force_local=True)
+mlflow_adapter.start_run("{exp_name}", run_name="04_transformation")
+transform_stats = {{}}'''),
+            self.cb.section("Load Silver Data"),
+            self.cb.code('''from customer_retention.integrations.adapters.factory import get_delta
+storage = get_delta(force_local=True)
+df = storage.read("./experiments/data/silver/customers_cleaned")
+print(f"Loaded shape: {df.shape}")
+mlflow_adapter.log_metric("input_rows", df.shape[0])
+mlflow_adapter.log_metric("input_columns", df.shape[1])'''),
+        ]
+
+        if transform_recs:
+            cells.append(self.cb.section("Transformation Recommendations from Exploration"))
+            cells.append(self.cb.code(f'''transform_recommendations = {transform_recs}
+print(f"Found transformation recommendations for {{len(transform_recommendations)}} columns")'''))
+
+        cells.extend([
+            self.cb.section("Initialize Transformer Manager"),
+            self.cb.code(f'''numeric_cols = {numeric_cols}
+categorical_cols = {categorical_cols}
+
+# TransformerManager ensures consistent transformations between training and scoring
+transformer_manager = TransformerManager(scaler_type="standard")'''),
+            self.cb.section("Fit and Transform Features"),
+            self.cb.code('''# Fit transformers and transform data in one step
+# Exclude identifier and target columns from transformation
+exclude_cols = ["customer_id", "target"]  # Adjust based on your data
+df = transformer_manager.fit_transform(
+    df,
+    numeric_columns=numeric_cols,
+    categorical_columns=categorical_cols,
+    exclude_columns=exclude_cols
+)
+
+# Log transformation statistics
+manifest = transformer_manager.manifest
+transform_stats["numeric_cols_scaled"] = len(manifest.numeric_columns)
+transform_stats["categorical_cols_encoded"] = len(manifest.categorical_columns)
+
+mlflow_adapter.log_params({
+    "scaler_type": manifest.scaler_type,
+    "encoder_type": manifest.encoder_type,
+    "scaled_columns": str(manifest.numeric_columns)[:250],
+    "encoded_columns": str(manifest.categorical_columns)[:250],
+})
+print(f"Scaled {len(manifest.numeric_columns)} numeric columns")
+print(f"Encoded {len(manifest.categorical_columns)} categorical columns")'''),
+            self.cb.section("Save Transformers as Artifacts"),
+            self.cb.code('''# Save transformers locally and to MLflow
+transformer_manager.save("./experiments/data/transformers/transformers.joblib")
+
+# Log to MLflow for scoring pipeline to retrieve
+import mlflow
+transformer_manager.log_to_mlflow(run_id=mlflow.active_run().info.run_id)
+print("Transformers saved locally and logged to MLflow")
+print("Scoring pipeline will use these same transformers for consistency")'''),
+            self.cb.section("Log Transformation Statistics"),
+            self.cb.code('''mlflow_adapter.log_metrics({
+    "output_rows": df.shape[0],
+    "output_columns": df.shape[1],
+    **{k: v for k, v in transform_stats.items() if isinstance(v, (int, float))}
+})
+print(f"Logged {len(transform_stats)} transformation statistics")'''),
+            self.cb.section("Save Transformed Data"),
+            self.cb.code('''storage.write(df, "./experiments/data/silver/customers_transformed")
+mlflow_adapter.end_run()
+print(f"Transformed data saved: {df.shape}")'''),
+        ])
+        return cells
+
+    def generate_databricks_cells(self) -> List[nbformat.NotebookNode]:
+        catalog = self.config.feature_store.catalog
+        schema = self.config.feature_store.schema
+        exp_name = self.config.mlflow.experiment_name
+        numeric_cols = self.get_numeric_columns()
+        categorical_cols = self.get_categorical_columns()
+        return self.header_cells() + [
+            self.cb.section("Setup MLflow Tracking"),
+            self.cb.code(f'''import mlflow
+
+mlflow.set_experiment("/Users/{{spark.conf.get('spark.databricks.notebook.username', 'default')}}/{exp_name}")
+mlflow.start_run(run_name="04_transformation")'''),
+            self.cb.section("Load Silver Data"),
+            self.cb.code(f'''df = spark.table("{catalog}.{schema}.silver_customers")
+input_count = df.count()
+mlflow.log_metric("input_rows", input_count)'''),
+            self.cb.section("Scale Numeric Features"),
+            self.cb.code(f'''from pyspark.ml.feature import StandardScaler, VectorAssembler
+
+numeric_cols = {numeric_cols}
+if numeric_cols:
+    assembler = VectorAssembler(inputCols=numeric_cols, outputCol="numeric_features")
+    df = assembler.transform(df)
+    scaler = StandardScaler(inputCol="numeric_features", outputCol="scaled_features", withStd=True, withMean=True)
+    scaler_model = scaler.fit(df)
+    df = scaler_model.transform(df)
+    mlflow.log_params({{"scaler_type": "standard", "scaled_columns_count": len(numeric_cols)}})
+    print("Numeric features scaled")'''),
+            self.cb.section("Encode Categorical Features"),
+            self.cb.code(f'''from pyspark.ml.feature import StringIndexer
+
+categorical_cols = {categorical_cols}
+for col_name in categorical_cols:
+    indexer = StringIndexer(inputCol=col_name, outputCol=f"{{col_name}}_idx")
+    df = indexer.fit(df).transform(df)
+mlflow.log_params({{"encoder_type": "string_indexer", "encoded_columns_count": len(categorical_cols)}})
+print(f"Encoded {{len(categorical_cols)}} categorical columns")'''),
+            self.cb.section("Log Statistics"),
+            self.cb.code('''output_count = df.count()
+mlflow.log_metrics({
+    "output_rows": output_count,
+    "columns_after_transform": len(df.columns),
+})'''),
+            self.cb.section("Save Transformed Data"),
+            self.cb.code(f'''df.write.format("delta").mode("overwrite").saveAsTable("{catalog}.{schema}.silver_transformed")
+mlflow.end_run()
+print("Transformed data saved")'''),
+        ]