churnkit 0.75.0a1__py3-none-any.whl

churnkit-0.75.0a1.data/data/share/churnkit/exploration_notebooks/02_column_deep_dive.ipynb

@@ -0,0 +1,1430 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "9359aa8a",
+   "metadata": {
+    "papermill": {
+     "duration": 0.003327,
+     "end_time": "2026-02-02T13:02:59.136885",
+     "exception": false,
+     "start_time": "2026-02-02T13:02:59.133558",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "# Chapter 2: Column Deep Dive\n",
+    "\n",
+    "**Purpose:** Analyze each column in detail with distribution analysis, value validation, and transformation recommendations.\n",
+    "\n",
+    "**What you'll learn:**\n",
+    "- How to validate value ranges for different column types\n",
+    "- How to interpret distribution shapes (skewness, kurtosis)\n",
+    "- When and why to apply transformations (log, sqrt, capping)\n",
+    "- How to detect zero-inflation and handle it\n",
+    "\n",
+    "**Outputs:**\n",
+    "- Value range validation results\n",
+    "- Per-column distribution visualizations with statistics\n",
+    "- Skewness/kurtosis analysis with transformation recommendations\n",
+    "- Zero-inflation detection\n",
+    "- Type confirmation/override capability\n",
+    "- Updated exploration findings"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2caa955f",
+   "metadata": {
+    "papermill": {
+     "duration": 0.002958,
+     "end_time": "2026-02-02T13:02:59.142413",
+     "exception": false,
+     "start_time": "2026-02-02T13:02:59.139455",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "## 2.1 Load Previous Findings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9dde1f33",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-02-02T13:02:59.148522Z",
+     "iopub.status.busy": "2026-02-02T13:02:59.148372Z",
+     "iopub.status.idle": "2026-02-02T13:03:00.961868Z",
+     "shell.execute_reply": "2026-02-02T13:03:00.961427Z"
+    },
+    "papermill": {
+     "duration": 1.817589,
+     "end_time": "2026-02-02T13:03:00.962754",
+     "exception": false,
+     "start_time": "2026-02-02T13:02:59.145165",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from customer_retention.analysis.notebook_progress import track_and_export_previous\n",
+    "track_and_export_previous(\"02_column_deep_dive.ipynb\")\n",
+    "\n",
+    "from customer_retention.analysis.auto_explorer import ExplorationFindings, RecommendationRegistry\n",
+    "from customer_retention.analysis.visualization import ChartBuilder, display_figure, display_table, console\n",
+    "from customer_retention.core.config.column_config import ColumnType\n",
+    "from customer_retention.stages.profiling import (\n",
+    "    DistributionAnalyzer, TransformationType,\n",
+    "    TemporalAnalyzer, TemporalGranularity,\n",
+    "    CategoricalDistributionAnalyzer, EncodingType\n",
+    ")\n",
+    "from customer_retention.stages.validation import DataValidator, RuleGenerator\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from scipy import stats\n",
+    "import plotly.graph_objects as go\n",
+    "import plotly.express as px\n",
+    "from plotly.subplots import make_subplots\n",
+    "from customer_retention.core.config.experiments import FINDINGS_DIR, EXPERIMENTS_DIR, OUTPUT_DIR, setup_experiments_structure\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18c54d15",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-02-02T13:03:00.968282Z",
+     "iopub.status.busy": "2026-02-02T13:03:00.968075Z",
+     "iopub.status.idle": "2026-02-02T13:03:01.104881Z",
+     "shell.execute_reply": "2026-02-02T13:03:01.104422Z"
+    },
+    "papermill": {
+     "duration": 0.140223,
+     "end_time": "2026-02-02T13:03:01.105512",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:00.965289",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# === CONFIGURATION ===\n",
+    "# Option 1: Set the exact path from notebook 01 output\n",
+    "# FINDINGS_PATH = \"../experiments/findings/customer_retention_retail_abc123_findings.yaml\"\n",
+    "\n",
+    "# Option 2: Auto-discover findings file (prefers aggregated over event-level)\n",
+    "from pathlib import Path\n",
+    "import os\n",
+    "\n",
+    "# FINDINGS_DIR imported from customer_retention.core.config.experiments\n",
+    "\n",
+    "# Find all findings files\n",
+    "findings_files = [f for f in FINDINGS_DIR.glob(\"*_findings.yaml\") if \"multi_dataset\" not in f.name]\n",
+    "if not findings_files:\n",
+    "    raise FileNotFoundError(f\"No findings files found in {FINDINGS_DIR}. Run notebook 01 first.\")\n",
+    "\n",
+    "# Prefer aggregated findings (from 01d) over event-level findings\n",
+    "# This ensures notebooks 02-10 work with entity-level data\n",
+    "# Pattern: *_aggregated_* in filename indicates aggregated data\n",
+    "aggregated_files = [f for f in findings_files if \"_aggregated\" in f.name]\n",
+    "non_aggregated_files = [f for f in findings_files if \"_aggregated\" not in f.name]\n",
+    "\n",
+    "if aggregated_files:\n",
+    "    # Use most recent aggregated file\n",
+    "    aggregated_files.sort(key=lambda f: f.stat().st_mtime, reverse=True)\n",
+    "    FINDINGS_PATH = str(aggregated_files[0])\n",
+    "    print(f\"Found {len(aggregated_files)} aggregated findings file(s)\")\n",
+    "    print(f\"Using: {FINDINGS_PATH}\")\n",
+    "    if non_aggregated_files:\n",
+    "        print(f\"   (Skipping {len(non_aggregated_files)} event-level findings)\")\n",
+    "else:\n",
+    "    # Fall back to most recent non-aggregated file\n",
+    "    non_aggregated_files.sort(key=lambda f: f.stat().st_mtime, reverse=True)\n",
+    "    FINDINGS_PATH = str(non_aggregated_files[0])\n",
+    "    print(f\"Found {len(non_aggregated_files)} findings file(s)\")\n",
+    "    print(f\"Using: {FINDINGS_PATH}\")\n",
+    "\n",
+    "findings = ExplorationFindings.load(FINDINGS_PATH)\n",
+    "print(f\"\\nLoaded findings for {findings.column_count} columns from {findings.source_path}\")\n",
+    "\n",
+    "# Warn if this is event-level data (should run 01d first)\n",
+    "if findings.is_time_series and \"_aggregated\" not in FINDINGS_PATH:\n",
+    "    ts_meta = findings.time_series_metadata\n",
+    "    print(f\"\\n⚠️  WARNING: This appears to be EVENT-LEVEL data\")\n",
+    "    print(f\"   Entity: {ts_meta.entity_column}, Time: {ts_meta.time_column}\")\n",
+    "    print(f\"   Recommendation: Run 01d_event_aggregation.ipynb first to create entity-level data\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "03167807",
+   "metadata": {
+    "papermill": {
+     "duration": 0.002163,
+     "end_time": "2026-02-02T13:03:01.110263",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:01.108100",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "## 2.2 Load Source Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b55e4eac",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-02-02T13:03:01.115492Z",
+     "iopub.status.busy": "2026-02-02T13:03:01.115384Z",
+     "iopub.status.idle": "2026-02-02T13:03:01.212413Z",
+     "shell.execute_reply": "2026-02-02T13:03:01.211972Z"
+    },
+    "papermill": {
+     "duration": 0.100207,
+     "end_time": "2026-02-02T13:03:01.212881",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:01.112674",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Load data - handle aggregated parquet files directly\n",
+    "from customer_retention.stages.temporal import load_data_with_snapshot_preference, TEMPORAL_METADATA_COLS\n",
+    "\n",
+    "# For aggregated data, load directly from the parquet source\n",
+    "if \"_aggregated\" in FINDINGS_PATH and findings.source_path.endswith('.parquet'):\n",
+    "    source_path = Path(findings.source_path)\n",
+    "    # Handle relative path from notebook directory\n",
+    "    if not source_path.is_absolute():\n",
+    "        source_path = Path(\"..\") / source_path.relative_to(\"..\") if str(source_path).startswith(\"..\") else FINDINGS_DIR / source_path.name\n",
+    "    df = pd.read_parquet(source_path)\n",
+    "    data_source = f\"aggregated:{source_path.name}\"\n",
+    "else:\n",
+    "    # Standard loading for event-level or entity-level data\n",
+    "    df, data_source = load_data_with_snapshot_preference(findings, output_dir=str(FINDINGS_DIR))\n",
+    "\n",
+    "print(f\"Loaded data from: {data_source}\")\n",
+    "print(f\"Shape: {df.shape}\")\n",
+    "\n",
+    "charts = ChartBuilder()\n",
+    "\n",
+    "# Initialize recommendation registry for this exploration\n",
+    "registry = RecommendationRegistry()\n",
+    "registry.init_bronze(findings.source_path)\n",
+    "\n",
+    "# Find target column for Gold layer initialization\n",
+    "target_col = next((name for name, col in findings.columns.items() if col.inferred_type == ColumnType.TARGET), None)\n",
+    "if target_col:\n",
+    "    registry.init_gold(target_col)\n",
+    "\n",
+    "# Find entity column for Silver layer initialization\n",
+    "entity_col = next((name for name, col in findings.columns.items() if col.inferred_type == ColumnType.IDENTIFIER), None)\n",
+    "if entity_col:\n",
+    "    registry.init_silver(entity_col)\n",
+    "\n",
+    "print(f\"Initialized recommendation registry (Bronze: {findings.source_path})\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fd863df6",
+   "metadata": {
+    "papermill": {
+     "duration": 0.002033,
+     "end_time": "2026-02-02T13:03:01.217382",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:01.215349",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "## 2.3 Value Range Validation\n",
+    "\n",
+    "**📖 Interpretation Guide:**\n",
+    "- **Percentage fields** (rates): Should be 0-100 or 0-1 depending on format\n",
+    "- **Binary fields**: Should only contain 0 and 1\n",
+    "- **Count fields**: Should be non-negative integers\n",
+    "- **Amount fields**: Should be non-negative (unless refunds are possible)\n",
+    "\n",
+    "**What to Watch For:**\n",
+    "- Rates > 100% suggest measurement or data entry errors\n",
+    "- Negative values in fields that should be positive\n",
+    "- Binary fields with values other than 0/1\n",
+    "\n",
+    "**Actions:**\n",
+    "- Cap rates at 100 if they exceed (or investigate cause)\n",
+    "- Flag records with impossible negative values\n",
+    "- Convert binary fields to proper 0/1 encoding"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "934c4565",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-02-02T13:03:01.221952Z",
+     "iopub.status.busy": "2026-02-02T13:03:01.221848Z",
+     "iopub.status.idle": "2026-02-02T13:03:01.230109Z",
+     "shell.execute_reply": "2026-02-02T13:03:01.229657Z"
+    },
+    "papermill": {
+     "duration": 0.011284,
+     "end_time": "2026-02-02T13:03:01.230585",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:01.219301",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "validator = DataValidator()\n",
+    "range_rules = RuleGenerator.from_findings(findings)\n",
+    "\n",
+    "console.start_section()\n",
+    "console.header(\"Value Range Validation\")\n",
+    "\n",
+    "if range_rules:\n",
+    "    range_results = validator.validate_value_ranges(df, range_rules)\n",
+    "    \n",
+    "    issues_found = []\n",
+    "    for r in range_results:\n",
+    "        detail = f\"{r.invalid_values} invalid\" if r.invalid_values > 0 else None\n",
+    "        console.check(f\"{r.column_name} ({r.rule_type})\", r.invalid_values == 0, detail)\n",
+    "        if r.invalid_values > 0:\n",
+    "            issues_found.append(r)\n",
+    "    \n",
+    "    all_invalid = sum(r.invalid_values for r in range_results)\n",
+    "    if all_invalid == 0:\n",
+    "        console.success(\"All value ranges valid\")\n",
+    "    else:\n",
+    "        console.error(f\"Found {all_invalid:,} values outside expected ranges\")\n",
+    "        \n",
+    "        console.info(\"Examples of invalid values:\")\n",
+    "        for r in issues_found[:3]:\n",
+    "            col = r.column_name\n",
+    "            if col in df.columns:\n",
+    "                if r.rule_type == 'binary':\n",
+    "                    invalid_mask = ~df[col].isin([0, 1, np.nan])\n",
+    "                    condition = \"value not in [0, 1]\"\n",
+    "                elif r.rule_type == 'non_negative':\n",
+    "                    invalid_mask = df[col] < 0\n",
+    "                    condition = \"value < 0\"\n",
+    "                elif r.rule_type == 'percentage':\n",
+    "                    invalid_mask = (df[col] < 0) | (df[col] > 100)\n",
+    "                    condition = \"value < 0 or value > 100\"\n",
+    "                elif r.rule_type == 'rate':\n",
+    "                    invalid_mask = (df[col] < 0) | (df[col] > 1)\n",
+    "                    condition = \"value < 0 or value > 1\"\n",
+    "                else:\n",
+    "                    continue\n",
+    "                \n",
+    "                invalid_values = df.loc[invalid_mask, col].dropna()\n",
+    "                if len(invalid_values) > 0:\n",
+    "                    examples = invalid_values.head(5).tolist()\n",
+    "                    console.metric(f\"  {col}\", f\"{examples}\")\n",
+    "                    \n",
+    "                    # Add filtering recommendation\n",
+    "                    registry.add_bronze_filtering(\n",
+    "                        column=col, condition=condition, action=\"cap\",\n",
+    "                        rationale=f\"{r.invalid_values} values violate {r.rule_type} constraint\",\n",
+    "                        source_notebook=\"02_column_deep_dive\"\n",
+    "                    )\n",
+    "    \n",
+    "    console.info(\"Rules auto-generated from detected column types\")\n",
+    "else:\n",
+    "    range_results = []\n",
+    "    console.info(\"No validation rules generated - no binary/numeric columns detected\")\n",
+    "\n",
+    "console.end_section()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d39a7123",
+   "metadata": {
+    "papermill": {
+     "duration": 0.002152,
+     "end_time": "2026-02-02T13:03:01.235280",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:01.233128",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "## 2.4 Numeric Columns Analysis\n",
+    "\n",
+    "**📖 How to Interpret These Charts:**\n",
+    "- **Red dashed line** = Mean (sensitive to outliers)\n",
+    "- **Green solid line** = Median (robust to outliers)\n",
+    "- **Large gap between mean and median** = Skewed distribution\n",
+    "- **Long right tail** = Positive skew (common in count/amount data)\n",
+    "\n",
+    "**📖 Understanding Distribution Metrics**\n",
+    "\n",
+    "| Metric | Interpretation | Action |\n",
+    "|--------|---------------|--------|\n",
+    "| **Skewness** | Measures asymmetry | \\|skew\\| > 1: Consider log transform |\n",
+    "| **Kurtosis** | Measures tail heaviness | kurt > 10: Cap outliers before transform |\n",
+    "| **Zero %** | Percentage of zeros | > 40%: Use zero-inflation handling |\n",
+    "\n",
+    "**📖 Transformation Decision Tree:**\n",
+    "1. If zeros > 40% → Create binary indicator + log(non-zeros)\n",
+    "2. If \\|skewness\\| > 1 AND kurtosis > 10 → Cap then log\n",
+    "3. If \\|skewness\\| > 1 → Log transform\n",
+    "4. If kurtosis > 10 → Cap outliers only\n",
+    "5. Otherwise → Standard scaling is sufficient"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13043429",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-02-02T13:03:01.240371Z",
+     "iopub.status.busy": "2026-02-02T13:03:01.240257Z",
+     "iopub.status.idle": "2026-02-02T13:03:02.330839Z",
+     "shell.execute_reply": "2026-02-02T13:03:02.330400Z"
+    },
+    "papermill": {
+     "duration": 1.093984,
+     "end_time": "2026-02-02T13:03:02.331518",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:01.237534",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Use framework's DistributionAnalyzer for comprehensive analysis\n",
+    "analyzer = DistributionAnalyzer()\n",
+    "\n",
+    "numeric_cols = [\n",
+    "    name for name, col in findings.columns.items()\n",
+    "    if col.inferred_type in [ColumnType.NUMERIC_CONTINUOUS, ColumnType.NUMERIC_DISCRETE]\n",
+    "    and name not in TEMPORAL_METADATA_COLS\n",
+    "]\n",
+    "\n",
+    "# Analyze all numeric columns using the framework\n",
+    "analyses = analyzer.analyze_dataframe(df, numeric_cols)\n",
+    "recommendations = {col: analyzer.recommend_transformation(analysis) \n",
+    "                   for col, analysis in analyses.items()}\n",
+    "\n",
+    "for col_name in numeric_cols:\n",
+    "    col_info = findings.columns[col_name]\n",
+    "    analysis = analyses.get(col_name)\n",
+    "    rec = recommendations.get(col_name)\n",
+    "    \n",
+    "    print(f\"\\n{'='*70}\")\n",
+    "    print(f\"Column: {col_name}\")\n",
+    "    print(f\"Type: {col_info.inferred_type.value} (Confidence: {col_info.confidence:.0%})\")\n",
+    "    print(f\"-\" * 70)\n",
+    "    \n",
+    "    if analysis:\n",
+    "        print(f\"📊 Distribution Statistics:\")\n",
+    "        print(f\"   Mean: {analysis.mean:.3f}  |  Median: {analysis.median:.3f}  |  Std: {analysis.std:.3f}\")\n",
+    "        print(f\"   Range: [{analysis.min_value:.3f}, {analysis.max_value:.3f}]\")\n",
+    "        print(f\"   Percentiles: 1%={analysis.percentiles['p1']:.3f}, 25%={analysis.q1:.3f}, 75%={analysis.q3:.3f}, 99%={analysis.percentiles['p99']:.3f}\")\n",
+    "        print(f\"\\n📈 Shape Analysis:\")\n",
+    "        skew_label = '(Right-skewed)' if analysis.skewness > 0.5 else '(Left-skewed)' if analysis.skewness < -0.5 else '(Symmetric)'\n",
+    "        print(f\"   Skewness: {analysis.skewness:.2f} {skew_label}\")\n",
+    "        kurt_label = '(Heavy tails/outliers)' if analysis.kurtosis > 3 else '(Light tails)'\n",
+    "        print(f\"   Kurtosis: {analysis.kurtosis:.2f} {kurt_label}\")\n",
+    "        print(f\"   Zeros: {analysis.zero_count:,} ({analysis.zero_percentage:.1f}%)\")\n",
+    "        print(f\"   Outliers (IQR): {analysis.outlier_count_iqr:,} ({analysis.outlier_percentage:.1f}%)\")\n",
+    "        \n",
+    "        if rec:\n",
+    "            print(f\"\\n🔧 Recommended Transformation: {rec.recommended_transform.value}\")\n",
+    "            print(f\"   Reason: {rec.reason}\")\n",
+    "            print(f\"   Priority: {rec.priority}\")\n",
+    "            if rec.warnings:\n",
+    "                for warn in rec.warnings:\n",
+    "                    print(f\"   ⚠️ {warn}\")\n",
+    "    \n",
+    "    # Create enhanced histogram with Plotly\n",
+    "    data = df[col_name].dropna()\n",
+    "    fig = go.Figure()\n",
+    "    \n",
+    "    fig.add_trace(go.Histogram(x=data, nbinsx=50, name='Distribution',\n",
+    "                                marker_color='steelblue', opacity=0.7))\n",
+    "    \n",
+    "    # Calculate mean and median\n",
+    "    mean_val = data.mean()\n",
+    "    median_val = data.median()\n",
+    "    \n",
+    "    # Position labels on opposite sides (left/right) to avoid overlap\n",
+    "    # The larger value gets right-justified, smaller gets left-justified\n",
+    "    mean_position = \"top right\" if mean_val >= median_val else \"top left\"\n",
+    "    median_position = \"top left\" if mean_val >= median_val else \"top right\"\n",
+    "    \n",
+    "    # Add mean line\n",
+    "    fig.add_vline(\n",
+    "        x=mean_val, \n",
+    "        line_dash=\"dash\", \n",
+    "        line_color=\"red\",\n",
+    "        annotation_text=f\"Mean: {mean_val:.2f}\",\n",
+    "        annotation_position=mean_position,\n",
+    "        annotation_font_color=\"red\",\n",
+    "        annotation_bgcolor=\"rgba(255,255,255,0.8)\"\n",
+    "    )\n",
+    "    \n",
+    "    # Add median line\n",
+    "    fig.add_vline(\n",
+    "        x=median_val, \n",
+    "        line_dash=\"solid\", \n",
+    "        line_color=\"green\",\n",
+    "        annotation_text=f\"Median: {median_val:.2f}\",\n",
+    "        annotation_position=median_position,\n",
+    "        annotation_font_color=\"green\",\n",
+    "        annotation_bgcolor=\"rgba(255,255,255,0.8)\"\n",
+    "    )\n",
+    "    \n",
+    "    # Add 99th percentile marker if there are outliers\n",
+    "    if analysis and analysis.outlier_percentage > 5:\n",
+    "        fig.add_vline(x=analysis.percentiles['p99'], line_dash=\"dot\", line_color=\"orange\",\n",
+    "                      annotation_text=f\"99th: {analysis.percentiles['p99']:.2f}\",\n",
+    "                      annotation_position=\"top right\",\n",
+    "                      annotation_font_color=\"orange\",\n",
+    "                      annotation_bgcolor=\"rgba(255,255,255,0.8)\")\n",
+    "    \n",
+    "    transform_label = rec.recommended_transform.value if rec else \"none\"\n",
+    "    fig.update_layout(\n",
+    "        title=f\"Distribution: {col_name}<br><sub>Skew: {analysis.skewness:.2f} | Kurt: {analysis.kurtosis:.2f} | Strategy: {transform_label}</sub>\",\n",
+    "        xaxis_title=col_name,\n",
+    "        yaxis_title=\"Count\",\n",
+    "        template='plotly_white',\n",
+    "        height=400\n",
+    "    )\n",
+    "    display_figure(fig)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "674ea32d",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-02-02T13:03:02.532435Z",
+     "iopub.status.busy": "2026-02-02T13:03:02.532307Z",
+     "iopub.status.idle": "2026-02-02T13:03:02.620535Z",
+     "shell.execute_reply": "2026-02-02T13:03:02.620110Z"
+    },
+    "papermill": {
+     "duration": 0.189181,
+     "end_time": "2026-02-02T13:03:02.621233",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:02.432052",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Numerical Feature Statistics Table\n",
+    "if numeric_cols:\n",
+    "    stats_data = []\n",
+    "    for col_name in numeric_cols:\n",
+    "        series = df[col_name].dropna()\n",
+    "        if len(series) > 0:\n",
+    "            stats_data.append({\n",
+    "                \"feature\": col_name,\n",
+    "                \"count\": len(series),\n",
+    "                \"mean\": series.mean(),\n",
+    "                \"std\": series.std(),\n",
+    "                \"min\": series.min(),\n",
+    "                \"25%\": series.quantile(0.25),\n",
+    "                \"50%\": series.quantile(0.50),\n",
+    "                \"75%\": series.quantile(0.75),\n",
+    "                \"95%\": series.quantile(0.95),\n",
+    "                \"99%\": series.quantile(0.99),\n",
+    "                \"max\": series.max(),\n",
+    "                \"skewness\": stats.skew(series),\n",
+    "                \"kurtosis\": stats.kurtosis(series)\n",
+    "            })\n",
+    "    \n",
+    "    stats_df = pd.DataFrame(stats_data)\n",
+    "    \n",
+    "    # Format for display\n",
+    "    display_stats = stats_df.copy()\n",
+    "    for col in [\"mean\", \"std\", \"min\", \"25%\", \"50%\", \"75%\", \"95%\", \"99%\", \"max\"]:\n",
+    "        display_stats[col] = display_stats[col].apply(lambda x: f\"{x:.3f}\")\n",
+    "    display_stats[\"skewness\"] = display_stats[\"skewness\"].apply(lambda x: f\"{x:.3f}\")\n",
+    "    display_stats[\"kurtosis\"] = display_stats[\"kurtosis\"].apply(lambda x: f\"{x:.3f}\")\n",
+    "    \n",
+    "    print(\"=\" * 80)\n",
+    "    print(\"NUMERICAL FEATURE STATISTICS\")\n",
+    "    print(\"=\" * 80)\n",
+    "    display(display_stats)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b53f6e24",
+   "metadata": {
+    "papermill": {
+     "duration": 0.099217,
+     "end_time": "2026-02-02T13:03:02.857720",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:02.758503",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "## 2.5 Distribution Summary & Transformation Plan\n",
+    "\n",
+    "This table summarizes all numeric columns with their recommended transformations."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "af3319f6",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-02-02T13:03:03.057083Z",
+     "iopub.status.busy": "2026-02-02T13:03:03.056972Z",
+     "iopub.status.idle": "2026-02-02T13:03:03.063404Z",
+     "shell.execute_reply": "2026-02-02T13:03:03.062883Z"
+    },
+    "papermill": {
+     "duration": 0.108346,
+     "end_time": "2026-02-02T13:03:03.064068",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:02.955722",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Build transformation summary table\n",
+    "summary_data = []\n",
+    "for col_name in numeric_cols:\n",
+    "    analysis = analyses.get(col_name)\n",
+    "    rec = recommendations.get(col_name)\n",
+    "    \n",
+    "    if analysis and rec:\n",
+    "        summary_data.append({\n",
+    "            \"Column\": col_name,\n",
+    "            \"Skewness\": f\"{analysis.skewness:.2f}\",\n",
+    "            \"Kurtosis\": f\"{analysis.kurtosis:.2f}\",\n",
+    "            \"Zeros %\": f\"{analysis.zero_percentage:.1f}%\",\n",
+    "            \"Outliers %\": f\"{analysis.outlier_percentage:.1f}%\",\n",
+    "            \"Transform\": rec.recommended_transform.value,\n",
+    "            \"Priority\": rec.priority\n",
+    "        })\n",
+    "        \n",
+    "        # Add Gold transformation recommendation if not \"none\"\n",
+    "        if rec.recommended_transform != TransformationType.NONE and registry.gold:\n",
+    "            registry.add_gold_transformation(\n",
+    "                column=col_name,\n",
+    "                transform=rec.recommended_transform.value,\n",
+    "                parameters=rec.parameters,\n",
+    "                rationale=rec.reason,\n",
+    "                source_notebook=\"02_column_deep_dive\"\n",
+    "            )\n",
+    "\n",
+    "if summary_data:\n",
+    "    summary_df = pd.DataFrame(summary_data)\n",
+    "    display_table(summary_df)\n",
+    "    \n",
+    "    # Show how many transformation recommendations were added\n",
+    "    transform_count = sum(1 for r in recommendations.values() if r and r.recommended_transform != TransformationType.NONE)\n",
+    "    if transform_count > 0 and registry.gold:\n",
+    "        print(f\"\\n✅ Added {transform_count} transformation recommendations to Gold layer\")\n",
+    "else:\n",
+    "    console.info(\"No numeric columns to summarize\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9e1f558b",
+   "metadata": {
+    "papermill": {
+     "duration": 0.098534,
+     "end_time": "2026-02-02T13:03:03.299429",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:03.200895",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "## 2.6 Categorical Columns Analysis\n",
+    "\n",
+    "**📖 Distribution Metrics (Analogues to Numeric Skewness/Kurtosis):**\n",
+    "\n",
+    "| Metric | Interpretation | Action |\n",
+    "|--------|---------------|--------|\n",
+    "| **Imbalance Ratio** | Largest / Smallest category count | > 10: Consider grouping rare categories |\n",
+    "| **Entropy** | Diversity measure (0 = one category, higher = more uniform) | Low entropy: May need stratified sampling |\n",
+    "| **Top-3 Concentration** | % of data in top 3 categories | > 90%: Rare categories may cause issues |\n",
+    "| **Rare Category %** | Categories with < 1% of data | High %: Group into \"Other\" category |\n",
+    "\n",
+    "**📖 Encoding Recommendations:**\n",
+    "- **Low cardinality (≤5)** → One-hot encoding\n",
+    "- **Medium cardinality (6-20)** → One-hot or Target encoding\n",
+    "- **High cardinality (>20)** → Target encoding or Frequency encoding\n",
+    "- **Cyclical (days, months)** → Sin/Cos encoding\n",
+    "\n",
+    "**⚠️ Common Issues:**\n",
+    "- Rare categories can cause overfitting with one-hot encoding\n",
+    "- High cardinality + one-hot = feature explosion\n",
+    "- Imbalanced categories may need special handling in train/test splits"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "740c5c5e",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-02-02T13:03:03.500506Z",
+     "iopub.status.busy": "2026-02-02T13:03:03.500396Z",
+     "iopub.status.idle": "2026-02-02T13:03:03.529538Z",
+     "shell.execute_reply": "2026-02-02T13:03:03.528970Z"
+    },
+    "papermill": {
+     "duration": 0.130148,
+     "end_time": "2026-02-02T13:03:03.530136",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:03.399988",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Use framework's CategoricalDistributionAnalyzer\n",
+    "cat_analyzer = CategoricalDistributionAnalyzer()\n",
+    "\n",
+    "categorical_cols = [\n",
+    "    name for name, col in findings.columns.items()\n",
+    "    if col.inferred_type in [ColumnType.CATEGORICAL_NOMINAL, ColumnType.CATEGORICAL_ORDINAL, ColumnType.CATEGORICAL_CYCLICAL]\n",
+    "    and col.inferred_type != ColumnType.TEXT  # TEXT columns processed separately in 02a\n",
+    "    and name not in TEMPORAL_METADATA_COLS\n",
+    "]\n",
+    "\n",
+    "# Analyze all categorical columns\n",
+    "cat_analyses = cat_analyzer.analyze_dataframe(df, categorical_cols)\n",
+    "\n",
+    "# Get encoding recommendations\n",
+    "cyclical_cols = [name for name, col in findings.columns.items() \n",
+    "                 if col.inferred_type == ColumnType.CATEGORICAL_CYCLICAL]\n",
+    "cat_recommendations = cat_analyzer.get_all_recommendations(df, categorical_cols, cyclical_columns=cyclical_cols)\n",
+    "\n",
+    "for col_name in categorical_cols:\n",
+    "    col_info = findings.columns[col_name]\n",
+    "    analysis = cat_analyses.get(col_name)\n",
+    "    rec = next((r for r in cat_recommendations if r.column_name == col_name), None)\n",
+    "    \n",
+    "    print(f\"\\n{'='*70}\")\n",
+    "    print(f\"Column: {col_name}\")\n",
+    "    print(f\"Type: {col_info.inferred_type.value} (Confidence: {col_info.confidence:.0%})\")\n",
+    "    print(f\"-\" * 70)\n",
+    "    \n",
+    "    if analysis:\n",
+    "        print(f\"\\n📊 Distribution Metrics:\")\n",
+    "        print(f\"   Categories: {analysis.category_count}\")\n",
+    "        print(f\"   Imbalance Ratio: {analysis.imbalance_ratio:.1f}x (largest/smallest)\")\n",
+    "        print(f\"   Entropy: {analysis.entropy:.2f} ({analysis.normalized_entropy*100:.0f}% of max)\")\n",
+    "        print(f\"   Top-1 Concentration: {analysis.top1_concentration:.1f}%\")\n",
+    "        print(f\"   Top-3 Concentration: {analysis.top3_concentration:.1f}%\")\n",
+    "        print(f\"   Rare Categories (<1%): {analysis.rare_category_count}\")\n",
+    "        \n",
+    "        # Interpretation\n",
+    "        print(f\"\\n📈 Interpretation:\")\n",
+    "        if analysis.has_low_diversity:\n",
+    "            print(f\"   ⚠️ LOW DIVERSITY: Distribution dominated by few categories\")\n",
+    "        elif analysis.normalized_entropy > 0.9:\n",
+    "            print(f\"   ✓ HIGH DIVERSITY: Categories are relatively balanced\")\n",
+    "        else:\n",
+    "            print(f\"   ✓ MODERATE DIVERSITY: Some category dominance but acceptable\")\n",
+    "        \n",
+    "        if analysis.imbalance_ratio > 100:\n",
+    "            print(f\"   🔴 SEVERE IMBALANCE: Rarest category has very few samples\")\n",
+    "        elif analysis.is_imbalanced:\n",
+    "            print(f\"   🟡 MODERATE IMBALANCE: Consider grouping rare categories\")\n",
+    "        \n",
+    "        # Recommendations\n",
+    "        if rec:\n",
+    "            print(f\"\\n🔧 Recommendations:\")\n",
+    "            print(f\"   Encoding: {rec.encoding_type.value}\")\n",
+    "            print(f\"   Reason: {rec.reason}\")\n",
+    "            print(f\"   Priority: {rec.priority}\")\n",
+    "            \n",
+    "            if rec.preprocessing_steps:\n",
+    "                print(f\"   Preprocessing:\")\n",
+    "                for step in rec.preprocessing_steps:\n",
+    "                    print(f\"      • {step}\")\n",
+    "            \n",
+    "            if rec.warnings:\n",
+    "                for warn in rec.warnings:\n",
+    "                    print(f\"   ⚠️ {warn}\")\n",
+    "    \n",
+    "    # Visualization\n",
+    "    value_counts = df[col_name].value_counts()\n",
+    "    subtitle = f\"Entropy: {analysis.normalized_entropy*100:.0f}% | Imbalance: {analysis.imbalance_ratio:.1f}x | Rare: {analysis.rare_category_count}\" if analysis else \"\"\n",
+    "    fig = charts.bar_chart(\n",
+    "        value_counts.head(10).index.tolist(), \n",
+    "        value_counts.head(10).values.tolist(),\n",
+    "        title=f\"Top Categories: {col_name}<br><sub>{subtitle}</sub>\"\n",
+    "    )\n",
+    "    display_figure(fig)\n",
+    "\n",
+    "# Summary table and add recommendations to registry\n",
+    "if cat_analyses:\n",
+    "    print(\"\\n\" + \"=\" * 70)\n",
+    "    print(\"CATEGORICAL COLUMNS SUMMARY\")\n",
+    "    print(\"=\" * 70)\n",
+    "    summary_data = []\n",
+    "    for col_name, analysis in cat_analyses.items():\n",
+    "        rec = next((r for r in cat_recommendations if r.column_name == col_name), None)\n",
+    "        summary_data.append({\n",
+    "            \"Column\": col_name,\n",
+    "            \"Categories\": analysis.category_count,\n",
+    "            \"Imbalance\": f\"{analysis.imbalance_ratio:.1f}x\",\n",
+    "            \"Entropy\": f\"{analysis.normalized_entropy*100:.0f}%\",\n",
+    "            \"Top-3 Conc.\": f\"{analysis.top3_concentration:.1f}%\",\n",
+    "            \"Rare (<1%)\": analysis.rare_category_count,\n",
+    "            \"Encoding\": rec.encoding_type.value if rec else \"N/A\"\n",
+    "        })\n",
+    "        \n",
+    "        # Add encoding recommendation to Gold layer\n",
+    "        if rec and registry.gold:\n",
+    "            registry.add_gold_encoding(\n",
+    "                column=col_name,\n",
+    "                method=rec.encoding_type.value,\n",
+    "                rationale=rec.reason,\n",
+    "                source_notebook=\"02_column_deep_dive\"\n",
+    "            )\n",
+    "    \n",
+    "    display_table(pd.DataFrame(summary_data))\n",
+    "    \n",
+    "    if registry.gold:\n",
+    "        print(f\"\\n✅ Added {len(cat_recommendations)} encoding recommendations to Gold layer\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "399d9419",
+   "metadata": {
+    "papermill": {
+     "duration": 0.101332,
+     "end_time": "2026-02-02T13:03:03.769208",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:03.667876",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "## 2.7 Datetime Columns Analysis\n",
+    "\n",
+    "**📖 Unlike numeric transformations, datetime analysis recommends NEW FEATURES to create:**\n",
+    "\n",
+    "| Recommendation Type | Purpose | Examples |\n",
+    "|---------------------|---------|----------|\n",
+    "| **Feature Engineering** | Create predictive features from dates | `days_since_signup`, `tenure_years`, `month_sin_cos` |\n",
+    "| **Modeling Strategy** | How to structure train/test | Time-based splits when trends detected |\n",
+    "| **Data Quality** | Issues to address before modeling | Placeholder dates (1/1/1900) to filter |\n",
+    "\n",
+    "**📖 Feature Engineering Strategies:**\n",
+    "- **Recency**: `days_since_X` - How recent was the event? (useful for predicting behavior)\n",
+    "- **Tenure**: `tenure_years` - How long has customer been active? (maturity/loyalty)\n",
+    "- **Duration**: `days_between_A_and_B` - Time between events (e.g., signup to first purchase)\n",
+    "- **Cyclical**: `month_sin`, `month_cos` - Preserves that December is near January\n",
+    "- **Categorical**: `is_weekend`, `is_quarter_end` - Behavioral indicators"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c1642bb7",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-02-02T13:03:03.974921Z",
+     "iopub.status.busy": "2026-02-02T13:03:03.974803Z",
+     "iopub.status.idle": "2026-02-02T13:03:03.984910Z",
+     "shell.execute_reply": "2026-02-02T13:03:03.984448Z"
+    },
+    "papermill": {
+     "duration": 0.113413,
+     "end_time": "2026-02-02T13:03:03.985554",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:03.872141",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from customer_retention.stages.profiling.temporal_analyzer import TemporalRecommendationType\n",
+    "\n",
+    "datetime_cols = [\n",
+    "    name for name, col in findings.columns.items()\n",
+    "    if col.inferred_type == ColumnType.DATETIME\n",
+    "    and name not in TEMPORAL_METADATA_COLS\n",
+    "]\n",
+    "\n",
+    "temporal_analyzer = TemporalAnalyzer()\n",
+    "\n",
+    "# Store all datetime recommendations grouped by type\n",
+    "feature_engineering_recs = []\n",
+    "modeling_strategy_recs = []\n",
+    "data_quality_recs = []\n",
+    "datetime_summaries = []\n",
+    "\n",
+    "for col_name in datetime_cols:\n",
+    "    col_info = findings.columns[col_name]\n",
+    "    print(f\"\\n{'='*70}\")\n",
+    "    print(f\"Column: {col_name}\")\n",
+    "    print(f\"Type: {col_info.inferred_type.value} (Confidence: {col_info.confidence:.0%})\")\n",
+    "    print(f\"{'='*70}\")\n",
+    "    \n",
+    "    date_series = pd.to_datetime(df[col_name], errors='coerce', format='mixed')\n",
+    "    valid_dates = date_series.dropna()\n",
+    "    \n",
+    "    print(f\"\\n📅 Date Range: {valid_dates.min()} to {valid_dates.max()}\")\n",
+    "    print(f\"   Nulls: {date_series.isna().sum():,} ({date_series.isna().mean()*100:.1f}%)\")\n",
+    "    \n",
+    "    # Basic temporal analysis\n",
+    "    analysis = temporal_analyzer.analyze(date_series)\n",
+    "    print(f\"   Auto-detected granularity: {analysis.granularity.value}\")\n",
+    "    print(f\"   Span: {analysis.span_days:,} days ({analysis.span_days/365:.1f} years)\")\n",
+    "    \n",
+    "    # Growth analysis\n",
+    "    growth = temporal_analyzer.calculate_growth_rate(date_series)\n",
+    "    if growth.get(\"has_data\"):\n",
+    "        print(f\"\\n📈 Growth Analysis:\")\n",
+    "        print(f\"   Trend: {growth['trend_direction'].upper()}\")\n",
+    "        print(f\"   Overall growth: {growth['overall_growth_pct']:+.1f}%\")\n",
+    "        print(f\"   Avg monthly growth: {growth['avg_monthly_growth']:+.1f}%\")\n",
+    "    \n",
+    "    # Seasonality analysis\n",
+    "    seasonality = temporal_analyzer.analyze_seasonality(date_series)\n",
+    "    if seasonality.has_seasonality:\n",
+    "        print(f\"\\n🔄 Seasonality Detected:\")\n",
+    "        print(f\"   Peak months: {', '.join(seasonality.peak_periods[:3])}\")\n",
+    "        print(f\"   Trough months: {', '.join(seasonality.trough_periods[:3])}\")\n",
+    "        print(f\"   Seasonal strength: {seasonality.seasonal_strength:.2f}\")\n",
+    "    \n",
+    "    # Get recommendations using framework\n",
+    "    other_dates = [c for c in datetime_cols if c != col_name]\n",
+    "    recommendations = temporal_analyzer.recommend_features(date_series, col_name, other_date_columns=other_dates)\n",
+    "    \n",
+    "    # Group by recommendation type\n",
+    "    col_feature_recs = [r for r in recommendations if r.recommendation_type == TemporalRecommendationType.FEATURE_ENGINEERING]\n",
+    "    col_modeling_recs = [r for r in recommendations if r.recommendation_type == TemporalRecommendationType.MODELING_STRATEGY]\n",
+    "    col_quality_recs = [r for r in recommendations if r.recommendation_type == TemporalRecommendationType.DATA_QUALITY]\n",
+    "    \n",
+    "    feature_engineering_recs.extend(col_feature_recs)\n",
+    "    modeling_strategy_recs.extend(col_modeling_recs)\n",
+    "    data_quality_recs.extend(col_quality_recs)\n",
+    "    \n",
+    "    # Display recommendations grouped by type\n",
+    "    if col_feature_recs:\n",
+    "        print(f\"\\n🛠️ FEATURES TO CREATE:\")\n",
+    "        for rec in col_feature_recs:\n",
+    "            priority_icon = \"🔴\" if rec.priority == \"high\" else \"🟡\" if rec.priority == \"medium\" else \"✓\"\n",
+    "            print(f\"   {priority_icon} {rec.feature_name} ({rec.category})\")\n",
+    "            print(f\"      Why: {rec.reason}\")\n",
+    "            if rec.code_hint:\n",
+    "                print(f\"      Code: {rec.code_hint}\")\n",
+    "    \n",
+    "    if col_modeling_recs:\n",
+    "        print(f\"\\n⚙️ MODELING CONSIDERATIONS:\")\n",
+    "        for rec in col_modeling_recs:\n",
+    "            priority_icon = \"🔴\" if rec.priority == \"high\" else \"🟡\" if rec.priority == \"medium\" else \"✓\"\n",
+    "            print(f\"   {priority_icon} {rec.feature_name}\")\n",
+    "            print(f\"      Why: {rec.reason}\")\n",
+    "    \n",
+    "    if col_quality_recs:\n",
+    "        print(f\"\\n⚠️ DATA QUALITY ISSUES:\")\n",
+    "        for rec in col_quality_recs:\n",
+    "            priority_icon = \"🔴\" if rec.priority == \"high\" else \"🟡\" if rec.priority == \"medium\" else \"✓\"\n",
+    "            print(f\"   {priority_icon} {rec.feature_name}\")\n",
+    "            print(f\"      Why: {rec.reason}\")\n",
+    "            if rec.code_hint:\n",
+    "                print(f\"      Code: {rec.code_hint}\")\n",
+    "    \n",
+    "    # Standard extractions always available\n",
+    "    print(f\"\\n   Standard extractions available: year, month, day, day_of_week, quarter\")\n",
+    "    \n",
+    "    # Store summary\n",
+    "    datetime_summaries.append({\n",
+    "        \"Column\": col_name,\n",
+    "        \"Span (days)\": analysis.span_days,\n",
+    "        \"Seasonality\": \"Yes\" if seasonality.has_seasonality else \"No\",\n",
+    "        \"Trend\": growth.get('trend_direction', 'N/A').capitalize() if growth.get(\"has_data\") else \"N/A\",\n",
+    "        \"Features to Create\": len(col_feature_recs),\n",
+    "        \"Modeling Notes\": len(col_modeling_recs),\n",
+    "        \"Quality Issues\": len(col_quality_recs)\n",
+    "    })\n",
+    "    \n",
+    "    # === VISUALIZATIONS ===\n",
+    "    \n",
+    "    if growth.get(\"has_data\"):\n",
+    "        fig = charts.growth_summary_indicators(growth, title=f\"Growth Summary: {col_name}\")\n",
+    "        display_figure(fig)\n",
+    "    \n",
+    "    chart_type = \"line\" if analysis.granularity in [TemporalGranularity.DAY, TemporalGranularity.WEEK] else \"bar\"\n",
+    "    fig = charts.temporal_distribution(analysis, title=f\"Records Over Time: {col_name}\", chart_type=chart_type)\n",
+    "    display_figure(fig)\n",
+    "    \n",
+    "    fig = charts.temporal_trend(analysis, title=f\"Trend Analysis: {col_name}\")\n",
+    "    display_figure(fig)\n",
+    "    \n",
+    "    yoy_data = temporal_analyzer.year_over_year_comparison(date_series)\n",
+    "    if len(yoy_data) > 1:\n",
+    "        fig = charts.year_over_year_lines(yoy_data, title=f\"Year-over-Year: {col_name}\")\n",
+    "        display_figure(fig)\n",
+    "        fig = charts.year_month_heatmap(yoy_data, title=f\"Records Heatmap: {col_name}\")\n",
+    "        display_figure(fig)\n",
+    "    \n",
+    "    if growth.get(\"has_data\"):\n",
+    "        fig = charts.cumulative_growth_chart(growth[\"cumulative\"], title=f\"Cumulative Records: {col_name}\")\n",
+    "        display_figure(fig)\n",
+    "    \n",
+    "    fig = charts.temporal_heatmap(date_series, title=f\"Day of Week Distribution: {col_name}\")\n",
+    "    display_figure(fig)\n",
+    "\n",
+    "# === DATETIME SUMMARY ===\n",
+    "if datetime_summaries:\n",
+    "    print(\"\\n\" + \"=\" * 70)\n",
+    "    print(\"DATETIME COLUMNS SUMMARY\")\n",
+    "    print(\"=\" * 70)\n",
+    "    display_table(pd.DataFrame(datetime_summaries))\n",
+    "    \n",
+    "    # Summary by recommendation type\n",
+    "    print(\"\\n📋 ALL RECOMMENDATIONS BY TYPE:\")\n",
+    "    \n",
+    "    if feature_engineering_recs:\n",
+    "        print(f\"\\n🛠️ FEATURES TO CREATE ({len(feature_engineering_recs)}):\")\n",
+    "        for i, rec in enumerate(feature_engineering_recs, 1):\n",
+    "            priority_icon = \"🔴\" if rec.priority == \"high\" else \"🟡\" if rec.priority == \"medium\" else \"✓\"\n",
+    "            print(f\"   {i}. {priority_icon} {rec.feature_name}\")\n",
+    "    \n",
+    "    if modeling_strategy_recs:\n",
+    "        print(f\"\\n⚙️ MODELING CONSIDERATIONS ({len(modeling_strategy_recs)}):\")\n",
+    "        for i, rec in enumerate(modeling_strategy_recs, 1):\n",
+    "            priority_icon = \"🔴\" if rec.priority == \"high\" else \"🟡\" if rec.priority == \"medium\" else \"✓\"\n",
+    "            print(f\"   {i}. {priority_icon} {rec.feature_name}: {rec.reason}\")\n",
+    "    \n",
+    "    if data_quality_recs:\n",
+    "        print(f\"\\n⚠️ DATA QUALITY TO ADDRESS ({len(data_quality_recs)}):\")\n",
+    "        for i, rec in enumerate(data_quality_recs, 1):\n",
+    "            priority_icon = \"🔴\" if rec.priority == \"high\" else \"🟡\" if rec.priority == \"medium\" else \"✓\"\n",
+    "            print(f\"   {i}. {priority_icon} {rec.feature_name}: {rec.reason}\")\n",
+    "    \n",
+    "    # Add recommendations to registry\n",
+    "    added_derived = 0\n",
+    "    added_modeling = 0\n",
+    "    \n",
+    "    # Add feature engineering recommendations to Silver layer (derived columns)\n",
+    "    if registry.silver:\n",
+    "        for rec in feature_engineering_recs:\n",
+    "            registry.add_silver_derived(\n",
+    "                column=rec.feature_name,\n",
+    "                expression=rec.code_hint or \"\",\n",
+    "                feature_type=rec.category,\n",
+    "                rationale=rec.reason,\n",
+    "                source_notebook=\"02_column_deep_dive\"\n",
+    "            )\n",
+    "            added_derived += 1\n",
+    "    \n",
+    "    # Add modeling strategy recommendations to Bronze layer\n",
+    "    seen_strategies = set()\n",
+    "    for rec in modeling_strategy_recs:\n",
+    "        if rec.feature_name not in seen_strategies:\n",
+    "            registry.add_bronze_modeling_strategy(\n",
+    "                strategy=rec.feature_name,\n",
+    "                column=datetime_cols[0] if datetime_cols else \"\",\n",
+    "                parameters={\"category\": rec.category},\n",
+    "                rationale=rec.reason,\n",
+    "                source_notebook=\"02_column_deep_dive\"\n",
+    "            )\n",
+    "            seen_strategies.add(rec.feature_name)\n",
+    "            added_modeling += 1\n",
+    "    \n",
+    "    print(f\"\\n✅ Added {added_derived} derived column recommendations to Silver layer\")\n",
+    "    print(f\"✅ Added {added_modeling} modeling strategy recommendations to Bronze layer\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "21b21acc",
+   "metadata": {
+    "papermill": {
+     "duration": 0.139274,
+     "end_time": "2026-02-02T13:03:04.226289",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:04.087015",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "## 2.8 Type Override (Optional)\n",
+    "\n",
+    "If any column types were incorrectly inferred, you can override them here.\n",
+    "\n",
+    "**Common overrides:**\n",
+    "- Binary columns detected as numeric → `ColumnType.BINARY`\n",
+    "- IDs detected as numeric → `ColumnType.IDENTIFIER`\n",
+    "- Ordinal categories detected as nominal → `ColumnType.CATEGORICAL_ORDINAL`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "00245d72",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-02-02T13:03:04.430049Z",
+     "iopub.status.busy": "2026-02-02T13:03:04.429934Z",
+     "iopub.status.idle": "2026-02-02T13:03:04.432555Z",
+     "shell.execute_reply": "2026-02-02T13:03:04.432110Z"
+    },
+    "papermill": {
+     "duration": 0.105193,
+     "end_time": "2026-02-02T13:03:04.433172",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:04.327979",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# === TYPE OVERRIDES ===\n",
+    "# Uncomment and modify to override any incorrectly inferred types\n",
+    "TYPE_OVERRIDES = {\n",
+    "    # \"column_name\": ColumnType.NEW_TYPE,\n",
+    "    # Examples:\n",
+    "    # \"is_active\": ColumnType.BINARY,\n",
+    "    # \"user_id\": ColumnType.IDENTIFIER,\n",
+    "    # \"satisfaction_level\": ColumnType.CATEGORICAL_ORDINAL,\n",
+    "}\n",
+    "\n",
+    "if TYPE_OVERRIDES:\n",
+    "    print(\"Applying type overrides:\")\n",
+    "    for col_name, new_type in TYPE_OVERRIDES.items():\n",
+    "        if col_name in findings.columns:\n",
+    "            old_type = findings.columns[col_name].inferred_type.value\n",
+    "            findings.columns[col_name].inferred_type = new_type\n",
+    "            findings.columns[col_name].confidence = 1.0\n",
+    "            findings.columns[col_name].evidence.append(\"Manually overridden\")\n",
+    "            print(f\"  {col_name}: {old_type} → {new_type.value}\")\n",
+    "else:\n",
+    "    print(\"No type overrides configured.\")\n",
+    "    print(\"To override a type, add entries to TYPE_OVERRIDES dictionary above.\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "96593591",
+   "metadata": {
+    "papermill": {
+     "duration": 0.102127,
+     "end_time": "2026-02-02T13:03:04.673253",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:04.571126",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "## 2.9 Data Segmentation Analysis\n",
+    "\n",
+    "**Purpose:** Determine if the dataset contains natural subgroups that might benefit from separate models.\n",
+    "\n",
+    "**📖 Why This Matters:**\n",
+    "- Some datasets have distinct customer segments with very different behaviors\n",
+    "- A single model might struggle to capture patterns that vary significantly across segments\n",
+    "- Segmented models can improve accuracy but add maintenance complexity\n",
+    "\n",
+    "**Recommendations:**\n",
+    "- **single_model** - Data is homogeneous; one model for all records\n",
+    "- **consider_segmentation** - Some variation exists; evaluate if complexity is worth it\n",
+    "- **strong_segmentation** - Distinct segments with different target rates; separate models likely beneficial\n",
+    "\n",
+    "**Important:** This is exploratory guidance only. The final decision depends on business context, model complexity tolerance, and available resources."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ad8d552a",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-02-02T13:03:04.877339Z",
+     "iopub.status.busy": "2026-02-02T13:03:04.877224Z",
+     "iopub.status.idle": "2026-02-02T13:03:05.087709Z",
+     "shell.execute_reply": "2026-02-02T13:03:05.086847Z"
+    },
+    "papermill": {
+     "duration": 0.313839,
+     "end_time": "2026-02-02T13:03:05.088528",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:04.774689",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from customer_retention.stages.profiling import SegmentAnalyzer\n",
+    "\n",
+    "# Initialize segment analyzer\n",
+    "segment_analyzer = SegmentAnalyzer()\n",
+    "\n",
+    "# Find target column if detected\n",
+    "target_col = None\n",
+    "for col_name, col_info in findings.columns.items():\n",
+    "    if col_info.inferred_type == ColumnType.TARGET:\n",
+    "        target_col = col_name\n",
+    "        break\n",
+    "\n",
+    "# Run segmentation analysis using numeric features\n",
+    "print(\"=\"*70)\n",
+    "print(\"DATA SEGMENTATION ANALYSIS\")\n",
+    "print(\"=\"*70)\n",
+    "\n",
+    "segmentation = segment_analyzer.analyze(\n",
+    "    df,\n",
+    "    target_col=target_col,\n",
+    "    feature_cols=numeric_cols if numeric_cols else None,\n",
+    "    max_segments=5\n",
+    ")\n",
+    "\n",
+    "print(f\"\\n🎯 Analysis Results:\")\n",
+    "print(f\"   Method: {segmentation.method.value}\")\n",
+    "print(f\"   Detected Segments: {segmentation.n_segments}\")\n",
+    "print(f\"   Cluster Quality Score: {segmentation.quality_score:.2f}\")\n",
+    "if segmentation.target_variance_ratio is not None:\n",
+    "    print(f\"   Target Variance Ratio: {segmentation.target_variance_ratio:.2f}\")\n",
+    "\n",
+    "print(f\"\\n📊 Segment Profiles:\")\n",
+    "for profile in segmentation.profiles:\n",
+    "    target_info = f\" | Target Rate: {profile.target_rate*100:.1f}%\" if profile.target_rate is not None else \"\"\n",
+    "    print(f\"   Segment {profile.segment_id}: {profile.size:,} records ({profile.size_pct:.1f}%){target_info}\")\n",
+    "\n",
+    "# Display recommendation card\n",
+    "fig = charts.segment_recommendation_card(segmentation)\n",
+    "display_figure(fig)\n",
+    "\n",
+    "# Display segment overview\n",
+    "fig = charts.segment_overview(segmentation, title=\"Segment Overview\")\n",
+    "display_figure(fig)\n",
+    "\n",
+    "# Display feature comparison if we have features\n",
+    "if segmentation.n_segments > 1 and any(p.defining_features for p in segmentation.profiles):\n",
+    "    fig = charts.segment_feature_comparison(segmentation, title=\"Feature Comparison Across Segments\")\n",
+    "    display_figure(fig)\n",
+    "\n",
+    "print(f\"\\n📝 Rationale:\")\n",
+    "for reason in segmentation.rationale:\n",
+    "    print(f\"   • {reason}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2f3e371f",
+   "metadata": {
+    "papermill": {
+     "duration": 0.144582,
+     "end_time": "2026-02-02T13:03:05.348148",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:05.203566",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "## 2.10 Save Updated Findings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fc32cba0",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-02-02T13:03:05.560506Z",
+     "iopub.status.busy": "2026-02-02T13:03:05.560391Z",
+     "iopub.status.idle": "2026-02-02T13:03:05.639998Z",
+     "shell.execute_reply": "2026-02-02T13:03:05.639277Z"
+    },
+    "papermill": {
+     "duration": 0.186106,
+     "end_time": "2026-02-02T13:03:05.640834",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:05.454728",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Save updated findings back to the same file\n",
+    "findings.save(FINDINGS_PATH)\n",
+    "print(f\"Updated findings saved to: {FINDINGS_PATH}\")\n",
+    "\n",
+    "# Save recommendations registry\n",
+    "recommendations_path = FINDINGS_PATH.replace(\"_findings.yaml\", \"_recommendations.yaml\")\n",
+    "registry.save(recommendations_path)\n",
+    "print(f\"Recommendations saved to: {recommendations_path}\")\n",
+    "\n",
+    "# Summary of recommendations\n",
+    "all_recs = registry.all_recommendations\n",
+    "print(f\"\\n📋 Recommendations Summary:\")\n",
+    "print(f\"   Bronze layer: {len(registry.get_by_layer('bronze'))} recommendations\")\n",
+    "print(f\"   Silver layer: {len(registry.get_by_layer('silver'))} recommendations\")\n",
+    "print(f\"   Gold layer: {len(registry.get_by_layer('gold'))} recommendations\")\n",
+    "print(f\"   Total: {len(all_recs)} recommendations\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4f8202b3",
+   "metadata": {
+    "papermill": {
+     "duration": 0.14125,
+     "end_time": "2026-02-02T13:03:05.892090",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:05.750840",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "---\n",
+    "\n",
+    "## Summary: What We Learned\n",
+    "\n",
+    "In this notebook, we performed a deep dive analysis that included:\n",
+    "\n",
+    "1. **Value Range Validation** - Validated rates, binary fields, and non-negative constraints\n",
+    "2. **Numeric Distribution Analysis** - Calculated skewness, kurtosis, and percentiles with transformation recommendations\n",
+    "3. **Categorical Distribution Analysis** - Calculated imbalance ratio, entropy, and concentration with encoding recommendations\n",
+    "4. **Datetime Analysis** - Analyzed seasonality, trends, and patterns with feature engineering recommendations\n",
+    "5. **Data Segmentation** - Evaluated if natural subgroups exist that might benefit from separate models\n",
+    "\n",
+    "## Key Metrics Reference\n",
+    "\n",
+    "**Numeric Columns:**\n",
+    "| Metric | Threshold | Action |\n",
+    "|--------|-----------|--------|\n",
+    "| Skewness | \\|skew\\| > 1 | Log transform |\n",
+    "| Kurtosis | > 10 | Cap outliers first |\n",
+    "| Zero % | > 40% | Zero-inflation handling |\n",
+    "\n",
+    "**Categorical Columns:**\n",
+    "| Metric | Threshold | Action |\n",
+    "|--------|-----------|--------|\n",
+    "| Imbalance Ratio | > 10x | Group rare categories |\n",
+    "| Entropy | < 50% | Stratified sampling |\n",
+    "| Rare Categories | > 0 | Group into \"Other\" |\n",
+    "\n",
+    "**Datetime Columns:**\n",
+    "| Finding | Action |\n",
+    "|---------|--------|\n",
+    "| Seasonality | Add cyclical month encoding |\n",
+    "| Strong trend | Time-based train/test split |\n",
+    "| Multiple dates | Calculate duration features |\n",
+    "| Placeholder dates | Filter or flag |\n",
+    "\n",
+    "## Transformation & Encoding Summary\n",
+    "\n",
+    "Review the summary tables above for:\n",
+    "- **Numeric**: Which columns need log transforms, capping, or zero-inflation handling\n",
+    "- **Categorical**: Which encoding to use and whether to group rare categories\n",
+    "- **Datetime**: Which temporal features to engineer based on detected patterns\n",
+    "\n",
+    "---\n",
+    "\n",
+    "## Next Steps\n",
+    "\n",
+    "Continue to **03_quality_assessment.ipynb** to:\n",
+    "- Analyze duplicate records and value conflicts\n",
+    "- Deep dive into missing value patterns\n",
+    "- Analyze outliers with IQR method\n",
+    "- Check data consistency\n",
+    "- Get cleaning recommendations\n",
+    "\n",
+    "Or jump to **05_feature_opportunities.ipynb** if you want to see derived feature recommendations."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f114cb0a",
+   "metadata": {
+    "papermill": {
+     "duration": 0.114526,
+     "end_time": "2026-02-02T13:03:06.112557",
+     "exception": false,
+     "start_time": "2026-02-02T13:03:05.998031",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "> **Save Reminder:** Save this notebook (Ctrl+S / Cmd+S) before running the next one.\n",
+    "> The next notebook will automatically export this notebook's HTML documentation from the saved file."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  },
+  "papermill": {
+   "default_parameters": {},
+   "duration": 8.078058,
+   "end_time": "2026-02-02T13:03:06.637909",
+   "environment_variables": {},
+   "exception": null,
+   "input_path": "/Users/Vital/python/CustomerRetention/exploration_notebooks/02_column_deep_dive.ipynb",
+   "output_path": "/Users/Vital/python/CustomerRetention/exploration_notebooks/02_column_deep_dive.ipynb",
+   "parameters": {},
+   "start_time": "2026-02-02T13:02:58.559851",
+   "version": "2.6.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}