dataface 0.1.2__py3-none-any.whl

dataface/core/compile/chart_type_detection.py

@@ -0,0 +1,489 @@
+"""Chart type auto-detection based on data structure.
+
+Stage: RENDER (Runtime)
+Purpose: Automatically select the most appropriate chart type based on data.
+
+Entry Points:
+    - detect_chart_type(data, x_field, y_field, color_field, column_descriptions) -> (str, str)
+    - detect_chart_type_full(...) -> DetectionResult (includes field suggestions)
+
+Detection considers:
+- Data types: numeric, categorical, temporal, identifier
+- Column count: chartable columns (excluding identifiers)
+- Cardinality: distinct values in each column
+- Column names: hints from naming conventions
+- Database types: cursor.description when available (trusts DB over value parsing)
+
+Dependencies:
+    - datetime (for temporal detection)
+    - re (for pattern matching)
+
+See also:
+    - types.py: Chart type definitions
+    - render/chart_decisions.py: Uses db_type_to_dtype for DB type mapping
+    - render/vega_lite.py: Uses detected chart type for rendering
+"""
+
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+# Chart type constants
+CHART_TYPE_KPI = "kpi"
+CHART_TYPE_LINE = "line"
+CHART_TYPE_BAR = "bar"
+CHART_TYPE_SCATTER = "scatter"
+CHART_TYPE_TABLE = "table"
+CHART_TYPE_HEATMAP = "heatmap"
+CHART_TYPE_AREA = "area"
+CHART_TYPE_PIE = "pie"
+
+# Identifier column name patterns
+_IDENTIFIER_PATTERNS = re.compile(
+    r"(^id$|_id$|_key$|_pk$|^pk$|^uuid$|^guid$|_uuid$|_guid$|^sk$|_sk$)",
+    re.IGNORECASE,
+)
+
+# DB types that are always identifiers regardless of name
+_IDENTIFIER_DB_TYPES = frozenset({"UUID", "SERIAL", "BIGSERIAL", "SMALLSERIAL"})
+
+# Chartable column threshold — above this, default to table
+_MAX_CHARTABLE_COLUMNS = 8
+
+
+@dataclass
+class DetectionResult:
+    """Result of chart type auto-detection, including field suggestions."""
+
+    chart_type: str
+    reasoning: str
+    x: str | None = None
+    y: str | list[str] | None = None
+    color: str | None = None
+    value: str | None = None
+    theta: str | None = None
+
+
+def detect_chart_type(
+    data: list[dict[str, Any]],
+    x_field: str | None = None,
+    y_field: str | None = None,
+    color_field: str | None = None,
+    value_field: str | None = None,
+    column_descriptions: dict[str, tuple] | None = None,
+) -> tuple[str, str]:
+    """Detect the most appropriate chart type based on data structure.
+
+    Returns (chart_type, reasoning).
+
+    Args:
+        data: Query result data as list of dicts
+        x_field: Optional x-axis field (if specified by user)
+        y_field: Optional y-axis field (if specified by user)
+        color_field: Optional color field (if specified by user)
+        value_field: Optional KPI value column reference (if specified by user)
+        column_descriptions: Optional PEP 249 cursor.description tuples keyed
+            by column name. Used for DB-type-aware classification.
+
+    Returns:
+        Tuple of (chart_type, reasoning)
+    """
+    result = detect_chart_type_full(
+        data, x_field, y_field, color_field, value_field, column_descriptions
+    )
+    return result.chart_type, result.reasoning
+
+
+def detect_chart_type_full(
+    data: list[dict[str, Any]],
+    x_field: str | None = None,
+    y_field: str | None = None,
+    color_field: str | None = None,
+    value_field: str | None = None,
+    column_descriptions: dict[str, tuple] | None = None,
+) -> DetectionResult:
+    """Full detection returning chart type AND field suggestions.
+
+    This is the richer API that also suggests x/y/color/value/theta fields.
+    The simpler detect_chart_type() wraps it for callers that only need
+    the chart type and reasoning.
+    """
+    if not data:
+        return DetectionResult(CHART_TYPE_TABLE, "Empty data defaults to table display")
+
+    columns = list(data[0].keys())
+    column_types = _analyze_column_types(data, columns, column_descriptions)
+    column_cardinalities = _analyze_cardinality(data, columns)
+
+    identifier_cols = [c for c, t in column_types.items() if t == "identifier"]
+    numeric_cols = [c for c, t in column_types.items() if t == "numeric"]
+    temporal_cols = [c for c, t in column_types.items() if t == "temporal"]
+    categorical_cols = [c for c, t in column_types.items() if t == "categorical"]
+
+    chartable_cols = [c for c in columns if c not in identifier_cols]
+
+    # ════════════════════════════════════════════════════════════════════
+    # HEURISTIC 0: Many chartable columns → Table
+    # ════════════════════════════════════════════════════════════════════
+    if len(chartable_cols) > _MAX_CHARTABLE_COLUMNS:
+        return DetectionResult(
+            CHART_TYPE_TABLE,
+            f"Many chartable columns ({len(chartable_cols)}) suggests tabular display",
+        )
+
+    # ════════════════════════════════════════════════════════════════════
+    # HEURISTIC 1: Single row with single numeric value → KPI
+    # ════════════════════════════════════════════════════════════════════
+    if len(data) == 1 and len(numeric_cols) == 1 and len(chartable_cols) <= 2:
+        return DetectionResult(
+            CHART_TYPE_KPI,
+            "Single numeric value suggests a KPI/big number display",
+            value=numeric_cols[0],
+        )
+
+    if value_field and len(data) == 1:
+        return DetectionResult(
+            CHART_TYPE_KPI,
+            f"Value field '{value_field}' with single row suggests KPI",
+            value=value_field,
+        )
+
+    # ════════════════════════════════════════════════════════════════════
+    # HEURISTIC 2: User specified both x and y fields
+    # ════════════════════════════════════════════════════════════════════
+    if x_field and y_field:
+        x_type = column_types.get(x_field, "categorical")
+        y_type = column_types.get(y_field, "numeric")
+
+        if x_type == "temporal" and y_type == "numeric":
+            return DetectionResult(
+                CHART_TYPE_LINE,
+                f"Temporal x-axis ({x_field}) with numeric y-axis ({y_field}) suggests a line chart",
+                x=x_field,
+                y=y_field,
+            )
+
+        if x_type == "numeric" and y_type == "numeric":
+            return DetectionResult(
+                CHART_TYPE_SCATTER,
+                f"Two numeric columns ({x_field}, {y_field}) suggests a scatter plot",
+                x=x_field,
+                y=y_field,
+            )
+
+        if x_type in ("categorical", "identifier") and y_type == "numeric":
+            return DetectionResult(
+                CHART_TYPE_BAR,
+                f"Categorical x-axis ({x_field}) with numeric y-axis ({y_field}) suggests a bar chart",
+                x=x_field,
+                y=y_field,
+            )
+
+    # ════════════════════════════════════════════════════════════════════
+    # HEURISTIC 3: Infer from data structure when fields not specified
+    # ════════════════════════════════════════════════════════════════════
+
+    # Temporal + numeric → line chart (always line, never area)
+    if len(temporal_cols) >= 1 and len(numeric_cols) >= 1:
+        x_pick = temporal_cols[0]
+        y_pick: str | list[str] = (
+            numeric_cols[0] if len(numeric_cols) == 1 else numeric_cols
+        )
+        return DetectionResult(
+            CHART_TYPE_LINE,
+            f"Temporal column ({x_pick}) with numeric data suggests a line chart",
+            x=x_pick,
+            y=y_pick,
+        )
+
+    # Temporal + categorical + numeric → heatmap (e.g. weekday × hour)
+    if (
+        len(temporal_cols) >= 1
+        and len(categorical_cols) >= 1
+        and len(numeric_cols) >= 1
+    ):
+        t_card = column_cardinalities.get(temporal_cols[0], 0)
+        c_card = column_cardinalities.get(categorical_cols[0], 0)
+        if t_card <= 12 and c_card <= 12:
+            return DetectionResult(
+                CHART_TYPE_HEATMAP,
+                f"Temporal ({temporal_cols[0]}) x categorical ({categorical_cols[0]}) with measure suggests a heatmap",
+                x=temporal_cols[0],
+                y=categorical_cols[0],
+            )
+
+    # 2+ categorical + numeric → heatmap (both low cardinality) or bar
+    if len(categorical_cols) >= 2 and len(numeric_cols) >= 1:
+        cat1_card = column_cardinalities.get(categorical_cols[0], 0)
+        cat2_card = column_cardinalities.get(categorical_cols[1], 0)
+        if cat1_card <= 10 and cat2_card <= 10:
+            return DetectionResult(
+                CHART_TYPE_HEATMAP,
+                f"Two categorical dimensions ({categorical_cols[0]}, {categorical_cols[1]}) with numeric measure suggests a heatmap",
+                x=categorical_cols[0],
+                y=categorical_cols[1],
+            )
+        return DetectionResult(
+            CHART_TYPE_BAR,
+            "Multiple categories with numeric measure suggests a bar chart",
+            x=categorical_cols[0],
+            y=numeric_cols[0],
+            color=categorical_cols[1],
+        )
+
+    # 1 categorical + 1+ numeric → pie (3-6 values) or bar
+    if len(categorical_cols) == 1 and len(numeric_cols) >= 1:
+        cardinality = column_cardinalities.get(categorical_cols[0], 0)
+        if 3 <= cardinality <= 6:
+            return DetectionResult(
+                CHART_TYPE_PIE,
+                f"Low cardinality categorical ({categorical_cols[0]}) with {cardinality} values suggests a pie chart",
+                theta=numeric_cols[0],
+                color=categorical_cols[0],
+            )
+        if cardinality <= 20:
+            return DetectionResult(
+                CHART_TYPE_BAR,
+                f"Categorical column ({categorical_cols[0]}) with numeric data suggests a bar chart",
+                x=categorical_cols[0],
+                y=numeric_cols[0],
+            )
+
+    # 2+ numeric, no temporal → scatter
+    if len(numeric_cols) >= 2 and len(temporal_cols) == 0:
+        return DetectionResult(
+            CHART_TYPE_SCATTER,
+            f"Multiple numeric columns ({', '.join(numeric_cols[:2])}) suggests a scatter plot",
+            x=numeric_cols[0],
+            y=numeric_cols[1],
+        )
+
+    # ════════════════════════════════════════════════════════════════════
+    # FALLBACK
+    # ════════════════════════════════════════════════════════════════════
+    if len(numeric_cols) >= 1:
+        x_cand = categorical_cols[0] if categorical_cols else None
+        return DetectionResult(
+            CHART_TYPE_BAR,
+            "Default to bar chart for numeric data",
+            x=x_cand,
+            y=numeric_cols[0],
+        )
+
+    return DetectionResult(
+        CHART_TYPE_TABLE,
+        "Unable to determine optimal chart type, defaulting to table",
+    )
+
+
+def _analyze_column_types(
+    data: list[dict[str, Any]],
+    columns: list[str],
+    column_descriptions: dict[str, tuple] | None = None,
+) -> dict[str, str]:
+    """Analyze and classify column types.
+
+    Categories: numeric, temporal, categorical, identifier.
+
+    When column_descriptions are available, database types take priority
+    over value-based heuristics (e.g. VARCHAR is always categorical even
+    if values look numeric like zip codes).
+    """
+    column_types: dict[str, str] = {}
+    row_count = len(data)
+
+    for col in columns:
+        sample_values = [row.get(col) for row in data[:100] if row.get(col) is not None]
+
+        # Layer 1: Use database type if available
+        db_dtype = None
+        db_type_str = None
+        if column_descriptions and col in column_descriptions:
+            desc = column_descriptions[col]
+            if desc and len(desc) > 1 and desc[1] is not None:
+                db_type_str = str(desc[1])
+                from dataface.core.render.chart.decisions import db_type_to_dtype
+
+                db_dtype = db_type_to_dtype(db_type_str)
+
+        # Layer 2: Value-based classification (fallback)
+        if db_dtype:
+            base_type = db_dtype
+        elif not sample_values:
+            base_type = "categorical"
+        else:
+            base_type = classify_column_type(col, sample_values, db_type_str)
+
+        # Layer 3: Identifier detection (overrides numeric/categorical)
+        if _is_identifier(col, base_type, sample_values, row_count, db_type_str):
+            column_types[col] = "identifier"
+        else:
+            column_types[col] = base_type
+
+    return column_types
+
+
+def _is_identifier(
+    col_name: str,
+    base_type: str,
+    sample_values: list[Any],
+    row_count: int,
+    db_type_str: str | None = None,
+) -> bool:
+    """Detect if a column is an identifier (primary key, UUID, etc.).
+
+    Identifiers are excluded from chart axis candidates.
+    """
+    if db_type_str:
+        upper = db_type_str.upper().split("(")[0].strip()
+        if upper in _IDENTIFIER_DB_TYPES:
+            return True
+
+    if not _IDENTIFIER_PATTERNS.search(col_name):
+        return False
+
+    if not sample_values or base_type not in ("numeric", "categorical"):
+        return False
+
+    distinct = len({str(v) for v in sample_values})
+    sample_size = min(len(sample_values), row_count)
+    if sample_size == 0:
+        return False
+
+    return distinct / sample_size > 0.9
+
+
+def classify_column_type(
+    column_name: str,
+    sample_values: list[Any],
+    db_type_str: str | None = None,
+) -> str:
+    """Classify a single column's type based on its values and name.
+
+    When db_type_str indicates VARCHAR/TEXT, string values are NOT parsed
+    as numbers — the database type is trusted.
+
+    Returns: "numeric", "temporal", or "categorical"
+    """
+    is_db_string = False
+    if db_type_str:
+        upper = db_type_str.upper().split("(")[0].strip()
+        if upper.startswith("VARCHAR") or upper in ("TEXT", "CHAR", "STRING", "ENUM"):
+            is_db_string = True
+
+    name_lower = column_name.lower()
+
+    temporal_patterns = [
+        r"date",
+        r"time",
+        r"timestamp",
+        r"created",
+        r"updated",
+        r"_at$",
+        r"day",
+        r"week",
+        r"month",
+        r"year",
+        r"quarter",
+        r"period",
+    ]
+    for pattern in temporal_patterns:
+        if re.search(pattern, name_lower) and any(
+            _is_temporal_value(v) for v in sample_values[:5]
+        ):
+            return "temporal"
+
+    numeric_count = 0
+    temporal_count = 0
+    categorical_count = 0
+
+    for val in sample_values:
+        if isinstance(val, bool):
+            categorical_count += 1
+        elif isinstance(val, (int, float)):
+            numeric_count += 1
+        elif isinstance(val, datetime):
+            temporal_count += 1
+        elif isinstance(val, str):
+            if _is_temporal_value(val):
+                temporal_count += 1
+            elif not is_db_string and _is_numeric_string(val):
+                numeric_count += 1
+            else:
+                categorical_count += 1
+        else:
+            categorical_count += 1
+
+    total = len(sample_values)
+    if total == 0:
+        return "categorical"
+
+    if numeric_count / total > 0.8:
+        return "numeric"
+    if temporal_count / total > 0.8:
+        return "temporal"
+
+    return "categorical"
+
+
+def _is_temporal_value(value: Any) -> bool:
+    """Check if a value appears to be temporal (date/time).
+
+    Recognises standard date formats and common period strings:
+    - ISO dates: 2024-01-15, 2024/01/15
+    - US dates: 01/15/2024, 01-15-2024
+    - Text dates: Jan 15, 2024
+    - Year-months: 2024-01, Jan 2024
+    - Quarters: 2024-Q1, Q1 2024, 2024Q1
+    - Plain years: 2024 (4-digit, in plausible range)
+    """
+    if isinstance(value, datetime):
+        return True
+    if not isinstance(value, str):
+        return False
+    date_patterns = [
+        r"^\d{4}-\d{2}-\d{2}",  # 2024-01-15
+        r"^\d{2}/\d{2}/\d{4}",  # 01/15/2024
+        r"^\d{2}-\d{2}-\d{4}",  # 01-15-2024
+        r"^\d{4}/\d{2}/\d{2}",  # 2024/01/15
+        r"^\w{3}\s+\d{1,2},?\s+\d{4}",  # Jan 15, 2024
+        r"^\d{4}-Q[1-4]$",  # 2024-Q1
+        r"^Q[1-4]\s*\d{4}$",  # Q1 2024, Q12024
+        r"^\d{4}Q[1-4]$",  # 2024Q1
+        r"^\d{4}-\d{2}$",  # 2024-01 (year-month)
+        r"^(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{4}$",  # Jan 2024
+    ]
+    return any(re.match(pattern, value, re.IGNORECASE) for pattern in date_patterns)
+
+
+def _is_numeric_string(value: str) -> bool:
+    """Check if a string value represents a number."""
+    try:
+        float(value.replace(",", "").replace("$", "").replace("%", ""))
+        return True
+    except (ValueError, AttributeError):
+        return False
+
+
+def _analyze_cardinality(
+    data: list[dict[str, Any]],
+    columns: list[str],
+) -> dict[str, int]:
+    """Calculate cardinality (distinct value count) for each column."""
+    cardinalities: dict[str, int] = {}
+    for col in columns:
+        distinct_values = set()
+        for row in data:
+            val = row.get(col)
+            if val is not None:
+                distinct_values.add(str(val))
+        cardinalities[col] = len(distinct_values)
+    return cardinalities
+
+
+def is_auto_chart_type(chart_type: str | None) -> bool:
+    """Check if chart type indicates auto-detection should be used."""
+    if chart_type is None:
+        return True
+    return chart_type.lower() == "auto"