dataface 0.1.2__py3-none-any.whl

dataface/core/inspect/cache_factory.py

@@ -0,0 +1,98 @@
+"""Cache-source factory and schema resolver builder.
+
+Builds a LayeredSchemaResolver from an adapter registry. When the private
+``dataface-super-schema`` package is installed, the resolver uses a warm
+SuperSchemaSource cache; otherwise it operates in dbt-only mode.
+
+This module provides factory helpers for building resolvers with optional
+super-schema caches. Other modules that only need a presence check use their
+own ``importlib.util.find_spec("dataface_super_schema")`` probe — that is
+intentional and avoids import cycles.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from dataface.core.execute.adapters import AdapterRegistry
+from dataface.core.inspect.resolver import LayeredSchemaResolver
+
+if TYPE_CHECKING:
+    from dataface_super_schema.inspect.sources.super_schema import SuperSchemaSource
+
+# Probe at module load time — single find_spec call per process.
+_SUPER_SCHEMA_AVAILABLE: bool = (
+    importlib.util.find_spec("dataface_super_schema") is not None
+)
+
+
+def project_root_for_registry(registry: AdapterRegistry) -> Path:
+    """Resolve the registry's project root without consulting cwd."""
+    raw = registry.project_root
+    if raw is not None:
+        return Path(raw).resolve()
+    raise ValueError("adapter_registry.project_root is required for schema requests")
+
+
+def make_cache_source(cache_path: Path) -> Any:
+    """Build a SuperSchemaSource when the private package is available.
+
+    Args:
+        cache_path: Absolute path to a ``super_schema.json`` cache file.
+
+    Returns ``None`` when ``dataface-super-schema`` is not installed.
+    The resolver degrades to dbt-only mode when ``None`` is returned.
+    """
+    if not _SUPER_SCHEMA_AVAILABLE:
+        return None
+    from dataface_super_schema.inspect.sources.super_schema import (  # noqa: PLC0415
+        SuperSchemaSource,
+    )
+    from dataface_super_schema.inspect.storage import InspectionStorage  # noqa: PLC0415
+
+    return SuperSchemaSource(InspectionStorage(output_path=cache_path))
+
+
+def make_cache_source_for_project(project_root: Path) -> Any:
+    """Build a SuperSchemaSource from the project's default cache path.
+
+    Equivalent to make_cache_source(project_root / InspectionStorage.DEFAULT_PATH)
+    without importing InspectionStorage in the caller.
+    """
+    if not _SUPER_SCHEMA_AVAILABLE:
+        return None
+    from dataface_super_schema.inspect.storage import InspectionStorage  # noqa: PLC0415
+
+    return make_cache_source(project_root / InspectionStorage.DEFAULT_PATH)
+
+
+def build_resolver(
+    adapter_registry: AdapterRegistry,
+    cache_path: Path | None = None,
+    cache_source: SuperSchemaSource | None = None,
+) -> LayeredSchemaResolver:
+    """Build a LayeredSchemaResolver from an adapter registry and optional cache.
+
+    Args:
+        adapter_registry: Registry providing source configs and project root.
+        cache_path: Explicit path to a ``super_schema.json`` file. When
+            ``None``, the cache is auto-derived from the registry's
+            project_root via ``make_cache_source_for_project``.
+        cache_source: Explicit warm-cache source. Takes priority over
+            ``cache_path`` when provided.
+    """
+    project_root = project_root_for_registry(adapter_registry)
+    resolved_cache: Any = cache_source
+    if resolved_cache is None:
+        resolved_cache = (
+            make_cache_source(cache_path)
+            if cache_path is not None
+            else make_cache_source_for_project(project_root)
+        )
+    return LayeredSchemaResolver(
+        cache=resolved_cache,
+        adapter_registry=adapter_registry,
+        project_root=project_root,
+    )

dataface/core/inspect/db_types.py

@@ -0,0 +1,162 @@
+"""Shared SQL type classification for the inspect module.
+
+Provides a single source of truth for categorising database column types
+as numeric, string, temporal, or complex.  Used by query_builder,
+quality_detector, semantic_detector, and inspector.
+"""
+
+from __future__ import annotations
+
+import re
+
+# ---------------------------------------------------------------------------
+# Base-type normaliser
+# ---------------------------------------------------------------------------
+
+# Postgres-specific aliases → standard SQL type names
+_POSTGRES_TYPE_MAP: dict[str, str] = {
+    "INT2": "SMALLINT",
+    "INT4": "INTEGER",
+    "INT8": "BIGINT",
+    "FLOAT4": "FLOAT",
+    "FLOAT8": "DOUBLE",
+    "BOOL": "BOOLEAN",
+    "BPCHAR": "CHAR",
+    "TIMESTAMPTZ": "TIMESTAMP",
+    "TIMETZ": "TIME",
+}
+
+
+def extract_base_type(db_type: str, *, normalize_aliases: bool = False) -> str:
+    """Extract base type name without precision, scale, or modifiers.
+
+    Args:
+        db_type: Raw database type string, e.g. ``"DECIMAL(18,2) NOT NULL"``.
+        normalize_aliases: When True, map Postgres aliases to standard
+            names (e.g. ``INT8`` → ``BIGINT``).
+
+    Examples:
+        >>> extract_base_type("DECIMAL(18,2)")
+        'DECIMAL'
+        >>> extract_base_type("int8", normalize_aliases=True)
+        'BIGINT'
+    """
+    base = re.sub(r"\([^)]*\)", "", db_type.upper()).strip()
+    for modifier in ("NOT NULL", "NULL", "PRIMARY KEY", "UNIQUE", "DEFAULT", "ARRAY"):
+        base = base.replace(modifier, "").strip()
+    base = base.split()[0] if base else base
+    if normalize_aliases:
+        base = _POSTGRES_TYPE_MAP.get(base, base)
+    return base
+
+
+# ---------------------------------------------------------------------------
+# Canonical type sets  (superset across all supported databases)
+# ---------------------------------------------------------------------------
+
+NUMERIC_TYPES: frozenset[str] = frozenset(
+    {
+        "INTEGER",
+        "INT",
+        "BIGINT",
+        "SMALLINT",
+        "TINYINT",
+        "DECIMAL",
+        "NUMERIC",
+        "FLOAT",
+        "DOUBLE",
+        "REAL",
+        "NUMBER",  # Oracle / Snowflake
+        "INT64",
+        "FLOAT64",  # BigQuery
+        "INT2",
+        "INT4",
+        "INT8",
+        "FLOAT4",
+        "FLOAT8",  # Postgres aliases
+        "SERIAL",
+        "BIGSERIAL",
+        "SMALLSERIAL",  # Postgres auto-increment
+        "HUGEINT",
+        "UBIGINT",
+        "UINTEGER",  # DuckDB unsigned
+        "USMALLINT",
+        "UTINYINT",
+        "MONEY",  # Postgres money
+    }
+)
+
+STRING_TYPES: frozenset[str] = frozenset(
+    {
+        "VARCHAR",
+        "CHAR",
+        "TEXT",
+        "STRING",
+        "CLOB",
+        "NVARCHAR",
+        "NCHAR",
+        "NTEXT",
+        "BPCHAR",  # Postgres blank-padded char
+    }
+)
+
+TEMPORAL_TYPES: frozenset[str] = frozenset(
+    {
+        "DATE",
+        "TIMESTAMP",
+        "DATETIME",
+        "TIME",
+        "TIMESTAMPTZ",
+        "TIMETZ",  # Postgres with timezone
+        "TIMESTAMP_NTZ",
+        "TIMESTAMP_LTZ",
+        "TIMESTAMP_TZ",  # Snowflake variants
+        "INTERVAL",
+    }
+)
+
+COMPLEX_TYPES: frozenset[str] = frozenset(
+    {
+        "ARRAY",
+        "STRUCT",
+        "RECORD",
+        "GEOGRAPHY",
+        "GEOMETRY",
+        "BYTES",
+        "JSON",  # BigQuery
+        "VARIANT",
+        "OBJECT",  # Snowflake
+        "MAP",  # Databricks
+    }
+)
+
+# Prefixes that indicate a complex parameterised type (e.g. ARRAY<INT64>)
+_COMPLEX_PREFIXES = ("ARRAY<", "STRUCT<", "RECORD<", "MAP<")
+
+
+# ---------------------------------------------------------------------------
+# Convenience helpers
+# ---------------------------------------------------------------------------
+
+
+def is_numeric(db_type: str) -> bool:
+    """Return True if *db_type* is a numeric column type."""
+    return extract_base_type(db_type) in NUMERIC_TYPES
+
+
+def is_string(db_type: str) -> bool:
+    """Return True if *db_type* is a string/text column type."""
+    return extract_base_type(db_type) in STRING_TYPES
+
+
+def is_temporal(db_type: str) -> bool:
+    """Return True if *db_type* is a date, time, or timestamp type."""
+    return extract_base_type(db_type) in TEMPORAL_TYPES
+
+
+def is_complex(db_type: str) -> bool:
+    """Return True if *db_type* is a complex type unsuitable for standard aggregates."""
+    upper = db_type.upper()
+    if upper.startswith(_COMPLEX_PREFIXES):
+        return True
+    return extract_base_type(db_type) in COMPLEX_TYPES

dataface/core/inspect/dbt_schema.py

@@ -0,0 +1,96 @@
+"""Parse dbt schema.yml files and map descriptions to inspected tables/columns.
+
+Scans ``models/**/schema.yml`` (and ``.yaml``) for model-level and
+column-level ``description`` fields, then maps them onto inspected
+table/column names using case-insensitive matching.
+
+Descriptions are stored as explicit source entries with
+``source=dbt_schema_yml`` — they never overwrite inferred metadata.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+
+@dataclass
+class DbtDescription:
+    """A single description extracted from a dbt schema.yml file."""
+
+    model_name: str
+    column_name: str | None  # None = table-level description
+    description: str
+    source: str = "dbt_schema_yml"
+
+
+class DbtSchemaParser:
+    """Discover and parse dbt schema.yml files under a project root."""
+
+    def __init__(self, project_root: Path) -> None:
+        self.project_root = project_root
+        self.warnings: list[str] = []
+
+    def parse(self) -> list[DbtDescription]:
+        """Scan ``models/**/schema.{yml,yaml}`` and extract descriptions."""
+        models_dir = self.project_root / "models"
+        if not models_dir.is_dir():
+            return []
+
+        descriptions: list[DbtDescription] = []
+        for pattern in ("**/*.yml", "**/*.yaml"):
+            for path in sorted(models_dir.glob(pattern)):
+                descriptions.extend(self._parse_file(path))
+        return descriptions
+
+    def _parse_file(self, path: Path) -> list[DbtDescription]:
+        """Parse a single schema file, returning descriptions found."""
+        try:
+            content = yaml.safe_load(path.read_text())
+        except yaml.YAMLError as e:
+            self.warnings.append(f"Failed to parse {path}: {e}")
+            return []
+
+        if not isinstance(content, dict):
+            return []
+
+        models = content.get("models")
+        if not isinstance(models, list):
+            return []
+
+        descriptions: list[DbtDescription] = []
+        for model in models:
+            if not isinstance(model, dict):
+                continue
+            model_name = model.get("name")
+            if not model_name:
+                continue
+
+            model_desc = model.get("description")
+            if model_desc:
+                descriptions.append(
+                    DbtDescription(
+                        model_name=model_name, column_name=None, description=model_desc
+                    )
+                )
+
+            columns = model.get("columns")
+            if not isinstance(columns, list):
+                continue
+            for col in columns:
+                if not isinstance(col, dict):
+                    continue
+                col_name = col.get("name")
+                col_desc = col.get("description")
+                if col_name and col_desc:
+                    descriptions.append(
+                        DbtDescription(
+                            model_name=model_name,
+                            column_name=col_name,
+                            description=col_desc,
+                        )
+                    )
+
+        return descriptions

dataface/core/inspect/defaults.yml

@@ -0,0 +1,37 @@
+inspector:
+  sample_size: 100
+  collect_top_values: true
+  top_values_limit: 10
+  top_values_max_cardinality: 1000
+  top_values_skip_types:
+    - INTEGER
+    - INT
+    - BIGINT
+    - SMALLINT
+    - TINYINT
+    - FLOAT
+    - DOUBLE
+    - REAL
+    - DECIMAL
+    - NUMERIC
+    - NUMBER
+    - DATE
+    - TIME
+    - TIMESTAMP
+    - DATETIME
+    - INTERVAL
+    - BOOLEAN
+    - BOOL
+    - UUID
+    - BLOB
+    - BINARY
+    - BYTEA
+    - VARBINARY
+    - BIT
+  deep_profile: false
+  collect_histogram_bins: false
+  collect_date_distribution: false
+  collect_enum_values: false
+  enum_cardinality_threshold: 20
+  bigquery_exact_threshold_bytes: 1073741824
+  fk_range_slack_percent: 2

dataface/core/inspect/fanout_risk.py

@@ -0,0 +1,109 @@
+"""Fanout risk scoring.
+
+Scores the risk of aggregate inflation when joining table pairs.
+Pure functions — no DB queries.
+
+Risk levels:
+- ``none``     — safe join, no action needed
+- ``low``      — join changes grain but may be intentional
+- ``medium``   — aggregation after this join may be incorrect
+- ``high``     — strong likelihood of aggregate inflation
+- ``critical`` — N:M join with aggregation, almost certainly wrong
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+# Fanout thresholds — public, imported by query_validator.py.
+HIGH_FANOUT_THRESHOLD = 10.0
+
+
+# ---------------------------------------------------------------------------
+# Data model
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class FanoutRisk:
+    """Fanout risk assessment for a join relationship."""
+
+    level: str  # "none" | "low" | "medium" | "high" | "critical"
+    reason: str
+    recommendation: str
+
+    def to_dict(self) -> dict[str, str]:
+        """Dict for serialization.
+
+        All keys are single words (no underscores) so snake_case and
+        camelCase are identical — one method serves both formats.
+        """
+        return {
+            "level": self.level,
+            "reason": self.reason,
+            "recommendation": self.recommendation,
+        }
+
+
+# ---------------------------------------------------------------------------
+# Scoring
+# ---------------------------------------------------------------------------
+
+
+def score_fanout_risk(
+    multiplicity: str,
+    fanout_factor: float,
+    has_aggregation: bool = False,
+) -> FanoutRisk:
+    """Score fanout risk for a join pattern.
+
+    Args:
+        multiplicity: Join cardinality ("one-to-one", "one-to-many",
+            "many-to-one", "many-to-many").
+        fanout_factor: Average row multiplication factor.
+        has_aggregation: Whether the query aggregates after the join.
+    """
+    # 1:1 — always safe
+    if multiplicity == "one-to-one":
+        return FanoutRisk("none", "1:1 join — safe", "No action needed")
+
+    # N:1 — dimension lookup, always safe
+    if multiplicity == "many-to-one":
+        return FanoutRisk("none", "N:1 dimension lookup — safe", "No action needed")
+
+    # N:M — always dangerous
+    if multiplicity == "many-to-many":
+        if has_aggregation:
+            return FanoutRisk(
+                "critical",
+                "N:M join with aggregation — almost certainly inflates results",
+                "Use a bridge table or pre-aggregate to the correct grain before joining",
+            )
+        return FanoutRisk(
+            "high",
+            "N:M join — row multiplication likely unintended",
+            "Consider using a bridge table or pre-aggregating before joining",
+        )
+
+    # 1:N — risk depends on aggregation context and fanout factor
+    if multiplicity == "one-to-many":
+        if not has_aggregation:
+            return FanoutRisk(
+                "low",
+                "1:N join without aggregation — row expansion expected",
+                "Verify this is a detail query, not an aggregate",
+            )
+        if fanout_factor > HIGH_FANOUT_THRESHOLD:
+            return FanoutRisk(
+                "high",
+                f"1:N join with aggregation and high fanout ({fanout_factor:.1f}x)",
+                "Pre-aggregate the many-side to the join key grain before joining",
+            )
+        return FanoutRisk(
+            "medium",
+            f"1:N join with aggregation ({fanout_factor:.1f}x fanout)",
+            "Pre-aggregate the many-side to the join key grain before joining",
+        )
+
+    # Unknown multiplicity — safe default
+    return FanoutRisk("none", "Unknown multiplicity", "Review join pattern")

dataface/core/inspect/manifest_utils.py

@@ -0,0 +1,77 @@
+"""Manifest I/O and template-comparison helpers for the inspect verb.
+
+Pure functions — no agent-api or Pydantic coupling. Used by
+``dataface.agent_api.inspect`` to persist ejection state and compare
+local templates against built-in versions.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from hashlib import sha256
+from pathlib import Path
+from typing import Any
+
+INSPECT_TEMPLATE_MANIFEST = ".inspect-template-manifest.json"
+MANIFEST_SCHEMA_VERSION = 1
+
+
+def load_manifest(target_dir: Path, *, dataface_version: str) -> dict:
+    """Load the inspect-template manifest from target_dir, or return a fresh one."""
+    manifest_path = target_dir / INSPECT_TEMPLATE_MANIFEST
+    if not manifest_path.exists():
+        return {
+            "schema_version": MANIFEST_SCHEMA_VERSION,
+            "dataface_version": dataface_version,
+            "generated_at": datetime.now(timezone.utc).isoformat(),
+            "templates": {},
+        }
+    with open(manifest_path) as f:
+        return json.load(f)
+
+
+def save_manifest(target_dir: Path, manifest: dict) -> None:
+    """Write the inspect-template manifest to target_dir."""
+    manifest_path = target_dir / INSPECT_TEMPLATE_MANIFEST
+    with open(manifest_path, "w") as f:
+        json.dump(manifest, f, indent=2, sort_keys=True)
+        f.write("\n")
+
+
+def compare_templates(
+    resolved: Path,
+    manifest_templates: dict[str, dict],
+    templates_pkg: Any,  # importlib.resources Traversable; Any avoids 3.10/3.11 compat shim
+) -> tuple[list[str], list[str], list[str], list[str]]:
+    """Compare ejected templates against built-in versions.
+
+    Returns ``(missing, upstream_changed, custom_safe, unchanged)`` name lists.
+    ``templates_pkg`` is an ``importlib.resources`` traversable (the package
+    containing the canonical ``.yml`` template files).
+    """
+    missing: list[str] = []
+    upstream_changed: list[str] = []
+    custom_safe: list[str] = []
+    unchanged: list[str] = []
+
+    for name, meta in sorted(manifest_templates.items()):
+        filename = meta.get("filename") or f"{name}.yml"
+        local_path = resolved / filename
+        if not local_path.exists():
+            missing.append(filename)
+            continue
+
+        built_in = templates_pkg.joinpath(filename).read_text()
+        built_in_hash = sha256(built_in.encode("utf-8")).hexdigest()
+        baseline_hash = meta.get("source_sha256")
+        local_hash = sha256(local_path.read_text().encode("utf-8")).hexdigest()
+
+        if local_hash == built_in_hash:
+            unchanged.append(name)
+        elif baseline_hash != built_in_hash:
+            upstream_changed.append(name)
+        else:
+            custom_safe.append(name)
+
+    return missing, upstream_changed, custom_safe, unchanged

dataface/core/inspect/partials/categorical.yml

@@ -0,0 +1,40 @@
+# Categorical Column Partial - Bar chart visualization
+#
+# A reusable partial for visualizing categorical column distributions.
+# Designed for use with foreach construct.
+#
+# Expected loop variable (col):
+#   - col.column: Column name to visualize
+#
+# Expected parent variables:
+#   - model: Table/model name
+#   - connection: DuckDB file path or :memory:
+
+title: "{{ col.column }}"
+
+queries:
+  category_counts:
+    source:
+      type: duckdb
+      path: "{{ connection }}"
+    sql: |
+      SELECT
+        COALESCE(CAST({{ col.column }} AS VARCHAR), '(null)') as category,
+        COUNT(*) as count
+      FROM {{ model }}
+      GROUP BY {{ col.column }}
+      ORDER BY COUNT(*) DESC
+      LIMIT 20
+
+charts:
+  bar_chart:
+    title: "{{ col.column }} Distribution"
+    type: bar
+    query: category_counts
+    x: category
+    y: count
+    style:
+      orientation: horizontal
+
+rows:
+  - bar_chart

dataface/core/inspect/partials/date.yml

@@ -0,0 +1,40 @@
+# Date Column Partial - Time series visualization
+#
+# A reusable partial for visualizing date column distributions.
+# Designed for use with foreach construct.
+#
+# Expected loop variable (col):
+#   - col.column: Column name to visualize
+#
+# Expected parent variables:
+#   - model: Table/model name
+#   - connection: DuckDB file path or :memory:
+
+title: "{{ col.column }}"
+
+queries:
+  daily_counts:
+    source:
+      type: duckdb
+      path: "{{ connection }}"
+    sql: |
+      SELECT
+        {{ col.column }}::DATE as date,
+        COUNT(*) as count
+      FROM {{ model }}
+      WHERE {{ col.column }} IS NOT NULL
+      GROUP BY {{ col.column }}::DATE
+      ORDER BY date
+
+charts:
+  time_series:
+    title: "{{ col.column }} Over Time"
+    type: line
+    query: daily_counts
+    x: date
+    y: count
+    x_label: Date
+    y_label: Count
+
+rows:
+  - time_series