dataface 0.1.2__py3-none-any.whl

dataface/core/inspect/sources/duckdb_utils.py

@@ -0,0 +1,66 @@
+"""DuckDB-specific helpers for schema inspection.
+
+When a DuckDB adapter connects to a file (e.g. ``dundersign.duckdb``),
+the database is attached under a non-system name (e.g. ``dundersign``).
+Passing ``None`` to ``adapter.list_relations(None, schema)`` returns an
+empty list for file-backed databases — the database argument must be the
+actual attach name.
+
+``PRAGMA database_list`` is the authoritative source: each row is
+``(seq, name, file)``.  We skip the two system databases (``system`` and
+``temp``); the remaining row is the user's attached database.  For
+``path=None`` or ``path=':memory:'`` the attach name is always ``memory``.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+
+def duckdb_resolve_database(adapter: Any, db_path: str | Path | None) -> str:
+    """Return the DuckDB attach name that ``list_relations`` must receive.
+
+    Reads ``PRAGMA database_list`` from the already-open adapter connection.
+    The caller must be inside a ``connection_named`` context.
+
+    ``db_path`` is the configured file path (or ``None`` / ``':memory:'`` for
+    in-memory).  When the path is in-memory we short-circuit and return
+    ``'memory'`` without a PRAGMA round-trip.
+
+    For file-backed databases we prefer the row whose ``file`` column matches
+    the absolute ``db_path``; if no row matches (e.g. path is relative or
+    symlinked) we fall back to the first non-system entry.
+
+    Raises ``ValueError`` if no non-system database is found in the pragma
+    output — this would indicate a DuckDB connection without an attached
+    database, which should be impossible for a live adapter.
+    """
+    path_str = str(db_path) if db_path is not None else None
+    if path_str is None or path_str == ":memory:":
+        return "memory"
+
+    _, result = adapter.execute("PRAGMA database_list", fetch=True)
+    # Rows are (seq, name, file).  Column order is stable in DuckDB.
+    # System databases to skip: 'system' and 'temp'.
+    _SKIP = frozenset({"system", "temp"})
+
+    abs_path = str(Path(path_str).resolve())
+    candidates: list[str] = []
+    for row in result.rows:
+        name = str(row[1])
+        file_col = str(row[2])
+        if name in _SKIP:
+            continue
+        # Prefer exact match on resolved absolute path.
+        if file_col == abs_path:
+            return name
+        candidates.append(name)
+
+    if candidates:
+        return candidates[0]
+
+    raise ValueError(
+        f"PRAGMA database_list returned no non-system database for path {path_str!r}. "
+        "This is unexpected for a live DuckDB file-backed adapter."
+    )

dataface/core/inspect/templates/__init__.py

@@ -0,0 +1 @@
+# Profile dashboard templates

dataface/core/inspect/templates/categorical_column.yml

@@ -0,0 +1,196 @@
+# Categorical Column Profile Template
+#
+# Analysis of a categorical column with low cardinality. Schema panel
+# pulls from the LayeredSchemaResolver; live aggregations stay on
+# warehouse SQL.
+#
+# URL: /inspect/categorical_column/?model=<table>&column=<col>
+#
+# Required variables:
+#   - model: Table/model name
+#   - column: Column name to analyze
+#
+# Optional variables:
+#   - connection: DuckDB file path or :memory: (default: :memory:)
+#
+# Theme is set via DFT_DEFAULT_THEME env var at serve startup.
+
+title: "Categorical: {{ model }}.{{ column }}"
+
+text: |
+  [Back to {{ model }}](/inspect/model/?model={{ model }}&connection={{ connection }})
+
+variables:
+  model:
+    input: text
+    visible: false
+    default: ""
+  column:
+    input: text
+    visible: false
+    default: ""
+  connection:
+    input: text
+    visible: false
+    default: ":memory:"
+  source_name:
+    input: text
+    visible: false
+    default: ""
+  schema_name:
+    input: text
+    visible: false
+    default: ""
+
+queries:
+  column_schema:
+    type: schema_resolver
+    source: "{{ source_name }}"
+    schema: "{{ schema_name }}"
+    table: "{{ model }}"
+    column: "{{ column }}"
+
+  # Basic category stats
+  stats:
+    source:
+      type: duckdb
+      path: "{{ connection }}"
+    sql: |
+      SELECT
+        COUNT(*) as total_count,
+        COUNT({{ column }}) as non_null_count,
+        COUNT(*) - COUNT({{ column }}) as null_count,
+        ROUND(100.0 * (COUNT(*) - COUNT({{ column }})) / COUNT(*), 2) as null_pct,
+        COUNT(DISTINCT {{ column }}) as category_count
+      FROM {{ model }}
+
+  # Category distribution
+  categories:
+    source:
+      type: duckdb
+      path: "{{ connection }}"
+    sql: |
+      SELECT
+        COALESCE(CAST({{ column }} AS VARCHAR), '(null)') as category,
+        COUNT(*) as count,
+        ROUND(100.0 * COUNT(*) / (SELECT COUNT(*) FROM {{ model }}), 2) as pct
+      FROM {{ model }}
+      GROUP BY {{ column }}
+      ORDER BY COUNT(*) DESC
+
+  # Category distribution for pie chart (top 10 + other)
+  categories_pie:
+    source:
+      type: duckdb
+      path: "{{ connection }}"
+    sql: |
+      WITH ranked AS (
+        SELECT
+          COALESCE(CAST({{ column }} AS VARCHAR), '(null)') as category,
+          COUNT(*) as count,
+          ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) as rn
+        FROM {{ model }}
+        GROUP BY {{ column }}
+      )
+      SELECT
+        CASE WHEN rn <= 10 THEN category ELSE 'Other' END as category,
+        SUM(count) as count
+      FROM ranked
+      GROUP BY CASE WHEN rn <= 10 THEN category ELSE 'Other' END
+      ORDER BY SUM(count) DESC
+
+charts:
+  schema_panel:
+    title: Schema
+    type: table
+    query: column_schema
+    style:
+      header_overflow: wrap-two
+      columns:
+        name:
+          label: Column
+        type:
+          label: Type
+        role:
+          label: Role
+        semantic_type:
+          label: Semantic
+        description:
+          label: Description
+
+  # Stats KPIs
+  total:
+    label: Total Records
+    type: kpi
+    query: stats
+    value: total_count
+    format: ",.0f"
+
+  categories:
+    label: Categories
+    type: kpi
+    query: stats
+    value: category_count
+    format: ",.0f"
+
+  null_pct:
+    label: "Null %"
+    type: kpi
+    query: stats
+    value: null_pct
+    format: ".1f"
+
+  # Pie chart
+  pie_chart:
+    title: Category Distribution
+    type: pie
+    query: categories_pie
+    theta: count
+    color: category
+
+  # Spark bar (compact distribution for profiler cards)
+  spark_bar_chart:
+    title: Distribution
+    type: spark_bar
+    query: categories
+    x: count
+    y: category
+
+  # Bar chart
+  bar_chart:
+    title: Category Counts
+    type: bar
+    query: categories
+    x: category
+    y: count
+    style:
+      orientation: horizontal
+
+  # Category table
+  category_table:
+    title: All Categories
+    type: table
+    query: categories
+    style:
+      header_overflow: wrap-two
+      columns:
+        category:
+          label: Category
+        count:
+          label: Count
+          format: ",.0f"
+        pct:
+          label: "%"
+          format: ".1f"
+
+rows:
+  - schema_panel
+  - cols:
+      - total
+      - categories
+      - null_pct
+  - cols:
+      - pie_chart
+      - spark_bar_chart
+  - bar_chart
+  - category_table

dataface/core/inspect/templates/charts.yml

@@ -0,0 +1,109 @@
+# Auto-Generated Charts Template
+#
+# Dynamically generates appropriate visualizations for each column type using
+# the foreach construct and board file imports.
+#
+# URL: /inspect/charts/?model=<table>
+#
+# Required variables:
+#   - model: Table/model name
+#
+# Optional variables:
+#   - connection: DuckDB file path or :memory: (default: :memory:)
+#
+# This template demonstrates the foreach + board imports pattern:
+# 1. Queries fetch column metadata grouped by type
+# 2. foreach iterates over each column
+# 3. Board imports load type-specific partials (numeric.yml, date.yml, etc.)
+# 4. Partials inherit parent variables (model, column, connection)
+#
+# Theme is set via DFT_DEFAULT_THEME env var at serve startup.
+
+title: "Charts: {{ model }}"
+
+
+variables:
+  model:
+    input: text
+    visible: false
+    default: ""
+  connection:
+    input: text
+    visible: false
+    default: ":memory:"
+
+queries:
+  # Get numeric columns for histograms
+  numeric_columns:
+    source:
+      type: duckdb
+      path: "{{ connection }}"
+    sql: |
+      SELECT column_name as column, 'numeric' as type
+      FROM information_schema.columns
+      WHERE table_name = '{{ model }}'
+        AND data_type IN ('INTEGER', 'BIGINT', 'SMALLINT', 'TINYINT',
+                          'DOUBLE', 'FLOAT', 'REAL', 'DECIMAL', 'NUMERIC')
+      ORDER BY ordinal_position
+    # Static data for compile-time iteration (used by foreach)
+    static_data: []
+
+  # Get date columns for time series
+  date_columns:
+    source:
+      type: duckdb
+      path: "{{ connection }}"
+    sql: |
+      SELECT column_name as column, 'date' as type
+      FROM information_schema.columns
+      WHERE table_name = '{{ model }}'
+        AND data_type IN ('DATE', 'TIMESTAMP', 'DATETIME', 'TIMESTAMPTZ')
+      ORDER BY ordinal_position
+    static_data: []
+
+  # Get categorical columns for bar charts
+  categorical_columns:
+    source:
+      type: duckdb
+      path: "{{ connection }}"
+    sql: |
+      SELECT column_name as column, 'categorical' as type
+      FROM information_schema.columns
+      WHERE table_name = '{{ model }}'
+        AND data_type IN ('VARCHAR', 'CHAR', 'TEXT', 'STRING', 'BOOLEAN')
+      ORDER BY ordinal_position
+    static_data: []
+
+rows:
+  # Numeric column histograms
+  - title: "Numeric Columns"
+    text: |
+      Distribution histograms for numeric columns.
+    rows:
+      - foreach:
+          query: numeric_columns
+          as: col
+          items:
+            - ../partials/numeric.yml
+
+  # Date column time series
+  - title: "Date Columns"
+    text: |
+      Time series for date/timestamp columns.
+    rows:
+      - foreach:
+          query: date_columns
+          as: col
+          items:
+            - ../partials/date.yml
+
+  # Categorical column bar charts
+  - title: "Categorical Columns"
+    text: |
+      Distribution for categorical columns.
+    rows:
+      - foreach:
+          query: categorical_columns
+          as: col
+          items:
+            - ../partials/categorical.yml

dataface/core/inspect/templates/date_column.yml

@@ -0,0 +1,248 @@
+# Date Column Profile Template
+#
+# Analysis of a date/timestamp column. Schema panel pulls from the
+# LayeredSchemaResolver; live time-series, gap, and distribution panels
+# stay on warehouse SQL.
+#
+# URL: /inspect/date_column/?model=<table>&column=<col>
+#
+# Required variables:
+#   - model: Table/model name
+#   - column: Column name to analyze
+#
+# Optional variables:
+#   - connection: DuckDB file path or :memory: (default: :memory:)
+#
+# Theme is set via DFT_DEFAULT_THEME env var at serve startup.
+
+title: "Date Column: {{ model }}.{{ column }}"
+
+text: |
+  [Back to {{ model }}](/inspect/model/?model={{ model }}&connection={{ connection }})
+
+variables:
+  model:
+    input: text
+    visible: false
+    default: ""
+  column:
+    input: text
+    visible: false
+    default: ""
+  connection:
+    input: text
+    visible: false
+    default: ":memory:"
+  source_name:
+    input: text
+    visible: false
+    default: ""
+  schema_name:
+    input: text
+    visible: false
+    default: ""
+
+queries:
+  column_schema:
+    type: schema_resolver
+    source: "{{ source_name }}"
+    schema: "{{ schema_name }}"
+    table: "{{ model }}"
+    column: "{{ column }}"
+
+  # Basic date stats
+  stats:
+    source:
+      type: duckdb
+      path: "{{ connection }}"
+    sql: |
+      SELECT
+        COUNT(*) as total_count,
+        COUNT({{ column }}) as non_null_count,
+        COUNT(*) - COUNT({{ column }}) as null_count,
+        ROUND(100.0 * (COUNT(*) - COUNT({{ column }})) / COUNT(*), 2) as null_pct,
+        MIN({{ column }})::DATE as min_date,
+        MAX({{ column }})::DATE as max_date,
+        COUNT(DISTINCT {{ column }}::DATE) as distinct_dates,
+        MAX({{ column }})::DATE - MIN({{ column }})::DATE as date_span_days
+      FROM {{ model }}
+
+  # Records per day
+  daily_counts:
+    source:
+      type: duckdb
+      path: "{{ connection }}"
+    sql: |
+      SELECT
+        {{ column }}::DATE as date,
+        COUNT(*) as count
+      FROM {{ model }}
+      WHERE {{ column }} IS NOT NULL
+      GROUP BY {{ column }}::DATE
+      ORDER BY date
+
+  # Day of week distribution
+  dow_distribution:
+    source:
+      type: duckdb
+      path: "{{ connection }}"
+    sql: |
+      SELECT
+        CASE DAYOFWEEK({{ column }})
+          WHEN 0 THEN 'Sunday'
+          WHEN 1 THEN 'Monday'
+          WHEN 2 THEN 'Tuesday'
+          WHEN 3 THEN 'Wednesday'
+          WHEN 4 THEN 'Thursday'
+          WHEN 5 THEN 'Friday'
+          WHEN 6 THEN 'Saturday'
+        END as day_of_week,
+        DAYOFWEEK({{ column }}) as dow_num,
+        COUNT(*) as count
+      FROM {{ model }}
+      WHERE {{ column }} IS NOT NULL
+      GROUP BY DAYOFWEEK({{ column }})
+      ORDER BY DAYOFWEEK({{ column }})
+
+  # Monthly distribution
+  monthly_counts:
+    source:
+      type: duckdb
+      path: "{{ connection }}"
+    sql: |
+      SELECT
+        DATE_TRUNC('month', {{ column }})::DATE as month,
+        COUNT(*) as count
+      FROM {{ model }}
+      WHERE {{ column }} IS NOT NULL
+      GROUP BY DATE_TRUNC('month', {{ column }})
+      ORDER BY month
+
+  # Date gaps (missing dates)
+  date_gaps:
+    source:
+      type: duckdb
+      path: "{{ connection }}"
+    sql: |
+      WITH date_range AS (
+        SELECT GENERATE_SERIES(
+          MIN({{ column }})::DATE,
+          MAX({{ column }})::DATE,
+          INTERVAL '1 day'
+        )::DATE as expected_date
+        FROM {{ model }}
+        WHERE {{ column }} IS NOT NULL
+      ),
+      actual_dates AS (
+        SELECT DISTINCT {{ column }}::DATE as actual_date
+        FROM {{ model }}
+        WHERE {{ column }} IS NOT NULL
+      )
+      SELECT
+        expected_date as missing_date
+      FROM date_range
+      WHERE expected_date NOT IN (SELECT actual_date FROM actual_dates)
+      ORDER BY expected_date
+      LIMIT 50
+
+charts:
+  schema_panel:
+    title: Schema
+    type: table
+    query: column_schema
+    style:
+      header_overflow: wrap-two
+      columns:
+        name:
+          label: Column
+        type:
+          label: Type
+        role:
+          label: Role
+        semantic_type:
+          label: Semantic
+        description:
+          label: Description
+
+  # Stats KPIs
+  min_date:
+    label: Earliest Date
+    type: kpi
+    query: stats
+    value: min_date
+
+  max_date:
+    label: Latest Date
+    type: kpi
+    query: stats
+    value: max_date
+
+  distinct_dates:
+    label: Distinct Dates
+    type: kpi
+    query: stats
+    value: distinct_dates
+    format: ",.0f"
+
+  date_span:
+    label: Date Span (Days)
+    type: kpi
+    query: stats
+    value: date_span_days
+    format: ",.0f"
+
+  null_pct:
+    label: "Null %"
+    type: kpi
+    query: stats
+    value: null_pct
+    format: ".1f"
+
+  # Time series
+  daily_chart:
+    title: Records per Day
+    type: line
+    query: daily_counts
+    x: date
+    y: count
+
+  # Monthly trend
+  monthly_chart:
+    title: Records per Month
+    type: bar
+    query: monthly_counts
+    x: month
+    y: count
+
+  # Day of week
+  dow_chart:
+    title: Day of Week Distribution
+    type: bar
+    query: dow_distribution
+    x: day_of_week
+    y: count
+
+  # Gaps table
+  gaps_table:
+    title: Missing Dates (first 50)
+    type: table
+    query: date_gaps
+    style:
+      header_overflow: wrap-two
+      columns:
+        missing_date:
+          label: Missing Date
+
+rows:
+  - schema_panel
+  - cols:
+      - min_date
+      - max_date
+      - distinct_dates
+      - date_span
+      - null_pct
+  - daily_chart
+  - cols:
+      - monthly_chart
+      - dow_chart
+  - gaps_table