dataface 0.3.2.dev111__py3-none-any.whl → 0.3.2.dev150__py3-none-any.whl

dataface/core/diagnostics/codes_render.py

@@ -3,9 +3,10 @@
 Error codes: KPI multirow, bar duplicate rows, map key mismatch, format
 unsupported, no layout, input invalid, Vega-Lite unsupported type, histogram
 non-numeric, histogram pre-aggregated, label/ticks validation, percent range,
-emitter not found, labels field not found, scale domain, concat overshoot.
+emitter not found, labels field not found, scale domain, concat overshoot,
+chart painted no marks.
 
-Warning codes: the 14 render-time detector codes, declared here (not in the
+Warning codes: every render-time detector code, declared here (not in the
 detector modules, which sit above this leaf) so the registry is complete on
 import of core.diagnostics. Detectors import their constants back from this
 package.
@@ -162,7 +163,7 @@ ERR_DUPLICATE_CHART_ID = REGISTRY.register(
         ),
         doc=(
             "Chart ids are unique within a face but not across a face tree: two "
-            "imported partials, or several foreach iterations of one partial, can "
+            "imported partials, including the same partial imported twice, can "
             "declare the same id. The flat output formats key charts by id, so a "
             "collision would silently drop every chart but the last. Rename the "
             "colliding chart, or render with a format that preserves the layout."
@@ -530,9 +531,31 @@ ERR_BOARD_RECORDING_MISMATCH = REGISTRY.register(
     )
 )
 
+ERR_CHART_PAINTED_NO_MARKS = REGISTRY.register(
+    ErrorCode(
+        code="ERR-CHART-PAINTED-NO-MARKS",
+        domain="render",
+        title="Chart received rows but painted no marks",
+        message_template=(
+            "Chart {chart_id!r} received {row_count} row(s) but every mark it drew "
+            "has zero width or height — check whether the x/y fields and scale "
+            "types match the data's actual shape."
+        ),
+        doc=(
+            "Fired when a plotting-family chart's query returns at least one row "
+            "but the rendered SVG contains no mark with visible extent — every "
+            "bar, line, area, point, wedge, or shape it drew is degenerate. "
+            "`WARN-QUERY-RETURNED-ZERO-ROWS` covers the honest empty case (no "
+            "rows); this covers the dishonest one — rows arrived, the renderer "
+            "just didn't paint anything visible with them."
+        ),
+        docs_topic="charts",
+    )
+)
+
 
 # ──────────────────────────────── Warning codes ───────────────────────────────
-# All 14 render-time warning codes declared here in the leaf; detectors
+# Every render-time warning code is declared here in the leaf; detectors
 # import their constants back rather than declaring codes in-module.
 
 WARN_BAR_BAND_WIDTH_TOO_NARROW = REGISTRY.register(
@@ -586,6 +609,37 @@ WARN_LAYERED_CHART_SHARED_Y_AXIS_SCALE_MISMATCH = REGISTRY.register(
     )
 )
 
+WARN_LAYOUT_MIN_EXCEEDS_HEIGHT = REGISTRY.register(
+    WarningCode(
+        code="WARN-LAYOUT-MIN-EXCEEDS-HEIGHT",
+        domain="render",
+        title="Chart's category count needs more height than its row allows",
+        message_template=(
+            "Chart {chart_id!r} has {n_categories} category bands, which need "
+            "at least {min_height:.0f}px to stay readable, but its row/tile "
+            "height caps it at {authored_height:.0f}px. The chart rendered at "
+            "{min_height:.0f}px anyway — the row's authored height was not honored, "
+            "and any sibling charts sharing the row grew to match."
+        ),
+        fix_template=(
+            "Raise the row's height to fit the category count, or reduce the "
+            "categories (filter, paginate, or roll up to a coarser grain) so "
+            "the readable minimum fits inside the authored height."
+        ),
+        doc=(
+            "Fires when a horizontal bar chart's category count forces a "
+            "minimum height — one readable band per category — that exceeds "
+            "the row/tile height its author assigned. The engine expands the "
+            "chart to the computed minimum regardless (squashing labels past "
+            "legibility to honor an impossible authored height would be worse), "
+            "so the row's authored height silently loses; any sibling chart in "
+            "the same row inherits the expansion. This warns so the author "
+            "learns why the row grew, instead of measuring it by hand."
+        ),
+        docs_topic="charts",
+    )
+)
+
 WARN_LIKELY_CURRENCY_OR_PERCENT_MISSING_FORMATTER = REGISTRY.register(
     WarningCode(
         code="WARN-LIKELY-CURRENCY-OR-PERCENT-MISSING-FORMATTER",

dataface/core/execute/adapters/__init__.py

@@ -10,7 +10,7 @@ Available Adapters:
 
 MetricFlow queries lower to SqlQuery at compile time (normalize_queries.py) by
 calling MetricFlow's own compiler against the dbt semantic manifest — there is
-no execute-time MetricFlow adapter (mirroring the LookML compile-time lowering seam).
+no execute-time MetricFlow adapter.
 """
 
 from dataface.core.execute.adapters.adapter_registry import (

dataface/core/execute/cache_keys.py

@@ -43,20 +43,15 @@ SECRET_FIELDS: frozenset[str] = frozenset(
     }
 )
 
-# Fields that describe caching policy, LookML explore scoping, or cost
-# attribution — not connection identity. Dropped so editing them doesn't
+# Fields that describe caching policy or cost attribution — not connection
+# identity. Dropped so editing them doesn't
 # cold-restart the whole source's cache (every cache entry keyed off that
 # source_hash would otherwise miss).
 # `cache`: the source's own authored cache: block (BaseSourceConfig.cache).
-# `models`/`lookml_path`: LookML model/explore grouping metadata
-# (LookMLModelsMixin) — it selects which explores a query can name, but does
-# not change the warehouse connection or the SQL a query executes.
 # `attribution`: cost-attribution metadata (BaseSourceConfig.attribution) — it
 # rides along to the warehouse as job labels but changes neither the connection
 # nor the rows a query returns.
-INCIDENTAL_FIELDS: frozenset[str] = frozenset(
-    {"cache", "models", "lookml_path", "attribution"}
-)
+INCIDENTAL_FIELDS: frozenset[str] = frozenset({"cache", "attribution"})
 
 
 def source_identity(
@@ -71,7 +66,7 @@ def source_identity(
     Identity-bearing fields (user, host, project, dataset, keyfile, etc.)
     are preserved so two users with the same source name but different
     credentials get distinct cache entries. Incidental fields (the source's
-    own ``cache:`` block, LookML ``models:``/``lookml_path``) are also
+    own ``cache:`` block, cost attribution) are also
     dropped — see ``INCIDENTAL_FIELDS``.
 
     Args:

dataface/core/execute/executor.py

@@ -61,12 +61,14 @@ from dataface.core.compile.variables import (
     coerce_variable_values,
     parse_variable_json_strings,
 )
+from dataface.core.diagnostics.base import DatafaceError
 from dataface.core.diagnostics.execution import ExecutionError, QueryError
 from dataface.core.execute.cache_backend import (
     CachedQueryFailure,
     CacheRows,
 )
 from dataface.core.execute.duckdb_cache import compute_cache_key
+from dataface.core.project import is_glob
 
 if TYPE_CHECKING:
     from dataface.core.compile.models.source import (
@@ -439,9 +441,15 @@ class Executor:
                 # Type narrower: _has_file_support (checked above) guarantees a
                 # non-None result, but the checker can't connect the two.
                 assert materializer is not None
+                # _resolve_file_source (above) only returns non-None when
+                # query.source is set — the assertion surfaces any future drift.
+                assert query.source is not None
                 try:
                     result_data = materializer.materialize_and_run(
-                        file_source, query.sql, variables or {}
+                        file_source,
+                        query.sql,
+                        variables or {},
+                        source_name=query.source,
                     )
                 except Exception as e:
                     if should_use_cache:
@@ -454,7 +462,11 @@ class Executor:
                             False,
                         )
                     )
-                    raise QueryError(str(e), query_name) from e
+                    raise QueryError(
+                        str(e),
+                        query_name,
+                        code=e.code if isinstance(e, DatafaceError) else None,
+                    ) from e
                 data_as_of = datetime.now(timezone.utc)
                 # Write through like any other query: the cache key now folds in the
                 # file's version token (Project.file_version), so an edited data file
@@ -888,9 +900,19 @@ class Executor:
         # Pre-hash each file's token to a fixed hex digest before joining, so the
         # combination is independent of any delimiter appearing inside a token
         # (the Project.file_version contract does not restrict the token alphabet).
+        # Glob patterns must be expanded first — file_version on a literal glob
+        # path would fail with FileNotFoundError on a local project.
+        relpaths: list[str] = []
+        for path_or_glob in file_source.files.values():
+            if is_glob(path_or_glob):
+                relpaths.extend(
+                    sorted(p.relpath for p in project.files.glob(path_or_glob))
+                )
+            else:
+                relpaths.append(path_or_glob)
         parts = sorted(
             hashlib.sha256(project.file_version(relpath).encode()).hexdigest()
-            for relpath in file_source.files.values()
+            for relpath in relpaths
         )
         version = hashlib.sha256("|".join(parts).encode()).hexdigest()[:16]
         self._source_versions[source_name] = version

dataface/core/execute/file_source_materializer.py

@@ -13,6 +13,11 @@ Cache key per file table:
                      cheap, so a warm table skips the read entirely.
     variables_hash = FILE_SOURCE_VARS_HASH  (file sources have no variables)
 
+For glob patterns (paths containing ``*`` or ``?``), the files: value is expanded
+via ``project.files.glob()``. The combined version key encodes the full set of
+matched relpaths and their individual version tokens, so the cache invalidates
+when any matched file changes OR when files are added/removed from the glob set.
+
 On a cache hit the table/view already exists and the file is never read; on a miss
 we read + parse + put + register. When the cache is a DuckDB-backed cache,
 disable_external_access() is called once at construction so author SQL cannot call
@@ -26,21 +31,32 @@ import io
 import logging
 import threading
 from collections.abc import Callable
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, TypeAlias
 
+from dataface.core.compile.config import get_execution_config
 from dataface.core.compile.models.source import (
     CsvSourceConfig,
     JsonSourceConfig,
     ParquetSourceConfig,
 )
 from dataface.core.compile.parameterized import render_parameterized
+from dataface.core.diagnostics.base import DatafaceError
+from dataface.core.diagnostics.codes_execute import (
+    ERR_GLOB_EMPTY,
+    ERR_GLOB_SCHEMA_MISMATCH,
+    ERR_GLOB_TOO_MANY,
+)
 from dataface.core.execute.cache_backend import FILE_SOURCE_VARS_HASH, CacheHit
 from dataface.core.execute.duckdb_cache import compute_source_hash
+from dataface.core.project import is_glob
 
 if TYPE_CHECKING:
     from dataface.core.execute.cache_backend import QueryResultCache
     from dataface.core.project import Project
 
+# JSON files carry runtime-dynamic column schemas; Any is the correct boundary type.
+_JsonRow: TypeAlias = dict[str, Any]
+
 logger = logging.getLogger(__name__)
 
 
@@ -73,6 +89,18 @@ def _version_key(token: str) -> str:
     return hashlib.sha256(token.encode()).hexdigest()[:16]
 
 
+def _glob_vkey(sorted_pairs: list[tuple[str, str]]) -> str:
+    """Composite version key for a glob table.
+
+    Encodes both the set of matched relpaths and their per-file version tokens
+    so the cache invalidates when any file changes content, is added, or is
+    removed from the glob match set.  ``sorted_pairs`` must be pre-sorted by
+    relpath (``project.files.glob`` yields sorted order).
+    """
+    combined = "\n".join(f"{relpath}:{vk}" for relpath, vk in sorted_pairs)
+    return hashlib.sha256(combined.encode()).hexdigest()[:16]
+
+
 class FileSourceMaterializer:
     """Materializes file-based sources into any QueryResultCache backend.
 
@@ -106,45 +134,124 @@ class FileSourceMaterializer:
         source: CsvSourceConfig | JsonSourceConfig | ParquetSourceConfig,
         sql: str,
         variables: dict[str, Any],
-    ) -> list[dict[str, Any]]:
+        source_name: str,
+    ) -> list[_JsonRow]:
         """Ensure all tables in *source* are in the cache, then execute *sql*.
 
+        Glob patterns (``*`` / ``?``) in ``source.files`` values are expanded via
+        ``project.files.glob()``.  Each glob must match at least one file and may
+        not exceed ``execution.max_glob_file_count``.  All matched files are loaded
+        and their rows concatenated into one table.
+
         Args:
             source: A CsvSourceConfig, JsonSourceConfig, or ParquetSourceConfig.
             sql: SQL template (may contain {{ variable }} Jinja expressions).
             variables: Variable values substituted into *sql* via render_parameterized.
+            source_name: Authored source name (for error messages).
 
         Returns:
             Rows returned by *sql*.
 
         Raises:
+            DatafaceError: Empty glob match or fan-out cap exceeded.
             RuntimeError: If SQL execution fails (propagated from execute_file_source_sql,
                 includes external-access violations in author SQL).
         """
         source_hash = compute_source_hash(source)
-
-        # Cheap per-table version token (a stat/OID lookup, NO file read). This
-        # is the file-identity for the cache key; the file itself is read only on
-        # a miss, inside the lock, so it is loaded at most once per render.
-        versions: list[tuple[str, str, str]] = [
-            (table_name, relpath, _version_key(self._project.file_version(relpath)))
-            for table_name, relpath in source.files.items()
-        ]
+        cap = get_execution_config().max_glob_file_count
+
+        # Each entry: (table_name, vkey, sorted_relpaths).
+        # For literal paths: one relpath, vkey from that file alone.
+        # For globs: sorted relpaths from glob expansion, composite vkey.
+        # Glob expansion and cap/empty checks happen here — before the lock,
+        # before any file reads, so a bad glob fails fast.
+        table_specs: list[tuple[str, str, list[str]]] = []
+        for table_name, path_or_glob in source.files.items():
+            if is_glob(path_or_glob):
+                matched = sorted(
+                    p.relpath for p in self._project.files.glob(path_or_glob)
+                )
+                if not matched:
+                    raise DatafaceError.from_code(
+                        ERR_GLOB_EMPTY,
+                        source_name=source_name,
+                        table_name=table_name,
+                        pattern=path_or_glob,
+                    )
+                if len(matched) > cap:
+                    raise DatafaceError.from_code(
+                        ERR_GLOB_TOO_MANY,
+                        source_name=source_name,
+                        table_name=table_name,
+                        pattern=path_or_glob,
+                        count=len(matched),
+                        cap=cap,
+                    )
+                pairs = [
+                    (r, _version_key(self._project.file_version(r))) for r in matched
+                ]
+                vkey = _glob_vkey(pairs)
+                table_specs.append((table_name, vkey, matched))
+            else:
+                vkey = _version_key(self._project.file_version(path_or_glob))
+                table_specs.append((table_name, vkey, [path_or_glob]))
 
         # Render {{ variable }} templates to parameterized SQL before executing.
         # profile_type="duckdb" → $1/$2/… placeholders (DuckDB positional style).
         parameterized = render_parameterized(sql, variables, profile_type="duckdb")
 
+        union_by_name = isinstance(source, JsonSourceConfig) and source.union_by_name
+
         with self._lock:
-            for table_name, relpath, vkey in versions:
-                # Cache miss → read + parse + store. A hit never opens the file.
+            for table_name, vkey, relpaths in table_specs:
+                # Cache miss → read + parse + store. A hit never opens any file.
                 if not isinstance(
                     self._cache.get(source_hash, vkey, FILE_SOURCE_VARS_HASH), CacheHit
                 ):
-                    rows = _parse(source, self._project.read_bytes(relpath))
+                    rows: list[_JsonRow] = []
+                    first_relpath = ""
+                    first_keys: frozenset[str] = frozenset()
+                    for relpath in relpaths:
+                        file_rows = _parse(
+                            source, self._project.read_bytes(relpath), relpath
+                        )
+                        if file_rows:
+                            # Schema check uses only the first row's keys.  This catches
+                            # header-level mismatches (the common authoring mistake) but
+                            # will not detect a mid-file column shift in irregular formats.
+                            file_keys = frozenset(file_rows[0])
+                            if not first_relpath:
+                                first_keys = file_keys
+                                first_relpath = relpath
+                            elif not union_by_name and file_keys != first_keys:
+                                extra = sorted(file_keys - first_keys)
+                                missing = sorted(first_keys - file_keys)
+                                detail = (f"Extra: {extra}. " if extra else "") + (
+                                    f"Missing: {missing}. " if missing else ""
+                                )
+                                raise DatafaceError.from_code(
+                                    ERR_GLOB_SCHEMA_MISMATCH,
+                                    source_name=source_name,
+                                    table_name=table_name,
+                                    path=relpath,
+                                    first_path=first_relpath,
+                                    detail=detail,
+                                )
+                            rows.extend(file_rows)
+                    if union_by_name and rows:
+                        # Union of column names in first-seen insertion order so
+                        # the created table's column order is deterministic across
+                        # processes (a set would vary under hash randomisation).
+                        all_keys = dict.fromkeys(k for row in rows for k in row)
+                        for row in rows:
+                            for k in all_keys:
+                                row.setdefault(k, None)
+                    # Individual glob-matched files with zero rows are silently
+                    # skipped (common for empty partitions); the error fires only
+                    # when *all* matched files together contribute no rows.
                     if not rows:
                         raise ValueError(
-                            f"File table '{table_name}': parsed file contains no rows. "
+                            f"File table {table_name!r}: parsed file(s) contain no rows. "
                             "Empty file sources are not supported — "
                             "provide a file with at least one data row."
                         )
@@ -171,19 +278,19 @@ class FileSourceMaterializer:
 
 
 def _parse(
-    source: CsvSourceConfig | JsonSourceConfig | ParquetSourceConfig, file_bytes: bytes
-) -> list[dict[str, Any]]:
+    source: CsvSourceConfig | JsonSourceConfig | ParquetSourceConfig,
+    file_bytes: bytes,
+    relpath: str,
+) -> list[_JsonRow]:
     """Dispatch to the correct parser for the given source type."""
     if isinstance(source, CsvSourceConfig):
         return _parse_csv(file_bytes, source.delimiter, source.encoding)
     if isinstance(source, JsonSourceConfig):
-        return _parse_json(file_bytes)
+        return _parse_json(file_bytes, relpath)
     return _parse_parquet(file_bytes)
 
 
-def _parse_csv(
-    file_bytes: bytes, delimiter: str, encoding: str
-) -> list[dict[str, Any]]:
+def _parse_csv(file_bytes: bytes, delimiter: str, encoding: str) -> list[_JsonRow]:
     """Parse CSV bytes to list[dict] using PyArrow's CSV reader.
 
     PyArrow infers column types (int, float, string, bool) — no-magic violation
@@ -199,30 +306,76 @@ def _parse_csv(
     return table.to_pylist()
 
 
-def _parse_json(file_bytes: bytes) -> list[dict[str, Any]]:
-    """Parse JSON bytes to list[dict].
+def _parse_json(
+    file_bytes: bytes,
+    relpath: str,
+) -> list[_JsonRow]:
+    """Parse JSON or NDJSON bytes to list[dict].
 
-    Accepts a top-level JSON array of objects (the common authored case).
-    Raises ValueError for non-array shapes (dicts, scalars) — use a warehouse
-    connection for complex JSON shapes.
+    The parser is chosen by extension: ``.jsonl`` → NDJSON, ``.json`` → standard
+    JSON.  The extension is validated at compile time, so only these two values
+    are possible at runtime.
+
+    Raises ValueError when the file cannot be parsed in the selected format.
     """
     import json
 
+    if relpath.lower().endswith(".jsonl"):
+        return _parse_ndjson(file_bytes)
+
+    try:
+        text = file_bytes.decode("utf-8")
+    except UnicodeDecodeError as e:
+        raise ValueError(f"JSON file is not valid UTF-8: {e}") from e
+
     try:
-        parsed = json.loads(file_bytes.decode("utf-8"))
-    except (json.JSONDecodeError, UnicodeDecodeError) as e:
-        raise ValueError(f"JSON file is not valid UTF-8 JSON: {e}") from e
+        parsed = json.loads(text)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"JSON file is not valid JSON: {e}") from e
 
+    if isinstance(parsed, dict):
+        return [parsed]
     if not isinstance(parsed, list):
         raise ValueError(
-            f"JSON file must contain a top-level array of objects, "
-            f"got: {type(parsed).__name__!r}. "
-            "Use a warehouse connection for complex JSON shapes."
+            f"JSON file must contain a top-level object or array of objects, "
+            f"got: {type(parsed).__name__!r}."
         )
     return parsed
 
 
-def _parse_parquet(file_bytes: bytes) -> list[dict[str, Any]]:
+def _parse_ndjson(file_bytes: bytes) -> list[_JsonRow]:
+    """Parse newline-delimited JSON (NDJSON) bytes to list[dict].
+
+    Each non-empty line must be a JSON object ``{...}``.
+
+    Raises ValueError on any line that is not a valid JSON object.
+    """
+    import json
+
+    try:
+        text = file_bytes.decode("utf-8")
+    except UnicodeDecodeError as e:
+        raise ValueError(f"NDJSON file is not valid UTF-8: {e}") from e
+
+    rows: list[_JsonRow] = []
+    for lineno, line in enumerate(text.splitlines(), start=1):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            obj = json.loads(line)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"NDJSON line {lineno} is not valid JSON: {e}") from e
+        if not isinstance(obj, dict):
+            raise ValueError(
+                f"NDJSON line {lineno} must be a JSON object, "
+                f"got: {type(obj).__name__!r}."
+            )
+        rows.append(obj)
+    return rows
+
+
+def _parse_parquet(file_bytes: bytes) -> list[_JsonRow]:
     """Parse Parquet bytes to list[dict] using PyArrow."""
     import pyarrow.parquet as pq
 

dataface/core/execute/trivial_local_cache.py

@@ -208,9 +208,12 @@ class TrivialDuckDBCache(_DuckDBResultCacheBase):
     def _ensure_result_table(self, table_name: str, data: list[dict[str, Any]]) -> None:
         if self._table_exists(table_name):
             return
-        sample = data[0]
+        # Scan all rows for the first non-None value per column so union_by_name
+        # padding (which fills row 0 with None for absent columns) doesn't collapse
+        # numeric columns to VARCHAR.
         cols = [
-            f'"{col_name}" {_infer_type(value)}' for col_name, value in sample.items()
+            f'"{col_name}" {_infer_type(next((r[col_name] for r in data if r.get(col_name) is not None), None))}'
+            for col_name in data[0]
         ]
         self.conn.execute(f"CREATE TABLE {_q(table_name)} ({', '.join(cols)})")
 

dataface/core/inspect/__init__.py

@@ -23,7 +23,6 @@ if TYPE_CHECKING:
 # Template files live in dataface/core/inspect/templates/{name}.yml
 INSPECT_TEMPLATES: list[str] = [
     "categorical_column",
-    "charts",
     "date_column",
     "model",
     "numeric_column",

dataface/core/project.py

@@ -126,7 +126,6 @@ IMAGE_ASSET_SUFFIXES: tuple[str, ...] = (
 #                  faces (.md/.markdown); composed in so this set can never
 #                  drift behind a new face suffix.
 #   IMAGE_ASSET_SUFFIXES — images embeddable from markdown/faces (see above).
-#   .lkml      — LookML models (LookMLModelsMixin.lookml_path)
 #   .csv/.json/.parquet — file sources (CsvSourceConfig etc.), whose `files:`
 #                  paths are required to carry these exact extensions
 #                  (models/source.py `_validate_files_mapping`)
@@ -136,7 +135,7 @@ IMAGE_ASSET_SUFFIXES: tuple[str, ...] = (
 SERVED_FILE_SUFFIXES: frozenset[str] = (
     frozenset(FACE_CANDIDATE_SUFFIXES)
     | frozenset(IMAGE_ASSET_SUFFIXES)
-    | {".lkml", ".csv", ".json", ".parquet"}
+    | {".csv", ".json", ".jsonl", ".parquet"}
 )
 
 
@@ -191,6 +190,11 @@ def glob_to_regex(pattern: str) -> re.Pattern[str]:
     return re.compile("".join(out) + r"\Z")
 
 
+def is_glob(path: str) -> bool:
+    """True when *path* contains a shell glob character (``*`` or ``?``)."""
+    return "*" in path or "?" in path
+
+
 def glob_literal_prefix(pattern: str) -> str:
     """The leading wildcard-free directory segments of a glob pattern.