dataface 0.3.2.dev111__py3-none-any.whl → 0.3.2.dev150__py3-none-any.whl

dataface/core/compile/models/source.py

@@ -111,29 +111,14 @@ def reject_credential_literals(source_name: str, raw_config: Mapping[str, Any])
         )
 
 
-def _validate_relative_path(class_name: str, field_label: str, value: str) -> str:
-    """Reject an empty, absolute, or root-escaping project-relative path.
-
-    Delegates the escape rule to ``assert_relpath`` — the single project-relative
-    path guard — and re-raises with the class/field context. Shared by every field
-    that stores a project-relative filesystem path (LookML model/lookml_path fields
-    today).
-    """
-    try:
-        assert_relpath(value)
-    except ValueError as e:
-        raise ValueError(f"{class_name}: {field_label} {e}") from e
-    return value
-
-
 def _validate_files_mapping(
-    class_name: str, files: dict[str, str], expected_extension: str
+    class_name: str, files: dict[str, str], expected_extension: str | tuple[str, ...]
 ) -> None:
     """Validate all keys and values in a file source's ``files:`` mapping.
 
     Keys must be valid SQL identifiers (letters, digits, underscores; no
     leading digit). Values must be non-empty relative POSIX paths that do not
-    escape above the project root after normalization, and must end in
+    escape above the project root after normalization, and must end in one of
     ``expected_extension`` (case-insensitive) — parsing dispatches on the
     declared source ``type``, not the file extension, so this is what
     guarantees every servable data file's extension matches its type (and
@@ -143,12 +128,18 @@ def _validate_files_mapping(
     Args:
         class_name: Source config class name (for error messages).
         files: The ``files:`` mapping from the authored YAML.
-        expected_extension: The canonical extension for this source type
-            (e.g. ``".csv"``), including the leading dot.
+        expected_extension: The canonical extension or tuple of extensions for
+            this source type (e.g. ``".csv"`` or ``(".json", ".jsonl")``),
+            including the leading dot.
 
     Raises:
         ValueError: On an invalid key, path value, or extension mismatch.
     """
+    exts = (
+        (expected_extension,)
+        if isinstance(expected_extension, str)
+        else expected_extension
+    )
     for table_name, file_path in files.items():
         if not _SQL_IDENTIFIER_RE.match(table_name):
             raise ValueError(
@@ -160,10 +151,11 @@ def _validate_files_mapping(
             assert_relpath(file_path)
         except ValueError as e:
             raise ValueError(f"{class_name}: files[{table_name!r}] {e}") from e
-        if not file_path.lower().endswith(expected_extension):
+        if not any(file_path.lower().endswith(ext) for ext in exts):
+            ext_msg = repr(exts[0]) if len(exts) == 1 else f"one of {list(exts)}"
             raise ValueError(
                 f"{class_name}: files[{table_name!r}] path {file_path!r} must end in "
-                f"{expected_extension!r} to match source type."
+                f"{ext_msg} to match source type."
             )
 
 
@@ -373,125 +365,12 @@ class DuckDBRetriesConfig(BaseModel):
     ] = Field(default_factory=_default_retryable_exceptions)
 
 
-# ============================================================================
-# LOOKML MODELS (nested under warehouse sources)
-# ============================================================================
-
-
-class LookMLModelConfig(BaseModel):
-    """One LookML model: a named collection of explores under a warehouse source.
-
-    Exactly one selector chooses which explores belong to the collection:
-
-    - ``lkml``: a specific ``.lkml`` model file. Validated as a relative path
-      that does not escape the project root. File-scoping within likeml is not
-      yet implemented, so using this selector raises a clear error at compile
-      time directing authors to use an explicit ``explores:`` list instead.
-      Disk existence is not checked at parse time.
-    - ``group_label``: explores tagged with this label in LookML. Accepted at
-      parse time but likeml carries no group_label metadata yet, so using it
-      raises a clear error at compile time.
-    - ``explores``: an explicit list of explore names. The only selector
-      enforced at compile time: an explore not in the list is rejected.
-    """
-
-    model_config = ConfigDict(extra="forbid")
-
-    lkml: str | None = Field(
-        default=None,
-        description="Path to a specific .lkml model file, relative to the project root.",
-    )
-    group_label: str | None = Field(
-        default=None,
-        description="LookML group_label selecting this model's explores. Not yet enforced at compile time.",
-    )
-    explores: list[str] | None = Field(
-        default=None, description="Explicit list of explore names in this model."
-    )
-
-    @field_validator("lkml")
-    @classmethod
-    def _validate_lkml(cls, v: str | None) -> str | None:
-        if v is None:
-            return v
-        return _validate_relative_path("LookMLModelConfig", "lkml", v)
-
-    @model_validator(mode="after")
-    def _require_one_selector(self) -> LookMLModelConfig:
-        selectors = [
-            name
-            for name, value in (
-                ("lkml", self.lkml),
-                ("group_label", self.group_label),
-                ("explores", self.explores),
-            )
-            if value
-        ]
-        if len(selectors) != 1:
-            got = ", ".join(selectors) if selectors else "none"
-            raise ValueError(
-                "LookMLModelConfig must set exactly one of 'lkml', 'group_label', "
-                f"'explores' (got {got})"
-            )
-        return self
-
-
-class LookMLModelsMixin(BaseModel):
-    """Shared nested ``models:`` field for warehouse sources with LookML explores.
-
-    A warehouse source with ``models:`` also declares ``lookml_path`` — the
-    directory of ``.lkml`` files the models' explores are parsed from. Each
-    warehouse source owns its own nested ``models:`` map; a model name must be
-    unique across the project (one model = one warehouse connection).
-
-    Example:
-        sources:
-          bigquery_prod:
-            type: bigquery
-            project: digital-arbor-400
-            dataset: transforms_bi
-            lookml_path: lkml
-            models:
-              finance: {lkml: fivetran.model.lkml}
-              core_kpis: {explores: [revenue, goals]}
-    """
-
-    model_config = ConfigDict(extra="forbid")
-
-    lookml_path: str | None = Field(
-        default=None,
-        description="Relative path to the directory of .lkml files backing this source's models.",
-    )
-    models: dict[str, LookMLModelConfig] = Field(
-        default_factory=dict,
-        description="Named LookML model collections nested under this source.",
-    )
-
-    @field_validator("lookml_path")
-    @classmethod
-    def _validate_lookml_path(cls, v: str | None) -> str | None:
-        if v is None:
-            return v
-        return _validate_relative_path("LookMLModelsMixin", "lookml_path", v)
-
-    @model_validator(mode="after")
-    def _require_lookml_path_with_models(self) -> LookMLModelsMixin:
-        if self.models and not self.lookml_path:
-            raise ValueError(
-                "'lookml_path' is required when 'models' is set — it names the "
-                "directory of .lkml files the models' explores are parsed from."
-            )
-        return self
-
-
 # ============================================================================
 # DATABASE SOURCE CONFIGS
 # ============================================================================
 
 
-class PostgresSourceConfig(
-    AttributedSourceConfig, LookMLModelsMixin, DatabaseSourceConfig
-):
+class PostgresSourceConfig(AttributedSourceConfig, DatabaseSourceConfig):
     """Postgres source configuration.
 
     Field names match dbt profiles.yml exactly.
@@ -567,9 +446,7 @@ class PostgresSourceConfig(
     ] = 1
 
 
-class SnowflakeSourceConfig(
-    AttributedSourceConfig, LookMLModelsMixin, DatabaseSourceConfig
-):
+class SnowflakeSourceConfig(AttributedSourceConfig, DatabaseSourceConfig):
     """Snowflake source configuration.
 
     Field names match dbt profiles.yml exactly.
@@ -702,9 +579,7 @@ class SnowflakeSourceConfig(
     ] = 0.0
 
 
-class BigQuerySourceConfig(
-    AttributedSourceConfig, LookMLModelsMixin, DatabaseSourceConfig
-):
+class BigQuerySourceConfig(AttributedSourceConfig, DatabaseSourceConfig):
     """BigQuery source configuration.
 
     Field names match dbt profiles.yml exactly.
@@ -870,9 +745,7 @@ class BigQuerySourceConfig(
         return self
 
 
-class RedshiftSourceConfig(
-    AttributedSourceConfig, LookMLModelsMixin, DatabaseSourceConfig
-):
+class RedshiftSourceConfig(AttributedSourceConfig, DatabaseSourceConfig):
     """Redshift source configuration.
 
     Field names match dbt profiles.yml exactly, except `autocommit`: Dataface
@@ -1025,9 +898,7 @@ class RedshiftSourceConfig(
     ] = None
 
 
-class MySQLSourceConfig(
-    AttributedSourceConfig, LookMLModelsMixin, DatabaseSourceConfig
-):
+class MySQLSourceConfig(AttributedSourceConfig, DatabaseSourceConfig):
     """MySQL source configuration.
 
     Field names match dbt profiles.yml exactly.
@@ -1048,9 +919,7 @@ class MySQLSourceConfig(
     password: str = Field(description="MySQL password.")
 
 
-class TrinoSourceConfig(
-    AttributedSourceConfig, LookMLModelsMixin, DatabaseSourceConfig
-):
+class TrinoSourceConfig(AttributedSourceConfig, DatabaseSourceConfig):
     """Trino / Presto source configuration.
 
     `database` names the Trino catalog; dbt-trino aliases `catalog` → `database`,
@@ -1195,7 +1064,7 @@ class TrinoSourceConfig(
     ] = None
 
 
-class DuckDBSourceConfig(LookMLModelsMixin, DatabaseSourceConfig):
+class DuckDBSourceConfig(DatabaseSourceConfig):
     """DuckDB source configuration.
 
     Read-only posture: handled in-process. DuckDBAdapter opens file-based DuckDB
@@ -1326,7 +1195,7 @@ class DuckDBSourceConfig(LookMLModelsMixin, DatabaseSourceConfig):
         return data
 
 
-class SQLiteSourceConfig(BaseSourceConfig, LookMLModelsMixin):
+class SQLiteSourceConfig(BaseSourceConfig):
     """SQLite source configuration.
 
     Read-only posture: handled in-process. SqliteAdapter opens SQLite connections
@@ -1354,6 +1223,9 @@ class CsvSourceConfig(BaseSourceConfig):
 
     A file source is a namespace of relations: each key in `files` is a table
     name; the value is the path to the CSV file (relative to the project root).
+    Glob patterns (``*`` and ``?``) are supported: every matched file is loaded
+    and its rows are concatenated into one table.  All matched files must share
+    the same column schema.  A glob that matches no files is an error.
 
     Example:
         sources:
@@ -1362,14 +1234,22 @@ class CsvSourceConfig(BaseSourceConfig):
             files:
               sales: assets/data/sales.csv
               returns: assets/data/returns.csv
+              monthly: assets/data/monthly_*.csv
     """
 
     source_category: ClassVar[str] = "file"
     type: Literal["csv"] = Field(description="Source type identifier.")
     files: dict[str, str] = Field(
         description=(
-            "Mapping of table_name → file path. "
-            "Each key is the SQL table name; the value is the path relative to the project root. "
+            "Mapping of table_name → relative path or glob pattern. "
+            "Each key is the SQL table name. "
+            "The value is a path relative to the project root, or a glob "
+            "pattern using ``*`` (matches within a directory) and ``?`` "
+            "(matches one character). "
+            "A glob must match at least one file; all matched files must share "
+            "the same column schema and are concatenated into one table. "
+            "The fan-out limit is ``execution.max_glob_file_count`` in "
+            "``dataface.yml`` (default 1000). "
             "Required and must not be empty."
         )
     )
@@ -1392,6 +1272,9 @@ class ParquetSourceConfig(BaseSourceConfig):
 
     A file source is a namespace of relations: each key in `files` is a table
     name; the value is the path to the Parquet file (relative to the project root).
+    Glob patterns (``*`` and ``?``) are supported: every matched file is loaded
+    and its rows are concatenated into one table.  All matched files must share
+    the same column schema.  A glob that matches no files is an error.
 
     Example:
         sources:
@@ -1399,14 +1282,22 @@ class ParquetSourceConfig(BaseSourceConfig):
             type: parquet
             files:
               events: assets/data/events.parquet
+              archive: assets/data/archive_*.parquet
     """
 
     source_category: ClassVar[str] = "file"
     type: Literal["parquet"] = Field(description="Source type identifier.")
     files: dict[str, str] = Field(
         description=(
-            "Mapping of table_name → file path. "
-            "Each key is the SQL table name; the value is the path relative to the project root. "
+            "Mapping of table_name → relative path or glob pattern. "
+            "Each key is the SQL table name. "
+            "The value is a path relative to the project root, or a glob "
+            "pattern using ``*`` (matches within a directory) and ``?`` "
+            "(matches one character). "
+            "A glob must match at least one file; all matched files must share "
+            "the same column schema and are concatenated into one table. "
+            "The fan-out limit is ``execution.max_glob_file_count`` in "
+            "``dataface.yml`` (default 1000). "
             "Required and must not be empty."
         )
     )
@@ -1427,6 +1318,12 @@ class JsonSourceConfig(BaseSourceConfig):
 
     A file source is a namespace of relations: each key in `files` is a table
     name; the value is the path to the JSON file (relative to the project root).
+    Both ``.json`` and ``.jsonl`` (newline-delimited JSON) extensions are accepted;
+    the parser is chosen by extension: ``.json`` → standard JSON array/object,
+    ``.jsonl`` → newline-delimited JSON (one object per line).
+    Glob patterns (``*`` and ``?``) are supported: every matched file is loaded
+    and its rows are concatenated into one table.  A glob that matches no files
+    is an error.
 
     Example:
         sources:
@@ -1434,17 +1331,34 @@ class JsonSourceConfig(BaseSourceConfig):
             type: json
             files:
               products: assets/data/products.json
+              events: assets/data/events_*.jsonl
     """
 
     source_category: ClassVar[str] = "file"
     type: Literal["json"] = Field(description="Source type identifier.")
     files: dict[str, str] = Field(
         description=(
-            "Mapping of table_name → file path. "
-            "Each key is the SQL table name; the value is the path relative to the project root. "
+            "Mapping of table_name → relative path or glob pattern. "
+            "Each key is the SQL table name. "
+            "The value is a path relative to the project root, or a glob "
+            "pattern using ``*`` (matches within a directory) and ``?`` "
+            "(matches one character). "
+            "A glob must match at least one file and all matched files are "
+            "concatenated into one table. "
+            "The fan-out limit is ``execution.max_glob_file_count`` in "
+            "``dataface.yml`` (default 1000). "
             "Required and must not be empty."
         )
     )
+    union_by_name: bool = Field(
+        default=False,
+        description=(
+            "When True, rows from different files in a glob set are unified by "
+            "column name: columns absent in a particular file are filled with "
+            "NULL.  When False (default), all files in a glob set must share the "
+            "same column schema — a mismatch raises an error."
+        ),
+    )
 
     @model_validator(mode="after")
     def _require_nonempty_files(self) -> JsonSourceConfig:
@@ -1453,7 +1367,7 @@ class JsonSourceConfig(BaseSourceConfig):
                 "JsonSourceConfig: files must not be empty — "
                 "provide at least one table_name → path entry."
             )
-        _validate_files_mapping("JsonSourceConfig", self.files, ".json")
+        _validate_files_mapping("JsonSourceConfig", self.files, (".json", ".jsonl"))
         return self
 
 

dataface/core/compile/models/style/theme/variables.py

@@ -107,9 +107,6 @@ class VariablesStyle(BaseModel):
     placeholder: VariablesPlaceholderStyle = Field(
         description="Style for unselected/hint text in variable inputs."
     )
-    line_height: float = Field(
-        description="Line height for variable control text in pixels."
-    )
     padding: float = Field(
         description="Outer padding around the variables panel in pixels."
     )

dataface/core/compile/normalize_charts.py

@@ -14,11 +14,9 @@ from typing import TYPE_CHECKING, Any, get_args
 if TYPE_CHECKING:
     from dataface.core.project import ProjectDirectory
 
-from collections.abc import Callable
 
 from pydantic import BaseModel, TypeAdapter, ValidationError
 
-from dataface.core.compile.channel import is_dotted_lookml_ref
 from dataface.core.compile.errors import CompilationError, ReferenceError
 from dataface.core.compile.jinja import extract_variable_dependencies
 from dataface.core.compile.models.cache import INHERIT_CACHE, CachePatch
@@ -46,8 +44,6 @@ from dataface.core.compile.models.face.authored import QueryOrRef
 from dataface.core.compile.models.face.normalized import Layout
 from dataface.core.compile.models.primitives import FormatConfig
 from dataface.core.compile.models.query.authored import (
-    AuthoredLookmlMergeQuery,
-    AuthoredLookmlQuery,
     AuthoredMetricflowQuery,
     TimeGrain,
     _BaseQueryFields,
@@ -661,8 +657,8 @@ def _collect_charts_from_layout(
 
 # Chart keys that carry data-binding field references (axis, encoding, geo, etc.).
 # Presentation-only keys (title, subtitle, description, link, format, ...) are
-# excluded — a dotted string in those positions is not a LookML field ref.
-_LOOKML_CHANNEL_KEYS: frozenset[str] = frozenset(
+# excluded — a dotted string in those positions is not a field ref.
+_FIELD_CHANNEL_KEYS: frozenset[str] = frozenset(
     {
         "x",
         "y",
@@ -681,150 +677,6 @@ _LOOKML_CHANNEL_KEYS: frozenset[str] = frozenset(
 )
 
 
-def _rewrite_channel_value(
-    value: object, collected: set[str], channel_alias: Callable[[str, str], str]
-) -> object:
-    """Rewrite dotted `view.field` refs in a channel value to their SQL alias.
-
-    A channel is a scalar ref OR a list of refs (multi-series `y: [a, b]`).
-    Every dotted ref in either shape is collected+rewritten — a list-valued
-    channel whose refs went uncollected would silently drop those measures
-    from the query while reporting success.
-
-    Returns the rewritten value, or None if `value` had no dotted ref to
-    rewrite (including when `value` itself is None).
-    """
-    if value is None:
-        return None
-    items = value if isinstance(value, list) else [value]
-    rewritten: list[object] = []
-    changed = False
-    for item in items:
-        if isinstance(item, str) and is_dotted_lookml_ref(item):
-            collected.add(item)
-            view, field = item.split(".", 1)
-            rewritten.append(channel_alias(view, field))
-            changed = True
-        else:
-            rewritten.append(item)
-    if not changed:
-        return None
-    return rewritten if isinstance(value, list) else rewritten[0]
-
-
-def collect_lookml_channel_refs(
-    charts: dict[str, Any],
-    query_registry: dict[str, QueryOrRef],
-    lookml_model_names: set[str],
-) -> None:
-    """Collect dotted view.field chart-channel refs into fields-less lookml queries.
-
-    A lookml query may omit `fields:`; authors instead name fields by dotted
-    `view.field` ref directly in chart channel positions (x, y, color, theta,
-    value, ...). This pass finds every named lookml query with no `fields`,
-    gathers the dotted refs from charts bound to it, injects them as the
-    query's `fields`, and rewrites each chart's channel value to the
-    resulting `view_field` alias so it matches the lowered SQL's output
-    column (the standard T1.2 lowering pass runs after this, unaware that
-    `fields` was collected rather than authored).
-
-    Must run before query normalization (which lowers lookml queries to
-    SqlQuery) and only sees named queries — inline per-chart queries own
-    their own `fields` and are not eligible for collection.
-
-    Raises:
-        CompilationError: A lookml query has no `fields` and no chart bound
-            to it (by name) uses a dotted-ref channel.
-    """
-    try:
-        # Lazy import: likeml is optional (pip install dataface[lookml]).
-        # It is an internal workspace package and not published to PyPI.
-        from likeml import channel_alias  # noqa: PLC0415
-    except ImportError as e:
-        raise CompilationError(
-            "likeml is not installed. Install with: pip install dataface[lookml]"
-        ) from e
-
-    for query_name, query_def in query_registry.items():
-        if not isinstance(query_def, (AuthoredLookmlQuery, AuthoredLookmlMergeQuery)):
-            continue
-        # source may be an inline dict (unhashable) — a `dict not in set` membership
-        # test would raise TypeError, so gate on str first. A non-str source is never
-        # a lookml source ref, so skip it.
-        if not isinstance(query_def.source, str):
-            continue
-        if query_def.source not in lookml_model_names:
-            continue
-        if query_def.fields:
-            continue
-
-        collected: set[str] = set()
-        for chart_def in charts.values():
-            is_model = isinstance(chart_def, BaseModel)
-            chart_dict = (
-                chart_def.model_dump(exclude_none=True) if is_model else chart_def
-            )
-            bound_to_query = chart_dict.get("query") == query_name
-            if bound_to_query:
-                for field_name, value in chart_dict.items():
-                    if field_name not in _LOOKML_CHANNEL_KEYS:
-                        continue
-                    new_value = _rewrite_channel_value(value, collected, channel_alias)
-                    if new_value is None:
-                        continue
-                    if is_model:
-                        setattr(chart_def, field_name, new_value)
-                    else:
-                        chart_dict[field_name] = new_value
-
-            # Layered charts carry their own channels inside `layers[]`, one query
-            # binding per layer (falls back to the chart-level query when a layer
-            # omits its own). Each layer is walked independently, mutating the
-            # actual Layer object (model or dict) in place, not the model_dump()
-            # copy — model_dump() flattens `layers` into plain dicts, so writes must
-            # go through the real chart_def.layers list to reach the pipeline.
-            layers = chart_dict.get("layers")
-            if not layers:
-                continue
-            # Use the live typed layer objects from the model so setattr writes
-            # propagate (model_dump() produces plain dicts — mutations to those are lost).
-            # `layers` is not on all chart types, so getattr is appropriate.
-            model_layers: list[CartesianLayer] | None = (
-                getattr(chart_def, "layers", None) if is_model else None
-            )
-            real_layers: list[CartesianLayer] | list[dict[str, Any]] = (
-                model_layers if model_layers else layers
-            )
-            for idx, layer_dict in enumerate(layers):
-                layer_query_name = layer_dict.get("query")
-                if layer_query_name is None:
-                    if not bound_to_query:
-                        continue
-                elif layer_query_name != query_name:
-                    continue
-                real_layer = real_layers[idx]
-                for field_name in ("x", "y", "color"):
-                    new_value = _rewrite_channel_value(
-                        layer_dict.get(field_name), collected, channel_alias
-                    )
-                    if new_value is None:
-                        continue
-                    if isinstance(real_layer, dict):
-                        real_layer[field_name] = new_value
-                    else:
-                        setattr(real_layer, field_name, new_value)
-
-        if not collected:
-            raise CompilationError(
-                f"lookml query '{query_name}' has no fields and no charts bind "
-                "dotted-ref channels to it. Either add `fields: [view.field, ...]` "
-                "to the query, or bind a chart channel (x, y, color, ...) to a "
-                "dotted `view.field` ref."
-            )
-
-        query_def.fields = sorted(collected)
-
-
 # Maps a raw grain token ("month") to the typed TimeGrain literal, so the
 # classifier returns a real TimeGrain without a cast.
 _GRAIN_BY_NAME: dict[str, TimeGrain] = {name: name for name in get_args(TimeGrain)}
@@ -1002,7 +854,7 @@ def collect_metricflow_channel_refs(
 ) -> None:
     """Populate a `model:`-synthesized metricflow query from its charts' channels.
 
-    The MetricFlow companion to `collect_lookml_channel_refs`: `model_by_query`
+    `model_by_query`
     maps each synthesized query name to the semantic model named in the chart's
     `model: <source>.<semantic_model>`. For every chart bound to that query, each
     bare channel field is classified against the manifest into a metric, an
@@ -1040,7 +892,7 @@ def collect_metricflow_channel_refs(
             if chart_dict.get("query") != query_name:
                 continue
             for field_name, value in chart_dict.items():
-                if field_name not in _LOOKML_CHANNEL_KEYS:
+                if field_name not in _FIELD_CHANNEL_KEYS:
                     continue
                 if not isinstance(value, (str, list)):
                     continue