dataface 0.3.2.dev22__py3-none-any.whl → 0.3.2.dev150__py3-none-any.whl

dataface/core/compile/models/refs.py

@@ -43,8 +43,6 @@ def infer_query_type_from_keys(query_dict: dict[str, Any]) -> str:
         return "values"
     if "metrics" in query_dict or "dimensions" in query_dict:
         return "metricflow"
-    if "explore" in query_dict or "merge" in query_dict:
-        return "lookml"
     if "url" in query_dict:
         return "http"
     return "sql"

dataface/core/compile/models/source.py

@@ -111,29 +111,14 @@ def reject_credential_literals(source_name: str, raw_config: Mapping[str, Any])
         )
 
 
-def _validate_relative_path(class_name: str, field_label: str, value: str) -> str:
-    """Reject an empty, absolute, or root-escaping project-relative path.
-
-    Delegates the escape rule to ``assert_relpath`` — the single project-relative
-    path guard — and re-raises with the class/field context. Shared by every field
-    that stores a project-relative filesystem path (LookML model/lookml_path fields
-    today).
-    """
-    try:
-        assert_relpath(value)
-    except ValueError as e:
-        raise ValueError(f"{class_name}: {field_label} {e}") from e
-    return value
-
-
 def _validate_files_mapping(
-    class_name: str, files: dict[str, str], expected_extension: str
+    class_name: str, files: dict[str, str], expected_extension: str | tuple[str, ...]
 ) -> None:
     """Validate all keys and values in a file source's ``files:`` mapping.
 
     Keys must be valid SQL identifiers (letters, digits, underscores; no
     leading digit). Values must be non-empty relative POSIX paths that do not
-    escape above the project root after normalization, and must end in
+    escape above the project root after normalization, and must end in one of
     ``expected_extension`` (case-insensitive) — parsing dispatches on the
     declared source ``type``, not the file extension, so this is what
     guarantees every servable data file's extension matches its type (and
@@ -143,12 +128,18 @@ def _validate_files_mapping(
     Args:
         class_name: Source config class name (for error messages).
         files: The ``files:`` mapping from the authored YAML.
-        expected_extension: The canonical extension for this source type
-            (e.g. ``".csv"``), including the leading dot.
+        expected_extension: The canonical extension or tuple of extensions for
+            this source type (e.g. ``".csv"`` or ``(".json", ".jsonl")``),
+            including the leading dot.
 
     Raises:
         ValueError: On an invalid key, path value, or extension mismatch.
     """
+    exts = (
+        (expected_extension,)
+        if isinstance(expected_extension, str)
+        else expected_extension
+    )
     for table_name, file_path in files.items():
         if not _SQL_IDENTIFIER_RE.match(table_name):
             raise ValueError(
@@ -160,10 +151,11 @@ def _validate_files_mapping(
             assert_relpath(file_path)
         except ValueError as e:
             raise ValueError(f"{class_name}: files[{table_name!r}] {e}") from e
-        if not file_path.lower().endswith(expected_extension):
+        if not any(file_path.lower().endswith(ext) for ext in exts):
+            ext_msg = repr(exts[0]) if len(exts) == 1 else f"one of {list(exts)}"
             raise ValueError(
                 f"{class_name}: files[{table_name!r}] path {file_path!r} must end in "
-                f"{expected_extension!r} to match source type."
+                f"{ext_msg} to match source type."
             )
 
 
@@ -373,125 +365,12 @@ class DuckDBRetriesConfig(BaseModel):
     ] = Field(default_factory=_default_retryable_exceptions)
 
 
-# ============================================================================
-# LOOKML MODELS (nested under warehouse sources)
-# ============================================================================
-
-
-class LookMLModelConfig(BaseModel):
-    """One LookML model: a named collection of explores under a warehouse source.
-
-    Exactly one selector chooses which explores belong to the collection:
-
-    - ``lkml``: a specific ``.lkml`` model file. Validated as a relative path
-      that does not escape the project root. File-scoping within likeml is not
-      yet implemented, so using this selector raises a clear error at compile
-      time directing authors to use an explicit ``explores:`` list instead.
-      Disk existence is not checked at parse time.
-    - ``group_label``: explores tagged with this label in LookML. Accepted at
-      parse time but likeml carries no group_label metadata yet, so using it
-      raises a clear error at compile time.
-    - ``explores``: an explicit list of explore names. The only selector
-      enforced at compile time: an explore not in the list is rejected.
-    """
-
-    model_config = ConfigDict(extra="forbid")
-
-    lkml: str | None = Field(
-        default=None,
-        description="Path to a specific .lkml model file, relative to the project root.",
-    )
-    group_label: str | None = Field(
-        default=None,
-        description="LookML group_label selecting this model's explores. Not yet enforced at compile time.",
-    )
-    explores: list[str] | None = Field(
-        default=None, description="Explicit list of explore names in this model."
-    )
-
-    @field_validator("lkml")
-    @classmethod
-    def _validate_lkml(cls, v: str | None) -> str | None:
-        if v is None:
-            return v
-        return _validate_relative_path("LookMLModelConfig", "lkml", v)
-
-    @model_validator(mode="after")
-    def _require_one_selector(self) -> LookMLModelConfig:
-        selectors = [
-            name
-            for name, value in (
-                ("lkml", self.lkml),
-                ("group_label", self.group_label),
-                ("explores", self.explores),
-            )
-            if value
-        ]
-        if len(selectors) != 1:
-            got = ", ".join(selectors) if selectors else "none"
-            raise ValueError(
-                "LookMLModelConfig must set exactly one of 'lkml', 'group_label', "
-                f"'explores' (got {got})"
-            )
-        return self
-
-
-class LookMLModelsMixin(BaseModel):
-    """Shared nested ``models:`` field for warehouse sources with LookML explores.
-
-    A warehouse source with ``models:`` also declares ``lookml_path`` — the
-    directory of ``.lkml`` files the models' explores are parsed from. Each
-    warehouse source owns its own nested ``models:`` map; a model name must be
-    unique across the project (one model = one warehouse connection).
-
-    Example:
-        sources:
-          bigquery_prod:
-            type: bigquery
-            project: digital-arbor-400
-            dataset: transforms_bi
-            lookml_path: lkml
-            models:
-              finance: {lkml: fivetran.model.lkml}
-              core_kpis: {explores: [revenue, goals]}
-    """
-
-    model_config = ConfigDict(extra="forbid")
-
-    lookml_path: str | None = Field(
-        default=None,
-        description="Relative path to the directory of .lkml files backing this source's models.",
-    )
-    models: dict[str, LookMLModelConfig] = Field(
-        default_factory=dict,
-        description="Named LookML model collections nested under this source.",
-    )
-
-    @field_validator("lookml_path")
-    @classmethod
-    def _validate_lookml_path(cls, v: str | None) -> str | None:
-        if v is None:
-            return v
-        return _validate_relative_path("LookMLModelsMixin", "lookml_path", v)
-
-    @model_validator(mode="after")
-    def _require_lookml_path_with_models(self) -> LookMLModelsMixin:
-        if self.models and not self.lookml_path:
-            raise ValueError(
-                "'lookml_path' is required when 'models' is set — it names the "
-                "directory of .lkml files the models' explores are parsed from."
-            )
-        return self
-
-
 # ============================================================================
 # DATABASE SOURCE CONFIGS
 # ============================================================================
 
 
-class PostgresSourceConfig(
-    AttributedSourceConfig, LookMLModelsMixin, DatabaseSourceConfig
-):
+class PostgresSourceConfig(AttributedSourceConfig, DatabaseSourceConfig):
     """Postgres source configuration.
 
     Field names match dbt profiles.yml exactly.
@@ -567,9 +446,7 @@ class PostgresSourceConfig(
     ] = 1
 
 
-class SnowflakeSourceConfig(
-    AttributedSourceConfig, LookMLModelsMixin, DatabaseSourceConfig
-):
+class SnowflakeSourceConfig(AttributedSourceConfig, DatabaseSourceConfig):
     """Snowflake source configuration.
 
     Field names match dbt profiles.yml exactly.
@@ -702,9 +579,7 @@ class SnowflakeSourceConfig(
     ] = 0.0
 
 
-class BigQuerySourceConfig(
-    AttributedSourceConfig, LookMLModelsMixin, DatabaseSourceConfig
-):
+class BigQuerySourceConfig(AttributedSourceConfig, DatabaseSourceConfig):
     """BigQuery source configuration.
 
     Field names match dbt profiles.yml exactly.
@@ -870,12 +745,14 @@ class BigQuerySourceConfig(
         return self
 
 
-class RedshiftSourceConfig(
-    AttributedSourceConfig, LookMLModelsMixin, DatabaseSourceConfig
-):
+class RedshiftSourceConfig(AttributedSourceConfig, DatabaseSourceConfig):
     """Redshift source configuration.
 
-    Field names match dbt profiles.yml exactly.
+    Field names match dbt profiles.yml exactly, except `autocommit`: Dataface
+    only reads on this connection, and a non-autocommit session holds
+    AccessShareLock on every table it read until the connection closes,
+    blocking any writer that needs ACCESS EXCLUSIVE. `build_adapter` forces
+    it on unconditionally, so it isn't part of the authored surface.
 
     Read-only posture: no native driver flag. Bind to a SELECT-only IAM role
     or database user at the Redshift/IAM layer.
@@ -957,10 +834,6 @@ class RedshiftSourceConfig(
         str | None,
         Field(description="AWS region containing the Redshift target."),
     ] = None
-    autocommit: Annotated[
-        bool | None,
-        Field(description="Whether Redshift statements autocommit."),
-    ] = True
     access_key_id: Annotated[
         str | None,
         Field(description="AWS access key identifier."),
@@ -1025,9 +898,7 @@ class RedshiftSourceConfig(
     ] = None
 
 
-class MySQLSourceConfig(
-    AttributedSourceConfig, LookMLModelsMixin, DatabaseSourceConfig
-):
+class MySQLSourceConfig(AttributedSourceConfig, DatabaseSourceConfig):
     """MySQL source configuration.
 
     Field names match dbt profiles.yml exactly.
@@ -1048,9 +919,7 @@ class MySQLSourceConfig(
     password: str = Field(description="MySQL password.")
 
 
-class TrinoSourceConfig(
-    AttributedSourceConfig, LookMLModelsMixin, DatabaseSourceConfig
-):
+class TrinoSourceConfig(AttributedSourceConfig, DatabaseSourceConfig):
     """Trino / Presto source configuration.
 
     `database` names the Trino catalog; dbt-trino aliases `catalog` → `database`,
@@ -1195,7 +1064,7 @@ class TrinoSourceConfig(
     ] = None
 
 
-class DuckDBSourceConfig(LookMLModelsMixin, DatabaseSourceConfig):
+class DuckDBSourceConfig(DatabaseSourceConfig):
     """DuckDB source configuration.
 
     Read-only posture: handled in-process. DuckDBAdapter opens file-based DuckDB
@@ -1326,7 +1195,7 @@ class DuckDBSourceConfig(LookMLModelsMixin, DatabaseSourceConfig):
         return data
 
 
-class SQLiteSourceConfig(BaseSourceConfig, LookMLModelsMixin):
+class SQLiteSourceConfig(BaseSourceConfig):
     """SQLite source configuration.
 
     Read-only posture: handled in-process. SqliteAdapter opens SQLite connections
@@ -1354,6 +1223,9 @@ class CsvSourceConfig(BaseSourceConfig):
 
     A file source is a namespace of relations: each key in `files` is a table
     name; the value is the path to the CSV file (relative to the project root).
+    Glob patterns (``*`` and ``?``) are supported: every matched file is loaded
+    and its rows are concatenated into one table.  All matched files must share
+    the same column schema.  A glob that matches no files is an error.
 
     Example:
         sources:
@@ -1362,14 +1234,22 @@ class CsvSourceConfig(BaseSourceConfig):
             files:
               sales: assets/data/sales.csv
               returns: assets/data/returns.csv
+              monthly: assets/data/monthly_*.csv
     """
 
     source_category: ClassVar[str] = "file"
     type: Literal["csv"] = Field(description="Source type identifier.")
     files: dict[str, str] = Field(
         description=(
-            "Mapping of table_name → file path. "
-            "Each key is the SQL table name; the value is the path relative to the project root. "
+            "Mapping of table_name → relative path or glob pattern. "
+            "Each key is the SQL table name. "
+            "The value is a path relative to the project root, or a glob "
+            "pattern using ``*`` (matches within a directory) and ``?`` "
+            "(matches one character). "
+            "A glob must match at least one file; all matched files must share "
+            "the same column schema and are concatenated into one table. "
+            "The fan-out limit is ``execution.max_glob_file_count`` in "
+            "``dataface.yml`` (default 1000). "
             "Required and must not be empty."
         )
     )
@@ -1392,6 +1272,9 @@ class ParquetSourceConfig(BaseSourceConfig):
 
     A file source is a namespace of relations: each key in `files` is a table
     name; the value is the path to the Parquet file (relative to the project root).
+    Glob patterns (``*`` and ``?``) are supported: every matched file is loaded
+    and its rows are concatenated into one table.  All matched files must share
+    the same column schema.  A glob that matches no files is an error.
 
     Example:
         sources:
@@ -1399,14 +1282,22 @@ class ParquetSourceConfig(BaseSourceConfig):
             type: parquet
             files:
               events: assets/data/events.parquet
+              archive: assets/data/archive_*.parquet
     """
 
     source_category: ClassVar[str] = "file"
     type: Literal["parquet"] = Field(description="Source type identifier.")
     files: dict[str, str] = Field(
         description=(
-            "Mapping of table_name → file path. "
-            "Each key is the SQL table name; the value is the path relative to the project root. "
+            "Mapping of table_name → relative path or glob pattern. "
+            "Each key is the SQL table name. "
+            "The value is a path relative to the project root, or a glob "
+            "pattern using ``*`` (matches within a directory) and ``?`` "
+            "(matches one character). "
+            "A glob must match at least one file; all matched files must share "
+            "the same column schema and are concatenated into one table. "
+            "The fan-out limit is ``execution.max_glob_file_count`` in "
+            "``dataface.yml`` (default 1000). "
             "Required and must not be empty."
         )
     )
@@ -1427,6 +1318,12 @@ class JsonSourceConfig(BaseSourceConfig):
 
     A file source is a namespace of relations: each key in `files` is a table
     name; the value is the path to the JSON file (relative to the project root).
+    Both ``.json`` and ``.jsonl`` (newline-delimited JSON) extensions are accepted;
+    the parser is chosen by extension: ``.json`` → standard JSON array/object,
+    ``.jsonl`` → newline-delimited JSON (one object per line).
+    Glob patterns (``*`` and ``?``) are supported: every matched file is loaded
+    and its rows are concatenated into one table.  A glob that matches no files
+    is an error.
 
     Example:
         sources:
@@ -1434,17 +1331,34 @@ class JsonSourceConfig(BaseSourceConfig):
             type: json
             files:
               products: assets/data/products.json
+              events: assets/data/events_*.jsonl
     """
 
     source_category: ClassVar[str] = "file"
     type: Literal["json"] = Field(description="Source type identifier.")
     files: dict[str, str] = Field(
         description=(
-            "Mapping of table_name → file path. "
-            "Each key is the SQL table name; the value is the path relative to the project root. "
+            "Mapping of table_name → relative path or glob pattern. "
+            "Each key is the SQL table name. "
+            "The value is a path relative to the project root, or a glob "
+            "pattern using ``*`` (matches within a directory) and ``?`` "
+            "(matches one character). "
+            "A glob must match at least one file and all matched files are "
+            "concatenated into one table. "
+            "The fan-out limit is ``execution.max_glob_file_count`` in "
+            "``dataface.yml`` (default 1000). "
             "Required and must not be empty."
         )
     )
+    union_by_name: bool = Field(
+        default=False,
+        description=(
+            "When True, rows from different files in a glob set are unified by "
+            "column name: columns absent in a particular file are filled with "
+            "NULL.  When False (default), all files in a glob set must share the "
+            "same column schema — a mismatch raises an error."
+        ),
+    )
 
     @model_validator(mode="after")
     def _require_nonempty_files(self) -> JsonSourceConfig:
@@ -1453,7 +1367,7 @@ class JsonSourceConfig(BaseSourceConfig):
                 "JsonSourceConfig: files must not be empty — "
                 "provide at least one table_name → path entry."
             )
-        _validate_files_mapping("JsonSourceConfig", self.files, ".json")
+        _validate_files_mapping("JsonSourceConfig", self.files, (".json", ".jsonl"))
         return self
 
 

dataface/core/compile/models/style/authored/__init__.py

@@ -42,25 +42,36 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator
 # Regression coverage: test_style_authored_bootstrap_order.py.
 from dataface.core.compile.models.style.authored._base import (
     AreaMarkStylePatch,
-    AxisDomainStylePatch,
-    AxisElementStylePatch,
     AxisGridStylePatch,
-    AxisStylePatch,
+    AxisLabelStylePatch,
+    AxisLineStylePatch,
     AxisTicksStylePatch,
+    AxisTitleStylePatch,
+    AxisXStylePatch,
+    AxisYStylePatch,
+    BandAxisStylePatch,
     BarMarkStylePatch,
+    BaseAxisGridStylePatch,
+    BaseAxisStylePatch,
+    BaseScaleStylePatch,
     DataTableStylePatch,
+    DimensionLabelStylePatch,
+    DimensionTicksStylePatch,
     EndpointLabelsConfig,
     EndpointLabelsConfigPatch,
     GlobalMarksStylePatch,
     LegendStylePatch,
     LineMarkStylePatch,
+    MeasureGridStylePatch,
     PaddingStylePatch,
     PointMarkStylePatch,
-    ScaleStylePatch,
+    QuantitativeAxisStylePatch,
+    ScaleContinuousStylePatch,
     StylePatch,
     TitleStylePatch,
     TooltipStylePatch,
     TotalStylePatch,
+    XScaleStylePatch,
 )
 from dataface.core.compile.models.style.authored.area import AreaChartStylePatch
 from dataface.core.compile.models.style.authored.bar import BarChartStylePatch
@@ -99,6 +110,7 @@ from dataface.core.compile.models.style.authored.table import (
     TableColumnConfig,
     TableColumnDefaultsConfig,
     _validate_glyph_pair as _validate_glyph_pair,
+    fill_table_column_defaults,
 )
 
 __all__ = [
@@ -106,11 +118,23 @@ __all__ = [
     "_validate_glyph_pair",
     "AreaChartStylePatch",
     "AreaMarkStylePatch",
-    "AxisDomainStylePatch",
-    "AxisElementStylePatch",
     "AxisGridStylePatch",
-    "AxisStylePatch",
+    "AxisLabelStylePatch",
+    "AxisLineStylePatch",
     "AxisTicksStylePatch",
+    "AxisTitleStylePatch",
+    "AxisXStylePatch",
+    "AxisYStylePatch",
+    "BandAxisStylePatch",
+    "BaseAxisGridStylePatch",
+    "BaseAxisStylePatch",
+    "BaseScaleStylePatch",
+    "DimensionLabelStylePatch",
+    "DimensionTicksStylePatch",
+    "MeasureGridStylePatch",
+    "QuantitativeAxisStylePatch",
+    "ScaleContinuousStylePatch",
+    "XScaleStylePatch",
     "BarChartStylePatch",
     "BarMarkStylePatch",
     "CalloutChartStylePatch",
@@ -131,7 +155,6 @@ __all__ = [
     "PieChartStylePatch",
     "PointMapChartStylePatch",
     "PointMarkStylePatch",
-    "ScaleStylePatch",
     "ScatterChartStylePatch",
     "SparkBarBarStylePatch",
     "SparkBarChartLabelStylePatch",
@@ -146,6 +169,7 @@ __all__ = [
     "TableColumnDefaultsConfig",
     "TitleStylePatch",
     "TooltipStylePatch",
+    "fill_table_column_defaults",
     "TotalStylePatch",
 ]