weaverstack 0.1.1__py3-none-any.whl

weaver/config.py

@@ -0,0 +1,178 @@
+"""Parse one Workspace configuration file into one Workspace value."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from .declaration.model import LAKEHOUSE, WAREHOUSE, WeaverItemId
+from .errors import ConfigError
+from .workspaces import (
+    FABRIC,
+    LOCAL,
+    WORKSPACE_TYPES,
+    ExecutionSettings,
+    FabricWorkspace,
+    LocalWorkspace,
+    TargetDeclaration,
+    Workspace,
+)
+
+_KEYS = {
+    "workspace",
+    "workspace_type",
+    "environment",
+    "weaver_lakehouse",
+    "execution",
+    "lakehouses",
+    "warehouses",
+}
+
+
+def load_workspace(path: str | Path) -> Workspace:
+    """Load one Workspace file; local paths resolve beside that file."""
+
+    import yaml
+
+    config_path = Path(path).expanduser().resolve()
+    if not config_path.is_file():
+        raise ConfigError(f"Workspace configuration not found: {config_path}")
+    payload = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
+    return parse_workspace(payload, base_dir=config_path.parent)
+
+
+def parse_workspace(payload: Any, base_dir: str | Path | None = None) -> Workspace:
+    """Parse a one-Workspace mapping."""
+
+    if not isinstance(payload, dict):
+        raise ConfigError("Workspace configuration must be a mapping")
+    unknown = set(payload) - _KEYS
+    if unknown:
+        raise ConfigError(
+            "Workspace configuration has unknown keys: " + ", ".join(sorted(unknown))
+        )
+    if "workspace" not in payload:
+        raise ConfigError("Workspace configuration must define 'workspace'")
+
+    workspace_type = payload.get("workspace_type", FABRIC)
+    if workspace_type not in WORKSPACE_TYPES:
+        raise ConfigError(
+            "workspace_type must be one of "
+            f"{', '.join(WORKSPACE_TYPES)}, got {workspace_type!r}"
+        )
+
+    workspace_value = payload["workspace"]
+    if workspace_type == LOCAL:
+        workspace_value = _local_path(workspace_value, base_dir)
+
+    common = {
+        "workspace": workspace_value,
+        "environment": payload.get("environment"),
+        "weaver_lakehouse": payload.get("weaver_lakehouse"),
+        "execution": _execution(payload.get("execution"), where="execution"),
+        "lakehouses": _targets(payload.get("lakehouses"), item_type=LAKEHOUSE),
+        "warehouses": _targets(payload.get("warehouses"), item_type=WAREHOUSE),
+    }
+    workspace_class = LocalWorkspace if workspace_type == LOCAL else FabricWorkspace
+    try:
+        return workspace_class(**common)
+    except TypeError as exc:
+        raise ConfigError(f"Workspace configuration is incomplete: {exc}") from exc
+
+
+def resolve_workspace(
+    *,
+    workspace: str | Path | None = None,
+    workspace_type: str | None = None,
+    environment: str | None = None,
+    weaver_lakehouse: str | None = None,
+    workspace_config: str | Path | None = None,
+) -> Workspace:
+    """Apply CLI-over-configuration precedence and return one Workspace."""
+
+    configured = load_workspace(workspace_config) if workspace_config else None
+    resolved_type = workspace_type or (
+        configured.workspace_type if configured is not None else FABRIC
+    )
+    if resolved_type not in WORKSPACE_TYPES:
+        raise ConfigError(
+            "workspace_type must be one of "
+            f"{', '.join(WORKSPACE_TYPES)}, got {resolved_type!r}"
+        )
+    resolved_identity = workspace if workspace is not None else (
+        configured.workspace if configured is not None else None
+    )
+    if resolved_identity is None:
+        raise ConfigError("give --workspace or --workspace-config containing workspace")
+
+    common = {
+        "workspace": resolved_identity,
+        "environment": environment
+        if environment is not None
+        else (configured.environment if configured is not None else None),
+        "weaver_lakehouse": weaver_lakehouse
+        if weaver_lakehouse is not None
+        else (configured.weaver_lakehouse if configured is not None else None),
+        "execution": configured.execution
+        if configured is not None
+        else ExecutionSettings(),
+        "lakehouses": configured.lakehouses if configured is not None else {},
+        "warehouses": configured.warehouses if configured is not None else {},
+    }
+    return (LocalWorkspace if resolved_type == LOCAL else FabricWorkspace)(**common)
+
+
+def _local_path(value: Any, base_dir: str | Path | None) -> Path:
+    if not isinstance(value, (str, Path)) or not str(value).strip():
+        raise ConfigError("local workspace must be a non-empty folder path")
+    path = Path(str(value).strip()).expanduser()
+    if not path.is_absolute() and base_dir is not None:
+        path = Path(base_dir) / path
+    return path
+
+
+def _execution(raw: Any, *, where: str) -> ExecutionSettings:
+    if raw is None:
+        return ExecutionSettings()
+    if not isinstance(raw, dict):
+        raise ConfigError(f"{where} must be a mapping")
+    unknown = set(raw) - {"parallel_workers"}
+    if unknown:
+        raise ConfigError(f"{where} has unknown keys: " + ", ".join(sorted(unknown)))
+    return ExecutionSettings(parallel_workers=raw.get("parallel_workers"))
+
+
+def _targets(raw: Any, *, item_type: str) -> dict[str, TargetDeclaration]:
+    field_name = "lakehouses" if item_type == LAKEHOUSE else "warehouses"
+    if raw is None:
+        return {}
+    if not isinstance(raw, dict):
+        raise ConfigError(f"{field_name} must be a mapping")
+    declarations: dict[str, TargetDeclaration] = {}
+    for physical_name, value in raw.items():
+        where = f"{field_name}[{physical_name!r}]"
+        if isinstance(value, str):
+            item_text = value
+            execution = ExecutionSettings()
+        elif isinstance(value, dict):
+            unknown = set(value) - {"item", "execution"}
+            if unknown:
+                raise ConfigError(
+                    f"{where} has unknown keys: " + ", ".join(sorted(unknown))
+                )
+            if "item" not in value:
+                raise ConfigError(f"{where} must define 'item'")
+            item_text = value["item"]
+            execution = _execution(value.get("execution"), where=f"{where}.execution")
+        else:
+            raise ConfigError(f"{where} must be an item name or mapping")
+        try:
+            item = WeaverItemId.parse(item_text)
+        except (TypeError, ValueError) as exc:
+            raise ConfigError(f"{where} has invalid logical item {item_text!r}") from exc
+        if item.item_type != item_type:
+            raise ConfigError(
+                f"{where} must name a {item_type} item, got {item}"
+            )
+        declarations[str(physical_name)] = TargetDeclaration(item, execution)
+    return declarations

weaver/declaration/__init__.py

@@ -0,0 +1,171 @@
+"""Weaver document policy — the contract an authored object must satisfy."""
+
+from __future__ import annotations
+
+from .dependencies import (
+    PythonImport,
+    RelationReference,
+    extract_python_references,
+    extract_sql_references,
+)
+from .columns import (
+    metadata_column_references,
+    resolve_build_columns,
+    validate_build_columns,
+)
+from .graph import Edge, Graph
+from .item_dependencies import project_bound_documents, resolve_item_dependencies
+from .model import (
+    FILES,
+    ITEM_TYPES,
+    ItemDependency,
+    LAKEHOUSE,
+    RepositoryAlias,
+    WAREHOUSE,
+    WeaverDocumentId,
+    WeaverItem,
+    WeaverItemId,
+    WeaverRepository,
+    WeaverSchemaId,
+)
+from .repository import (
+    build_internal_graph,
+    effective_dependencies,
+    parse_item_repository,
+    unresolved_references,
+)
+from .schemas import (
+    SCHEMAS_DIRECTORY,
+    SchemaSes,
+    is_schema_file,
+    parse_schema_document,
+    read_schema_document,
+)
+from .source import (
+    SourceDocument,
+    SqlAnalysis,
+    analyse_sql,
+    content_hash,
+    language_for_filename,
+    object_id_for_filename,
+    read_source_document,
+)
+from .references import (
+    IDENTITY_COLUMN_NOTE,
+    ResolvedText,
+    column_note,
+    declared_column_notes,
+    resolve_text,
+    validate_repository_metadata,
+)
+from .metadata import (
+    AUDIT_COLUMNS,
+    AUDIT_DELETE,
+    AUDIT_INSERT,
+    AUDIT_UPDATE,
+    FOLDER,
+    DELTA_LANGUAGES,
+    LAKEHOUSE_NAMESPACE,
+    LANGUAGES,
+    NAMESPACES,
+    OBJECT_KINDS,
+    PYTHON,
+    SPARK_SQL,
+    SQL,
+    TABLE,
+    VIEW,
+    WAREHOUSE_NAMESPACE,
+    Column,
+    ForeignKey,
+    MetadataText,
+    ObjectId,
+    Reference,
+    Revision,
+    SesDocument,
+    WeaverDocument,
+    audit_column_name,
+    extract_python_metadata,
+    extract_sql_metadata_and_body,
+    namespace_for_target,
+    parse_document,
+    parse_python_document,
+    parse_sql_document,
+)
+
+__all__ = [
+    "RelationReference",
+    "PythonImport",
+    "extract_python_references",
+    "extract_sql_references",
+    "WeaverRepository",
+    "WeaverItem",
+    "WeaverItemId",
+    "WeaverSchemaId",
+    "WeaverDocumentId",
+    "RepositoryAlias",
+    "ItemDependency",
+    "LAKEHOUSE",
+    "WAREHOUSE",
+    "ITEM_TYPES",
+    "FILES",
+    "parse_item_repository",
+    "Graph",
+    "resolve_item_dependencies",
+    "project_bound_documents",
+    "Edge",
+    "build_internal_graph",
+    "effective_dependencies",
+    "unresolved_references",
+    "SchemaSes",
+    "SCHEMAS_DIRECTORY",
+    "is_schema_file",
+    "parse_schema_document",
+    "read_schema_document",
+    "LAKEHOUSE_NAMESPACE",
+    "WAREHOUSE_NAMESPACE",
+    "NAMESPACES",
+    "namespace_for_target",
+    "SourceDocument",
+    "SqlAnalysis",
+    "analyse_sql",
+    "content_hash",
+    "language_for_filename",
+    "object_id_for_filename",
+    "read_source_document",
+    "FOLDER",
+    "TABLE",
+    "VIEW",
+    "OBJECT_KINDS",
+    "PYTHON",
+    "SQL",
+    "SPARK_SQL",
+    "LANGUAGES",
+    "DELTA_LANGUAGES",
+    "AUDIT_COLUMNS",
+    "AUDIT_INSERT",
+    "AUDIT_UPDATE",
+    "AUDIT_DELETE",
+    "audit_column_name",
+    "Column",
+    "ForeignKey",
+    "MetadataText",
+    "ObjectId",
+    "Reference",
+    "Revision",
+    "SesDocument",
+    "WeaverDocument",
+    "IDENTITY_COLUMN_NOTE",
+    "ResolvedText",
+    "column_note",
+    "declared_column_notes",
+    "resolve_text",
+    "validate_repository_metadata",
+    "extract_python_metadata",
+    "extract_sql_metadata_and_body",
+    "parse_document",
+    "parse_python_document",
+    "parse_sql_document",
+    "metadata_column_references",
+    "resolve_build_columns",
+    "validate_build_columns",
+]

weaver/declaration/columns.py

@@ -0,0 +1,223 @@
+"""The one column-validation model both build engines share.
+
+A SQL-backed table's physical business columns are known in two ways. A declared
+schema states them up front and is validated at parse time by
+:func:`weaver.ses.metadata._validate_columns`. An *inferred* table has no
+declared schema, so the same guards cannot run until the query's output shape is
+known — which, per how-does-build-work §2, is at build, inside the install action.
+
+This module is that deferred guard, expressed once so Spark and T-SQL agree. The
+Spark executor calls :func:`resolve_build_columns` directly with the columns its
+``DataFrame`` reported; the generated T-SQL script mirrors the identical rules in
+SQL because a Warehouse only knows its query's shape server-side. Both draw the
+set of column-referencing metadata from :func:`metadata_column_references`, so
+neither can drift from the other.
+
+A column name is an exact, case-sensitive Weaver contract, even where the target
+engine would fold case. The rules:
+
+- a query may not produce two columns whose names collide **case-insensitively**
+  — ``CustomerId`` and ``customerid`` together are ambiguous, and no unambiguous
+  table could be built from them;
+- when a schema is declared, the declared column set and the query's output set
+  must be equivalent **by exact name** — every declared column returned under the
+  same spelling, no undeclared column produced — while types, order, width,
+  precision and nullability are not compared, so a declaration may deliberately
+  be wider or more stable;
+- every column named by ``Primary key``, ``Not null``, ``Comparison columns`` or
+  ``Column notes`` must exist among the business columns **under exactly its
+  declared spelling** (the ``Identity`` column counts as present here — see below).
+
+Case is thus exact for naming and equivalence, but case-insensitive for the
+ambiguity guard — a name that only sometimes matches is not a name Weaver can
+rely on.
+
+``Identity`` is not in that list: it names a **Weaver-managed** surrogate column
+build adds, not a business column, so it must *not* clash with the query's output
+— but the primary key may name it when the surrogate is the key.
+"""
+
+from __future__ import annotations
+
+from ..errors import BuildError
+from .metadata import SesDocument
+
+
+def metadata_column_references(document: SesDocument) -> tuple[tuple[str, str], ...]:
+    """The ``(label, column)`` pairs this document's metadata references.
+
+    Every pair must resolve to a produced business column. The set is the same
+    whether the table is declared or inferred; only *when* it is checked differs.
+    """
+
+    references: list[tuple[str, str]] = []
+    references.extend(("Primary key", column) for column in document.primary_key)
+    references.extend(
+        ("Unique keys", column)
+        for unique_key in document.unique_keys
+        for column in unique_key
+    )
+    # Only this side of a relationship: the parent's columns belong to the parent
+    # and are checked when it is built, not here.
+    references.extend(
+        ("Foreign keys", column)
+        for foreign_key in document.foreign_keys
+        for column in foreign_key.columns
+    )
+    references.extend(("Not null", column) for column in document.declared_not_null)
+    references.extend(
+        ("Comparison columns", column)
+        for column in document.declared_comparison_columns
+    )
+    references.extend(
+        ("Column notes", column.name)
+        for column in document.schema
+        if column.note is not None
+    )
+    # For an inferred table the notes live on no declared column, so read them
+    # from the raw metadata block instead.
+    if not document.has_declared_schema:
+        notes = document.raw.get("Column notes") or {}
+        if isinstance(notes, dict):
+            references.extend(("Column notes", str(name)) for name in notes)
+    return tuple(references)
+
+
+def resolve_build_columns(
+    document: SesDocument, query_columns: tuple[str, ...]
+) -> tuple[str, ...]:
+    """Validate a built table's columns from a live document — the tests' entry.
+
+    A convenience wrapper over :func:`validate_build_columns` that reads the
+    declared columns and metadata references off ``document``. The installer never
+    uses this — it holds no document — and calls the data-level function with the
+    values the bundle froze (how-does-build-work §2).
+    """
+
+    declared = (
+        tuple(column.name for column in document.schema)
+        if document.has_declared_schema
+        else None
+    )
+    return validate_build_columns(
+        document.qualified,
+        query_columns,
+        declared_columns=declared,
+        references=metadata_column_references(document),
+        identity=document.identity,
+    )
+
+
+def validate_build_columns(
+    qualified: str,
+    query_columns: tuple[str, ...],
+    *,
+    declared_columns: tuple[str, ...] | None,
+    references: tuple[tuple[str, str], ...],
+    identity: str | None = None,
+) -> tuple[str, ...]:
+    """The rule core, over plain data so the frozen payload can drive it.
+
+    ``query_columns`` are what the engine reported for the object's query.
+    ``declared_columns`` are the declared business-column names, or ``None`` when
+    the table is inferred. ``references`` are the ``(label, column)`` pairs from
+    :func:`metadata_column_references`. ``identity`` names the Weaver-managed
+    surrogate column, when one is declared: it is not a business column, so it may
+    not clash with the query's output, but the primary key may name it. Returns
+    the physical business columns in order — declared names when declared
+    (authoritative), else the query's own — and raises :class:`BuildError` on any
+    violation.
+    """
+
+    _reject_duplicate_query_columns(qualified, query_columns)
+
+    if declared_columns is not None:
+        _require_set_equivalence(qualified, declared_columns, query_columns)
+        business_columns = tuple(declared_columns)
+    else:
+        business_columns = tuple(query_columns)
+
+    _reject_identity_collision(qualified, identity, business_columns)
+    # The identity column is Weaver's own, so the primary key may name it even
+    # though it is not a business column.
+    available = business_columns + ((identity,) if identity is not None else ())
+    _require_references_exist(qualified, available, references)
+    return business_columns
+
+
+def _reject_identity_collision(
+    qualified: str, identity: str | None, business_columns: tuple[str, ...]
+) -> None:
+    if identity is None:
+        return
+    if any(identity.lower() == name.lower() for name in business_columns):
+        raise BuildError(
+            f"{qualified}: Identity {identity!r} collides with a business column — "
+            "the identity column is Weaver-managed and must not be one the query "
+            "produces or the schema declares."
+        )
+
+
+def _reject_duplicate_query_columns(
+    qualified: str, query_columns: tuple[str, ...]
+) -> None:
+    # Case-insensitive on purpose: two names that differ only by case cannot be
+    # told apart reliably downstream, so they are ambiguous, not distinct.
+    groups: dict[str, list[str]] = {}
+    for column in query_columns:
+        groups.setdefault(column.lower(), []).append(column)
+    colliding = sorted(
+        ", ".join(names) for names in groups.values() if len(names) > 1
+    )
+    if colliding:
+        raise BuildError(
+            f"{qualified}: the query produces columns that collide by name "
+            "(case-insensitively): "
+            + "; ".join(colliding)
+            + " — no unambiguous table can be built. Give them distinct names."
+        )
+
+
+def _require_set_equivalence(
+    qualified: str,
+    declared: tuple[str, ...],
+    query_columns: tuple[str, ...],
+) -> None:
+    # Exact, case-sensitive: a declared name and a query name that differ only by
+    # case are two different columns, and the declaration must win exactly.
+    declared_set = set(declared)
+    query_set = set(query_columns)
+
+    missing = [name for name in declared if name not in query_set]
+    if missing:
+        raise BuildError(
+            f"{qualified}: declared column(s) not returned by the query under the "
+            "same case: "
+            + ", ".join(missing)
+            + ". A declared schema must match the query's column set exactly by "
+            "name (types aside)."
+        )
+
+    extra = [name for name in query_columns if name not in declared_set]
+    if extra:
+        raise BuildError(
+            f"{qualified}: the query returns column(s) not in the declared schema "
+            "(names are case-sensitive): "
+            + ", ".join(extra)
+            + ". Declare them under the same spelling, or drop them from the query."
+        )
+
+
+def _require_references_exist(
+    qualified: str,
+    business_columns: tuple[str, ...],
+    references: tuple[tuple[str, str], ...],
+) -> None:
+    available = set(business_columns)
+    for label, column in references:
+        if column not in available:
+            raise BuildError(
+                f"{qualified}: {label} names column {column!r}, which the built "
+                "table does not have under that exact name (names are "
+                "case-sensitive)."
+            )