weaverstack 0.1.1__py3-none-any.whl

weaver/build_bundle/workflow.py

@@ -0,0 +1,585 @@
+"""Source preparation, state handover, bundle generation, and installation.
+
+A repository source is independent of the target estate. Remote sources are
+materialised once onto the current process's local filesystem; parsing and
+request validation finish before target state is read. Native builds keep all
+four stages in-environment. A desktop Fabric build serialises only ``BuildState``
+out and hands only a completed archive back for execution.
+"""
+
+from __future__ import annotations
+
+import stat
+import tempfile
+import zipfile
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path, PurePosixPath
+from typing import Iterator, Mapping
+
+from ..errors import BuildError
+from ..locations import Location
+from ..declaration.model import WeaverItemId, WeaverRepository
+from ..declaration.repository import parse_item_repository
+from ..store import LocalStore, Store
+from .bundle import BuildBundle, load_bundle
+from .installer import InstallationEnvironment, install_bundle
+from .planner import generate_item_build_bundle
+from .models import BuildPlan
+from .report import InstallationReport
+from .targets import ItemBindings, LakehouseBinding
+from .targets import WAREHOUSE_TARGET
+from .prune import (
+    TargetInventory,
+    read_lakehouse_inventory,
+    read_warehouse_inventory,
+)
+from ..catalogue.state import (
+    Catalogue,
+    Reconciliation,
+    read_catalogue_state,
+    reconcile_catalogue_state,
+)
+
+ARCHIVE_SUFFIX = ".weaver.zip"
+
+
+@dataclass(frozen=True)
+class MaterialisedTree:
+    """A source tree copied onto the current process's local filesystem."""
+
+    location: Location
+    store: LocalStore
+
+
+@dataclass(frozen=True)
+class PreparedRepository:
+    """A parsed repository and the process-local store that owns its files."""
+
+    repository: WeaverRepository
+    store: LocalStore
+
+
+@dataclass(frozen=True)
+class ItemBuildResult:
+    """Durable in-memory result of a temporary in-environment build."""
+
+    plan: BuildPlan
+    report: InstallationReport
+    repository_signature: str
+    item_signatures: Mapping[WeaverItemId, str]
+    archive: Location | None = None
+
+    @property
+    def bundle_id(self) -> str:
+        return self.plan.bundle_id
+
+
+@dataclass(frozen=True)
+class BuildState:
+    """Authoritative target state handed from Fabric to a local planner."""
+
+    catalogue: Catalogue
+    target_inventories: Mapping[WeaverItemId, TargetInventory]
+
+    def to_mapping(self) -> dict[str, object]:
+        return {
+            "format_version": 1,
+            "catalogue": self.catalogue.to_mapping(),
+            "target_inventories": [
+                {
+                    "item": str(item),
+                    "inventory": inventory.to_mapping(),
+                }
+                for item, inventory in sorted(
+                    self.target_inventories.items(), key=lambda pair: str(pair[0])
+                )
+            ],
+        }
+
+    @classmethod
+    def from_mapping(cls, mapping) -> "BuildState":
+        version = mapping.get("format_version")
+        if version != 1:
+            raise BuildError(
+                f"unsupported build state format_version {version!r}; expected 1"
+            )
+        return cls(
+            catalogue=Catalogue.from_mapping(mapping["catalogue"]),
+            target_inventories={
+                WeaverItemId.parse(entry["item"]): TargetInventory.from_mapping(
+                    entry["inventory"]
+                )
+                for entry in mapping.get("target_inventories", ())
+            },
+        )
+
+
+def catalogue_items_for_build(
+    repository: WeaverRepository, bindings: ItemBindings
+) -> tuple[WeaverItemId, ...]:
+    """Catalogue scope needed for selection and cross-item alias freshness."""
+
+    bound = set(bindings.by_item)
+    items = bound | {
+        alias.source.item
+        for alias in repository.aliases
+        if alias.destination.item in bound and alias.source.item not in bound
+    }
+    return tuple(sorted(items, key=str))
+
+
+def validate_build_request(
+    repository: WeaverRepository,
+    bindings: ItemBindings,
+    *,
+    control_lakehouse: LakehouseBinding,
+) -> tuple[WeaverItemId, ...]:
+    """Validate repository-dependent input before any target is contacted."""
+
+    if control_lakehouse is None:
+        raise BuildError("every build needs an explicit control-plane Lakehouse")
+    if not bindings.entries:
+        raise BuildError("at least one Weaver item must be bound")
+    known = {item.identity for item in repository.items}
+    unknown = set(bindings.by_item) - known
+    if unknown:
+        raise BuildError(
+            "binding names item(s) absent from the repository: "
+            + ", ".join(sorted(map(str, unknown)))
+        )
+    placed = {item for layer in repository.item_layers for item in layer}
+    missing = set(bindings.by_item) - placed
+    if missing:
+        raise BuildError(
+            "bound item(s) absent from the repository item graph: "
+            + ", ".join(sorted(map(str, missing)))
+        )
+    builtin = WeaverItemId.parse("Lakehouse/_weaver")
+    binding = bindings.by_item.get(builtin)
+    if binding is not None and not isinstance(binding.target, LakehouseBinding):
+        raise BuildError("Lakehouse/_weaver requires a Lakehouse binding")
+    if (
+        binding is not None
+        and binding.target.lakehouse.name != control_lakehouse.lakehouse.name
+    ):
+        raise BuildError(
+            "Lakehouse/_weaver must be bound to the explicit control-plane Lakehouse"
+        )
+    return catalogue_items_for_build(repository, bindings)
+
+
+def read_build_state(
+    bindings: ItemBindings,
+    *,
+    required_catalogue_items,
+    environment: InstallationEnvironment,
+    sql_by_item=None,
+) -> BuildState:
+    """Read only the authoritative state a source-independent planner needs."""
+
+    inventories = read_target_inventories(
+        bindings, environment=environment, sql_by_item=sql_by_item
+    )
+    if environment.spark is None:
+        raise BuildError("every build needs Spark to read and publish the catalogue")
+    workspace = environment.workspace
+    if workspace is None or not workspace.weaver_lakehouse:
+        raise BuildError("every build needs a Workspace with a Weaver Lakehouse")
+    from ..spark import SparkCatalogue
+    from ..targets import ItemRef
+
+    catalogue = SparkCatalogue(
+        environment.spark,
+        environment.resolver.spark_destination(ItemRef(workspace.weaver_lakehouse)),
+    )
+    return BuildState(
+        catalogue=read_catalogue_state(catalogue, required_catalogue_items),
+        target_inventories=inventories,
+    )
+
+
+@contextmanager
+def materialise_tree(
+    source: Location,
+    *,
+    store: Store,
+    prefix: str = "weaver-source-",
+) -> Iterator[MaterialisedTree]:
+    """Copy a store tree once to a temporary local directory.
+
+    FabricStore uses one recursive ``notebookutils.fs.cp`` operation. A generic
+    Store falls back to one listing and exactly one read per file, which makes the
+    same contract testable without Fabric.
+    """
+
+    if not store.exists(source):
+        raise BuildError(f"source does not exist: {source.value}")
+    if not store.is_directory(source):
+        raise BuildError(f"source is not a directory: {source.value}")
+
+    with tempfile.TemporaryDirectory(prefix=prefix) as temporary:
+        destination = Path(temporary) / source.name
+        copier = getattr(store, "copy_to_local", None)
+        if callable(copier):
+            copier(source, destination)
+        else:
+            _copy_tree_through_store(source, store, destination)
+        if not destination.is_dir():
+            raise BuildError(
+                f"materialising {source.value} did not create {destination}"
+            )
+        yield MaterialisedTree(Location(destination.as_posix()), LocalStore())
+
+
+@contextmanager
+def prepare_repository(
+    source: Location,
+    *,
+    source_store: Store,
+) -> Iterator[PreparedRepository]:
+    """Make a repository process-local when needed, then parse it completely."""
+
+    with _local_tree(source, source_store, prefix="weaver-repository-") as root:
+        store = LocalStore()
+        repository = parse_item_repository(Location(root.as_posix()), store=store)
+        yield PreparedRepository(repository=repository, store=store)
+
+
+def _copy_tree_through_store(source: Location, store: Store, destination: Path) -> None:
+    destination.mkdir(parents=True)
+    prefix = source.value.rstrip("/") + "/"
+    entries = store.list(source, recursive=True)
+    for entry in entries:
+        relative = entry.location.value[len(prefix) :]
+        target = destination.joinpath(*relative.split("/"))
+        if entry.is_directory:
+            target.mkdir(parents=True, exist_ok=True)
+    for entry in entries:
+        if entry.is_directory:
+            continue
+        relative = entry.location.value[len(prefix) :]
+        target = destination.joinpath(*relative.split("/"))
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_bytes(store.read(entry.location))
+
+
+def timestamped_archive_name(at: datetime | None = None) -> str:
+    """A sortable, collision-resistant physical name for an optional record."""
+
+    at = at or datetime.now(timezone.utc)
+    if at.tzinfo is None:
+        at = at.replace(tzinfo=timezone.utc)
+    stamp = at.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
+    return f"{stamp}{ARCHIVE_SUFFIX}"
+
+
+def persist_bundle_archive(
+    bundle: BuildBundle,
+    destination: Location,
+    *,
+    store: Store,
+) -> Location:
+    """Persist a complete bundle as one deterministic ZIP file."""
+
+    if not destination.name.endswith(ARCHIVE_SUFFIX):
+        raise BuildError(
+            f"bundle archive must end with {ARCHIVE_SUFFIX!r}: {destination.value}"
+        )
+    bundle_store = bundle.store or store
+    with _local_tree(bundle.location, bundle_store, prefix="weaver-bundle-source-") as root:
+        with tempfile.TemporaryDirectory(prefix="weaver-bundle-archive-") as temporary:
+            archive = Path(temporary) / destination.name
+            _write_archive(root, archive)
+            parent_value, separator, _ = destination.value.rpartition("/")
+            if separator:
+                parent = Location(parent_value)
+                if not store.exists(parent):
+                    store.make_directory(parent)
+            copier = getattr(store, "copy_from_local", None)
+            if callable(copier):
+                copier(archive, destination)
+            else:
+                store.write(destination, archive.read_bytes())
+    return destination
+
+
+@contextmanager
+def materialise_bundle_archive(
+    archive: Location,
+    *,
+    store: Store,
+) -> Iterator[BuildBundle]:
+    """Copy one archive locally, extract safely, and load its validated bundle."""
+
+    if not archive.name.endswith(ARCHIVE_SUFFIX):
+        raise BuildError(f"not a Weaver bundle archive: {archive.value}")
+    with tempfile.TemporaryDirectory(prefix="weaver-bundle-install-") as temporary:
+        temporary_path = Path(temporary)
+        local_archive = temporary_path / archive.name
+        copier = getattr(store, "copy_to_local", None)
+        if callable(copier):
+            copier(archive, local_archive)
+        else:
+            local_archive.write_bytes(store.read(archive))
+        root = temporary_path / "bundle"
+        root.mkdir()
+        _extract_archive(local_archive, root)
+        local_store = LocalStore()
+        yield load_bundle(Location(root.as_posix()), store=local_store)
+
+
+def install_bundle_archive(
+    archive: Location,
+    *,
+    archive_store: Store,
+    environment: InstallationEnvironment,
+) -> InstallationReport:
+    """Install a handover archive entirely from its temporary local extraction."""
+
+    with materialise_bundle_archive(archive, store=archive_store) as bundle:
+        return install_bundle(bundle, environment=environment)
+
+
+def build_item_repository(
+    repository: WeaverRepository,
+    *,
+    bindings: ItemBindings,
+    target_inventories: Mapping[WeaverItemId, TargetInventory],
+    reconciliation: Reconciliation,
+    environment: InstallationEnvironment,
+    source_store: Store,
+    control_lakehouse: LakehouseBinding,
+    archive: Location | None = None,
+    archive_store: Store | None = None,
+) -> ItemBuildResult:
+    """Generate and install from already parsed source and already read state.
+
+    This is the planner/executor seam.  It deliberately cannot materialise or
+    parse authored files, inspect a Workspace, or discover target state.
+    """
+
+    validate_build_request(
+        repository, bindings, control_lakehouse=control_lakehouse
+    )
+
+    with tempfile.TemporaryDirectory(prefix="weaver-build-") as temporary:
+        bundle = generate_item_build_bundle(
+            repository,
+            bindings=bindings,
+            output=Location((Path(temporary) / "bundle").as_posix()),
+            store=source_store,
+            target_inventories=target_inventories,
+            catalogue=reconciliation.catalogue,
+            stale_claims=reconciliation.stale_claims,
+            control_lakehouse=control_lakehouse,
+        )
+        report = install_bundle(bundle, environment=environment)
+        persisted = None
+        if archive is not None:
+            persisted = persist_bundle_archive(
+                bundle,
+                archive,
+                store=archive_store or environment.store,
+            )
+        return ItemBuildResult(
+            plan=bundle.plan,
+            report=report,
+            repository_signature=repository.signature,
+            item_signatures={item.identity: item.signature for item in repository.items},
+            archive=persisted,
+        )
+
+
+def build_uploaded_item_repository(
+    repository_root: Location,
+    *,
+    bindings: ItemBindings,
+    environment: InstallationEnvironment,
+    control_lakehouse: LakehouseBinding,
+    archive: Location | None = None,
+    archive_store: Store | None = None,
+    sql_by_item=None,
+) -> ItemBuildResult:
+    """Compatibility wrapper for a repository stored with the target estate."""
+
+    return build_item_repository_source(
+        repository_root,
+        source_store=environment.store,
+        bindings=bindings,
+        environment=environment,
+        control_lakehouse=control_lakehouse,
+        archive=archive,
+        archive_store=archive_store,
+        sql_by_item=sql_by_item,
+    )
+
+
+def build_item_repository_source(
+    source: Location,
+    *,
+    source_store: Store,
+    bindings: ItemBindings,
+    environment: InstallationEnvironment,
+    control_lakehouse: LakehouseBinding,
+    archive: Location | None = None,
+    archive_store: Store | None = None,
+    sql_by_item=None,
+) -> ItemBuildResult:
+    """Prepare an explicit source independently from the target, then build it."""
+
+    with prepare_repository(source, source_store=source_store) as prepared:
+        repository = prepared.repository
+        validate_build_request(
+            repository, bindings, control_lakehouse=control_lakehouse
+        )
+        inventories = read_target_inventories(
+            bindings, environment=environment, sql_by_item=sql_by_item
+        )
+        reconciled = read_reconciled_catalogue(
+            bindings,
+            inventories=inventories,
+            environment=environment,
+            repository=repository,
+        )
+        return build_item_repository(
+            repository,
+            bindings=bindings,
+            target_inventories=inventories,
+            reconciliation=reconciled,
+            environment=environment,
+            source_store=prepared.store,
+            control_lakehouse=control_lakehouse,
+            archive=archive,
+            archive_store=archive_store,
+        )
+
+
+def read_reconciled_catalogue(
+    bindings: ItemBindings,
+    *,
+    inventories,
+    environment: InstallationEnvironment,
+    repository=None,
+) -> Reconciliation:
+    """Read the Weaver Lakehouse catalogue and prove selected claims physically.
+
+    The read covers the bound items and, when a ``repository`` is given, the
+    items that *produce* what those items alias. Those producers are not being
+    built and nothing about them will be written — but their Registry rows carry
+    the build that published them, and comparing that against the alias's own row
+    is the only way to learn that a producer moved on while this consumer was not
+    looking (see
+    :func:`~weaver.build_bundle.incremental.stale_alias_destinations`).
+
+    They are read without an inventory, so nothing about them is reconciled away:
+    a build has no business proving claims about a target it was not pointed at.
+    """
+
+    items = {binding.item for binding in bindings.entries}
+    if repository is not None:
+        items |= {
+            alias.source.item
+            for alias in repository.aliases
+            if alias.destination.item in items and alias.source.item not in items
+        }
+
+    if environment.spark is None:
+        raise BuildError("every build needs Spark to read and publish the catalogue")
+    workspace = environment.workspace
+    if workspace is None or not workspace.weaver_lakehouse:
+        raise BuildError("every build needs a Workspace with a Weaver Lakehouse")
+    from ..spark import SparkCatalogue
+    from ..targets import ItemRef
+
+    catalogue = SparkCatalogue(
+        environment.spark,
+        environment.resolver.spark_destination(ItemRef(workspace.weaver_lakehouse)),
+    )
+    state = read_catalogue_state(catalogue, sorted(items, key=str))
+    return reconcile_catalogue_state(state, inventories=inventories)
+
+
+def read_target_inventories(
+    bindings: ItemBindings,
+    *,
+    environment: InstallationEnvironment,
+    sql_by_item=None,
+) -> dict:
+    """Read every selected physical target before planning begins."""
+
+    supplied_sql = sql_by_item or {}
+    inventories = {}
+    owned = []
+    try:
+        for binding in bindings.entries:
+            target = binding.to_bound_target()
+            if target.kind == WAREHOUSE_TARGET:
+                sql = supplied_sql.get(binding.item)
+                if sql is None:
+                    if environment.workspace is None:
+                        raise BuildError(
+                            f"reading Warehouse inventory for {binding.item} needs a Workspace"
+                        )
+                    from ..fabric.sql import fabric_sql_executor
+                    from ..targets import WarehouseTarget
+
+                    sql = fabric_sql_executor(
+                        WarehouseTarget.parse(target.item_id), environment.workspace
+                    )
+                    owned.append(sql)
+                inventories[binding.item] = read_warehouse_inventory(target, sql=sql)
+            else:
+                inventories[binding.item] = read_lakehouse_inventory(
+                    target,
+                    resolver=environment.resolver,
+                    store=environment.store,
+                    spark=environment.spark,
+                )
+        return inventories
+    finally:
+        for sql in owned:
+            if hasattr(sql, "close"):
+                sql.close()
+
+
+@contextmanager
+def _local_tree(
+    source: Location,
+    store: Store,
+    *,
+    prefix: str,
+) -> Iterator[Path]:
+    if isinstance(store, LocalStore) and not source.is_url:
+        yield source.path.resolve()
+        return
+    with materialise_tree(source, store=store, prefix=prefix) as tree:
+        yield tree.location.path
+
+
+def _write_archive(root: Path, destination: Path) -> None:
+    with zipfile.ZipFile(
+        destination, mode="w", compression=zipfile.ZIP_DEFLATED, compresslevel=9
+    ) as zipped:
+        for path in sorted(candidate for candidate in root.rglob("*") if candidate.is_file()):
+            relative = path.relative_to(root).as_posix()
+            info = zipfile.ZipInfo(relative, date_time=(1980, 1, 1, 0, 0, 0))
+            info.compress_type = zipfile.ZIP_DEFLATED
+            info.external_attr = (stat.S_IFREG | 0o644) << 16
+            zipped.writestr(info, path.read_bytes(), compresslevel=9)
+
+
+def _extract_archive(archive: Path, destination: Path) -> None:
+    with zipfile.ZipFile(archive) as zipped:
+        for info in zipped.infolist():
+            path = PurePosixPath(info.filename)
+            mode = info.external_attr >> 16
+            if (
+                path.is_absolute()
+                or not path.parts
+                or any(part in ("", ".", "..") for part in path.parts)
+                or stat.S_ISLNK(mode)
+            ):
+                raise BuildError(f"unsafe path in bundle archive: {info.filename!r}")
+        zipped.extractall(destination)

weaver/catalogue/__init__.py

@@ -0,0 +1,73 @@
+"""Weaver's central catalogue, scoped by logical item.
+
+An installation is identified by ``(item_type, item_name)`` and an object by
+that item identity plus ``(schema_name, object_name)``. Files objects use
+``Files/<schema>`` as their catalogue schema, so the same four-part identity
+covers them without a namespace column.
+"""
+
+from __future__ import annotations
+
+from .render import (
+    InstallationScope,
+    Row,
+    column_set,
+    identifier,
+    literal,
+    qualified_name,
+    render_delete_obsolete,
+    render_delete_scope,
+    render_merge,
+    sorted_rows,
+    typed_literal,
+)
+from .tables import (
+    ALIAS,
+    AUDIT_COLUMN_NAMES,
+    CATALOGUE_SCHEMA,
+    CATALOGUE_TABLES,
+    COLUMN_DICTIONARY,
+    DEPENDENCY,
+    DICTIONARY_TABLES,
+    FOLDER_DICTIONARY,
+    FOREIGN_KEY_DICTIONARY,
+    INDEX_DICTIONARY,
+    INSTALLATION,
+    ITEM_SCOPE_COLUMNS,
+    KEY_PRIMARY,
+    KEY_UNIQUE,
+    OBJECT_ROLES,
+    OBJECT_TYPES,
+    REGISTRY,
+    ROLE_DATA,
+    ROLE_LOAD,
+    SCHEMA_DICTIONARY,
+    SIGNATURE,
+    TABLE_DICTIONARY,
+    CatalogueColumn,
+    CatalogueTable,
+    table,
+)
+from .state import (
+    RegisteredDocument,
+    Catalogue,
+    Reconciliation,
+    for_targets,
+    retaining,
+    read_catalogue_state,
+    reconcile_catalogue_state,
+)
+
+__all__ = [
+    "ALIAS", "AUDIT_COLUMN_NAMES", "CATALOGUE_SCHEMA", "CATALOGUE_TABLES",
+    "COLUMN_DICTIONARY", "CatalogueColumn", "CatalogueTable", "DEPENDENCY",
+    "DICTIONARY_TABLES", "FOLDER_DICTIONARY", "FOREIGN_KEY_DICTIONARY",
+    "INDEX_DICTIONARY", "INSTALLATION", "ITEM_SCOPE_COLUMNS",
+    "InstallationScope", "KEY_PRIMARY", "KEY_UNIQUE",
+    "OBJECT_ROLES", "OBJECT_TYPES", "REGISTRY", "ROLE_DATA", "ROLE_LOAD", "Row",
+    "SCHEMA_DICTIONARY", "SIGNATURE", "TABLE_DICTIONARY", "column_set",
+    "identifier", "literal", "qualified_name", "render_delete_obsolete",
+    "render_delete_scope", "render_merge", "sorted_rows", "table", "typed_literal",
+    "RegisteredDocument", "Catalogue", "Reconciliation", "retaining", "for_targets",
+    "read_catalogue_state", "reconcile_catalogue_state",
+]