weaverstack 0.1.1__py3-none-any.whl

weaver/fabric/store.py

@@ -0,0 +1,198 @@
+"""Session-native storage for Weaver running inside Microsoft Fabric.
+
+This is the within-workspace counterpart to :class:`OneLakeDfsClient`.  It uses the
+``notebookutils.fs`` object already present in a Fabric Spark session and never
+authenticates back across the workspace boundary.
+
+Directory operations came first, for wipe. Byte reads and writes came next, for
+installing a build bundle in-session — the installer reads ``plan.yml`` and the
+generated payloads from OneLake and writes an install report back. They go
+through ``notebookutils.fs.head``/``put``, which exchange UTF-8 text: a build
+bundle's manifest and payloads are UTF-8, so this is exact for them. Arbitrary
+binary artefacts do not pass through those methods: recursive repository
+materialisation and bundle-archive persistence use ``notebookutils.fs.cp``
+between OneLake and the driver's ``file:/tmp`` filesystem.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from ..errors import CommandError
+from ..locations import Location
+from ..store import Entry, StoreError
+
+#: notebookutils.fs.head reads up to this many bytes. Bundle files are tiny; this
+#: ceiling is only a guard against an unexpectedly large one.
+_MAX_READ_BYTES = 256 * 1024 * 1024
+
+
+class FabricStore:
+    """Within-workspace Fabric storage, backed by ``notebookutils.fs``."""
+
+    def __init__(self, fs: Any | None = None) -> None:
+        if fs is None:
+            try:
+                from notebookutils import fs as notebook_fs
+            except ImportError as exc:
+                raise CommandError(
+                    "FabricStore is available only inside a Fabric session; "
+                    "desktop access uses OneLakeDfsClient explicitly"
+                ) from exc
+            fs = notebook_fs
+        self.fs = fs
+
+    @staticmethod
+    def _path(location: Location) -> str:
+        if not isinstance(location, Location):
+            raise CommandError(
+                f"store operations take a Location, got {type(location).__name__}"
+            )
+        if not location.value.startswith("abfss://"):
+            raise CommandError(
+                f"FabricStore needs an abfss location, got {location.value!r}"
+            )
+        return location.value
+
+    def exists(self, location: Location) -> bool:
+        return bool(self.fs.exists(self._path(location)))
+
+    def is_directory(self, location: Location) -> bool:
+        path = self._path(location)
+        if not self.fs.exists(path):
+            return False
+        parent, _, name = path.rstrip("/").rpartition("/")
+        if not parent:
+            return True
+        for info in self.fs.ls(parent):
+            info_path = str(getattr(info, "path", "")).rstrip("/")
+            info_name = str(getattr(info, "name", "")).rstrip("/")
+            if info_path == path.rstrip("/") or info_name == name:
+                return bool(getattr(info, "isDir", False))
+        return False
+
+    def list(self, location: Location, *, recursive: bool = False) -> list[Entry]:
+        path = self._path(location)
+        if not self.fs.exists(path):
+            raise StoreError(f"cannot list a location that does not exist: {path}")
+
+        entries = self._list_once(location)
+        if not recursive:
+            return entries
+
+        found = list(entries)
+        pending = [entry.location for entry in entries if entry.is_directory]
+        while pending:
+            directory = pending.pop()
+            children = self._list_once(directory)
+            found.extend(children)
+            pending.extend(
+                child.location for child in children if child.is_directory
+            )
+        return found
+
+    def _list_once(self, location: Location) -> list[Entry]:
+        entries = []
+        for info in self.fs.ls(self._path(location)):
+            name = str(getattr(info, "name", "")).rstrip("/")
+            raw_path = str(getattr(info, "path", "")).rstrip("/")
+            child = (
+                Location(raw_path)
+                if raw_path.startswith("abfss://")
+                else location / name
+            )
+            is_directory = bool(getattr(info, "isDir", False))
+            entries.append(
+                Entry(
+                    location=child,
+                    is_directory=is_directory,
+                    size=None if is_directory else int(getattr(info, "size", 0)),
+                )
+            )
+        return entries
+
+    def read(self, location: Location) -> bytes:
+        """The file's bytes, decoded as the UTF-8 text a bundle is made of."""
+
+        path = self._path(location)
+        if not self.fs.exists(path):
+            raise StoreError(f"cannot read a location that does not exist: {path}")
+        try:
+            text = self.fs.head(path, _MAX_READ_BYTES)
+        except Exception as exc:  # notebookutils raises a bare Py4J error
+            raise StoreError(f"cannot read {location.value}: {exc}") from exc
+        return text.encode("utf-8")
+
+    def write(self, location: Location, data: bytes) -> None:
+        """Write UTF-8 text (a bundle manifest, a payload, an install report)."""
+
+        path = self._path(location)
+        try:
+            self.fs.put(path, data.decode("utf-8"), True)
+        except Exception as exc:
+            raise StoreError(f"cannot write {location.value}: {exc}") from exc
+
+    def delete(self, location: Location, *, recursive: bool = False) -> None:
+        if not self.fs.rm(self._path(location), recurse=recursive):
+            raise StoreError(f"could not delete {location.value}")
+
+    def make_directory(self, location: Location) -> None:
+        if not self.fs.mkdirs(self._path(location)):
+            raise StoreError(f"could not create directory {location.value}")
+
+    def copy_to_local(self, source: Location, destination: Path) -> None:
+        """Copy OneLake content to the driver and remove Hadoop checksum debris.
+
+        ``notebookutils.fs.cp`` writes ``.filename.crc`` sidecars through
+        Hadoop's checked local filesystem. They are transport metadata, not
+        repository content. A same-shaped file that really exists in OneLake is
+        retained, so strict discovery still sees authored input exactly.
+        """
+
+        source_is_directory = self.is_directory(source)
+        retained_checksums: set[str] = set()
+        if source_is_directory:
+            prefix = source.value.rstrip("/") + "/"
+            retained_checksums = {
+                entry.location.value[len(prefix) :]
+                for entry in self.list(source, recursive=True)
+                if not entry.is_directory and _is_checksum_sidecar(entry.name)
+            }
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        target = f"file:{destination.as_posix()}"
+        try:
+            copied = self.fs.cp(
+                self._path(source), target, source_is_directory
+            )
+        except Exception as exc:
+            raise StoreError(
+                f"cannot materialise {source.value} at {destination}: {exc}"
+            ) from exc
+        if copied is False:
+            raise StoreError(f"could not materialise {source.value} at {destination}")
+        if source_is_directory:
+            for sidecar in destination.rglob(".*.crc"):
+                relative = sidecar.relative_to(destination).as_posix()
+                if relative not in retained_checksums:
+                    sidecar.unlink()
+        else:
+            generated = destination.parent / f".{destination.name}.crc"
+            generated.unlink(missing_ok=True)
+
+    def copy_from_local(self, source: Path, destination: Location) -> None:
+        """Copy one driver-local file or tree into OneLake without text decoding."""
+
+        origin = f"file:{source.as_posix()}"
+        try:
+            copied = self.fs.cp(origin, self._path(destination), source.is_dir())
+        except Exception as exc:
+            raise StoreError(
+                f"cannot persist {source} at {destination.value}: {exc}"
+            ) from exc
+        if copied is False:
+            raise StoreError(f"could not persist {source} at {destination.value}")
+
+
+def _is_checksum_sidecar(name: str) -> bool:
+    return name.startswith(".") and name.endswith(".crc")

weaver/initialise.py

@@ -0,0 +1,209 @@
+"""Bootstrapping the Weaver Lakehouse — Weaver installing its own control plane.
+
+Initialisation composes the package-owned catalogue item into the parsed
+repository in memory and builds it through the *ordinary* planner and installer. There is
+deliberately no second "create the control tables" path: if the catalogue needed
+privileged machinery to exist, the claim that a catalogue table is an ordinary
+Weaver object would be false, and every later assumption resting on that claim
+would be resting on nothing.
+
+The bootstrap looks circular and is not. One bundle does the whole of it, because
+the barriers already order it correctly:
+
+.. code-block:: text
+
+    create schema `_` and the catalogue tables
+    publish dictionaries and Installation as one batch
+    certify them in Registry last
+
+The catalogue's own DML runs after the tables it writes to exist, so no special
+first-run mode is needed and generation reads nothing — the statements are
+rendered from the projection and are correct against an absent catalogue as much
+as a populated one.
+
+The built-in ``_weaver`` item's inventory is scoped to the reserved ``_`` schema,
+so the ordinary authoritative prune cannot touch application schemas that happen
+to share the control Lakehouse.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+import tempfile
+from typing import Any
+
+from .build_bundle.models import BuildPlan
+from .build_bundle.installer import InstallationEnvironment, install_bundle
+from .build_bundle.planner import generate_item_build_bundle
+from .build_bundle.report import InstallationReport
+from .build_bundle.targets import ItemBinding, ItemBindings, LakehouseBinding
+from .build_bundle.workflow import read_reconciled_catalogue, read_target_inventories
+from .catalogue.tables import CATALOGUE_TABLES
+from .declaration import parse_item_repository
+from .declaration.model import WeaverItemId
+from .errors import CommandError
+from .locations import Location
+from .resolution import resolver_for
+from .store import LocalStore, Store
+from .targets import ItemRef
+from .workspaces import FabricWorkspace, LocalWorkspace
+
+#: The driver-local bundle directory used while initialisation runs. A fixed
+#: name is enough because the containing temporary directory is per invocation.
+INITIALISE_BUNDLE_NAME = "weaver-initialise"
+
+
+@dataclass(frozen=True)
+class InitialiseResult:
+    """What initialisation did, in terms a caller can print or assert on."""
+
+    item: str
+    weaver_lakehouse: str
+    plan: BuildPlan
+    report: InstallationReport
+
+    @property
+    def succeeded(self) -> bool:
+        return self.report.status == "succeeded"
+
+    @property
+    def tables(self) -> tuple[str, ...]:
+        return tuple(table.qualified for table in CATALOGUE_TABLES)
+
+    def to_mapping(self) -> dict[str, Any]:
+        """A plain structure, for a CLI to serialise. The CLI owns no semantics."""
+
+        return {
+            "item": self.item,
+            "weaver_lakehouse": self.weaver_lakehouse,
+            "bundle_id": self.plan.bundle_id,
+            "status": self.report.status,
+            "tables": list(self.tables),
+        }
+
+
+@dataclass(frozen=True)
+class PreparedWeaverLakehouse:
+    workspace: str
+    weaver_lakehouse: str
+    created: bool
+
+
+def prepare_weaver_lakehouse(
+    workspace,
+    *,
+    exists_ok: bool = False,
+    store: Store | None = None,
+    client=None,
+) -> PreparedWeaverLakehouse:
+    """Create the configured Weaver Lakehouse and its required Files areas."""
+
+    if not workspace.weaver_lakehouse:
+        raise CommandError("initialise requires a configured Weaver Lakehouse")
+    name = workspace.weaver_lakehouse
+    if isinstance(workspace, LocalWorkspace):
+        from .store import LocalStore
+
+        store = store or LocalStore()
+        resolver = resolver_for(workspace)
+        existed = store.exists(resolver.weaver_lakehouse)
+        if existed and not exists_ok:
+            raise CommandError(
+                f"Weaver Lakehouse {name!r} already exists; pass --exists-ok"
+            )
+        store.make_directory(resolver.files_root(ItemRef(name)))
+        store.make_directory(resolver.tables_root(ItemRef(name)))
+        return PreparedWeaverLakehouse(str(workspace.workspace), name, not existed)
+
+    if isinstance(workspace, FabricWorkspace):
+        from .fabric.resources import (
+            LAKEHOUSE,
+            ItemNotFoundError,
+            create_lakehouse,
+            find_item,
+            find_workspace,
+        )
+
+        physical_workspace = find_workspace(workspace.workspace, client=client)
+        try:
+            find_item(physical_workspace, name, item_type=LAKEHOUSE, client=client)
+        except ItemNotFoundError:
+            create_lakehouse(physical_workspace, name, client=client)
+            created = True
+        else:
+            if not exists_ok:
+                raise CommandError(
+                    f"Weaver Lakehouse {name!r} already exists; pass --exists-ok"
+                )
+            created = False
+        return PreparedWeaverLakehouse(workspace.workspace, name, created)
+
+    raise CommandError(f"unsupported Workspace type: {type(workspace).__name__}")
+
+
+def initialise_weaver_lakehouse(
+    *,
+    weaver_lakehouse: ItemRef,
+    workspace,
+    store: Store,
+    spark: Any = None,
+    output: Location | None = None,
+) -> InitialiseResult:
+    """Install Weaver's catalogue into the Weaver Lakehouse, through the normal build.
+
+    Idempotent to re-run in *shape*: the same package produces the same bundle, and
+    the catalogue's own reconciliation is a no-op when nothing changed.
+
+    An unchanged incremental plan emits no physical table work, so re-running
+    initialisation preserves existing catalogue rows while its catalogue tail
+    reconciles the built-in item.
+    """
+
+    resolver = resolver_for(workspace)
+    control = LakehouseBinding(lakehouse=weaver_lakehouse)
+    bindings = ItemBindings(
+        (
+            ItemBinding(
+                WeaverItemId.parse("Lakehouse/_weaver"),
+                control,
+            ),
+        )
+    )
+    environment = InstallationEnvironment(
+        store=store, resolver=resolver, spark=spark, workspace=workspace
+    )
+    with tempfile.TemporaryDirectory(prefix="weaver-initialise-") as temporary:
+        local_store = LocalStore()
+        repository_root = Path(temporary) / "repository"
+        repository_root.mkdir()
+        repository = parse_item_repository(
+            Location(repository_root.as_posix()), store=local_store
+        )
+        inventories = read_target_inventories(bindings, environment=environment)
+        reconciled = read_reconciled_catalogue(
+            bindings,
+            inventories=inventories,
+            environment=environment,
+            repository=repository,
+        )
+        bundle = generate_item_build_bundle(
+            repository,
+            bindings=bindings,
+            output=output
+            or Location((Path(temporary) / INITIALISE_BUNDLE_NAME).as_posix()),
+            store=local_store,
+            control_lakehouse=control,
+            target_inventories=inventories,
+            catalogue=reconciled.catalogue,
+            stale_claims=reconciled.stale_claims,
+        )
+
+        report = install_bundle(bundle, environment=environment)
+
+    return InitialiseResult(
+        item="Lakehouse/_weaver",
+        weaver_lakehouse=weaver_lakehouse.name,
+        plan=bundle.plan,
+        report=report,
+    )