weaverstack 0.1.1__py3-none-any.whl

weaver/store.py

@@ -0,0 +1,209 @@
+"""File transport — the operations Weaver performs on locations.
+
+This is transport, never policy. It knows how to list, read, write, delete and
+move; it has no opinion about what *should* be copied or deleted. Push,
+deployment and Folder reconciliation each carry their own rules and sit on top
+of these primitives:
+
+- push owns its destination subtree, so a file missing from source is deleted;
+- Folder reconciliation deletes only within its ``File key`` scope, and under
+  ``Incremental`` deletes nothing — governed by load policy, not by transport.
+
+Collapsing those into one ``sync(delete_missing=...)`` would put a data-correctness
+decision behind a transport flag, so they stay separate.
+
+**Listing carries metadata.** :class:`Entry` reports size, modification time and
+etag, because every incremental strategy needs them. A listing of bare names
+would foreclose all of them.
+
+There is deliberately no move operation. Staging promotion, destination
+replacement and atomic publication are all real needs, but their contract comes
+from the load algorithm, not from a guess made here — and the mechanisms differ
+enough (a local rename, a OneLake copy, an in-session ``notebookutils.fs.mv``)
+that a single reassuring name would hide materially different cost. Load
+introduces whatever it actually needs, named for what it does.
+"""
+
+from __future__ import annotations
+
+import shutil
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+from .errors import CommandError, WeaverError
+from .locations import Location
+
+
+class StoreError(WeaverError):
+    """Raised when a store operation fails."""
+
+
+@dataclass(frozen=True)
+class Entry:
+    """One listed item, with enough metadata to diff without reading."""
+
+    location: Location
+    is_directory: bool
+    size: int | None = None
+    modified: datetime | None = None
+    etag: str | None = None
+
+    @property
+    def name(self) -> str:
+        return self.location.name
+
+
+@runtime_checkable
+class Store(Protocol):
+    """File transport within one workspace.
+
+    A within-workspace store operates beneath a local root or through Fabric's
+    session-native utilities. A cross-boundary caller may also implement this
+    protocol (the desktop's OneLake DFS client) and inject it explicitly, but
+    moving files from a laptop into Fabric remains CLI orchestration rather than
+    a workspace default.
+    """
+
+    def exists(self, location: Location) -> bool: ...
+
+    def is_directory(self, location: Location) -> bool: ...
+
+    def list(self, location: Location, *, recursive: bool = False) -> list[Entry]: ...
+
+    def read(self, location: Location) -> bytes: ...
+
+    def write(self, location: Location, data: bytes) -> None: ...
+
+    def delete(self, location: Location, *, recursive: bool = False) -> None: ...
+
+    def make_directory(self, location: Location) -> None: ...
+
+
+class LocalStore:
+    """Filesystem implementation.
+
+    Not sandboxed to a workspace root, because push reads from arbitrary source
+    directories. Containment comes from name validation in
+    :mod:`weaver.targets`, which rejects separators and traversal.
+    """
+
+    def _local(self, location: Location) -> Path:
+        if not isinstance(location, Location):
+            raise CommandError(
+                f"store operations take a Location, got {type(location).__name__}"
+            )
+        if location.is_url:
+            raise CommandError(f"LocalStore cannot address the URL location {location.value!r}")
+        return location.path
+
+    def exists(self, location: Location) -> bool:
+        return self._local(location).exists()
+
+    def is_directory(self, location: Location) -> bool:
+        return self._local(location).is_dir()
+
+    def list(self, location: Location, *, recursive: bool = False) -> list[Entry]:
+        root = self._local(location)
+        if not root.exists():
+            raise StoreError(f"cannot list a location that does not exist: {location.value}")
+        if not root.is_dir():
+            raise StoreError(f"cannot list a file: {location.value}")
+        paths = sorted(root.rglob("*") if recursive else root.glob("*"))
+        return [self._entry(path, location, root) for path in paths]
+
+    def _entry(self, path: Path, root_location: Location, root: Path) -> Entry:
+        relative = path.relative_to(root).as_posix()
+        info = path.stat()
+        is_directory = path.is_dir()
+        return Entry(
+            location=root_location.join(*relative.split("/")),
+            is_directory=is_directory,
+            size=None if is_directory else info.st_size,
+            modified=datetime.fromtimestamp(info.st_mtime, tz=timezone.utc),
+        )
+
+    def read(self, location: Location) -> bytes:
+        path = self._local(location)
+        try:
+            return path.read_bytes()
+        except OSError as exc:
+            raise StoreError(f"cannot read {location.value}: {exc}") from exc
+
+    def write(self, location: Location, data: bytes) -> None:
+        path = self._local(location)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        try:
+            path.write_bytes(data)
+        except OSError as exc:
+            raise StoreError(f"cannot write {location.value}: {exc}") from exc
+
+    def delete(self, location: Location, *, recursive: bool = False) -> None:
+        path = self._local(location)
+        if path.is_symlink():
+            # A link is removed, never followed: the alias it stands for is
+            # disposable, the object it points at belongs to another item.
+            path.unlink()
+            return
+        if not path.exists():
+            return
+        if path.is_dir():
+            if not recursive:
+                raise StoreError(
+                    f"{location.value} is a directory — pass recursive=True to delete it"
+                )
+            shutil.rmtree(path)
+        else:
+            path.unlink()
+
+    def make_directory(self, location: Location) -> None:
+        self._local(location).mkdir(parents=True, exist_ok=True)
+
+    def copy_to_local(self, source: Location, destination: Path) -> None:
+        """Copy one local file or tree to an exact driver-local path."""
+
+        source_path = self._local(source)
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        if source_path.is_dir():
+            shutil.copytree(source_path, destination)
+        else:
+            shutil.copy2(source_path, destination)
+
+    def copy_from_local(self, source: Path, destination: Location) -> None:
+        """Copy one driver-local file or tree to an exact local location."""
+
+        destination_path = self._local(destination)
+        destination_path.parent.mkdir(parents=True, exist_ok=True)
+        if source.is_dir():
+            shutil.copytree(source, destination_path)
+        else:
+            shutil.copy2(source, destination_path)
+
+    def link(self, source: Location, destination: Location) -> None:
+        """Make ``destination`` refer to ``source`` without copying it.
+
+        The emulator's counterpart of a OneLake shortcut, and the reason it is a
+        link rather than a copy: a shortcut has no bytes of its own, so a copy
+        would drift the moment the source was rebuilt and would make the emulator
+        stop reproducing what Fabric does.
+
+        Not part of the :class:`Store` protocol. A shortcut in Fabric is made
+        through the workspace API, not through file transport, so an environment
+        either offers this or offers that — see
+        :mod:`weaver.build_bundle.executors.alias`.
+        """
+
+        source_path = self._local(source)
+        if not source_path.exists():
+            raise StoreError(f"cannot link to something that does not exist: {source.value}")
+        destination_path = self._local(destination)
+        destination_path.parent.mkdir(parents=True, exist_ok=True)
+        if destination_path.is_symlink():
+            destination_path.unlink()
+        try:
+            destination_path.symlink_to(source_path, target_is_directory=source_path.is_dir())
+        except OSError as exc:
+            raise StoreError(
+                f"cannot link {destination.value} to {source.value}: {exc}"
+            ) from exc

weaver/targets.py

@@ -0,0 +1,257 @@
+"""Physical identities — the third level of the four-level model.
+
+Weaver names things the way SQL does::
+
+    Server . Database . Schema . Object
+
+    4         3          2        1
+
++-------+-------------------+-------------------------------+
+| Level | Fabric            | Local                         |
++=======+===================+===============================+
+| 4     | workspace         | root directory                |
+| 3     | Lakehouse,        | subdirectory                  |
+|       | Warehouse,        |                               |
+|       | Environment       |                               |
+| 2     | schema            | schema directory              |
+| 1     | table, view,      | table or folder               |
+|       | folder, procedure |                               |
++-------+-------------------+-------------------------------+
+
+Level 4 is the only level written down in Workspace configuration. Level 3
+needs no alias because an item is *uniquely
+identifiable within its workspace* — so it is referred to by its real name, never by
+an alias. That is uniqueness, not invariance: promoting ``Dev_Lakehouse`` to
+``Prod_Lakehouse`` inside one workspace is ordinary, so level-3 names are always
+supplied explicitly at the call site and never inferred.
+
+Levels 2 and 1 come from the object's own metadata (``Schema.Object``) and do
+not appear here.
+
+This module is pure identity. Nothing here resolves an item to a path, an ID or
+an endpoint — that is the local or Fabric resolver's job.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .errors import IdentityError
+
+#: The Lakehouse area holding folder materialisations. Written explicitly in a
+#: folder target because it is what the user sees in the Fabric UI. The Delta
+#: area (``Tables``) is implicit for the same reason: a Delta target names a
+#: Lakehouse, and the area follows from the object kind.
+FILES_AREA = "Files"
+
+_ILLEGAL_IN_NAME = ("/", "\\", ":", "*", "?", '"', "<", ">", "|")
+
+
+def validate_name(value: object, *, what: str) -> str:
+    """Validate one level-3 or path name and return it stripped."""
+
+    if not isinstance(value, str):
+        raise IdentityError(f"{what} must be a string, got {type(value).__name__}")
+    name = value.strip()
+    if not name:
+        raise IdentityError(f"{what} must not be empty")
+    for character in _ILLEGAL_IN_NAME:
+        if character in name:
+            raise IdentityError(f"{what} must not contain {character!r}: {value!r}")
+    if set(name) == {"."}:
+        raise IdentityError(f"{what} must not be {name!r}")
+    return name
+
+
+def _split(text: object, *, what: str) -> list[str]:
+    if not isinstance(text, str):
+        raise IdentityError(f"{what} must be a string, got {type(text).__name__}")
+    if not text.strip():
+        raise IdentityError(f"{what} must not be empty")
+    return [segment for segment in text.strip().strip("/").split("/")]
+
+
+@dataclass(frozen=True)
+class ItemRef:
+    """A uniquely-named item within a workspace — level three.
+
+    A Lakehouse, a Warehouse or a Fabric Environment. Which of those it must be
+    is decided by the slot it is used in, never by the name itself: the same
+    string passed as a ``delta_target`` names a Lakehouse and passed as a
+    ``sql_target`` names a Warehouse.
+    """
+
+    name: str
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "name", validate_name(self.name, what="item name"))
+
+    @classmethod
+    def parse(cls, text: str) -> "ItemRef":
+        segments = _split(text, what="item name")
+        if len(segments) != 1:
+            raise IdentityError(f"item name must be a single name, got {text!r}")
+        return cls(name=segments[0])
+
+    def __str__(self) -> str:
+        return self.name
+
+
+@dataclass(frozen=True)
+class FolderTarget:
+    """A Lakehouse Files area — ``Sales/Files``, and nothing further.
+
+    A folder object's physical location is derived from its identity alone:
+    ``Files/<Schema>/<Object>``. A binding-level subpath used to be accepted here,
+    and it made that derivation untrue — the same object landed in different
+    places depending on how its item was bound, so authored code could not compose
+    its own path and neither could anything else without carrying the binding
+    around. One deterministic location per object is worth more than a
+    configurable root.
+    """
+
+    lakehouse: ItemRef
+
+    @classmethod
+    def parse(cls, text: str) -> "FolderTarget":
+        segments = _split(text, what="folder target")
+        if len(segments) != 2:
+            raise IdentityError(
+                f"folder target must be '<Lakehouse>/{FILES_AREA}', got {text!r}"
+                + (
+                    " — a folder object lands at Files/<Schema>/<Object>, so there is "
+                    "nothing to configure beneath the area"
+                    if len(segments) > 2
+                    else ""
+                )
+            )
+        if segments[1] != FILES_AREA:
+            raise IdentityError(
+                f"folder target must name the {FILES_AREA!r} area after the Lakehouse, "
+                f"got {segments[1]!r} in {text!r}"
+            )
+        return cls(lakehouse=ItemRef(segments[0]))
+
+    def __str__(self) -> str:
+        return f"{self.lakehouse.name}/{FILES_AREA}"
+
+
+@dataclass(frozen=True)
+class DeltaTarget:
+    """A Lakehouse holding Delta tables.
+
+    Named bare — ``Sales``. The ``Tables`` area is implicit because the object
+    kind already determines it.
+    """
+
+    lakehouse: ItemRef
+
+    @classmethod
+    def parse(cls, text: str) -> "DeltaTarget":
+        segments = _split(text, what="delta target")
+        if len(segments) != 1:
+            raise IdentityError(
+                "delta target must name a Lakehouse only — the 'Tables' area is implicit, "
+                f"got {text!r}"
+            )
+        return cls(lakehouse=ItemRef(segments[0]))
+
+    def __str__(self) -> str:
+        return self.lakehouse.name
+
+
+@dataclass(frozen=True)
+class WarehouseTarget:
+    """A Warehouse holding SQL tables, views and generated load procedures."""
+
+    warehouse: ItemRef
+
+    @classmethod
+    def parse(cls, text: str) -> "WarehouseTarget":
+        segments = _split(text, what="warehouse target")
+        if len(segments) != 1:
+            raise IdentityError(f"warehouse target must name a Warehouse only, got {text!r}")
+        return cls(warehouse=ItemRef(segments[0]))
+
+    def __str__(self) -> str:
+        return self.warehouse.name
+
+
+# --- the one typed physical grammar the public operations share ---------------
+#
+# ``Lakehouse/Name`` and ``Warehouse/Name`` are what a caller writes at every
+# boundary that names a whole physical item: a build binding's left-hand side, a
+# wipe target, an unbind target, a load target. One parser, deliberately — four
+# spellings of one grammar is four places for it to drift, and the drift would
+# show up as one operation accepting a target another refuses.
+#
+# It returns the *existing* typed targets rather than a fifth wrapper, so what a
+# caller gets back is what the resolvers and executors already take.
+
+LAKEHOUSE_KIND = "Lakehouse"
+WAREHOUSE_KIND = "Warehouse"
+
+#: How the grammar spells each kind, in the order the error message lists them.
+PHYSICAL_KINDS = (LAKEHOUSE_KIND, WAREHOUSE_KIND)
+
+_PHYSICAL_TYPES = {LAKEHOUSE_KIND: DeltaTarget, WAREHOUSE_KIND: WarehouseTarget}
+
+
+def parse_physical_target(
+    text: object, *, what: str = "target", error: type[Exception] = IdentityError
+):
+    """``Lakehouse/Name`` or ``Warehouse/Name``, as the typed physical target.
+
+    ``what`` names the caller's own noun so the message reads in that operation's
+    vocabulary — "a wipe target must …", "a load target must …". ``error`` is the
+    class the caller's boundary raises, because *which* error a malformed request
+    produces belongs to the operation and not to the grammar.
+    """
+
+    if not isinstance(text, str):
+        raise error(f"{what}s must be strings, got {type(text).__name__}")
+    parts = text.strip().strip("/").split("/")
+    if len(parts) != 2 or not all(part.strip() for part in parts):
+        raise error(
+            f"a {what} must name a whole physical item as "
+            + " or ".join(f"{kind}/Name" for kind in PHYSICAL_KINDS)
+            + f", got {text!r}"
+        )
+    kind, name = parts[0].strip(), parts[1].strip()
+    if kind not in _PHYSICAL_TYPES:
+        raise error(
+            f"a {what} must start with "
+            + " or ".join(PHYSICAL_KINDS)
+            + f", got {kind!r}"
+        )
+    return _PHYSICAL_TYPES[kind](ItemRef.parse(name))
+
+
+def physical_kind(target) -> str:
+    """``Lakehouse`` or ``Warehouse`` for one typed physical target."""
+
+    if isinstance(target, DeltaTarget):
+        return LAKEHOUSE_KIND
+    if isinstance(target, WarehouseTarget):
+        return WAREHOUSE_KIND
+    raise IdentityError(
+        f"{type(target).__name__} is not a typed physical target"
+    )
+
+
+def physical_item(target) -> ItemRef:
+    """The item one typed physical target names."""
+
+    if isinstance(target, DeltaTarget):
+        return target.lakehouse
+    if isinstance(target, WarehouseTarget):
+        return target.warehouse
+    raise IdentityError(
+        f"{type(target).__name__} is not a typed physical target"
+    )
+
+
+def physical_target_text(target) -> str:
+    """One typed physical target, spelled back in the grammar it was parsed from."""
+
+    return f"{physical_kind(target)}/{physical_item(target).name}"

weaver/task_logging.py

@@ -0,0 +1,215 @@
+"""Immutable file evidence for one top-level Weaver task.
+
+Generic across Weaver's five top-level tasks — ``wipe``, ``mirror``, ``build``,
+``load`` and ``test`` — because a log that knew what a load was would need a
+second one the first time anything else wanted evidence, and the two would drift.
+Nothing here understands a DAG: a task writes a plan, some steps and a
+completion, and what those contain is the task's business.
+
+.. code-block:: text
+
+    Files/_/Log/
+    └── task_date=2026-08-03/
+        └── 20260803T091522.123456Z_load_<task-uuid>/
+            ├── plan.json
+            ├── 20260803T091523.012345Z_load_<step-uuid>.json
+            ├── 20260803T091530.441092Z_refresh_<step-uuid>.json
+            └── 20260803T091540.102334Z_complete_<task-uuid>.json
+
+**The folder is a declared Weaver document, not a path this module knows.**
+``_.Log`` is an ordinary ``Folder`` in the built-in control-plane item, so it is
+projected into the catalogue, inventoried, installed, converged on and protected
+from prune by the machinery that already exists — and this module asks a resolver
+where that folder is rather than composing ``Files/_/Log`` for itself. A special
+path known only to the logger would need its own creation rule, its own prune
+exemption and its own removal rule, each of which is a rule nothing else has.
+
+**Nothing is ever rewritten.** A step's file is written when the step finishes
+and is never touched again, which is what makes the log usable after an
+interruption: the plan says what was intended, the step files say what completed,
+and the *absence* of a completion file is how you know the task did not finish.
+A log the runner updated in place could not say that — a crashed task and a
+finished one would look the same.
+
+**A dry run writes nothing at all.** Validation is not execution, and a folder of
+plausible-looking steps that never ran is worse than no folder: it is evidence of
+work nobody did. The dry run's complete result is returned in memory instead.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Callable, Mapping
+
+from .catalogue.builtin import LOG_FOLDER, LOG_FOLDER_ID
+from .catalogue.tables import CATALOGUE_SCHEMA
+from .errors import CommandError
+from .locations import Location
+from .store import Store
+from .targets import FolderTarget, ItemRef
+
+#: Weaver's top-level tasks. A task type is part of a folder name, so a reader
+#: can see what ran without opening anything.
+TASK_TYPES = ("wipe", "mirror", "build", "load", "test")
+
+#: The date partition's key. ``task_date`` is the UTC date the task *started*; a
+#: task that crosses midnight stays in the partition it began in, because a run
+#: is one thing and splitting it across two partitions would make it two.
+DATE_PARTITION = "task_date"
+
+PLAN_FILE = "plan.json"
+COMPLETE_STEP = "complete"
+
+_TIMESTAMP = "%Y%m%dT%H%M%S.%f"
+
+
+def log_folder(resolver: Any, weaver_lakehouse: ItemRef | str) -> Location:
+    """Where the declared ``_.Log`` folder materialises, per the resolver.
+
+    Derived from the folder's *identity* through the same resolution every other
+    Folder object goes through, so the logger writes where the build installed
+    rather than where the logger guessed.
+    """
+
+    item = ItemRef(weaver_lakehouse) if isinstance(weaver_lakehouse, str) else weaver_lakehouse
+    return resolver.folder_object(FolderTarget(item), CATALOGUE_SCHEMA, LOG_FOLDER)
+
+
+@dataclass(frozen=True)
+class TaskLog:
+    """One task's evidence folder, open for writing.
+
+    Holds no mutable run state beyond the files it has written — which is the
+    point. Every method appends; nothing amends.
+    """
+
+    task_id: str
+    task_type: str
+    started: datetime
+    root: Location
+    store: Store
+    clock: Callable[[], datetime] = field(default=lambda: datetime.now(timezone.utc))
+    #: Every file written, in the order it was written. A record for the caller,
+    #: never read back to decide anything.
+    written: list[Location] = field(default_factory=list)
+
+    @property
+    def partition(self) -> str:
+        return f"{DATE_PARTITION}={self.started.date().isoformat()}"
+
+    def write_plan(self, plan: Mapping[str, Any]) -> Location:
+        """Write the complete intended task, once, before execution begins."""
+
+        return self._write(PLAN_FILE, {**plan, **self._identity()})
+
+    def write_step(self, step_type: str, result: Mapping[str, Any]) -> Location:
+        """Write one executed step's immutable result.
+
+        ``step_type`` is the broad kind, and it is in the *filename* so the folder
+        reads as a sequence without opening anything. The exact identity — which
+        object, in which target, through which primitive — is in the JSON, where
+        it can be as precise as it needs to be.
+        """
+
+        name = (
+            f"{self._stamp()}_{_slug(step_type)}_{uuid.uuid4().hex}.json"
+        )
+        return self._write(name, {**result, **self._identity(), "step_type": step_type})
+
+    def write_completion(self, summary: Mapping[str, Any]) -> Location:
+        """Write the one file whose presence means the task finished normally."""
+
+        name = f"{self._stamp()}_{COMPLETE_STEP}_{self.task_id}.json"
+        return self._write(
+            name,
+            {
+                **summary,
+                **self._identity(),
+                "started_at": _iso(self.started),
+                "ended_at": _iso(self.clock()),
+            },
+        )
+
+    # --- writing --------------------------------------------------------------
+
+    def _identity(self) -> dict[str, str]:
+        return {"task_id": self.task_id, "task_type": self.task_type}
+
+    def _stamp(self) -> str:
+        return _stamp(self.clock())
+
+    def _write(self, name: str, payload: Mapping[str, Any]) -> Location:
+        location = self.root / name
+        self.store.make_directory(self.root)
+        self.store.write(
+            location,
+            json.dumps(payload, indent=2, sort_keys=True, default=str).encode("utf-8"),
+        )
+        self.written.append(location)
+        return location
+
+
+def open_task_log(
+    *,
+    task_type: str,
+    folder: Location,
+    store: Store,
+    task_id: str | None = None,
+    clock: Callable[[], datetime] | None = None,
+) -> TaskLog:
+    """Create one task's folder beneath the declared log folder.
+
+    ``folder`` is where ``_.Log`` materialises — see :func:`log_folder`. Taking
+    the location rather than resolving it means the logger can be tested against
+    its folder abstraction, with no control plane anywhere near it.
+    """
+
+    if task_type not in TASK_TYPES:
+        raise CommandError(
+            f"{task_type!r} is not a Weaver task type; expected one of "
+            + ", ".join(TASK_TYPES)
+        )
+    clock = clock or (lambda: datetime.now(timezone.utc))
+    started = clock()
+    task_id = task_id or uuid.uuid4().hex
+    partition = f"{DATE_PARTITION}={started.date().isoformat()}"
+    name = f"{_stamp(started)}_{_slug(task_type)}_{task_id}"
+    root = folder / partition / name
+    store.make_directory(root)
+    return TaskLog(
+        task_id=task_id,
+        task_type=task_type,
+        started=started,
+        root=root,
+        store=store,
+        clock=clock,
+    )
+
+
+def _stamp(moment: datetime) -> str:
+    """``20260803T091522.123456Z`` — sortable, and unambiguous about its zone."""
+
+    return moment.astimezone(timezone.utc).strftime(_TIMESTAMP) + "Z"
+
+
+def _iso(moment: datetime) -> str:
+    return moment.astimezone(timezone.utc).isoformat()
+
+
+def _slug(value: str) -> str:
+    return str(value).replace("/", "-").replace(" ", "-")
+
+
+__all__ = [
+    "COMPLETE_STEP",
+    "DATE_PARTITION",
+    "LOG_FOLDER_ID",
+    "PLAN_FILE",
+    "TASK_TYPES",
+    "TaskLog",
+    "log_folder",
+    "open_task_log",
+]