weaverstack 0.1.1__py3-none-any.whl

weaver/load_resolution.py

@@ -0,0 +1,386 @@
+"""Locating what each planned node would dispatch, without dispatching it.
+
+The seam between "what should run" and "run it", and it exists as its own layer
+because the two questions fail for entirely different reasons. A planning fault
+is a wrong graph; a resolution fault is a graph that is right about an estate
+that is not there — a procedure that was never installed, a module deleted from
+the runtime tree, a Warehouse that has been wiped. Keeping them apart means a
+failure names which of the two it is.
+
+The question this layer answers is exactly:
+
+    Can the orchestrator locate what it would dispatch?
+
+and deliberately not *does the primitive work*. Each primitive is independently
+runnable and independently tested; re-proving that here would make orchestration
+the owner of behaviour it does not implement.
+
+**Physical state arrives as an inventory, not as a live connection.** The same
+:class:`~weaver.build_bundle.prune.TargetInventory` a build reads before planning
+answers every existence question here — is the target there, is the procedure
+there, is the deployed file there, does the table it loads into exist. So this
+module is pure: a graph and some observed state in, a resolved plan out. The
+reading happens once, above, where the session already is.
+
+Dry run is this layer plus nothing. §11's list — read the catalogue, reverse the
+bindings, build the graph, check it is acyclic, order it, resolve every node,
+verify targets, procedures, files, modules and refresh capability — is the whole
+orchestration path with dispatch removed, which is why a dry run can be complete
+and still touch nothing.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Mapping
+
+from .build_bundle.prune import TargetInventory
+from .etl import load_procedure_name
+from .load_plan import (
+    ENDPOINT_REFRESH,
+    PYTHON_FOLDER,
+    PYTHON_TABLE,
+    SPARK_SQL_FILE,
+    WAREHOUSE_PROCEDURE,
+    LoadDag,
+    LoadNode,
+    PhysicalTargetRef,
+)
+from .load_report import (
+    BLOCKED,
+    DEPENDENCY_BLOCKED,
+    DISPATCH_LOCATION_MISSING,
+    INVALID,
+    MODULE_IMPORT_FAILURE,
+    TARGET_MISSING,
+    VALIDATED,
+    LoadMessage,
+    LoadNodeReport,
+    error,
+    warning,
+)
+from .targets import ItemRef
+
+#: What a refresh resolves to when the host can perform one. Not a physical
+#: object — a Lakehouse's SQL analytics endpoint is a capability of the item, so
+#: the location names the item and the capability rather than a path.
+ENDPOINT_SUFFIX = "sql_endpoint"
+
+#: Why a refresh could not be resolved on this host. The emulator has no SQL
+#: analytics endpoint at all, which is an honest absence rather than a fault —
+#: the build's own executor skips for the same reason.
+REFRESH_UNSUPPORTED = "SQL endpoint refresh is unsupported in this environment"
+
+
+@dataclass(frozen=True)
+class LoadEnvironment:
+    """Runtime services and the observed physical state of one load run.
+
+    ``inventories`` is keyed by the physical target's public spelling —
+    ``Lakehouse/Raw_LH`` — because that is what a caller wrote and what a report
+    prints, so a key that could not be read back would put a second vocabulary
+    between the request and the answer. A target with no entry is a target that
+    is not there.
+
+    ``sql`` is per Warehouse rather than one connection, because a run may span
+    two of them and a Warehouse is reached over its own TDS endpoint.
+    """
+
+    resolver: Any = None
+    inventories: Mapping[str, TargetInventory] = field(default_factory=dict)
+    store: Any = None
+    spark: Any = None
+    sql: Mapping[str, Any] = field(default_factory=dict)
+    workspace: Any = None
+
+    def inventory(self, target: PhysicalTargetRef) -> TargetInventory | None:
+        return self.inventories.get(str(target))
+
+    def sql_for(self, target: PhysicalTargetRef) -> Any:
+        return self.sql.get(target.name)
+
+    def can_refresh(self) -> bool:
+        return callable(getattr(self.resolver, "refresh_sql_endpoint", None))
+
+
+@dataclass(frozen=True)
+class ResolvedLoadNode:
+    """One node, and what the orchestrator would actually reach for.
+
+    ``target_exists`` and ``primitive_exists`` are separate answers because they
+    are separate failures: a Warehouse that has been wiped and a procedure that
+    was never generated both stop this node, and telling a reader which one it
+    was is the whole value of resolving ahead of dispatching.
+    """
+
+    node: LoadNode
+    dispatch_location: str | None = None
+    target_exists: bool = False
+    primitive_exists: bool = False
+    #: The class a deployed Python module must define, for the two Python kinds.
+    expected_class: str | None = None
+    validation_messages: tuple[LoadMessage, ...] = ()
+    #: A capability this host does not have, so the node is omitted rather than
+    #: failed. Only an endpoint refresh in an environment with no endpoint.
+    unsupported: bool = False
+
+    @property
+    def node_id(self) -> str:
+        return self.node.node_id
+
+    @property
+    def valid(self) -> bool:
+        return not any(
+            message.severity == "error" for message in self.validation_messages
+        )
+
+
+@dataclass(frozen=True)
+class ResolvedLoadPlan:
+    """A whole graph resolved against one environment."""
+
+    dag: LoadDag
+    nodes: tuple[ResolvedLoadNode, ...]
+
+    @property
+    def by_id(self) -> Mapping[str, ResolvedLoadNode]:
+        return {node.node_id: node for node in self.nodes}
+
+    @property
+    def order(self) -> tuple[ResolvedLoadNode, ...]:
+        resolved = self.by_id
+        return tuple(resolved[node.node_id] for node in self.dag.order())
+
+    @property
+    def blocked(self) -> Mapping[str, frozenset[str]]:
+        """Which invalid node blocks each node that may not run because of it."""
+
+        blocked: dict[str, set[str]] = {}
+        for resolved in self.nodes:
+            if resolved.valid:
+                continue
+            for downstream in self.dag.descendants(resolved.node_id):
+                blocked.setdefault(downstream, set()).add(resolved.node_id)
+        return {node: frozenset(causes) for node, causes in blocked.items()}
+
+
+def resolve_load_plan(
+    dag: LoadDag, *, environment: LoadEnvironment
+) -> ResolvedLoadPlan:
+    """Resolve every node in ``dag`` to the installed primitive it would run."""
+
+    return ResolvedLoadPlan(
+        dag=dag,
+        nodes=tuple(_resolve_node(node, environment) for node in dag.order()),
+    )
+
+
+def _resolve_node(node: LoadNode, environment: LoadEnvironment) -> ResolvedLoadNode:
+    inventory = environment.inventory(node.physical_target)
+    if node.primitive_kind == ENDPOINT_REFRESH:
+        return _resolve_refresh(node, environment, inventory)
+    messages: list[LoadMessage] = []
+    if inventory is None:
+        messages.append(
+            error(
+                TARGET_MISSING,
+                f"{node.physical_target} is not present, so {node.node_id} has "
+                "nowhere to run",
+                source="load_resolution",
+            )
+        )
+    location, expected_class = _dispatch_location(node, environment)
+    if location is None:
+        messages.append(
+            error(
+                DISPATCH_LOCATION_MISSING,
+                f"{node.node_id} names primitive kind {node.primitive_kind!r}, "
+                "which this environment cannot address",
+                source="load_resolution",
+            )
+        )
+    primitive_exists = _holds(inventory, node.primitive_object)
+    if inventory is not None and not primitive_exists:
+        messages.append(
+            error(
+                DISPATCH_LOCATION_MISSING,
+                f"{node.node_id} would dispatch {location}, which is not installed "
+                f"in {node.physical_target}",
+                source="load_resolution",
+            )
+        )
+    if inventory is not None and not _holds(inventory, node.physical_object):
+        messages.append(
+            error(
+                TARGET_MISSING,
+                f"{node.physical_target} does not hold {node.physical_object}, "
+                "which this node loads into",
+                source="load_resolution",
+            )
+        )
+    if expected_class is None and node.primitive_kind in (PYTHON_TABLE, PYTHON_FOLDER):
+        messages.append(
+            error(
+                MODULE_IMPORT_FAILURE,
+                f"{node.node_id} names a deployed module whose expected class "
+                "cannot be derived from its filename",
+                source="load_resolution",
+            )
+        )
+    return ResolvedLoadNode(
+        node=node,
+        dispatch_location=location,
+        target_exists=inventory is not None,
+        primitive_exists=primitive_exists,
+        expected_class=expected_class,
+        validation_messages=tuple(messages),
+    )
+
+
+def _resolve_refresh(
+    node: LoadNode, environment: LoadEnvironment, inventory
+) -> ResolvedLoadNode:
+    """A barrier resolves to a capability, and its absence is not a failure."""
+
+    messages: list[LoadMessage] = []
+    if inventory is None:
+        messages.append(
+            error(
+                TARGET_MISSING,
+                f"{node.physical_target} is not present, so its SQL endpoint "
+                "cannot be refreshed",
+                source="load_resolution",
+            )
+        )
+    supported = environment.can_refresh()
+    if not supported:
+        messages.append(
+            warning(
+                DISPATCH_LOCATION_MISSING,
+                f"{REFRESH_UNSUPPORTED}; {node.node_id} will be skipped",
+                source="load_resolution",
+            )
+        )
+    return ResolvedLoadNode(
+        node=node,
+        dispatch_location=f"{node.physical_target}/{ENDPOINT_SUFFIX}",
+        target_exists=inventory is not None,
+        primitive_exists=supported,
+        validation_messages=tuple(messages),
+        unsupported=not supported,
+    )
+
+
+def _dispatch_location(
+    node: LoadNode, environment: LoadEnvironment
+) -> tuple[str | None, str | None]:
+    """Where the primitive is, and the class a deployed module must define."""
+
+    if node.primitive_kind == WAREHOUSE_PROCEDURE:
+        procedure = load_procedure_name(node.logical_id.object_id)
+        return f"{node.physical_target}/{procedure}", None
+    if node.primitive_kind in (SPARK_SQL_FILE, PYTHON_TABLE, PYTHON_FOLDER):
+        location = installed_file_location(node, environment)
+        expected = (
+            _module_class(node.primitive_object.object)
+            if node.primitive_kind in (PYTHON_TABLE, PYTHON_FOLDER)
+            else None
+        )
+        return location, expected
+    return None, None
+
+
+def installed_file_location(node: LoadNode, environment: LoadEnvironment) -> str | None:
+    """The deployed artefact's location, as this environment addresses it.
+
+    Resolved rather than composed, so the desktop, the emulator and a Fabric
+    session each get their own spelling of the same file from the one place that
+    knows how a name becomes a location.
+    """
+
+    resolver = environment.resolver
+    files_root = getattr(resolver, "files_root", None)
+    if files_root is None or node.primitive_object is None:
+        return None
+    root = files_root(ItemRef(node.physical_target.name))
+    relative = f"{node.primitive_object.schema}/{node.primitive_object.object}"
+    return root.join(*relative.split("/")).value
+
+
+def _module_class(filename: str) -> str | None:
+    """``Sales__Order.py`` names class ``Sales__Order``.
+
+    The same rule the authoring surface applies to a class name and the
+    repository parser applies to a filename, so a deployed module's class is
+    found by the rule that put it there rather than by importing and looking.
+    """
+
+    if not filename.endswith(".py"):
+        return None
+    stem = filename[: -len(".py")]
+    return stem or None
+
+
+def _holds(inventory: TargetInventory | None, where) -> bool:
+    if inventory is None or where is None:
+        return False
+    return inventory.has_object(where.schema, where.object, where.object_type)
+
+
+# --- the dry-run report -------------------------------------------------------
+
+
+def dry_run_reports(plan: ResolvedLoadPlan) -> tuple[LoadNodeReport, ...]:
+    """A resolved plan as node reports, with nothing executed.
+
+    A validation status is never an execution status: a node that resolved is
+    ``validated``, not ``succeeded``, and ``executed`` is false on every one of
+    them. Reporting a dry run as a successful load would put a claim in a report
+    that nothing performed — and the report shape is otherwise identical to a
+    real run's, which is precisely what makes a dry run worth inspecting.
+    """
+
+    blocked = plan.blocked
+    reports = []
+    for resolved in plan.order:
+        node = resolved.node
+        causes = blocked.get(node.node_id, frozenset())
+        messages = list(resolved.validation_messages)
+        if resolved.valid and causes:
+            status = BLOCKED
+            messages.append(
+                error(
+                    DEPENDENCY_BLOCKED,
+                    f"{node.node_id} cannot run: "
+                    + ", ".join(sorted(causes))
+                    + " did not validate",
+                    source="load_resolution",
+                )
+            )
+        else:
+            status = VALIDATED if resolved.valid else INVALID
+        reports.append(
+            LoadNodeReport(
+                node_id=node.node_id,
+                logical_id=str(node.logical_id) if node.logical_id else None,
+                physical_target=str(node.physical_target),
+                primitive_kind=node.primitive_kind,
+                dispatch_location=resolved.dispatch_location,
+                status=status,
+                executed=False,
+                messages=tuple(messages),
+            )
+        )
+    return tuple(reports)
+
+
+__all__ = [
+    "ENDPOINT_SUFFIX",
+    "LoadEnvironment",
+    "REFRESH_UNSUPPORTED",
+    "ResolvedLoadNode",
+    "ResolvedLoadPlan",
+    "dry_run_reports",
+    "installed_file_location",
+    "resolve_load_plan",
+]

weaver/locations.py

@@ -0,0 +1,164 @@
+"""Resolved physical locations.
+
+A :class:`Location` is what a workspace resolution produces: the concrete place an
+item, folder or table lives. It exists because ``pathlib.Path`` cannot be the
+common currency — ``Path("abfss://ws@onelake.dfs.fabric.microsoft.com/lh")``
+silently collapses the double slash and yields a broken root with no error.
+
+So a location always carries a string and always joins by string. ``.path`` is
+available when, and only when, the location is a filesystem path; asking a URL
+location for a ``Path`` is a mistake worth raising on rather than corrupting.
+
+Local resolution (checkpoint 2) produces filesystem locations. Fabric
+resolution (checkpoint 7) will produce ``abfss://`` and OneLake URL locations
+through the same type, so everything downstream is written once.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from .errors import IdentityError
+
+_URL_MARKER = "://"
+
+
+@dataclass(frozen=True)
+class Location:
+    """One resolved location — a filesystem path or a URL."""
+
+    value: str
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.value, str):
+            raise IdentityError(f"location must be a string, got {type(self.value).__name__}")
+        value = self.value.strip()
+        if not value:
+            raise IdentityError("location must not be empty")
+        # One separator, everywhere. A Windows caller reaches this with
+        # backslashes — `LocalWorkspace` normalises its root through `Path`, and
+        # `str()` of a `WindowsPath` uses them — while everything downstream
+        # treats "/" as the only separator: `join`, `name`, and the segment
+        # splitting in the Weaver document reader. Left alone, a repository read from a
+        # Windows checkout takes its whole path as its catalogue name.
+        #
+        # Safe against real names: "\" is rejected in object and schema names
+        # (see targets._ILLEGAL_IN_NAME), so a backslash here is always a
+        # separator. URLs never carry one either.
+        value = value.replace("\\", "/")
+        if len(value) > 1:
+            value = value.rstrip("/")
+        object.__setattr__(self, "value", value)
+
+    @property
+    def is_url(self) -> bool:
+        return _URL_MARKER in self.value
+
+    @property
+    def path(self) -> Path:
+        """The filesystem path. Raises for URL locations."""
+
+        if self.is_url:
+            raise IdentityError(
+                f"{self.value!r} is a URL location and has no filesystem path — "
+                "use a Store to read or write it"
+            )
+        return Path(self.value)
+
+    def join(self, *parts: str) -> "Location":
+        """Append path segments. Always a string join, never ``Path``."""
+
+        joined = self.value
+        for part in parts:
+            if not isinstance(part, str):
+                raise IdentityError(f"location segment must be a string, got {part!r}")
+            segment = part.strip().strip("/")
+            if not segment:
+                raise IdentityError(f"location segment must not be empty: {part!r}")
+            joined = f"{joined.rstrip('/')}/{segment}"
+        return Location(joined)
+
+    def __truediv__(self, part: str) -> "Location":
+        return self.join(part)
+
+    @property
+    def name(self) -> str:
+        """The final segment."""
+
+        return self.value.rstrip("/").rsplit("/", 1)[-1]
+
+    def __str__(self) -> str:
+        return self.value
+
+
+@dataclass(frozen=True)
+class LakehouseSparkLocation:
+    """One destination Lakehouse's physical roots, resolved once.
+
+    The Spark session is attached to the **Weaver Lakehouse** — that is the fixed
+    control-plane context, and it is why Weaver's own catalogue tables are reached
+    as ordinary two-part names in schema ``_``. Destination Lakehouses are the
+    variable data plane, so they are reached through explicit roots instead, and
+    never by making the session point somewhere else.
+
+    That distinction is what lets one session build several Lakehouses in one
+    invocation. Switching the current catalogue between targets would make two
+    destinations that share a schema name indistinguishable; resolving each
+    target's roots keeps them separate by construction::
+
+        locations = {
+            target: resolver.lakehouse_spark_location(target)
+            for target in bound_lakehouse_targets
+        }
+
+    Roots are plain strings rather than :class:`Location` values because this is
+    what *Spark* addresses: an ``abfss://`` URL on Fabric, a filesystem path in
+    the local emulator. Same contract, different transport.
+
+    A resolved location is deliberately **not** carried in a build bundle. It is
+    derived from the item at install time, because on Fabric it embeds workspace
+    and item ids and locally it embeds a temporary directory — and a bundle whose
+    identity moved with a temporary path would not be comparable between
+    environments (how-does-build-work §15).
+    """
+
+    #: The Lakehouse this resolves, by its logical name.
+    item: str
+    tables_root: str
+    files_root: str
+
+    def schema_root(self, schema: str) -> str:
+        """Where a schema's managed tables live."""
+
+        return f"{self.tables_root.rstrip('/')}/{_segment(schema)}"
+
+    def table_path(self, schema: str, name: str) -> str:
+        """Where one managed Delta table lives."""
+
+        return f"{self.schema_root(schema)}/{_segment(name)}"
+
+    def folder_path(self, schema: str, name: str) -> str:
+        """Where one managed folder lives, under the Files area."""
+
+        return (
+            f"{self.files_root.rstrip('/')}/{_segment(schema)}/{_segment(name)}"
+        )
+
+    def __str__(self) -> str:
+        return f"{self.item} (tables={self.tables_root}, files={self.files_root})"
+
+
+def _segment(value: str) -> str:
+    """One path segment, checked rather than trusted.
+
+    These strings are concatenated into paths Spark writes through, so a segment
+    that escaped its parent would write outside the Lakehouse it names.
+    """
+
+    segment = value.strip().strip("/")
+    if not segment or segment in (".", ".."):
+        raise IdentityError(f"path segment must be a real name, got {value!r}")
+    if "/" in segment or "\\" in segment:
+        raise IdentityError(f"path segment must not contain a separator: {value!r}")
+    return segment