weaverstack 0.1.1__py3-none-any.whl

weaver/fabric/resources.py

@@ -0,0 +1,245 @@
+"""Resolving Fabric workspace items by name.
+
+Item names are unique within a workspace, which is the whole reason level three
+needs no configuration. This is where that assumption meets the API.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Iterable
+
+from ..errors import CommandError
+from .client import FabricClient, FabricError
+
+LAKEHOUSE = "Lakehouse"
+WAREHOUSE = "Warehouse"
+ENVIRONMENT = "Environment"
+NOTEBOOK = "Notebook"
+SQL_ENDPOINT = "SQLEndpoint"
+
+#: A Lakehouse grows a SQLEndpoint sibling of the same name, a little after it
+#: is created. That is a facet of the Lakehouse rather than an item anyone
+#: addresses, so it is ignored when a name is resolved without a type — item
+#: names are unique per type, not across types.
+FACET_TYPES = frozenset({SQL_ENDPOINT})
+
+
+class ItemNotFoundError(CommandError):
+    """Raised when an item lookup returns no match."""
+
+
+@dataclass(frozen=True)
+class Workspace:
+    id: str
+    name: str
+
+    def __str__(self) -> str:
+        return f"{self.name} ({self.id})"
+
+
+@dataclass(frozen=True)
+class Item:
+    """One workspace item — a Lakehouse, a Warehouse, an Environment."""
+
+    id: str
+    name: str
+    type: str
+    workspace_id: str
+
+    def __str__(self) -> str:
+        return f"{self.type} {self.name} ({self.id})"
+
+
+def find_workspace(name: str, *, client: FabricClient | None = None) -> Workspace:
+    """The workspace with this name."""
+
+    client = client or FabricClient()
+    matches = [
+        workspace
+        for workspace in client.paged("workspaces")
+        if workspace.get("displayName") == name
+    ]
+    if not matches:
+        available = ", ".join(
+            sorted(w.get("displayName", "?") for w in client.paged("workspaces"))
+        )
+        raise CommandError(
+            f"no workspace named {name!r} — found: {available or 'none'}"
+        )
+    if len(matches) > 1:
+        raise CommandError(f"more than one workspace named {name!r}")
+    return Workspace(id=matches[0]["id"], name=name)
+
+
+def list_items(
+    workspace: Workspace, *, item_type: str | None = None, client: FabricClient | None = None
+) -> tuple[Item, ...]:
+    client = client or FabricClient()
+    path = f"workspaces/{workspace.id}/items"
+    if item_type:
+        path += f"?type={item_type}"
+    return tuple(
+        Item(
+            id=item["id"],
+            name=item.get("displayName", ""),
+            type=item.get("type", ""),
+            workspace_id=workspace.id,
+        )
+        for item in client.paged(path)
+    )
+
+
+def find_item(
+    workspace: Workspace,
+    name: str,
+    *,
+    item_type: str | None = None,
+    client: FabricClient | None = None,
+) -> Item:
+    """The item with this name, which is unique within a workspace."""
+
+    matches = [
+        item
+        for item in list_items(workspace, item_type=item_type, client=client)
+        if item.name == name and (item_type is None or item.type == item_type)
+    ]
+    if item_type is None and len(matches) > 1:
+        matches = [item for item in matches if item.type not in FACET_TYPES] or matches
+    if not matches:
+        raise ItemNotFoundError(
+            f"no {item_type or 'item'} named {name!r} in workspace {workspace.name!r}"
+        )
+    if len(matches) > 1:
+        found = ", ".join(sorted(item.type for item in matches))
+        raise CommandError(
+            f"more than one item named {name!r} in {workspace.name!r} ({found}) — "
+            "say which type is meant"
+        )
+    return matches[0]
+
+
+def create_lakehouse(
+    workspace: Workspace, name: str, *, client: FabricClient | None = None
+) -> Item:
+    """Create a **schema-enabled** Lakehouse. Returns the existing one if the name is taken.
+
+    Schemas are not optional and so are not a parameter. Weaver's catalogue lives
+    in a schema called ``_``, and a Lakehouse created without schema support
+    cannot hold one — so a Weaver Lakehouse made without ``enableSchemas`` is
+    unusable, and a destination made without it puts managed tables somewhere
+    other than ``Tables/<schema>/<table>``, which is the layout every resolved
+    location assumes. There is no Weaver use for a Lakehouse that has neither.
+
+    Fabric decides this only at creation, so getting it wrong means deleting the
+    item and making it again.
+    """
+
+    client = client or FabricClient()
+    try:
+        return find_item(workspace, name, item_type=LAKEHOUSE, client=client)
+    except CommandError:
+        pass
+
+    # Fabric holds a deleted item's name for some minutes and answers 409
+    # ItemDisplayNameNotAvailableYet until it is free again.
+    response = client.request(
+        "POST",
+        f"workspaces/{workspace.id}/lakehouses",
+        payload={"displayName": name, "creationPayload": {"enableSchemas": True}},
+        expected=(200, 201, 202, 409),
+    )
+    if response.status_code == 409:
+        raise CommandError(
+            f"cannot create Lakehouse {name!r} in {workspace.name!r}: "
+            + (response.json().get("message") or response.text.strip()[:200])
+        )
+    if response.status_code == 202:
+        # Long-running create: the item exists once the operation settles.
+        return _await_item(workspace, name, LAKEHOUSE, client=client)
+    body = response.json()
+    return Item(id=body["id"], name=name, type=LAKEHOUSE, workspace_id=workspace.id)
+
+
+def create_warehouse(
+    workspace: Workspace, name: str, *, client: FabricClient | None = None
+) -> Item:
+    """Create a disposable Warehouse, returning an existing typed match."""
+
+    client = client or FabricClient()
+    try:
+        return find_item(workspace, name, item_type=WAREHOUSE, client=client)
+    except ItemNotFoundError:
+        pass
+
+    response = client.request(
+        "POST",
+        f"workspaces/{workspace.id}/warehouses",
+        payload={"displayName": name},
+        expected=(200, 201, 202, 409),
+    )
+    if response.status_code == 409:
+        raise CommandError(
+            f"cannot create Warehouse {name!r} in {workspace.name!r}: "
+            + (response.json().get("message") or response.text.strip()[:200])
+        )
+    if response.status_code == 202:
+        return _await_item(workspace, name, WAREHOUSE, client=client)
+    body = response.json()
+    return Item(id=body["id"], name=name, type=WAREHOUSE, workspace_id=workspace.id)
+
+
+def delete_item(item: Item, *, client: FabricClient | None = None) -> None:
+    client = client or FabricClient()
+    client.request(
+        "DELETE",
+        f"workspaces/{item.workspace_id}/items/{item.id}",
+        expected=(200, 202, 204),
+    )
+
+
+def refresh_sql_endpoint_metadata(
+    endpoint: Item, *, client: FabricClient | None = None
+) -> dict:
+    """Refresh every table in one SQL analytics endpoint and await completion."""
+
+    if endpoint.type != SQL_ENDPOINT:
+        raise CommandError(
+            f"SQL endpoint refresh needs a {SQL_ENDPOINT} item, got {endpoint.type!r}"
+        )
+    client = client or FabricClient()
+    response = client.request(
+        "POST",
+        f"workspaces/{endpoint.workspace_id}/sqlEndpoints/{endpoint.id}/refreshMetadata",
+        payload={"recreateTables": False},
+        expected=(200, 202),
+    )
+    result = client.wait_for_operation(response)
+    return {
+        "lakehouse": endpoint.name,
+        "sql_endpoint_id": endpoint.id,
+        "operation_id": response.headers.get("x-ms-operation-id"),
+        "status": result.get("status", "Succeeded"),
+    }
+
+
+def _await_item(
+    workspace: Workspace,
+    name: str,
+    item_type: str,
+    *,
+    client: FabricClient,
+    attempts: int = 30,
+    pause: float = 2.0,
+) -> Item:
+    import time
+
+    for _ in range(attempts):
+        try:
+            return find_item(workspace, name, item_type=item_type, client=client)
+        except CommandError:
+            time.sleep(pause)
+    raise FabricError(
+        f"{item_type} {name!r} did not appear in {workspace.name!r} after "
+        f"{int(attempts * pause)}s"
+    )

weaver/fabric/session.py

@@ -0,0 +1,148 @@
+"""Resolution from inside a Microsoft Fabric session.
+
+The desktop resolver crosses into Fabric through REST. This resolver stays
+inside the current workspace: NotebookUtils supplies the workspace identity and
+resolves Lakehouse names, and the resulting locations are native ``abfss``.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Mapping
+
+from ..errors import CommandError
+from ..workspaces import FabricWorkspace
+from ..locations import Location
+from ..targets import ItemRef
+from .onelake import abfss_root
+from .resolution import FabricResolver
+from .resources import LAKEHOUSE, WAREHOUSE, Item, Workspace, find_item
+
+
+def _value(record: Any, name: str) -> Any:
+    if isinstance(record, Mapping):
+        return record.get(name)
+    return getattr(record, name, None)
+
+
+class FabricSessionResolver(FabricResolver):
+    """Resolve names without leaving the Fabric session."""
+
+    def __init__(
+        self,
+        workspace: FabricWorkspace,
+        *,
+        runtime: Any | None = None,
+        lakehouse: Any | None = None,
+        credentials: Any | None = None,
+        client: Any | None = None,
+    ) -> None:
+        if not isinstance(workspace, FabricWorkspace):
+            raise CommandError(
+                f"FabricSessionResolver needs a FabricWorkspace, got {type(workspace).__name__}"
+            )
+        if runtime is None or lakehouse is None:
+            try:
+                from notebookutils import lakehouse as notebook_lakehouse
+                from notebookutils import runtime as notebook_runtime
+            except ImportError as exc:
+                raise CommandError(
+                    "FabricSessionResolver is available only inside a Fabric session"
+                ) from exc
+            runtime = runtime or notebook_runtime
+            lakehouse = lakehouse or notebook_lakehouse
+
+        context = runtime.context
+        if callable(context):
+            context = context()
+        workspace_name = _value(context, "currentWorkspaceName")
+        workspace_id = _value(context, "currentWorkspaceId")
+        if not workspace_name or not workspace_id:
+            raise CommandError("Fabric runtime context carries no current workspace")
+        if workspace_name != workspace.workspace:
+            raise CommandError(
+                f"this session runs in workspace {workspace_name!r}, "
+                f"not configured Workspace {workspace.workspace!r}"
+            )
+
+        self.configuration = workspace
+        self._workspace = Workspace(id=str(workspace_id), name=str(workspace_name))
+        self._lakehouse_utils = lakehouse
+        self._credentials = credentials
+        self.client = client
+        self._items: dict[str, Item] = {}
+
+    @property
+    def workspace(self) -> Workspace:
+        return self._workspace
+
+    @property
+    def root(self) -> Location:
+        return Location(
+            f"abfss://{self.workspace.id}@onelake.dfs.fabric.microsoft.com"
+        )
+
+    def resolve(self, item: ItemRef, *, item_type: str) -> Item:
+        if item_type == WAREHOUSE:
+            key = f"{item.name}:{item_type}"
+            if key not in self._items:
+                self._items[key] = find_item(
+                    self.workspace,
+                    item.name,
+                    item_type=WAREHOUSE,
+                    client=self._rest_client(),
+                )
+            return self._items[key]
+        if item_type != LAKEHOUSE:
+            raise CommandError(
+                f"session-native resolution for {item_type} is not implemented"
+            )
+        key = f"{item.name}:{item_type}"
+        if key not in self._items:
+            artifact = self._lakehouse_utils.get(
+                item.name, workspaceId=self.workspace.id
+            )
+            item_id = _value(artifact, "id")
+            display_name = _value(artifact, "displayName")
+            if not item_id:
+                raise CommandError(
+                    f"no Lakehouse named {item.name!r} in workspace "
+                    f"{self.workspace.name!r}"
+                )
+            self._items[key] = Item(
+                id=str(item_id),
+                name=str(display_name or item.name),
+                type=LAKEHOUSE,
+                workspace_id=self.workspace.id,
+            )
+        return self._items[key]
+
+    def lakehouse(self, item: ItemRef) -> Location:
+        resolved = self.resolve(item, item_type=LAKEHOUSE)
+        return Location(abfss_root(self.workspace.id, resolved.id))
+
+    def sql_endpoint(self, target):
+        self.client = self._rest_client()
+        return super().sql_endpoint(target)
+
+
+    def _rest_client(self):
+        """Fabric REST using the identity of this Fabric session."""
+
+        if self.client is None:
+            credentials = self._credentials
+            if credentials is None:
+                try:
+                    from notebookutils import credentials as notebook_credentials
+                except ImportError as exc:
+                    raise CommandError(
+                        "Warehouse resolution is available only inside a Fabric session"
+                    ) from exc
+                credentials = notebook_credentials
+            from .client import FabricClient
+
+            # A callable, not the string it answers: a session-native token
+            # expires like any other, and an install running inside Fabric can
+            # outlive one. NotebookUtils serves from its own cache, so asking per
+            # request is cheap.
+            self.client = FabricClient(token=lambda: credentials.getToken("pbi"))
+        return self.client

weaver/fabric/shortcuts.py

@@ -0,0 +1,120 @@
+"""OneLake shortcuts — Fabric's own way of pointing one item at another's data.
+
+A shortcut is how an alias exists in Fabric. It has no bytes: the destination
+Lakehouse gains a table (or a folder) under its own name, and reads pass through
+to the item that owns the data. That is precisely what a Weaver alias claims —
+the destination item owns the name, the source stays the canonical producer — so
+the two map onto each other exactly.
+
+Created by replacement. A shortcut carries no data of its own, so removing and
+remaking one loses nothing, and a build that could not re-run over its own aliases
+would not be re-runnable.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from urllib.parse import quote
+
+from .client import FabricClient
+from .resources import Item
+
+
+@dataclass(frozen=True)
+class Shortcut:
+    """One shortcut an item holds: where it appears, and what it points at."""
+
+    path: str
+    name: str
+    target_workspace_id: str | None = None
+    target_item_id: str | None = None
+    target_path: str | None = None
+
+    @property
+    def qualified(self) -> str:
+        return f"{self.path}/{self.name}"
+
+
+def list_shortcuts(item: Item, *, client: FabricClient) -> tuple[Shortcut, ...]:
+    """Every shortcut this item holds.
+
+    Fabric echoes a path back rooted — ``/Tables/DWG`` for the ``Tables/DWG`` it
+    was given — so the leading separator is normalised here rather than by every
+    caller.
+    """
+
+    found = []
+    for entry in client.paged(
+        f"workspaces/{item.workspace_id}/items/{item.id}/shortcuts"
+    ):
+        onelake = (entry.get("target") or {}).get("oneLake") or {}
+        found.append(
+            Shortcut(
+                path=(entry.get("path") or "").strip("/"),
+                name=entry.get("name") or "",
+                target_workspace_id=onelake.get("workspaceId"),
+                target_item_id=onelake.get("itemId"),
+                target_path=onelake.get("path"),
+            )
+        )
+    return tuple(sorted(found, key=lambda shortcut: shortcut.qualified))
+
+
+def create_shortcut(
+    destination: Item,
+    *,
+    path: str,
+    name: str,
+    source: Item,
+    source_path: str,
+    client: FabricClient,
+) -> dict:
+    """Point ``destination``'s ``path/name`` at ``source``'s ``source_path``."""
+
+    delete_shortcut(destination, path=path, name=name, client=client)
+    response = client.request(
+        "POST",
+        f"workspaces/{destination.workspace_id}/items/{destination.id}/shortcuts",
+        payload={
+            "path": path,
+            "name": name,
+            "target": {
+                "oneLake": {
+                    "workspaceId": source.workspace_id,
+                    "itemId": source.id,
+                    "path": source_path,
+                }
+            },
+        },
+    )
+    return {
+        "shortcut": f"{path}/{name}",
+        "in": destination.name,
+        "target": f"{source.name}/{source_path}",
+        # Reported because it says which contract Fabric honoured. Creating one
+        # shortcut is documented as synchronous — a 201 — while bulk creation is
+        # not; so a 202 here would mean the shortcut itself is still being made,
+        # which is a different thing from the destination Lakehouse not yet having
+        # registered it as a table. Only the second is what the readability wait
+        # in `weaver.build_bundle.executors.alias` exists for.
+        "status": response.status_code,
+    }
+
+
+def delete_shortcut(
+    destination: Item, *, path: str, name: str, client: FabricClient
+) -> None:
+    """Remove a shortcut if it is there. A 404 is the intended state, not a fault.
+
+    Removing the shortcut is not removing what it points at: the data belongs to
+    the item that produced it, and this only takes away this item's name for it.
+    That distinction is the whole reason a wipe must remove shortcuts *through the
+    workspace* rather than by deleting a directory (see :mod:`weaver.physical_wipe`).
+    """
+
+    client.request(
+        "DELETE",
+        f"workspaces/{destination.workspace_id}/items/{destination.id}/shortcuts/"
+        f"{quote(path.strip('/'), safe='')}/{quote(name, safe='')}",
+        expected=(200, 202, 204, 404),
+    )

weaver/fabric/sql.py

@@ -0,0 +1,118 @@
+"""Explicit desktop and Fabric-session SQL capability factories."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..errors import CommandError
+from ..sql import (
+    DEFAULT_MAX_CONNECTIONS,
+    AccessTokenAuthentication,
+    PooledSqlExecutor,
+    SqlConnectionPool,
+)
+from .auth import SQL_SCOPE, get_token
+
+# NotebookUtils accepts the SQL resource audience, not an OAuth ``.default``
+# scope.  Keeping the two constants separate prevents a caller-boundary mix-up.
+FABRIC_SQL_AUDIENCE = "https://database.windows.net/"
+
+
+def desktop_sql_pool(
+    target,
+    workspace,
+    *,
+    credential=None,
+    resolver=None,
+    max_connections: int = DEFAULT_MAX_CONNECTIONS,
+    connection_factory=None,
+) -> SqlConnectionPool:
+    """Cross from a desktop caller into a resolved Fabric Warehouse."""
+
+    from ..resolution import resolver_for
+    from ..sql.connection import connect
+
+    resolver = resolver or resolver_for(workspace)
+    endpoint = resolver.sql_endpoint(target)
+    authentication = AccessTokenAuthentication(
+        lambda: get_token(SQL_SCOPE, credential)
+    )
+    return SqlConnectionPool(
+        endpoint,
+        authentication,
+        max_connections=max_connections,
+        connection_factory=connection_factory or connect,
+    )
+
+
+def desktop_sql_executor(target, workspace, **kwargs) -> PooledSqlExecutor:
+    """An explicitly cross-boundary desktop executor."""
+
+    return PooledSqlExecutor(
+        desktop_sql_pool(target, workspace, **kwargs),
+        owns_pool=True,
+    )
+
+
+def fabric_sql_pool(
+    target,
+    workspace,
+    *,
+    resolver=None,
+    runtime: Any | None = None,
+    lakehouse: Any | None = None,
+    credentials: Any | None = None,
+    max_connections: int = DEFAULT_MAX_CONNECTIONS,
+    connection_factory=None,
+) -> SqlConnectionPool:
+    """SQL access using the identity made available by a Fabric session."""
+
+    from ..sql.connection import connect
+    from .session import FabricSessionResolver
+
+    if resolver is None:
+        try:
+            resolver = FabricSessionResolver(
+                workspace,
+                runtime=runtime,
+                lakehouse=lakehouse,
+                credentials=credentials,
+            )
+        except CommandError as exc:
+            raise CommandError(
+                "Fabric-native SQL is available only inside a supported Fabric "
+                "notebook or Livy session"
+            ) from exc
+    if not isinstance(resolver, FabricSessionResolver):
+        raise CommandError(
+            "Fabric-native SQL needs a FabricSessionResolver; desktop callers "
+            "must use desktop_sql_executor explicitly"
+        )
+
+    endpoint = resolver.sql_endpoint(target)
+    notebook_credentials = credentials or getattr(resolver, "_credentials", None)
+    if notebook_credentials is None:
+        try:
+            from notebookutils import credentials as notebook_credentials
+        except ImportError as exc:
+            raise CommandError(
+                "Fabric-native SQL cannot acquire the Fabric session identity"
+            ) from exc
+    authentication = AccessTokenAuthentication(
+        lambda: notebook_credentials.getToken(FABRIC_SQL_AUDIENCE)
+    )
+    return SqlConnectionPool(
+        endpoint,
+        authentication,
+        max_connections=max_connections,
+        connection_factory=connection_factory or connect,
+    )
+
+
+def fabric_sql_executor(target, workspace, **kwargs) -> PooledSqlExecutor:
+    """An explicitly within-Fabric executor."""
+
+    return PooledSqlExecutor(
+        fabric_sql_pool(target, workspace, **kwargs),
+        owns_pool=True,
+    )