weaverstack 0.1.1__py3-none-any.whl

weaver/fabric/onelake.py

@@ -0,0 +1,263 @@
+"""OneLake as a :class:`~weaver.store.Store`, over the DFS API.
+
+This is the **desktop's** way of reaching into a Fabric workspace: authenticated
+HTTPS against the ADLS Gen2 DFS endpoint. It is what the CLI uses to push
+repository files up and to inspect results from a test on the laptop. It is not
+the path Weaver should use when it is *running inside* Fabric — there the native
+in-session mechanisms (``notebookutils.fs``, Spark) apply — so this store is a
+cross-into-Fabric transport, not the canonical in-workspace Fabric implementation.
+
+Two things OneLake does over DFS that a plain filesystem does not:
+
+**Writing is three calls, not one.** Create the file, append the bytes, flush at
+the final offset.
+
+**Listing is paged.** A large directory returns a continuation token rather than
+everything. Pagination is not implemented yet, so a paged listing fails loudly
+(see :meth:`OneLakeDfsClient.list`) rather than silently returning a first page —
+which would quietly truncate a wipe, a sync or a reconciliation.
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass
+from datetime import timezone
+from datetime import datetime
+from urllib.parse import quote, urlencode
+
+from ..errors import CommandError
+from ..locations import Location
+from ..store import Entry, StoreError
+from .auth import STORAGE_SCOPE, token_source
+from .client import ONELAKE_DFS
+
+STORAGE_API_VERSION = "2023-11-03"
+DEFAULT_TIMEOUT = 120.0
+
+
+def lakehouse_artifact_segment(item: str) -> str:
+    """A OneLake path segment for a **Lakehouse**, by id or by name.
+
+    A GUID stands alone; a name needs its item type, as ``Weaver.Lakehouse``.
+    The rule is Lakehouse-specific — the ``.Lakehouse`` suffix — which is why the
+    name says so. OneLake file paths only ever address Lakehouses; a Warehouse
+    is reached over TDS, not here.
+    """
+
+    try:
+        uuid.UUID(item)
+        return item
+    except ValueError:
+        return f"{item}.Lakehouse"
+
+
+def onelake_url(
+    workspace: str,
+    item: str,
+    relative_path: str = "",
+    *,
+    base_url: str = ONELAKE_DFS,
+    query: dict[str, str] | None = None,
+) -> str:
+    """A DFS URL beneath one item, e.g. ``…/{ws}/{lh}/Files/weaver_items/x``."""
+
+    parts = [workspace, lakehouse_artifact_segment(item)]
+    parts.extend(part for part in relative_path.strip("/").split("/") if part)
+    url = f"{base_url.rstrip('/')}/" + "/".join(quote(part, safe="") for part in parts)
+    return f"{url}?{urlencode(query)}" if query else url
+
+
+def abfss_root(workspace_id: str, item_id: str) -> str:
+    """The Spark-facing root for an item.
+
+    Proven to list, read and write Lakehouses that are not attached to the
+    notebook, which is the whole reason destination roots are explicit.
+    """
+
+    return f"abfss://{workspace_id}@onelake.dfs.fabric.microsoft.com/{item_id}"
+
+
+@dataclass(frozen=True)
+class OneLakePath:
+    """A OneLake location split back into the parts DFS needs."""
+
+    workspace: str
+    item: str
+    relative: str
+
+
+def parse_onelake(location: Location, *, base_url: str = ONELAKE_DFS) -> OneLakePath:
+    prefix = base_url.rstrip("/") + "/"
+    if not location.value.startswith(prefix):
+        raise CommandError(
+            f"{location.value!r} is not a OneLake location — expected it to start "
+            f"with {prefix}"
+        )
+    parts = [part for part in location.value[len(prefix):].split("/") if part]
+    if len(parts) < 2:
+        raise CommandError(f"{location.value!r} names no item beneath its workspace")
+    return OneLakePath(workspace=parts[0], item=parts[1], relative="/".join(parts[2:]))
+
+
+class OneLakeDfsClient:
+    """An ADLS Gen2 DFS client for one workspace, used **from outside Fabric**.
+
+    This is how a local caller — the CLI, or a Fabric integration test — reaches
+    into a workspace: authenticated HTTPS to the OneLake DFS endpoint. It
+    satisfies the :class:`~weaver.store.Store` protocol so the CLI can hand it to
+    the same code a ``LocalStore`` drives, but it is *cross-boundary access*, not
+    the store Weaver uses when it runs inside Fabric. The in-Fabric,
+    session-native store is a separate implementation for when it exists.
+
+    Because it crosses a boundary, it is constructed explicitly by the caller
+    that crosses — never returned by a workspace-to-store factory, which returns the
+    NotebookUtils-backed ``FabricStore`` only inside a Fabric session.
+    """
+
+    def __init__(
+        self,
+        *,
+        base_url: str = ONELAKE_DFS,
+        token: str | None = None,
+        timeout: float = DEFAULT_TIMEOUT,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+        self._token_source = token_source(token, scope=STORAGE_SCOPE)
+
+    @property
+    def token(self) -> str:
+        """A currently-valid bearer, renewed when it is close to expiring.
+
+        A push or a repository upload can run for a long time on one client, so
+        the token has to be read per request rather than snapshotted.
+        """
+
+        return self._token_source()
+
+    def _request(
+        self,
+        method: str,
+        url: str,
+        *,
+        data: bytes | None = None,
+        headers: dict[str, str] | None = None,
+        expected: tuple[int, ...] = (200, 201, 202),
+    ):
+        import requests
+
+        merged = {
+            "Authorization": f"Bearer {self.token}",
+            "x-ms-version": STORAGE_API_VERSION,
+        }
+        merged.update(headers or {})
+        response = requests.request(
+            method, url, headers=merged, data=data, timeout=self.timeout
+        )
+        if response.status_code not in expected:
+            raise StoreError(
+                f"{method} {url.split('?')[0]} returned {response.status_code}: "
+                f"{response.text.strip()[:300] or 'no body'}"
+            )
+        return response
+
+    def _url(self, location: Location, query: dict[str, str] | None = None) -> str:
+        parsed = parse_onelake(location, base_url=self.base_url)
+        return onelake_url(
+            parsed.workspace,
+            parsed.item,
+            parsed.relative,
+            base_url=self.base_url,
+            query=query,
+        )
+
+    # --- the Store protocol ----------------------------------------------
+
+    def exists(self, location: Location) -> bool:
+        return self._request("HEAD", self._url(location), expected=(200, 404)).status_code == 200
+
+    def is_directory(self, location: Location) -> bool:
+        response = self._request("HEAD", self._url(location), expected=(200, 404))
+        if response.status_code != 200:
+            return False
+        return response.headers.get("x-ms-resource-type") == "directory"
+
+    def list(self, location: Location, *, recursive: bool = False) -> list[Entry]:
+        parsed = parse_onelake(location, base_url=self.base_url)
+        directory = "/".join(
+            part for part in (lakehouse_artifact_segment(parsed.item), parsed.relative) if part
+        )
+        url = f"{self.base_url}/{quote(parsed.workspace, safe='')}?" + urlencode(
+            {
+                "resource": "filesystem",
+                "recursive": "true" if recursive else "false",
+                "directory": directory,
+            }
+        )
+        response = self._request("GET", url, expected=(200, 404))
+        if response.status_code == 404:
+            raise StoreError(f"cannot list a location that does not exist: {location}")
+
+        # A large directory pages, returning a continuation token. Until that is
+        # handled, returning only the first page would silently truncate a wipe,
+        # a sync or a reconciliation, so fail before returning anything.
+        if response.headers.get("x-ms-continuation"):
+            raise NotImplementedError("OneLake listing pagination is not implemented")
+
+        entries: list[Entry] = []
+        prefix = f"{lakehouse_artifact_segment(parsed.item)}/"
+        for path in response.json().get("paths", []):
+            name = path.get("name", "")
+            relative = name[len(prefix):] if name.startswith(prefix) else name
+            entries.append(
+                Entry(
+                    location=Location(
+                        f"{self.base_url}/{parsed.workspace}/"
+                        f"{lakehouse_artifact_segment(parsed.item)}/{relative}"
+                    ),
+                    is_directory=str(path.get("isDirectory", "false")).lower() == "true",
+                    size=int(path["contentLength"]) if path.get("contentLength") else None,
+                    modified=_parse_time(path.get("lastModified")),
+                    etag=path.get("etag"),
+                )
+            )
+        return entries
+
+    def read(self, location: Location) -> bytes:
+        return self._request("GET", self._url(location), expected=(200,)).content
+
+    def write(self, location: Location, data: bytes) -> None:
+        url = self._url(location)
+        self._request("PUT", f"{url}?resource=file", expected=(201,))
+        if data:
+            self._request(
+                "PATCH",
+                f"{url}?action=append&position=0",
+                data=data,
+                headers={"Content-Length": str(len(data))},
+                expected=(202,),
+            )
+        self._request("PATCH", f"{url}?action=flush&position={len(data)}", expected=(200,))
+
+    def delete(self, location: Location, *, recursive: bool = False) -> None:
+        query = "?recursive=true" if recursive else ""
+        self._request(
+            "DELETE", f"{self._url(location)}{query}", expected=(200, 202, 204, 404)
+        )
+
+    def make_directory(self, location: Location) -> None:
+        self._request(
+            "PUT", f"{self._url(location)}?resource=directory", expected=(201, 409)
+        )
+
+
+def _parse_time(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    from email.utils import parsedate_to_datetime
+
+    try:
+        return parsedate_to_datetime(value).astimezone(timezone.utc)
+    except (TypeError, ValueError):
+        return None

weaver/fabric/resolution.py

@@ -0,0 +1,344 @@
+"""Fabric workspace resolution — names to OneLake locations.
+
+The twin of :class:`~weaver.resolution.LocalResolver`, and deliberately the same
+surface, so everything above resolution is written once and neither build nor
+wipe learns which workspace it is talking to.
+
+The difference is that a Fabric name has to be *asked about* — a name maps to a
+GUID only by consulting the workspace. It is asked about **with its type**:
+identity is ``workspace + type + name``, so a Lakehouse and a Warehouse may share a
+display name (indeed a Lakehouse grows a same-named SQL endpoint), and the
+caller always knows the type from the slot. Answers are cached, because asking
+costs an API call.
+"""
+
+from __future__ import annotations
+
+from ..errors import CommandError
+from ..workspaces import BUILD_BUNDLES_AREA, CLI_AREA, WEAVER_ITEMS_AREA, FabricWorkspace
+from ..locations import LakehouseSparkLocation, Location
+from ..resolution import TABLES_AREA
+from ..spark import SparkDestination, fabric_destination
+from ..targets import (
+    FILES_AREA,
+    DeltaTarget,
+    FolderTarget,
+    ItemRef,
+    WarehouseTarget,
+    validate_name,
+)
+from .client import ONELAKE_DFS, FabricClient
+from .onelake import abfss_root, lakehouse_artifact_segment
+from .resources import (
+    LAKEHOUSE,
+    SQL_ENDPOINT,
+    WAREHOUSE,
+    Item,
+    Workspace,
+    find_item,
+    find_workspace,
+    refresh_sql_endpoint_metadata,
+)
+
+
+class FabricResolver:
+    """Resolves level-three names against one Fabric workspace."""
+
+    def __init__(
+        self,
+        workspace: FabricWorkspace,
+        *,
+        client: FabricClient | None = None,
+        base_url: str = ONELAKE_DFS,
+    ) -> None:
+        if not isinstance(workspace, FabricWorkspace):
+            raise CommandError(
+                f"FabricResolver needs a FabricWorkspace, got {type(workspace).__name__}"
+            )
+        self.configuration = workspace
+        self.client = client or FabricClient()
+        self.base_url = base_url.rstrip("/")
+        self._workspace: Workspace | None = None
+        self._items: dict[str, Item] = {}
+
+    # --- level four -------------------------------------------------------
+
+    @property
+    def workspace(self) -> Workspace:
+        if self._workspace is None:
+            self._workspace = find_workspace(
+                self.configuration.workspace, client=self.client
+            )
+        return self._workspace
+
+    @property
+    def root(self) -> Location:
+        """The workspace root. Everything resolved sits beneath it."""
+
+        return Location(f"{self.base_url}/{self.workspace.id}")
+
+    # --- level three ------------------------------------------------------
+
+    def resolve(self, item: ItemRef, *, item_type: str) -> Item:
+        """The workspace item of this name and type. Cached.
+
+        A type is required: identity is ``workspace + type + name``, and asking the
+        workspace what a bare name *is* would make a caller depend on ambiguous
+        name inference. The caller knows the type from the slot — a
+        ``DeltaTarget`` is a Lakehouse, a ``WarehouseTarget`` is a Warehouse.
+        """
+
+        key = f"{item.name}:{item_type}"
+        if key not in self._items:
+            self._items[key] = find_item(
+                self.workspace, item.name, item_type=item_type, client=self.client
+            )
+        return self._items[key]
+
+    def _rest_client(self) -> FabricClient:
+        return self.client
+
+    def refresh_sql_endpoint(self, item: ItemRef) -> dict:
+        """Refresh the SQL analytics endpoint paired with a named Lakehouse."""
+
+        client = self._rest_client()
+        endpoint = find_item(
+            self.workspace,
+            item.name,
+            item_type=SQL_ENDPOINT,
+            client=client,
+        )
+        return refresh_sql_endpoint_metadata(endpoint, client=client)
+
+    def lakehouse(self, item: ItemRef) -> Location:
+        return self.root / lakehouse_artifact_segment(
+            self.resolve(item, item_type=LAKEHOUSE).id
+        )
+
+    def files_root(self, item: ItemRef) -> Location:
+        return self.lakehouse(item) / FILES_AREA
+
+    def tables_root(self, item: ItemRef) -> Location:
+        return self.lakehouse(item) / TABLES_AREA
+
+    def spark_root(self, item: ItemRef) -> str:
+        """The ``abfss://`` root Spark writes through, for a Lakehouse.
+
+        Explicit, so a session never needs the item attached.
+        """
+
+        return abfss_root(self.workspace.id, self.resolve(item, item_type=LAKEHOUSE).id)
+
+    # --- targets ----------------------------------------------------------
+
+    def folder_root(self, target: FolderTarget) -> Location:
+        return self.files_root(target.lakehouse)
+
+    def folder_object(self, target: FolderTarget, schema: str, name: str) -> Location:
+        return self.folder_root(target).join(
+            validate_name(schema, what="schema"),
+            validate_name(name, what="object name"),
+        )
+
+    def folder_staging(self, target: FolderTarget, schema: str, name: str) -> Location:
+        destination = self.folder_object(target, schema, name)
+        return Location(f"{destination.value}_Staging")
+
+    def delta_table(self, target: DeltaTarget, schema: str, name: str) -> Location:
+        return self.tables_root(target.lakehouse).join(
+            validate_name(schema, what="schema"),
+            validate_name(name, what="object name"),
+        )
+
+    def warehouse(self, target: WarehouseTarget) -> Item:
+        """The Warehouse item. Its SQL endpoint is reached over TDS, not OneLake."""
+
+        return self.resolve(target.warehouse, item_type=WAREHOUSE)
+
+    # --- what a Fabric workspace can do that a filesystem cannot -----------
+    #
+    # Two operations an installed bundle needs that are neither path arithmetic
+    # nor a SQL statement: pointing one Lakehouse at another's data, and asking a
+    # Lakehouse's SQL analytics endpoint to catch up. Both belong here because
+    # this is the adapter that already knows how to reach this workspace, and the
+    # local resolver deliberately offers neither — the emulator links files, and
+    # has no SQL endpoint at all.
+
+    def create_onelake_shortcut(
+        self,
+        item: ItemRef,
+        *,
+        path: str,
+        name: str,
+        source: ItemRef,
+        source_path: str,
+    ) -> dict:
+        """Point ``item``'s ``path/name`` at ``source``'s ``source_path``."""
+
+        from .shortcuts import create_shortcut
+
+        return create_shortcut(
+            self.resolve(item, item_type=LAKEHOUSE),
+            path=path,
+            name=name,
+            source=self.resolve(source, item_type=LAKEHOUSE),
+            source_path=source_path,
+            client=self._rest_client(),
+        )
+
+    def onelake_shortcuts(self, item: ItemRef) -> tuple:
+        """Every shortcut a Lakehouse holds. Scoping is the caller's business."""
+
+        from .shortcuts import list_shortcuts
+
+        return list_shortcuts(
+            self.resolve(item, item_type=LAKEHOUSE), client=self._rest_client()
+        )
+
+    def remove_onelake_shortcut(self, item: ItemRef, *, path: str, name: str) -> None:
+        """Take away this Lakehouse's name for another item's data.
+
+        Never the data: see :func:`weaver.fabric.shortcuts.delete_shortcut`.
+        """
+
+        from .shortcuts import delete_shortcut
+
+        delete_shortcut(
+            self.resolve(item, item_type=LAKEHOUSE),
+            path=path,
+            name=name,
+            client=self._rest_client(),
+        )
+
+
+    def sql_endpoint(self, target: WarehouseTarget):
+        """Resolve a typed Warehouse to the common SQL endpoint record."""
+
+        from ..sql import SqlEndpoint
+
+        warehouse = self.warehouse(target)
+        payload = self.client.get_json(
+            f"workspaces/{self.workspace.id}/warehouses/"
+            f"{warehouse.id}/connectionString"
+        )
+        value = payload.get("connectionString")
+        if not isinstance(value, str) or not value.strip():
+            raise CommandError(
+                f"Fabric returned no SQL connection string for Warehouse "
+                f"{warehouse.name!r}"
+            )
+        return SqlEndpoint(
+            server=_server_name(value),
+            database=warehouse.name,
+            workspace_id=self.workspace.id,
+            warehouse_id=warehouse.id,
+            warehouse_name=warehouse.name,
+        )
+
+    # --- the weaver lakehouse ---------------------------------------------
+
+    def _weaver_lakehouse(self) -> ItemRef:
+        name = self.configuration.weaver_lakehouse
+        if name is None:
+            raise CommandError(
+                "no Weaver Lakehouse for this Workspace — set weaver_lakehouse on the Workspace "
+                "or supply it explicitly"
+            )
+        return ItemRef(name)
+
+    @property
+    def weaver_lakehouse(self) -> Location:
+        return self.lakehouse(self._weaver_lakehouse())
+
+    @property
+    def weaver_items_root(self) -> Location:
+        """The workspace's one declaration, with item types directly below it."""
+
+        return self.files_root(self._weaver_lakehouse()) / WEAVER_ITEMS_AREA
+
+    @property
+    def build_bundles_root(self) -> Location:
+        return self.files_root(self._weaver_lakehouse()) / BUILD_BUNDLES_AREA
+
+    def build_bundle(self, name: str) -> Location:
+        from ..targets import validate_name
+
+        return self.build_bundles_root / validate_name(name, what="bundle name")
+
+    @property
+    def cli_root(self) -> Location:
+        return self.files_root(self._weaver_lakehouse()) / CLI_AREA
+
+    def cli_execution(self, execution_id: str) -> Location:
+        return self.cli_root / validate_name(execution_id, what="execution id")
+
+    def cli_bundle(self, execution_id: str) -> Location:
+        return self.cli_execution(execution_id) / "install.weaver.zip"
+
+    def lakehouse_spark_location(self, item: ItemRef) -> LakehouseSparkLocation:
+        """One destination Lakehouse's ``abfss://`` roots, for Spark to address.
+
+        Built from :meth:`spark_root`, which exists precisely so a session never
+        needs the item attached. The session stays attached to the Weaver
+        Lakehouse — the control plane — and destinations are reached explicitly.
+        """
+
+        root = self.spark_root(item).rstrip("/")
+        return LakehouseSparkLocation(
+            item=item.name,
+            tables_root=f"{root}/{TABLES_AREA}",
+            files_root=f"{root}/{FILES_AREA}",
+        )
+
+    def spark_destination(self, item: ItemRef) -> SparkDestination:
+        """One Lakehouse, as Fabric's Spark catalogue names it.
+
+        Fabric's namespace is the fundamental representation:
+        ``workspace.lakehouse.schema.object``. One session addresses every
+        Lakehouse in the workspace through it, so nothing has to be attached and
+        nothing has to be switched — and a schema-enabled Lakehouse pins its own
+        managed tables, which is why no path appears in the destination.
+
+        Display names, because that is what the namespace is spelled with. The
+        ids stay in resolution and in the bundle's target block.
+        """
+
+        return fabric_destination(
+            workspace=self.workspace.name,
+            lakehouse=self.resolve(item, item_type=LAKEHOUSE).name,
+        )
+
+    @property
+    def control_tables_root(self) -> Location:
+        return self.tables_root(self._weaver_lakehouse())
+
+
+def _server_name(value: str) -> str:
+    """Extract a server from Fabric's endpoint response.
+
+    The current Warehouse API returns a bare FQDN.  Accepting the familiar full
+    connection-string form as well makes the boundary tolerant without keeping
+    a credential-bearing connection string as endpoint identity.
+    """
+
+    text = value.strip().strip(";")
+    if ";" in text or "=" in text:
+        fields = {}
+        for part in text.split(";"):
+            if "=" in part:
+                key, item = part.split("=", 1)
+                fields[key.strip().lower()] = item.strip()
+        text = (
+            fields.get("server")
+            or fields.get("data source")
+            or fields.get("address")
+            or fields.get("addr")
+            or fields.get("network address")
+            or ""
+        )
+    text = text.removeprefix("tcp:").strip()
+    if "," in text:
+        text = text.rsplit(",", 1)[0].strip()
+    if not text:
+        raise CommandError("Fabric returned an invalid Warehouse SQL endpoint")
+    return text