blobhub 0.2.0__py3-none-any.whl

blobhub/__init__.py

@@ -0,0 +1,57 @@
+"""BlobHub Python SDK: blobs, revisions and typed data access for the BlobHub platform."""
+
+__version__ = "0.2.0"
+
+import importlib
+
+from blobhub._blob import Blob
+from blobhub._client import Client, Identity, connect
+from blobhub._operation import Operation
+from blobhub._revision import Revision
+from blobhub.errors import (
+    AuthError,
+    BlobHubError,
+    CommandError,
+    ConfigError,
+    Conflict,
+    NetworkError,
+    NotFound,
+    OperationFailed,
+    PermissionDenied,
+    RateLimited,
+    RequestNotSent,
+    ServerError,
+    TypeMismatch,
+    WaitTimeout,
+)
+
+__all__ = [
+    "AuthError",
+    "Blob",
+    "BlobHubError",
+    "Client",
+    "CommandError",
+    "ConfigError",
+    "Conflict",
+    "Identity",
+    "NetworkError",
+    "NotFound",
+    "Operation",
+    "OperationFailed",
+    "PermissionDenied",
+    "RateLimited",
+    "RequestNotSent",
+    "Revision",
+    "ServerError",
+    "TypeMismatch",
+    "WaitTimeout",
+    "__version__",
+    "connect",
+]
+
+
+def __getattr__(name: str):
+    """Resolve `blobhub.notebooks` lazily so `import blobhub` stays light."""
+    if name == "notebooks":
+        return importlib.import_module("blobhub.notebooks")
+    raise AttributeError(f"module 'blobhub' has no attribute {name!r}")

blobhub/_auth.py

@@ -0,0 +1,83 @@
+"""How a request is authenticated: an API key, or an anonymous token pair held in memory."""
+
+import time
+from collections.abc import Callable
+
+from blobhub.errors import AuthError, ServerError
+
+REFRESH_EARLY_MS = 60_000
+
+Fetch = Callable[[str, str, dict | None], dict]
+
+
+class KeyAuth:
+    """Sends the key as `X-API-Key`; a rejected key is final, never a reason to fall back to anonymous."""
+
+    def __init__(self, api_key: str) -> None:
+        self._api_key = api_key
+
+    def __repr__(self) -> str:
+        return "KeyAuth()"
+
+    def headers(self) -> dict[str, str]:
+        """The headers that authenticate one request."""
+        return {"X-API-Key": self._api_key}
+
+    def renew(self) -> bool:
+        """A key cannot be renewed; returns False."""
+        return False
+
+
+class AnonymousAuth:
+    """Mints an anonymous token pair on first use and renews it; both tokens live in memory only."""
+
+    def __init__(self, fetch: Fetch, *, wall_clock: Callable[[], float] = time.time) -> None:
+        self._fetch = fetch
+        self._wall_clock = wall_clock
+        self._access: str | None = None
+        self._refresh: str | None = None
+        self._expires_at_ms: int | None = None
+
+    def __repr__(self) -> str:
+        return "AnonymousAuth()"
+
+    def headers(self) -> dict[str, str]:
+        """A bearer header, minting the pair first or refreshing a token within a minute of `expires_at`."""
+        if self._access is None:
+            self._mint()
+        elif self._expires_at_ms is not None and self._wall_clock() * 1000 >= self._expires_at_ms - REFRESH_EARLY_MS:
+            self._renew()
+        return {"Authorization": f"Bearer {self._access}"}
+
+    def renew(self) -> bool:
+        """Refresh the access token, or mint a new pair if the refresh is refused; returns True."""
+        self._renew()
+        return True
+
+    def _mint(self) -> None:
+        body = self._fetch("GET", "auth/core/anonymous", None)
+        self._refresh = _token(body, "refresh_token")
+        self._store(body)
+
+    def _renew(self) -> None:
+        if self._refresh is None:
+            self._mint()
+            return
+        try:
+            body = self._fetch("POST", "auth/core/refresh", {"refresh_token": self._refresh})
+        except AuthError:
+            self._mint()
+            return
+        self._store(body)
+
+    def _store(self, body: dict) -> None:
+        self._access = _token(body, "access_token")
+        expires_at = body.get("expires_at")
+        self._expires_at_ms = expires_at if isinstance(expires_at, int) else None
+
+
+def _token(body: dict, name: str) -> str:
+    value = body.get(name)
+    if not isinstance(value, str) or not value:
+        raise ServerError("invalid_response", f"the token response has no {name}")
+    return value

blobhub/_blob.py

@@ -0,0 +1,96 @@
+"""`Blob`: one blob's record, its revisions, and revision create."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import TYPE_CHECKING
+
+from blobhub._revision import Revision, settled
+from blobhub._transport import required, segment
+from blobhub.errors import ServerError
+
+if TYPE_CHECKING:
+    from blobhub._client import Client
+
+
+class Blob:
+    """A blob's record; after the first fetch it is addressed by its canonical org and blob ids."""
+
+    def __init__(self, client: Client, record: dict) -> None:
+        self.client = client
+        self.record = record
+
+    def __repr__(self) -> str:
+        return f"Blob(id={self.id!r}, alias={self.alias!r}, type={self.type!r})"
+
+    @property
+    def id(self) -> str:
+        return self.record["id"]
+
+    @property
+    def alias(self) -> str | None:
+        return self.record.get("alias")
+
+    @property
+    def org_id(self) -> str:
+        return self.record["org_id"]
+
+    @property
+    def type(self) -> str:
+        return self.record["type"]
+
+    @property
+    def visibility(self) -> str | None:
+        return self.record.get("visibility")
+
+    @property
+    def status(self) -> str | None:
+        return self.record.get("status")
+
+    @property
+    def default_revision_id(self) -> str | None:
+        return self.record.get("default_revision_id")
+
+    @property
+    def _path(self) -> str:
+        return f"blobs/{segment(self.org_id)}/{segment(self.id)}"
+
+    def refresh(self) -> Blob:
+        """Re-read the record (`GET /blobs/{org_id}/{id}`)."""
+        self.record = required(self.client._send("GET", self._path)[0], "blob")
+        return self
+
+    def revision(self, ref: str = "default") -> Revision:
+        """The revision `ref`, a revision id or `default` (`GET …/revisions/{ref}`)."""
+        body = self.client._send("GET", f"{self._path}/revisions/{segment(ref)}")[0]
+        return Revision(self, required(body, "revision"))
+
+    def revisions(self, *, page_size: int = 50) -> Iterator[Revision]:
+        """Every revision, newest first, following the cursor until the server returns none.
+
+        :param page_size: the `limit` sent with every page, 1 to 200.
+        """
+        cursor = None
+        sent: set[str] = set()
+        while True:
+            params = {"limit": page_size, "cursor": cursor}
+            body = self.client._send("GET", f"{self._path}/revisions", params=params)[0]
+            for record in body.get("revisions", []):
+                yield Revision(self, record)
+            following = body.get("cursor")
+            if not following:
+                return
+            if following in sent:
+                raise ServerError("cursor_loop", "the revision listing returned a cursor it had already been sent")
+            sent.add(following)
+            cursor = following
+
+    def create_revision(self, comment: str, *, timeout: float = 600.0) -> Revision:
+        """Create a revision from the committed default (`POST …/revisions`) and wait until it is `ready`.
+
+        :param comment: the new revision's comment, 1 to 200 characters.
+        :param timeout: seconds to wait before WaitTimeout.
+        """
+        body = self.client._send("POST", f"{self._path}/revisions", json_body={"comment": comment})[0]
+        revision = Revision(self, {"id": required(body, "revision_id"), "blob_id": self.id, "status": "creating"})
+        return revision._wait_for("creating revision", timeout, settled)

blobhub/_cache.py

@@ -0,0 +1,83 @@
+"""D8: where downloaded artifacts are cached, and when a cached copy may be reused.
+
+Write protocol: call `invalidate` before (re)writing an artifact, write it to a temp file in the same
+directory and `os.replace` it into place, then call `remember` last. A copy with no valid stamp is never reused.
+"""
+
+import json
+import os
+import tempfile
+from pathlib import Path, PurePosixPath, PureWindowsPath
+
+SIDECAR_SUFFIX = ".blobhub.json"
+
+
+def cache_root() -> Path:
+    """`$XDG_CACHE_HOME/blobhub` when that is an absolute path, else `~/.cache/blobhub`; read at call time."""
+    base = os.environ.get("XDG_CACHE_HOME", "")
+    root = Path(base) if base and Path(base).is_absolute() else Path.home() / ".cache"
+    return root / "blobhub"
+
+
+def plain(name: str, what: str) -> str:
+    """`name` itself when it is a plain file name; ValueError for empty, `.`, `..`, a separator, drive or NUL."""
+    if (
+        not name
+        or name in (".", "..")
+        or ":" in name
+        or "\x00" in name
+        or PurePosixPath(name).name != name
+        or PureWindowsPath(name).name != name
+    ):
+        raise ValueError(f"{what} {name!r} is not a plain file name")
+    return name
+
+
+def artifact_path(revision_id: str, name: str) -> Path:
+    """Where the cached copy of artifact `name` of revision `revision_id` lives."""
+    if name.endswith(SIDECAR_SUFFIX):
+        raise ValueError(f"artifact name {name!r} is not a plain file name")
+    return cache_root() / "revisions" / plain(revision_id, "revision id") / plain(name, "artifact name")
+
+
+def is_reusable(path: Path, revision: dict) -> bool:
+    """Whether `path` is a complete copy taken from this exact committed state of `revision`."""
+    if revision.get("phase") != "commit" or revision.get("status") != "ready" or not path.is_file():
+        return False
+    if revision.get("id") is None or revision.get("updated_at") is None:
+        return False
+    try:
+        stamp = json.loads(_sidecar(path).read_text())
+    except (OSError, ValueError):
+        return False
+    return (
+        isinstance(stamp, dict)
+        and stamp.get("revision_id") == revision.get("id")
+        and stamp.get("updated_at") == revision.get("updated_at")
+        and stamp.get("size") == path.stat().st_size
+    )
+
+
+def remember(path: Path, revision: dict) -> None:
+    """Record, atomically, which revision state `path` was downloaded from; call it last, after `invalidate`."""
+    for key in ("id", "updated_at"):
+        if revision.get(key) is None:
+            raise ValueError(f"revision has no {key}")
+    stamp = {"revision_id": revision["id"], "updated_at": revision["updated_at"], "size": path.stat().st_size}
+    fd, temp = tempfile.mkstemp(dir=path.parent, prefix=".stamp-")
+    try:
+        with os.fdopen(fd, "w") as handle:
+            handle.write(json.dumps(stamp))
+        os.replace(temp, _sidecar(path))
+    except BaseException:
+        Path(temp).unlink(missing_ok=True)
+        raise
+
+
+def invalidate(path: Path) -> None:
+    """Drop `path`'s stamp so it is not reused; call it before rewriting the artifact. Absent is fine."""
+    _sidecar(path).unlink(missing_ok=True)
+
+
+def _sidecar(path: Path) -> Path:
+    return path.with_name(path.name + SIDECAR_SUFFIX)

blobhub/_client.py

@@ -0,0 +1,126 @@
+"""`connect()` and `Client`: resolved credentials, the transport, and the entry points to the platform."""
+
+import random
+import time
+from collections.abc import Callable
+from typing import NamedTuple
+
+import httpx
+
+from blobhub import _settings
+from blobhub._auth import AnonymousAuth, KeyAuth
+from blobhub._blob import Blob
+from blobhub._operation import Operation
+from blobhub._transport import Transport, required, segment
+
+
+class Identity(NamedTuple):
+    """Where the client's key came from, and the API it talks to; never the key itself."""
+
+    source: str
+    api_url: str
+
+
+class Client:
+    """A connection to the BlobHub API. Build one with `blobhub.connect()`; use one per thread."""
+
+    def __init__(
+        self,
+        settings: _settings.Settings,
+        *,
+        transport: httpx.BaseTransport | None = None,
+        sleep: Callable[[float], None] = time.sleep,
+        clock: Callable[[], float] = time.monotonic,
+        wall_clock: Callable[[], float] = time.time,
+        rng: Callable[[], float] = random.random,
+    ) -> None:
+        self._identity = Identity(source=settings.source, api_url=settings.api_url)
+        self._sleep = sleep
+        self._clock = clock
+        self._transport = Transport(settings.api_url, transport=transport, sleep=sleep, rng=rng)
+        if settings.api_key is None:
+            self._transport.auth = AnonymousAuth(self._unauthenticated, wall_clock=wall_clock)
+        else:
+            self._transport.auth = KeyAuth(settings.api_key)
+
+    def __repr__(self) -> str:
+        return f"Client(source={self._identity.source!r}, api_url={self._identity.api_url!r})"
+
+    def __enter__(self) -> "Client":
+        return self
+
+    def __exit__(self, *exc_info: object) -> None:
+        self.close()
+
+    @property
+    def identity(self) -> Identity:
+        """Where the key came from and which API this client talks to."""
+        return self._identity
+
+    @property
+    def anonymous(self) -> bool:
+        """Whether this client reads with an anonymous token."""
+        return self._identity.source == "anonymous"
+
+    def close(self) -> None:
+        """Close the underlying HTTP connections."""
+        self._transport.close()
+
+    def whoami(self) -> dict | None:
+        """The calling user's record (`GET /users/me`), or None for an anonymous client."""
+        if self.anonymous:
+            return None
+        return required(self.request("GET", "users/me"), "user")
+
+    def blob(self, org: str, blob: str) -> Blob:
+        """The blob `org/blob`, each part an id or an alias, fetched once (`GET /blobs/{org}/{blob}`)."""
+        return Blob(self, required(self._send("GET", f"blobs/{segment(org)}/{segment(blob)}")[0], "blob"))
+
+    def operation(self, operation_id: str) -> Operation:
+        """The operation `operation_id`, fetched once (`GET /operations/{id}`)."""
+        body = self._send("GET", f"operations/{segment(operation_id)}")[0]
+        return Operation(self, required(body, "operation"))
+
+    def request(self, method: str, path: str, *, json: dict | None = None, params: dict | None = None) -> dict:
+        """Send one request relative to the API URL and return its success envelope.
+
+        :param json: the JSON body, sent exactly as given.
+        :param params: query parameters; None values are left out.
+        """
+        return self._send(method.upper(), path, json_body=json, params=params)[0]
+
+    def upload_part(self, part: dict, data: bytes) -> str:
+        """PUT one part of a presigned upload plan and return its ETag; no credential is sent."""
+        upload = part["upload"]
+        return self._transport.put_part(upload["url"], dict(upload.get("headers") or {}), data)
+
+    def download_part(self, part: dict) -> bytes:
+        """GET one part of a presigned download plan and check its length; no credential is sent."""
+        download = part["download"]
+        return self._transport.get_part(download["url"], dict(download.get("headers") or {}), part["size"])
+
+    def _send(self, method: str, path: str, **options: object) -> tuple[dict, bytes | None]:
+        return self._transport.send(method, path, **options)
+
+    def _unauthenticated(self, method: str, path: str, body: dict | None) -> dict:
+        return self._transport.send(method, path, json_body=body, authenticated=False)[0]
+
+
+def connect(
+    *,
+    profile: str | None = None,
+    api_key: str | None = None,
+    api_url: str | None = None,
+    anonymous: bool = False,
+    transport: httpx.BaseTransport | None = None,
+) -> Client:
+    """A client for the first credential the ladder finds: argument, environment, profile, then anonymous.
+
+    :param profile: a profile name from ~/.blobhub/credentials.yaml.
+    :param api_key: an API key; outranks BLOBHUB_API_KEY and the profile.
+    :param api_url: the API base URL; outranks BLOBHUB_API_URL and the profile.
+    :param anonymous: read public content with an anonymous token, ignoring the profile file.
+    :param transport: an httpx transport for API and part requests (tests, proxies).
+    """
+    settings = _settings.resolve(profile=profile, api_key=api_key, api_url=api_url, anonymous=anonymous)
+    return Client(settings, transport=transport)

blobhub/_credentials.py

@@ -0,0 +1,78 @@
+"""Read-only access to the profile file blobhub-cli writes: `~/.blobhub/credentials.yaml`."""
+
+import stat
+from pathlib import Path
+from typing import NamedTuple
+
+import yaml
+
+from blobhub.errors import ConfigError
+
+BLOBHUB_HOME = Path.home() / ".blobhub"
+
+
+class Profile(NamedTuple):
+    """One stored profile: its API key and API URL."""
+
+    key: str
+    url: str
+
+    def __repr__(self) -> str:
+        return f"Profile(key='***', url={self.url!r})"
+
+
+def credentials_path() -> Path:
+    """The profile file's path, read from `BLOBHUB_HOME` at call time."""
+    return BLOBHUB_HOME / "credentials.yaml"
+
+
+def load() -> tuple[dict[str, Profile], str | None]:
+    """The file's profiles and its `default`; `({}, None)` when the file does not exist. Never writes it."""
+    path = credentials_path()
+    text = _read(path)
+    if text is None:
+        return {}, None
+    try:
+        data = yaml.safe_load(text)
+    except yaml.YAMLError:
+        # PyYAML's problem marks quote the offending line, which may hold a key.
+        raise ConfigError("invalid_credentials_file", f"{path} is not valid YAML; run `blobhub login`") from None
+    if not isinstance(data, dict) or not isinstance(data.get("profiles"), dict):
+        raise ConfigError("invalid_credentials_file", f"{path} is not a credentials file; run `blobhub login`")
+    profiles = {}
+    for name, entry in data["profiles"].items():
+        if not isinstance(entry, dict) or not _filled(entry.get("key")) or not _filled(entry.get("url")):
+            raise ConfigError(
+                "invalid_credentials_file", f"{path} has a malformed profile {name!r}; run `blobhub login`"
+            )
+        profiles[str(name)] = Profile(key=entry["key"], url=entry["url"])
+    default = data.get("default")
+    if default is not None and not isinstance(default, str):
+        raise ConfigError(
+            "invalid_credentials_file", f"{path} has a default that is not a profile name; run `blobhub login`"
+        )
+    return profiles, default
+
+
+def _read(path: Path) -> str | None:
+    """The file's text once its mode is owner-only, or None when it does not exist."""
+    try:
+        if not path.exists():
+            return None
+        mode = stat.S_IMODE(path.stat().st_mode)
+        if mode & 0o077:
+            raise ConfigError(
+                "insecure_credentials_file",
+                f"{path} is readable by group or other (mode {oct(mode)}); run `chmod 600 {path}` or `blobhub login`",
+            )
+        return path.read_text(encoding="utf-8")
+    except OSError as error:
+        reason = error.strerror or type(error).__name__
+        raise ConfigError("unreadable_credentials_file", f"{path} cannot be read ({reason})") from None
+    except UnicodeDecodeError:
+        # The decode error holds the file's bytes, keys included.
+        raise ConfigError("invalid_credentials_file", f"{path} is not UTF-8 text; run `blobhub login`") from None
+
+
+def _filled(value: object) -> bool:
+    return isinstance(value, str) and bool(value)

blobhub/_operation.py

@@ -0,0 +1,61 @@
+"""`Operation`: a long-running server-side operation, such as the processing of an ONNX upload."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from blobhub._transport import required, segment
+from blobhub._wait import INTERVAL, wait_until
+from blobhub.errors import OperationFailed
+
+if TYPE_CHECKING:
+    from blobhub._client import Client
+
+TERMINAL = ("completed", "failed")
+
+
+class Operation:
+    """One operation's record (`GET /operations/{id}`) and a way to wait for its outcome."""
+
+    def __init__(self, client: Client, record: dict) -> None:
+        self.client = client
+        self.record = record
+
+    def __repr__(self) -> str:
+        return f"Operation(id={self.id!r}, status={self.status!r})"
+
+    @property
+    def id(self) -> str:
+        return self.record["id"]
+
+    @property
+    def status(self) -> str | None:
+        return self.record.get("status")
+
+    @property
+    def type(self) -> str | None:
+        return self.record.get("type")
+
+    @property
+    def revision_id(self) -> str | None:
+        return self.record.get("revision_id")
+
+    def refresh(self) -> Operation:
+        """Re-read the record."""
+        self.record = required(self.client._send("GET", f"operations/{segment(self.id)}")[0], "operation")
+        return self
+
+    def wait(self, *, timeout: float = 600.0, interval: float = INTERVAL) -> Operation:
+        """Poll until the operation completes; OperationFailed if it fails, WaitTimeout past `timeout` seconds."""
+        wait_until(
+            self.refresh,
+            lambda operation: operation.status in TERMINAL,
+            what=f"operation {self.id}",
+            timeout=timeout,
+            interval=interval,
+            sleep=self.client._sleep,
+            clock=self.client._clock,
+        )
+        if self.status == "failed":
+            raise OperationFailed("operation_failed", f"operation {self.id} failed")
+        return self

blobhub/_retry.py

@@ -0,0 +1,52 @@
+"""D5: which requests may be retried after which failures, and how long to wait between attempts."""
+
+import email.utils
+import math
+import time
+from collections.abc import Callable
+
+from blobhub.errors import BlobHubError, NetworkError, RateLimited, RequestNotSent, ServerError
+
+ATTEMPTS = 3
+BASE_DELAY = 0.5
+MAX_DELAY = 8.0
+MAX_RETRY_AFTER = 60.0
+
+SAFE = "safe"
+UNSAFE = "unsafe"
+
+
+def request_class(method: str, path: str) -> str:
+    """SAFE for a GET or a `/data/query`, which can run twice harmlessly; UNSAFE for every other request."""
+    route = path.split("?", 1)[0].rstrip("/")
+    if method.upper() == "GET" or route.endswith("/data/query"):
+        return SAFE
+    return UNSAFE
+
+
+def should_retry(error: BlobHubError, kind: str) -> bool:
+    """Whether a failed attempt of a request of class `kind` may be repeated."""
+    if isinstance(error, (RateLimited, RequestNotSent)):
+        return True
+    return kind == SAFE and isinstance(error, (ServerError, NetworkError))
+
+
+def backoff(attempt: int, rng: Callable[[], float]) -> float:
+    """The full-jitter delay after failed attempt number `attempt`, counting from 0."""
+    return rng() * min(MAX_DELAY, BASE_DELAY * 2**attempt)
+
+
+def parse_retry_after(value: str | None, *, now: Callable[[], float] = time.time) -> float | None:
+    """Seconds named by a Retry-After value (delta-seconds or HTTP-date), clamped to [0, 60]; None if unusable."""
+    if not value:
+        return None
+    try:
+        seconds = float(value)
+    except ValueError:
+        try:
+            seconds = email.utils.parsedate_to_datetime(value).timestamp() - now()
+        except (TypeError, ValueError):
+            return None
+    if not math.isfinite(seconds):
+        return None
+    return min(max(seconds, 0.0), MAX_RETRY_AFTER)