pandion-api 0.1.0__py3-none-any.whl

pandion_api/__init__.py

@@ -0,0 +1,15 @@
+"""pandion-api: Pandion's public API over HTTP, with one Policy for a site's rules.
+
+`create_app(ApiSettings(data_dir=...), policy)` gives a FastAPI app. Every
+endpoint that takes a pipeline takes the pipeline file's shape as JSON.
+`create_app(settings, policy, mode="serve", pipeline="pipeline.yaml")` instead
+serves one fixed pipeline to a site's users. See docs/pandion-api/index.md.
+"""
+
+from pandion_api.app import create_app
+from pandion_api.policy import Policy
+from pandion_api.serve import ServeSettings
+from pandion_api.settings import ApiSettings
+from pandion_api.version import VERSION as __version__
+
+__all__ = ["ApiSettings", "Policy", "ServeSettings", "create_app", "__version__"]

pandion_api/app.py

@@ -0,0 +1,67 @@
+"""create_app: a FastAPI app over Pandion's public API, with a site's Policy."""
+
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Any
+
+from fastapi import FastAPI
+
+from pandion_api.documents import DocumentStore
+from pandion_api.policy import Policy
+from pandion_api.routes import ask, documents, evaluations, health, projects, questions, results, sort, strategies
+from pandion_api.routes import runs as run_routes
+from pandion_api.routes import settings as settings_routes
+from pandion_api.runs import RunManager
+from pandion_api.settings import ApiSettings
+
+#: Every router, in the order docs/pandion-api/index.md lists them.
+ROUTERS = (
+    health.router, strategies.router, documents.router, questions.router, run_routes.router, sort.router, ask.router,
+    evaluations.router, results.router, settings_routes.router, projects.router,
+)
+
+
+def create_app(
+    settings: ApiSettings,
+    policy: Policy | None = None,
+    *,
+    mode: str = "lab",
+    pipeline: str | Path | None = None,
+    conversations: Any = None,
+    serve: Any = None,
+) -> FastAPI:
+    """The app. `policy` holds the site's rules; the default allows everything.
+
+    `mode="lab"` (the default) takes a pipeline with each request. `mode="serve"`
+    loads the one pipeline file at `pipeline` and answers POST /chat; it keeps
+    conversations in `conversations` (a Pandion conversation store, else SQL in
+    the data folder) and takes its chat settings from `serve` (`ServeSettings`).
+    See docs/pandion-api/serve-mode.md, "Serve mode"."""
+    if mode == "serve":
+        from pandion_api.serve import create_serve_app
+
+        return create_serve_app(settings, policy, pipeline, conversations, serve)
+    if mode != "lab":
+        raise ValueError(f"mode must be 'lab' or 'serve', not '{mode}'.")
+    policy = policy or Policy()
+    manager = RunManager(limit=policy.max_active_runs)
+
+    @asynccontextmanager
+    async def lifespan(app: FastAPI):
+        await policy.on_startup(app)
+        try:
+            yield
+        finally:
+            await manager.shutdown()
+            await policy.on_shutdown(app)
+
+    app = FastAPI(title=settings.title, lifespan=lifespan)
+    app.state.settings = settings
+    app.state.policy = policy
+    app.state.runs = manager
+    app.state.documents = DocumentStore(settings.sources)
+    for router in ROUTERS:
+        app.include_router(router, prefix=settings.prefix, dependencies=policy.dependencies())
+    return app

pandion_api/checks.py

@@ -0,0 +1,92 @@
+"""Read a pipeline dict step by step, for a site's rules (see Policy.check_run).
+
+Every function here reads the pipeline file's shape, as `Pipeline.to_dict`
+gives it, and the strategies' settings as `pandion.catalog()` describes them.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pandion
+
+#: (section, key, stage) for each step that is one block.
+SINGLE = (
+    ("index", "source", "source"),
+    ("index", "parse", "parse"),
+    ("index", "chunk", "chunk"),
+    ("index", "enrich", "enrich"),
+    ("index", "store", "index"),
+    ("retrieve", "rewrite", "query"),
+    ("retrieve", "search", "retrieve"),
+)
+#: (section, key, stage) for each step that is a list of blocks.
+LISTS = (("index", "clean", "clean"), ("retrieve", "rerank", "rerank"))
+
+
+def _block(block: Any) -> dict[str, Any] | None:
+    return block if isinstance(block, dict) and isinstance(block.get("use"), str) else None
+
+
+def _section(config: dict[str, Any], name: str) -> dict[str, Any]:
+    section = config.get(name)
+    return section if isinstance(section, dict) else {}
+
+
+def steps(config: dict[str, Any]) -> list[tuple[str, str, str, dict[str, Any]]]:
+    """(where, stage, use, settings) for every step in `config`."""
+    found: list[tuple[str, str, str, dict[str, Any]]] = []
+
+    def add(where: str, stage: str, block: dict[str, Any]) -> None:
+        found.append((where, stage, block["use"], {k: v for k, v in block.items() if k != "use"}))
+
+    # index.embed is not visited: to_dict folds the embedder's settings into it,
+    # and no embedder has a read-only setting or a custom endpoint today.
+    for section, key, stage in SINGLE:
+        block = _block(_section(config, section).get(key))
+        if block:
+            add(f"{section}.{key}", stage, block)
+    for section, key, stage in LISTS:
+        blocks = _section(config, section).get(key)
+        for n, block in enumerate(blocks if isinstance(blocks, list) else [], start=1):
+            if _block(block):
+                add(f"{section}.{key}.{n}", stage, block)
+    answer = _block(config.get("answer"))
+    if answer:
+        add("answer", "use_case", answer)
+    return found
+
+
+def _properties(catalog: dict[str, Any], stage: str, use: str) -> dict[str, Any]:
+    return catalog.get(stage, {}).get(use, {}).get("config_schema", {}).get("properties", {})
+
+
+def chat_providers(catalog: dict[str, Any] | None = None) -> dict[str, str]:
+    """{chat model id: its provider}, as the chat step's `model` setting lists them."""
+    catalog = catalog if catalog is not None else pandion.catalog()
+    return dict((_properties(catalog, "use_case", "chat").get("model") or {}).get("x-providers") or {})
+
+
+def custom_endpoint_steps(config: dict[str, Any], catalog: dict[str, Any] | None = None) -> list[str]:
+    """The steps set to call a model at an address the request chose (an
+    OpenAI-compatible endpoint): a server would send a request there."""
+    catalog = catalog if catalog is not None else pandion.catalog()
+    found = []
+    for where, stage, use, settings in steps(config):
+        field = _properties(catalog, stage, use).get("model") or {}
+        providers = field.get("x-providers") or {}
+        model = settings.get("model")
+        if isinstance(model, str) and providers.get(model) == "openai_compatible":
+            found.append(where)
+    return found
+
+
+def changed_read_only_settings(config: dict[str, Any], catalog: dict[str, Any] | None = None) -> list[str]:
+    """Every setting marked readOnly (a prompt, a model revision) set to anything but its default."""
+    catalog = catalog if catalog is not None else pandion.catalog()
+    found = []
+    for where, stage, use, settings in steps(config):
+        for key, prop in _properties(catalog, stage, use).items():
+            if prop.get("readOnly") and key in settings and settings[key] != prop.get("default"):
+                found.append(f"{where}.{key}")
+    return found

pandion_api/credentials.py

@@ -0,0 +1,58 @@
+"""Model keys: which header and variable each provider's key comes in, and
+keeping keys out of everything a run shows.
+
+A key is never stored, echoed, hashed or logged. A run gets the keys
+`Policy.credentials(request)` resolves, holds them only in its worker's
+closure, and every event and result it gives passes through `redact`.
+"""
+
+from __future__ import annotations
+
+import traceback
+from dataclasses import dataclass
+from typing import Any, Callable, Iterable
+
+
+@dataclass(frozen=True)
+class ProviderKey:
+    header: str  # the request header a client sends a key in
+    env_var: str  # the server's environment variable
+    extra: str  # the key's name in Pandion's `credentials=`
+
+
+PROVIDERS: dict[str, ProviderKey] = {
+    "anthropic": ProviderKey("X-Anthropic-Api-Key", "ANTHROPIC_API_KEY", "anthropic_api_key"),
+    "openai": ProviderKey("X-OpenAI-Api-Key", "OPENAI_API_KEY", "openai_api_key"),
+    "custom": ProviderKey("X-Custom-Api-Key", "OPENAI_COMPATIBLE_API_KEY", "custom_api_key"),
+    "openrouter": ProviderKey("X-OpenRouter-Api-Key", "OPENROUTER_API_KEY", "openrouter_api_key"),
+}
+
+#: What a scrubbed key is replaced with.
+REDACTED = "[redacted]"
+
+
+def redact(value: Any, key: str | Iterable[str] | None) -> Any:
+    """Replace every occurrence of `key` (or of each of several keys) in the
+    strings inside `value`."""
+    keys = [key] if isinstance(key, str) else [k for k in (key or []) if k]
+    if not keys:
+        return value
+    if isinstance(value, str):
+        for k in sorted(keys, key=len, reverse=True):
+            value = value.replace(k, REDACTED)
+        return value
+    if isinstance(value, dict):
+        return {k: redact(v, keys) for k, v in value.items()}
+    if isinstance(value, (list, tuple)):
+        return type(value)(redact(v, keys) for v in value)
+    return value
+
+
+def guarded(keys: list[str], fn: Callable[[], Any]) -> Any:
+    """Run `fn`; a crash is raised again with every key scrubbed from its traceback."""
+    try:
+        return fn()
+    except Exception:
+        if not keys:
+            raise
+        raise RuntimeError(redact(traceback.format_exc(), keys)) from None

pandion_api/deps.py

@@ -0,0 +1,26 @@
+"""Plain FastAPI dependencies: what a route gets from the app it runs in."""
+
+from __future__ import annotations
+
+from fastapi import Request
+
+from pandion_api.documents import DocumentStore
+from pandion_api.policy import Policy
+from pandion_api.runs import RunManager
+from pandion_api.settings import ApiSettings
+
+
+def get_policy(request: Request) -> Policy:
+    return request.app.state.policy
+
+
+def get_settings(request: Request) -> ApiSettings:
+    return request.app.state.settings
+
+
+def get_runs(request: Request) -> RunManager:
+    return request.app.state.runs
+
+
+def get_documents(request: Request) -> DocumentStore:
+    return request.app.state.documents

pandion_api/documents.py

@@ -0,0 +1,206 @@
+"""Documents: each uploaded file once, by its fingerprint, with a record beside it.
+
+A file is kept as `<sha256><suffix>` in the `sources/` folder under the
+server's data folder, the name Pandion's own upload step looks for. Its record
+(`sources/.meta/<sha>.json`) holds the name it came with, its size and its
+content type, the link it was fetched from for a URL document (`origin_url`),
+and any other key a site's Policy adds, such as who uploaded it; a client is
+shown only the first five. A question set written for it lives in
+`sources/questions/<sha>.json`.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import threading
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import pandion
+from pandion.extend import FORMATS, format_of
+
+#: A document id is a sha256 in lowercase hex. Checked before an id reaches a
+#: path, so a crafted one such as `../../x` never leaves the folder.
+SHA = re.compile(r"^[0-9a-f]{64}\Z")
+#: The 404 for a document that is not there, or not for this caller. It never
+#: repeats the id the request sent.
+NO_SUCH_DOCUMENT = "There is no document with that fingerprint."
+META_DIR = ".meta"
+QUESTIONS_DIR = "questions"
+#: Held around every change to a record, here, in a Policy and in a site's own
+#: clean-up, so two uploads of the same bytes, or an upload and a clean-up,
+#: never lose what the other wrote.
+SIDECAR_LOCK = threading.Lock()
+#: The part of a request's file name that reaches a path: a plain suffix such
+#: as `.pdf`. Anything else (a `:` that names a Windows stream, a space, a
+#: separator) is dropped, and the file is kept under its sha alone.
+SAFE_SUFFIX = re.compile(r"^\.[A-Za-z0-9_-]{1,16}$")
+
+#: The refusal for a name ending in `.part`: a stored file with that suffix
+#: would look like a half-written one, which every reader skips.
+PART_REFUSED = "A file name ending in .part cannot be uploaded; rename it and try again."
+
+_PUBLIC = ("sha", "filename", "size", "content_type", "origin_url")
+
+
+def readable_formats() -> list[str] | None:
+    """Every format some parser here declares it reads, such as ["pdf", "docx"],
+    or None when a parser declares none (it is not checked, so any file may do)."""
+    found: set[str] = set()
+    for parser in pandion.catalog()["parse"].values():
+        formats = (parser.get("provides") or {}).get("formats")
+        if not formats:
+            return None
+        found.update(formats)
+    return [name for name in FORMATS if name in found]
+
+
+def unreadable(filename: str) -> str | None:
+    """The 415 sentence for a file no parser here reads, or None. A name
+    whose suffix the store would not keep (see `suffix`) is not checked, the
+    same as Pandion leaves a file with no extension unchecked."""
+    formats = readable_formats()
+    if formats is None or not suffix(filename) or format_of(filename) in formats:
+        return None
+    names = [FORMATS[name][0] for name in formats]
+    listed = names[0] if len(names) == 1 else f"{', '.join(names[:-1])} and {names[-1]}"
+    return f"This server reads {listed} files, and this file is not one of them."
+
+
+def suffix(filename: str) -> str:
+    """The suffix a file is stored with: its name's, when it is a plain one, else none."""
+    found = Path(filename).suffix
+    return found if SAFE_SUFFIX.match(found) else ""
+
+
+def safe_name(filename: str) -> str:
+    """`filename` as a run names its document. Pandion keeps its own copy of a
+    document as `<sha><the name's suffix>`, so the name's suffix must be the
+    one the store kept (`suffix`). When the store dropped it (a space, a `:`
+    that names a Windows stream, too long), every dot becomes `_`, leaving the
+    name no suffix at all. A plain name such as `report.pdf` is unchanged, so
+    its result ids are the old routes' ones."""
+    return filename if Path(filename).suffix == suffix(filename) else filename.replace(".", "_")
+
+
+def is_part(filename: str) -> bool:
+    """Whether the name ends in `.part`, in any case (see PART_REFUSED)."""
+    return filename.lower().endswith(".part")
+
+
+def read_sidecar(path: Path) -> dict[str, Any]:
+    """A record's JSON, or {} when it is missing, unreadable, or not an object."""
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+@dataclass(frozen=True)
+class Document:
+    sha: str
+    filename: str
+    size: int
+    content_type: str
+    #: The link a URL document was fetched from; None for an upload.
+    origin_url: str | None = None
+    #: Every other key in the record, for the site's Policy. Never shown.
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    def record(self) -> dict[str, Any]:
+        """What a client is shown: `origin_url` only for a URL document."""
+        shown = {"sha": self.sha, "filename": self.filename, "size": self.size, "content_type": self.content_type}
+        return {**shown, "origin_url": self.origin_url} if self.origin_url else shown
+
+    @classmethod
+    def of(cls, body: dict[str, Any]) -> Document | None:
+        """The document a record describes, or None for one without a valid id and name."""
+        sha, filename = body.get("sha"), body.get("filename")
+        if not isinstance(sha, str) or not SHA.match(sha) or not isinstance(filename, str):
+            return None
+        size, content_type, origin = body.get("size"), body.get("content_type"), body.get("origin_url")
+        return cls(
+            sha=sha,
+            filename=filename,
+            size=size if isinstance(size, int) else 0,
+            content_type=content_type if isinstance(content_type, str) else "application/octet-stream",
+            origin_url=origin if isinstance(origin, str) else None,
+            extra={k: v for k, v in body.items() if k not in _PUBLIC},
+        )
+
+
+class DocumentStore:
+    lock = SIDECAR_LOCK
+
+    def __init__(self, folder: str | Path) -> None:
+        self.folder = Path(folder)
+
+    def sidecar(self, sha: str) -> Path:
+        return self.folder / META_DIR / f"{sha}.json"
+
+    def questions(self, sha: str) -> Path:
+        return self.folder / QUESTIONS_DIR / f"{sha}.json"
+
+    def save(self, data: bytes, filename: str) -> str:
+        """Keep `data` as `<sha><suffix of filename>`, unless a file with this sha
+        is here already, under any suffix: the same bytes are never stored
+        twice, whatever they are called. Only a plain suffix comes from the
+        name (see `suffix`). The bytes go to a side file first and are renamed
+        into place, so a reader never sees half a file. Returns the sha."""
+        sha = hashlib.sha256(data).hexdigest()
+        dest = self.folder / f"{sha}{suffix(filename)}"
+        if self.file(sha) is None:
+            self.folder.mkdir(parents=True, exist_ok=True)
+            part = dest.with_name(dest.name + ".part")
+            part.write_bytes(data)
+            os.replace(part, dest)
+        return sha
+
+    def write(
+        self, sha: str, filename: str, size: int, content_type: str, origin_url: str | None = None
+    ) -> Document:
+        """Write the record. Keys a site added stay, and so does an `origin_url`
+        an earlier fetch of the same bytes wrote. Call with `lock` held."""
+        path = self.sidecar(sha)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        body = {**read_sidecar(path), "sha": sha, "filename": filename, "size": size, "content_type": content_type}
+        if origin_url:
+            body["origin_url"] = origin_url
+        path.write_text(json.dumps(body), encoding="utf-8")
+        return Document.of(body)
+
+    def get(self, sha: str) -> Document | None:
+        if not SHA.match(sha):
+            return None
+        return Document.of(read_sidecar(self.sidecar(sha)))
+
+    def all(self) -> list[Document]:
+        meta = self.folder / META_DIR
+        if not meta.is_dir():
+            return []
+        found = (Document.of(read_sidecar(p)) for p in meta.glob("*.json"))
+        return [d for d in found if d is not None]
+
+    def file(self, sha: str) -> Path | None:
+        """The stored file, or None. A seeded file with no record is found too."""
+        if not SHA.match(sha) or not self.folder.is_dir():
+            return None
+        for p in sorted(self.folder.glob(f"{sha}*")):
+            if p.is_file() and not p.name.endswith(".part"):
+                return p
+        return None
+
+    def remove(self, sha: str) -> None:
+        """Delete the file, its record and its question set. Call with `lock` held."""
+        if not SHA.match(sha):
+            return
+        for p in self.folder.glob(f"{sha}*"):
+            if p.is_file():
+                p.unlink()
+        self.sidecar(sha).unlink(missing_ok=True)
+        self.questions(sha).unlink(missing_ok=True)

pandion_api/errors.py

@@ -0,0 +1,20 @@
+"""What a serve caller sees when something fails: one sentence and a trace id.
+The detail goes to the log under the same id."""
+
+from __future__ import annotations
+
+import logging
+import uuid
+
+logger = logging.getLogger("pandion_api")
+
+
+def generic(trace_id: str) -> str:
+    return f"Something went wrong. Quote this trace id when you ask for help: {trace_id}."
+
+
+def hide(detail: str, trace_id: str | None = None) -> str:
+    """Log `detail` at ERROR with a trace id and return the sentence for the caller."""
+    trace_id = trace_id or uuid.uuid4().hex
+    logger.error("Serve error (trace %s): %s", trace_id, detail)
+    return generic(trace_id)

pandion_api/jobs.py

@@ -0,0 +1,141 @@
+"""Starting a run, and the documents it reads."""
+
+from __future__ import annotations
+
+import threading
+from typing import Annotated, Any, Callable
+
+from fastapi import HTTPException, Request
+from pydantic import BaseModel, Field
+
+from pandion_api.credentials import PROVIDERS, guarded, redact
+from pandion_api.errors import hide
+from pandion_api.documents import NO_SUCH_DOCUMENT, SHA, safe_name
+from pandion_api.pipelines import problems
+
+#: What a run's work does on the worker thread: given the event callback, the
+#: cancel event and the keys (None when there are none), it returns
+#: (ok, cancelled, result) with a plain JSON result.
+Work = Callable[[Callable[[dict[str, Any]], None], threading.Event, dict[str, str] | None], tuple[bool, bool, Any]]
+
+
+#: One document's labels: at most 50, a name up to 2,000 characters, a value up to 10,000.
+Labels = Annotated[dict[Annotated[str, Field(max_length=2000)], Annotated[str, Field(max_length=10000)]], Field(max_length=50)]
+
+
+class SortedIn(BaseModel):
+    """A request body that names documents and may say which category each
+    goes to. Both maps are keyed by the document's sha, as `documents` is."""
+
+    #: Pin a document to a category; this wins over the route.
+    categories: dict[str, str] = {}
+    #: Labels for a document, read by the route's `metadata` check.
+    metadata: dict[str, Labels] = {}
+
+
+def assign_to(p: Any, body: SortedIn, names: list[str], request: Request) -> None:
+    """Give `p` the pins and labels of `body`, and the link each URL document
+    was fetched from. `names` are the document ids the pipeline knows, in the
+    order of `body.documents`. A pin or label for a document not in
+    `documents`, or a pin to an unknown category, is a 422."""
+    ids = dict(zip(body.documents, names))
+    store = request.app.state.documents
+    links = {name: d.origin_url for sha, name in ids.items() if (d := store.get(sha)) and d.origin_url}
+    for sha in [*body.categories, *body.metadata]:
+        if sha not in ids:
+            raise problems(f"categories and metadata name document {sha}, which is not one of the documents sent.")
+    try:
+        p.index.assign(
+            categories={ids[s]: c for s, c in body.categories.items()},
+            metadata={ids[s]: m for s, m in body.metadata.items()},
+            links=links,
+        )
+    except ValueError as exc:
+        raise problems(str(exc)) from None
+
+
+def documents_of(shas: list[str], request: Request) -> dict[str, str]:
+    """{document name: stored file} for a run, named as each was uploaded, so
+    the same bytes under the same name always give the same result ids. A file
+    with no record takes the name `Policy.document_name` gives it, else its
+    stored file's name."""
+    store = request.app.state.documents
+    policy = request.app.state.policy
+    out: dict[str, str] = {}
+    for sha in shas:
+        if not SHA.match(sha):
+            raise HTTPException(status_code=404, detail=NO_SUCH_DOCUMENT)
+        if not policy.can_read(sha, request):
+            raise HTTPException(status_code=403, detail=policy.not_readable)
+        path = store.file(sha)
+        if path is None:
+            raise HTTPException(status_code=404, detail=NO_SUCH_DOCUMENT)
+        document = store.get(sha)
+        given = policy.document_name(sha, request) if document is None else None
+        name = safe_name(document.filename) if document is not None else safe_name(given) if given else path.name
+        if name in out:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Two documents are both named {name}. Upload one of them under another name.",
+            )
+        out[name] = str(path)
+    return out
+
+
+def scrub_errors(value: Any, scrub: Callable[[str], str]) -> Any:
+    """`value` with `scrub` applied to every string under an `error` key, at any depth."""
+    if isinstance(value, dict):
+        return {
+            k: scrub(v) if k == "error" and isinstance(v, str) else scrub_errors(v, scrub)
+            for k, v in value.items()
+        }
+    if isinstance(value, (list, tuple)):
+        return type(value)(scrub_errors(v, scrub) for v in value)
+    return value
+
+
+def too_many_runs(most: int) -> str:
+    """The 429 for an owner who already has `most` runs going."""
+    runs = "run" if most == 1 else "runs"
+    return f"You already have {most} {runs} going. Wait for one to finish, or stop one."
+
+
+def start(request: Request, kind: str, work: Work) -> dict[str, str]:
+    """Start `work` on the app's run manager with this request's keys, as this
+    request's owner (`Policy.run_owner`). Every event and the result are
+    scrubbed of the keys, and their error text goes through `Policy.scrub`; a
+    `run_cancelled` event marks the run cancelled. An owner who already has
+    `Policy.max_runs_per_owner` runs working or waiting is refused with a 429."""
+    policy = request.app.state.policy
+    owner = policy.run_owner(request)
+    most = policy.max_runs_per_owner(request)
+    if owner is not None and most is not None:
+        going = sum(1 for s in request.app.state.runs.active() if s.owner == owner)
+        if going >= most:
+            raise HTTPException(status_code=429, detail=too_many_runs(most))
+    serving = hasattr(request.app.state, "serve")  # serve mode: env keys only, generic errors
+    if serving:
+        credentials = {
+            spec.extra: key for p, spec in PROVIDERS.items() if (key := policy.server_key(p)[0])
+        }
+    else:
+        credentials = policy.credentials(request)
+    keys = list(credentials.values())
+
+    def clean(value: Any) -> Any:
+        return scrub_errors(redact(value, keys), policy.scrub)
+
+    def job(emit, cancel):
+        seen: list[bool] = []
+
+        def on_event(e: dict[str, Any]) -> None:
+            if e.get("event") == "run_cancelled":
+                seen.append(True)
+            emit(clean(e))
+
+        ok, cancelled, result = guarded(keys, lambda: work(on_event, cancel, credentials or None))
+        return ok, cancelled or bool(seen), clean(result)
+
+    scrub = (lambda text: hide(redact(text, keys), getattr(request.state, "trace_id", None))) if serving else (lambda text: policy.scrub(redact(text, keys)))
+    state = request.app.state.runs.start(kind, job, owner=owner, scrub=scrub)
+    return {"run_id": state.run_id}