spanlight 0.1.0__py3-none-any.whl

spanlight/_experiments_api.py

@@ -0,0 +1,182 @@
+"""The slice of the dashboard API (``/api/v1``) the experiment runner reads and writes.
+
+Authenticated with a personal access token. Every failure becomes an :class:`ExperimentError`
+whose message names the request and the server's problem ``code`` and ``detail`` and never the
+token or a response body that is not a problem document.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import quote, urlsplit
+
+import httpx
+
+from ._experiments_parse import PromptRef
+from ._version import __version__
+
+_PAGE_SIZE = 200
+# A guard against a server that keeps returning cursors: 50 pages of 200 is 10 000 rows.
+_MAX_PAGES = 50
+_ESCAPE = "~"
+_LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
+
+
+class ExperimentError(Exception):
+    """The run could not be set up or finished; the message is safe to print."""
+
+
+@dataclass(frozen=True)
+class ResolvedPrompt:
+    """One prompt version the experiment will run."""
+
+    name: str
+    version: int
+    version_id: str
+
+
+def _require_safe_host(host: str) -> None:
+    """Refuse to send the access token over plain http, except to the local machine."""
+    parts = urlsplit(host)
+    if parts.scheme == "https" and parts.hostname:
+        return
+    if parts.scheme == "http" and parts.hostname in _LOCAL_HOSTS:
+        return
+    raise ExperimentError(
+        "the host must be an https:// URL (http:// is allowed only for localhost); "
+        "the access token is never sent in clear text"
+    )
+
+
+def wire_name(name: str) -> str:
+    """A prompt name as it travels in a path: the server strips exactly one leading ``~``."""
+    escaped = f"{_ESCAPE}{name}" if name in {".", ".."} or name.startswith(_ESCAPE) else name
+    return quote(escaped, safe="/")
+
+
+class Api:
+    """A small JSON client for one Spanlight host."""
+
+    def __init__(
+        self,
+        *,
+        host: str,
+        token: str,
+        project_id: str,
+        timeout: float = 30.0,
+        transport: httpx.BaseTransport | None = None,
+    ) -> None:
+        _require_safe_host(host)
+        self.project_id = project_id
+        self._client = httpx.Client(
+            base_url=f"{host.rstrip('/')}/api/v1",
+            headers={
+                "Authorization": f"Bearer {token}",
+                "Accept": "application/json",
+                "User-Agent": f"spanlight-python/{__version__} experiments",
+            },
+            timeout=timeout,
+            transport=transport,
+        )
+
+    def close(self) -> None:
+        """Release the connection pool."""
+        self._client.close()
+
+    def project(self, path: str = "") -> str:
+        """A path under this project."""
+        return f"/projects/{self.project_id}{path}"
+
+    def request(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: dict[str, Any] | None = None,
+        headers: dict[str, str] | None = None,
+        body: dict[str, Any] | None = None,
+    ) -> Any:
+        """Send one request and return the decoded JSON body."""
+        try:
+            response = self._client.request(method, path, params=params, headers=headers, json=body)
+        except httpx.HTTPError as error:
+            raise ExperimentError(
+                f"{method} {path}: cannot reach the server ({type(error).__name__})"
+            ) from None
+        if response.is_success:
+            try:
+                return response.json()
+            except ValueError:
+                raise ExperimentError(f"{method} {path}: the answer is not JSON") from None
+        raise ExperimentError(f"{method} {path}: {_problem(response)}")
+
+    def pages(self, path: str, params: dict[str, Any] | None = None) -> Iterator[dict[str, Any]]:
+        """Every item of a keyset-paginated list, newest first."""
+        cursor: str | None = None
+        for _ in range(_MAX_PAGES):
+            query: dict[str, Any] = {**(params or {}), "limit": _PAGE_SIZE}
+            if cursor:
+                query["cursor"] = cursor
+            page = self.request("GET", path, params=query)
+            yield from page["items"]
+            cursor = page.get("next_cursor")
+            if not cursor:
+                return
+
+
+def _problem(response: httpx.Response) -> str:
+    """``401 UNAUTHENTICATED: detail`` from a problem document, else just the status."""
+    try:
+        body = response.json()
+        code, detail = body.get("code"), body.get("detail")
+    except (ValueError, AttributeError):
+        return str(response.status_code)
+    parts = [str(response.status_code)]
+    if isinstance(code, str):
+        parts.append(code)
+    text = " ".join(parts)
+    return f"{text}: {detail}" if isinstance(detail, str) and detail else text
+
+
+def find_by_name(api: Api, path: str, name: str) -> dict[str, Any] | None:
+    """The listed item called ``name``, or ``None``."""
+    return next((item for item in api.pages(path) if item.get("name") == name), None)
+
+
+def resolve_dataset(api: Api, name: str) -> dict[str, Any]:
+    """The dataset called ``name`` (the server filters by exact name)."""
+    for item in api.pages(api.project("/datasets"), {"name": name}):
+        if item.get("name") == name:
+            return item
+    raise ExperimentError(f"dataset {name!r} not found in the project")
+
+
+def resolve_prompt(api: Api, ref: PromptRef) -> ResolvedPrompt:
+    """Pin a prompt reference to one version: its label (``production`` by default) or number."""
+    base = api.project(f"/prompts/{wire_name(ref.name)}")
+    version = ref.version
+    if version is None:
+        label = ref.label or "production"
+        prompt = api.request("GET", base)
+        if label == "latest":
+            version = int(prompt["latest"]["version"])
+        else:
+            labels = {entry["label"]: int(entry["version"]) for entry in prompt["labels"]}
+            if label not in labels:
+                raise ExperimentError(f"prompt {ref.name!r} has no label {label!r}")
+            version = labels[label]
+    found = api.request("GET", f"{base}/versions/{version}")
+    return ResolvedPrompt(name=ref.name, version=version, version_id=found["id"])
+
+
+def resolve_evaluators(api: Api, names: list[str]) -> list[str]:
+    """Evaluator ids for ``names``, in order; an unknown name is an error."""
+    if not names:
+        return []
+    by_name = {item["name"]: item["id"] for item in api.pages(api.project("/evaluators"))}
+    missing = [name for name in names if name not in by_name]
+    if missing:
+        raise ExperimentError(f"evaluator not found in the project: {', '.join(missing)}")
+    return [by_name[name] for name in names]

spanlight/_experiments_parse.py

@@ -0,0 +1,68 @@
+"""Parsing the experiment runner's command-line values. Pure: no I/O."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from decimal import Decimal, InvalidOperation
+
+# Prompt names never contain `@` or `:`, so either one can only start a label or a version.
+_VERSION = re.compile(r"^[1-9][0-9]{0,9}$")
+_LABEL = re.compile(r"^[a-z0-9][a-z0-9_-]{0,31}$")
+_SCORE_NAME = re.compile(r"^[^\s=,]{1,100}$")
+
+
+@dataclass(frozen=True)
+class PromptRef:
+    """A prompt and the version to run: a label, a version number, or neither (``production``)."""
+
+    name: str
+    label: str | None
+    version: int | None
+
+
+def parse_fail_below(spec: str) -> dict[str, Decimal]:
+    """Parse ``relevance=0.8,format=1`` into ``{"relevance": 0.8, "format": 1}``.
+
+    An empty spec means no thresholds. Raises ``ValueError`` for a malformed pair, a repeated
+    score name or a value that is not a finite number.
+    """
+    thresholds: dict[str, Decimal] = {}
+    for pair in spec.split(","):
+        pair = pair.strip()
+        if not pair:
+            continue
+        name, separator, raw = pair.partition("=")
+        name, raw = name.strip(), raw.strip()
+        if not separator or not _SCORE_NAME.match(name):
+            raise ValueError(f"expected score=number, got {pair!r}")
+        if name in thresholds:
+            raise ValueError(f"score {name!r} appears twice")
+        try:
+            value = Decimal(raw)
+        except InvalidOperation:
+            raise ValueError(f"threshold for {name!r} is not a number: {raw!r}") from None
+        if not value.is_finite():
+            raise ValueError(f"threshold for {name!r} is not a finite number: {raw!r}")
+        thresholds[name] = value
+    return thresholds
+
+
+def parse_prompt_ref(ref: str) -> PromptRef:
+    """Parse ``support/answer@production``, ``support/answer:3`` or a bare ``support/answer``."""
+    ref = ref.strip()
+    if "@" in ref and ":" in ref:
+        raise ValueError(f"give a label or a version, not both: {ref!r}")
+    if "@" in ref:
+        name, _, label = ref.rpartition("@")
+        if not name or not _LABEL.match(label):
+            raise ValueError(f"not a prompt label reference: {ref!r}")
+        return PromptRef(name, label, None)
+    if ":" in ref:
+        name, _, number = ref.rpartition(":")
+        if not name or not _VERSION.match(number):
+            raise ValueError(f"not a prompt version reference: {ref!r}")
+        return PromptRef(name, None, int(number))
+    if not ref:
+        raise ValueError("the prompt reference is empty")
+    return PromptRef(ref, None, None)

spanlight/_experiments_report.py

@@ -0,0 +1,100 @@
+"""Scoring a finished run against its thresholds, and the Markdown report of it. Pure: no I/O."""
+
+from __future__ import annotations
+
+from decimal import Decimal
+from typing import Any
+
+_PLACES = Decimal("0.000001")
+_UNKNOWN = "—"
+
+
+def fmt(value: Decimal) -> str:
+    """A number without trailing zeros: ``0.75``, ``1``, ``10``."""
+    return format(value.quantize(_PLACES).normalize(), "f")
+
+
+def _mean(stat: Any) -> Decimal | None:
+    """The mean of a numeric score; ``None`` for a categorical score or a missing one."""
+    mean = stat.get("mean") if isinstance(stat, dict) else None
+    if isinstance(mean, bool) or not isinstance(mean, int | float):
+        return None
+    return Decimal(str(mean))
+
+
+def check_thresholds(totals: dict[str, Any], fail_below: dict[str, Decimal]) -> list[str]:
+    """One line per score whose mean is below its threshold or that the run did not produce.
+
+    A score nobody recorded (every item errored, or the evaluator was over budget) fails the
+    gate: an unknown value never counts as a pass.
+    """
+    scores = totals.get("scores") or {}
+    failures: list[str] = []
+    for name, threshold in fail_below.items():
+        mean = _mean(scores.get(name))
+        if mean is None:
+            failures.append(f"{name}: no numeric score recorded < {fmt(threshold)}")
+        elif mean < threshold:
+            failures.append(f"{name}: {fmt(mean)} < {fmt(threshold)}")
+    return failures
+
+
+def _score_rows(totals: dict[str, Any], fail_below: dict[str, Decimal]) -> list[str]:
+    scores = totals.get("scores") or {}
+    rows = ["| Score | Mean | Items | Threshold | Result |", "| --- | --- | --- | --- | --- |"]
+    for name in sorted({*scores, *fail_below}):
+        stat = scores.get(name)
+        mean = _mean(stat)
+        count = stat.get("count") if isinstance(stat, dict) else None
+        threshold = fail_below.get(name)
+        if threshold is None:
+            result = ""
+        elif mean is not None and mean >= threshold:
+            result = "pass"
+        else:
+            result = "FAIL"
+        rows.append(
+            f"| {name} | {fmt(mean) if mean is not None else _UNKNOWN} "
+            f"| {count if count is not None else _UNKNOWN} "
+            f"| {'>= ' + fmt(threshold) if threshold is not None else _UNKNOWN} | {result} |"
+        )
+    return rows
+
+
+def _number(value: Any) -> str:
+    return _UNKNOWN if value is None else str(value)
+
+
+def render_markdown(
+    *,
+    title: str,
+    status: str,
+    error: str | None,
+    url: str,
+    totals: dict[str, Any] | None,
+    fail_below: dict[str, Decimal],
+    failures: list[str],
+) -> str:
+    """The report printed to the log and appended to the GitHub step summary."""
+    verdict = "passed" if status == "done" and not failures else "failed"
+    lines = [
+        f"### Spanlight eval gate: {title}",
+        "",
+        f"Gate **{verdict}**. [Open the run]({url})",
+        "",
+    ]
+    if status != "done":
+        lines += [f"The run did not finish ({status}): {error or 'no reason given'}.", ""]
+    if totals is not None:
+        lines += [
+            f"Items {_number(totals.get('items'))}, errors {_number(totals.get('errors'))}, "
+            f"cost {_number(totals.get('cost_usd'))} USD, "
+            f"p50 {_number(totals.get('latency_p50_ms'))} ms, "
+            f"p95 {_number(totals.get('latency_p95_ms'))} ms.",
+            "",
+            *_score_rows(totals, fail_below),
+            "",
+        ]
+    if failures:
+        lines += ["Below threshold:", "", *[f"- {line}" for line in failures], ""]
+    return "\n".join(lines)

spanlight/_experiments_runs.py

@@ -0,0 +1,122 @@
+"""Starting a run and following it to its end, and how a run ends."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from decimal import Decimal
+from typing import Any
+
+from ._experiments_api import Api, ExperimentError
+from ._experiments_report import check_thresholds
+
+# Consecutive failed polls before the runner gives up: a deploy of the server must not fail a gate.
+_POLL_RETRIES = 3
+
+
+@dataclass(frozen=True)
+class ExperimentOutcome:
+    """How one run ended.
+
+    ``status`` is ``done``, ``failed`` or ``timeout``; ``totals`` is the run's totals from the
+    API (``None`` when it never produced any); ``failures`` lists the thresholds that were missed.
+    """
+
+    run_id: str
+    totals: dict[str, Any] | None
+    failures: list[str]
+    url: str
+    status: str = "done"
+    error: str | None = None
+    title: str = field(default="", compare=False)
+
+    @property
+    def passed(self) -> bool:
+        """Whether the run finished and every threshold held."""
+        return self.status == "done" and not self.failures
+
+
+def outcome_of(
+    final: dict[str, Any] | None,
+    run_id: str,
+    url: str,
+    title: str,
+    timeout_s: int,
+    fail_below: dict[str, Decimal],
+) -> ExperimentOutcome:
+    """Turn the last state of a run (``None``: still going at the deadline) into an outcome."""
+    if final is None:
+        return ExperimentOutcome(
+            run_id, None, [], url, "timeout", f"no result after {timeout_s} s", title
+        )
+    totals = final.get("totals")
+    if final["status"] == "failed":
+        return ExperimentOutcome(run_id, totals, [], url, "failed", final.get("error"), title)
+    failures = check_thresholds(totals or {}, fail_below)
+    return ExperimentOutcome(run_id, totals, failures, url, "done", None, title)
+
+
+def org_of(api: Api) -> str:
+    return str(api.request("GET", api.project())["org_id"])
+
+
+def start_run(
+    api: Api, experiment_id: str, version_id: str, env: Mapping[str, str]
+) -> dict[str, Any]:
+    """Queue a run. Inside GitHub Actions a retried request of one attempt starts only one run."""
+    headers: dict[str, str] = {}
+    run_id, attempt = env.get("GITHUB_RUN_ID"), env.get("GITHUB_RUN_ATTEMPT")
+    if run_id:
+        # The whole experiment id keeps two gate steps of one workflow run from sharing a key.
+        headers["Idempotency-Key"] = f"{run_id}-{attempt or '1'}-{experiment_id}"
+    path = api.project(f"/experiments/{experiment_id}/runs")
+    # The run names its prompt version, so a moved label runs the new version in this experiment.
+    body = {"target_override": {"prompt_version_id": version_id}}
+    started: dict[str, Any] = api.request("POST", path, headers=headers, body=body)
+    return started
+
+
+def wait_idle(
+    api: Api,
+    experiment_id: str,
+    deadline: float,
+    poll_s: float,
+    sleep: Callable[[float], None],
+    clock: Callable[[], float],
+) -> None:
+    """Wait until the experiment has no queued or running run: it allows only one at a time."""
+    path = api.project(f"/experiments/{experiment_id}/runs")
+    while True:
+        latest = api.request("GET", path, params={"limit": 1})["items"]
+        if not latest or latest[0]["status"] not in {"queued", "running"}:
+            return
+        if clock() >= deadline:
+            raise ExperimentError("another run of this experiment did not finish in time")
+        sleep(poll_s)
+
+
+def wait(
+    api: Api,
+    experiment_id: str,
+    run_id: str,
+    deadline: float,
+    poll_s: float,
+    sleep: Callable[[float], None],
+    clock: Callable[[], float],
+) -> dict[str, Any] | None:
+    """Poll until the run is ``done`` or ``failed``; ``None`` when the deadline passes first."""
+    path = api.project(f"/experiments/{experiment_id}/runs/{run_id}")
+    errors = 0
+    while True:
+        try:
+            run: dict[str, Any] = api.request("GET", path)
+            errors = 0
+            if run["status"] in {"done", "failed"}:
+                return run
+        except ExperimentError:
+            errors += 1
+            if errors >= _POLL_RETRIES:
+                raise
+        if clock() >= deadline:
+            return None
+        sleep(poll_s)

spanlight/_experiments_setup.py

@@ -0,0 +1,85 @@
+"""Find or create the experiment a CI run belongs to."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from decimal import Decimal
+from typing import Any
+
+from ._experiments_api import Api, ExperimentError, ResolvedPrompt, find_by_name
+
+# The server's limit on an experiment name.
+_MAX_NAME = 200
+
+
+def normalize_credential_id(value: str) -> str:
+    """The credential id in canonical lower-case form, so a pasted upper-case id still matches."""
+    try:
+        return str(uuid.UUID(value.strip()))
+    except ValueError:
+        raise ExperimentError("--credential-id is not a UUID") from None
+
+
+def experiment_name(
+    dataset: str, prompt: str, evaluator_names: list[str], credential_id: str, model: str
+) -> str:
+    """``ci/<dataset>/<prompt>/<8 hex>``; the digest follows the evaluators, credential and model.
+
+    An experiment cannot be edited, so a gate that changes one of these gets its own experiment
+    instead of silently running under the old settings.
+    """
+    settings = json.dumps(
+        [sorted(set(evaluator_names)), credential_id, model], separators=(",", ":")
+    )
+    digest = hashlib.sha256(settings.encode()).hexdigest()[:8]
+    name = f"ci/{dataset}/{prompt}/{digest}"
+    if len(name) > _MAX_NAME:
+        raise ExperimentError(
+            f"the experiment name would be {len(name)} characters; the limit is {_MAX_NAME}. "
+            "Use a shorter dataset or prompt name"
+        )
+    return name
+
+
+def experiment_for(
+    api: Api,
+    dataset: dict[str, Any],
+    version: ResolvedPrompt,
+    evaluator_names: list[str],
+    evaluator_ids: list[str],
+    *,
+    credential_id: str,
+    model: str,
+    max_cost_usd: Decimal | None,
+) -> dict[str, Any]:
+    """The experiment for this dataset, prompt and settings, created on first use.
+
+    Runs of one experiment stay comparable. It is created with the resolved version as its base
+    target; every run then names the version it wants through ``target_override``, so a label that
+    moved runs the new version. ``max_cost_usd`` applies only when the experiment is created.
+    """
+    name = experiment_name(dataset["name"], version.name, evaluator_names, credential_id, model)
+    path = api.project("/experiments")
+    existing = find_by_name(api, path, name)
+    if existing is not None:
+        return existing
+    config: dict[str, Any] = {
+        "target": {
+            "kind": "prompt_version",
+            "prompt_version_id": version.version_id,
+            "credential_id": credential_id,
+            "model": model,
+        }
+    }
+    if max_cost_usd is not None:
+        config["max_cost_usd"] = f"{max_cost_usd:f}"
+    body = {
+        "name": name,
+        "dataset_id": dataset["id"],
+        "config": config,
+        "evaluator_ids": evaluator_ids,
+    }
+    created: dict[str, Any] = api.request("POST", path, body=body)
+    return created