annoscript 0.1.0__py3-none-any.whl

annoscript/job_protocol.py

@@ -0,0 +1,180 @@
+"""Python 3.10 job wire contracts. No transport or execution dependencies."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import math
+from email.utils import parsedate_to_datetime
+
+from annoscript._data import snapshot
+from annoscript.remote import RemoteProtocolError, _json, ir_sha256
+
+PROTOCOL_VERSION = 1
+TERMINAL = frozenset(("completed", "cancelled", "failed"))
+STATES = TERMINAL | {"queued", "running", "paused"}
+
+
+def encode(value, *, canonical=False):
+    return json.dumps(value, ensure_ascii=False, allow_nan=False, sort_keys=canonical,
+                      separators=(",", ":")).encode("utf-8")
+
+
+def decode(value):
+    if isinstance(value, (bytes, bytearray)):
+        try:
+            value = value.decode("utf-8")
+        except UnicodeError:
+            raise RemoteProtocolError("JSON must be UTF-8") from None
+    return _json(value)
+
+
+def nonempty(value):
+    return type(value) is str and bool(value.strip())
+
+
+def integer(value, minimum=0):
+    return type(value) is int and value >= minimum
+
+
+def validate_request(value):
+    value = snapshot(value)
+    required = {"protocol_version", "ir", "ir_sha256", "mode", "inputs"}
+    optional = {"release_id", "application_revision", "session_id"}
+    if type(value) is not dict or not required <= value.keys() or value.keys() - required - optional:
+        raise ValueError("invalid job request fields")
+    if type(value["protocol_version"]) is not int or value["protocol_version"] != 1:
+        raise ValueError("unsupported job protocol version")
+    if type(value["ir"]) is not dict or ir_sha256(value["ir"]) != value["ir_sha256"]:
+        raise ValueError("IR digest mismatch")
+    if value["mode"] not in ("eager", "strict", "speculative"):
+        raise ValueError("invalid execution mode")
+    if type(value["inputs"]) is not list or not value["inputs"]:
+        raise ValueError("a job needs at least one input")
+    indices = set()
+    for item in value["inputs"]:
+        if (type(item) is not dict or set(item) != {"index", "state"}
+                or not integer(item["index"]) or item["index"] in indices):
+            raise ValueError("inputs require unique nonnegative indices and states")
+        snapshot(item["state"], state=True)
+        indices.add(item["index"])
+    for name in optional & value.keys():
+        if not nonempty(value[name]):
+            raise ValueError(f"{name} must be a nonempty string")
+    return value
+
+
+def request_sha256(request):
+    # Normalize protocol field order and IR encoding, but NEVER sort state keys:
+    # an IR program can iterate over the keys of an input object.
+    value = {key: request[key] for key in ("protocol_version", "ir_sha256", "mode")}
+    value["inputs"] = [{"index": item["index"], "state": item["state"]}
+                       for item in request["inputs"]]
+    for key in ("release_id", "application_revision", "session_id"):
+        if key in request:
+            value[key] = request[key]
+    return hashlib.sha256(encode(value)).hexdigest()
+
+
+def cursor_for(job_id, seq):
+    return base64.urlsafe_b64encode(encode([1, job_id, seq])).decode("ascii").rstrip("=")
+
+
+def cursor_seq(cursor, job_id):
+    if cursor is None:
+        return 0
+    try:
+        raw = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True)
+        value = decode(raw)
+        if (type(value) is not list or len(value) != 3 or type(value[0]) is not int
+                or value[0] != 1 or value[1] != job_id or not integer(value[2])
+                or cursor_for(job_id, value[2]) != cursor):
+            raise ValueError()
+        return value[2]
+    except (ValueError, TypeError, UnicodeError, RemoteProtocolError):
+        raise ValueError("invalid cursor for this job") from None
+
+
+def retry_after_seconds(value, now):
+    if value is None:
+        return 0.0
+    try:
+        seconds = float(value)
+        if math.isfinite(seconds):
+            return max(0.0, seconds)
+    except (ValueError, TypeError):
+        pass
+    try:
+        return max(0.0, parsedate_to_datetime(value).timestamp() - now)
+    except (ValueError, TypeError, OverflowError):
+        return 0.0
+
+
+def validate_status(job):
+    from annoscript.remote import _error
+    fields = {"job_id", "ir_sha256", "release_id", "mode", "input_indices", "state", "counts",
+              "pause_generation", "pause_reason", "created_at", "updated_at"}
+    if type(job) is not dict or set(job) != fields:
+        raise RemoteProtocolError("invalid job status fields")
+    if (not all(nonempty(job[k]) for k in ("job_id", "ir_sha256", "release_id"))
+            or len(job["ir_sha256"]) != 64 or any(c not in "0123456789abcdef" for c in job["ir_sha256"])
+            or job["mode"] not in ("strict", "eager", "speculative")
+            or job["state"] not in STATES or not integer(job["pause_generation"])):
+        raise RemoteProtocolError("invalid job status")
+    indices, counts = job["input_indices"], job["counts"]
+    if (type(indices) is not list or not indices or any(not integer(i) for i in indices)
+            or len(set(indices)) != len(indices) or type(counts) is not dict
+            or set(counts) != {"total", "completed", "success", "error"}
+            or any(not integer(n) for n in counts.values())
+            or counts["total"] != len(indices)
+            or counts["completed"] != counts["success"] + counts["error"]
+            or counts["completed"] > counts["total"]
+            or ((job["state"] in TERMINAL) != (counts["completed"] == counts["total"]))):
+        raise RemoteProtocolError("invalid job indices or counts")
+    if job["pause_reason"] is not None:
+        _error(job["pause_reason"])
+    if (job["state"] == "paused") != (job["pause_reason"] is not None):
+        raise RemoteProtocolError("inconsistent paused job")
+    if any(type(job[k]) not in (int, float) or not math.isfinite(job[k])
+           for k in ("created_at", "updated_at")):
+        raise RemoteProtocolError("invalid job timestamps")
+    return job
+
+
+def validate_page(page, job_id, cursor=None):
+    from annoscript._client_common import Result
+    from annoscript.remote import _error
+    if (type(page) is not dict
+            or set(page) != {"protocol_version", "job", "results", "next_cursor", "has_more", "done"}
+            or type(page["protocol_version"]) is not int or page["protocol_version"] != 1
+            or type(page["results"]) is not list
+            or type(page["has_more"]) is not bool or type(page["done"]) is not bool):
+        raise RemoteProtocolError("invalid result page")
+    job = validate_status(page["job"])
+    if job["job_id"] != job_id:
+        raise RemoteProtocolError("page belongs to another job")
+    try:
+        last = cursor_seq(cursor, job_id)
+        next_seq = cursor_seq(page["next_cursor"], job_id)
+    except ValueError as exc:
+        raise RemoteProtocolError(str(exc)) from None
+    seen, expected = set(), set(job["input_indices"])
+    for item in page["results"]:
+        if (type(item) is not dict or set(item) != {"seq", "index", "status", "value", "error", "attempts"}
+                or not integer(item["seq"], 1) or item["seq"] != last + 1
+                or not integer(item["index"]) or item["index"] not in expected or item["index"] in seen
+                or not integer(item["attempts"])):
+            raise RemoteProtocolError("invalid, missing or duplicate page result")
+        try:
+            Result(item["index"], item["status"], item["value"],
+                   None if item["error"] is None else _error(item["error"]))
+        except ValueError as exc:
+            raise RemoteProtocolError(str(exc)) from None
+        last = item["seq"]
+        seen.add(item["index"])
+    if (next_seq != last or last > job["counts"]["completed"]
+            or page["has_more"] != (last < job["counts"]["completed"])
+            or page["done"] != (job["state"] in TERMINAL and not page["has_more"])):
+        raise RemoteProtocolError("inconsistent result cursor or completion")
+    return page

annoscript/jobs.py

@@ -0,0 +1,255 @@
+"""Python 3.10+ durable job HTTP client. Closing it does not cancel durable jobs."""
+
+from __future__ import annotations
+
+import math
+import hashlib
+import random
+import threading
+import time
+from urllib.parse import quote
+
+from annoscript.job_protocol import (decode, encode, nonempty, retry_after_seconds,
+                              validate_page, validate_request, validate_status)
+from annoscript.remote import (RemoteProtocolError, RemoteSubmissionError,
+                        RemoteTransportError, ir_sha256, _error)
+
+
+class RemoteJobPausedError(RemoteSubmissionError):
+    def __init__(self, job):
+        reason = job["pause_reason"]
+        super().__init__(reason["code"], reason["message"])
+        self.job = job
+
+
+class RemoteCancellationError(RemoteTransportError):
+    """Cancellation was requested but could not be confirmed before closing."""
+
+
+def job_id(job):
+    value = job.get("job_id") if type(job) is dict else job
+    if not nonempty(value):
+        raise ValueError("job must be a job status or nonempty job ID")
+    return value
+
+
+class AnnoJobClient:
+    def __init__(self, base_url, auth_token=None, *, timeout=30.0, transport=None,
+                 _clock=time.monotonic, _wall_clock=time.time, _sleep=time.sleep,
+                 _random=random.random):
+        import httpx
+        url = httpx.URL(base_url)
+        if url.scheme not in ("http", "https") or not url.host or url.userinfo or url.query or url.fragment:
+            raise ValueError("base_url must be an HTTP(S) URL without credentials, query or fragment")
+        if auth_token is not None and (not nonempty(auth_token) or "\r" in auth_token or "\n" in auth_token):
+            raise ValueError("auth_token must be a nonempty single-line string")
+        if type(timeout) not in (int, float) or not math.isfinite(timeout) or timeout <= 0:
+            raise ValueError("timeout must be finite and positive")
+        self.base_url, self.timeout = str(url).rstrip("/"), timeout
+        self.clock, self.wall_clock, self.sleep, self.random = _clock, _wall_clock, _sleep, _random
+        headers = {"Accept": "application/json", "Content-Type": "application/json"}
+        if auth_token is not None:
+            headers["Authorization"] = auth_token if auth_token.lower().startswith("bearer ") else "Bearer " + auth_token
+        self._http = httpx.Client(headers=headers, timeout=timeout, follow_redirects=False,
+                                  transport=transport if transport is not None else httpx.HTTPTransport(retries=0))
+        self.closed = False
+
+    def __enter__(self):
+        if self.closed:
+            raise RuntimeError("client is closed")
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+    def close(self):
+        if not self.closed:
+            self.closed = True
+            self._http.close()
+
+    def _request(self, method, path, *, payload=None, key=None, params=None,
+                 budget=120.0, attempts=5):
+        # httpx timeouts are per network phase; DNS or a slowly progressing
+        # response must not turn a bounded reconnect budget into an endless wait.
+        # A late request can still reach the server, so all mutations are either
+        # idempotent or guarded by a closed ownership session.
+        outcome, done = [], threading.Event()
+        def run():
+            try:
+                outcome.append((True, self._request_with_retries(method, path, payload=payload,
+                    key=key, params=params, budget=budget, attempts=attempts)))
+            except BaseException as exc:
+                outcome.append((False, exc))
+            finally:
+                done.set()
+        threading.Thread(target=run, name="anno-job-http", daemon=True).start()
+        if not done.wait(max(0.0, budget)):
+            raise RemoteTransportError("remote reconnect deadline exceeded")
+        success, value = outcome[0]
+        if not success:
+            raise value
+        return value
+
+    def _request_with_retries(self, method, path, *, payload=None, key=None, params=None,
+                              budget=120.0, attempts=5):
+        import httpx
+        if self.closed:
+            raise RuntimeError("client is closed")
+        if key is not None and (not nonempty(key) or len(key) > 256 or "\r" in key or "\n" in key):
+            raise ValueError("invalid idempotency key")
+        content = encode(payload) if payload is not None else None
+        deadline = self.clock() + budget
+        for attempt in range(attempts):
+            remaining = deadline - self.clock()
+            if remaining <= 0:
+                raise RemoteTransportError("remote reconnect budget exhausted")
+            retry_after = None
+            try:
+                with self._http.stream(method, self.base_url + path, content=content, params=params,
+                                       headers={"Idempotency-Key": key} if key is not None else None,
+                                       timeout=min(self.timeout, remaining)) as response:
+                    chunks = []
+                    for chunk in response.iter_bytes():
+                        if self.clock() > deadline:
+                            raise httpx.ReadTimeout("request deadline exceeded")
+                        chunks.append(chunk)
+                    raw = b"".join(chunks)
+                    if not 200 <= response.status_code < 300:
+                        code, message = "http_error", f"remote HTTP error {response.status_code}"
+                        try:
+                            error = _error(decode(raw)["error"])
+                            code, message = error.code, error.message
+                        except (RemoteProtocolError, KeyError, TypeError):
+                            pass
+                        error = RemoteSubmissionError(code, message, http_status=response.status_code,
+                                                      retry_after=response.headers.get("Retry-After"))
+                        if response.status_code not in (408, 429) and not 500 <= response.status_code <= 599:
+                            raise error
+                        failure, retry_after = error, error.retry_after
+                    else:
+                        if response.headers.get("Content-Type", "").split(";", 1)[0].lower() != "application/json":
+                            raise RemoteProtocolError("expected an application/json response")
+                        value = decode(raw)
+                        if type(value) is not dict or type(value.get("protocol_version")) is not int or value["protocol_version"] != 1:
+                            raise RemoteProtocolError("unsupported job protocol version")
+                        return value
+            except httpx.RequestError as exc:
+                failure = RemoteTransportError("remote HTTP request interrupted")
+                failure.__cause__ = exc
+            if attempt + 1 == attempts:
+                raise failure
+            delay = max(self.random() * min(10.0, 0.5 * 2 ** attempt),
+                        retry_after_seconds(retry_after, self.wall_clock()))
+            if self.clock() + delay >= deadline:
+                raise failure
+            self.sleep(delay)
+        raise RemoteTransportError("remote reconnect budget exhausted")
+
+    @staticmethod
+    def _status(response, expected_id=None):
+        if set(response) != {"protocol_version", "job"}:
+            raise RemoteProtocolError("invalid job envelope")
+        job = validate_status(response["job"])
+        if expected_id is not None and job["job_id"] != expected_id:
+            raise RemoteProtocolError("response belongs to another job")
+        return job
+
+    @staticmethod
+    def _binding(expected, actual):
+        if type(expected) is dict:
+            for field in ("job_id", "ir_sha256", "release_id", "mode", "input_indices"):
+                if expected[field] != actual[field]:
+                    raise RemoteProtocolError("immutable job binding changed")
+        return actual
+
+    def submit_ir(self, ir, inputs, *, idempotency_key, mode="eager", release_id=None,
+                  application_revision=None, session_id=None):
+        ir = ir.to_dict() if hasattr(ir, "to_dict") else ir
+        payload = {"protocol_version": 1, "ir": ir, "ir_sha256": ir_sha256(ir), "mode": mode, "inputs": inputs}
+        for name, value in (("release_id", release_id), ("application_revision", application_revision), ("session_id", session_id)):
+            if value is not None:
+                payload[name] = value
+        payload = validate_request(payload)
+        job = self._status(self._request("POST", "/v1/jobs", payload=payload, key=idempotency_key))
+        if (job["ir_sha256"] != payload["ir_sha256"] or job["mode"] != mode
+                or job["input_indices"] != [item["index"] for item in payload["inputs"]]
+                or (release_id is not None and job["release_id"] != release_id)):
+            raise RemoteProtocolError("accepted job does not match submitted IR, inputs or release")
+        return job
+
+    def lookup(self, idempotency_key, **options):
+        return self._status(self._request("GET", "/v1/jobs:lookup", key=idempotency_key, **options))
+
+    def release(self, release_id):
+        """Read an immutable release without importing server implementation."""
+        if not nonempty(release_id):
+            raise ValueError("release_id must be nonempty")
+        response = self._request("GET", f"/v1/releases/{quote(release_id, safe='')}")
+        if (set(response) != {"protocol_version", "release_id", "manifest", "available"}
+                or response["release_id"] != release_id or type(response["available"]) is not bool
+                or type(response["manifest"]) is not dict
+                or hashlib.sha256(encode(response["manifest"], canonical=True)).hexdigest() != release_id):
+            raise RemoteProtocolError("invalid release manifest or digest")
+        return response
+
+    def status(self, job):
+        jid = job_id(job)
+        return self._binding(job, self._status(self._request("GET", f"/v1/jobs/{quote(jid, safe='')}"), jid))
+
+    def fetch_results(self, job, *, cursor=None, limit=100):
+        jid = job_id(job)
+        params = {"limit": limit}
+        if cursor is not None:
+            params["cursor"] = cursor
+        page = validate_page(self._request("GET", f"/v1/jobs/{quote(jid, safe='')}/results", params=params), jid, cursor)
+        self._binding(job, page["job"])
+        return page
+
+    def metrics(self, job):
+        """Read a consistent metric snapshot without loading server modules."""
+        from annoscript.metrics import validate_metrics
+        jid = job_id(job)
+        value = validate_metrics(self._request("GET", f"/v1/jobs/{quote(jid, safe='')}/metrics"), jid)
+        self._binding(job, value["job"])
+        return value
+
+    def resume(self, job, *, pause_generation, idempotency_key):
+        jid = job_id(job)
+        return self._binding(job, self._status(self._request("POST", f"/v1/jobs/{quote(jid, safe='')}:resume",
+                                          payload={"pause_generation": pause_generation}, key=idempotency_key), jid))
+
+    def cancel(self, job, **options):
+        jid = job_id(job)
+        result = self._status(self._request("POST", f"/v1/jobs/{quote(jid, safe='')}:cancel", payload={}, **options), jid)
+        self._binding(job, result)
+        if result["state"] not in ("completed", "cancelled", "failed"):
+            raise RemoteProtocolError("cancellation was not persisted")
+        return result
+
+    def _session(self, response):
+        if (set(response) != {"protocol_version", "session_id", "state", "expires_at", "heartbeat_interval"}
+                or not nonempty(response["session_id"]) or response["state"] not in ("active", "closed", "expired")
+                or any(type(response[k]) not in (int, float) or not math.isfinite(response[k]) or response[k] <= 0
+                       for k in ("expires_at", "heartbeat_interval"))):
+            raise RemoteProtocolError("invalid session response")
+        return response
+
+    def _create_session(self, key):
+        response = self._session(self._request("POST", "/v1/sessions", payload={}, key=key))
+        if response["state"] != "active":
+            raise RemoteSubmissionError("session_closed", "client session has ended")
+        return response
+
+    def _heartbeat(self, session_id):
+        response = self._session(self._request("POST", f"/v1/sessions/{quote(session_id, safe='')}:heartbeat",
+                                               payload={}, attempts=1, budget=30))
+        if response["session_id"] != session_id or response["state"] != "active":
+            raise RemoteProtocolError("invalid session heartbeat")
+        return response
+
+    def _close_session(self, session_id, budget):
+        response = self._session(self._request("POST", f"/v1/sessions/{quote(session_id, safe='')}:close",
+                                               payload={}, attempts=1, budget=budget))
+        if response["session_id"] != session_id or response["state"] == "active":
+            raise RemoteProtocolError("session cancellation was not persisted")
+        return response

annoscript/metrics.py

@@ -0,0 +1,153 @@
+"""Version 1 metric values and validation (stdlib only, Python 3.10+).
+
+Observed subtotals are never substituted for unknown totals. Token details such
+as reasoning/cached tokens are subsets, not additional billable tokens.
+"""
+
+from collections import Counter
+import math
+
+from annoscript.job_protocol import TERMINAL, integer, validate_status
+from annoscript.remote import RemoteProtocolError
+
+
+def normalize_usage(usage):
+    result = {}
+    for field, aliases in (("input_tokens", ("input_tokens", "prompt_tokens")),
+                           ("output_tokens", ("output_tokens", "completion_tokens"))):
+        present = [key for key in aliases if type(usage) is dict and key in usage]
+        values = [usage[key] for key in present]
+        reason = ("missing" if not present else "invalid" if not all(integer(v) for v in values)
+                  else "conflicting" if len(set(values)) != 1 else None)
+        result[field] = {"value": values[0] if reason is None else None,
+                         "sources": present, "reason": reason}
+    return result
+
+
+def quantile(values, fraction):
+    return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)] if values else None
+
+
+def measure(values, *, history=True, duration=False):
+    known = [value for value in values if value is not None]
+    result = {"observed": sum(known), "total": sum(known) if history and len(known) == len(values) else None,
+              "known": len(known), "unknown": len(values) - len(known)}
+    if duration:
+        result.update(p50=quantile(known, .5), p95=quantile(known, .95))
+    return result
+
+
+def aggregate(attempts, requests, *, history, waits):
+    """Aggregate sanitized persisted records; never distribute batch usage by question."""
+    history = history and all(a["metered"] for a in attempts)
+    result = {
+        "history_complete": bool(history),
+        "attempts": {"count": len(attempts), "statuses": dict(sorted(Counter(a["status"] for a in attempts).items())),
+                     "seconds": measure([a["duration_seconds"] for a in attempts], history=history, duration=True)},
+        "queue_wait_seconds": measure(waits, history=history, duration=True),
+        "backends": {},
+    }
+    for backend in ("decision", "generation"):
+        rows = [r for r in requests if r["backend"] == backend]
+        unknown = sum(r["http_status"] is None for r in rows)
+        item = {"requests": len(rows), "confirmed_requests": len(rows) - unknown,
+                "unknown_requests": unknown, "total_requests": len(rows) if history and not unknown else None,
+                "questions": sum(r["questions"] for r in rows),
+                "confirmed_questions": sum(r["questions"] for r in rows if r["http_status"] is not None),
+                "total_questions": sum(r["questions"] for r in rows) if history and not unknown else None,
+                "http_seconds": measure([r["seconds"] for r in rows], history=history, duration=True)}
+        for field in ("input_tokens", "output_tokens"):
+            tokens = [r["usage"][field] for r in rows]
+            item[field] = {**measure([t["value"] for t in tokens], history=history),
+                           "sources": sorted({s for t in tokens for s in t["sources"]}),
+                           "reasons": dict(sorted(Counter(t["reason"] for t in tokens if t["reason"]).items()))}
+        result["backends"][backend] = item
+    return result
+
+
+def validate_metrics(value, job_id):
+    """Reject all malformed JSON shapes through the public protocol error type."""
+    try:
+        return _validate_metrics(value, job_id)
+    except RemoteProtocolError:
+        raise
+    except (KeyError, TypeError, ValueError, OverflowError, RecursionError):
+        raise RemoteProtocolError("invalid job metrics") from None
+
+
+def _validate_metrics(value, job_id):
+    def require(ok):
+        if not ok:
+            raise RemoteProtocolError("invalid job metrics")
+
+    def number(v):
+        return type(v) in (int, float) and 0 <= v < float("inf")
+
+    def metric(m, *, duration=False, tokens=False):
+        keys = {"observed", "total", "known", "unknown"}
+        if duration:
+            keys |= {"p50", "p95"}
+        if tokens:
+            keys |= {"sources", "reasons"}
+        require(type(m) is dict and set(m) == keys)
+        require(integer(m["known"]) and integer(m["unknown"]) and number(m["observed"]))
+        require(m["total"] is None or (number(m["total"]) and m["total"] == m["observed"] and not m["unknown"]))
+        require(m["known"] != 0 or m["observed"] == 0)
+        if tokens:
+            require(integer(m["observed"]) and (m["total"] is None or integer(m["total"])))
+            require(type(m["sources"]) is list and all(s in ("input_tokens", "prompt_tokens", "output_tokens", "completion_tokens") for s in m["sources"]))
+            require(len(set(m["sources"])) == len(m["sources"]))
+            require(type(m["reasons"]) is dict and all(k in ("missing", "invalid", "conflicting") and integer(v, 1) for k, v in m["reasons"].items()))
+            require(sum(m["reasons"].values()) == m["unknown"])
+        if duration:
+            require(all((m[k] is None if not m["known"] else number(m[k])) for k in ("p50", "p95")))
+            require(not m["known"] or m["p50"] <= m["p95"] <= m["observed"] + 1e-9)
+
+    def summary(s):
+        require(type(s) is dict and set(s) == {"history_complete", "attempts", "queue_wait_seconds", "backends"})
+        require(type(s["history_complete"]) is bool)
+        a = s["attempts"]
+        require(type(a) is dict and set(a) == {"count", "statuses", "seconds"} and integer(a["count"]))
+        require(type(a["statuses"]) is dict and all(k in ("running", "finished", "failed", "cancelled", "abandoned") and integer(v, 1) for k, v in a["statuses"].items()))
+        require(sum(a["statuses"].values()) == a["count"])
+        metric(a["seconds"], duration=True)
+        require(a["seconds"]["known"] + a["seconds"]["unknown"] == a["count"])
+        metric(s["queue_wait_seconds"], duration=True)
+        require(s["history_complete"] or (a["seconds"]["total"] is None and s["queue_wait_seconds"]["total"] is None))
+        require(type(s["backends"]) is dict and set(s["backends"]) == {"decision", "generation"})
+        for b in s["backends"].values():
+            require(type(b) is dict and set(b) == {"requests", "confirmed_requests", "unknown_requests", "total_requests", "questions", "confirmed_questions", "total_questions", "http_seconds", "input_tokens", "output_tokens"})
+            require(all(integer(b[k]) for k in ("requests", "confirmed_requests", "unknown_requests", "questions", "confirmed_questions")))
+            require(b["confirmed_requests"] + b["unknown_requests"] == b["requests"] and b["confirmed_questions"] <= b["questions"])
+            complete = s["history_complete"] and not b["unknown_requests"]
+            require(all(b[k] is None or integer(b[k]) for k in ("total_requests", "total_questions")))
+            require(b["total_requests"] == (b["requests"] if complete else None))
+            require(b["total_questions"] == (b["questions"] if complete else None))
+            for key in ("http_seconds", "input_tokens", "output_tokens"):
+                metric(b[key], duration=key == "http_seconds", tokens=key != "http_seconds")
+                require(b[key]["known"] + b[key]["unknown"] == b["requests"])
+                require(s["history_complete"] or b[key]["total"] is None)
+        return s
+
+    require(type(value) is dict and set(value) == {"protocol_version", "metrics_version", "job", "provisional", "summary", "inputs"})
+    require(type(value["protocol_version"]) is int and value["protocol_version"] == 1)
+    require(type(value["metrics_version"]) is int and value["metrics_version"] == 1)
+    job = validate_status(value["job"])
+    require(job["job_id"] == job_id and type(value["provisional"]) is bool and value["provisional"] == (job["state"] not in TERMINAL))
+    summary(value["summary"])
+    require(type(value["inputs"]) is list and len(value["inputs"]) == len(job["input_indices"]))
+    indices = []
+    for item in value["inputs"]:
+        require(type(item) is dict and set(item) == {"index", "metrics"} and integer(item["index"]))
+        indices.append(item["index"])
+        summary(item["metrics"])
+    require(indices == job["input_indices"])
+    require(value["summary"]["attempts"]["count"] == sum(i["metrics"]["attempts"]["count"] for i in value["inputs"]))
+    for backend in ("decision", "generation"):
+        for k in ("requests", "questions", "unknown_requests", "confirmed_requests", "confirmed_questions"):
+            require(value["summary"]["backends"][backend][k] == sum(i["metrics"]["backends"][backend][k] for i in value["inputs"]))
+        for k in ("input_tokens", "output_tokens", "http_seconds"):
+            observed = sum(i["metrics"]["backends"][backend][k]["observed"] for i in value["inputs"])
+            actual = value["summary"]["backends"][backend][k]["observed"]
+            require(math.isclose(actual, observed, rel_tol=1e-12, abs_tol=1e-9) if k == "http_seconds" else actual == observed)
+    return value