assay-evals 0.1.0__tar.gz

assay_evals-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 tap222
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

assay_evals-0.1.0/PKG-INFO

@@ -0,0 +1,82 @@
+Metadata-Version: 2.4
+Name: assay-evals
+Version: 0.1.0
+Summary: Record what your AI system does, and how it went, in Assay: runs, agent steps, feedback and test results
+Author: tap222
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/tap222/docai-eval
+Project-URL: Documentation, https://github.com/tap222/docai-eval/tree/main/sdk/python#readme
+Project-URL: Event schema, https://github.com/tap222/docai-eval/blob/main/docs/event-schema.md
+Project-URL: Issues, https://github.com/tap222/docai-eval/issues
+Keywords: llm,agents,evaluation,observability,tracing,ai
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# assay-evals
+
+Record what your AI system does, and how it went, in
+[Assay](https://github.com/tap222/docai-eval): runs, agent steps, user feedback and test
+results. Standard library only, Python 3.9+.
+
+```bash
+pip install assay-evals
+```
+
+You need an Assay server to send to. See the
+[setup guide](https://github.com/tap222/docai-eval#setup-guide).
+
+```python
+import assay_sdk as assay
+
+assay.init("https://assay.example.com", key="ak_...")   # or set ASSAY_URL / ASSAY_KEY
+
+# An agent
+with assay.run("refund_request", input=message, version={"prompt": "support@v5", "model": "claude-sonnet-5"}) as run:
+    run.llm(model="claude-sonnet-5", tokens_in=620, tokens_out=180, cost_usd=0.0024)
+    order = run.call("get_order", get_order, order_id="O-17")    # runs it; records the result or the error
+    run.state("refund:O-17", "create", {"amount": order["price"]})
+    run.answer(f"Refunded ${order['price']}.")
+
+# A pipeline
+with assay.run("invoice", kind="pipeline", input_ref="s3://inbox/inv-9.pdf") as run:
+    with run.stage("extract", prompt="extract_fields@v13") as s:
+        run.llm(model="claude-sonnet-5", cost_usd=0.01)          # nested under the stage
+        s.outputs.update(fields)
+
+# Outcomes, whenever they're known
+assay.feedback(run.id, "thumbs_down")
+assay.correction(run.id, "total", expected="1240.00", observed="1204.00")
+assay.check("nightly-0924", "case-17", "fail", run_id=run.id, field="total", expected="1240.00", actual="1204.00")
+assay.expect("case-17", calls=[{"tool": "get_order", "args": {"order_id": "O-17"}}], answer="27.61")
+```
+
+| Call | Records |
+|---|---|
+| `assay.run(task, kind="agent"\|"pipeline", input=, version=, test={"run", "case", "attempt"})` | one run; an exception ends it as failed |
+| `run.llm(...)`, `run.tool(name, args, result)`, `run.call(name, fn, **args)`, `run.state(obj, op, value)`, `run.answer(text)`, `with run.stage(name) as s` | its steps, in order |
+| `assay.feedback`, `assay.check`, `assay.correction`, `assay.expect` | outcomes, sent whenever they're known |
+| `assay.flush()` | send now (short-lived scripts); also happens every second and at exit |
+
+These options go to `init()`:
+- `redact`: a function applied to inputs, arguments, results, text and outputs before they
+  leave the process.
+- `sample=0.1`: record one run in ten. A run is recorded whole or not at all, and outcomes
+  are always sent.
+- `strict=True`: raise send errors while developing. Otherwise the SDK never raises into
+  your code.
+- `enabled=False`: the SDK does nothing, e.g. in unit tests.
+
+Events follow the
+[Assay event schema v1](https://github.com/tap222/docai-eval/blob/main/docs/event-schema.md). They stream to
+`POST /v1/ingest` in the background, so a run that crashes still shows every step up to
+the crash.

assay_evals-0.1.0/README.md

@@ -0,0 +1,58 @@
+# assay-evals
+
+Record what your AI system does, and how it went, in
+[Assay](https://github.com/tap222/docai-eval): runs, agent steps, user feedback and test
+results. Standard library only, Python 3.9+.
+
+```bash
+pip install assay-evals
+```
+
+You need an Assay server to send to. See the
+[setup guide](https://github.com/tap222/docai-eval#setup-guide).
+
+```python
+import assay_sdk as assay
+
+assay.init("https://assay.example.com", key="ak_...")   # or set ASSAY_URL / ASSAY_KEY
+
+# An agent
+with assay.run("refund_request", input=message, version={"prompt": "support@v5", "model": "claude-sonnet-5"}) as run:
+    run.llm(model="claude-sonnet-5", tokens_in=620, tokens_out=180, cost_usd=0.0024)
+    order = run.call("get_order", get_order, order_id="O-17")    # runs it; records the result or the error
+    run.state("refund:O-17", "create", {"amount": order["price"]})
+    run.answer(f"Refunded ${order['price']}.")
+
+# A pipeline
+with assay.run("invoice", kind="pipeline", input_ref="s3://inbox/inv-9.pdf") as run:
+    with run.stage("extract", prompt="extract_fields@v13") as s:
+        run.llm(model="claude-sonnet-5", cost_usd=0.01)          # nested under the stage
+        s.outputs.update(fields)
+
+# Outcomes, whenever they're known
+assay.feedback(run.id, "thumbs_down")
+assay.correction(run.id, "total", expected="1240.00", observed="1204.00")
+assay.check("nightly-0924", "case-17", "fail", run_id=run.id, field="total", expected="1240.00", actual="1204.00")
+assay.expect("case-17", calls=[{"tool": "get_order", "args": {"order_id": "O-17"}}], answer="27.61")
+```
+
+| Call | Records |
+|---|---|
+| `assay.run(task, kind="agent"\|"pipeline", input=, version=, test={"run", "case", "attempt"})` | one run; an exception ends it as failed |
+| `run.llm(...)`, `run.tool(name, args, result)`, `run.call(name, fn, **args)`, `run.state(obj, op, value)`, `run.answer(text)`, `with run.stage(name) as s` | its steps, in order |
+| `assay.feedback`, `assay.check`, `assay.correction`, `assay.expect` | outcomes, sent whenever they're known |
+| `assay.flush()` | send now (short-lived scripts); also happens every second and at exit |
+
+These options go to `init()`:
+- `redact`: a function applied to inputs, arguments, results, text and outputs before they
+  leave the process.
+- `sample=0.1`: record one run in ten. A run is recorded whole or not at all, and outcomes
+  are always sent.
+- `strict=True`: raise send errors while developing. Otherwise the SDK never raises into
+  your code.
+- `enabled=False`: the SDK does nothing, e.g. in unit tests.
+
+Events follow the
+[Assay event schema v1](https://github.com/tap222/docai-eval/blob/main/docs/event-schema.md). They stream to
+`POST /v1/ingest` in the background, so a run that crashes still shows every step up to
+the crash.

assay_evals-0.1.0/assay_evals.egg-info/PKG-INFO

@@ -0,0 +1,82 @@
+Metadata-Version: 2.4
+Name: assay-evals
+Version: 0.1.0
+Summary: Record what your AI system does, and how it went, in Assay: runs, agent steps, feedback and test results
+Author: tap222
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/tap222/docai-eval
+Project-URL: Documentation, https://github.com/tap222/docai-eval/tree/main/sdk/python#readme
+Project-URL: Event schema, https://github.com/tap222/docai-eval/blob/main/docs/event-schema.md
+Project-URL: Issues, https://github.com/tap222/docai-eval/issues
+Keywords: llm,agents,evaluation,observability,tracing,ai
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# assay-evals
+
+Record what your AI system does, and how it went, in
+[Assay](https://github.com/tap222/docai-eval): runs, agent steps, user feedback and test
+results. Standard library only, Python 3.9+.
+
+```bash
+pip install assay-evals
+```
+
+You need an Assay server to send to. See the
+[setup guide](https://github.com/tap222/docai-eval#setup-guide).
+
+```python
+import assay_sdk as assay
+
+assay.init("https://assay.example.com", key="ak_...")   # or set ASSAY_URL / ASSAY_KEY
+
+# An agent
+with assay.run("refund_request", input=message, version={"prompt": "support@v5", "model": "claude-sonnet-5"}) as run:
+    run.llm(model="claude-sonnet-5", tokens_in=620, tokens_out=180, cost_usd=0.0024)
+    order = run.call("get_order", get_order, order_id="O-17")    # runs it; records the result or the error
+    run.state("refund:O-17", "create", {"amount": order["price"]})
+    run.answer(f"Refunded ${order['price']}.")
+
+# A pipeline
+with assay.run("invoice", kind="pipeline", input_ref="s3://inbox/inv-9.pdf") as run:
+    with run.stage("extract", prompt="extract_fields@v13") as s:
+        run.llm(model="claude-sonnet-5", cost_usd=0.01)          # nested under the stage
+        s.outputs.update(fields)
+
+# Outcomes, whenever they're known
+assay.feedback(run.id, "thumbs_down")
+assay.correction(run.id, "total", expected="1240.00", observed="1204.00")
+assay.check("nightly-0924", "case-17", "fail", run_id=run.id, field="total", expected="1240.00", actual="1204.00")
+assay.expect("case-17", calls=[{"tool": "get_order", "args": {"order_id": "O-17"}}], answer="27.61")
+```
+
+| Call | Records |
+|---|---|
+| `assay.run(task, kind="agent"\|"pipeline", input=, version=, test={"run", "case", "attempt"})` | one run; an exception ends it as failed |
+| `run.llm(...)`, `run.tool(name, args, result)`, `run.call(name, fn, **args)`, `run.state(obj, op, value)`, `run.answer(text)`, `with run.stage(name) as s` | its steps, in order |
+| `assay.feedback`, `assay.check`, `assay.correction`, `assay.expect` | outcomes, sent whenever they're known |
+| `assay.flush()` | send now (short-lived scripts); also happens every second and at exit |
+
+These options go to `init()`:
+- `redact`: a function applied to inputs, arguments, results, text and outputs before they
+  leave the process.
+- `sample=0.1`: record one run in ten. A run is recorded whole or not at all, and outcomes
+  are always sent.
+- `strict=True`: raise send errors while developing. Otherwise the SDK never raises into
+  your code.
+- `enabled=False`: the SDK does nothing, e.g. in unit tests.
+
+Events follow the
+[Assay event schema v1](https://github.com/tap222/docai-eval/blob/main/docs/event-schema.md). They stream to
+`POST /v1/ingest` in the background, so a run that crashes still shows every step up to
+the crash.

assay_evals-0.1.0/assay_evals.egg-info/SOURCES.txt

@@ -0,0 +1,9 @@
+LICENSE
+README.md
+pyproject.toml
+assay_evals.egg-info/PKG-INFO
+assay_evals.egg-info/SOURCES.txt
+assay_evals.egg-info/dependency_links.txt
+assay_evals.egg-info/top_level.txt
+assay_sdk/__init__.py
+assay_sdk/py.typed

assay_evals-0.1.0/assay_evals.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

assay_evals-0.1.0/assay_evals.egg-info/top_level.txt

@@ -0,0 +1 @@
+assay_sdk

assay_evals-0.1.0/assay_sdk/__init__.py

@@ -0,0 +1,361 @@
+"""Assay SDK: record what your AI system does, and how it went. Standard library only.
+
+    import assay_sdk as assay
+
+    assay.init("https://assay.example.com", key="ak_...")   # or ASSAY_URL / ASSAY_KEY
+
+    with assay.run("refund_request", input=message, version={"prompt": "support@v5"}) as run:
+        run.llm(model="claude-sonnet-5", tokens_in=620, tokens_out=180, cost_usd=0.0024)
+        order = run.call("get_order", get_order, order_id="O-17")   # runs it, records result or error
+        run.state("refund:O-17", "create", {"amount": 27.61})
+        run.answer("Refunded $27.61.")
+
+    assay.feedback(run.id, "thumbs_up")                       # later, from your UI
+    assay.check("nightly-0924", "case-17", "pass", run_id=run.id, field="answer")   # from your tests
+
+Events follow the Assay event schema v1 (docs/event-schema.md) and stream to
+POST /v1/ingest in the background: a run that crashes still shows every step up
+to that point. Every event has an id, so retries never duplicate anything. The
+SDK never raises into your code (pass strict=True to init while developing).
+"""
+from __future__ import annotations
+
+import atexit
+import json
+import logging
+import os
+import random
+import threading
+import time
+import urllib.error
+import urllib.request
+import uuid
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from typing import Any, Callable, Dict, List, Optional
+
+__all__ = ["init", "run", "feedback", "check", "correction", "expect", "flush", "shutdown", "Run"]
+__version__ = "0.1.0"
+
+log = logging.getLogger("assay_sdk")
+SCHEMA = 1
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _ts(t: Optional[datetime]) -> Optional[str]:
+    if t is None:
+        return None
+    if t.tzinfo is None:
+        t = t.replace(tzinfo=timezone.utc)
+    return t.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _json_safe(v):
+    """Make a value JSON-serializable without failing: unknown objects become their repr."""
+    try:
+        json.dumps(v)
+        return v
+    except (TypeError, ValueError):
+        if isinstance(v, dict):
+            return {str(k): _json_safe(x) for k, x in v.items()}
+        if isinstance(v, (list, tuple, set)):
+            return [_json_safe(x) for x in v]
+        if isinstance(v, datetime):
+            return _ts(v)
+        return repr(v)
+
+
+class _Client:
+    def __init__(self, url: str, key: Optional[str], tenant: Optional[str], redact: Optional[Callable],
+                 sample: float, flush_interval: float, batch_size: int, max_queue: int, strict: bool,
+                 transport: Optional[Callable[[List[dict]], None]], enabled: bool):
+        self.url, self.key, self.tenant = url.rstrip("/"), key, tenant
+        self.redact, self.sample, self.strict, self.enabled = redact, sample, strict, enabled
+        self.batch_size, self.max_queue = batch_size, max_queue
+        self._send = transport or self._http
+        self._queue: List[dict] = []
+        self._lock = threading.Lock()
+        self._wake = threading.Event()
+        self._stop = False
+        self.dropped = 0
+        self._thread = threading.Thread(target=self._loop, args=(flush_interval,), name="assay-evals", daemon=True)
+        self._thread.start()
+
+    def emit(self, event: dict) -> None:
+        if not self.enabled:
+            return
+        event = {"v": SCHEMA, "id": uuid.uuid4().hex, "ts": _now(), **event}
+        with self._lock:
+            if len(self._queue) >= self.max_queue:
+                self._queue.pop(0)
+                self.dropped += 1
+                if self.dropped in (1, 100, 10000):
+                    log.warning("Assay: queue full, dropped %d events so far (is the server reachable?)", self.dropped)
+            self._queue.append({k: v for k, v in event.items() if v is not None})
+            full = len(self._queue) >= self.batch_size
+        if full:
+            self._wake.set()
+
+    def clean(self, v):
+        """Redact (if configured), then make JSON-safe."""
+        if v is None:
+            return None
+        if self.redact:
+            try:
+                v = self.redact(v)
+            except Exception:
+                log.exception("Assay: redact() failed; dropping the value rather than sending it unredacted")
+                return "<redaction failed>"
+        return _json_safe(v)
+
+    def flush(self) -> bool:
+        """Send everything queued. True if the queue is empty afterwards."""
+        while True:
+            with self._lock:
+                batch, self._queue = self._queue[:self.batch_size], self._queue[self.batch_size:]
+            if not batch:
+                return True
+            try:
+                self._send(batch)
+            except Exception:
+                with self._lock:  # keep them, in order, for the next try
+                    self._queue = batch + self._queue
+                if self.strict:
+                    raise
+                log.warning("Assay: couldn't send %d events; will retry", len(batch), exc_info=True)
+                return False
+
+    def _loop(self, interval: float) -> None:
+        backoff = interval
+        while not self._stop:
+            self._wake.wait(backoff)
+            self._wake.clear()
+            ok = self.flush() if not self.strict else self._flush_quiet()
+            backoff = interval if ok else min(backoff * 2, 60.0)
+
+    def _flush_quiet(self) -> bool:
+        try:
+            return self.flush()
+        except Exception:
+            return False
+
+    def close(self) -> None:
+        self._stop = True
+        self._wake.set()
+        self.flush()
+
+    def _http(self, batch: List[dict]) -> None:
+        headers = {"Content-Type": "application/json", "User-Agent": f"assay-evals-python/{__version__}"}
+        if self.key:
+            headers["Authorization"] = f"Bearer {self.key}"
+        if self.tenant:
+            headers["X-Tenant"] = self.tenant
+        req = urllib.request.Request(self.url + "/v1/ingest", data=json.dumps({"events": batch}).encode(),
+                                     headers=headers, method="POST")
+        for attempt in range(3):
+            try:
+                with urllib.request.urlopen(req, timeout=10) as r:
+                    r.read()
+                return
+            except urllib.error.HTTPError as e:
+                if e.code == 422:  # a malformed event won't improve with retries: log it and drop the batch
+                    log.error("Assay rejected events: %s", e.read().decode()[:1000])
+                    return
+                if e.code < 500 and e.code != 429 or attempt == 2:
+                    raise
+            except OSError:
+                if attempt == 2:
+                    raise
+            time.sleep(0.5 * 2 ** attempt)
+
+
+_client: Optional[_Client] = None
+
+
+def init(url: Optional[str] = None, key: Optional[str] = None, *, tenant: Optional[str] = None,
+         redact: Optional[Callable[[Any], Any]] = None, sample: float = 1.0, flush_interval: float = 1.0,
+         batch_size: int = 500, max_queue: int = 100_000, strict: bool = False,
+         transport: Optional[Callable[[List[dict]], None]] = None, enabled: bool = True) -> None:
+    """Configure the SDK once, at startup.
+
+    url / key      default to the ASSAY_URL / ASSAY_KEY environment variables
+    redact         applied to inputs, arguments, results, text and outputs before they leave the process
+    sample         share of runs to record (0.1 = one in ten); outcomes are always sent
+    strict         raise send errors instead of logging them (for development)
+    enabled        False turns the SDK into a no-op (e.g. in unit tests)
+    """
+    global _client
+    if _client is not None:
+        _client.close()
+    url = url or os.environ.get("ASSAY_URL")
+    if not url and transport is None and enabled:
+        raise ValueError("Give init() a url, or set ASSAY_URL.")
+    _client = _Client(url or "http://localhost", key or os.environ.get("ASSAY_KEY"), tenant, redact, sample,
+                      flush_interval, batch_size, max_queue, strict, transport, enabled)
+
+
+def _c() -> _Client:
+    if _client is None:
+        init()
+    return _client
+
+
+class Run:
+    """One run of your system on one input. Use through `assay.run(...)`."""
+
+    def __init__(self, client: _Client, run_id: str, recorded: bool):
+        self.id, self._c, self._on = run_id, client, recorded
+        self._seq, self._lock, self._parents = 0, threading.Lock(), []
+        self.answer_text: Optional[str] = None
+
+    def _step(self, kind: str, started: Optional[datetime] = None, **fields) -> int:
+        with self._lock:
+            seq, self._seq = self._seq, self._seq + 1
+            parent = self._parents[-1] if self._parents else None
+        if self._on:
+            self._c.emit({"type": "step", "run_id": self.id, "seq": seq, "kind": kind, "parent_seq": parent,
+                          "ts": _ts(started) or _now(), **fields})
+        return seq
+
+    def llm(self, model: Optional[str] = None, tokens_in: Optional[int] = None, tokens_out: Optional[int] = None,
+            cost_usd: Optional[float] = None, prompt: Optional[str] = None, text: Optional[str] = None,
+            started: Optional[datetime] = None, ended: Optional[datetime] = None, error: Optional[str] = None) -> None:
+        """A model call. prompt is "id@version"; text is the output (or a summary of it)."""
+        self._step("llm", started, model=model, tokens_in=tokens_in, tokens_out=tokens_out, cost_usd=cost_usd,
+                   prompt=prompt, text=self._c.clean(text), ended_at=_ts(ended),
+                   status="error" if error else "ok", error=error)
+
+    def tool(self, name: str, args: Optional[Dict[str, Any]] = None, result: Any = None, error: Optional[str] = None,
+             started: Optional[datetime] = None, ended: Optional[datetime] = None) -> None:
+        """A tool call you've already made."""
+        self._step("tool", started, name=name, args=self._c.clean(args or {}), result=self._c.clean(result),
+                   ended_at=_ts(ended), status="error" if error else "ok", error=error)
+
+    def call(self, name: str, fn: Callable, *positional, **args):
+        """Call fn(*positional, **args), record it as a tool call (result or error, and timing), and
+        return its result. Exceptions are recorded, then re-raised."""
+        started = datetime.now(timezone.utc)
+        try:
+            out = fn(*positional, **args)
+        except Exception as e:
+            self.tool(name, args, error=f"{type(e).__name__}: {e}"[:2000], started=started,
+                      ended=datetime.now(timezone.utc))
+            raise
+        self.tool(name, args, out, started=started, ended=datetime.now(timezone.utc))
+        return out
+
+    def state(self, obj: str, op: str = "update", value: Any = None) -> None:
+        """A change to the world, e.g. state("order:17", "update", {"qty": 3}). op: create, update, delete."""
+        self._step("state", name=obj, op=op, value=self._c.clean(value))
+
+    def answer(self, text: str) -> None:
+        self.answer_text = text
+        self._step("answer", text=self._c.clean(text))
+
+    @contextmanager
+    def stage(self, name: str, prompt: Optional[str] = None):
+        """A pipeline stage. Record what it produced in .outputs; llm/tool calls inside are nested under it.
+
+            with run.stage("extract") as s:
+                s.outputs.update(extract(text))
+        """
+        handle = _Stage()
+        started = datetime.now(timezone.utc)
+        with self._lock:
+            seq, self._seq = self._seq, self._seq + 1
+            self._parents.append(seq)
+        error = None
+        try:
+            yield handle
+        except Exception as e:
+            error = f"{type(e).__name__}: {e}"[:2000]
+            raise
+        finally:
+            with self._lock:
+                self._parents.remove(seq)
+            if self._on:
+                self._c.emit({"type": "step", "run_id": self.id, "seq": seq, "kind": "stage", "name": name,
+                              "ts": _ts(started), "ended_at": _now(), "status": "error" if error else "ok",
+                              "error": error, "outputs": self._c.clean(handle.outputs) or None,
+                              "did_work": handle.did_work, "prompt": prompt})
+
+
+class _Stage:
+    def __init__(self):
+        self.outputs: Dict[str, Any] = {}
+        self.did_work: Optional[bool] = True
+
+
+@contextmanager
+def run(task: Optional[str] = None, *, run_id: Optional[str] = None, kind: str = "agent", input: Any = None,
+        input_ref: Optional[str] = None, version: Optional[Dict[str, str]] = None, segment: Optional[str] = None,
+        test: Optional[Dict[str, Any]] = None, parent: Optional[Run] = None, tags: Optional[Dict[str, Any]] = None):
+    """Record one run. kind is "agent" (llm/tool/state/answer steps) or "pipeline" (stages).
+    test={"run": "nightly-0924", "case": "case-17", "attempt": 0} marks a test-case run.
+    An exception inside the block ends the run as failed (and is re-raised)."""
+    c = _c()
+    rid = run_id or uuid.uuid4().hex
+    recorded = c.sample >= 1 or random.random() < c.sample
+    r = Run(c, rid, recorded)
+    if recorded:
+        c.emit({"type": "run.start", "run_id": rid, "kind": kind, "task": task, "segment": segment,
+                "input": c.clean(input), "input_ref": input_ref, "version": version, "test": test,
+                "parent_run_id": parent.id if parent else None, "tags": tags})
+    status, error = "completed", None
+    try:
+        yield r
+    except BaseException as e:
+        status, error = "failed", f"{type(e).__name__}: {e}"[:2000]
+        raise
+    finally:
+        if recorded:
+            c.emit({"type": "run.end", "run_id": rid, "status": status, "error": error})
+
+
+def feedback(run_id: str, kind: str, note: Optional[str] = None) -> None:
+    """What a user did: thumbs_up, thumbs_down, retry, escalation or complaint."""
+    _c().emit({"type": "feedback", "run_id": run_id, "kind": kind, "note": note})
+
+
+def check(test_run: str, case: str, status: str, *, attempt: Optional[int] = None, run_id: Optional[str] = None,
+          field: Optional[str] = None, expected: Any = None, actual: Any = None, evaluator: Optional[str] = None,
+          score: Optional[float] = None, reason: Optional[str] = None,
+          version: Optional[Dict[str, str]] = None) -> None:
+    """One result from a test run: status pass, fail, or error (the check couldn't run). Send passes too."""
+    s = lambda v: None if v is None else v if isinstance(v, str) else json.dumps(_json_safe(v))
+    _c().emit({"type": "check", "test": {k: v for k, v in {"run": test_run, "case": case, "attempt": attempt}.items()
+                                         if v is not None},
+               "status": status, "run_id": run_id, "field": field, "expected": s(expected), "actual": s(actual),
+               "evaluator": evaluator, "score": score, "reason": reason, "version": version})
+
+
+def correction(run_id: str, field: str, expected: Any = None, observed: Any = None, kind: str = "wrong",
+               reporter: Optional[str] = None) -> None:
+    """Someone found a wrong value: Assay traces it to the step it started at."""
+    s = lambda v: None if v is None else str(v)
+    _c().emit({"type": "correction", "run_id": run_id, "field": field, "expected": s(expected),
+               "observed": s(observed), "kind": kind, "reporter": reporter})
+
+
+def expect(case: str, *, calls: Optional[List[Dict[str, Any]]] = None, answer: Optional[str] = None,
+           state: Optional[List[Dict[str, Any]]] = None, allow_extra: Optional[List[str]] = None,
+           max_steps: Optional[int] = None, answer_match: str = "contains") -> None:
+    """What a test case should do: the tool calls, the answer, and the end state."""
+    _c().emit({"type": "expect", "case": case, "calls": calls or [], "answer": answer, "answer_match": answer_match,
+               "state": state or [], "allow_extra": allow_extra or [], "max_steps": max_steps})
+
+
+def flush() -> bool:
+    """Send everything now (e.g. before a short-lived process exits). True if nothing is left."""
+    return _c().flush() if _client else True
+
+
+def shutdown() -> None:
+    if _client:
+        _client.close()
+
+
+atexit.register(lambda: _client and _client.close())

assay_evals-0.1.0/pyproject.toml

@@ -0,0 +1,37 @@
+[project]
+name = "assay-evals"
+version = "0.1.0"
+description = "Record what your AI system does, and how it went, in Assay: runs, agent steps, feedback and test results"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "tap222" }]
+requires-python = ">=3.9"
+dependencies = []
+keywords = ["llm", "agents", "evaluation", "observability", "tracing", "ai"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+  "Topic :: Software Development :: Testing",
+  "Typing :: Typed",
+]
+
+[project.urls]
+Homepage = "https://github.com/tap222/docai-eval"
+Documentation = "https://github.com/tap222/docai-eval/tree/main/sdk/python#readme"
+"Event schema" = "https://github.com/tap222/docai-eval/blob/main/docs/event-schema.md"
+Issues = "https://github.com/tap222/docai-eval/issues"
+
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+packages = ["assay_sdk"]
+
+[tool.setuptools.package-data]
+assay_sdk = ["py.typed"]

assay_evals-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+