openrunner-sdk 2.37.4__tar.gz → 2.37.5__tar.gz

{openrunner_sdk-2.37.4 → openrunner_sdk-2.37.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrunner-sdk
-Version: 2.37.4
+Version: 2.37.5
 Summary: OpenRunner SDK - W&B-compatible ML experiment tracking client
 Project-URL: Homepage, https://github.com/jqueguiner/openrunner
 Project-URL: Repository, https://github.com/jqueguiner/openrunner

{openrunner_sdk-2.37.4 → openrunner_sdk-2.37.5}/openrunner/__init__.py

@@ -181,7 +181,7 @@ launch.from_run = _launch_from_run  # type: ignore[attr-defined]
 # openrunner.trace.patch_openai() syntax
 trace.patch_openai = _patch_openai  # type: ignore[attr-defined]
 
-__version__ = "2.37.4"
+__version__ = "2.37.5"
 
 logger = logging.getLogger("openrunner")
 

{openrunner_sdk-2.37.4 → openrunner_sdk-2.37.5}/openrunner/install_commands.py

@@ -442,7 +442,7 @@ allowed-tools:
 Upgrade openrunner-sdk and reinstall slash commands:
 
 ```bash
-pip install --upgrade openrunner-sdk 2>&1 | tail -3 && python3 -c "
+pip install --upgrade --no-cache-dir openrunner-sdk 2>&1 | tail -3 && python3 -c "
 import sys, os
 from pathlib import Path
 for p in ([os.path.join(d, 'site-packages') for d in __import__('glob').glob(os.path.expanduser('~/.local/lib/python3.*/')) + __import__('glob').glob(os.path.expanduser('~/Library/Python/3.*/lib/python/'))]):
@@ -2596,6 +2596,128 @@ pulls** -- `openrunner gpu run --image ...` resolves the org's stored registry.)
 """
 
 
+DRIFT_CMD = """---
+name: {prefix}drift
+description: Instrument an inference API/service to report per-prediction events to OpenRunner inference-drift monitoring (distribution shift / PSI), segment by metadata, then watch + alert -- no raw payloads stored, only the metrics you choose
+argument-hint: "[project org/name]"
+allowed-tools:
+  - Bash
+  - Read
+  - Edit
+  - AskUserQuestion
+---
+
+Add **inference-drift monitoring** to an API/service: after each prediction,
+report a lightweight event (input features + output metrics) to OpenRunner, which
+tracks distribution shift (PSI), per-segment health and A/B. Only the numbers +
+tags you send are stored -- never the raw inputs/outputs.
+
+## The event shape (one per inference)
+
+- `inference_id` -- your request/prediction id (str).
+- `metadata` -- dict of scalars OR lists (the axes you'll slice by), e.g.
+  `{{"model": "v3", "lang": ["fr", "en"], "route": "asr", "region": "eu"}}`.
+- `metrics` -- dict of floats to watch, e.g.
+  `{{"confidence": 0.92, "latency_ms": 180, "tokens": 57, "input_len": 12.4}}`.
+- `monitor` (optional) -- the drift stream name; use one per model/endpoint.
+- `event_time` (optional) -- ISO timestamp; defaults to now.
+
+Choose metrics that reveal drift: output confidence/score, input length / feature
+stats, latency, class distribution. Metadata = segments to localise a drift.
+
+## Step 1 -- resolve the project id
+
+```bash
+openrunner login
+python3 -c "
+from openrunner.api_client import APIClient
+import json, os
+c = APIClient(base_url=os.environ.get('OPENRUNNER_BASE_URL','https://openrun.gladia.io'),
+              api_key=os.environ['OPENRUNNER_API_KEY'])
+print([(p['name'], p['id']) for p in c.list_projects()])
+"
+```
+
+## Step 2 -- instrument (NON-BLOCKING) -- FastAPI example
+
+Report AFTER responding, batched on a background thread, so you never add latency
+to the request. The ingest endpoint answers **429 + Retry-After** under load (the
+client backs off) -- so drop events rather than block the request path.
+
+```python
+import os, threading, queue, atexit
+from openrunner.api_client import APIClient
+
+PROJECT_ID = "..."   # from step 1
+_client = APIClient(base_url=os.environ.get("OPENRUNNER_BASE_URL", "https://openrun.gladia.io"),
+                    api_key=os.environ["OPENRUNNER_API_KEY"])
+_q: queue.Queue = queue.Queue(maxsize=10000)
+
+def _flush_loop():
+    buf = []
+    while True:
+        ev = _q.get()
+        if ev is not None:
+            buf.append(ev)
+        if buf and (ev is None or len(buf) >= 200):
+            _client.post_inference_events(PROJECT_ID, buf, monitor="asr-prod")
+            buf = []
+        if ev is None:
+            return
+
+threading.Thread(target=_flush_loop, daemon=True).start()
+atexit.register(lambda: _q.put(None))
+
+def record_inference(req_id, metadata, metrics):
+    try:
+        _q.put_nowait({{"inference_id": req_id, "metadata": metadata, "metrics": metrics}})
+    except queue.Full:
+        pass  # never block or fail the request because of monitoring
+
+# ... in your endpoint, AFTER producing the result:
+record_inference(
+    req_id,
+    {{"model": "v3", "lang": lang, "route": "asr"}},
+    {{"confidence": conf, "latency_ms": dt_ms, "audio_s": duration}},
+)
+```
+
+**Non-Python service?** POST the events directly:
+
+```
+POST /api/v1/projects/PROJECT_ID/inference-drift/ingest
+Authorization: Bearer <API_KEY>
+{{ "monitor": "asr-prod",
+   "events": [ {{"inference_id": "r1", "metadata": {{"lang": "fr"}}, "metrics": {{"confidence": 0.9}}}} ] }}
+-> {{"ingested": n}}   (429 + Retry-After under load -> back off)
+```
+
+**Batch offline** from a JSON file instead of instrumenting:
+
+```bash
+openrunner drift report events.json --project ORG/NAME --monitor asr-prod
+```
+
+## Step 3 -- watch + alert
+
+- UI: the **Drift** menu -> overview / timeseries / per-segment PSI / A/B.
+- CLI:
+  - `openrunner drift monitors --project ORG/NAME`
+  - `openrunner drift check --project ORG/NAME --metric confidence` (drift verdict)
+  - `openrunner drift ab --project ORG/NAME --metric confidence --variant-a v2 --variant-b v3`
+  - `openrunner drift ask --project ORG/NAME "did latency drift this week?"`
+- MCP (if the OpenRunner server is registered): the drift tools answer the same in-session.
+
+## Notes
+
+- **Nothing raw is stored** -- only the metrics + metadata you send.
+- One `monitor` per model/endpoint; segment with metadata (lang / region / route /
+  model version) so a drift can be traced to a slice.
+- **Sample** at high QPS (report 1 in N) -- PSI needs volume, not every event.
+- Old days age out to S3 parquet automatically (a hot window stays in Postgres).
+"""
+
+
 def install_claude_code() -> list[str]:
     """Install commands in ~/.claude/commands/openrunner/."""
     cmd_dir = Path.home() / ".claude" / "commands" / "openrunner"
@@ -2630,6 +2752,7 @@ def install_claude_code() -> list[str]:
         "derive-dataset.md": DERIVE_DATASET_CMD,
         "launch-gpu.md": LAUNCH_GPU_CMD,
         "registry.md": REGISTRY_CMD,
+        "drift.md": DRIFT_CMD,
     }
 
     files = []
@@ -2832,6 +2955,7 @@ def _install_all_commands_to_dir(cmd_dir: Path, prefix: str = "openrunner:") ->
         "derive-dataset.md": DERIVE_DATASET_CMD,
         "launch-gpu.md": LAUNCH_GPU_CMD,
         "registry.md": REGISTRY_CMD,
+        "drift.md": DRIFT_CMD,
     }
 
     files = []

{openrunner_sdk-2.37.4 → openrunner_sdk-2.37.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "openrunner-sdk"
-version = "2.37.4"
+version = "2.37.5"
 description = "OpenRunner SDK - W&B-compatible ML experiment tracking client"
 readme = "README.md"
 license = {text = "MIT"}