runlab-client 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""runlab-client: Python client for the RunLab research API.
|
|
2
|
+
|
|
3
|
+
from runlab_client import RunLab
|
|
4
|
+
|
|
5
|
+
rl = RunLab() # reads RUNLAB_URL and RUNLAB_TOKEN
|
|
6
|
+
rl.plan("demo", "treatment", "baseline", "acc", hypothesis="the change helps", n_max=10,
|
|
7
|
+
stopping="fixed")
|
|
8
|
+
rl.result("demo", "treatment", acc=0.91, seed=0)
|
|
9
|
+
rl.result("demo", "baseline", acc=0.89, seed=0)
|
|
10
|
+
... # more seeds
|
|
11
|
+
v = rl.compare("demo", "treatment", "baseline", "acc")
|
|
12
|
+
print(v["verdict"], v["text"])
|
|
13
|
+
|
|
14
|
+
Standard library only, Python 3.10+. Each method is one operation of openapi.json.
|
|
15
|
+
Errors raise RunLabError with the HTTP status, RunLab's message and the request id.
|
|
16
|
+
GET requests are retried on connection errors and on 429, 502, 503 and 504 with exponential
|
|
17
|
+
backoff; POST requests are never retried.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import time
|
|
25
|
+
import urllib.error
|
|
26
|
+
import urllib.parse
|
|
27
|
+
import urllib.request
|
|
28
|
+
import uuid
|
|
29
|
+
from typing import Any
|
|
30
|
+
|
|
31
|
+
__all__ = ["RunLab", "RunLabError", "result_line"]
|
|
32
|
+
__version__ = "0.1.0"
|
|
33
|
+
|
|
34
|
+
RETRY_STATUS = (429, 502, 503, 504)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class RunLabError(RuntimeError):
|
|
38
|
+
"""An HTTP error (`status` 400-599) or RunLab unreachable (`status` 0)."""
|
|
39
|
+
|
|
40
|
+
def __init__(self, status: int, message: str, request_id: str = "") -> None:
|
|
41
|
+
super().__init__(f"HTTP {status}: {message}"
|
|
42
|
+
+ (f" (request {request_id})" if request_id else ""))
|
|
43
|
+
self.status = status
|
|
44
|
+
self.message = message
|
|
45
|
+
self.request_id = request_id
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class RunLab:
|
|
49
|
+
"""A connection to RunLab.
|
|
50
|
+
|
|
51
|
+
url: your RunLab URL, e.g. "https://runlab.example.com" (default: env RUNLAB_URL).
|
|
52
|
+
token: your API token (default: env RUNLAB_TOKEN).
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
def __init__(self, url: str | None = None, token: str | None = None, *,
|
|
56
|
+
timeout: float = 30.0, retries: int = 3, backoff_s: float = 0.5) -> None:
|
|
57
|
+
url = url or os.environ.get("RUNLAB_URL", "")
|
|
58
|
+
if not url:
|
|
59
|
+
raise ValueError("no RunLab URL: pass url= or set RUNLAB_URL")
|
|
60
|
+
self.url = url.rstrip("/")
|
|
61
|
+
self.token = token if token is not None else os.environ.get("RUNLAB_TOKEN", "")
|
|
62
|
+
self.timeout = timeout
|
|
63
|
+
self.retries = retries
|
|
64
|
+
self.backoff_s = backoff_s
|
|
65
|
+
|
|
66
|
+
# ------------------------------------------------------------------ HTTP
|
|
67
|
+
|
|
68
|
+
def request(self, method: str, path: str, body: dict[str, Any] | None = None,
|
|
69
|
+
params: dict[str, Any] | None = None) -> Any:
|
|
70
|
+
"""One call to `/api/v1{path}`; returns the decoded JSON body."""
|
|
71
|
+
q = {k: v for k, v in (params or {}).items() if v is not None}
|
|
72
|
+
url = f"{self.url}/api/v1{path}" + (f"?{urllib.parse.urlencode(q)}" if q else "")
|
|
73
|
+
rid = uuid.uuid4().hex[:16]
|
|
74
|
+
headers = {"Accept": "application/json", "X-Request-Id": rid,
|
|
75
|
+
"User-Agent": f"runlab-client/{__version__}"}
|
|
76
|
+
if self.token:
|
|
77
|
+
headers["Authorization"] = f"Bearer {self.token}"
|
|
78
|
+
data = None
|
|
79
|
+
if method == "POST":
|
|
80
|
+
headers["Content-Type"] = "application/json"
|
|
81
|
+
data = json.dumps(body or {}).encode("utf-8")
|
|
82
|
+
attempts = 1 + (self.retries if method == "GET" else 0)
|
|
83
|
+
for i in range(attempts):
|
|
84
|
+
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
|
85
|
+
try:
|
|
86
|
+
with urllib.request.urlopen(req, timeout=self.timeout) as r: # noqa: S310
|
|
87
|
+
return json.loads(r.read() or b"{}")
|
|
88
|
+
except urllib.error.HTTPError as exc:
|
|
89
|
+
if exc.code in RETRY_STATUS and i + 1 < attempts:
|
|
90
|
+
time.sleep(self.backoff_s * 2 ** i)
|
|
91
|
+
continue
|
|
92
|
+
try:
|
|
93
|
+
msg = json.loads(exc.read() or b"{}").get("error") or exc.reason
|
|
94
|
+
except ValueError:
|
|
95
|
+
msg = exc.reason
|
|
96
|
+
raise RunLabError(exc.code, str(msg),
|
|
97
|
+
exc.headers.get("X-Request-Id", rid)) from None
|
|
98
|
+
except (urllib.error.URLError, OSError) as exc:
|
|
99
|
+
if i + 1 < attempts:
|
|
100
|
+
time.sleep(self.backoff_s * 2 ** i)
|
|
101
|
+
continue
|
|
102
|
+
raise RunLabError(0, f"cannot reach {self.url}: "
|
|
103
|
+
f"{getattr(exc, 'reason', exc)}", rid) from None
|
|
104
|
+
raise AssertionError("unreachable") # pragma: no cover
|
|
105
|
+
|
|
106
|
+
# --------------------------------------------------------------- account
|
|
107
|
+
|
|
108
|
+
def me(self) -> dict[str, Any]:
|
|
109
|
+
"""Who the token belongs to: `name` and `role`. A quick check that access works."""
|
|
110
|
+
return self.request("GET", "/me")
|
|
111
|
+
|
|
112
|
+
# -------------------------------------------------------------- research
|
|
113
|
+
|
|
114
|
+
def plan(self, project: str, a: str, b: str, metric: str, hypothesis: str, n_max: int,
|
|
115
|
+
**options: Any) -> dict[str, Any]:
|
|
116
|
+
"""Pre-register the comparison before any seed runs. Returns `plan`, `power_check`
|
|
117
|
+
and `text`. options: better ("auto" | "higher" | "lower"), alpha, n_min,
|
|
118
|
+
stopping ("msprt" | "fixed"), amend (a reason, to replace an earlier plan)."""
|
|
119
|
+
return self.request("POST", "/research/plans", {
|
|
120
|
+
"project": project, "a": a, "b": b, "metric": metric,
|
|
121
|
+
"hypothesis": hypothesis, "n_max": n_max, **options})
|
|
122
|
+
|
|
123
|
+
def record(self, project: str, results: str | list[str]) -> dict[str, Any]:
|
|
124
|
+
"""Store RESULT lines (one string, or a list of lines from `result_line`).
|
|
125
|
+
Returns `stored` and `already` (lines RunLab had already)."""
|
|
126
|
+
text = results if isinstance(results, str) else "\n".join(results)
|
|
127
|
+
return self.request("POST", "/research/results",
|
|
128
|
+
{"project": project, "results": text})
|
|
129
|
+
|
|
130
|
+
def result(self, project: str, experiment: str, *, seed: int | None = None,
|
|
131
|
+
params: dict[str, Any] | None = None, **metrics: float) -> dict[str, Any]:
|
|
132
|
+
"""One seed's numbers for one arm: `rl.result("demo", "treatment", acc=0.91, seed=3)`.
|
|
133
|
+
Sends one RESULT line per metric in a single call."""
|
|
134
|
+
if not metrics:
|
|
135
|
+
raise ValueError("give at least one metric, e.g. acc=0.91")
|
|
136
|
+
return self.record(project, [result_line(experiment, m, v, seed=seed, **(params or {}))
|
|
137
|
+
for m, v in metrics.items()])
|
|
138
|
+
|
|
139
|
+
def compare(self, project: str, a: str, b: str, metric: str,
|
|
140
|
+
**options: Any) -> dict[str, Any]:
|
|
141
|
+
"""Judge arm `a` against arm `b` on `metric`. Returns `verdict`, the test's numbers
|
|
142
|
+
and `text`. options: better ("auto" | "higher" | "lower"), alpha, margin."""
|
|
143
|
+
return self.request("POST", "/research/compare", {
|
|
144
|
+
"project": project, "a": a, "b": b, "metric": metric, **options})
|
|
145
|
+
|
|
146
|
+
def verdicts(self, project: str = "", limit: int = 50,
|
|
147
|
+
recheck: bool = False) -> dict[str, Any]:
|
|
148
|
+
"""Recorded verdicts, newest first. recheck=True judges each pair again on today's
|
|
149
|
+
results (`stale` says whether one changed)."""
|
|
150
|
+
return self.request("GET", "/research/verdicts", params={
|
|
151
|
+
"project": project or None, "limit": limit, "recheck": "1" if recheck else None})
|
|
152
|
+
|
|
153
|
+
def verdict(self, verdict_id: int) -> dict[str, Any]:
|
|
154
|
+
"""One recorded verdict in full: each seed's values, the re-derived difference and
|
|
155
|
+
whether it still holds on today's results."""
|
|
156
|
+
return self.request("GET", f"/verdicts/{int(verdict_id)}")
|
|
157
|
+
|
|
158
|
+
def report(self, project: str, *, arm: str = "", metric: str = "",
|
|
159
|
+
q: float = 0.05) -> dict[str, Any]:
|
|
160
|
+
"""Which verdicts of a project you can quote, after correcting for the number of
|
|
161
|
+
comparisons (false discovery rate `q`, Benjamini-Hochberg)."""
|
|
162
|
+
return self.request("GET", "/research/report", params={
|
|
163
|
+
"project": project, "arm": arm or None, "metric": metric or None, "q": q})
|
|
164
|
+
|
|
165
|
+
# ----------------------------------------------------------- seed studies
|
|
166
|
+
|
|
167
|
+
def start_study(self, project: str, command: str, a: str, b: str, metric: str,
|
|
168
|
+
max_usd: float, *, params: dict[str, Any] | None = None,
|
|
169
|
+
arm_params: dict[str, dict[str, Any]] | None = None) -> dict[str, Any]:
|
|
170
|
+
"""Let RunLab run the seeds of a planned comparison under a hard dollar cap.
|
|
171
|
+
`command` names the training command to run; `params` fill its placeholders
|
|
172
|
+
for both arms, `arm_params` per arm, keyed by arm name: {"mixup": {"alpha": "0.2"}}.
|
|
173
|
+
A refusal raises RunLabError (400) before anything runs."""
|
|
174
|
+
body: dict[str, Any] = {"project": project, "command": command, "a": a, "b": b,
|
|
175
|
+
"metric": metric, "max_usd": max_usd}
|
|
176
|
+
if params:
|
|
177
|
+
body["params"] = params
|
|
178
|
+
if arm_params:
|
|
179
|
+
body["arm_params"] = arm_params
|
|
180
|
+
return self.request("POST", "/research/studies", body)
|
|
181
|
+
|
|
182
|
+
def study(self, study_id: str) -> dict[str, Any]:
|
|
183
|
+
"""One study by id (`st_...`)."""
|
|
184
|
+
return self.request("GET", f"/research/studies/{urllib.parse.quote(study_id, safe='')}")
|
|
185
|
+
|
|
186
|
+
def studies(self, project: str = "", limit: int = 50) -> list[dict[str, Any]]:
|
|
187
|
+
"""Studies, newest first."""
|
|
188
|
+
return self.request("GET", "/research/studies", params={
|
|
189
|
+
"project": project or None, "limit": limit})["studies"]
|
|
190
|
+
|
|
191
|
+
def wait_study(self, study_id: str, timeout: float = 24 * 3600,
|
|
192
|
+
poll_s: float = 30.0) -> dict[str, Any]:
|
|
193
|
+
"""Poll until the study stops running and return it. Raises TimeoutError after
|
|
194
|
+
`timeout` seconds (the study keeps running in RunLab)."""
|
|
195
|
+
deadline = time.monotonic() + timeout
|
|
196
|
+
while True:
|
|
197
|
+
s = self.study(study_id)
|
|
198
|
+
if s.get("state") != "running":
|
|
199
|
+
return s
|
|
200
|
+
if time.monotonic() + poll_s > deadline:
|
|
201
|
+
raise TimeoutError(f"study {study_id} still running after {timeout:.0f}s")
|
|
202
|
+
time.sleep(poll_s)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def result_line(experiment: str, metric: str, value: float, *, seed: int | None = None,
|
|
206
|
+
**params: Any) -> str:
|
|
207
|
+
"""`RESULT: <experiment> <metric>=<value> seed=<n> [<param>=<value> ...]`.
|
|
208
|
+
|
|
209
|
+
Names use letters, digits, `_`, `.` and `-`; parameter values cannot hold spaces or `=`.
|
|
210
|
+
"""
|
|
211
|
+
for k, v in params.items():
|
|
212
|
+
if k == "reported": # RunLab reads it as a paper's number
|
|
213
|
+
raise ValueError("param 'reported' is reserved")
|
|
214
|
+
if " " in str(v) or "=" in str(v):
|
|
215
|
+
raise ValueError(f"param {k}={v!r}: values cannot hold spaces or '='")
|
|
216
|
+
extra = "".join(f" {k}={v}" for k, v in params.items())
|
|
217
|
+
return (f"RESULT: {experiment} {metric}={float(value)!r}"
|
|
218
|
+
+ ("" if seed is None else f" seed={int(seed)}") + extra)
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: runlab-client
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python client for the RunLab research API: pre-registered A/B comparisons over seeds, verdicts with confidence intervals. Standard library only.
|
|
5
|
+
Author: omer-nachman
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/omer-nachman/runlab-python
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Dynamic: license-file
|
|
12
|
+
|
|
13
|
+
# runlab-client
|
|
14
|
+
|
|
15
|
+
**Did my change actually help?** You record each seed's metric for two arms: your change and
|
|
16
|
+
its baseline. RunLab gives back a verdict you can quote, built from:
|
|
17
|
+
|
|
18
|
+
- a pre-registered plan;
|
|
19
|
+
- a paired test across seeds, with a 95 % confidence interval;
|
|
20
|
+
- checks that refuse a comparison that would mislead, e.g. arms tuned with unequal budgets.
|
|
21
|
+
|
|
22
|
+
This repo has the Python client (standard library only), the
|
|
23
|
+
[OpenAPI spec](openapi.json) ([rendered](api.html)) and examples. Install it and set your
|
|
24
|
+
RunLab URL and API token:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
pip install runlab-client # Python 3.10+, no dependencies
|
|
28
|
+
# before its first PyPI release: pip install "git+https://github.com/omer-nachman/runlab-python"
|
|
29
|
+
export RUNLAB_URL=https://runlab.example.com RUNLAB_TOKEN=<your token>
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Quickstart
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from runlab_client import RunLab
|
|
36
|
+
|
|
37
|
+
rl = RunLab() # reads RUNLAB_URL and RUNLAB_TOKEN
|
|
38
|
+
rl.me() # {'name': 'ada', 'role': 'agent', ...}
|
|
39
|
+
|
|
40
|
+
# 1. Pre-register, before any seed runs.
|
|
41
|
+
rl.plan("resnet-aug", "mixup", "baseline", "acc",
|
|
42
|
+
hypothesis="mixup raises top-1 accuracy", n_max=10, stopping="fixed")
|
|
43
|
+
|
|
44
|
+
# 2. Record each seed's metric for both arms (use the same seeds for both).
|
|
45
|
+
for seed in range(10):
|
|
46
|
+
rl.result("resnet-aug", "mixup", acc=train("mixup", seed), seed=seed)
|
|
47
|
+
rl.result("resnet-aug", "baseline", acc=train("baseline", seed), seed=seed)
|
|
48
|
+
|
|
49
|
+
# 3. Compare once, after the planned seeds ran.
|
|
50
|
+
v = rl.compare("resnet-aug", "mixup", "baseline", "acc")
|
|
51
|
+
print(v["verdict"], v["prereg"]["status"]) # a_better ok
|
|
52
|
+
print(v["text"])
|
|
53
|
+
# mixup vs baseline on acc (higher is better): mixup is better
|
|
54
|
+
# paired_t: diff +0.0178 [95% CI +0.0094, +0.0262], p=0.00098, n=10 (A 10, B 10)
|
|
55
|
+
# ... (relative effect, rank test, seed stability, fragility, plan, equal budget)
|
|
56
|
+
|
|
57
|
+
# 4. May a write-up quote it?
|
|
58
|
+
for c in rl.report("resnet-aug")["claims"]:
|
|
59
|
+
print(c["status"], c["quote"]) # quotable mixup − baseline on acc = ...
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`train` is your own training run. The numbers above are an illustration. A runnable version
|
|
63
|
+
with a simulated `train` is [examples/quickstart.py](examples/quickstart.py), and
|
|
64
|
+
[examples/curl.sh](examples/curl.sh) does the same loop over plain HTTP.
|
|
65
|
+
|
|
66
|
+
## The loop
|
|
67
|
+
|
|
68
|
+
| Step | Client | HTTP | What RunLab does |
|
|
69
|
+
|---|---|---|---|
|
|
70
|
+
| 1. Plan | `plan(project, a, b, metric, hypothesis, n_max)` | `POST /api/v1/research/plans` | Stores the plan and its hash. Estimates power from seeds already recorded (`power_check`). |
|
|
71
|
+
| 2. Record | `result(project, arm, seed=s, metric=value)` or `record(project, lines)` | `POST /api/v1/research/results` | Stores one row per arm, seed and metric. A line it already has is skipped, so retries are safe. |
|
|
72
|
+
| 3. Compare | `compare(project, a, b, metric)` | `POST /api/v1/research/compare` | Runs the paired t-test on a − b over shared seeds (Welch's test if fewer than 2 are shared). Checks the plan and records the verdict. |
|
|
73
|
+
| 4. Quote | `report(project)` | `GET /api/v1/research/report` | Says which verdicts a write-up may quote, after the false-discovery-rate correction across every comparison in the project. |
|
|
74
|
+
| Audit | `verdicts(project)`, `verdict(id)` | `GET /api/v1/research/verdicts`, `GET /api/v1/verdicts/{id}` | Shows every recorded verdict, each seed's values, and whether it still holds on today's results. |
|
|
75
|
+
| Delegate | `start_study(...)`, `wait_study(id)` | `POST /api/v1/research/studies` | Runs the seeds itself under a dollar cap and stops once the evidence decides (see below). |
|
|
76
|
+
|
|
77
|
+
`a` is the treatment and `b` the baseline. `diff` is the mean of a − b, so it is positive when
|
|
78
|
+
a scored higher. Whether higher is better comes from the metric name: lower for names
|
|
79
|
+
containing `loss`, `error`, `err`, `perplexity`, `ppl`, `wer`, `cer`, `mse`, `mae`, `rmse`,
|
|
80
|
+
`nll`, `latency`, `fid` or `regret`, higher otherwise. Pass `better="higher"` or
|
|
81
|
+
`better="lower"` to override.
|
|
82
|
+
|
|
83
|
+
### Results
|
|
84
|
+
|
|
85
|
+
RunLab stores each result as one line:
|
|
86
|
+
|
|
87
|
+
```
|
|
88
|
+
RESULT: <arm> <metric>=<number> seed=<n> [<param>=<value> ...]
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
`result()` builds these lines, so you only write them yourself when your training script
|
|
92
|
+
prints them (send its log to `record()`) or when you call the HTTP API directly.
|
|
93
|
+
|
|
94
|
+
- Names use letters, digits, `_`, `.` and `-`; arm names start with a letter.
|
|
95
|
+
- Lines that do not start with `RESULT:` are ignored, so a whole training log can go to
|
|
96
|
+
`record()` as is.
|
|
97
|
+
- If one RESULT line is malformed, or there is none, the call stores nothing and fails with
|
|
98
|
+
a 400.
|
|
99
|
+
- Project names are case-insensitive.
|
|
100
|
+
- Params are optional; keep an arm's params the same on every seed. Each distinct set of
|
|
101
|
+
params counts as one tuned configuration. A compare whose arms differ by more than 1.25x
|
|
102
|
+
in configurations is refused (`refused_by: equal_budget`).
|
|
103
|
+
|
|
104
|
+
### Verdicts
|
|
105
|
+
|
|
106
|
+
| `verdict` | Meaning |
|
|
107
|
+
|---|---|
|
|
108
|
+
| `a_better` / `b_better` | p < alpha (default 0.05) and the difference favours that arm. |
|
|
109
|
+
| `no_detectable_difference` | p ≥ alpha. This is not evidence that the arms are equal; for that, pass `margin`. |
|
|
110
|
+
| `equivalent` | With `margin`: the 95 % CI lies inside ±margin. |
|
|
111
|
+
| `non_inferior` | With `margin`: a is worse than b by less than margin. |
|
|
112
|
+
| `insufficient` | An arm has fewer than 2 seeds, so there is no test. |
|
|
113
|
+
| `refused` | A check found that any verdict would mislead. `refused_by` names the check and `why` says what to fix. |
|
|
114
|
+
|
|
115
|
+
Read these fields next to the verdict:
|
|
116
|
+
|
|
117
|
+
- `prereg.status`:
|
|
118
|
+
- `ok`;
|
|
119
|
+
- `deviation`, with `issues`, e.g. compared before `n_max` seeds, or the plan was registered
|
|
120
|
+
after more than 3 seeds had been seen;
|
|
121
|
+
- `missing`: no plan;
|
|
122
|
+
- `tampered`.
|
|
123
|
+
- `warnings`, e.g. `peeking`.
|
|
124
|
+
- `report()`, before you quote anything.
|
|
125
|
+
|
|
126
|
+
### What keeps a verdict honest
|
|
127
|
+
|
|
128
|
+
| Rule | Why | What you see |
|
|
129
|
+
|---|---|---|
|
|
130
|
+
| Plan first | Choosing the metric or the number of seeds after seeing results inflates false positives. | `prereg.status` is `deviation` or `missing`. Amending a plan needs a reason, and every version is kept. |
|
|
131
|
+
| Look once | `compare` is a fixed-n test. Re-reading its p-value as seeds arrive breaks its alpha. | Each call on grown data is a new "look". From the second look on, the response warns `peeking`, and `report()` marks the pair `not_quotable`. A call on unchanged data is the same look. To stop early validly, use a study. |
|
|
132
|
+
| Equal budgets | A tuned arm against an untuned baseline measures the tuning, not the change. | With more than 1.25x the configurations on one arm, the verdict is `refused` (`refused_by: equal_budget`). |
|
|
133
|
+
| Many comparisons | At alpha 0.05, about 1 in 20 pairs with no real difference still comes out a "win". | `report()` applies Benjamini-Hochberg across all pairs in the project. Each claim is `quotable`, `quotable_with_caveat` or `not_quotable`, and carries a ready `quote`. |
|
|
134
|
+
| Results can go bad | A seed or dataset later found broken undermines verdicts built on it. | `verdicts()` counts verdicts built on results since marked bad in `stale`; `verdicts(recheck=True)` judges each pair again on today's results (`unchanged`, `inputs_changed` or `flipped`). |
|
|
135
|
+
|
|
136
|
+
## Studies: let RunLab run the seeds
|
|
137
|
+
|
|
138
|
+
RunLab can also run the seeds for you.
|
|
139
|
+
It runs both arms one seed pair at a time and stops as soon as an always-valid sequential test
|
|
140
|
+
(mSPRT) decides. It also stops at `n_max`, or at your dollar cap, whichever comes first.
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
rl.plan("resnet-aug", "mixup", "baseline", "acc",
|
|
144
|
+
hypothesis="mixup raises top-1 accuracy", n_max=20) # stopping="msprt", the default
|
|
145
|
+
s = rl.start_study("resnet-aug", command="train", a="mixup", b="baseline", metric="acc",
|
|
146
|
+
max_usd=5, arm_params={"mixup": {"alpha": "0.2"}})
|
|
147
|
+
s = rl.wait_study(s["id"])
|
|
148
|
+
print(s["state"], s["winner"], s["pairs"], s["usd"]) # e.g. decided mixup 6 0.84
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
- `command` names the training command to run. `params` and `arm_params` fill its
|
|
152
|
+
placeholders, using values of letters, digits and `_.:+=-` only.
|
|
153
|
+
- Every check runs before anything starts. A refusal raises `RunLabError` with status 400
|
|
154
|
+
and costs nothing, e.g. an unknown command, no msprt plan, or a cap that is too high.
|
|
155
|
+
- A study never spends more than `max_usd`.
|
|
156
|
+
- `state` is one of:
|
|
157
|
+
|
|
158
|
+
| `state` | Meaning |
|
|
159
|
+
|---|---|
|
|
160
|
+
| `running` | Still running seeds. |
|
|
161
|
+
| `decided` | The sequential test stopped. `decision` is `A` or `B` (`winner` names the arm), or `futile` (no detectable difference at this budget) or `equivalent` (`winner` is null). |
|
|
162
|
+
| `undecided` | `n_max` seeds ran without a decision. |
|
|
163
|
+
| `budget` | The run's compute budget was spent before a decision. |
|
|
164
|
+
| `killed` | Stopped at the dollar cap. |
|
|
165
|
+
| `failed` | The command failed or printed no result; see `detail`. |
|
|
166
|
+
| `lost` | Stopped reporting long past its deadline. |
|
|
167
|
+
| `refused` | Refused at the start (that call raised a 400); listed for the record. |
|
|
168
|
+
|
|
169
|
+
## Errors
|
|
170
|
+
|
|
171
|
+
Every failure raises `RunLabError` with:
|
|
172
|
+
|
|
173
|
+
- `.status`: the HTTP status, or 0 when RunLab cannot be reached;
|
|
174
|
+
- `.message`: RunLab's `error` text;
|
|
175
|
+
- `.request_id`: quote it when reporting a problem.
|
|
176
|
+
|
|
177
|
+
| Status | Meaning |
|
|
178
|
+
|---|---|
|
|
179
|
+
| 400 | Bad input or a refused action. The message says what to fix; an unknown arm, metric or project names the closest match. |
|
|
180
|
+
| 401 | Missing or wrong token. |
|
|
181
|
+
| 403 | Your role cannot do this: writes need `agent` or above. |
|
|
182
|
+
| 404 | No such verdict or study. |
|
|
183
|
+
| 413 | Body over 64 KiB; send results in several calls. |
|
|
184
|
+
|
|
185
|
+
The client retries GET requests 3 times with exponential backoff, on connection errors and on
|
|
186
|
+
429, 502, 503 and 504. It never retries POST requests. Recording the same result line twice is
|
|
187
|
+
harmless anyway.
|
|
188
|
+
|
|
189
|
+
## Client reference
|
|
190
|
+
|
|
191
|
+
`RunLab(url=None, token=None, *, timeout=30, retries=3, backoff_s=0.5)`. Every method returns
|
|
192
|
+
the decoded JSON of one operation in [openapi.json](openapi.json).
|
|
193
|
+
|
|
194
|
+
| Method | Operation | Returns |
|
|
195
|
+
|---|---|---|
|
|
196
|
+
| `me()` | `getMe` | `name`, `role` |
|
|
197
|
+
| `plan(project, a, b, metric, hypothesis, n_max, **options)` | `registerPlan` | `plan`, `power_check`, `text`. Options: `better`, `alpha`, `n_min`, `stopping`, `amend`. |
|
|
198
|
+
| `record(project, results)` | `recordResults` | `stored`, `already` |
|
|
199
|
+
| `result(project, arm, *, seed=None, params=None, **metrics)` | `recordResults` | one line per metric, in one call |
|
|
200
|
+
| `compare(project, a, b, metric, **options)` | `compareArms` | `verdict`, `diff`, `ci`, `p`, `test`, `n`, `prereg`, `warnings`, `text`. Options: `better`, `alpha`, `margin`. |
|
|
201
|
+
| `verdicts(project="", limit=50, recheck=False)` | `listVerdicts` | `verdicts`, `stale`, and `recheck` when asked |
|
|
202
|
+
| `verdict(id)` | `getVerdict` | `seeds`, `check`, `now` |
|
|
203
|
+
| `report(project, *, arm="", metric="", q=0.05)` | `getResearchReport` | `claims`, the counts per status, `text` |
|
|
204
|
+
| `start_study(project, command, a, b, metric, max_usd, *, params=None, arm_params=None)` | `startStudy` | the study |
|
|
205
|
+
| `study(id)`, `studies(project="", limit=50)`, `wait_study(id, timeout, poll_s)` | `getStudy`, `listStudies` | the study, or a list of studies |
|
|
206
|
+
| `result_line(arm, metric, value, *, seed=None, **params)` | none | one `RESULT:` line, for `record()` |
|
|
207
|
+
|
|
208
|
+
## Coding agents
|
|
209
|
+
|
|
210
|
+
[AGENTS.md](AGENTS.md) is the short version for Claude Code, Codex and other agents.
|
|
211
|
+
[llms.txt](llms.txt) indexes this repo for LLM tools.
|
|
212
|
+
|
|
213
|
+
## License
|
|
214
|
+
|
|
215
|
+
MIT ([LICENSE](LICENSE)).
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
runlab_client/__init__.py,sha256=-_Bw_EqM3koYq4FwR4kr9Tn7kzckmwR1AXyIW6IV7jw,10823
|
|
2
|
+
runlab_client-0.1.0.dist-info/licenses/LICENSE,sha256=nL6ZsFKVKQsvSs2_j2q5SIKYqicB2SODeKllhtvJSHk,1069
|
|
3
|
+
runlab_client-0.1.0.dist-info/METADATA,sha256=eHxfnbKnzDveMeClTgFBRNNFQAqNG-Q_ED01ClT2Zk0,11743
|
|
4
|
+
runlab_client-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
5
|
+
runlab_client-0.1.0.dist-info/top_level.txt,sha256=yYFShr7VWb487_pMzH2gZ-mPkdp4NhBHayUKv9JEyEg,14
|
|
6
|
+
runlab_client-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 omer-nachman
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
runlab_client
|