stateweave-sdk 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
stateweave/__init__.py ADDED
@@ -0,0 +1,311 @@
1
+ """StateWeave's client for your own agent's harness. Start a run of a test, connect your agent to the fresh copies of
2
+ the apps the run gets, then hand in its answer: the studio grades the run and shows it, with every call your agent
3
+ made, on its page.
4
+
5
+ import stateweave
6
+
7
+ run = stateweave.start("t-3fa2c1", agent="my-agent@0.3")
8
+ answer = my_agent(run.prompt, run.apps) # your harness, your agent, your code
9
+ result = run.finish(answer)
10
+ print(result.passed, result.reasons, result.url)
11
+
12
+ Every test of a scenario, each run closed even when your agent raises:
13
+
14
+ for test in stateweave.tests(project="p-9c41d2"):
15
+ with stateweave.start(test["id"], agent="my-agent@0.3") as run:
16
+ run.finish(my_agent(run.prompt, run.apps))
17
+
18
+ Besides runs, the client does what the studio's page does with a scenario (its environments), by API key:
19
+ projects(), create_project(), data() and add_data() (data of your own, twin/additions.py), write_tests() (your
20
+ requests, word for word) and suggest_tests() (Claude's ideas), faults() and fault_targets() (what tests may inject),
21
+ job(), recheck().
22
+
23
+ The studio's address and an API key (made on its Runs page) come from Client(url=..., api_key=...), else from
24
+ STATEWEAVE_URL and STATEWEAVE_API_KEY, else from what `sw login` saved (config.py); the address is
25
+ http://127.0.0.1:8700 when none says otherwise. Only the standard library is used. The command line is `sw`
26
+ (python -m stateweave)."""
27
+ from __future__ import annotations
28
+
29
+ import dataclasses
30
+ import json
31
+ import os
32
+ import time
33
+ import urllib.error
34
+ import urllib.parse
35
+ import urllib.request
36
+
37
+ from . import config
38
+
39
+ __all__ = ["Client", "Run", "Result", "StateWeaveError", "RunEnded", "start", "tests"]
40
+
41
+ __version__ = "0.3.0"
42
+
43
+ DEFAULT_URL = "http://127.0.0.1:8700"
44
+ RETRIES = 3 # times a request the studio rate-limited (429) is sent again
45
+
46
+
47
+ class StateWeaveError(Exception):
48
+ """The studio refused a request: `status` is the HTTP status, `body` what it said."""
49
+
50
+ def __init__(self, status: int, body: dict):
51
+ self.status, self.body = status, body
52
+ said = body.get("message") or "; ".join(body.get("problems") or []) or body.get("hint") or body.get("error")
53
+ super().__init__(f"HTTP {status}: {said or body}")
54
+
55
+
56
+ class RunEnded(StateWeaveError):
57
+ """The run had ended already (finished, timed out or cancelled): `run` is how it ended."""
58
+
59
+ @property
60
+ def run(self) -> dict | None:
61
+ return self.body.get("run")
62
+
63
+
64
+ @dataclasses.dataclass
65
+ class Result:
66
+ """How a run ended. status: passed, failed, timed_out, cancelled, interrupted or error (the test's own code
67
+ failed, not your agent)."""
68
+ id: str
69
+ status: str
70
+ passed: bool | None
71
+ reasons: list[str]
72
+ calls: int
73
+ url: str
74
+
75
+
76
+ class Client:
77
+ def __init__(self, url: str | None = None, api_key: str | None = None, timeout: float = 300):
78
+ saved = config.load()
79
+ self.url = (url or os.environ.get("STATEWEAVE_URL") or saved.get("url") or DEFAULT_URL).rstrip("/")
80
+ self.api_key = api_key or os.environ.get("STATEWEAVE_API_KEY") or saved.get("api_key")
81
+ self.timeout = timeout # starting a run serves its apps, and finishing grades it: both take a few seconds
82
+ if not self.api_key:
83
+ raise ValueError("no API key: run `sw login`, or make one on the studio's Runs page and set "
84
+ "STATEWEAVE_API_KEY")
85
+
86
+ # ---------------------------------------------------------------- who and where
87
+ def me(self) -> dict:
88
+ """{"studio": its address, "name": the hosted studio's name or None, "key": {"id", "name"}}."""
89
+ return self.request("GET", "/api/v1/me")
90
+
91
+ # ---------------------------------------------------------------- scenarios (projects) and their data
92
+ def projects(self) -> list[dict]:
93
+ """Every scenario, newest first: {"id", "title", "status", "apps", "envs": {app: {"counts", "summary",
94
+ "state"}}, ...}; no credentials."""
95
+ return self.request("GET", "/api/v1/projects")["projects"]
96
+
97
+ def project(self, project: str) -> dict:
98
+ return self.request("GET", f"/api/v1/projects/{_q(project)}")
99
+
100
+ def create_project(self, description: str, apps: list[str], *, company: str | None = None,
101
+ people: int | None = None, weeks: int | None = None, content: str | None = None) -> dict:
102
+ """A new scenario, generated in the background: follow it with project(id) until its status is ready.
103
+ `content`: "auto" (Claude when the studio has it), "llm" or "templates"."""
104
+ options = {k: v for k, v in (("company", company), ("people", people), ("weeks", weeks),
105
+ ("content", content)) if v is not None}
106
+ return self.request("POST", "/api/v1/projects", {"description": description, "apps": apps,
107
+ "options": options})
108
+
109
+ def data(self, project: str) -> dict:
110
+ """{"kinds": {app: {kind: {"title", "fields", "example"}}}, "additions": [what was added]}."""
111
+ return self.request("GET", f"/api/v1/projects/{_q(project)}/data")
112
+
113
+ def add_data(self, project: str, app: str, kind: str, items: list[dict]) -> dict:
114
+ """Add items of your own (e.g. app "gmail", kind "email"); the app's tests then need a re-check.
115
+ Returns {"addition", "counts", "stale"}; a bad item raises StateWeaveError with every problem."""
116
+ return self.request("POST", f"/api/v1/projects/{_q(project)}/data", {"app": app, "kind": kind,
117
+ "items": items})
118
+
119
+ # ---------------------------------------------------------------- writing tests
120
+ def write_tests(self, project: str, requests: list[str | dict], faults: list[str] | None = None, *,
121
+ noise: dict | None = None, pinned: list[dict] | None = None,
122
+ difficulty: str | None = None) -> dict:
123
+ """One test per request you wrote, kept word for word (a string, or {"text", "criteria"?, "kind"?,
124
+ "apps"?}; criteria: what a right outcome must meet, in your words); returns the job: follow it with
125
+ job(project, id). A request that cannot be a test as written ends "rejected", with why and a wording that
126
+ would work. Faults, one way at most: `faults` (preset ids, from faults()), `noise` (random injection:
127
+ {"rate"?, "kinds"?, "max"?, "max_hang"?, "max_outage"?}, {} for the defaults) or `pinned` (faults at
128
+ endpoints: [{"app", "endpoint", "kind", "nth"?, "seconds"?, "cut"?}], from fault_targets())."""
129
+ return self.request("POST", f"/api/v1/projects/{_q(project)}/tests",
130
+ {"requests": requests, **_faults(faults, noise, pinned),
131
+ **({"difficulty": difficulty} if difficulty else {})})
132
+
133
+ def suggest_tests(self, project: str, count: int, focus: str = "", faults: list[str] | None = None,
134
+ difficulty: str = "normal", *, noise: dict | None = None,
135
+ pinned: list[dict] | None = None) -> dict:
136
+ """`count` tests (1 to 10) from Claude's own ideas, with an optional focus; returns the job. Faults as in
137
+ write_tests()."""
138
+ return self.request("POST", f"/api/v1/projects/{_q(project)}/tests",
139
+ {"count": count, "focus": focus, "difficulty": difficulty,
140
+ **_faults(faults, noise, pinned)})
141
+
142
+ def faults(self, project: str) -> list[dict]:
143
+ """The fault presets the project's tests may inject: {"id", "app", "kind", "title", "observed" (seen on the
144
+ real app, else carried over from another app's), "evidence"}."""
145
+ return self.request("GET", f"/api/v1/projects/{_q(project)}/faults")["presets"]
146
+
147
+ def fault_targets(self, project: str) -> dict:
148
+ """Where faults can be pinned: {"apps": {app: {"title", "endpoints": [...]}}, "kinds": {kind: what it does},
149
+ "noise": {"defaults", "limits"}}."""
150
+ return self.request("GET", f"/api/v1/projects/{_q(project)}/endpoints")
151
+
152
+ def job(self, project: str, job: str) -> dict:
153
+ """A job writing tests: its status and each item's ("pending", "writing", "checking", "ready", "failed",
154
+ "rejected")."""
155
+ return self.request("GET", f"/api/v1/projects/{_q(project)}/jobs/{_q(job)}")
156
+
157
+ def recheck(self, project: str, tests: list[str] | None = None) -> list[str]:
158
+ """Re-check stale tests (all of the project's, or these) on the data as it is now; returns their ids."""
159
+ return self.request("POST", f"/api/v1/projects/{_q(project)}/tests/recheck",
160
+ {"tests": tests} if tests else {})["rechecking"]
161
+
162
+ # ---------------------------------------------------------------- runs
163
+
164
+ def tests(self, project: str | None = None) -> list[dict]:
165
+ """The tests you can run: {"id", "project", "title", "prompt", "kind", "apps", "answer_format", ...}."""
166
+ query = f"?{urllib.parse.urlencode({'project': project})}" if project else ""
167
+ return self.request("GET", f"/api/v1/tests{query}")["tests"]
168
+
169
+ def start(self, test: str, *, agent: str, project: str | None = None, timeout: float | None = None,
170
+ meta: dict | None = None) -> Run:
171
+ """Start a run of `test` for `agent` (a label your runs are compared by, e.g. "my-agent@0.3"). `timeout`:
172
+ seconds before the studio ends it unfinished (30 minutes when unset); `meta`: anything you want shown with
173
+ it (model, prompt version, commit)."""
174
+ body = {"test": test, "agent": agent, "project": project, "timeout": timeout, "meta": meta}
175
+ return Run(self, self.request("POST", "/api/v1/runs", {k: v for k, v in body.items() if v is not None}))
176
+
177
+ def run(self, run_id: str) -> Run:
178
+ """A run you started, as it stands now."""
179
+ return Run(self, self.request("GET", f"/api/v1/runs/{run_id}"))
180
+
181
+ def next_run(self, worker: str, *, agent: str, meta: dict | None = None, timeout: float | None = None,
182
+ wait: float = 25) -> Run | None:
183
+ """As a worker (`worker`: an id of your own, the same on every call), wait up to `wait` seconds for a run
184
+ someone asked for on the studio's pages (Run again); it comes started, labelled `agent`, like start()'s. None
185
+ when none came. Calling it is what keeps the worker listed as connected there."""
186
+ body = {"worker": worker, "agent": agent, "meta": meta, "timeout": timeout, "wait": wait}
187
+ data = self.request("POST", "/api/v1/workers/next", {k: v for k, v in body.items() if v is not None})
188
+ return Run(self, data) if data.get("id") else None
189
+
190
+ def leave(self, worker: str) -> None:
191
+ """The worker stops: the studio no longer lists it or queues runs for it."""
192
+ self.request("POST", f"/api/v1/workers/{_q(worker)}/leave")
193
+
194
+ def request(self, method: str, path: str, body: dict | None = None) -> dict:
195
+ data = None if body is None else json.dumps(body).encode()
196
+ req = urllib.request.Request(self.url + path, data=data, method=method,
197
+ headers={"Authorization": f"Bearer {self.api_key}",
198
+ "Content-Type": "application/json", "Accept": "application/json"})
199
+ for attempt in range(RETRIES + 1):
200
+ try:
201
+ with urllib.request.urlopen(req, timeout=self.timeout) as res:
202
+ return json.loads(res.read() or b"{}")
203
+ except urllib.error.HTTPError as e:
204
+ try:
205
+ said = json.loads(e.read() or b"{}")
206
+ except ValueError:
207
+ said = {"error": e.reason}
208
+ if e.code == 429 and said.get("error") == "rate_limited" and attempt < RETRIES:
209
+ # a hosted studio's per-key rate limit, not a limit of the workspace's: wait as it says
210
+ time.sleep(min(float(e.headers.get("retry-after") or 1), 10))
211
+ continue
212
+ raise (RunEnded if e.code == 409 and said.get("run") else StateWeaveError)(e.code, said) from None
213
+ except urllib.error.URLError as e: # no answer at all: a wrong address, or the studio is down
214
+ raise ConnectionError(f"cannot reach {self.url}: {getattr(e.reason, 'strerror', None) or e.reason}") \
215
+ from None
216
+ raise AssertionError("unreachable") # the loop returns or raises
217
+
218
+
219
+ class Run:
220
+ """A run of a test. While it runs: `prompt` (what to ask your agent), `kind` ("answer": the agent answers;
221
+ "action": it changes the apps), `answer_format`, `persona` (whom the agent acts as) and `apps`: per app its
222
+ base_url, endpoints, credentials and the MCP server those credentials open ({"url", "headers"}, or None)."""
223
+
224
+ def __init__(self, client: Client, data: dict):
225
+ self.client = client
226
+ self._set(data)
227
+
228
+ def _set(self, data: dict) -> None:
229
+ self.data = data
230
+ self.id: str = data["id"]
231
+ self.test: str = data["test"]
232
+ self.status: str = data["status"]
233
+ self.url: str = data.get("url", "")
234
+ self.prompt: str = data.get("prompt", "")
235
+ self.kind: str = data.get("kind", "")
236
+ self.answer_format: str = data.get("answer_format", "")
237
+ self.persona: dict = data.get("persona") or {}
238
+ self.apps: dict = data.get("apps") or {}
239
+ self.expires_at: float | None = data.get("expires_at")
240
+
241
+ def credential(self, app: str, kind: str = "user") -> str | None:
242
+ """The first credential of `app` whose label starts with `kind` ("user", "bot", "app", "mcp", "setup")."""
243
+ creds = self.apps.get(app, {}).get("credentials", {})
244
+ return next((v for label, v in creds.items() if label.split(":", 1)[0] == kind), None)
245
+
246
+ def mcp_servers(self) -> dict:
247
+ """The apps' MCP servers as Claude Code and the Claude Agent SDK take them: {app: {"type": "http", "url",
248
+ "headers"}}."""
249
+ return {app: {"type": "http", **a["mcp"]} for app, a in self.apps.items() if a.get("mcp")}
250
+
251
+ def finish(self, answer=None) -> Result:
252
+ """Hand in the agent's answer (the JSON value answer_format describes; nothing for an action test): the
253
+ studio grades the run on what the agent left in the apps and the answer."""
254
+ data = self.client.request("POST", f"/api/v1/runs/{self.id}/finish", {"answer": answer})
255
+ self._set(data)
256
+ return self.result()
257
+
258
+ def cancel(self) -> Result:
259
+ """Give the run up: its apps stop, it is not graded."""
260
+ self._set(self.client.request("POST", f"/api/v1/runs/{self.id}/cancel"))
261
+ return self.result()
262
+
263
+ def result(self) -> Result:
264
+ d = self.data
265
+ return Result(d["id"], d["status"], d.get("passed"), list(d.get("reasons") or []), d.get("calls") or 0,
266
+ self.url)
267
+
268
+ def __enter__(self) -> Run:
269
+ return self
270
+
271
+ def __exit__(self, *exc) -> None:
272
+ if self.status == "running": # left without finishing, or the agent raised: give it up, keep what it did
273
+ try:
274
+ self.cancel()
275
+ except RunEnded:
276
+ pass
277
+
278
+ def __repr__(self) -> str:
279
+ return f"<Run {self.id} {self.test} {self.status}>"
280
+
281
+
282
+ def _q(part: str) -> str:
283
+ return urllib.parse.quote(part, safe="")
284
+
285
+
286
+ def _faults(faults: list[str] | None, noise: dict | None, pinned: list[dict] | None) -> dict:
287
+ if sum(x is not None and x != [] for x in (faults, noise, pinned)) > 1:
288
+ raise ValueError("faults, noise and pinned: one of them at most")
289
+ return {"faults": faults} if faults else {"noise": noise} if noise is not None else \
290
+ {"pinned": pinned} if pinned else {}
291
+
292
+
293
+ _default: Client | None = None
294
+
295
+
296
+ def _client() -> Client:
297
+ global _default
298
+ if _default is None:
299
+ _default = Client()
300
+ return _default
301
+
302
+
303
+ def start(test: str, *, agent: str, project: str | None = None, timeout: float | None = None,
304
+ meta: dict | None = None) -> Run:
305
+ """Client().start(...) with the address and key from the environment."""
306
+ return _client().start(test, agent=agent, project=project, timeout=timeout, meta=meta)
307
+
308
+
309
+ def tests(project: str | None = None) -> list[dict]:
310
+ """Client().tests(...) with the address and key from the environment."""
311
+ return _client().tests(project)
stateweave/__main__.py ADDED
@@ -0,0 +1,537 @@
1
+ """`sw`, StateWeave from a shell (python -m stateweave works too).
2
+
3
+ sw login [--url URL] [--key sw_…] check the key with the studio, then keep both (config.py)
4
+ sw whoami | sw logout | sw --version
5
+
6
+ sw env ls | sw env show ENV
7
+ sw env new --apps gmail,slack --describe "…" [--company NAME] [--people N] [--weeks N] [--templates] [--no-wait]
8
+
9
+ sw data kinds ENV what each app of ENV takes, field by field
10
+ sw data add ENV APP KIND --FIELD VALUE … one item, e.g. sw data add p-1 gmail email --from dana@x.com
11
+ --subject "Credit?" --body-file mail.txt (--X-file reads X from a file)
12
+ sw data add ENV APP KIND --file items.json a JSON list of items
13
+
14
+ sw task ls [--env ENV]
15
+ sw task new ENV --request "…" [--criteria "…"] | --request … --request … | --file requests.txt
16
+ | --suggest N [--focus "…"] [--hard]
17
+ [--faults ID,ID | --random-faults [RATE]] [--no-wait]
18
+ sw task faults ENV the fault presets ENV's tests may inject
19
+ sw task recheck ENV [TASK …]
20
+
21
+ sw run TASK --agent LABEL [--env ENV] [--timeout S] [--json] -- COMMAND …
22
+ sw worker --agent LABEL [--timeout S] [--once] [--json] -- COMMAND …
23
+ sw mcp-config RUN a live run's MCP servers, as Claude Code's --mcp-config takes them
24
+
25
+ sw tests | start | finish | cancel the run endpoints one call at a time, as JSON
26
+
27
+ `sw run` starts a run of TASK, runs COMMAND (your agent, in any language) with the run in its environment, then
28
+ hands in the answer and prints the result:
29
+
30
+ STATEWEAVE_RUN the run's id
31
+ STATEWEAVE_PROMPT what to ask your agent
32
+ STATEWEAVE_KIND answer (it finds something out) or action (it changes the apps)
33
+ STATEWEAVE_ANSWER_FORMAT for an answer task, the JSON value to answer with
34
+ STATEWEAVE_RUN_FILE the whole run as JSON: per app its base_url, endpoints, credentials, MCP server
35
+ STATEWEAVE_MCP_CONFIG {"mcpServers": {...}}: claude -p "$STATEWEAVE_PROMPT" --mcp-config "$STATEWEAVE_MCP_CONFIG"
36
+ STATEWEAVE_ANSWER_FILE where to write the answer (JSON, or plain text); nothing for an action task
37
+
38
+ The run is graded when COMMAND exits, whatever its exit code; Ctrl-C gives the run up instead. `sw run` exits 0
39
+ when the run passed, 1 when it did not.
40
+
41
+ `sw worker` waits for the runs someone asks for on the studio's pages (Run again on a run's page) and drives each the
42
+ same way, as LABEL, until Ctrl-C. COMMAND starts afresh for every run, so a change to your agent counts from the next
43
+ run on, without restarting the worker. The runs show the command, this machine's name and, in a git checkout, the
44
+ commit the worker runs at."""
45
+ from __future__ import annotations
46
+
47
+ import argparse
48
+ import dataclasses
49
+ import getpass
50
+ import json
51
+ import os
52
+ import pathlib
53
+ import secrets
54
+ import shutil
55
+ import socket
56
+ import subprocess
57
+ import sys
58
+ import tempfile
59
+ import time
60
+
61
+ from . import DEFAULT_URL, Client, RunEnded, StateWeaveError, __version__, config
62
+
63
+ POLL = 2.0 # seconds between looks at something the studio does in the background
64
+ AWAY = 5.0 # seconds a worker waits before asking again when the studio cannot be reached
65
+
66
+
67
+ def main(argv=None) -> int:
68
+ argv = list(sys.argv[1:] if argv is None else argv)
69
+ command = []
70
+ if "--" in argv: # sw run TASK … -- the agent's own command line
71
+ cut = argv.index("--")
72
+ argv, command = argv[:cut], argv[cut + 1:]
73
+ parser = _parser()
74
+ a, extra = parser.parse_known_args(argv)
75
+ if extra and not (a.command == "data" and a.what == "add"):
76
+ parser.error(f"unrecognized arguments: {' '.join(extra)}")
77
+ a.extra, a.agent_command = extra, command
78
+ try:
79
+ return COMMANDS[a.command](a)
80
+ except StateWeaveError as e:
81
+ said, problems = e.body.get("message"), e.body.get("problems") or []
82
+ # the problems one to a line, each once (the error's own text is the first of them when there is no message)
83
+ print(f"sw: {e}" if said or not problems else f"sw: the studio refused it (HTTP {e.status}):", file=sys.stderr)
84
+ for problem in problems:
85
+ if problem != said:
86
+ print(f" {problem}", file=sys.stderr)
87
+ return 1
88
+ except (ValueError, OSError) as e:
89
+ print(f"sw: {e}", file=sys.stderr)
90
+ return 1
91
+ except KeyboardInterrupt:
92
+ return 130
93
+
94
+
95
+ def _parser() -> argparse.ArgumentParser:
96
+ p = argparse.ArgumentParser(prog="sw", description="StateWeave: environments, tests and runs of your own agent",
97
+ epilog="Documentation: https://stateweave.net/docs/")
98
+ p.add_argument("--version", action="version", version=f"sw {__version__}")
99
+ sub = p.add_subparsers(dest="command", required=True)
100
+
101
+ login = sub.add_parser("login", help="keep the studio's address and your API key")
102
+ login.add_argument("--url", help=f"the studio's address (default: the one kept, else {DEFAULT_URL})")
103
+ login.add_argument("--key", help="the API key (default: asked for, not echoed)")
104
+ sub.add_parser("logout", help="forget them")
105
+ sub.add_parser("whoami", help="which studio and key")
106
+
107
+ env = sub.add_parser("env", help="scenarios: their environments")
108
+ env_sub = env.add_subparsers(dest="what", required=True)
109
+ env_sub.add_parser("ls", help="every scenario")
110
+ show = env_sub.add_parser("show", help="one scenario")
111
+ show.add_argument("env")
112
+ new = env_sub.add_parser("new", help="generate a scenario")
113
+ new.add_argument("--apps", required=True, help="comma-separated, e.g. gmail,slack")
114
+ new.add_argument("--describe", required=True, help="the company and what the agent will be tested on")
115
+ new.add_argument("--company")
116
+ new.add_argument("--people", type=int)
117
+ new.add_argument("--weeks", type=int)
118
+ new.add_argument("--templates", action="store_true", help="templates only, no Claude (quick, plainer text)")
119
+ new.add_argument("--no-wait", action="store_true", help="print its id and return while it generates")
120
+
121
+ data = sub.add_parser("data", help="data of your own in a scenario's apps")
122
+ data_sub = data.add_subparsers(dest="what", required=True)
123
+ kinds = data_sub.add_parser("kinds", help="what each app takes")
124
+ kinds.add_argument("env")
125
+ add = data_sub.add_parser("add", help="add items (--FIELD VALUE …, or --file)", allow_abbrev=False)
126
+ add.add_argument("env")
127
+ add.add_argument("app")
128
+ add.add_argument("kind")
129
+ add.add_argument("--file", help="a JSON list of items (or one item)")
130
+
131
+ task = sub.add_parser("task", help="tests")
132
+ task_sub = task.add_subparsers(dest="what", required=True)
133
+ ls = task_sub.add_parser("ls", help="the tests you can run")
134
+ ls.add_argument("--env")
135
+ tnew = task_sub.add_parser("new", help="write tests: yours, word for word, or Claude's")
136
+ tnew.add_argument("env")
137
+ tnew.add_argument("--request", action="append", help="a request, as the agent will be asked it (repeatable)")
138
+ tnew.add_argument("--criteria", help="with one --request: what a right outcome must meet, in your words")
139
+ tnew.add_argument("--file", help="requests, a blank line between them")
140
+ tnew.add_argument("--suggest", type=int, metavar="N", help="instead: N tests from Claude's own ideas")
141
+ tnew.add_argument("--focus", default="", help="with --suggest: what they should be about")
142
+ tnew.add_argument("--hard", action="store_true", help="with --suggest: hard tests")
143
+ tnew.add_argument("--faults", help="comma-separated fault presets the tests may inject (sw task faults ENV)")
144
+ tnew.add_argument("--random-faults", nargs="?", const=-1.0, type=float, metavar="RATE",
145
+ help="instead: faults injected at random into every app the tests use, on about RATE of the "
146
+ "calls (default 0.05)")
147
+ tnew.add_argument("--no-wait", action="store_true", help="print the job's id and return while it writes")
148
+ faults = task_sub.add_parser("faults", help="the fault presets a scenario's tests may inject")
149
+ faults.add_argument("env")
150
+ recheck = task_sub.add_parser("recheck", help="re-check tests whose data changed")
151
+ recheck.add_argument("env")
152
+ recheck.add_argument("tasks", nargs="*")
153
+
154
+ run = sub.add_parser("run", help="run your agent on a test: sw run TASK --agent LABEL -- COMMAND …")
155
+ run.add_argument("task")
156
+ run.add_argument("--agent", required=True, help="the label runs are compared by, e.g. my-agent@0.3")
157
+ run.add_argument("--env", help="the scenario, when the task id is in several")
158
+ run.add_argument("--timeout", type=float, help="seconds before the studio ends it unfinished (default 1800)")
159
+ run.add_argument("--json", action="store_true", help="print the result as JSON")
160
+ worker = sub.add_parser("worker", help="drive the runs asked for on the studio's pages: sw worker --agent LABEL "
161
+ "-- COMMAND …")
162
+ worker.add_argument("--agent", required=True, help="the label its runs are compared by, e.g. my-agent@0.4")
163
+ worker.add_argument("--timeout", type=float, help="seconds before the studio ends a run unfinished (default: the "
164
+ "time limit of the run it repeats)")
165
+ worker.add_argument("--once", action="store_true", help="drive one run, then stop")
166
+ worker.add_argument("--json", action="store_true", help="print each result as JSON")
167
+ mcp = sub.add_parser("mcp-config", help="a live run's MCP servers")
168
+ mcp.add_argument("run")
169
+
170
+ t = sub.add_parser("tests", help="the tests you can run, one per line")
171
+ t.add_argument("--project")
172
+ s = sub.add_parser("start", help="start a run; prints the request and the apps to connect to, as JSON")
173
+ s.add_argument("test")
174
+ s.add_argument("--agent", required=True, help="the label your runs are compared by, e.g. my-agent@0.3")
175
+ s.add_argument("--project")
176
+ s.add_argument("--timeout", type=float, help="seconds before the studio ends it unfinished (default 1800)")
177
+ f = sub.add_parser("finish", help="hand in the answer; prints the result")
178
+ f.add_argument("run")
179
+ f.add_argument("--answer", help="the answer as JSON (nothing for an action test)")
180
+ c = sub.add_parser("cancel", help="give a run up")
181
+ c.add_argument("run")
182
+ return p
183
+
184
+
185
+ # ------------------------------------------------------------------ login
186
+ def cmd_login(a) -> int:
187
+ url = (a.url or os.environ.get("STATEWEAVE_URL") or config.load().get("url") or DEFAULT_URL).rstrip("/")
188
+ key = a.key or getpass.getpass(f"API key for {url} (made on the studio's Runs page): ").strip()
189
+ try:
190
+ me = Client(url=url, api_key=key).me() # a wrong key or address fails here, and nothing is kept
191
+ except StateWeaveError as e:
192
+ if e.status == 401:
193
+ raise ValueError(f"{url} did not accept that key: make one on its Runs page") from None
194
+ raise
195
+ path = config.save(url, key)
196
+ print(f"Signed in to {me.get('name') or me['studio']} as key “{me['key']['name']}”. Kept in {path}.")
197
+ return 0
198
+
199
+
200
+ def cmd_logout(a) -> int:
201
+ print("Forgot the address and key." if config.clear() else "Nothing was kept.")
202
+ return 0
203
+
204
+
205
+ def cmd_whoami(a) -> int:
206
+ client = Client()
207
+ me = client.me()
208
+ print(f"{client.url} key “{me['key']['name']}” ({me['key']['id']})")
209
+ return 0
210
+
211
+
212
+ # ------------------------------------------------------------------ scenarios
213
+ def cmd_env(a) -> int:
214
+ client = Client()
215
+ if a.what == "ls":
216
+ _table([(p["id"], p["status"], ",".join(p["apps"]), p["title"]) for p in client.projects()])
217
+ return 0
218
+ if a.what == "show":
219
+ _show_project(client.project(a.env))
220
+ return 0
221
+ made = client.create_project(a.describe, [x.strip() for x in a.apps.split(",") if x.strip()],
222
+ company=a.company, people=a.people, weeks=a.weeks,
223
+ content="templates" if a.templates else None)
224
+ print(made["id"], flush=True)
225
+ if a.no_wait:
226
+ return 0
227
+ project = _follow_project(client, made["id"])
228
+ _show_project(project)
229
+ return 0 if project["status"] in ("ready", "partial") else 1
230
+
231
+
232
+ def _follow_project(client: Client, pid: str) -> dict:
233
+ said: dict[str, str] = {}
234
+ while True:
235
+ project = client.project(pid)
236
+ for step in project["steps"]:
237
+ line = f"{step['title']}: {step['status']}" + (f" ({step['detail']})" if step.get("detail") else "")
238
+ if said.get(step["key"]) != step["status"]:
239
+ said[step["key"]] = step["status"]
240
+ print(f" {line}", file=sys.stderr, flush=True)
241
+ if project["status"] != "generating":
242
+ return project
243
+ time.sleep(POLL)
244
+
245
+
246
+ def _show_project(p: dict) -> None:
247
+ print(f"{p['id']} {p['title']} ({p['status']}{': ' + p['error'] if p.get('error') else ''})")
248
+ _table([(app, env["state"], ", ".join(f"{n} {k}" for k, n in env["counts"].items()))
249
+ for app, env in p["envs"].items()], indent=" ")
250
+
251
+
252
+ def _table(rows: list[tuple], indent: str = "") -> None:
253
+ """Rows in columns as wide as their widest cell; the last column is left as it is."""
254
+ widths = [max(len(str(row[i])) for row in rows) for i in range(len(rows[0]) - 1)] if rows else []
255
+ for row in rows:
256
+ print(indent + " ".join([str(c).ljust(w) for c, w in zip(row, widths)] + [str(row[-1])]))
257
+
258
+
259
+ # ------------------------------------------------------------------ data
260
+ def cmd_data(a) -> int:
261
+ client = Client()
262
+ kinds = client.data(a.env)["kinds"]
263
+ if a.what == "kinds":
264
+ for app, offered in kinds.items():
265
+ for kind, spec in offered.items():
266
+ print(f"{app} {kind}: {spec['title']}")
267
+ for name, f in spec["fields"].items():
268
+ print(f" --{name.replace('_', '-'):<20} {'' if f['required'] else '(optional) '}{f['help']}")
269
+ return 0
270
+ spec = kinds.get(a.app, {}).get(a.kind)
271
+ if a.file:
272
+ if a.extra:
273
+ raise ValueError("give the items in --file or as --FIELD VALUE, not both")
274
+ items = json.loads(pathlib.Path(a.file).read_text())
275
+ items = items if isinstance(items, list) else [items]
276
+ else:
277
+ items = [_item(a.extra, spec)]
278
+ out = client.add_data(a.env, a.app, a.kind, items)
279
+ stale = out.get("stale") or 0
280
+ print(f"Added {out['addition']['count']} {a.kind}(s) to {a.app}" +
281
+ (f"; {stale} test(s) need a re-check: sw task recheck {a.env}" if stale else "") + ".")
282
+ return 0
283
+
284
+
285
+ def _item(tokens: list[str], spec: dict | None) -> dict:
286
+ """--FIELD VALUE pairs as one item; --FIELD-file PATH reads the value from a file. Lists (people, labels) are
287
+ comma-separated or JSON; objects are JSON. Field names may use - for _."""
288
+ fields = (spec or {}).get("fields", {})
289
+ item: dict = {}
290
+ if len(tokens) % 2:
291
+ raise ValueError(f"every --FIELD needs a value: {' '.join(tokens)}")
292
+ for flag, value in zip(tokens[::2], tokens[1::2]):
293
+ if not flag.startswith("--"):
294
+ raise ValueError(f"expected --FIELD VALUE, got {flag!r}")
295
+ name = flag[2:].replace("-", "_")
296
+ if name.endswith("_file") and name not in fields: # --body-file mail.txt: the body, read from the file
297
+ name, value = name[:-5], pathlib.Path(value).read_text()
298
+ kind = fields.get(name, {}).get("type")
299
+ if kind in ("people", "list") and not value.lstrip().startswith("["):
300
+ item[name] = [v.strip() for v in value.split(",") if v.strip()]
301
+ elif kind in ("people", "list", "object") or value.lstrip()[:1] in ("[", "{") and kind is None:
302
+ item[name] = json.loads(value)
303
+ else:
304
+ item[name] = value
305
+ return item
306
+
307
+
308
+ # ------------------------------------------------------------------ tests
309
+ def cmd_task(a) -> int:
310
+ client = Client()
311
+ if a.what == "ls":
312
+ return cmd_tests(argparse.Namespace(project=a.env))
313
+ if a.what == "recheck":
314
+ ids = client.recheck(a.env, a.tasks or None)
315
+ print(f"Re-checking {len(ids)} test(s): {' '.join(ids)}" if ids else "No test needs a re-check.")
316
+ return 0
317
+ if a.what == "faults":
318
+ _table([(f["id"], f["title"], "seen live" if f["observed"] else "carried over (未验证)")
319
+ for f in client.faults(a.env)])
320
+ return 0
321
+ faults = [f.strip() for f in (a.faults or "").split(",") if f.strip()] or None
322
+ if faults and a.random_faults is not None:
323
+ raise ValueError("give --faults or --random-faults, not both")
324
+ noise = None if a.random_faults is None else {} if a.random_faults < 0 else {"rate": a.random_faults}
325
+ requests: list = list(a.request or [])
326
+ if a.file:
327
+ requests += [r.strip() for r in pathlib.Path(a.file).read_text().split("\n\n") if r.strip()]
328
+ if a.suggest and requests:
329
+ raise ValueError("give requests (--request, --file) or --suggest N, not both")
330
+ if a.criteria:
331
+ if len(requests) != 1:
332
+ raise ValueError("--criteria goes with one request: give a single --request")
333
+ requests = [{"text": requests[0], "criteria": a.criteria}]
334
+ if a.suggest:
335
+ job = client.suggest_tests(a.env, a.suggest, a.focus, faults, "hard" if a.hard else "normal", noise=noise)
336
+ elif requests:
337
+ job = client.write_tests(a.env, requests, faults, noise=noise)
338
+ else:
339
+ raise ValueError("give the requests (--request \"…\", or --file), or --suggest N")
340
+ print(job["id"], flush=True)
341
+ if a.no_wait:
342
+ return 0
343
+ job = _follow_job(client, a.env, job["id"])
344
+ for item in job["items"]:
345
+ if item["status"] == "ready":
346
+ print(f"{item['test_id']} kept {item['title']}")
347
+ elif item["status"] == "rejected":
348
+ print(f"{'-':<9} not a test {item.get('request') or item['title']}\n"
349
+ f"{'':<11}why: {item.get('detail')}\n{'':<11}try: {item.get('suggestion')}")
350
+ else:
351
+ print(f"{'-':<9} dropped {item['title']}: {item.get('detail')}")
352
+ return 0 if any(i["status"] == "ready" for i in job["items"]) else 1
353
+
354
+
355
+ def _follow_job(client: Client, project: str, jid: str) -> dict:
356
+ said: dict[int, str] = {}
357
+ while True:
358
+ job = client.job(project, jid)
359
+ for i, item in enumerate(job["items"]):
360
+ if said.get(i) != item["status"]:
361
+ said[i] = item["status"]
362
+ print(f" [{i + 1}/{len(job['items'])}] {item['status']}: {item['title']}", file=sys.stderr,
363
+ flush=True)
364
+ if job["status"] != "running":
365
+ if job.get("error") and not job["items"]:
366
+ raise ValueError(job["error"])
367
+ return job
368
+ time.sleep(POLL)
369
+
370
+
371
+ # ------------------------------------------------------------------ runs
372
+ def cmd_run(a) -> int:
373
+ if not a.agent_command:
374
+ raise ValueError("give your agent's command after --, e.g. sw run t-1a2b3c --agent mine@1 -- python agent.py")
375
+ _runnable(a.agent_command)
376
+ client = Client()
377
+ run = client.start(a.task, agent=a.agent, project=a.env, timeout=a.timeout,
378
+ meta={"command": " ".join(a.agent_command)[:200]})
379
+ print(f"sw: run {run.id} of {run.test} started: {run.url}", file=sys.stderr, flush=True)
380
+ try:
381
+ answer, code = _drive(run, a.agent_command)
382
+ except KeyboardInterrupt:
383
+ return 130
384
+ result = run.finish(answer)
385
+ _report(result, code, a.json)
386
+ return 0 if result.passed else 1
387
+
388
+
389
+ def cmd_worker(a) -> int:
390
+ if not a.agent_command:
391
+ raise ValueError("give your agent's command after --, e.g. sw worker --agent mine@1 -- python agent.py")
392
+ _runnable(a.agent_command)
393
+ client = Client()
394
+ worker, host = f"w-{secrets.token_hex(6)}", socket.gethostname()
395
+ print(f"sw: {a.agent} on {host} is waiting for runs: press Run again on a run's page ({client.url}/#/runs). "
396
+ "Ctrl-C stops.", file=sys.stderr, flush=True)
397
+ away = False
398
+ try:
399
+ while True:
400
+ try:
401
+ run = client.next_run(worker, agent=a.agent, meta=_worker_meta(a.agent_command, host),
402
+ timeout=a.timeout)
403
+ except OSError as e: # the studio is down or restarting: keep asking
404
+ if not away:
405
+ said = e if isinstance(e, ConnectionError) else f"cannot reach {client.url} ({e})"
406
+ print(f"sw: {said}; trying again", file=sys.stderr, flush=True)
407
+ away = True
408
+ time.sleep(AWAY)
409
+ continue
410
+ except StateWeaveError as e:
411
+ if e.status < 500:
412
+ raise
413
+ time.sleep(AWAY)
414
+ continue
415
+ if away:
416
+ print("sw: reached the studio again", file=sys.stderr, flush=True)
417
+ away = False
418
+ if run is None:
419
+ continue
420
+ print(f"sw: run {run.id} of {run.test}: {run.url}", file=sys.stderr, flush=True)
421
+ answer, code = _drive(run, a.agent_command)
422
+ try:
423
+ result = run.finish(answer)
424
+ except RunEnded as e: # cancelled on the page, or out of time, while the agent worked
425
+ print(f"sw: run {run.id} had ended ({(e.run or {}).get('status', 'ended')})", file=sys.stderr)
426
+ continue
427
+ _report(result, code, a.json)
428
+ if a.once:
429
+ return 0 if result.passed else 1
430
+ except KeyboardInterrupt:
431
+ return 130
432
+ finally:
433
+ try:
434
+ client.leave(worker)
435
+ except (StateWeaveError, OSError):
436
+ pass
437
+
438
+
439
+ def _runnable(command: list[str]) -> None:
440
+ """Refuse, before a run starts, a command that cannot start: a run would be made only to be given up."""
441
+ if shutil.which(command[0]) is None:
442
+ raise ValueError(f"could not start {command[0]}: no such command, or not executable")
443
+
444
+
445
+ def _drive(run, command: list[str]) -> tuple[object, int]:
446
+ """Run COMMAND, the agent, with `run` in its environment (above); returns the answer it wrote and its exit code.
447
+ Ctrl-C, or a command that does not start, gives the run up."""
448
+ with tempfile.TemporaryDirectory(prefix="sw-run-") as folder:
449
+ folder = pathlib.Path(folder)
450
+ (folder / "run.json").write_text(json.dumps(run.data, indent=1))
451
+ (folder / "mcp.json").write_text(json.dumps({"mcpServers": run.mcp_servers()}, indent=1))
452
+ answer_file = folder / "answer.json"
453
+ env = {**os.environ, "STATEWEAVE_RUN": run.id, "STATEWEAVE_PROMPT": run.prompt, "STATEWEAVE_KIND": run.kind,
454
+ "STATEWEAVE_ANSWER_FORMAT": run.answer_format, "STATEWEAVE_RUN_FILE": str(folder / "run.json"),
455
+ "STATEWEAVE_MCP_CONFIG": str(folder / "mcp.json"), "STATEWEAVE_ANSWER_FILE": str(answer_file)}
456
+ try:
457
+ code = subprocess.call(command, env=env)
458
+ except KeyboardInterrupt:
459
+ run.cancel()
460
+ print(f"sw: gave run {run.id} up", file=sys.stderr)
461
+ raise
462
+ except OSError as e:
463
+ run.cancel()
464
+ raise ValueError(f"could not start {command[0]}: {e.strerror or e}") from None
465
+ answer = None
466
+ if answer_file.exists():
467
+ text = answer_file.read_text().strip()
468
+ try:
469
+ answer = json.loads(text) if text else None
470
+ except ValueError:
471
+ answer = text # plain text: handed in as a string
472
+ return answer, code
473
+
474
+
475
+ def _report(result, code: int, as_json: bool) -> None:
476
+ if as_json:
477
+ print(json.dumps({**dataclasses.asdict(result), "agent_exit": code}, indent=2), flush=True)
478
+ return
479
+ print(f"{'passed' if result.passed else result.status} {result.calls} calls {result.url}")
480
+ for reason in result.reasons:
481
+ print(f" {reason}")
482
+ if code:
483
+ print(f" (your agent exited with {code})")
484
+ sys.stdout.flush()
485
+
486
+
487
+ def _worker_meta(command: list[str], host: str) -> dict:
488
+ """What a worker's runs show with them: the command, this machine, and the commit of the git checkout it runs
489
+ in (with a note when it has changes not committed yet), so two versions of an agent can be told apart."""
490
+ meta = {"command": " ".join(command)[:200], "host": host}
491
+ try:
492
+ head = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, timeout=5)
493
+ if head.returncode == 0:
494
+ changed = subprocess.run(["git", "status", "--porcelain", "--untracked-files=no"], capture_output=True,
495
+ text=True, timeout=5).stdout.strip()
496
+ meta["commit"] = head.stdout.strip() + (" + uncommitted changes" if changed else "")
497
+ except (OSError, subprocess.SubprocessError):
498
+ pass
499
+ return meta
500
+
501
+
502
+ def cmd_mcp_config(a) -> int:
503
+ run = Client().run(a.run)
504
+ if run.status != "running":
505
+ raise ValueError(f"run {run.id} has ended ({run.status}): its apps are gone")
506
+ print(json.dumps({"mcpServers": run.mcp_servers()}, indent=2))
507
+ return 0
508
+
509
+
510
+ def cmd_tests(a) -> int:
511
+ _table([(t["id"], t["kind"], ",".join(t["apps"]), t["title"]) for t in Client().tests(a.project)])
512
+ return 0
513
+
514
+
515
+ def cmd_start(a) -> int:
516
+ print(json.dumps(Client().start(a.test, agent=a.agent, project=a.project, timeout=a.timeout).data, indent=2))
517
+ return 0
518
+
519
+
520
+ def cmd_finish(a) -> int:
521
+ answer = json.loads(a.answer) if a.answer is not None else None
522
+ print(json.dumps(dataclasses.asdict(Client().run(a.run).finish(answer)), indent=2))
523
+ return 0
524
+
525
+
526
+ def cmd_cancel(a) -> int:
527
+ print(json.dumps(dataclasses.asdict(Client().run(a.run).cancel()), indent=2))
528
+ return 0
529
+
530
+
531
+ COMMANDS = {"login": cmd_login, "logout": cmd_logout, "whoami": cmd_whoami, "env": cmd_env, "data": cmd_data,
532
+ "task": cmd_task, "run": cmd_run, "worker": cmd_worker, "mcp-config": cmd_mcp_config, "tests": cmd_tests,
533
+ "start": cmd_start,
534
+ "finish": cmd_finish, "cancel": cmd_cancel}
535
+
536
+ if __name__ == "__main__":
537
+ sys.exit(main())
stateweave/config.py ADDED
@@ -0,0 +1,45 @@
1
+ """Where `sw login` keeps the studio's address and your API key: ~/.config/stateweave/config.json (or
2
+ $XDG_CONFIG_HOME/stateweave/config.json, or the file $STATEWEAVE_CONFIG names), readable by you alone.
3
+
4
+ {"url": "https://acme.stateweave.net", "api_key": "sw_…"}
5
+
6
+ STATEWEAVE_URL and STATEWEAVE_API_KEY, when set, win over the file, and Client(url=..., api_key=...) over both."""
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import os
11
+ import pathlib
12
+
13
+
14
+ def path() -> pathlib.Path:
15
+ if os.environ.get("STATEWEAVE_CONFIG"):
16
+ return pathlib.Path(os.environ["STATEWEAVE_CONFIG"])
17
+ base = os.environ.get("XDG_CONFIG_HOME") or pathlib.Path.home() / ".config"
18
+ return pathlib.Path(base) / "stateweave" / "config.json"
19
+
20
+
21
+ def load() -> dict:
22
+ try:
23
+ data = json.loads(path().read_text())
24
+ except (OSError, ValueError):
25
+ return {}
26
+ return data if isinstance(data, dict) else {}
27
+
28
+
29
+ def save(url: str, api_key: str) -> pathlib.Path:
30
+ p = path()
31
+ p.parent.mkdir(parents=True, exist_ok=True)
32
+ tmp = p.with_suffix(".tmp")
33
+ tmp.touch(mode=0o600)
34
+ tmp.chmod(0o600) # before the key is in it
35
+ tmp.write_text(json.dumps({"url": url, "api_key": api_key}, indent=1) + "\n")
36
+ tmp.replace(p)
37
+ return p
38
+
39
+
40
+ def clear() -> bool:
41
+ p = path()
42
+ if not p.exists():
43
+ return False
44
+ p.unlink()
45
+ return True
@@ -0,0 +1,121 @@
1
+ Metadata-Version: 2.5
2
+ Name: stateweave-sdk
3
+ Version: 0.3.0
4
+ Summary: Run your own AI agent on StateWeave: environments, tests, data of your own, and graded runs
5
+ Project-URL: Homepage, https://stateweave.net
6
+ Project-URL: Documentation, https://stateweave.net/docs/
7
+ Author-email: StateWeave <mikewei@stateweave.net>
8
+ License-Expression: Apache-2.0
9
+ License-File: LICENSE
10
+ Keywords: agent evaluation,ai agents,digital twin,mcp,salesforce,shopify,slack,testing
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3 :: Only
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Software Development :: Testing
21
+ Requires-Python: >=3.10
22
+ Description-Content-Type: text/markdown
23
+
24
+ # stateweave-sdk
25
+
26
+ Run your own AI agent on [StateWeave](https://stateweave.net). StateWeave gives you working copies of the apps your
27
+ agent uses (Slack, Gmail, Google Calendar, Sheets and Drive, Salesforce, Shopify, Zendesk), filled with a company's
28
+ data, and tests with answer keys.
29
+ Your agent stays on your machine and connects to the apps over their real APIs or MCP. Every run is graded and shown,
30
+ call by call, on your StateWeave page.
31
+
32
+ ```bash
33
+ pip install stateweave-sdk
34
+ ```
35
+
36
+ This installs the `sw` command and the `stateweave` Python module. It uses only the standard library and needs
37
+ Python 3.10 or later. The full documentation is at [stateweave.net/docs](https://stateweave.net/docs/).
38
+
39
+ ## Five minutes
40
+
41
+ ```bash
42
+ sw login --url https://<you>.app.stateweave.net # asks for an API key: make one under Connect an agent
43
+ sw env ls # your scenarios
44
+ sw task ls # the tests you can run
45
+ sw run t-3fa2c1 --agent my-agent@0.1 -- python my_agent.py
46
+ ```
47
+
48
+ `sw run` starts a run, runs your command with the run in its environment, then grades it:
49
+
50
+ | variable | what it holds |
51
+ |---|---|
52
+ | `STATEWEAVE_PROMPT` | what to ask your agent |
53
+ | `STATEWEAVE_KIND` | `answer` (it finds something out) or `action` (it changes the apps) |
54
+ | `STATEWEAVE_ANSWER_FORMAT` | for an answer task, the JSON value to answer with |
55
+ | `STATEWEAVE_RUN_FILE` | the whole run as JSON: each app's `base_url`, `endpoints`, `credentials` and MCP server |
56
+ | `STATEWEAVE_MCP_CONFIG` | `{"mcpServers": {...}}`, for any MCP client |
57
+ | `STATEWEAVE_ANSWER_FILE` | where your agent writes its answer (JSON, or plain text); nothing for an action task |
58
+ | `STATEWEAVE_RUN` | the run's id |
59
+
60
+ The run is graded when your command exits, whatever its exit code. Ctrl-C gives the run up instead. `sw run` exits
61
+ 0 when the run passed.
62
+
63
+ With Claude Code as the agent, on a test of Slack and Gmail:
64
+
65
+ ```bash
66
+ sw run t-3fa2c1 --agent claude-code -- sh -c 'claude -p "$STATEWEAVE_PROMPT" \
67
+ --mcp-config "$STATEWEAVE_MCP_CONFIG" --strict-mcp-config --allowedTools "mcp__slack__*" "mcp__gmail__*"'
68
+ ```
69
+
70
+ In print mode Claude Code refuses any tool nobody approved, so `--allowedTools` must name each app the test uses.
71
+ `--strict-mcp-config` keeps your own MCP servers out of the run. The
72
+ [MCP servers](https://stateweave.net/docs/mcp.html#claude-code) page has a script that works out the apps by itself
73
+ and hands in the answer for answer tasks.
74
+
75
+ Give each version of your agent its own `--agent` label. The Runs page compares the labels.
76
+
77
+ ## Your own scenarios, tests and data
78
+
79
+ ```bash
80
+ sw env new --apps gmail,zendesk --describe "A 40-person logistics software company after a carrier outage"
81
+ sw task new p-9c41d2 --request "Which enterprise customers were hit by last week's outage?"
82
+ sw task new p-9c41d2 --request "Refund #1042." --criteria "Only order #1042 is refunded, in full."
83
+ sw task new p-9c41d2 --suggest 5 --focus "refunds promised but never processed" --random-faults
84
+ sw task faults p-9c41d2
85
+ sw data kinds p-9c41d2
86
+ sw data add p-9c41d2 gmail email --from dana@northwind.example --subject "Credit?" --body-file mail.txt
87
+ sw task recheck p-9c41d2
88
+ ```
89
+
90
+ - **Your own requests.** A request you write is kept word for word. StateWeave builds its answer key and checker,
91
+ or tells you why the request can't be a test as written and suggests a rewording. `--criteria` says, in your words,
92
+ what a right outcome must meet.
93
+ - **Faults.** Tests can inject the failures real APIs have: presets (`--faults`, listed by `sw task faults`) or
94
+ random ones (`--random-faults`).
95
+ - **Your own data.** Added data joins the app's data as if the scenario had generated it. Only the scenario's
96
+ people and contacts may appear in it.
97
+ - **Re-checking.** After you add data, that app's tests need a re-check before they run again.
98
+
99
+ ## In Python
100
+
101
+ ```python
102
+ import stateweave
103
+
104
+ with stateweave.start("t-3fa2c1", agent="my-agent@0.1") as run:
105
+ answer = my_agent(run.prompt, run.apps) # run.apps: each app's URLs, credentials and MCP server
106
+ result = run.finish(answer)
107
+ print(result.passed, result.reasons, result.url)
108
+ ```
109
+
110
+ `stateweave.Client` also has `projects()`, `create_project()`, `data()`, `add_data()`, `write_tests()`,
111
+ `suggest_tests()`, `faults()`, `fault_targets()`, `job()` and `recheck()`.
112
+
113
+ ## Where the address and key come from
114
+
115
+ The first of these that is set wins:
116
+
117
+ 1. `Client(url=..., api_key=...)`
118
+ 2. `STATEWEAVE_URL` and `STATEWEAVE_API_KEY`
119
+ 3. What `sw login` kept in `~/.config/stateweave/config.json`, which only you can read
120
+
121
+ To forget it: `sw logout`.
@@ -0,0 +1,8 @@
1
+ stateweave/__init__.py,sha256=WyN3WKZrAoUsfRBfyABWVfjtDTFubl2htcZWd_bCfew,16376
2
+ stateweave/__main__.py,sha256=VMjGazWFPuZs3O8nVKiKtqkI5kxDLkn85VcybEciH0k,26089
3
+ stateweave/config.py,sha256=-N2lBXvAzsKExRnqUDPqbCQrcKL4JlrB7KUApnapXrU,1362
4
+ stateweave_sdk-0.3.0.dist-info/METADATA,sha256=QNZh9r7KoXt3kd_gj1dIK4jUt3Vl-QjnasR825-zQKs,5622
5
+ stateweave_sdk-0.3.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
6
+ stateweave_sdk-0.3.0.dist-info/entry_points.txt,sha256=tI-nhocURnnberY4NPY5g4YWBH0Uzm3xyRc_UH3osOo,48
7
+ stateweave_sdk-0.3.0.dist-info/licenses/LICENSE,sha256=MoaPew_bCDvTLu9xHBnheNth1s8ZXqSS6czkUPvCvpo,11341
8
+ stateweave_sdk-0.3.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ sw = stateweave.__main__:main
@@ -0,0 +1,202 @@
1
+
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
5
+
6
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
+
8
+ 1. Definitions.
9
+
10
+ "License" shall mean the terms and conditions for use, reproduction,
11
+ and distribution as defined by Sections 1 through 9 of this document.
12
+
13
+ "Licensor" shall mean the copyright owner or entity authorized by
14
+ the copyright owner that is granting the License.
15
+
16
+ "Legal Entity" shall mean the union of the acting entity and all
17
+ other entities that control, are controlled by, or are under common
18
+ control with that entity. For the purposes of this definition,
19
+ "control" means (i) the power, direct or indirect, to cause the
20
+ direction or management of such entity, whether by contract or
21
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
22
+ outstanding shares, or (iii) beneficial ownership of such entity.
23
+
24
+ "You" (or "Your") shall mean an individual or Legal Entity
25
+ exercising permissions granted by this License.
26
+
27
+ "Source" form shall mean the preferred form for making modifications,
28
+ including but not limited to software source code, documentation
29
+ source, and configuration files.
30
+
31
+ "Object" form shall mean any form resulting from mechanical
32
+ transformation or translation of a Source form, including but
33
+ not limited to compiled object code, generated documentation,
34
+ and conversions to other media types.
35
+
36
+ "Work" shall mean the work of authorship, whether in Source or
37
+ Object form, made available under the License, as indicated by a
38
+ copyright notice that is included in or attached to the work
39
+ (an example is provided in the Appendix below).
40
+
41
+ "Derivative Works" shall mean any work, whether in Source or Object
42
+ form, that is based on (or derived from) the Work and for which the
43
+ editorial revisions, annotations, elaborations, or other modifications
44
+ represent, as a whole, an original work of authorship. For the purposes
45
+ of this License, Derivative Works shall not include works that remain
46
+ separable from, or merely link (or bind by name) to the interfaces of,
47
+ the Work and Derivative Works thereof.
48
+
49
+ "Contribution" shall mean any work of authorship, including
50
+ the original version of the Work and any modifications or additions
51
+ to that Work or Derivative Works thereof, that is intentionally
52
+ submitted to Licensor for inclusion in the Work by the copyright owner
53
+ or by an individual or Legal Entity authorized to submit on behalf of
54
+ the copyright owner. For the purposes of this definition, "submitted"
55
+ means any form of electronic, verbal, or written communication sent
56
+ to the Licensor or its representatives, including but not limited to
57
+ communication on electronic mailing lists, source code control systems,
58
+ and issue tracking systems that are managed by, or on behalf of, the
59
+ Licensor for the purpose of discussing and improving the Work, but
60
+ excluding communication that is conspicuously marked or otherwise
61
+ designated in writing by the copyright owner as "Not a Contribution."
62
+
63
+ "Contributor" shall mean Licensor and any individual or Legal Entity
64
+ on behalf of whom a Contribution has been received by Licensor and
65
+ subsequently incorporated within the Work.
66
+
67
+ 2. Grant of Copyright License. Subject to the terms and conditions of
68
+ this License, each Contributor hereby grants to You a perpetual,
69
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70
+ copyright license to reproduce, prepare Derivative Works of,
71
+ publicly display, publicly perform, sublicense, and distribute the
72
+ Work and such Derivative Works in Source or Object form.
73
+
74
+ 3. Grant of Patent License. Subject to the terms and conditions of
75
+ this License, each Contributor hereby grants to You a perpetual,
76
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77
+ (except as stated in this section) patent license to make, have made,
78
+ use, offer to sell, sell, import, and otherwise transfer the Work,
79
+ where such license applies only to those patent claims licensable
80
+ by such Contributor that are necessarily infringed by their
81
+ Contribution(s) alone or by combination of their Contribution(s)
82
+ with the Work to which such Contribution(s) was submitted. If You
83
+ institute patent litigation against any entity (including a
84
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
85
+ or a Contribution incorporated within the Work constitutes direct
86
+ or contributory patent infringement, then any patent licenses
87
+ granted to You under this License for that Work shall terminate
88
+ as of the date such litigation is filed.
89
+
90
+ 4. Redistribution. You may reproduce and distribute copies of the
91
+ Work or Derivative Works thereof in any medium, with or without
92
+ modifications, and in Source or Object form, provided that You
93
+ meet the following conditions:
94
+
95
+ (a) You must give any other recipients of the Work or
96
+ Derivative Works a copy of this License; and
97
+
98
+ (b) You must cause any modified files to carry prominent notices
99
+ stating that You changed the files; and
100
+
101
+ (c) You must retain, in the Source form of any Derivative Works
102
+ that You distribute, all copyright, patent, trademark, and
103
+ attribution notices from the Source form of the Work,
104
+ excluding those notices that do not pertain to any part of
105
+ the Derivative Works; and
106
+
107
+ (d) If the Work includes a "NOTICE" text file as part of its
108
+ distribution, then any Derivative Works that You distribute must
109
+ include a readable copy of the attribution notices contained
110
+ within such NOTICE file, excluding those notices that do not
111
+ pertain to any part of the Derivative Works, in at least one
112
+ of the following places: within a NOTICE text file distributed
113
+ as part of the Derivative Works; within the Source form or
114
+ documentation, if provided along with the Derivative Works; or,
115
+ within a display generated by the Derivative Works, if and
116
+ wherever such third-party notices normally appear. The contents
117
+ of the NOTICE file are for informational purposes only and
118
+ do not modify the License. You may add Your own attribution
119
+ notices within Derivative Works that You distribute, alongside
120
+ or as an addendum to the NOTICE text from the Work, provided
121
+ that such additional attribution notices cannot be construed
122
+ as modifying the License.
123
+
124
+ You may add Your own copyright statement to Your modifications and
125
+ may provide additional or different license terms and conditions
126
+ for use, reproduction, or distribution of Your modifications, or
127
+ for any such Derivative Works as a whole, provided Your use,
128
+ reproduction, and distribution of the Work otherwise complies with
129
+ the conditions stated in this License.
130
+
131
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
132
+ any Contribution intentionally submitted for inclusion in the Work
133
+ by You to the Licensor shall be under the terms and conditions of
134
+ this License, without any additional terms or conditions.
135
+ Notwithstanding the above, nothing herein shall supersede or modify
136
+ the terms of any separate license agreement you may have executed
137
+ with Licensor regarding such Contributions.
138
+
139
+ 6. Trademarks. This License does not grant permission to use the trade
140
+ names, trademarks, service marks, or product names of the Licensor,
141
+ except as required for reasonable and customary use in describing the
142
+ origin of the Work and reproducing the content of the NOTICE file.
143
+
144
+ 7. Disclaimer of Warranty. Unless required by applicable law or
145
+ agreed to in writing, Licensor provides the Work (and each
146
+ Contributor provides its Contributions) on an "AS IS" BASIS,
147
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148
+ implied, including, without limitation, any warranties or conditions
149
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150
+ PARTICULAR PURPOSE. You are solely responsible for determining the
151
+ appropriateness of using or redistributing the Work and assume any
152
+ risks associated with Your exercise of permissions under this License.
153
+
154
+ 8. Limitation of Liability. In no event and under no legal theory,
155
+ whether in tort (including negligence), contract, or otherwise,
156
+ unless required by applicable law (such as deliberate and grossly
157
+ negligent acts) or agreed to in writing, shall any Contributor be
158
+ liable to You for damages, including any direct, indirect, special,
159
+ incidental, or consequential damages of any character arising as a
160
+ result of this License or out of the use or inability to use the
161
+ Work (including but not limited to damages for loss of goodwill,
162
+ work stoppage, computer failure or malfunction, or any and all
163
+ other commercial damages or losses), even if such Contributor
164
+ has been advised of the possibility of such damages.
165
+
166
+ 9. Accepting Warranty or Additional Liability. While redistributing
167
+ the Work or Derivative Works thereof, You may choose to offer,
168
+ and charge a fee for, acceptance of support, warranty, indemnity,
169
+ or other liability obligations and/or rights consistent with this
170
+ License. However, in accepting such obligations, You may act only
171
+ on Your own behalf and on Your sole responsibility, not on behalf
172
+ of any other Contributor, and only if You agree to indemnify,
173
+ defend, and hold each Contributor harmless for any liability
174
+ incurred by, or claims asserted against, such Contributor by reason
175
+ of your accepting any such warranty or additional liability.
176
+
177
+ END OF TERMS AND CONDITIONS
178
+
179
+ APPENDIX: How to apply the Apache License to your work.
180
+
181
+ To apply the Apache License to your work, attach the following
182
+ boilerplate notice, with the fields enclosed by brackets "[]"
183
+ replaced with your own identifying information. (Don't include
184
+ the brackets!) The text should be enclosed in the appropriate
185
+ comment syntax for the file format. We also recommend that a
186
+ file or class name and description of purpose be included on the
187
+ same "printed page" as the copyright notice for easier
188
+ identification within third-party archives.
189
+
190
+ Copyright 2026 StateWeave
191
+
192
+ Licensed under the Apache License, Version 2.0 (the "License");
193
+ you may not use this file except in compliance with the License.
194
+ You may obtain a copy of the License at
195
+
196
+ http://www.apache.org/licenses/LICENSE-2.0
197
+
198
+ Unless required by applicable law or agreed to in writing, software
199
+ distributed under the License is distributed on an "AS IS" BASIS,
200
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201
+ See the License for the specific language governing permissions and
202
+ limitations under the License.