wspx 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. wspx/__init__.py +24 -0
  2. wspx/_version.py +1 -0
  3. wspx/addressing.py +415 -0
  4. wspx/auth.py +77 -0
  5. wspx/cli/__init__.py +1 -0
  6. wspx/cli/client.py +124 -0
  7. wspx/cli/config.py +89 -0
  8. wspx/cli/connect.py +88 -0
  9. wspx/cli/main.py +848 -0
  10. wspx/cloud/__init__.py +20 -0
  11. wspx/cloud/app.py +191 -0
  12. wspx/cloud/backend.py +141 -0
  13. wspx/cloud/compose.py +85 -0
  14. wspx/cloud/deploy.py +102 -0
  15. wspx/config.py +189 -0
  16. wspx/control/__init__.py +5 -0
  17. wspx/control/app.py +488 -0
  18. wspx/control/backend.py +244 -0
  19. wspx/control/gateway_client.py +79 -0
  20. wspx/control/mcp_server.py +71 -0
  21. wspx/control/sessions.py +424 -0
  22. wspx/control/tracedb.py +183 -0
  23. wspx/control/ui/assets/index-BJFweZAn.css +1 -0
  24. wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
  25. wspx/control/ui/index.html +13 -0
  26. wspx/control/views.py +180 -0
  27. wspx/diagnostics.py +146 -0
  28. wspx/driver/__init__.py +7 -0
  29. wspx/driver/loop.py +255 -0
  30. wspx/driver/messages.py +62 -0
  31. wspx/driver/providers/__init__.py +84 -0
  32. wspx/driver/providers/anthropic.py +149 -0
  33. wspx/driver/providers/compat.py +115 -0
  34. wspx/driver/providers/google.py +138 -0
  35. wspx/driver/providers/openai.py +125 -0
  36. wspx/driver/run.py +134 -0
  37. wspx/errors.py +69 -0
  38. wspx/gateway/__init__.py +8 -0
  39. wspx/gateway/app.py +626 -0
  40. wspx/gateway/ledger.py +536 -0
  41. wspx/gateway/prices.py +75 -0
  42. wspx/gateway/usage.py +284 -0
  43. wspx/handles.py +117 -0
  44. wspx/integrations/AGENTS.snippet.md +8 -0
  45. wspx/integrations/SKILL.md +52 -0
  46. wspx/integrations/__init__.py +17 -0
  47. wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
  48. wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
  49. wspx/integrations/codex-hooks.toml +9 -0
  50. wspx/lab/__init__.py +5 -0
  51. wspx/lab/analysis.py +485 -0
  52. wspx/lab/check.py +489 -0
  53. wspx/lab/evaluate.py +108 -0
  54. wspx/lab/export.py +275 -0
  55. wspx/lab/grep.py +754 -0
  56. wspx/lab/index.py +239 -0
  57. wspx/lab/ir/__init__.py +1 -0
  58. wspx/lab/ir/capture.py +299 -0
  59. wspx/lab/ir/graph.py +148 -0
  60. wspx/lab/ir/model_ir.py +522 -0
  61. wspx/lab/ir/semantics.py +1255 -0
  62. wspx/lab/ir/tape.py +853 -0
  63. wspx/lab/loader.py +204 -0
  64. wspx/lab/merge.py +200 -0
  65. wspx/lab/ops/__init__.py +76 -0
  66. wspx/lab/ops/builtin.py +379 -0
  67. wspx/lab/patch.py +320 -0
  68. wspx/lab/probe.py +206 -0
  69. wspx/lab/results.py +114 -0
  70. wspx/lab/sdk.py +120 -0
  71. wspx/lab/server.py +236 -0
  72. wspx/lab/session.py +1574 -0
  73. wspx/lab/tensors.py +119 -0
  74. wspx/lab/text.py +69 -0
  75. wspx/lab/tree.py +302 -0
  76. wspx/loops.py +127 -0
  77. wspx/project.py +143 -0
  78. wspx/py.typed +0 -0
  79. wspx/safety/__init__.py +15 -0
  80. wspx/safety/kill.py +47 -0
  81. wspx/safety/sweeper.py +68 -0
  82. wspx/store/__init__.py +64 -0
  83. wspx/store/objects.py +130 -0
  84. wspx/store/overlay.py +209 -0
  85. wspx/store/refs.py +150 -0
  86. wspx/testing/__init__.py +21 -0
  87. wspx/testing/api.py +155 -0
  88. wspx/testing/builtin.py +139 -0
  89. wspx/testing/runner.py +257 -0
  90. wspx/tools/__init__.py +17 -0
  91. wspx/tools/dispatch.py +129 -0
  92. wspx/tools/schemas.py +313 -0
  93. wspx/vcs/__init__.py +17 -0
  94. wspx/vcs/repo.py +431 -0
  95. wspx-0.1.0.dev0.dist-info/METADATA +278 -0
  96. wspx-0.1.0.dev0.dist-info/RECORD +99 -0
  97. wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
  98. wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
  99. wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/testing/runner.py ADDED
@@ -0,0 +1,257 @@
1
+ """Running suites: discovery, paired runs, caching and the held-out lock."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import importlib.util
6
+ import io
7
+ import json
8
+ import math
9
+ import random
10
+ import sys
11
+ import time
12
+ from collections.abc import Callable, Iterable
13
+ from dataclasses import dataclass, field
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ from wspx.errors import HeldOutAccess, WspxError
18
+ from wspx.testing import api
19
+ from wspx.testing.api import Check, Suite
20
+
21
+ PREVIEW_FAILURES = 5
22
+
23
+
24
+ def discover(paths: Iterable[str | Path], root: Path | None = None) -> list[Suite]:
25
+ """Load suites from ``.py`` and ``.yaml``/``.yml`` files under ``paths``."""
26
+ root = root or Path.cwd()
27
+ suites: list[Suite] = []
28
+ for entry in paths:
29
+ base = (root / entry) if not Path(entry).is_absolute() else Path(entry)
30
+ files = sorted(base.rglob("*")) if base.is_dir() else [base]
31
+ for path in files:
32
+ if path.suffix == ".py" and not path.name.startswith("_"):
33
+ suites.extend(_load_python(path))
34
+ elif path.suffix in (".yaml", ".yml"):
35
+ suites.append(_load_yaml(path))
36
+ names = [s.name for s in suites]
37
+ dupes = {n for n in names if names.count(n) > 1}
38
+ if dupes:
39
+ raise WspxError(f"duplicate suite names: {', '.join(sorted(dupes))}")
40
+ return suites
41
+
42
+
43
+ def _load_python(path: Path) -> list[Suite]:
44
+ api.collect()
45
+ spec = importlib.util.spec_from_file_location(f"wtests_{path.stem}_{abs(hash(path))}", path)
46
+ if spec is None or spec.loader is None:
47
+ raise WspxError(f"cannot import {path}")
48
+ module = importlib.util.module_from_spec(spec)
49
+ sys.path.insert(0, str(path.parent))
50
+ try:
51
+ spec.loader.exec_module(module)
52
+ finally:
53
+ sys.path.remove(str(path.parent))
54
+ found = api.collect()
55
+ for s in found:
56
+ s.path = str(path)
57
+ return found
58
+
59
+
60
+ def _load_yaml(path: Path) -> Suite:
61
+ try:
62
+ import yaml # type: ignore[import-untyped]
63
+ except ImportError: # pragma: no cover - PyYAML ships with the lab image via transformers
64
+ raise WspxError("YAML suites need PyYAML") from None
65
+ data = yaml.safe_load(path.read_text())
66
+ if not isinstance(data, dict) or "items" not in data:
67
+ raise WspxError(f"{path}: a YAML suite needs 'name' and 'items'")
68
+ return Suite(
69
+ name=str(data.get("name", path.stem)),
70
+ fn=None,
71
+ split=data.get("split", "dev"),
72
+ seed=int(data.get("seed", 0)),
73
+ source=path.read_text(),
74
+ items=list(data["items"]),
75
+ path=str(path),
76
+ threshold=data.get("threshold"),
77
+ )
78
+
79
+
80
+ def yaml_checks(suite: Suite, model: Any) -> Iterable[Check]:
81
+ for i, item in enumerate(suite.items):
82
+ prompt = item["prompt"]
83
+ out = model.generate(prompt, max_tokens=int(item.get("max_tokens", 64)))
84
+ spec = item.get("expect", {})
85
+ if not isinstance(spec, dict) or len(spec) != 1:
86
+ raise WspxError(f"{suite.name} item {i}: 'expect' needs exactly one checker")
87
+ ((kind, arg),) = spec.items()
88
+ checker = getattr(api.expect, kind, None)
89
+ if checker is None or kind in ("logprob", "logit_diff", "kl", "true"):
90
+ raise WspxError(f"{suite.name} item {i}: unknown YAML checker {kind!r}")
91
+ check = checker(out, arg)
92
+ check.item = str(item.get("id", i))
93
+ yield check
94
+
95
+
96
+ @dataclass
97
+ class RunResult:
98
+ suite: str
99
+ split: str
100
+ seed: int
101
+ tree: str
102
+ """Hash of the weight state the run used."""
103
+ n: int
104
+ passed: int
105
+ mean_score: float
106
+ checks: list[dict[str, Any]] = field(default_factory=list)
107
+ seconds: float = 0.0
108
+ data_hashes: dict[str, str] = field(default_factory=dict)
109
+ cached: bool = False
110
+
111
+ @property
112
+ def pass_rate(self) -> float:
113
+ return self.passed / self.n if self.n else math.nan
114
+
115
+ def scores(self) -> list[float]:
116
+ return [float(c["score"]) for c in self.checks]
117
+
118
+ def to_json(self) -> bytes:
119
+ return json.dumps(self.__dict__, sort_keys=True, default=str).encode()
120
+
121
+ @classmethod
122
+ def from_json(cls, data: bytes) -> RunResult:
123
+ return cls(**json.loads(data))
124
+
125
+
126
+ def run_suite(
127
+ suite: Suite, model: Any, tree: str, *, progress: Callable[[int], None] | None = None
128
+ ) -> RunResult:
129
+ random.seed(suite.seed)
130
+ try:
131
+ import torch
132
+
133
+ torch.manual_seed(suite.seed)
134
+ except ImportError: # pragma: no cover
135
+ pass
136
+ api._DATA_HASHES.clear()
137
+ started = time.perf_counter()
138
+ checks: list[dict[str, Any]] = []
139
+ source = yaml_checks(suite, model) if suite.fn is None else suite.fn(model)
140
+ for i, check in enumerate(source):
141
+ if not isinstance(check, Check):
142
+ raise WspxError(
143
+ f"{suite.name}: yielded {type(check).__name__}, expected a Check from expect.*"
144
+ )
145
+ if check.item is None:
146
+ check.item = str(i)
147
+ checks.append(check.to_dict())
148
+ if progress is not None:
149
+ progress(i + 1)
150
+ passed = sum(1 for c in checks if c["passed"])
151
+ scores = [c["score"] for c in checks if not math.isnan(c["score"])]
152
+ return RunResult(
153
+ suite=suite.name,
154
+ split=suite.split,
155
+ seed=suite.seed,
156
+ tree=tree,
157
+ n=len(checks),
158
+ passed=passed,
159
+ mean_score=sum(scores) / len(scores) if scores else math.nan,
160
+ checks=checks,
161
+ seconds=time.perf_counter() - started,
162
+ data_hashes=dict(api._DATA_HASHES),
163
+ )
164
+
165
+
166
+ # --------------------------------------------------------------------------- paired statistics
167
+
168
+
169
+ @dataclass
170
+ class Paired:
171
+ n: int
172
+ mean_delta: float
173
+ ci_low: float
174
+ ci_high: float
175
+ flips_up: int
176
+ """Items that failed on the reference and pass now."""
177
+ flips_down: int
178
+
179
+
180
+ def paired(
181
+ current: RunResult, reference: RunResult, *, resamples: int = 2000, seed: int = 0
182
+ ) -> Paired:
183
+ """Per-item paired differences with a percentile bootstrap 95% interval."""
184
+ ref = {c["item"]: c for c in reference.checks}
185
+ pairs = [(c, ref[c["item"]]) for c in current.checks if c["item"] in ref]
186
+ if not pairs:
187
+ raise WspxError("no common items to pair; are these the same suite and seed?")
188
+ deltas = [
189
+ float(a["score"]) - float(b["score"])
190
+ for a, b in pairs
191
+ if not (math.isnan(a["score"]) or math.isnan(b["score"]))
192
+ ]
193
+ up = sum(1 for a, b in pairs if a["passed"] and not b["passed"])
194
+ down = sum(1 for a, b in pairs if b["passed"] and not a["passed"])
195
+ if not deltas:
196
+ return Paired(len(pairs), math.nan, math.nan, math.nan, up, down)
197
+ rng = random.Random(seed)
198
+ n = len(deltas)
199
+ means = sorted(sum(rng.choice(deltas) for _ in range(n)) / n for _ in range(resamples))
200
+ mean = sum(deltas) / n
201
+ return Paired(
202
+ n, mean, means[int(0.025 * resamples)], means[int(0.975 * resamples) - 1], up, down
203
+ )
204
+
205
+
206
+ # --------------------------------------------------------------------------- reporting
207
+
208
+
209
+ def summary(
210
+ result: RunResult, *, vs: RunResult | None = None, show_failures: int = PREVIEW_FAILURES
211
+ ) -> str:
212
+ """What the driver sees: aggregate, paired delta and (dev only) a few failing examples."""
213
+ head = (
214
+ f"{result.suite} [{result.split}] {result.passed}/{result.n} passed "
215
+ f"({result.pass_rate:.1%}) · mean {result.mean_score:.3f}"
216
+ + (" · cached" if result.cached else f" · {result.seconds:.1f}s")
217
+ )
218
+ lines = [head]
219
+ if vs is not None:
220
+ p = paired(result, vs)
221
+ lines.append(
222
+ f" vs {vs.tree[:9]}: Δ {p.mean_delta:+.3f} [95% CI {p.ci_low:+.3f}, {p.ci_high:+.3f}] "
223
+ f"· {p.flips_up} fixed, {p.flips_down} broken (n={p.n})"
224
+ )
225
+ if result.split == "heldout":
226
+ lines.append(" (held-out: per-item results are visible in the UI only)")
227
+ return "\n".join(lines)
228
+ failures = [c for c in result.checks if not c["passed"]][:show_failures]
229
+ for c in failures:
230
+ out = (c.get("output") or "")[:160].replace("\n", "⏎")
231
+ lines.append(f" ✗ item {c['item']} [{c['kind']} {c.get('detail', '')}] → {out!r}")
232
+ more = result.n - result.passed - len(failures)
233
+ if more > 0:
234
+ lines.append(f" … {more} more failures")
235
+ return "\n".join(lines)
236
+
237
+
238
+ def driver_view(result: RunResult) -> dict[str, Any]:
239
+ """The result as the driver may see it. Held-out runs carry aggregates only."""
240
+ data = {k: v for k, v in result.__dict__.items() if k != "checks"}
241
+ if result.split != "heldout":
242
+ data["failures"] = [c for c in result.checks if not c["passed"]][:PREVIEW_FAILURES]
243
+ return data
244
+
245
+
246
+ def heldout_items(result: RunResult, *, caller: str) -> list[dict[str, Any]]:
247
+ """Per-item held-out data, for the UI only."""
248
+ if result.split == "heldout" and caller != "ui":
249
+ raise HeldOutAccess("held-out items and outputs are visible in the UI only")
250
+ return result.checks
251
+
252
+
253
+ def jsonl(rows: list[dict[str, Any]]) -> bytes:
254
+ buf = io.StringIO()
255
+ for row in rows:
256
+ buf.write(json.dumps(row) + "\n")
257
+ return buf.getvalue().encode()
wspx/tools/__init__.py ADDED
@@ -0,0 +1,17 @@
1
+ """The tool surface (§4.2): one schema per tool, shared by every driver and the CLI.
2
+
3
+ Schemas are Pydantic models; MCP and provider tool definitions are generated
4
+ from them, so the built-in driver, external agents over MCP and humans at the
5
+ CLI all see the same tools with the same arguments.
6
+ """
7
+
8
+ from wspx.tools.schemas import (
9
+ TOOLS,
10
+ ToolSpec,
11
+ anthropic_tools,
12
+ gemini_tools,
13
+ mcp_tools,
14
+ openai_tools,
15
+ )
16
+
17
+ __all__ = ["TOOLS", "ToolSpec", "anthropic_tools", "gemini_tools", "mcp_tools", "openai_tools"]
wspx/tools/dispatch.py ADDED
@@ -0,0 +1,129 @@
1
+ """Call a tool on a lab session: validate, run, budget the answer, never raise."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Callable
6
+ from typing import TYPE_CHECKING, Any
7
+
8
+ from pydantic import ValidationError
9
+
10
+ from wspx.errors import CheckFailed, WspxError
11
+ from wspx.tools.schemas import TOOLS
12
+
13
+ if TYPE_CHECKING:
14
+ from wspx.lab.session import Lab, ToolResult
15
+
16
+
17
+ def call(
18
+ lab: Lab,
19
+ name: str,
20
+ arguments: dict[str, Any] | None = None,
21
+ *,
22
+ jobs: Callable[[str | None], ToolResult] | None = None,
23
+ ) -> ToolResult:
24
+ from wspx.lab.session import ToolResult
25
+
26
+ spec = TOOLS.get(name)
27
+ if spec is None:
28
+ return ToolResult(f"unknown tool {name!r}; tools: {', '.join(TOOLS)}", ok=False)
29
+ try:
30
+ args = spec.args.model_validate(arguments or {})
31
+ except ValidationError as exc:
32
+ problems = "; ".join(
33
+ f"{'.'.join(str(p) for p in e['loc']) or name}: {e['msg']}" for e in exc.errors()
34
+ )
35
+ return ToolResult(f"{name}: invalid arguments: {problems}", ok=False)
36
+ try:
37
+ return _run(lab, name, args, jobs)
38
+ except CheckFailed as exc:
39
+ text = "\n".join(d.render() for d in exc.diagnostics)
40
+ return ToolResult(
41
+ f"{name} blocked by {len(exc.diagnostics)} error(s):\n{text}",
42
+ {"diagnostics": [d.model_dump(mode="json") for d in exc.diagnostics]},
43
+ ok=False,
44
+ )
45
+ except WspxError as exc:
46
+ return ToolResult(f"{name}: {exc}", ok=False)
47
+
48
+
49
+ def _run(
50
+ lab: Lab, name: str, a: Any, jobs: Callable[[str | None], ToolResult] | None
51
+ ) -> ToolResult:
52
+ from wspx.lab.session import ToolResult
53
+
54
+ if name == "info":
55
+ return lab.info(handles=a.handles)
56
+ if name == "ls":
57
+ return lab.ls(a.scope, role=a.role, layers=a.layers, limit=a.limit)
58
+ if name == "grep":
59
+ hits = lab.grep(
60
+ a.expression, scope=a.scope, axis=a.axis, top=a.top, name=a.name, fires_on=a.fires_on
61
+ )
62
+ return ToolResult(
63
+ hits.summary, {"handle": hits.handle, "hits": [h.__dict__ for h in hits]}, [hits.handle]
64
+ )
65
+ if name == "show":
66
+ return lab.show(a.address, values=a.values)
67
+ if name == "contrast":
68
+ d = lab.contrast(a.positive, a.negative, name=a.name, layer=a.layer, position=a.position)
69
+ return ToolResult(
70
+ d.summary, {"handle": d.handle, "separation": d.direction.separation}, [d.handle]
71
+ )
72
+ if name == "generate":
73
+ return lab.generate(
74
+ a.prompt, max_tokens=a.max_tokens, temperature=a.temperature, seed=a.seed
75
+ )
76
+ if name == "trace":
77
+ return lab.trace(a.prompt, vs=a.vs, reference=a.reference, target=a.target, top=a.top)
78
+ if name == "edit":
79
+ p = lab.edit_op(a.op, precision=a.precision, scope=a.scope, **a.args)
80
+ return ToolResult(p.summary, {"handle": p.handle, "record": p.record.oid}, [p.handle])
81
+ if name == "check":
82
+ report = lab.check(a.patch, level=a.level)
83
+ return ToolResult(report.summary(), report.model_dump(mode="json"), ok=report.ok)
84
+ if name == "test":
85
+ return lab.test(a.suite, vs=a.vs)
86
+ if name == "jobs":
87
+ if jobs is None:
88
+ return ToolResult("no background jobs in a local session")
89
+ return jobs(a.job)
90
+ if name == "vcs":
91
+ return _vcs(lab, a)
92
+ if name == "export":
93
+ return lab.export(a.kind, a.path, base_model=a.base_model, max_rank=a.max_rank)
94
+ if name == "run":
95
+ return lab.run(a.script, args=a.args)
96
+ raise WspxError(f"tool {name} has no handler") # pragma: no cover - TOOLS and handlers agree
97
+
98
+
99
+ def _vcs(lab: Lab, a: Any) -> ToolResult:
100
+ action = a.action
101
+ if action == "status":
102
+ return lab.status()
103
+ if action == "commit":
104
+ if not a.message:
105
+ raise WspxError("vcs commit needs a message")
106
+ return lab.commit(a.message, force=a.force)
107
+ if action == "log":
108
+ return lab.log(a.rev or "HEAD", a.n)
109
+ if action == "checkout":
110
+ return lab.checkout(_need(a.rev, "rev"), force=a.force)
111
+ if action == "branch":
112
+ return lab.branch(a.name, at=a.rev or "HEAD")
113
+ if action == "revert":
114
+ return lab.revert(a.rev or "HEAD")
115
+ if action == "diff":
116
+ return lab.diff(a.a or "HEAD", a.b)
117
+ if action == "bisect":
118
+ return lab.bisect(_need(a.suite, "suite"), good=a.good or "base", bad=a.bad or "HEAD")
119
+ if action == "blame":
120
+ return lab.blame(_need(a.address, "address"))
121
+ if action == "merge":
122
+ return lab.merge(_need(a.name, "name"), strategies=a.strategies, message=a.message)
123
+ raise WspxError(f"unknown vcs action {action}") # pragma: no cover - Literal guards it
124
+
125
+
126
+ def _need(value: str | None, what: str) -> str:
127
+ if not value:
128
+ raise WspxError(f"this action needs {what}")
129
+ return value