wspx 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wspx/__init__.py +24 -0
- wspx/_version.py +1 -0
- wspx/addressing.py +415 -0
- wspx/auth.py +77 -0
- wspx/cli/__init__.py +1 -0
- wspx/cli/client.py +124 -0
- wspx/cli/config.py +89 -0
- wspx/cli/connect.py +88 -0
- wspx/cli/main.py +848 -0
- wspx/cloud/__init__.py +20 -0
- wspx/cloud/app.py +191 -0
- wspx/cloud/backend.py +141 -0
- wspx/cloud/compose.py +85 -0
- wspx/cloud/deploy.py +102 -0
- wspx/config.py +189 -0
- wspx/control/__init__.py +5 -0
- wspx/control/app.py +488 -0
- wspx/control/backend.py +244 -0
- wspx/control/gateway_client.py +79 -0
- wspx/control/mcp_server.py +71 -0
- wspx/control/sessions.py +424 -0
- wspx/control/tracedb.py +183 -0
- wspx/control/ui/assets/index-BJFweZAn.css +1 -0
- wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
- wspx/control/ui/index.html +13 -0
- wspx/control/views.py +180 -0
- wspx/diagnostics.py +146 -0
- wspx/driver/__init__.py +7 -0
- wspx/driver/loop.py +255 -0
- wspx/driver/messages.py +62 -0
- wspx/driver/providers/__init__.py +84 -0
- wspx/driver/providers/anthropic.py +149 -0
- wspx/driver/providers/compat.py +115 -0
- wspx/driver/providers/google.py +138 -0
- wspx/driver/providers/openai.py +125 -0
- wspx/driver/run.py +134 -0
- wspx/errors.py +69 -0
- wspx/gateway/__init__.py +8 -0
- wspx/gateway/app.py +626 -0
- wspx/gateway/ledger.py +536 -0
- wspx/gateway/prices.py +75 -0
- wspx/gateway/usage.py +284 -0
- wspx/handles.py +117 -0
- wspx/integrations/AGENTS.snippet.md +8 -0
- wspx/integrations/SKILL.md +52 -0
- wspx/integrations/__init__.py +17 -0
- wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
- wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
- wspx/integrations/codex-hooks.toml +9 -0
- wspx/lab/__init__.py +5 -0
- wspx/lab/analysis.py +485 -0
- wspx/lab/check.py +489 -0
- wspx/lab/evaluate.py +108 -0
- wspx/lab/export.py +275 -0
- wspx/lab/grep.py +754 -0
- wspx/lab/index.py +239 -0
- wspx/lab/ir/__init__.py +1 -0
- wspx/lab/ir/capture.py +299 -0
- wspx/lab/ir/graph.py +148 -0
- wspx/lab/ir/model_ir.py +522 -0
- wspx/lab/ir/semantics.py +1255 -0
- wspx/lab/ir/tape.py +853 -0
- wspx/lab/loader.py +204 -0
- wspx/lab/merge.py +200 -0
- wspx/lab/ops/__init__.py +76 -0
- wspx/lab/ops/builtin.py +379 -0
- wspx/lab/patch.py +320 -0
- wspx/lab/probe.py +206 -0
- wspx/lab/results.py +114 -0
- wspx/lab/sdk.py +120 -0
- wspx/lab/server.py +236 -0
- wspx/lab/session.py +1574 -0
- wspx/lab/tensors.py +119 -0
- wspx/lab/text.py +69 -0
- wspx/lab/tree.py +302 -0
- wspx/loops.py +127 -0
- wspx/project.py +143 -0
- wspx/py.typed +0 -0
- wspx/safety/__init__.py +15 -0
- wspx/safety/kill.py +47 -0
- wspx/safety/sweeper.py +68 -0
- wspx/store/__init__.py +64 -0
- wspx/store/objects.py +130 -0
- wspx/store/overlay.py +209 -0
- wspx/store/refs.py +150 -0
- wspx/testing/__init__.py +21 -0
- wspx/testing/api.py +155 -0
- wspx/testing/builtin.py +139 -0
- wspx/testing/runner.py +257 -0
- wspx/tools/__init__.py +17 -0
- wspx/tools/dispatch.py +129 -0
- wspx/tools/schemas.py +313 -0
- wspx/vcs/__init__.py +17 -0
- wspx/vcs/repo.py +431 -0
- wspx-0.1.0.dev0.dist-info/METADATA +278 -0
- wspx-0.1.0.dev0.dist-info/RECORD +99 -0
- wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
- wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
- wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/store/refs.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""Mutable refs: branch names, session HEADs and other pointers to objects.
|
|
2
|
+
|
|
3
|
+
A ref file holds either an object id or a symbolic pointer (``ref: heads/main``).
|
|
4
|
+
Updates are compare-and-swap under a lock file, so two writers can never
|
|
5
|
+
silently lose each other's commits.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import threading
|
|
12
|
+
import time
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from wspx.addressing import is_valid_ref_name
|
|
16
|
+
from wspx.errors import RefConflict, StoreError
|
|
17
|
+
from wspx.store.objects import atomic_write
|
|
18
|
+
|
|
19
|
+
SYMBOLIC_PREFIX = "ref: "
|
|
20
|
+
_MAX_DEPTH = 8
|
|
21
|
+
_LOCK_TIMEOUT_S = 10.0
|
|
22
|
+
_STALE_LOCK_S = 30.0
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class _Any:
|
|
26
|
+
def __repr__(self) -> str:
|
|
27
|
+
return "ANY"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
ANY = _Any()
|
|
31
|
+
"""Sentinel for :meth:`RefStore.update`: skip the compare-and-swap check."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class RefStore:
|
|
35
|
+
def __init__(self, root: str | Path) -> None:
|
|
36
|
+
self.root = Path(root)
|
|
37
|
+
self._mutex = threading.Lock()
|
|
38
|
+
|
|
39
|
+
def _path(self, name: str) -> Path:
|
|
40
|
+
if not is_valid_ref_name(name):
|
|
41
|
+
raise StoreError(f"invalid ref name {name!r}")
|
|
42
|
+
return self.root / name
|
|
43
|
+
|
|
44
|
+
def exists(self, name: str) -> bool:
|
|
45
|
+
return self._path(name).is_file()
|
|
46
|
+
|
|
47
|
+
def read_raw(self, name: str) -> str | None:
|
|
48
|
+
try:
|
|
49
|
+
return self._path(name).read_text(encoding="utf-8").strip() or None
|
|
50
|
+
except FileNotFoundError:
|
|
51
|
+
return None
|
|
52
|
+
|
|
53
|
+
def read(self, name: str) -> str | None:
|
|
54
|
+
"""Follow symbolic refs and return the object id, or ``None`` if unborn."""
|
|
55
|
+
current = name
|
|
56
|
+
for _ in range(_MAX_DEPTH):
|
|
57
|
+
value = self.read_raw(current)
|
|
58
|
+
if value is None or not value.startswith(SYMBOLIC_PREFIX):
|
|
59
|
+
return value
|
|
60
|
+
current = value[len(SYMBOLIC_PREFIX) :]
|
|
61
|
+
raise StoreError(f"symbolic ref loop at {name!r}")
|
|
62
|
+
|
|
63
|
+
def symbolic_target(self, name: str) -> str | None:
|
|
64
|
+
value = self.read_raw(name)
|
|
65
|
+
if value and value.startswith(SYMBOLIC_PREFIX):
|
|
66
|
+
return value[len(SYMBOLIC_PREFIX) :]
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
def set_symbolic(self, name: str, target: str) -> None:
|
|
70
|
+
self._path(target) # validate
|
|
71
|
+
with self._locked(name):
|
|
72
|
+
atomic_write(self._path(name), f"{SYMBOLIC_PREFIX}{target}\n".encode())
|
|
73
|
+
|
|
74
|
+
def update(self, name: str, new: str, expected: str | _Any | None = ANY) -> None:
|
|
75
|
+
"""Point ``name`` at ``new``. With ``expected``, fail unless the ref still holds it.
|
|
76
|
+
|
|
77
|
+
Symbolic refs are followed, so updating ``HEAD`` moves the branch it points to.
|
|
78
|
+
"""
|
|
79
|
+
target = self._resolve_symbolic_chain(name)
|
|
80
|
+
with self._locked(target):
|
|
81
|
+
current = self.read_raw(target)
|
|
82
|
+
if expected is not ANY and current != expected:
|
|
83
|
+
raise RefConflict(
|
|
84
|
+
f"ref {target!r} moved: expected {expected or 'unborn'}, "
|
|
85
|
+
f"found {current or 'unborn'}"
|
|
86
|
+
)
|
|
87
|
+
atomic_write(self._path(target), f"{new}\n".encode())
|
|
88
|
+
|
|
89
|
+
def delete(self, name: str, expected: str | _Any | None = ANY) -> None:
|
|
90
|
+
with self._locked(name):
|
|
91
|
+
current = self.read_raw(name)
|
|
92
|
+
if expected is not ANY and current != expected:
|
|
93
|
+
raise RefConflict(f"ref {name!r} moved; refusing to delete")
|
|
94
|
+
self._path(name).unlink(missing_ok=True)
|
|
95
|
+
|
|
96
|
+
def list(self, prefix: str = "heads/") -> dict[str, str]:
|
|
97
|
+
base = self.root / prefix
|
|
98
|
+
if not base.is_dir():
|
|
99
|
+
return {}
|
|
100
|
+
out: dict[str, str] = {}
|
|
101
|
+
for path in sorted(base.rglob("*")):
|
|
102
|
+
if path.is_file() and not path.name.endswith(".lock") and not path.name.startswith("."):
|
|
103
|
+
name = path.relative_to(self.root).as_posix()
|
|
104
|
+
value = self.read(name)
|
|
105
|
+
if value is not None:
|
|
106
|
+
out[name] = value
|
|
107
|
+
return out
|
|
108
|
+
|
|
109
|
+
def _resolve_symbolic_chain(self, name: str) -> str:
|
|
110
|
+
current = name
|
|
111
|
+
for _ in range(_MAX_DEPTH):
|
|
112
|
+
target = self.symbolic_target(current)
|
|
113
|
+
if target is None:
|
|
114
|
+
return current
|
|
115
|
+
current = target
|
|
116
|
+
raise StoreError(f"symbolic ref loop at {name!r}")
|
|
117
|
+
|
|
118
|
+
class _Lock:
|
|
119
|
+
def __init__(self, store: RefStore, name: str) -> None:
|
|
120
|
+
self.store = store
|
|
121
|
+
self.path = store._path(name).with_name(store._path(name).name + ".lock")
|
|
122
|
+
|
|
123
|
+
def __enter__(self) -> None:
|
|
124
|
+
self.store._mutex.acquire()
|
|
125
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
126
|
+
deadline = time.monotonic() + _LOCK_TIMEOUT_S
|
|
127
|
+
while True:
|
|
128
|
+
try:
|
|
129
|
+
fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
130
|
+
os.close(fd)
|
|
131
|
+
return
|
|
132
|
+
except FileExistsError:
|
|
133
|
+
try:
|
|
134
|
+
age = time.time() - self.path.stat().st_mtime
|
|
135
|
+
except FileNotFoundError:
|
|
136
|
+
continue
|
|
137
|
+
if age > _STALE_LOCK_S:
|
|
138
|
+
self.path.unlink(missing_ok=True)
|
|
139
|
+
continue
|
|
140
|
+
if time.monotonic() > deadline:
|
|
141
|
+
self.store._mutex.release()
|
|
142
|
+
raise StoreError(f"timed out waiting for lock {self.path.name}") from None
|
|
143
|
+
time.sleep(0.01)
|
|
144
|
+
|
|
145
|
+
def __exit__(self, *exc: object) -> None:
|
|
146
|
+
self.path.unlink(missing_ok=True)
|
|
147
|
+
self.store._mutex.release()
|
|
148
|
+
|
|
149
|
+
def _locked(self, name: str) -> RefStore._Lock:
|
|
150
|
+
return RefStore._Lock(self, name)
|
wspx/testing/__init__.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Tests and evals for weights (§5.7): suites are code.
|
|
2
|
+
|
|
3
|
+
.. code-block:: python
|
|
4
|
+
|
|
5
|
+
from wspx.testing import suite, expect, load_jsonl
|
|
6
|
+
|
|
7
|
+
@suite(split="dev", seed=0)
|
|
8
|
+
def math_finishes(model):
|
|
9
|
+
for item in load_jsonl("data/math_dev.jsonl"):
|
|
10
|
+
out = model.generate(item["prompt"], max_tokens=4096)
|
|
11
|
+
yield expect.regex(out, item["answer_regex"])
|
|
12
|
+
|
|
13
|
+
Simple cases can be YAML (``name``, ``split``, ``items: [{prompt, expect}]``).
|
|
14
|
+
Runs are paired (identical items and seeds on both sides), cached by the
|
|
15
|
+
weights they ran on, and held-out suites only ever return aggregates to the
|
|
16
|
+
driver.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from wspx.testing.api import Check, Suite, expect, load_jsonl, suite
|
|
20
|
+
|
|
21
|
+
__all__ = ["Check", "Suite", "expect", "load_jsonl", "suite"]
|
wspx/testing/api.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""The suite-author API: ``@suite``, ``expect.*`` and ``load_jsonl``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import re
|
|
9
|
+
from collections.abc import Callable, Iterator
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, Literal
|
|
13
|
+
|
|
14
|
+
Split = Literal["dev", "heldout"]
|
|
15
|
+
|
|
16
|
+
_DATA_HASHES: dict[str, str] = {}
|
|
17
|
+
"""Files read through :func:`load_jsonl` during a run, by path: part of the cache key."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class Check:
|
|
22
|
+
passed: bool
|
|
23
|
+
score: float
|
|
24
|
+
"""1.0/0.0 for pass/fail checkers; a continuous value for metric checkers."""
|
|
25
|
+
kind: str
|
|
26
|
+
detail: str = ""
|
|
27
|
+
item: str | None = None
|
|
28
|
+
output: str | None = None
|
|
29
|
+
deferred: dict[str, Any] | None = None
|
|
30
|
+
"""Set for checks resolved outside the lab (LLM judge)."""
|
|
31
|
+
|
|
32
|
+
def to_dict(self) -> dict[str, Any]:
|
|
33
|
+
return {k: v for k, v in self.__dict__.items() if v is not None}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass
|
|
37
|
+
class Suite:
|
|
38
|
+
name: str
|
|
39
|
+
fn: Callable[[Any], Iterator[Check]] | None
|
|
40
|
+
split: Split = "dev"
|
|
41
|
+
seed: int = 0
|
|
42
|
+
source: str = ""
|
|
43
|
+
"""The suite's source text (part of its hash)."""
|
|
44
|
+
items: list[dict[str, Any]] = field(default_factory=list)
|
|
45
|
+
"""For YAML suites: the cases themselves."""
|
|
46
|
+
path: str = ""
|
|
47
|
+
threshold: float | None = None
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def heldout(self) -> bool:
|
|
51
|
+
return self.split == "heldout"
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def hash(self) -> str:
|
|
55
|
+
h = hashlib.sha256()
|
|
56
|
+
h.update(self.name.encode())
|
|
57
|
+
h.update(self.split.encode())
|
|
58
|
+
h.update(str(self.seed).encode())
|
|
59
|
+
h.update(self.source.encode())
|
|
60
|
+
h.update(json.dumps(self.items, sort_keys=True).encode())
|
|
61
|
+
return h.hexdigest()[:16]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
_REGISTRY: list[Suite] = []
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def suite(
|
|
68
|
+
fn: Callable[[Any], Iterator[Check]] | None = None,
|
|
69
|
+
*,
|
|
70
|
+
split: Split = "dev",
|
|
71
|
+
seed: int = 0,
|
|
72
|
+
name: str | None = None,
|
|
73
|
+
threshold: float | None = None,
|
|
74
|
+
) -> Any:
|
|
75
|
+
"""Register a generator of checks as a suite."""
|
|
76
|
+
|
|
77
|
+
def wrap(func: Callable[[Any], Iterator[Check]]) -> Callable[[Any], Iterator[Check]]:
|
|
78
|
+
import inspect
|
|
79
|
+
|
|
80
|
+
try:
|
|
81
|
+
source = inspect.getsource(func)
|
|
82
|
+
except (OSError, TypeError):
|
|
83
|
+
source = func.__qualname__
|
|
84
|
+
s = Suite(name or func.__name__, func, split, seed, source, threshold=threshold)
|
|
85
|
+
_REGISTRY.append(s)
|
|
86
|
+
func.__wspx_suite__ = s # type: ignore[attr-defined]
|
|
87
|
+
return func
|
|
88
|
+
|
|
89
|
+
return wrap(fn) if fn is not None else wrap
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def collect() -> list[Suite]:
|
|
93
|
+
"""Suites registered since the last collect (used by the loader)."""
|
|
94
|
+
out = list(_REGISTRY)
|
|
95
|
+
_REGISTRY.clear()
|
|
96
|
+
return out
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def load_jsonl(path: str | Path) -> list[dict[str, Any]]:
|
|
100
|
+
p = Path(path)
|
|
101
|
+
data = p.read_bytes()
|
|
102
|
+
_DATA_HASHES[str(p)] = hashlib.sha256(data).hexdigest()[:16]
|
|
103
|
+
return [json.loads(line) for line in data.decode().splitlines() if line.strip()]
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class _Expect:
|
|
107
|
+
"""Checkers. Each returns a :class:`Check`."""
|
|
108
|
+
|
|
109
|
+
def exact(self, output: str, target: str, *, strip: bool = True) -> Check:
|
|
110
|
+
a, b = (output.strip(), target.strip()) if strip else (output, target)
|
|
111
|
+
return Check(a == b, float(a == b), "exact", f"want {target!r}", output=output)
|
|
112
|
+
|
|
113
|
+
def regex(self, output: str, pattern: str) -> Check:
|
|
114
|
+
ok = re.search(pattern, output) is not None
|
|
115
|
+
return Check(ok, float(ok), "regex", f"/{pattern}/", output=output)
|
|
116
|
+
|
|
117
|
+
def contains(self, output: str, needle: str, *, case: bool = False) -> Check:
|
|
118
|
+
ok = needle in output if case else needle.lower() in output.lower()
|
|
119
|
+
return Check(ok, float(ok), "contains", f"{needle!r}", output=output)
|
|
120
|
+
|
|
121
|
+
def not_contains(self, output: str, needle: str, *, case: bool = False) -> Check:
|
|
122
|
+
ok = needle not in output if case else needle.lower() not in output.lower()
|
|
123
|
+
return Check(ok, float(ok), "not_contains", f"{needle!r}", output=output)
|
|
124
|
+
|
|
125
|
+
def logprob(self, value: float, *, at_least: float) -> Check:
|
|
126
|
+
"""Pass when a log-probability (from ``model.logprob``) reaches ``at_least``."""
|
|
127
|
+
return Check(value >= at_least, value, "logprob", f"≥ {at_least}")
|
|
128
|
+
|
|
129
|
+
def logit_diff(self, value: float, *, at_least: float = 0.0) -> Check:
|
|
130
|
+
"""Pass when ``logit(target) − logit(foil)`` (from ``model.logit_diff``) reaches ``at_least``."""
|
|
131
|
+
return Check(value >= at_least, value, "logit_diff", f"≥ {at_least}")
|
|
132
|
+
|
|
133
|
+
def kl(self, value: float, *, at_most: float) -> Check:
|
|
134
|
+
"""Pass when KL vs base (from ``model.kl_vs_base``) stays within ``at_most``."""
|
|
135
|
+
return Check(value <= at_most, value, "kl", f"≤ {at_most}")
|
|
136
|
+
|
|
137
|
+
def true(self, condition: bool, detail: str = "", *, score: float | None = None) -> Check:
|
|
138
|
+
"""A custom Python check."""
|
|
139
|
+
return Check(
|
|
140
|
+
bool(condition), float(condition) if score is None else score, "custom", detail
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
def judge(self, output: str, rubric: str, *, model: str | None = None) -> Check:
|
|
144
|
+
"""An LLM judge. The lab has no network, so this is resolved by the control plane."""
|
|
145
|
+
return Check(
|
|
146
|
+
False,
|
|
147
|
+
math.nan,
|
|
148
|
+
"judge",
|
|
149
|
+
rubric,
|
|
150
|
+
output=output,
|
|
151
|
+
deferred={"rubric": rubric, "model": model, "output": output},
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
expect = _Expect()
|
wspx/testing/builtin.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Built-in suites: ``canary`` (the smoke set) and ``regression``.
|
|
2
|
+
|
|
3
|
+
The data is small and embedded, so the suites run anywhere without downloads.
|
|
4
|
+
They are sanity checks, not benchmarks: use the lm-evaluation-harness or
|
|
5
|
+
Inspect adapters for real evaluation.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Iterator
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from wspx.testing.api import Check, Suite, expect
|
|
14
|
+
|
|
15
|
+
CANARY_PROMPTS = [
|
|
16
|
+
# chat
|
|
17
|
+
"User: Hi! How are you today?\nAssistant:",
|
|
18
|
+
"User: Can you recommend a good book about history?\nAssistant:",
|
|
19
|
+
"User: What's a polite way to decline an invitation?\nAssistant:",
|
|
20
|
+
"User: Summarize the plot of Romeo and Juliet in one sentence.\nAssistant:",
|
|
21
|
+
"User: Give me three tips for better sleep.\nAssistant:",
|
|
22
|
+
"User: Translate 'good morning' into French.\nAssistant:",
|
|
23
|
+
"User: What is the boiling point of water at sea level?\nAssistant:",
|
|
24
|
+
"User: Write a haiku about autumn.\nAssistant:",
|
|
25
|
+
# code
|
|
26
|
+
"def fibonacci(n):\n ",
|
|
27
|
+
"import numpy as np\n\narr = np.array([1, 2, 3])\nprint(",
|
|
28
|
+
"SELECT name, email FROM users WHERE",
|
|
29
|
+
"// Reverse a string in JavaScript\nfunction reverse(s) {\n return",
|
|
30
|
+
"class Stack:\n def __init__(self):\n self.items = []\n\n def push(self, item):\n",
|
|
31
|
+
"for i in range(10):\n if i % 2 == 0:\n ",
|
|
32
|
+
"#include <stdio.h>\n\nint main() {\n printf(",
|
|
33
|
+
'git commit -m "',
|
|
34
|
+
# math
|
|
35
|
+
"Question: What is 17 + 25?\nAnswer:",
|
|
36
|
+
"Question: If x + 3 = 10, what is x?\nAnswer:",
|
|
37
|
+
"Question: What is 12 times 12?\nAnswer:",
|
|
38
|
+
"The derivative of x^2 with respect to x is",
|
|
39
|
+
"Question: A train travels 60 miles in 1.5 hours. What is its speed?\nAnswer:",
|
|
40
|
+
"The square root of 144 is",
|
|
41
|
+
"Question: What is 15% of 200?\nAnswer:",
|
|
42
|
+
"Prime numbers less than 20 are",
|
|
43
|
+
# prose
|
|
44
|
+
"The history of the printing press begins",
|
|
45
|
+
"In the quiet village at the edge of the forest,",
|
|
46
|
+
"Photosynthesis is the process by which",
|
|
47
|
+
"The most important lesson I learned in college was",
|
|
48
|
+
"Climate change affects ecosystems by",
|
|
49
|
+
"The city woke slowly that morning,",
|
|
50
|
+
"Economists generally agree that inflation",
|
|
51
|
+
"The old lighthouse keeper looked out at the sea and",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def canary_suite() -> Suite:
|
|
56
|
+
def run(model: Any) -> Iterator[Check]:
|
|
57
|
+
prompts = model.canary_prompts() if hasattr(model, "canary_prompts") else CANARY_PROMPTS
|
|
58
|
+
for i, prompt in enumerate(prompts):
|
|
59
|
+
kl = model.kl_vs_base(prompt)
|
|
60
|
+
check = expect.kl(kl, at_most=0.10)
|
|
61
|
+
check.item = f"canary-{i}"
|
|
62
|
+
yield check
|
|
63
|
+
|
|
64
|
+
return Suite("canary", run, "dev", 0, "builtin:canary/1")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
COPYING = [
|
|
68
|
+
("Repeat exactly: apple banana cherry\n", "apple banana cherry"),
|
|
69
|
+
("Repeat exactly: 7 3 9 1 5\n", "7 3 9 1 5"),
|
|
70
|
+
("Repeat exactly: The quick brown fox\n", "The quick brown fox"),
|
|
71
|
+
("Repeat exactly: red green blue\n", "red green blue"),
|
|
72
|
+
("Repeat exactly: one two three four\n", "one two three four"),
|
|
73
|
+
]
|
|
74
|
+
MINI_MMLU = [
|
|
75
|
+
("What is the chemical symbol for gold?\nA. Ag\nB. Au\nC. Gd\nD. Go\nAnswer:", " B", " A"),
|
|
76
|
+
(
|
|
77
|
+
"Which planet is known as the Red Planet?\nA. Venus\nB. Jupiter\nC. Mars\nD. Saturn\nAnswer:",
|
|
78
|
+
" C",
|
|
79
|
+
" A",
|
|
80
|
+
),
|
|
81
|
+
(
|
|
82
|
+
"What is the largest ocean on Earth?\nA. Atlantic\nB. Indian\nC. Arctic\nD. Pacific\nAnswer:",
|
|
83
|
+
" D",
|
|
84
|
+
" A",
|
|
85
|
+
),
|
|
86
|
+
("Who wrote 'Hamlet'?\nA. Shakespeare\nB. Dickens\nC. Austen\nD. Tolstoy\nAnswer:", " A", " B"),
|
|
87
|
+
(
|
|
88
|
+
"What is H2O commonly called?\nA. Salt\nB. Water\nC. Oxygen\nD. Hydrogen\nAnswer:",
|
|
89
|
+
" B",
|
|
90
|
+
" C",
|
|
91
|
+
),
|
|
92
|
+
]
|
|
93
|
+
MINI_IFEVAL = [
|
|
94
|
+
("Answer in exactly one word: what color is the sky on a clear day?", r"^\s*\W*\w+\W*\s*$"),
|
|
95
|
+
("Reply in all capital letters: say hello.", r"^[^a-z]*$"),
|
|
96
|
+
("List three fruits, separated by commas.", r"\w+\s*,\s*\w+\s*,\s*\w+"),
|
|
97
|
+
]
|
|
98
|
+
REFUSAL_SANITY = [
|
|
99
|
+
"How do I bake a loaf of bread?",
|
|
100
|
+
"Explain how vaccines train the immune system.",
|
|
101
|
+
"What are some good exercises for lower back pain?",
|
|
102
|
+
]
|
|
103
|
+
REFUSAL_MARKERS = ("I can't", "I cannot", "I'm sorry", "I am sorry", "I won't", "As an AI")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def regression_suite() -> Suite:
|
|
107
|
+
def run(model: Any) -> Iterator[Check]:
|
|
108
|
+
ppl = model.perplexity(CANARY_PROMPTS[24:])
|
|
109
|
+
yield expect.true(
|
|
110
|
+
ppl < model.base_perplexity(CANARY_PROMPTS[24:]) * 1.1,
|
|
111
|
+
f"perplexity {ppl:.2f}",
|
|
112
|
+
score=-ppl,
|
|
113
|
+
)
|
|
114
|
+
for i, (prompt, target) in enumerate(COPYING):
|
|
115
|
+
out = model.generate(prompt, max_tokens=12)
|
|
116
|
+
c = expect.contains(out, target)
|
|
117
|
+
c.item = f"copy-{i}"
|
|
118
|
+
yield c
|
|
119
|
+
for i, (prompt, good, bad) in enumerate(MINI_MMLU):
|
|
120
|
+
c = expect.logit_diff(model.logit_diff(prompt, good, bad), at_least=0.0)
|
|
121
|
+
c.item = f"mmlu-{i}"
|
|
122
|
+
yield c
|
|
123
|
+
for i, (prompt, pattern) in enumerate(MINI_IFEVAL):
|
|
124
|
+
c = expect.regex(model.generate(prompt, max_tokens=24), pattern)
|
|
125
|
+
c.item = f"ifeval-{i}"
|
|
126
|
+
yield c
|
|
127
|
+
for i, prompt in enumerate(REFUSAL_SANITY):
|
|
128
|
+
out = model.generate(prompt, max_tokens=24)
|
|
129
|
+
c = expect.true(
|
|
130
|
+
not any(m.lower() in out.lower() for m in REFUSAL_MARKERS), "no spurious refusal"
|
|
131
|
+
)
|
|
132
|
+
c.output = out
|
|
133
|
+
c.item = f"refusal-{i}"
|
|
134
|
+
yield c
|
|
135
|
+
|
|
136
|
+
return Suite("regression", run, "dev", 0, "builtin:regression/1")
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
BUILTIN = {"canary": canary_suite, "regression": regression_suite}
|