recursa 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- recursa/__init__.py +25 -0
- recursa/adapter.py +134 -0
- recursa/auth.py +141 -0
- recursa/cli.py +378 -0
- recursa/data.py +238 -0
- recursa/evals.py +256 -0
- recursa/isolation.py +79 -0
- recursa/learner.py +553 -0
- recursa/llm.py +42 -0
- recursa/optimize.py +133 -0
- recursa/project.py +75 -0
- recursa/redact.py +45 -0
- recursa/reflect.py +112 -0
- recursa/stats.py +143 -0
- recursa/store.py +295 -0
- recursa/trace.py +149 -0
- recursa/types.py +152 -0
- recursa/ui.py +173 -0
- recursa/worker.py +80 -0
- recursa-0.1.0.dist-info/METADATA +157 -0
- recursa-0.1.0.dist-info/RECORD +25 -0
- recursa-0.1.0.dist-info/WHEEL +5 -0
- recursa-0.1.0.dist-info/entry_points.txt +2 -0
- recursa-0.1.0.dist-info/licenses/LICENSE +202 -0
- recursa-0.1.0.dist-info/top_level.txt +1 -0
recursa/__init__.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Recursa: agents that get better every run.
|
|
2
|
+
|
|
3
|
+
v1 proves one claim: on one reference agent, Recursa proposes a system-prompt
|
|
4
|
+
change that a human approves and that improves a pre-declared primary metric on a
|
|
5
|
+
sealed evaluation set, with no guardrail regression.
|
|
6
|
+
"""
|
|
7
|
+
from . import evals, trace
|
|
8
|
+
from .adapter import Agent, Context, wrap
|
|
9
|
+
from .evals import EvalSuite
|
|
10
|
+
from .project import Config, Project, load_config
|
|
11
|
+
from .stats import GateConfig
|
|
12
|
+
from .trace import SandboxViolation, tool
|
|
13
|
+
from .types import Example, Result
|
|
14
|
+
|
|
15
|
+
__all__ = ["Agent", "Config", "Context", "EvalSuite", "Example", "GateConfig", "Project", "Result",
|
|
16
|
+
"SandboxViolation", "evals", "load_config", "tool", "trace", "wrap", "Learner"]
|
|
17
|
+
__version__ = "0.1.0"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def __getattr__(name: str):
|
|
21
|
+
if name == "Learner": # imported lazily so the optimizer worker stays light
|
|
22
|
+
from .learner import Learner
|
|
23
|
+
|
|
24
|
+
return Learner
|
|
25
|
+
raise AttributeError(name)
|
recursa/adapter.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Adapter contract for v1: an async Python function plus learnable surfaces.
|
|
2
|
+
|
|
3
|
+
async def my_agent(task: dict, ctx: rc.Context) -> Any: ...
|
|
4
|
+
agent = rc.wrap(my_agent, name="triage",
|
|
5
|
+
surfaces={"system_prompt": PROMPT},
|
|
6
|
+
model={"id": "claude-opus-5-5"},
|
|
7
|
+
tools=[lookup_order])
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
import copy
|
|
13
|
+
import hashlib
|
|
14
|
+
import secrets
|
|
15
|
+
import time
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from typing import Any, Awaitable, Callable
|
|
18
|
+
|
|
19
|
+
from . import trace as _trace
|
|
20
|
+
from .types import Result, Run, Step, new_id
|
|
21
|
+
|
|
22
|
+
V1_SURFACES = ("system_prompt",)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class Context:
|
|
27
|
+
"""What the agent sees at run time: the effective configuration of one version."""
|
|
28
|
+
|
|
29
|
+
surfaces: dict[str, Any]
|
|
30
|
+
model: dict[str, Any]
|
|
31
|
+
version: str
|
|
32
|
+
mode: str
|
|
33
|
+
tools: dict[str, Callable[..., Any]] = field(default_factory=dict)
|
|
34
|
+
# Deterministic per (example, sample) in eval mode; pass it to providers that accept a seed.
|
|
35
|
+
seed: int = 0
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def system_prompt(self) -> str:
|
|
39
|
+
return self.surfaces["system_prompt"]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
AgentFn = Callable[[dict[str, Any], Context], Awaitable[Any]]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Agent:
|
|
46
|
+
def __init__(self, fn: AgentFn, name: str, surfaces: dict[str, Any], model: dict[str, Any] | None = None,
|
|
47
|
+
tools: list[Callable[..., Any]] | None = None, max_attempts: int = 1):
|
|
48
|
+
unknown = set(surfaces) - set(V1_SURFACES)
|
|
49
|
+
if unknown:
|
|
50
|
+
raise ValueError(f"v1 supports only these learnable surfaces: {V1_SURFACES}; got {sorted(unknown)}")
|
|
51
|
+
if "system_prompt" not in surfaces:
|
|
52
|
+
raise ValueError("the adapter must expose the 'system_prompt' surface")
|
|
53
|
+
self.fn = fn
|
|
54
|
+
self.name = name
|
|
55
|
+
self._surfaces = dict(surfaces)
|
|
56
|
+
self.model = dict(model or {})
|
|
57
|
+
self.tools = list(tools or [])
|
|
58
|
+
self.tool_manifest = _trace.tool_manifest(self.tools)
|
|
59
|
+
self.max_attempts = max(1, int(max_attempts))
|
|
60
|
+
|
|
61
|
+
# -- learnable surfaces ---------------------------------------------------
|
|
62
|
+
def get_surface(self, name: str) -> Any:
|
|
63
|
+
return self._surfaces[name]
|
|
64
|
+
|
|
65
|
+
def surfaces(self) -> dict[str, Any]:
|
|
66
|
+
return dict(self._surfaces)
|
|
67
|
+
|
|
68
|
+
def default_config(self) -> dict[str, Any]:
|
|
69
|
+
"""The full effective runtime configuration an AgentVersion pins."""
|
|
70
|
+
return {
|
|
71
|
+
"surfaces": copy.deepcopy(self._surfaces),
|
|
72
|
+
"model": copy.deepcopy(self.model),
|
|
73
|
+
"tools": copy.deepcopy(self.tool_manifest),
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
# -- running --------------------------------------------------------------
|
|
77
|
+
async def run(self, task: dict[str, Any], config: dict[str, Any], version: str, mode: str = "online",
|
|
78
|
+
example_id: str | None = None, sample: int = 0) -> Run:
|
|
79
|
+
"""Run once. Retries are attempts nested inside one run id."""
|
|
80
|
+
if config.get("tools", self.tool_manifest) != self.tool_manifest:
|
|
81
|
+
raise RuntimeError("version pins a different tool manifest than this agent registers; refusing to run")
|
|
82
|
+
ctx = Context(surfaces=dict(config["surfaces"]), model=dict(config.get("model", {})), version=version,
|
|
83
|
+
mode=mode, tools={m["name"]: t for m, t in zip(self.tool_manifest, sorted(
|
|
84
|
+
self.tools, key=lambda t: t.__recursa_tool__["name"]))},
|
|
85
|
+
seed=_seed(example_id, sample) if mode == "eval" else secrets.randbits(31))
|
|
86
|
+
rec = _trace.RunRecorder(mode=mode)
|
|
87
|
+
started = time.time()
|
|
88
|
+
t0 = time.perf_counter()
|
|
89
|
+
output: Any = None
|
|
90
|
+
status = "error"
|
|
91
|
+
error: str | None = None
|
|
92
|
+
with _trace.recording(rec):
|
|
93
|
+
for attempt in range(self.max_attempts):
|
|
94
|
+
rec.attempt = attempt
|
|
95
|
+
try:
|
|
96
|
+
out = await self.fn(task, ctx)
|
|
97
|
+
res = out if isinstance(out, Result) else Result(output=out)
|
|
98
|
+
output, status, error = res.output, res.status, None
|
|
99
|
+
if status == "ok":
|
|
100
|
+
break
|
|
101
|
+
except asyncio.CancelledError:
|
|
102
|
+
status, error = "cancelled", "cancelled"
|
|
103
|
+
break
|
|
104
|
+
except _trace.SandboxViolation as e:
|
|
105
|
+
status, error = "error", f"SandboxViolation: {e}"
|
|
106
|
+
break # fail closed, never retry a sandbox breach
|
|
107
|
+
except Exception as e: # noqa: BLE001 - agent code may raise anything
|
|
108
|
+
status, error = "error", f"{type(e).__name__}: {e}"
|
|
109
|
+
wall_ms = (time.perf_counter() - t0) * 1000
|
|
110
|
+
steps: list[Step] = rec.steps
|
|
111
|
+
return Run(
|
|
112
|
+
id=new_id("run"), agent=self.name, agent_version=version, mode=mode, input=task, output=output,
|
|
113
|
+
status=status, steps=steps, cost_usd=sum(s.cost_usd for s in steps),
|
|
114
|
+
latency_ms=_run_latency(steps, wall_ms), started_at=started,
|
|
115
|
+
context_snapshot={"surfaces": ctx.surfaces, "model": ctx.model}, error=error,
|
|
116
|
+
example_id=example_id, sample=sample,
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _seed(example_id: str | None, sample: int) -> int:
|
|
121
|
+
return int(hashlib.sha256(f"{example_id}:{sample}".encode()).hexdigest()[:8], 16)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _run_latency(steps: list[Step], wall_ms: float) -> float:
|
|
125
|
+
# Simulators report per-step latency; when every step does, use their sum so
|
|
126
|
+
# benchmark latency is deterministic. Otherwise wall-clock time is the truth.
|
|
127
|
+
if steps and all(s.provider_metadata.get("simulated_latency") for s in steps):
|
|
128
|
+
return max(sum(s.latency_ms for s in steps), 0.0)
|
|
129
|
+
return wall_ms
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def wrap(fn: AgentFn, name: str, surfaces: dict[str, Any], model: dict[str, Any] | None = None,
|
|
133
|
+
tools: list[Callable[..., Any]] | None = None, max_attempts: int = 1) -> Agent:
|
|
134
|
+
return Agent(fn, name=name, surfaces=surfaces, model=model, tools=tools, max_attempts=max_attempts)
|
recursa/auth.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Reviewer identity for v1: local passphrase accounts and short-lived signed tokens.
|
|
2
|
+
|
|
3
|
+
* ``reviewers.yaml`` (in the repo) names reviewers, their surfaces and the quorum.
|
|
4
|
+
* ``<home>/auth/credentials.json`` holds scrypt hashes of reviewer passphrases.
|
|
5
|
+
* ``<home>/auth/secret`` (0600) signs short-lived HMAC tokens (default 15 minutes).
|
|
6
|
+
|
|
7
|
+
OIDC/SSO is the post-v1 identity provider; the token format is the seam for it.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import base64
|
|
12
|
+
import hashlib
|
|
13
|
+
import hmac
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import secrets
|
|
17
|
+
import time
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
import yaml
|
|
22
|
+
|
|
23
|
+
TOKEN_TTL_S = 15 * 60
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class AuthError(PermissionError):
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class Reviewers:
|
|
32
|
+
reviewers: dict[str, dict[str, Any]]
|
|
33
|
+
quorum: dict[str, int]
|
|
34
|
+
|
|
35
|
+
@classmethod
|
|
36
|
+
def load(cls, path: str) -> "Reviewers":
|
|
37
|
+
if not os.path.exists(path):
|
|
38
|
+
raise AuthError(f"no reviewers file at {path}; every promotion needs a named human reviewer")
|
|
39
|
+
with open(path) as f:
|
|
40
|
+
raw = yaml.safe_load(f) or {}
|
|
41
|
+
revs = {r["id"]: r for r in raw.get("reviewers", [])}
|
|
42
|
+
if not revs:
|
|
43
|
+
raise AuthError("reviewers.yaml lists no reviewers")
|
|
44
|
+
quorum = {"default": 1, **{k: int(v) for k, v in (raw.get("quorum") or {}).items()}}
|
|
45
|
+
return cls(revs, quorum)
|
|
46
|
+
|
|
47
|
+
def can_review(self, reviewer: str, surface: str) -> bool:
|
|
48
|
+
r = self.reviewers.get(reviewer)
|
|
49
|
+
return bool(r) and (surface in r.get("surfaces", [surface]) or "*" in r.get("surfaces", []))
|
|
50
|
+
|
|
51
|
+
def quorum_for(self, surface: str) -> int:
|
|
52
|
+
return max(1, self.quorum.get(surface, self.quorum["default"]))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class Auth:
|
|
56
|
+
def __init__(self, auth_dir: str):
|
|
57
|
+
self.dir = auth_dir
|
|
58
|
+
os.makedirs(auth_dir, mode=0o700, exist_ok=True)
|
|
59
|
+
self.cred_path = os.path.join(auth_dir, "credentials.json")
|
|
60
|
+
self.secret_path = os.path.join(auth_dir, "secret")
|
|
61
|
+
if not os.path.exists(self.secret_path):
|
|
62
|
+
fd = os.open(self.secret_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
63
|
+
with os.fdopen(fd, "wb") as f:
|
|
64
|
+
f.write(secrets.token_bytes(32))
|
|
65
|
+
|
|
66
|
+
def _secret(self) -> bytes:
|
|
67
|
+
with open(self.secret_path, "rb") as f:
|
|
68
|
+
return f.read()
|
|
69
|
+
|
|
70
|
+
def _creds(self) -> dict[str, Any]:
|
|
71
|
+
if not os.path.exists(self.cred_path):
|
|
72
|
+
return {}
|
|
73
|
+
with open(self.cred_path) as f:
|
|
74
|
+
return json.load(f)
|
|
75
|
+
|
|
76
|
+
def set_passphrase(self, reviewer: str, passphrase: str) -> None:
|
|
77
|
+
if len(passphrase) < 10:
|
|
78
|
+
raise AuthError("passphrase must be at least 10 characters")
|
|
79
|
+
salt = secrets.token_bytes(16)
|
|
80
|
+
h = hashlib.scrypt(passphrase.encode(), salt=salt, n=2 ** 14, r=8, p=1)
|
|
81
|
+
creds = self._creds()
|
|
82
|
+
creds[reviewer] = {"salt": salt.hex(), "hash": h.hex()}
|
|
83
|
+
fd = os.open(self.cred_path + ".tmp", os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
84
|
+
with os.fdopen(fd, "w") as f:
|
|
85
|
+
json.dump(creds, f)
|
|
86
|
+
os.replace(self.cred_path + ".tmp", self.cred_path)
|
|
87
|
+
|
|
88
|
+
def verify_passphrase(self, reviewer: str, passphrase: str) -> bool:
|
|
89
|
+
c = self._creds().get(reviewer)
|
|
90
|
+
if not c:
|
|
91
|
+
return False
|
|
92
|
+
h = hashlib.scrypt(passphrase.encode(), salt=bytes.fromhex(c["salt"]), n=2 ** 14, r=8, p=1)
|
|
93
|
+
return hmac.compare_digest(h.hex(), c["hash"])
|
|
94
|
+
|
|
95
|
+
def issue_token(self, reviewer: str, passphrase: str, channel: str = "cli", ttl_s: int = TOKEN_TTL_S) -> str:
|
|
96
|
+
if not self.verify_passphrase(reviewer, passphrase):
|
|
97
|
+
raise AuthError("invalid reviewer or passphrase")
|
|
98
|
+
body = {"sub": reviewer, "ch": channel, "iat": int(time.time()), "exp": int(time.time()) + ttl_s,
|
|
99
|
+
"n": secrets.token_hex(8)}
|
|
100
|
+
raw = base64.urlsafe_b64encode(json.dumps(body, sort_keys=True).encode()).decode()
|
|
101
|
+
sig = hmac.new(self._secret(), raw.encode(), hashlib.sha256).hexdigest()
|
|
102
|
+
return f"{raw}.{sig}"
|
|
103
|
+
|
|
104
|
+
def verify_token(self, token: str | None) -> dict[str, Any]:
|
|
105
|
+
if not token or "." not in token:
|
|
106
|
+
raise AuthError("not logged in: run `recursa auth login <reviewer>`")
|
|
107
|
+
raw, sig = token.rsplit(".", 1)
|
|
108
|
+
want = hmac.new(self._secret(), raw.encode(), hashlib.sha256).hexdigest()
|
|
109
|
+
if not hmac.compare_digest(sig, want):
|
|
110
|
+
raise AuthError("invalid token signature")
|
|
111
|
+
body = json.loads(base64.urlsafe_b64decode(raw.encode()))
|
|
112
|
+
if body["exp"] < time.time():
|
|
113
|
+
raise AuthError("token expired: log in again")
|
|
114
|
+
return body
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def token_file(project: str) -> str:
|
|
118
|
+
override = os.environ.get("RECURSA_TOKEN_FILE")
|
|
119
|
+
if override:
|
|
120
|
+
return override
|
|
121
|
+
return os.path.expanduser(os.path.join("~", ".recursa", "tokens", f"{project}.token"))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def save_token(project: str, token: str) -> str:
|
|
125
|
+
path = token_file(project)
|
|
126
|
+
os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
|
|
127
|
+
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
128
|
+
with os.fdopen(fd, "w") as f:
|
|
129
|
+
f.write(token)
|
|
130
|
+
return path
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def load_token(project: str) -> str | None:
|
|
134
|
+
env = os.environ.get("RECURSA_TOKEN")
|
|
135
|
+
if env:
|
|
136
|
+
return env
|
|
137
|
+
try:
|
|
138
|
+
with open(token_file(project)) as f:
|
|
139
|
+
return f.read().strip()
|
|
140
|
+
except FileNotFoundError:
|
|
141
|
+
return None
|
recursa/cli.py
ADDED
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
"""``recursa`` command-line interface."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import difflib
|
|
6
|
+
import getpass
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import sys
|
|
10
|
+
import textwrap
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from .auth import Auth, AuthError, Reviewers, load_token, save_token
|
|
14
|
+
from .data import OpenSplits, Paths, SealedSets, load_jsonl, split_and_seal
|
|
15
|
+
from .project import load_config
|
|
16
|
+
|
|
17
|
+
CONFIG_TEMPLATE = """\
|
|
18
|
+
project: {project}
|
|
19
|
+
factory: {factory} # module:function returning recursa.Project
|
|
20
|
+
home: .recursa
|
|
21
|
+
reviewers: reviewers.yaml
|
|
22
|
+
expire_after_days: 7
|
|
23
|
+
dev_samples: 1
|
|
24
|
+
gate:
|
|
25
|
+
min_gain: 0.02 # +2 points absolute on task success
|
|
26
|
+
alpha: 0.05
|
|
27
|
+
budget: 20 # candidate evaluations per gate-set version
|
|
28
|
+
min_examples: 150
|
|
29
|
+
confirm_min_examples: 100
|
|
30
|
+
samples: 3
|
|
31
|
+
bootstrap: 10000
|
|
32
|
+
max_cost_increase: 0.10
|
|
33
|
+
max_latency_increase: 0.15
|
|
34
|
+
max_safety_regressions: 0
|
|
35
|
+
optimizer:
|
|
36
|
+
strategy: heuristic # or: claude
|
|
37
|
+
candidates: 4
|
|
38
|
+
gate_top_k: 1
|
|
39
|
+
model: claude-opus-5-5
|
|
40
|
+
budget:
|
|
41
|
+
max_cost_usd_per_cycle: 5.0
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
REVIEWERS_TEMPLATE = """\
|
|
45
|
+
reviewers:
|
|
46
|
+
- id: {reviewer}
|
|
47
|
+
name: {reviewer}
|
|
48
|
+
surfaces: [system_prompt]
|
|
49
|
+
quorum:
|
|
50
|
+
system_prompt: 1
|
|
51
|
+
default: 1
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _passphrase(prompt: str) -> str:
|
|
56
|
+
return os.environ.get("RECURSA_PASSPHRASE") or getpass.getpass(prompt)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _learner(args: argparse.Namespace):
|
|
60
|
+
from .learner import Learner
|
|
61
|
+
|
|
62
|
+
return Learner(load_config(args.config))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _print(obj: Any, as_json: bool = False) -> None:
|
|
66
|
+
if as_json or not isinstance(obj, str):
|
|
67
|
+
print(json.dumps(obj, indent=2, default=str))
|
|
68
|
+
else:
|
|
69
|
+
print(obj)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _maybe_remove(path: str, keep: bool) -> None:
|
|
73
|
+
if not keep:
|
|
74
|
+
os.remove(path)
|
|
75
|
+
print(f"removed {path} so the sealed labels exist only in encrypted form (use --keep-source to keep it)")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# -- commands ------------------------------------------------------------------------------
|
|
79
|
+
def cmd_init(args: argparse.Namespace) -> None:
|
|
80
|
+
if not os.path.exists(args.config):
|
|
81
|
+
if not (args.project and args.factory):
|
|
82
|
+
sys.exit("no recursa.yaml: pass --project and --factory to scaffold one")
|
|
83
|
+
with open(args.config, "w") as f:
|
|
84
|
+
f.write(CONFIG_TEMPLATE.format(project=args.project, factory=args.factory))
|
|
85
|
+
print(f"wrote {args.config}")
|
|
86
|
+
cfg = load_config(args.config)
|
|
87
|
+
if not os.path.exists(cfg.reviewers_path):
|
|
88
|
+
with open(cfg.reviewers_path, "w") as f:
|
|
89
|
+
f.write(REVIEWERS_TEMPLATE.format(reviewer=args.reviewer or getpass.getuser()))
|
|
90
|
+
print(f"wrote {cfg.reviewers_path}")
|
|
91
|
+
gi = os.path.join(cfg.root, ".gitignore")
|
|
92
|
+
lines = open(gi).read().splitlines() if os.path.exists(gi) else []
|
|
93
|
+
if cfg.home not in lines and f"{cfg.home}/" not in lines:
|
|
94
|
+
with open(gi, "a") as f:
|
|
95
|
+
f.write(f"{cfg.home}/\n")
|
|
96
|
+
learner = _learner(args)
|
|
97
|
+
vid = learner.bootstrap(args.reviewer or getpass.getuser())
|
|
98
|
+
print(f"initialised {cfg.project}: current version {vid}")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def cmd_data(args: argparse.Namespace) -> None:
|
|
102
|
+
cfg = load_config(args.config)
|
|
103
|
+
paths = Paths(cfg.home_path)
|
|
104
|
+
if args.data_cmd == "split":
|
|
105
|
+
out = split_and_seal(paths, load_jsonl(args.pool), seed=args.seed)
|
|
106
|
+
_print(out)
|
|
107
|
+
_maybe_remove(args.pool, args.keep_source)
|
|
108
|
+
elif args.data_cmd == "seal":
|
|
109
|
+
paths.ensure()
|
|
110
|
+
version = SealedSets(paths).seal(args.kind, load_jsonl(args.file))
|
|
111
|
+
print(f"sealed {args.kind} set {version}")
|
|
112
|
+
_maybe_remove(args.file, args.keep_source)
|
|
113
|
+
elif args.data_cmd == "rotate":
|
|
114
|
+
sealed = SealedSets(paths)
|
|
115
|
+
new_dev = sealed.retire(args.kind, OpenSplits(paths))
|
|
116
|
+
version = sealed.seal(args.kind, load_jsonl(args.file))
|
|
117
|
+
print(f"retired the previous {args.kind} set into dev (dev {new_dev}); sealed {version}")
|
|
118
|
+
_maybe_remove(args.file, args.keep_source)
|
|
119
|
+
elif args.data_cmd == "status":
|
|
120
|
+
from .store import Store
|
|
121
|
+
|
|
122
|
+
sealed = SealedSets(paths)
|
|
123
|
+
m = sealed.manifest()
|
|
124
|
+
ss = Store(paths.sealed_db)
|
|
125
|
+
for entry in m.get("gate", []):
|
|
126
|
+
entry["evaluations_used"] = ss.ledger_count(entry["version"], "gate")
|
|
127
|
+
entry["budget"] = cfg.gate.budget
|
|
128
|
+
_print({"open": OpenSplits(paths).manifest(), "sealed": m})
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def cmd_auth(args: argparse.Namespace) -> None:
|
|
132
|
+
cfg = load_config(args.config)
|
|
133
|
+
auth = Auth(Paths(cfg.home_path).auth)
|
|
134
|
+
if args.auth_cmd == "add-reviewer":
|
|
135
|
+
reviewers = Reviewers.load(cfg.reviewers_path)
|
|
136
|
+
if args.reviewer not in reviewers.reviewers:
|
|
137
|
+
sys.exit(f"{args.reviewer} is not listed in {cfg.reviewers}; add them there first")
|
|
138
|
+
p1 = _passphrase(f"new passphrase for {args.reviewer}: ")
|
|
139
|
+
if not os.environ.get("RECURSA_PASSPHRASE") and p1 != getpass.getpass("repeat: "):
|
|
140
|
+
sys.exit("passphrases differ")
|
|
141
|
+
auth.set_passphrase(args.reviewer, p1)
|
|
142
|
+
print(f"set passphrase for {args.reviewer}")
|
|
143
|
+
elif args.auth_cmd == "login":
|
|
144
|
+
token = auth.issue_token(args.reviewer, _passphrase(f"passphrase for {args.reviewer}: "))
|
|
145
|
+
path = save_token(cfg.project, token)
|
|
146
|
+
print(f"logged in as {args.reviewer} for 15 minutes (token in {path})")
|
|
147
|
+
elif args.auth_cmd == "whoami":
|
|
148
|
+
body = auth.verify_token(load_token(cfg.project))
|
|
149
|
+
print(f"{body['sub']} (expires in {int(body['exp'] - __import__('time').time())}s)")
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def cmd_eval(args: argparse.Namespace) -> None:
|
|
153
|
+
learner = _learner(args)
|
|
154
|
+
res = learner.eval(args.version, args.split, args.samples, use_cache=not args.no_cache)
|
|
155
|
+
_print(res.summary())
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def cmd_cycle(args: argparse.Namespace) -> None:
|
|
159
|
+
learner = _learner(args)
|
|
160
|
+
_print(learner.cycle(actor=args.actor, resume=args.resume))
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def cmd_gate(args: argparse.Namespace) -> None:
|
|
164
|
+
learner = _learner(args)
|
|
165
|
+
out = learner.confirm_pending(args.candidate) if args.confirm else learner.gate(args.candidate, actor="gate")
|
|
166
|
+
_print(out)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def render_packet(h: str, p: dict[str, Any]) -> str:
|
|
170
|
+
g, c = p["gate_report"], p["confirmation_report"]
|
|
171
|
+
gp, cp = g["primary"], c["primary"]
|
|
172
|
+
lines = [
|
|
173
|
+
f"Candidate {p['candidate_id']} (surface: {p['surface']}, baseline {p['baseline_version']}, "
|
|
174
|
+
f"source: {p['source']})",
|
|
175
|
+
f"Packet hash: {h}",
|
|
176
|
+
"",
|
|
177
|
+
"Rationale: " + p["rationale"],
|
|
178
|
+
"",
|
|
179
|
+
"Diff:", p["diff"] or "(no change)",
|
|
180
|
+
"Gate set {gv} (n={n}, attempt {a}/{b} on this version, alpha={al:.4f}):".format(
|
|
181
|
+
gv=g["gate_version"], n=g["n_examples"], a=g["attempts_on_version"], b=g["budget"], al=g["alpha"]),
|
|
182
|
+
f" success {gp['baseline']:.3f} -> {gp['candidate']:.3f} gain {gp['gain']:+.3f} "
|
|
183
|
+
f"95% CI [{gp['ci95'][0]:+.3f}, {gp['ci95'][1]:+.3f}] lower bound {gp['lower_bound']:+.3f}",
|
|
184
|
+
" cost ratio {:.3f} (max {:.2f}), p95 latency ratio {:.3f} (max {:.2f}), safety regressions {}".format(
|
|
185
|
+
g["guardrails"]["cost"]["ratio"], g["guardrails"]["cost"]["max_ratio"],
|
|
186
|
+
g["guardrails"]["latency_p95"]["ratio"], g["guardrails"]["latency_p95"]["max_ratio"],
|
|
187
|
+
g["guardrails"]["safety"]["regressions"]),
|
|
188
|
+
f"Confirmation set {c['confirm_version']} (n={c['n_examples']}): gain {cp['gain']:+.3f}, "
|
|
189
|
+
f"lower bound {cp['lower_bound']:+.3f}, verdict {c['verdict']}",
|
|
190
|
+
"",
|
|
191
|
+
"Insights:",
|
|
192
|
+
]
|
|
193
|
+
for i in p["insights"]:
|
|
194
|
+
lines.append(f" - {i['statement']} (x{i['frequency']}{'' if i['verified'] else ', UNVERIFIED'}; "
|
|
195
|
+
f"evidence {', '.join(i['evidence_run_ids'][:3])})")
|
|
196
|
+
lines.append("")
|
|
197
|
+
lines.append("Side by side (sampled gate cases; outputs shown are the samples that differ):")
|
|
198
|
+
for case in p["side_by_side"]:
|
|
199
|
+
lines.append(f" [{case['change']}] {json.dumps(case['input'])[:110]}")
|
|
200
|
+
lines.append(f" expected {json.dumps(case['expected'])}")
|
|
201
|
+
for who in ("baseline", "candidate"):
|
|
202
|
+
passes = case.get(f"{who}_passes")
|
|
203
|
+
tag = ""
|
|
204
|
+
if passes is not None:
|
|
205
|
+
marks = "".join("✓" if x else "✗" for x in passes)
|
|
206
|
+
tag = f"[{sum(passes)}/{len(passes)} {marks}, sample {case[f'{who}_sample'] + 1}] "
|
|
207
|
+
lines.append(f" {who:<9} {tag}{json.dumps(case[f'{who}_output'], default=str)[:100]}")
|
|
208
|
+
return "\n".join(lines)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def cmd_review(args: argparse.Namespace) -> None:
|
|
212
|
+
learner = _learner(args)
|
|
213
|
+
if args.review_cmd == "list":
|
|
214
|
+
q = learner.review_queue()
|
|
215
|
+
if not q:
|
|
216
|
+
print("no candidates awaiting review")
|
|
217
|
+
for c in q:
|
|
218
|
+
h, _ = learner.packet(c.id)
|
|
219
|
+
print(f"{c.id} baseline {c.baseline_version} packet {h[:12]} {c.rationale[:80]}")
|
|
220
|
+
elif args.review_cmd == "show":
|
|
221
|
+
h, p = learner.packet(args.candidate)
|
|
222
|
+
if args.json:
|
|
223
|
+
_print({"packet_hash": h, **p}, True)
|
|
224
|
+
else:
|
|
225
|
+
print(render_packet(h, p))
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _resolve_hash(learner, candidate: str, prefix: str) -> str:
|
|
229
|
+
h, _ = learner.packet(candidate)
|
|
230
|
+
if not prefix or not h.startswith(prefix) or len(prefix) < 12:
|
|
231
|
+
raise AuthError("pass --packet with at least 12 characters of the packet hash you reviewed "
|
|
232
|
+
"(see `recursa review show`)")
|
|
233
|
+
return h
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def cmd_decide(args: argparse.Namespace) -> None:
|
|
237
|
+
learner = _learner(args)
|
|
238
|
+
token = load_token(learner.cfg.project)
|
|
239
|
+
h = _resolve_hash(learner, args.candidate, args.packet)
|
|
240
|
+
edited = open(args.prompt_file).read() if getattr(args, "prompt_file", None) else None
|
|
241
|
+
decision = {"approve": "approve", "promote": "approve", "reject": "reject",
|
|
242
|
+
"request-changes": "request_changes"}[args.cmd]
|
|
243
|
+
_print(learner.decide(args.candidate, decision, token, h, reason=args.reason or "", edited_prompt=edited))
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def cmd_rollback(args: argparse.Namespace) -> None:
|
|
247
|
+
learner = _learner(args)
|
|
248
|
+
_print(learner.rollback(args.version, load_token(learner.cfg.project), args.reason))
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def cmd_versions(args: argparse.Namespace) -> None:
|
|
252
|
+
learner = _learner(args)
|
|
253
|
+
current = learner.store.current_version()
|
|
254
|
+
for v in learner.store.list_versions():
|
|
255
|
+
mark = "*" if v["id"] == current else " "
|
|
256
|
+
print(f"{mark} {v['id']} parent={v['parent']} candidate={v['candidate_id']} "
|
|
257
|
+
f"approvers={','.join(v['approvers'])} hash={v['config_hash'][:12]}")
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def cmd_diff(args: argparse.Namespace) -> None:
|
|
261
|
+
learner = _learner(args)
|
|
262
|
+
a, b = learner.version_config(args.a), learner.version_config(args.b)
|
|
263
|
+
for k in sorted(set(a) | set(b)):
|
|
264
|
+
if a.get(k) == b.get(k):
|
|
265
|
+
continue
|
|
266
|
+
if k == "surfaces":
|
|
267
|
+
for s in sorted(set(a[k]) | set(b[k])):
|
|
268
|
+
print("".join(difflib.unified_diff(str(a[k].get(s, "")).splitlines(True),
|
|
269
|
+
str(b[k].get(s, "")).splitlines(True), f"{args.a}:{s}",
|
|
270
|
+
f"{args.b}:{s}")))
|
|
271
|
+
else:
|
|
272
|
+
print(f"{k}: {json.dumps(a.get(k))} -> {json.dumps(b.get(k))}")
|
|
273
|
+
v = learner.store.get_version(args.b)
|
|
274
|
+
if v and v["candidate_id"]:
|
|
275
|
+
c = learner.store.get_candidate(v["candidate_id"])
|
|
276
|
+
print(f"why: {c.rationale}\napproved by: {', '.join(v['approvers'])}")
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def cmd_audit(args: argparse.Namespace) -> None:
|
|
280
|
+
learner = _learner(args)
|
|
281
|
+
if args.verify:
|
|
282
|
+
ok = learner.store.verify_audit_chain()
|
|
283
|
+
print("audit chain intact" if ok else "AUDIT CHAIN BROKEN")
|
|
284
|
+
sys.exit(0 if ok else 1)
|
|
285
|
+
for r in learner.store.audit_log()[-args.limit:]:
|
|
286
|
+
print(f"{r['seq']:>5} {r['actor']:<16} {r['action']:<24} {r['subject']} {json.dumps(r['details'])[:100]}")
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def cmd_ui(args: argparse.Namespace) -> None:
|
|
290
|
+
from .ui import serve
|
|
291
|
+
|
|
292
|
+
serve(_learner(args), host=args.host, port=args.port)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
296
|
+
p = argparse.ArgumentParser(prog="recursa", description="Eval-gated, human-approved agent improvement.")
|
|
297
|
+
p.add_argument("--config", default="recursa.yaml")
|
|
298
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
299
|
+
|
|
300
|
+
s = sub.add_parser("init", help="scaffold config and create the initial agent version")
|
|
301
|
+
s.add_argument("--project"); s.add_argument("--factory"); s.add_argument("--reviewer")
|
|
302
|
+
s.set_defaults(fn=cmd_init)
|
|
303
|
+
|
|
304
|
+
d = sub.add_parser("data", help="split, seal and rotate evaluation sets")
|
|
305
|
+
dsub = d.add_subparsers(dest="data_cmd", required=True)
|
|
306
|
+
x = dsub.add_parser("split"); x.add_argument("pool"); x.add_argument("--seed", default="recursa")
|
|
307
|
+
x.add_argument("--keep-source", action="store_true")
|
|
308
|
+
x = dsub.add_parser("seal"); x.add_argument("kind", choices=["gate", "confirm"]); x.add_argument("file")
|
|
309
|
+
x.add_argument("--keep-source", action="store_true")
|
|
310
|
+
x = dsub.add_parser("rotate"); x.add_argument("kind", choices=["gate", "confirm"]); x.add_argument("file")
|
|
311
|
+
x.add_argument("--keep-source", action="store_true")
|
|
312
|
+
dsub.add_parser("status")
|
|
313
|
+
d.set_defaults(fn=cmd_data)
|
|
314
|
+
|
|
315
|
+
a = sub.add_parser("auth", help="reviewer accounts and login")
|
|
316
|
+
asub = a.add_subparsers(dest="auth_cmd", required=True)
|
|
317
|
+
x = asub.add_parser("add-reviewer"); x.add_argument("reviewer")
|
|
318
|
+
x = asub.add_parser("login"); x.add_argument("reviewer")
|
|
319
|
+
asub.add_parser("whoami")
|
|
320
|
+
a.set_defaults(fn=cmd_auth)
|
|
321
|
+
|
|
322
|
+
s = sub.add_parser("eval", help="evaluate a version on train or dev")
|
|
323
|
+
s.add_argument("--version"); s.add_argument("--split", default="dev", choices=["train", "dev"])
|
|
324
|
+
s.add_argument("--samples", type=int); s.add_argument("--no-cache", action="store_true")
|
|
325
|
+
s.set_defaults(fn=cmd_eval)
|
|
326
|
+
|
|
327
|
+
s = sub.add_parser("cycle", help="run one learning cycle (proposes candidates for review)")
|
|
328
|
+
s.add_argument("--resume"); s.add_argument("--actor", default="cli")
|
|
329
|
+
s.set_defaults(fn=cmd_cycle)
|
|
330
|
+
|
|
331
|
+
s = sub.add_parser("gate", help="gate one proposed candidate (e.g. a reviewer's edit)")
|
|
332
|
+
s.add_argument("candidate"); s.add_argument("--confirm", action="store_true",
|
|
333
|
+
help="run the pending confirmation step")
|
|
334
|
+
s.set_defaults(fn=cmd_gate)
|
|
335
|
+
|
|
336
|
+
r = sub.add_parser("review", help="list or show candidates awaiting approval")
|
|
337
|
+
rsub = r.add_subparsers(dest="review_cmd", required=True)
|
|
338
|
+
rsub.add_parser("list")
|
|
339
|
+
x = rsub.add_parser("show"); x.add_argument("candidate"); x.add_argument("--json", action="store_true")
|
|
340
|
+
r.set_defaults(fn=cmd_review)
|
|
341
|
+
|
|
342
|
+
for name in ("approve", "promote", "reject", "request-changes"):
|
|
343
|
+
s = sub.add_parser(name, help=f"{name} a candidate (requires `recursa auth login`)")
|
|
344
|
+
s.add_argument("candidate")
|
|
345
|
+
s.add_argument("--packet", required=True, help="hash (prefix >= 12 chars) of the packet you reviewed")
|
|
346
|
+
s.add_argument("--reason", default="")
|
|
347
|
+
if name == "request-changes":
|
|
348
|
+
s.add_argument("--prompt-file", help="your edited system prompt; becomes a new candidate to gate")
|
|
349
|
+
s.set_defaults(fn=cmd_decide)
|
|
350
|
+
|
|
351
|
+
s = sub.add_parser("rollback", help="restore a previous human-approved version")
|
|
352
|
+
s.add_argument("version"); s.add_argument("--reason", required=True)
|
|
353
|
+
s.set_defaults(fn=cmd_rollback)
|
|
354
|
+
|
|
355
|
+
s = sub.add_parser("versions", help="list agent versions"); s.set_defaults(fn=cmd_versions)
|
|
356
|
+
s = sub.add_parser("diff", help="show what changed between two versions and why")
|
|
357
|
+
s.add_argument("a"); s.add_argument("b"); s.set_defaults(fn=cmd_diff)
|
|
358
|
+
s = sub.add_parser("audit", help="show or verify the audit log")
|
|
359
|
+
s.add_argument("--verify", action="store_true"); s.add_argument("--limit", type=int, default=50)
|
|
360
|
+
s.set_defaults(fn=cmd_audit)
|
|
361
|
+
s = sub.add_parser("ui", help="local review dashboard")
|
|
362
|
+
s.add_argument("--host", default="127.0.0.1"); s.add_argument("--port", type=int, default=8765)
|
|
363
|
+
s.set_defaults(fn=cmd_ui)
|
|
364
|
+
return p
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def main(argv: list[str] | None = None) -> None:
|
|
368
|
+
args = build_parser().parse_args(argv)
|
|
369
|
+
try:
|
|
370
|
+
args.fn(args)
|
|
371
|
+
except (AuthError, PermissionError) as e:
|
|
372
|
+
sys.exit(f"denied: {e}")
|
|
373
|
+
except (RuntimeError, KeyError, ValueError) as e:
|
|
374
|
+
sys.exit(f"error: {textwrap.shorten(str(e), 400)}")
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
if __name__ == "__main__":
|
|
378
|
+
main()
|