quirework 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- quirework/__init__.py +3 -0
- quirework/__main__.py +3 -0
- quirework/cli.py +431 -0
- quirework/core/__init__.py +1 -0
- quirework/core/artifact_paths.py +86 -0
- quirework/core/checks.py +235 -0
- quirework/core/frontmatter.py +83 -0
- quirework/core/globs.py +45 -0
- quirework/core/pinned.py +169 -0
- quirework/core/usage.py +604 -0
- quirework/delivery/__init__.py +1 -0
- quirework/delivery/artifacts.py +210 -0
- quirework/delivery/text.py +166 -0
- quirework/manuscript/__init__.py +1 -0
- quirework/manuscript/backfill.py +1972 -0
- quirework/manuscript/editions.py +111 -0
- quirework/manuscript/footprint.py +95 -0
- quirework/manuscript/formats.py +208 -0
- quirework/manuscript/lifecycle.py +191 -0
- quirework/manuscript/notes.py +2301 -0
- quirework/manuscript/pages.py +284 -0
- quirework/manuscript/quarto.py +416 -0
- quirework/manuscript/quotes.py +512 -0
- quirework/manuscript/rename.py +309 -0
- quirework/manuscript/sources.py +286 -0
- quirework/manuscript/trail.py +1875 -0
- quirework/manuscript/units.py +416 -0
- quirework/project.py +645 -0
- quirework/provenance/__init__.py +1 -0
- quirework/provenance/agyfilter.py +436 -0
- quirework/provenance/backup/__init__.py +469 -0
- quirework/provenance/backup/base.py +290 -0
- quirework/provenance/backup/command.py +107 -0
- quirework/provenance/backup/directory.py +142 -0
- quirework/provenance/backup/git.py +150 -0
- quirework/provenance/backup/rclone.py +153 -0
- quirework/provenance/capture/__init__.py +1 -0
- quirework/provenance/capture/commands.py +28 -0
- quirework/provenance/capture/extract_artifacts.py +110 -0
- quirework/provenance/capture/extract_files.py +118 -0
- quirework/provenance/capture/fetch_conversation.py +127 -0
- quirework/provenance/capture/fetch_gemini.py +149 -0
- quirework/provenance/capture/render_transcript.py +126 -0
- quirework/provenance/capture/transform.py +22 -0
- quirework/provenance/catalog.py +733 -0
- quirework/provenance/ccfilter.py +1126 -0
- quirework/provenance/codexfilter.py +883 -0
- quirework/provenance/coldread.py +1731 -0
- quirework/provenance/curate.py +67 -0
- quirework/provenance/humaninput.py +811 -0
- quirework/provenance/ingest.py +31 -0
- quirework/provenance/ingestion/__init__.py +1 -0
- quirework/provenance/ingestion/agy.py +192 -0
- quirework/provenance/ingestion/claude.py +493 -0
- quirework/provenance/ingestion/codex.py +296 -0
- quirework/provenance/lineage.py +148 -0
- quirework/provenance/metrics.py +1247 -0
- quirework/provenance/rulings.py +478 -0
- quirework/provenance/scope.py +73 -0
- quirework/provenance/search.py +455 -0
- quirework/provenance/semantic.py +475 -0
- quirework/provenance/show.py +1213 -0
- quirework/provenance/store.py +1271 -0
- quirework/provenance/trees.py +133 -0
- quirework/provenance/worktrees.py +199 -0
- quirework/providers/__init__.py +1 -0
- quirework/providers/blind.py +2192 -0
- quirework/providers/canary.py +685 -0
- quirework/providers/catalog.py +44 -0
- quirework/providers/claude_login.py +92 -0
- quirework/providers/exits.py +196 -0
- quirework/providers/harness.py +830 -0
- quirework/providers/images.py +92 -0
- quirework/providers/models.py +1284 -0
- quirework/providers/quota.py +588 -0
- quirework/providers/reads.py +474 -0
- quirework/providers/replies.py +175 -0
- quirework/publish/__init__.py +1 -0
- quirework/publish/backends.py +112 -0
- quirework/publish/bookstage.py +1086 -0
- quirework/publish/build.py +60 -0
- quirework/publish/editions.py +48 -0
- quirework/publish/epubcheck.py +287 -0
- quirework/publish/experiment.py +1097 -0
- quirework/publish/figfonts.py +234 -0
- quirework/publish/fignumbers.py +293 -0
- quirework/publish/figures.py +729 -0
- quirework/publish/folios.py +62 -0
- quirework/publish/fonts.py +307 -0
- quirework/publish/graybuild.py +73 -0
- quirework/publish/hugo.py +117 -0
- quirework/publish/kindle.js +411 -0
- quirework/publish/kindle.py +1393 -0
- quirework/publish/panelkit.py +918 -0
- quirework/publish/plans.py +651 -0
- quirework/publish/proof.py +792 -0
- quirework/publish/redpen.py +1654 -0
- quirework/publish/render.py +942 -0
- quirework/publish/targets.py +216 -0
- quirework/publish/variant.py +918 -0
- quirework/runs/__init__.py +1 -0
- quirework/runs/__main__.py +15 -0
- quirework/runs/commands.py +278 -0
- quirework/runs/coverage.py +86 -0
- quirework/runs/engine.py +2143 -0
- quirework/runs/feed.py +853 -0
- quirework/runs/observe.py +379 -0
- quirework/runs/storage.py +155 -0
- quirework/settings.py +492 -0
- quirework/sops/__init__.py +1 -0
- quirework/sops/catalog.py +386 -0
- quirework/sops/commands.py +567 -0
- quirework/sops/composition.py +382 -0
- quirework/sops/instruments.py +184 -0
- quirework/sops/personas.py +296 -0
- quirework/sops/review.py +2568 -0
- quirework/sops/room.py +2791 -0
- quirework/sops/roundtable.py +2578 -0
- quirework/sops/sections.py +169 -0
- quirework/sops/workshop.py +1629 -0
- quirework/templates/backup.toml +39 -0
- quirework/templates/book.toml +88 -0
- quirework/templates/run.toml +32 -0
- quirework/templates/site.toml +90 -0
- quirework/templates/tells.toml +64 -0
- quirework/templates/terminology.toml +20 -0
- quirework/templates/tmpfiles.conf +42 -0
- quirework/testing.py +345 -0
- quirework/text/__init__.py +1 -0
- quirework/text/jargon.py +545 -0
- quirework/text/mdwrap.py +1001 -0
- quirework/text/tells.py +547 -0
- quirework-0.1.0.dist-info/METADATA +444 -0
- quirework-0.1.0.dist-info/RECORD +138 -0
- quirework-0.1.0.dist-info/WHEEL +5 -0
- quirework-0.1.0.dist-info/entry_points.txt +2 -0
- quirework-0.1.0.dist-info/licenses/LICENSE +21 -0
- quirework-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1629 @@
|
|
|
1
|
+
"""Persistent prose experiments. Procedure: docs/workflows.md."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import fcntl
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import random
|
|
11
|
+
import re
|
|
12
|
+
import shutil
|
|
13
|
+
import signal
|
|
14
|
+
import subprocess
|
|
15
|
+
import sys
|
|
16
|
+
import tempfile
|
|
17
|
+
import time
|
|
18
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
19
|
+
from contextlib import contextmanager
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from quirework.core import usage as usage_records
|
|
23
|
+
from quirework.providers import blind
|
|
24
|
+
from quirework.providers import claude_login
|
|
25
|
+
from quirework.providers import exits
|
|
26
|
+
from quirework.sops import personas
|
|
27
|
+
from quirework.providers import quota
|
|
28
|
+
from quirework.providers import models as drivers
|
|
29
|
+
from quirework.providers.harness import bus_flags
|
|
30
|
+
from quirework.project import ConfigurationError, current_project, submit
|
|
31
|
+
from quirework.runs import feed
|
|
32
|
+
from quirework.runs.storage import shown
|
|
33
|
+
|
|
34
|
+
SCHEMA = 1
|
|
35
|
+
# `quota` owns the tier vocabulary and the order it implies, because the
|
|
36
|
+
# reason for the order is the provider's wall and that module owns the wall.
|
|
37
|
+
TIERS = quota.TIERS
|
|
38
|
+
|
|
39
|
+
# The author's default seats, owned as data so a plan can omit `writers` and
|
|
40
|
+
# the SOP does not carry a second copy of the list to rot.
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _settings():
|
|
44
|
+
"""Resolve paths from the current operation, without import-time state."""
|
|
45
|
+
from types import SimpleNamespace
|
|
46
|
+
|
|
47
|
+
DEFAULT_ROOT = current_project().state_root
|
|
48
|
+
ROSTER = current_project().path("sops.roster")
|
|
49
|
+
return SimpleNamespace(DEFAULT_ROOT=DEFAULT_ROOT, ROSTER=ROSTER)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def digest(path):
|
|
53
|
+
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def read_json(path):
|
|
57
|
+
return json.loads(Path(path).read_text())
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def write_json(path, value):
|
|
61
|
+
path = Path(path)
|
|
62
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
63
|
+
with tempfile.NamedTemporaryFile(mode="w", dir=path.parent, delete=False) as f:
|
|
64
|
+
json.dump(value, f, indent=2, ensure_ascii=False)
|
|
65
|
+
f.write("\n")
|
|
66
|
+
f.flush()
|
|
67
|
+
os.fsync(f.fileno())
|
|
68
|
+
temporary = f.name
|
|
69
|
+
os.replace(temporary, path)
|
|
70
|
+
directory_fd = os.open(path.parent, os.O_DIRECTORY)
|
|
71
|
+
try:
|
|
72
|
+
os.fsync(directory_fd)
|
|
73
|
+
finally:
|
|
74
|
+
os.close(directory_fd)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def safe_rel(value):
|
|
78
|
+
p = Path(value)
|
|
79
|
+
if p.is_absolute() or not value or ".." in p.parts or p == Path("."):
|
|
80
|
+
raise ValueError(f"expected a relative path without traversal: {value!r}")
|
|
81
|
+
return p
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def identifier(value):
|
|
85
|
+
if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*", value):
|
|
86
|
+
raise ValueError(f"invalid experiment/writer identifier: {value!r}")
|
|
87
|
+
return value
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def model_directory_name(harness, model):
|
|
91
|
+
"""Readable model identity, with the harness named once."""
|
|
92
|
+
model = model.lower()
|
|
93
|
+
stem = model if model.startswith(harness + "-") else f"{harness}-{model}"
|
|
94
|
+
stem = re.sub(r"[^a-z0-9._-]+", "-", stem).strip("-._")
|
|
95
|
+
return identifier(stem)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def tree_hashes(root):
|
|
99
|
+
root = Path(root)
|
|
100
|
+
if root.is_symlink():
|
|
101
|
+
raise ValueError(f"symlink at tree root: {root}")
|
|
102
|
+
out = {}
|
|
103
|
+
for p in sorted(root.rglob("*")):
|
|
104
|
+
if p.is_symlink():
|
|
105
|
+
raise ValueError(f"symlink in frozen material: {p}")
|
|
106
|
+
if p.is_file():
|
|
107
|
+
out[str(p.relative_to(root))] = digest(p)
|
|
108
|
+
elif not p.is_dir():
|
|
109
|
+
raise ValueError(f"non-regular file: {p}")
|
|
110
|
+
return out
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def process_identity(pid):
|
|
114
|
+
try:
|
|
115
|
+
fields = Path(f"/proc/{pid}/stat").read_text().rsplit(") ", 1)[1].split()
|
|
116
|
+
return None if fields[0] == "Z" else fields[19]
|
|
117
|
+
except FileNotFoundError:
|
|
118
|
+
return None
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@contextmanager
|
|
122
|
+
def lock(path):
|
|
123
|
+
with Path(path).open("a+") as f:
|
|
124
|
+
try:
|
|
125
|
+
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
126
|
+
except BlockingIOError as exc:
|
|
127
|
+
raise RuntimeError(f"another operator holds {path}") from exc
|
|
128
|
+
yield
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
@contextmanager
|
|
132
|
+
def harness_slot(harness):
|
|
133
|
+
from quirework.runs.storage import slot
|
|
134
|
+
|
|
135
|
+
with slot(harness):
|
|
136
|
+
yield
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _git(repo, *args):
|
|
140
|
+
return subprocess.check_output(["git", "-C", str(repo), *args])
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def prepare(plan_path, destination, *, resolve_models=True):
|
|
144
|
+
"""Plan: assignment, brief, files, required_reads, outputs, writers.
|
|
145
|
+
|
|
146
|
+
File entries have path (inside input/) and source (inside source_root).
|
|
147
|
+
A source_revision freezes Git blobs; omitted uses the actual working file.
|
|
148
|
+
local_source instead names an explicit file relative to the plan directory.
|
|
149
|
+
Outputs are paths under drafts/, with optional min_bytes. Media are opaque
|
|
150
|
+
bytes: the assignment and chosen harness determine how to use them.
|
|
151
|
+
"""
|
|
152
|
+
plan_path, destination = Path(plan_path).resolve(), Path(destination).absolute()
|
|
153
|
+
plan = read_json(plan_path)
|
|
154
|
+
identifier(destination.name)
|
|
155
|
+
if destination.exists():
|
|
156
|
+
raise ValueError(f"experiment already exists: {destination}")
|
|
157
|
+
if not plan.get("assignment", "").strip() or not plan.get("brief", "").strip():
|
|
158
|
+
raise ValueError("assignment and brief must both contain text")
|
|
159
|
+
repo = Path(plan.get("source_root", current_project().root)).resolve()
|
|
160
|
+
revision = plan.get("source_revision")
|
|
161
|
+
if revision:
|
|
162
|
+
revision = (
|
|
163
|
+
_git(repo, "rev-parse", "--verify", revision + "^{commit}").decode().strip()
|
|
164
|
+
)
|
|
165
|
+
outputs = []
|
|
166
|
+
for entry in plan.get("outputs", []):
|
|
167
|
+
item = {"path": entry} if isinstance(entry, str) else dict(entry)
|
|
168
|
+
rel = safe_rel(item["path"])
|
|
169
|
+
if rel.parts[0] != "drafts" or len(rel.parts) < 2:
|
|
170
|
+
raise ValueError("outputs must be files below drafts/")
|
|
171
|
+
item.setdefault("min_bytes", 100)
|
|
172
|
+
if not isinstance(item["min_bytes"], int) or item["min_bytes"] < 1:
|
|
173
|
+
raise ValueError("output min_bytes must be a positive integer")
|
|
174
|
+
outputs.append(item)
|
|
175
|
+
if not outputs or len({i["path"] for i in outputs}) != len(outputs):
|
|
176
|
+
raise ValueError("declare unique output files")
|
|
177
|
+
roster_default = not plan.get("writers")
|
|
178
|
+
# The roster is the fallback for a plan that names no writers, so a plan
|
|
179
|
+
# naming its own needs none. One that is declared still lends its persona
|
|
180
|
+
# to a writer that names neither its own nor the plan's.
|
|
181
|
+
if roster_default or current_project().get("sops.roster") is not None:
|
|
182
|
+
roster_data = read_json(_settings().ROSTER)
|
|
183
|
+
else:
|
|
184
|
+
roster_data = {}
|
|
185
|
+
supplied_writers = roster_data["writers"] if roster_default else plan["writers"]
|
|
186
|
+
writers = []
|
|
187
|
+
for supplied in supplied_writers:
|
|
188
|
+
item = dict(supplied)
|
|
189
|
+
item["persona"] = personas.resolve(
|
|
190
|
+
item.get(
|
|
191
|
+
"persona",
|
|
192
|
+
plan.get("persona", roster_data.get("persona", "workshop-writer")),
|
|
193
|
+
)
|
|
194
|
+
)
|
|
195
|
+
if "id" in item:
|
|
196
|
+
identifier(item["id"])
|
|
197
|
+
if item.get("mode", "auto") not in ("auto", "manual"):
|
|
198
|
+
raise ValueError("writer mode must be auto or manual")
|
|
199
|
+
if "requested_model" not in item and "model" not in item:
|
|
200
|
+
raise ValueError("writer needs requested_model or model")
|
|
201
|
+
# Either name alone is enough. Requiring both made the common plan --
|
|
202
|
+
# copied from the roster minus its resolved spelling -- die on a bare
|
|
203
|
+
# KeyError that named a field the help text never called required.
|
|
204
|
+
item.setdefault("requested_model", item.get("model"))
|
|
205
|
+
item.setdefault("model", item["requested_model"])
|
|
206
|
+
if resolve_models:
|
|
207
|
+
resolved = drivers.resolve(
|
|
208
|
+
item["harness"],
|
|
209
|
+
item["requested_model"],
|
|
210
|
+
item["effort"],
|
|
211
|
+
item["model"],
|
|
212
|
+
)
|
|
213
|
+
resolved["requested_effort"] = item.get("requested_effort", item["effort"])
|
|
214
|
+
item.update(resolved)
|
|
215
|
+
item.setdefault("mode", "auto")
|
|
216
|
+
item.setdefault("tier", "standard")
|
|
217
|
+
if item["tier"] not in TIERS:
|
|
218
|
+
raise ValueError(f"writer tier must be one of {TIERS}: {item['tier']!r}")
|
|
219
|
+
item["capabilities"] = list(
|
|
220
|
+
item.get("capabilities", plan.get("capabilities", []))
|
|
221
|
+
)
|
|
222
|
+
unknown = set(item["capabilities"]) - {"image-generation"}
|
|
223
|
+
if unknown:
|
|
224
|
+
raise ValueError(f"unsupported capabilities: {sorted(unknown)}")
|
|
225
|
+
if "image-generation" in item["capabilities"] and item["harness"] != "agy":
|
|
226
|
+
raise ValueError(
|
|
227
|
+
"native image generation currently requires the agy adapter"
|
|
228
|
+
)
|
|
229
|
+
writers.append(item)
|
|
230
|
+
explicit = [item["id"] for item in writers if "id" in item]
|
|
231
|
+
if not writers or len(set(explicit)) != len(explicit):
|
|
232
|
+
raise ValueError("declare unique writers")
|
|
233
|
+
reserved = set(explicit)
|
|
234
|
+
for item in writers:
|
|
235
|
+
if "id" in item:
|
|
236
|
+
continue
|
|
237
|
+
stem = model_directory_name(item["harness"], item["model"])
|
|
238
|
+
name, attempt = stem, 2
|
|
239
|
+
while name in reserved:
|
|
240
|
+
name = f"{stem}-{attempt:02d}"
|
|
241
|
+
attempt += 1
|
|
242
|
+
item["id"] = name
|
|
243
|
+
reserved.add(name)
|
|
244
|
+
if resolve_models:
|
|
245
|
+
from quirework.providers import canary as canaries
|
|
246
|
+
|
|
247
|
+
canaries.require_models((f"writer {w['id']}", w["harness"]) for w in writers)
|
|
248
|
+
# Validate before creating the destination. Failed setup never masquerades
|
|
249
|
+
# as a launchable partial experiment.
|
|
250
|
+
files = {
|
|
251
|
+
"assignment.md": plan["assignment"].encode(),
|
|
252
|
+
"orientation.md": plan["brief"].encode(),
|
|
253
|
+
}
|
|
254
|
+
for entry in plan.get("files", []):
|
|
255
|
+
rel = safe_rel(entry["path"])
|
|
256
|
+
if str(rel) in files or any(
|
|
257
|
+
part
|
|
258
|
+
in ("AGENTS.md", "CLAUDE.md", "CODEX.md", "GEMINI.md", ".codex", ".claude")
|
|
259
|
+
for part in rel.parts
|
|
260
|
+
):
|
|
261
|
+
raise ValueError(f"duplicate or injected instruction path: {rel}")
|
|
262
|
+
if "local_source" in entry:
|
|
263
|
+
p = plan_path.parent / entry["local_source"]
|
|
264
|
+
if p.is_symlink():
|
|
265
|
+
raise ValueError(f"source is a symlink: {p}")
|
|
266
|
+
content = p.read_bytes()
|
|
267
|
+
else:
|
|
268
|
+
src = safe_rel(entry["source"])
|
|
269
|
+
if revision:
|
|
270
|
+
mode = (
|
|
271
|
+
_git(repo, "ls-tree", revision, "--", str(src)).decode().split()[0]
|
|
272
|
+
)
|
|
273
|
+
if mode != "100644" and mode != "100755":
|
|
274
|
+
raise ValueError(f"source is not a regular Git blob: {src}")
|
|
275
|
+
content = _git(repo, "show", f"{revision}:{src}")
|
|
276
|
+
else:
|
|
277
|
+
p = repo / src
|
|
278
|
+
if p.is_symlink():
|
|
279
|
+
raise ValueError(f"source is a symlink: {p}")
|
|
280
|
+
content = p.read_bytes()
|
|
281
|
+
files[str(rel)] = content
|
|
282
|
+
order_path = plan.get("reading_order", "READING_ORDER.md")
|
|
283
|
+
if order_path and str(safe_rel(order_path)) not in files:
|
|
284
|
+
# A library's order is visible in no file it contains, so it is the
|
|
285
|
+
# kind of convention that cannot transmit: a writer given thirty
|
|
286
|
+
# chapters cannot tell which follows which. Generated rather than
|
|
287
|
+
# asked for, because the one packet built by hand forgot it.
|
|
288
|
+
sources = [
|
|
289
|
+
safe_rel(entry["source"]).as_posix()
|
|
290
|
+
for entry in plan.get("files", [])
|
|
291
|
+
if "source" in entry
|
|
292
|
+
]
|
|
293
|
+
listing = _reading_order(repo, sources)
|
|
294
|
+
if listing:
|
|
295
|
+
files[str(safe_rel(order_path))] = listing.encode()
|
|
296
|
+
for rel in plan.get("required_reads", []):
|
|
297
|
+
if str(safe_rel(rel)) not in files:
|
|
298
|
+
raise ValueError(f"required reading is absent: {rel}")
|
|
299
|
+
destination.mkdir(parents=True, mode=0o700)
|
|
300
|
+
source = destination / "source"
|
|
301
|
+
source.mkdir()
|
|
302
|
+
for rel, content in files.items():
|
|
303
|
+
p = source / rel
|
|
304
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
305
|
+
p.write_bytes(content)
|
|
306
|
+
frozen = dict(
|
|
307
|
+
plan,
|
|
308
|
+
schema=SCHEMA,
|
|
309
|
+
writers_from_roster=roster_default,
|
|
310
|
+
source_root=str(repo),
|
|
311
|
+
source_revision=revision,
|
|
312
|
+
writers=writers,
|
|
313
|
+
outputs=outputs,
|
|
314
|
+
input_hashes=tree_hashes(source),
|
|
315
|
+
source_head=_git(repo, "rev-parse", "HEAD").decode().strip(),
|
|
316
|
+
prepared_at=time.time(),
|
|
317
|
+
supplied_bytes=sum(len(v) for v in files.values()),
|
|
318
|
+
)
|
|
319
|
+
write_json(destination / "plan.json", frozen)
|
|
320
|
+
(destination / "evaluation").mkdir()
|
|
321
|
+
(destination / "records").mkdir()
|
|
322
|
+
(destination / "experiment.md").write_text(
|
|
323
|
+
f"# {plan.get('title', destination.name)}\n\n"
|
|
324
|
+
f"Source revision: {revision or 'working files, frozen by hash'}.\n\n"
|
|
325
|
+
"Private experiment. plan.json freezes the assignment and inputs;\n"
|
|
326
|
+
"records/<writer>/state.json carries progress. Read the assignment\n"
|
|
327
|
+
"before deciding whether to run, continue manually, or evaluate.\n"
|
|
328
|
+
)
|
|
329
|
+
for item in writers:
|
|
330
|
+
_prepare_writer(destination, frozen, item)
|
|
331
|
+
write_json(
|
|
332
|
+
destination / "plan-digest.json", {"sha256": digest(destination / "plan.json")}
|
|
333
|
+
)
|
|
334
|
+
return frozen
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _reading_order(repo, sources):
|
|
338
|
+
"""The order of the staged units, from the work's own registry.
|
|
339
|
+
|
|
340
|
+
Matched by the path each file was staged from, never by its name, since
|
|
341
|
+
every post of a site is an `index.md`. Silent when no manuscript is
|
|
342
|
+
configured, or when fewer than three staged files are units: an
|
|
343
|
+
experiment supplying one passage needs no order.
|
|
344
|
+
"""
|
|
345
|
+
from quirework.manuscript import units as _units
|
|
346
|
+
|
|
347
|
+
try:
|
|
348
|
+
registry = _units.registry(current_project().path_at(repo, "manuscript.source"))
|
|
349
|
+
except (ConfigurationError, OSError):
|
|
350
|
+
return ""
|
|
351
|
+
staged = set(sources)
|
|
352
|
+
rows = [c for c in registry if c.rel in staged]
|
|
353
|
+
if len(rows) < 3:
|
|
354
|
+
return ""
|
|
355
|
+
lines = ["# Reading order", ""]
|
|
356
|
+
for c in rows:
|
|
357
|
+
lines.append(f"{c.number:>3}. {c.title}" if c.number else f" {c.title}")
|
|
358
|
+
if any(c.number for c in rows):
|
|
359
|
+
lines += ["", "Front and back matter carry no number."]
|
|
360
|
+
return "\n".join(lines + [""])
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def anonymize(root, output=None, *, seed=None, include_partial=False):
|
|
364
|
+
"""Shuffle collected drafts into evaluation/anon/ under sealed letters.
|
|
365
|
+
|
|
366
|
+
The key is written and never returned or printed. An editor who reads the
|
|
367
|
+
mapping before the drafts has spent the step and kept none of it, and a
|
|
368
|
+
reshuffle cannot undo that, because draft sizes are near-unique per writer.
|
|
369
|
+
Neither can a reshuffle undo per-writer byte counts seen any other way, so
|
|
370
|
+
seal before inspecting sizes; `report` withholds them until this has run.
|
|
371
|
+
|
|
372
|
+
`output` defaults to the plan's first declared output, because a default
|
|
373
|
+
naming one experiment's file silently fails in every later one. Only
|
|
374
|
+
writers whose turn actually completed are shuffled in: a failed partial
|
|
375
|
+
looks chapter-complete and is not a candidate. Pass include_partial to
|
|
376
|
+
read them anyway, deliberately.
|
|
377
|
+
|
|
378
|
+
Candidates come from the records on disk rather than from `plan["writers"]`.
|
|
379
|
+
A retry is a new writer that `retry` cannot add to the plan, because `load`
|
|
380
|
+
hash-verifies the plan and a changed one is refused. Reading the frozen list
|
|
381
|
+
here therefore dropped every retried seat from the seal, and dropped it
|
|
382
|
+
SILENTLY: a seven-seat run whose premium seat failed on quota and was
|
|
383
|
+
retried sealed six drafts and reported six, with nothing saying which seat
|
|
384
|
+
was gone or why. The failed original is still excluded by the completed
|
|
385
|
+
check, so a retried seat contributes its retry and not its partial.
|
|
386
|
+
"""
|
|
387
|
+
root = Path(root)
|
|
388
|
+
plan = read_json(root / "plan.json")
|
|
389
|
+
if output is None:
|
|
390
|
+
output = Path(plan["outputs"][0]["path"]).name
|
|
391
|
+
|
|
392
|
+
def _completed(writer):
|
|
393
|
+
state = root / "records" / writer / "state.json"
|
|
394
|
+
return state.is_file() and read_json(state).get("status") == "completed"
|
|
395
|
+
|
|
396
|
+
# Sorted for a deterministic candidate order before the shuffle; the
|
|
397
|
+
# shuffle is what seals, not the enumeration order.
|
|
398
|
+
candidates = sorted(p.name for p in (root / "writers").iterdir() if p.is_dir())
|
|
399
|
+
present = [
|
|
400
|
+
writer
|
|
401
|
+
for writer in candidates
|
|
402
|
+
if (root / "writers" / writer / "drafts" / output).is_file()
|
|
403
|
+
and (root / "writers" / writer / "drafts" / output).stat().st_size
|
|
404
|
+
and (include_partial or _completed(writer))
|
|
405
|
+
]
|
|
406
|
+
if not present:
|
|
407
|
+
raise ValueError(
|
|
408
|
+
f"no completed writer produced drafts/{output}"
|
|
409
|
+
if not include_partial
|
|
410
|
+
else f"no writer produced drafts/{output}"
|
|
411
|
+
)
|
|
412
|
+
letters = [c for c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"[: len(present)]]
|
|
413
|
+
shuffled = list(present)
|
|
414
|
+
random.Random(seed).shuffle(shuffled)
|
|
415
|
+
anon = root / "evaluation" / "anon"
|
|
416
|
+
if anon.exists():
|
|
417
|
+
shutil.rmtree(anon)
|
|
418
|
+
anon.mkdir(parents=True)
|
|
419
|
+
key = {}
|
|
420
|
+
for letter, writer in zip(letters, shuffled):
|
|
421
|
+
shutil.copyfile(
|
|
422
|
+
root / "writers" / writer / "drafts" / output,
|
|
423
|
+
anon / f"{letter}{Path(output).suffix}",
|
|
424
|
+
)
|
|
425
|
+
key[letter] = writer
|
|
426
|
+
write_json(root / "evaluation" / "identity-key.json", key)
|
|
427
|
+
return {
|
|
428
|
+
"drafts": len(key),
|
|
429
|
+
"letters": letters,
|
|
430
|
+
"anon": str(anon),
|
|
431
|
+
"key": str(root / "evaluation" / "identity-key.json"),
|
|
432
|
+
"warning": "read the drafts before opening the key; it is not printed",
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _prepare_writer(root, plan, item):
|
|
437
|
+
work = root / "writers" / item["id"]
|
|
438
|
+
work.mkdir(parents=True)
|
|
439
|
+
shutil.copytree(root / "source", work / "input")
|
|
440
|
+
(work / "drafts").mkdir()
|
|
441
|
+
required = ["orientation.md", "assignment.md", *plan.get("required_reads", [])]
|
|
442
|
+
text = "# Your assignment\n\n" + "Read these files in full before writing:\n\n"
|
|
443
|
+
text += "".join(f"- {work / 'input' / rel}\n" for rel in dict.fromkeys(required))
|
|
444
|
+
text += "\nOther supplied book files are available as reference.\n\n"
|
|
445
|
+
text += "The assignment determines what to deliver. Save these files:\n\n"
|
|
446
|
+
text += "".join(f"- {work / entry['path']}\n" for entry in plan["outputs"])
|
|
447
|
+
text += (
|
|
448
|
+
"\nWork with the supplied material. Stay inside this workspace; use no\n"
|
|
449
|
+
"web, external services or other agents. Inputs are read-only.\n"
|
|
450
|
+
"Write your contribution in drafts/. Keep any further alternatives\n"
|
|
451
|
+
"as separate files. Finish the assignment without asking for approval.\n"
|
|
452
|
+
)
|
|
453
|
+
if "image-generation" in item.get("capabilities", []):
|
|
454
|
+
text += "\nThis assignment additionally permits the harness image-generation tool.\n"
|
|
455
|
+
if item.get("persona"):
|
|
456
|
+
text = personas.compose(text, item["persona"])
|
|
457
|
+
(work / "START.md").write_text(text)
|
|
458
|
+
record = root / "records" / item["id"]
|
|
459
|
+
record.mkdir()
|
|
460
|
+
write_json(
|
|
461
|
+
record / "state.json",
|
|
462
|
+
{"status": "prepared", "config": item, "start_sha": digest(work / "START.md")},
|
|
463
|
+
)
|
|
464
|
+
# A human-operated session uses the same controller and adapters. Quote
|
|
465
|
+
# every path with shell quoting, not JSON or shell interpolation.
|
|
466
|
+
import shlex
|
|
467
|
+
|
|
468
|
+
command = [
|
|
469
|
+
sys.executable,
|
|
470
|
+
"-m",
|
|
471
|
+
"quirework.sops.workshop",
|
|
472
|
+
"manual",
|
|
473
|
+
str(root),
|
|
474
|
+
item["id"],
|
|
475
|
+
]
|
|
476
|
+
launcher = (
|
|
477
|
+
"#!/bin/sh\nset -eu\ncd "
|
|
478
|
+
+ shlex.quote(str(current_project().root))
|
|
479
|
+
+ "\n"
|
|
480
|
+
+ "export PYTHONPATH="
|
|
481
|
+
+ shlex.quote(str(Path(__file__).resolve().parents[2]))
|
|
482
|
+
+ "\nexec "
|
|
483
|
+
+ shlex.join(command)
|
|
484
|
+
+ "\n"
|
|
485
|
+
)
|
|
486
|
+
(record / "manual.sh").write_text(launcher)
|
|
487
|
+
(record / "manual.sh").chmod(0o700)
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def load(root, writer=None):
|
|
491
|
+
root = Path(root).resolve()
|
|
492
|
+
plan = read_json(root / "plan.json")
|
|
493
|
+
if plan.get("schema") != SCHEMA:
|
|
494
|
+
raise ValueError("unsupported experiment schema")
|
|
495
|
+
if digest(root / "plan.json") != read_json(root / "plan-digest.json")["sha256"]:
|
|
496
|
+
raise ValueError("frozen plan changed")
|
|
497
|
+
if writer is None:
|
|
498
|
+
return root, plan
|
|
499
|
+
identifier(writer)
|
|
500
|
+
record, work = root / "records" / writer, root / "writers" / writer
|
|
501
|
+
state = read_json(record / "state.json")
|
|
502
|
+
if tree_hashes(work / "input") != plan["input_hashes"]:
|
|
503
|
+
raise ValueError(f"writer input changed: {writer}")
|
|
504
|
+
if digest(work / "START.md") != state["start_sha"]:
|
|
505
|
+
raise ValueError(f"writer entry instructions changed: {writer}")
|
|
506
|
+
return root, plan, record, work, state
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _snapshot_driver(record):
|
|
510
|
+
directory = record / "driver"
|
|
511
|
+
directory.mkdir(exist_ok=True)
|
|
512
|
+
for p in (
|
|
513
|
+
Path(__file__),
|
|
514
|
+
Path(drivers.__file__),
|
|
515
|
+
Path(blind.__file__),
|
|
516
|
+
Path(__file__).resolve().parents[1] / "project.py",
|
|
517
|
+
):
|
|
518
|
+
shutil.copyfile(p, directory / p.name)
|
|
519
|
+
write_json(
|
|
520
|
+
directory / "versions.json",
|
|
521
|
+
{
|
|
522
|
+
"python": sys.version,
|
|
523
|
+
"implementation_version": __import__("quirework").__version__,
|
|
524
|
+
"files": tree_hashes(directory),
|
|
525
|
+
},
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def _archive_drafts(work, record):
|
|
530
|
+
"""Copy every draft, complete or not, before anything can fail."""
|
|
531
|
+
outputs = record / "artifacts"
|
|
532
|
+
outputs.mkdir(exist_ok=True)
|
|
533
|
+
for rel in tree_hashes(work / "drafts"):
|
|
534
|
+
src, dst = work / "drafts" / rel, outputs / rel
|
|
535
|
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
536
|
+
if dst.exists() and digest(src) != digest(dst):
|
|
537
|
+
raise RuntimeError(f"archived output changed: {rel}; create a new attempt")
|
|
538
|
+
if not dst.exists():
|
|
539
|
+
shutil.copyfile(src, dst)
|
|
540
|
+
return tree_hashes(outputs)
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def _missing_outputs(plan, work):
|
|
544
|
+
return [
|
|
545
|
+
item["path"]
|
|
546
|
+
for item in plan["outputs"]
|
|
547
|
+
if not (work / item["path"]).is_file()
|
|
548
|
+
or (work / item["path"]).stat().st_size < item["min_bytes"]
|
|
549
|
+
]
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def _collect(root, writer, *, external=None):
|
|
553
|
+
root, plan, record, work, state = load(root, writer)
|
|
554
|
+
if state.get("pid") and process_identity(state["pid"]) == state.get(
|
|
555
|
+
"process_identity"
|
|
556
|
+
):
|
|
557
|
+
raise RuntimeError("writer process is still alive; wait, do not resubmit")
|
|
558
|
+
if (
|
|
559
|
+
state["status"] == "running"
|
|
560
|
+
and not state.get("pid")
|
|
561
|
+
and not state.get("ended_at")
|
|
562
|
+
):
|
|
563
|
+
raise RuntimeError(
|
|
564
|
+
"pending launch has no process identity; inspect the host before retiring or collecting it"
|
|
565
|
+
)
|
|
566
|
+
if state["status"] == "completed":
|
|
567
|
+
if tree_hashes(record / "transcripts") != state["record_hashes"]:
|
|
568
|
+
raise RuntimeError("archived transcript integrity failure")
|
|
569
|
+
if tree_hashes(record / "artifacts") != state["output_hashes"]:
|
|
570
|
+
raise RuntimeError("archived output integrity failure")
|
|
571
|
+
return state
|
|
572
|
+
try:
|
|
573
|
+
outputs = _archive_drafts(work, record)
|
|
574
|
+
if external:
|
|
575
|
+
result = dict(
|
|
576
|
+
external,
|
|
577
|
+
transport="external import; harness completion not independently verified",
|
|
578
|
+
)
|
|
579
|
+
if (
|
|
580
|
+
not result.get("session_id")
|
|
581
|
+
or not result.get("model")
|
|
582
|
+
or not result.get("provenance")
|
|
583
|
+
):
|
|
584
|
+
raise ValueError(
|
|
585
|
+
"external collection needs session_id, model and provenance"
|
|
586
|
+
)
|
|
587
|
+
else:
|
|
588
|
+
private = blind.state_root(work) / state["config"]["harness"]
|
|
589
|
+
result = drivers.receipt(
|
|
590
|
+
state["config"],
|
|
591
|
+
record,
|
|
592
|
+
private,
|
|
593
|
+
manual=state.get("manual", False),
|
|
594
|
+
resumed=state.get("resumed_from"),
|
|
595
|
+
)
|
|
596
|
+
result["transport"] = "confined harness session"
|
|
597
|
+
if stopped := exits.recorded(record):
|
|
598
|
+
result["stopped_after_answer"] = stopped
|
|
599
|
+
# An Antigravity writer reaches the keyring through the bus its
|
|
600
|
+
# launch opened. The receipt says so, as every procedure's does.
|
|
601
|
+
result["red_flags"] = result.get("red_flags", []) + bus_flags(record)
|
|
602
|
+
if missing := _missing_outputs(plan, work):
|
|
603
|
+
raise RuntimeError(f"missing or undersized declared outputs: {missing}")
|
|
604
|
+
transcripts = record / "transcripts"
|
|
605
|
+
transcripts.mkdir(exist_ok=True)
|
|
606
|
+
for p in result.get("native_transcripts", []):
|
|
607
|
+
src = Path(p)
|
|
608
|
+
target = transcripts / src.name
|
|
609
|
+
if target.exists() and digest(target) != digest(src):
|
|
610
|
+
raise RuntimeError("native transcript changed after collection")
|
|
611
|
+
if not target.exists():
|
|
612
|
+
shutil.copyfile(src, target)
|
|
613
|
+
if external and external.get("transcript"):
|
|
614
|
+
src = Path(external["transcript"])
|
|
615
|
+
shutil.copyfile(src, transcripts / src.name)
|
|
616
|
+
answer = result.pop("answer", "")
|
|
617
|
+
if answer:
|
|
618
|
+
(record / "response.md").write_text(answer)
|
|
619
|
+
write_json(record / "receipt.json", result)
|
|
620
|
+
state.update(
|
|
621
|
+
status="completed",
|
|
622
|
+
completed_at=time.time(),
|
|
623
|
+
output_hashes=outputs,
|
|
624
|
+
record_hashes=tree_hashes(transcripts),
|
|
625
|
+
receipt=result,
|
|
626
|
+
)
|
|
627
|
+
state.pop("error", None)
|
|
628
|
+
except (RuntimeError, ValueError, OSError, StopIteration) as exc:
|
|
629
|
+
state.update(status="failed", error=str(exc), collected_at=time.time())
|
|
630
|
+
write_json(record / "state.json", state)
|
|
631
|
+
raise
|
|
632
|
+
write_json(record / "state.json", state)
|
|
633
|
+
# A completed writer is never resumed; a failed one may be, after a
|
|
634
|
+
# quota stop, so it keeps its home until it is retried.
|
|
635
|
+
blind.release(work)
|
|
636
|
+
return state
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def collect(root, writer, *, external=None):
|
|
640
|
+
root, _, record, _, _ = load(root, writer)
|
|
641
|
+
with lock(record / "operator.lock"):
|
|
642
|
+
return _collect(root, writer, external=external)
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def _pty_run(args, env, record, bus=None):
|
|
646
|
+
"""Forward an interactive terminal while recording the actual child PID."""
|
|
647
|
+
import errno
|
|
648
|
+
import pty
|
|
649
|
+
import select
|
|
650
|
+
import termios
|
|
651
|
+
import tty
|
|
652
|
+
|
|
653
|
+
if not sys.stdin.isatty():
|
|
654
|
+
raise RuntimeError(
|
|
655
|
+
"manual mode requires a terminal; use the prepared manual.sh"
|
|
656
|
+
)
|
|
657
|
+
master, slave = pty.openpty()
|
|
658
|
+
size = fcntl.ioctl(sys.stdin.fileno(), termios.TIOCGWINSZ, b"\0" * 8)
|
|
659
|
+
fcntl.ioctl(slave, termios.TIOCSWINSZ, size)
|
|
660
|
+
settings = termios.tcgetattr(sys.stdin)
|
|
661
|
+
proc = blind.popen(
|
|
662
|
+
args,
|
|
663
|
+
bus,
|
|
664
|
+
stdin=slave,
|
|
665
|
+
stdout=slave,
|
|
666
|
+
stderr=slave,
|
|
667
|
+
env=env,
|
|
668
|
+
start_new_session=True,
|
|
669
|
+
)
|
|
670
|
+
os.close(slave)
|
|
671
|
+
state = read_json(record / "state.json")
|
|
672
|
+
state.update(pid=proc.pid, process_identity=process_identity(proc.pid))
|
|
673
|
+
write_json(record / "state.json", state)
|
|
674
|
+
try:
|
|
675
|
+
tty.setraw(sys.stdin.fileno())
|
|
676
|
+
with (record / "terminal.log").open("ab") as log:
|
|
677
|
+
while True:
|
|
678
|
+
ready, _, _ = select.select([master, sys.stdin.fileno()], [], [], 1)
|
|
679
|
+
if master in ready:
|
|
680
|
+
try:
|
|
681
|
+
data = os.read(master, 65536)
|
|
682
|
+
except OSError as exc:
|
|
683
|
+
if exc.errno != errno.EIO:
|
|
684
|
+
raise
|
|
685
|
+
break
|
|
686
|
+
if not data:
|
|
687
|
+
break
|
|
688
|
+
log.write(data)
|
|
689
|
+
log.flush()
|
|
690
|
+
os.write(sys.stdout.fileno(), data)
|
|
691
|
+
if sys.stdin.fileno() in ready:
|
|
692
|
+
data = os.read(sys.stdin.fileno(), 65536)
|
|
693
|
+
os.write(master, data or b"\x04")
|
|
694
|
+
if proc.poll() is not None and not ready:
|
|
695
|
+
break
|
|
696
|
+
return proc.wait()
|
|
697
|
+
finally:
|
|
698
|
+
termios.tcsetattr(sys.stdin, termios.TCSADRAIN, settings)
|
|
699
|
+
os.close(master)
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def _terminate(proc):
|
|
703
|
+
"""Stop a writer and everything it started, then let it be reaped.
|
|
704
|
+
|
|
705
|
+
The child is its own session leader (``start_new_session``), so the
|
|
706
|
+
harness's own subprocesses die with it instead of surviving as orphans
|
|
707
|
+
holding the writer's workspace open.
|
|
708
|
+
"""
|
|
709
|
+
for signal_number in (signal.SIGTERM, signal.SIGKILL):
|
|
710
|
+
try:
|
|
711
|
+
os.killpg(os.getpgid(proc.pid), signal_number)
|
|
712
|
+
except (ProcessLookupError, PermissionError):
|
|
713
|
+
return
|
|
714
|
+
try:
|
|
715
|
+
proc.wait(timeout=20)
|
|
716
|
+
return
|
|
717
|
+
except subprocess.TimeoutExpired:
|
|
718
|
+
continue
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
def run_writer(root, writer, *, manual=False, timeout=None, resume=None):
|
|
722
|
+
root, _plan, record, work, state = load(root, writer)
|
|
723
|
+
with lock(record / "operator.lock"), harness_slot(state["config"]["harness"]):
|
|
724
|
+
if state["status"] == "completed":
|
|
725
|
+
return _collect(root, writer)
|
|
726
|
+
if state["status"] != "prepared":
|
|
727
|
+
raise RuntimeError(
|
|
728
|
+
f"{writer} is {state['status']}; collect existing evidence or retry as a new writer"
|
|
729
|
+
)
|
|
730
|
+
if manual and not sys.stdin.isatty():
|
|
731
|
+
raise RuntimeError("manual mode requires a terminal")
|
|
732
|
+
if resume:
|
|
733
|
+
# The session already holds the assignment, the reading and the
|
|
734
|
+
# draft so far. Re-sending START.md would invite a cold restart,
|
|
735
|
+
# which is the expensive thing this path exists to avoid.
|
|
736
|
+
prompt = quota.continuation(
|
|
737
|
+
f"Your draft is where you left it, under {work / 'drafts'}. "
|
|
738
|
+
"Continue writing there.",
|
|
739
|
+
cause="auth" if state.get("quota_scope") == "auth" else "quota",
|
|
740
|
+
)
|
|
741
|
+
else:
|
|
742
|
+
prompt = (
|
|
743
|
+
f"Read {work / 'START.md'} and complete the assignment it specifies."
|
|
744
|
+
)
|
|
745
|
+
proven = drivers.canary(state["config"], f"workshop writer {writer}")
|
|
746
|
+
# Open before the jail is built, since its argv names the proxy's
|
|
747
|
+
# socket, and closed only after the writer's process has exited.
|
|
748
|
+
with blind.session_bus([state["config"]["harness"]]) as bus:
|
|
749
|
+
args, payload = drivers.command(
|
|
750
|
+
state["config"], work, prompt, manual=manual, resume=resume
|
|
751
|
+
)
|
|
752
|
+
jail, env, private = drivers.launch_context(
|
|
753
|
+
state["config"], work, bus, proven
|
|
754
|
+
)
|
|
755
|
+
blind.hold(
|
|
756
|
+
work,
|
|
757
|
+
f"workshop {root.name}: writer {writer} can be resumed until it "
|
|
758
|
+
"completes or is retried",
|
|
759
|
+
)
|
|
760
|
+
_snapshot_driver(record)
|
|
761
|
+
(record / "prompt.md").write_text(prompt + "\n")
|
|
762
|
+
write_json(
|
|
763
|
+
record / "launch.json",
|
|
764
|
+
{
|
|
765
|
+
"argv": args,
|
|
766
|
+
"jail": jail,
|
|
767
|
+
"harness_version": blind.runner_version(state["config"]["harness"]),
|
|
768
|
+
"private_runtime": str(private),
|
|
769
|
+
"manual": manual,
|
|
770
|
+
"session_bus": bus.receipt() if bus else None,
|
|
771
|
+
"canary": proven,
|
|
772
|
+
},
|
|
773
|
+
)
|
|
774
|
+
state.update(status="running", started_at=time.time(), manual=manual)
|
|
775
|
+
write_json(record / "state.json", state)
|
|
776
|
+
usage_records.seat("workshop", state["config"])
|
|
777
|
+
try:
|
|
778
|
+
if manual:
|
|
779
|
+
state["exit_code"] = _pty_run(jail + args, env, record, bus)
|
|
780
|
+
else:
|
|
781
|
+
with (
|
|
782
|
+
(record / "events.jsonl").open("xb") as out,
|
|
783
|
+
(record / "stderr.log").open("xb") as err,
|
|
784
|
+
):
|
|
785
|
+
proc = blind.popen(
|
|
786
|
+
jail + args,
|
|
787
|
+
bus,
|
|
788
|
+
stdin=subprocess.PIPE,
|
|
789
|
+
stdout=out,
|
|
790
|
+
stderr=err,
|
|
791
|
+
cwd=current_project().root,
|
|
792
|
+
env=env,
|
|
793
|
+
start_new_session=True,
|
|
794
|
+
)
|
|
795
|
+
state.update(
|
|
796
|
+
pid=proc.pid, process_identity=process_identity(proc.pid)
|
|
797
|
+
)
|
|
798
|
+
write_json(record / "state.json", state)
|
|
799
|
+
why = exits.wait(
|
|
800
|
+
proc,
|
|
801
|
+
payload,
|
|
802
|
+
harness=state["config"]["harness"],
|
|
803
|
+
events=record / "events.jsonl",
|
|
804
|
+
timeout=timeout,
|
|
805
|
+
)
|
|
806
|
+
if why:
|
|
807
|
+
# Killed, not abandoned: collection below archives
|
|
808
|
+
# whatever prose exists and then reports the run as
|
|
809
|
+
# failed, so a stalled writer costs one slot, not the
|
|
810
|
+
# whole experiment. A writer that answered and
|
|
811
|
+
# stayed is collected like one that exited.
|
|
812
|
+
_terminate(proc)
|
|
813
|
+
proc.communicate()
|
|
814
|
+
if why == exits.TIMEOUT:
|
|
815
|
+
state["timed_out_after"] = timeout
|
|
816
|
+
if why == exits.ANSWERED:
|
|
817
|
+
state["stopped_after_answer"] = exits.record(
|
|
818
|
+
record,
|
|
819
|
+
harness=state["config"]["harness"],
|
|
820
|
+
grace=exits.EXIT_GRACE,
|
|
821
|
+
exit_code=proc.returncode,
|
|
822
|
+
)
|
|
823
|
+
state["exit_code"] = proc.returncode
|
|
824
|
+
state.update(ended_at=time.time())
|
|
825
|
+
state.pop("pid", None)
|
|
826
|
+
state.pop("process_identity", None)
|
|
827
|
+
write_json(record / "state.json", state)
|
|
828
|
+
return _collect(root, writer)
|
|
829
|
+
except BaseException as exc:
|
|
830
|
+
# Do not mark a still-running external process dead. Recovery
|
|
831
|
+
# inspects its identity and receipts instead of blindly retrying.
|
|
832
|
+
state = read_json(record / "state.json")
|
|
833
|
+
state["error"] = str(exc)
|
|
834
|
+
if state.get("ended_at") or (
|
|
835
|
+
state.get("pid")
|
|
836
|
+
and process_identity(state["pid"]) != state.get("process_identity")
|
|
837
|
+
):
|
|
838
|
+
state["status"] = "failed"
|
|
839
|
+
write_json(record / "state.json", state)
|
|
840
|
+
raise
|
|
841
|
+
|
|
842
|
+
|
|
843
|
+
def launch_order(writers):
|
|
844
|
+
"""The declared seats in spend order; `quota.spend_order` owns the rule."""
|
|
845
|
+
return quota.spend_order(writers)
|
|
846
|
+
|
|
847
|
+
|
|
848
|
+
def run(root, writers=None, jobs=1, timeout=None):
|
|
849
|
+
"""Run the selected writers, cheapest first and one at a time by default.
|
|
850
|
+
|
|
851
|
+
`jobs` stays available because the author may want a batch, but its
|
|
852
|
+
default is one: seats sharing an account exhaust a five-hour window
|
|
853
|
+
several times faster in parallel, and the wall then lands mid-turn on
|
|
854
|
+
every one of them at once instead of on one.
|
|
855
|
+
"""
|
|
856
|
+
root, plan = load(root)
|
|
857
|
+
if jobs < 1:
|
|
858
|
+
raise ValueError("jobs must be positive")
|
|
859
|
+
if writers:
|
|
860
|
+
# An explicit list is the operator's own order and is left alone.
|
|
861
|
+
selected = list(writers)
|
|
862
|
+
else:
|
|
863
|
+
selected = [
|
|
864
|
+
w["id"] for w in launch_order(plan["writers"]) if w.get("mode") == "auto"
|
|
865
|
+
]
|
|
866
|
+
|
|
867
|
+
def task(writer):
|
|
868
|
+
try:
|
|
869
|
+
return writer, run_writer(root, writer, timeout=timeout)["status"]
|
|
870
|
+
except (RuntimeError, ValueError, OSError, subprocess.SubprocessError) as exc:
|
|
871
|
+
return writer, f"error: {exc}"
|
|
872
|
+
|
|
873
|
+
results = {}
|
|
874
|
+
if jobs == 1:
|
|
875
|
+
# Sequential means sequential: a one-worker pool would still hide the
|
|
876
|
+
# order, and the order is the point.
|
|
877
|
+
for writer in selected:
|
|
878
|
+
writer, status = task(writer)
|
|
879
|
+
results[writer] = status
|
|
880
|
+
return results
|
|
881
|
+
with ThreadPoolExecutor(max_workers=jobs) as pool:
|
|
882
|
+
for future in as_completed([submit(pool, task, w) for w in selected]):
|
|
883
|
+
writer, status = future.result()
|
|
884
|
+
results[writer] = status
|
|
885
|
+
return results
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
def _archive_attempt(record, state, reason):
|
|
889
|
+
"""Move the stopped attempt's whole record aside before the next one runs.
|
|
890
|
+
|
|
891
|
+
Everything the attempt produced is kept: its events, its stderr, the
|
|
892
|
+
prompt it was given, the argv it was launched with, the partial draft
|
|
893
|
+
`_collect` already archived, and the driver snapshot that ran it. Nothing is
|
|
894
|
+
deleted and nothing is overwritten, so the run can still be read as two
|
|
895
|
+
attempts rather than one that changed its mind. This is also what unblocks the collector, whose refusal to let an
|
|
896
|
+
archived output change is correct for one attempt and wrong across two.
|
|
897
|
+
"""
|
|
898
|
+
number = len(state.get("attempts", [])) + 1
|
|
899
|
+
folder = record / "attempts" / f"{number:02d}"
|
|
900
|
+
folder.mkdir(parents=True)
|
|
901
|
+
for name in (
|
|
902
|
+
"events.jsonl",
|
|
903
|
+
"stderr.log",
|
|
904
|
+
"prompt.md",
|
|
905
|
+
"launch.json",
|
|
906
|
+
"artifacts",
|
|
907
|
+
"response.md",
|
|
908
|
+
"receipt.json",
|
|
909
|
+
"driver",
|
|
910
|
+
):
|
|
911
|
+
source = record / name
|
|
912
|
+
if source.exists():
|
|
913
|
+
shutil.move(str(source), str(folder / name))
|
|
914
|
+
state.setdefault("attempts", []).append(
|
|
915
|
+
{
|
|
916
|
+
"attempt": number,
|
|
917
|
+
"reason": reason,
|
|
918
|
+
"error": state.get("error"),
|
|
919
|
+
"exit_code": state.get("exit_code"),
|
|
920
|
+
"record": str(folder),
|
|
921
|
+
"hashes": tree_hashes(folder),
|
|
922
|
+
}
|
|
923
|
+
)
|
|
924
|
+
return folder
|
|
925
|
+
|
|
926
|
+
|
|
927
|
+
def resume(root, writer, *, timeout=None):
|
|
928
|
+
"""Continue a writer a provider quota limit stopped, in its own session.
|
|
929
|
+
|
|
930
|
+
NOT `retry`. A retry is a new writer with a new session that re-derives
|
|
931
|
+
everything from a cold start; this hands the same session a short notice
|
|
932
|
+
and lets it finish. The difference is measurable: one seat's 31 requests
|
|
933
|
+
before a wall spent 58 percent of their cost on output and reasoning and
|
|
934
|
+
4.5 percent on cached context reads, so a restart pays the expensive half
|
|
935
|
+
again to recover what the session still holds.
|
|
936
|
+
|
|
937
|
+
A Claude writer whose borrowed login was renewed under it mid-turn is
|
|
938
|
+
resumed the same way, once the host's login has room for another launch
|
|
939
|
+
(``claude_login``): that stop says exactly what ran, as a quota stop does.
|
|
940
|
+
|
|
941
|
+
Refuses on anything else. A timeout, a crash or a model that simply gave
|
|
942
|
+
up are all ambiguous about what actually ran, and resuming one of those
|
|
943
|
+
would ask a writer to continue from a state nobody has established.
|
|
944
|
+
"""
|
|
945
|
+
root, _plan, record, work, state = load(root, writer)
|
|
946
|
+
with lock(record / "operator.lock"):
|
|
947
|
+
if state.get("pid") and process_identity(state["pid"]) == state.get(
|
|
948
|
+
"process_identity"
|
|
949
|
+
):
|
|
950
|
+
raise RuntimeError("writer process is still alive; wait, do not resume")
|
|
951
|
+
if state["status"] not in ("failed", "running"):
|
|
952
|
+
raise RuntimeError(
|
|
953
|
+
f"{writer} is {state['status']}; only a stopped writer is resumed"
|
|
954
|
+
)
|
|
955
|
+
harness = state["config"]["harness"]
|
|
956
|
+
stop = quota.classify(harness, record) or quota.auth_stop(harness, record)
|
|
957
|
+
if not stop:
|
|
958
|
+
raise RuntimeError(
|
|
959
|
+
f"{writer} does not carry a provider quota stop or a renewed "
|
|
960
|
+
"login in its event stream; inspect the record, and use `retry` "
|
|
961
|
+
"if the work has to start over"
|
|
962
|
+
)
|
|
963
|
+
if stop["scope"] == "auth":
|
|
964
|
+
try:
|
|
965
|
+
claude_login.host_login()
|
|
966
|
+
except claude_login.LoginError as exc:
|
|
967
|
+
raise RuntimeError(
|
|
968
|
+
f"{writer} stopped on a revoked or expired Claude login, and "
|
|
969
|
+
f"{exc}; then resume it again"
|
|
970
|
+
) from None
|
|
971
|
+
session = stop["session_id"]
|
|
972
|
+
if stop["phase"] == "interrupted" and not session:
|
|
973
|
+
raise RuntimeError(
|
|
974
|
+
"the limit interrupted this writer after the model ran, but its "
|
|
975
|
+
"event stream names no single session to continue; preserve the "
|
|
976
|
+
"record and retry as a new writer"
|
|
977
|
+
)
|
|
978
|
+
# A refusal means the provider never ran the model, so there is no
|
|
979
|
+
# reasoning to keep and nothing to continue FROM. The same assignment
|
|
980
|
+
# simply goes again, in a new session, which is a relaunch and not a
|
|
981
|
+
# restart of work that happened.
|
|
982
|
+
resumed = session if stop["phase"] == "interrupted" else None
|
|
983
|
+
if resumed:
|
|
984
|
+
# Before the stopped attempt is archived, so a refusal leaves the
|
|
985
|
+
# writer exactly as it was.
|
|
986
|
+
blind.require_session(work, harness, resumed)
|
|
987
|
+
cause = "login" if stop["scope"] == "auth" else "quota"
|
|
988
|
+
_archive_attempt(record, state, f"{cause} {stop['phase']}: {stop['scope']}")
|
|
989
|
+
state.update(
|
|
990
|
+
status="prepared",
|
|
991
|
+
resumed_from=resumed,
|
|
992
|
+
quota_stop=stop["phase"],
|
|
993
|
+
quota_scope=stop["scope"],
|
|
994
|
+
)
|
|
995
|
+
for key in (
|
|
996
|
+
"error",
|
|
997
|
+
"exit_code",
|
|
998
|
+
"started_at",
|
|
999
|
+
"ended_at",
|
|
1000
|
+
"collected_at",
|
|
1001
|
+
"timed_out_after",
|
|
1002
|
+
"receipt",
|
|
1003
|
+
):
|
|
1004
|
+
state.pop(key, None)
|
|
1005
|
+
write_json(record / "state.json", state)
|
|
1006
|
+
return run_writer(root, writer, timeout=timeout, resume=resumed)
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
def retry(root, writer, new_id):
|
|
1010
|
+
root, plan, record, _, state = load(root, writer)
|
|
1011
|
+
identifier(new_id)
|
|
1012
|
+
with lock(record / "operator.lock"):
|
|
1013
|
+
if state.get("pid") and process_identity(state["pid"]) == state.get(
|
|
1014
|
+
"process_identity"
|
|
1015
|
+
):
|
|
1016
|
+
raise RuntimeError("old writer is still running")
|
|
1017
|
+
if state["status"] == "running":
|
|
1018
|
+
raise RuntimeError(
|
|
1019
|
+
"reconcile the submitted attempt before creating a retry"
|
|
1020
|
+
)
|
|
1021
|
+
if (root / "writers" / new_id).exists():
|
|
1022
|
+
raise ValueError("new writer already exists")
|
|
1023
|
+
item = dict(state["config"], id=new_id, retry_of=writer)
|
|
1024
|
+
_prepare_writer(root, plan, item)
|
|
1025
|
+
# The retry starts over in a session of its own, so the old writer's
|
|
1026
|
+
# is never resumed.
|
|
1027
|
+
blind.release(root / "writers" / writer)
|
|
1028
|
+
return item
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
# The keys an editor reads to decide what to do next. The rest of a writer's
|
|
1032
|
+
# config is the resolution evidence: a whole Codex catalog entry per writer,
|
|
1033
|
+
# which buries ten writers' status in several hundred lines. It stays in
|
|
1034
|
+
# plan.json and state.json; --full prints it here.
|
|
1035
|
+
BRIEF_CONFIG = (
|
|
1036
|
+
"harness",
|
|
1037
|
+
"model",
|
|
1038
|
+
"effort",
|
|
1039
|
+
"mode",
|
|
1040
|
+
"tier",
|
|
1041
|
+
"requested_model",
|
|
1042
|
+
"requested_effort",
|
|
1043
|
+
"capabilities",
|
|
1044
|
+
"retry_of",
|
|
1045
|
+
)
|
|
1046
|
+
|
|
1047
|
+
|
|
1048
|
+
def roster(*, check=False, plan=None):
|
|
1049
|
+
"""The default seats, optionally resolved against the installed catalogs.
|
|
1050
|
+
|
|
1051
|
+
Resolution is what makes a stale entry visible: providers rename models
|
|
1052
|
+
often, and a seat that has quietly disappeared should fail here rather
|
|
1053
|
+
than halfway through an experiment.
|
|
1054
|
+
|
|
1055
|
+
`plan` resolves that plan's own writers instead of the default roster.
|
|
1056
|
+
Departing from the roster is normal, and a run that adds a seat had no
|
|
1057
|
+
supported way to check it before launch: the answer was to call
|
|
1058
|
+
drivers.resolve by hand, which no reader of the SOP would find.
|
|
1059
|
+
"""
|
|
1060
|
+
if plan is not None:
|
|
1061
|
+
data = {"source": str(plan), "writers": read_json(plan).get("writers", [])}
|
|
1062
|
+
else:
|
|
1063
|
+
data = read_json(_settings().ROSTER)
|
|
1064
|
+
if not check:
|
|
1065
|
+
return data
|
|
1066
|
+
for item in data["writers"]:
|
|
1067
|
+
# Either name alone is enough here too. `prepare` normalizes the pair
|
|
1068
|
+
# before resolving; this path did not, so the one command the SOP says
|
|
1069
|
+
# to run against a hand-written plan died on a bare KeyError naming a
|
|
1070
|
+
# field the plan template calls optional.
|
|
1071
|
+
item.setdefault("requested_model", item.get("model"))
|
|
1072
|
+
item.setdefault("model", item.get("requested_model"))
|
|
1073
|
+
# `effort` is the third resolver input and was the one still indexed
|
|
1074
|
+
# bare, so the fix above left the identical crash reachable under a
|
|
1075
|
+
# different key. The plan template marks the field optional and
|
|
1076
|
+
# `prepare` fills it; this path has to as well.
|
|
1077
|
+
item.setdefault("effort", None)
|
|
1078
|
+
try:
|
|
1079
|
+
item["resolved"] = drivers.resolve(
|
|
1080
|
+
item["harness"],
|
|
1081
|
+
item["requested_model"],
|
|
1082
|
+
item["effort"],
|
|
1083
|
+
item["model"],
|
|
1084
|
+
)["model"]
|
|
1085
|
+
except (ValueError, OSError, subprocess.SubprocessError) as exc:
|
|
1086
|
+
item["resolved"] = f"UNAVAILABLE: {exc}"
|
|
1087
|
+
data["unavailable"] = [
|
|
1088
|
+
i["model"]
|
|
1089
|
+
for i in data["writers"]
|
|
1090
|
+
if str(i["resolved"]).startswith("UNAVAILABLE")
|
|
1091
|
+
]
|
|
1092
|
+
return data
|
|
1093
|
+
|
|
1094
|
+
|
|
1095
|
+
def inspect(root, *, full=False):
|
|
1096
|
+
root, plan = load(root)
|
|
1097
|
+
writers = {}
|
|
1098
|
+
for p in sorted((root / "records").glob("*/state.json")):
|
|
1099
|
+
s = read_json(p)
|
|
1100
|
+
item = {
|
|
1101
|
+
k: s[k]
|
|
1102
|
+
for k in (
|
|
1103
|
+
"status",
|
|
1104
|
+
"config",
|
|
1105
|
+
"error",
|
|
1106
|
+
"started_at",
|
|
1107
|
+
"ended_at",
|
|
1108
|
+
"quota_stop",
|
|
1109
|
+
"quota_scope",
|
|
1110
|
+
"resumed_from",
|
|
1111
|
+
)
|
|
1112
|
+
if k in s
|
|
1113
|
+
}
|
|
1114
|
+
# Named outright, because the recovery differs from every other
|
|
1115
|
+
# failure: this writer is resumed, not retried.
|
|
1116
|
+
if s.get("status") == "failed" and not s.get("attempts"):
|
|
1117
|
+
stopped = root / "records" / p.parent.name
|
|
1118
|
+
stop = quota.classify(s["config"]["harness"], stopped) or quota.auth_stop(
|
|
1119
|
+
s["config"]["harness"], stopped
|
|
1120
|
+
)
|
|
1121
|
+
if stop:
|
|
1122
|
+
item["quota_stop"] = stop["phase"]
|
|
1123
|
+
# "auth" for a login renewed under the writer, which `resume`
|
|
1124
|
+
# continues like a quota stop once the host's login has room.
|
|
1125
|
+
item["stop_scope"] = stop["scope"]
|
|
1126
|
+
# Not `bool(session_id)`: a refusal records a session too, so
|
|
1127
|
+
# that test called every refused writer resumable and put it in
|
|
1128
|
+
# the bucket the SOP uses to choose between continue and
|
|
1129
|
+
# relaunch. `quota` owns which stops can be continued.
|
|
1130
|
+
item["resumable"] = bool(quota.resumable_session(stop))
|
|
1131
|
+
if s.get("attempts"):
|
|
1132
|
+
item["attempts"] = [
|
|
1133
|
+
{k: a.get(k) for k in ("attempt", "reason", "record")}
|
|
1134
|
+
for a in s["attempts"]
|
|
1135
|
+
]
|
|
1136
|
+
# From the receipt once the writer is collected, and before that from
|
|
1137
|
+
# the launch, so a writer still running or stopped shows its bus too.
|
|
1138
|
+
if flags := (s.get("receipt") or {}).get("red_flags") or bus_flags(p.parent):
|
|
1139
|
+
item["red_flags"] = flags
|
|
1140
|
+
item["persona"] = personas.identity(s.get("config", {}).get("persona"))
|
|
1141
|
+
if not full and "config" in item:
|
|
1142
|
+
item["config"] = {
|
|
1143
|
+
k: v
|
|
1144
|
+
for k, v in item["config"].items()
|
|
1145
|
+
if k in BRIEF_CONFIG and v not in (None, [], "")
|
|
1146
|
+
}
|
|
1147
|
+
if s.get("pid"):
|
|
1148
|
+
item["process_alive"] = process_identity(s["pid"]) == s.get(
|
|
1149
|
+
"process_identity"
|
|
1150
|
+
)
|
|
1151
|
+
# Presence and packet integrity, never byte counts. A completion check
|
|
1152
|
+
# needs both, and reporting sizes here would hand over the identity key
|
|
1153
|
+
# before `anonymize` has sealed it, which is what `report` now refuses
|
|
1154
|
+
# to do. See docs/workflows.md, "Seal first, inspect afterwards".
|
|
1155
|
+
work = root / "writers" / p.parent.name
|
|
1156
|
+
item["declared_outputs"] = {
|
|
1157
|
+
o["path"]: (
|
|
1158
|
+
"ok"
|
|
1159
|
+
if (work / o["path"]).is_file()
|
|
1160
|
+
and (work / o["path"]).stat().st_size >= o["min_bytes"]
|
|
1161
|
+
else "missing or undersized"
|
|
1162
|
+
)
|
|
1163
|
+
for o in plan["outputs"]
|
|
1164
|
+
}
|
|
1165
|
+
item["packet_unchanged"] = tree_hashes(work / "input") == plan["input_hashes"]
|
|
1166
|
+
writers[p.parent.name] = item
|
|
1167
|
+
return {
|
|
1168
|
+
"experiment": str(root),
|
|
1169
|
+
"supplied_bytes": plan["supplied_bytes"],
|
|
1170
|
+
"writers": writers,
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1173
|
+
|
|
1174
|
+
def _normalize_usage(harness, usage):
|
|
1175
|
+
"""One token shape across three harnesses, plus what each cannot report.
|
|
1176
|
+
|
|
1177
|
+
These are provider counters summed over every request in the session, so
|
|
1178
|
+
they are neither context occupancy nor comparable billing across
|
|
1179
|
+
harnesses. Only Claude reports a price; a subscription harness reports
|
|
1180
|
+
none, and inventing one from public rates would put a number the run did
|
|
1181
|
+
not measure next to numbers it did.
|
|
1182
|
+
"""
|
|
1183
|
+
out = {
|
|
1184
|
+
"input": None,
|
|
1185
|
+
"cached_input": None,
|
|
1186
|
+
"output": None,
|
|
1187
|
+
"reasoning": None,
|
|
1188
|
+
"cost_usd": None,
|
|
1189
|
+
"cost_evidence": "not reported by this harness",
|
|
1190
|
+
}
|
|
1191
|
+
if not usage:
|
|
1192
|
+
return out
|
|
1193
|
+
if harness == "claude":
|
|
1194
|
+
# A dict keyed by served model: the writing model plus whatever small
|
|
1195
|
+
# preflight the harness ran. Summed here, itemized in by_model so the
|
|
1196
|
+
# preflight stays visible instead of inflating the writer's figure.
|
|
1197
|
+
out["by_model"] = {
|
|
1198
|
+
name: {
|
|
1199
|
+
"input": row.get("inputTokens"),
|
|
1200
|
+
"output": row.get("outputTokens"),
|
|
1201
|
+
"reasoning": row.get("thinkingTokens"),
|
|
1202
|
+
"cost_usd": row.get("costUSD"),
|
|
1203
|
+
}
|
|
1204
|
+
for name, row in usage.items()
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
def total(k):
|
|
1208
|
+
return sum(r.get(k) or 0 for r in usage.values())
|
|
1209
|
+
|
|
1210
|
+
out.update(
|
|
1211
|
+
input=total("inputTokens"),
|
|
1212
|
+
cached_input=total("cacheReadInputTokens")
|
|
1213
|
+
+ total("cacheCreationInputTokens"),
|
|
1214
|
+
output=total("outputTokens"),
|
|
1215
|
+
reasoning=total("thinkingTokens"),
|
|
1216
|
+
cost_usd=round(total("costUSD"), 4),
|
|
1217
|
+
cost_evidence="harness modelUsage costUSD, list basis",
|
|
1218
|
+
)
|
|
1219
|
+
elif harness == "codex":
|
|
1220
|
+
out.update(
|
|
1221
|
+
input=usage.get("input_tokens"),
|
|
1222
|
+
cached_input=usage.get("cached_input_tokens"),
|
|
1223
|
+
output=usage.get("output_tokens"),
|
|
1224
|
+
reasoning=usage.get("reasoning_output_tokens"),
|
|
1225
|
+
)
|
|
1226
|
+
else:
|
|
1227
|
+
out.update(
|
|
1228
|
+
input=usage.get("input_tokens"),
|
|
1229
|
+
cached_input=usage.get("cache_read_tokens"),
|
|
1230
|
+
output=usage.get("output_tokens"),
|
|
1231
|
+
reasoning=usage.get("thinking_tokens"),
|
|
1232
|
+
)
|
|
1233
|
+
return out
|
|
1234
|
+
|
|
1235
|
+
|
|
1236
|
+
def report(root, *, sizes=None):
|
|
1237
|
+
"""Cost of each writer beside what it returned, for the editor's ROI note.
|
|
1238
|
+
|
|
1239
|
+
Everything here is transport and spend. Yield is the editorial reading;
|
|
1240
|
+
output_bytes counts what came back, never whether it was worth reading.
|
|
1241
|
+
|
|
1242
|
+
Byte counts are near-unique per writer, so printing them before the drafts
|
|
1243
|
+
are sealed hands the editor the identity key by arithmetic. They are
|
|
1244
|
+
therefore withheld until `anonymize` has run. Pass sizes=True to override
|
|
1245
|
+
deliberately, sizes=False to keep them out of a sealed run's write-up.
|
|
1246
|
+
"""
|
|
1247
|
+
root, plan = load(root)
|
|
1248
|
+
if sizes is None:
|
|
1249
|
+
sizes = (Path(root) / "evaluation" / "identity-key.json").is_file()
|
|
1250
|
+
declared = [item["path"] for item in plan["outputs"]]
|
|
1251
|
+
rows = []
|
|
1252
|
+
for p in sorted((root / "records").glob("*/state.json")):
|
|
1253
|
+
s, writer = read_json(p), p.parent.name
|
|
1254
|
+
config, receipt = s.get("config", {}), s.get("receipt", {})
|
|
1255
|
+
artifacts = root / "records" / writer / "artifacts"
|
|
1256
|
+
produced = {
|
|
1257
|
+
str(q.relative_to(artifacts)): q.stat().st_size
|
|
1258
|
+
for q in sorted(artifacts.rglob("*"))
|
|
1259
|
+
if q.is_file()
|
|
1260
|
+
}
|
|
1261
|
+
started, ended = s.get("started_at"), s.get("ended_at")
|
|
1262
|
+
rows.append(
|
|
1263
|
+
{
|
|
1264
|
+
"writer": writer,
|
|
1265
|
+
"harness": config.get("harness"),
|
|
1266
|
+
"model": receipt.get("model") or config.get("model"),
|
|
1267
|
+
"selected_model": config.get("model"),
|
|
1268
|
+
"effort": config.get("effort"),
|
|
1269
|
+
"requested_effort": config.get("requested_effort"),
|
|
1270
|
+
"status": s.get("status"),
|
|
1271
|
+
"seconds": round(ended - started, 1) if started and ended else None,
|
|
1272
|
+
"usage": _normalize_usage(config.get("harness"), receipt.get("usage")),
|
|
1273
|
+
"context": drivers.context_peak(
|
|
1274
|
+
config.get("harness"), root / "records" / writer
|
|
1275
|
+
),
|
|
1276
|
+
"output_bytes": {
|
|
1277
|
+
"declared": sum(
|
|
1278
|
+
produced.get(d.split("/", 1)[1], 0) for d in declared
|
|
1279
|
+
),
|
|
1280
|
+
"all_files": sum(produced.values()),
|
|
1281
|
+
"extra_files": sorted(
|
|
1282
|
+
set(produced) - {d.split("/", 1)[1] for d in declared}
|
|
1283
|
+
),
|
|
1284
|
+
}
|
|
1285
|
+
if sizes
|
|
1286
|
+
else {"withheld": "run anonymize first, or pass --sizes"},
|
|
1287
|
+
"error": s.get("error"),
|
|
1288
|
+
}
|
|
1289
|
+
)
|
|
1290
|
+
return {"experiment": str(root), "sizes_shown": sizes, "writers": rows}
|
|
1291
|
+
|
|
1292
|
+
|
|
1293
|
+
def report_table(data):
|
|
1294
|
+
"""A markdown table for the editor's write-up; JSON stays the record."""
|
|
1295
|
+
head = (
|
|
1296
|
+
"| Writer | Harness | Model | Effort | Status | Seconds | "
|
|
1297
|
+
"Peak request | Window | In | Cached in | Out | Reasoning | USD | "
|
|
1298
|
+
"Draft bytes |"
|
|
1299
|
+
)
|
|
1300
|
+
lines = [head, "|" + "---|" * 14]
|
|
1301
|
+
|
|
1302
|
+
def cell(value):
|
|
1303
|
+
if value in (None, ""):
|
|
1304
|
+
return "-"
|
|
1305
|
+
return f"{value:,}" if isinstance(value, int) else str(value)
|
|
1306
|
+
|
|
1307
|
+
for r in data["writers"]:
|
|
1308
|
+
u = r["usage"]
|
|
1309
|
+
effort = r["effort"]
|
|
1310
|
+
if r["requested_effort"] and r["requested_effort"] != r["effort"]:
|
|
1311
|
+
effort += f" (asked {r['requested_effort']})"
|
|
1312
|
+
lines.append(
|
|
1313
|
+
"| "
|
|
1314
|
+
+ " | ".join(
|
|
1315
|
+
[
|
|
1316
|
+
r["writer"],
|
|
1317
|
+
r["harness"] or "-",
|
|
1318
|
+
r["model"] or "-",
|
|
1319
|
+
effort or "-",
|
|
1320
|
+
r["status"] or "-",
|
|
1321
|
+
cell(r["seconds"]),
|
|
1322
|
+
cell(r["context"]["peak_request"]),
|
|
1323
|
+
cell(r["context"]["window"]),
|
|
1324
|
+
cell(u["input"]),
|
|
1325
|
+
cell(u["cached_input"]),
|
|
1326
|
+
cell(u["output"]),
|
|
1327
|
+
cell(u["reasoning"]),
|
|
1328
|
+
cell(u["cost_usd"]),
|
|
1329
|
+
cell(r["output_bytes"].get("declared")),
|
|
1330
|
+
]
|
|
1331
|
+
)
|
|
1332
|
+
+ " |"
|
|
1333
|
+
)
|
|
1334
|
+
return "\n".join(lines)
|
|
1335
|
+
|
|
1336
|
+
|
|
1337
|
+
# The writers' words and how much they wrote, withheld until asked for. A
|
|
1338
|
+
# draft compared blind is identified by either before the seal, which is why
|
|
1339
|
+
# `report` withholds byte counts, and what a writer says while it works is
|
|
1340
|
+
# the draft's own voice.
|
|
1341
|
+
WITHHELD = feed.Show(words=False, sizes=False)
|
|
1342
|
+
# A running command's own log, under the same rule.
|
|
1343
|
+
NARRATION = feed.Show(words=False, actions=False, sizes=False)
|
|
1344
|
+
|
|
1345
|
+
|
|
1346
|
+
class Feed:
|
|
1347
|
+
"""A workshop told as it runs: each writer's start, actions and finish.
|
|
1348
|
+
|
|
1349
|
+
The writers keep no trail, so progress is read from each writer's state,
|
|
1350
|
+
which names when its launch began and how it ended, and from its stream.
|
|
1351
|
+
The feed names the files a writer read and wrote and nothing they
|
|
1352
|
+
contain, unless the words are asked for deliberately (``WITHHELD``).
|
|
1353
|
+
"""
|
|
1354
|
+
|
|
1355
|
+
def __init__(self, root, *, history="all", show=WITHHELD, width=None):
|
|
1356
|
+
self.root, self.plan = load(root)
|
|
1357
|
+
self.show = show
|
|
1358
|
+
self.started, self.active, self.first = {}, {}, True
|
|
1359
|
+
names = [p.parent.name for p in self._states()]
|
|
1360
|
+
self.page = feed.Page(width, who=max(map(len, names), default=0))
|
|
1361
|
+
if history == "none":
|
|
1362
|
+
for path in self._states():
|
|
1363
|
+
self.started[path.parent.name] = read_json(path).get("started_at")
|
|
1364
|
+
|
|
1365
|
+
def _states(self):
|
|
1366
|
+
return sorted((self.root / "records").glob("*/state.json"))
|
|
1367
|
+
|
|
1368
|
+
def header(self):
|
|
1369
|
+
lines = [
|
|
1370
|
+
f"following the workshop at {shown(self.root)}",
|
|
1371
|
+
" (interrupt to stop)",
|
|
1372
|
+
]
|
|
1373
|
+
if self.show == WITHHELD:
|
|
1374
|
+
lines.append(
|
|
1375
|
+
" the writers' words and draft sizes are withheld; --words shows them"
|
|
1376
|
+
)
|
|
1377
|
+
return lines
|
|
1378
|
+
|
|
1379
|
+
def poll(self):
|
|
1380
|
+
catching_up, self.first = self.first, False
|
|
1381
|
+
stamp = None if catching_up else time.time()
|
|
1382
|
+
lines = []
|
|
1383
|
+
for follower in self.active.values():
|
|
1384
|
+
lines += follower.poll(stamp) if follower else []
|
|
1385
|
+
for path in self._states():
|
|
1386
|
+
writer, state = path.parent.name, read_json(path)
|
|
1387
|
+
began = state.get("started_at")
|
|
1388
|
+
if began and self.started.get(writer) != began:
|
|
1389
|
+
self.started[writer] = began
|
|
1390
|
+
if self.active.get(writer):
|
|
1391
|
+
lines += self.active[writer].finish(stamp)
|
|
1392
|
+
lines += self._start(writer, state)
|
|
1393
|
+
if self.active[writer]:
|
|
1394
|
+
lines += self.active[writer].poll(None if catching_up else stamp)
|
|
1395
|
+
if writer in self.active and state.get("status") in ("completed", "failed"):
|
|
1396
|
+
follower = self.active.pop(writer)
|
|
1397
|
+
lines += (follower.finish(stamp) if follower else []) + self._ended(
|
|
1398
|
+
writer, state
|
|
1399
|
+
)
|
|
1400
|
+
return lines
|
|
1401
|
+
|
|
1402
|
+
def _start(self, writer, state):
|
|
1403
|
+
config = state["config"]
|
|
1404
|
+
served = f"{config['harness']}/{drivers.model_text(config)}"
|
|
1405
|
+
again = (
|
|
1406
|
+
f", attempt {len(state['attempts']) + 1}" if state.get("attempts") else ""
|
|
1407
|
+
)
|
|
1408
|
+
if state.get("manual"):
|
|
1409
|
+
self.active[writer] = None
|
|
1410
|
+
return self.page.event(
|
|
1411
|
+
state["started_at"],
|
|
1412
|
+
f"{writer} [{served}] starts writing by hand{again}",
|
|
1413
|
+
)
|
|
1414
|
+
work = self.root / "writers" / writer
|
|
1415
|
+
self.active[writer] = feed.Turn(
|
|
1416
|
+
self.page,
|
|
1417
|
+
writer,
|
|
1418
|
+
config["harness"],
|
|
1419
|
+
self.root / "records" / writer / "events.jsonl",
|
|
1420
|
+
places=[(work, "", work)],
|
|
1421
|
+
private=blind.state_root(work) / config["harness"],
|
|
1422
|
+
show=self.show,
|
|
1423
|
+
)
|
|
1424
|
+
return self.page.event(
|
|
1425
|
+
state["started_at"], f"{writer} [{served}] starts writing{again}"
|
|
1426
|
+
)
|
|
1427
|
+
|
|
1428
|
+
def _ended(self, writer, state):
|
|
1429
|
+
took = (
|
|
1430
|
+
feed.duration(state["ended_at"] - state["started_at"])
|
|
1431
|
+
if state.get("ended_at") and state.get("started_at")
|
|
1432
|
+
else None
|
|
1433
|
+
)
|
|
1434
|
+
if state["status"] == "completed":
|
|
1435
|
+
text = f"{writer} finished" + (f" in {took}" if took else "")
|
|
1436
|
+
else:
|
|
1437
|
+
stop = state.get("quota_stop")
|
|
1438
|
+
text = (
|
|
1439
|
+
f"{writer} "
|
|
1440
|
+
+ ("stopped on a provider limit" if stop else "failed")
|
|
1441
|
+
+ (f" after {took}" if took else "")
|
|
1442
|
+
)
|
|
1443
|
+
if state.get("error") and not stop:
|
|
1444
|
+
text += f": {state['error']}"
|
|
1445
|
+
return self.page.event(state.get("ended_at") or state.get("collected_at"), text)
|
|
1446
|
+
|
|
1447
|
+
|
|
1448
|
+
def main(argv=None):
|
|
1449
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
1450
|
+
sub = parser.add_subparsers(dest="action", required=True)
|
|
1451
|
+
p = sub.add_parser(
|
|
1452
|
+
"prepare", help="stage a writing experiment", description=prepare.__doc__
|
|
1453
|
+
)
|
|
1454
|
+
p.add_argument("plan")
|
|
1455
|
+
p.add_argument("destination")
|
|
1456
|
+
p = sub.add_parser("run", help="run the prepared writers")
|
|
1457
|
+
p.add_argument("experiment")
|
|
1458
|
+
p.add_argument("writers", nargs="*")
|
|
1459
|
+
p.add_argument(
|
|
1460
|
+
"--jobs",
|
|
1461
|
+
type=int,
|
|
1462
|
+
default=1,
|
|
1463
|
+
help="writers at once; one by default, and naming writers keeps your "
|
|
1464
|
+
"order instead of the roster's cheap-then-premium one",
|
|
1465
|
+
)
|
|
1466
|
+
p.add_argument(
|
|
1467
|
+
"--timeout",
|
|
1468
|
+
type=float,
|
|
1469
|
+
help="seconds before a writer is killed; its partial prose is kept",
|
|
1470
|
+
)
|
|
1471
|
+
p = sub.add_parser(
|
|
1472
|
+
"roster",
|
|
1473
|
+
help="the default seats, optionally resolved",
|
|
1474
|
+
description=roster.__doc__,
|
|
1475
|
+
)
|
|
1476
|
+
p.add_argument(
|
|
1477
|
+
"--check", action="store_true", help="resolve each seat against the catalogs"
|
|
1478
|
+
)
|
|
1479
|
+
p.add_argument(
|
|
1480
|
+
"--plan",
|
|
1481
|
+
help="resolve this plan's writers instead of the default roster; "
|
|
1482
|
+
"departing from the roster is normal and its seats need checking too",
|
|
1483
|
+
)
|
|
1484
|
+
p = sub.add_parser("inspect", help="each writer's status and model")
|
|
1485
|
+
p.add_argument("experiment")
|
|
1486
|
+
p.add_argument(
|
|
1487
|
+
"--full", action="store_true", help="include full model-resolution evidence"
|
|
1488
|
+
)
|
|
1489
|
+
p = sub.add_parser(
|
|
1490
|
+
"report",
|
|
1491
|
+
help="each writer's cost beside its output",
|
|
1492
|
+
description=report.__doc__,
|
|
1493
|
+
)
|
|
1494
|
+
p.add_argument("experiment")
|
|
1495
|
+
p.add_argument("--table", action="store_true", help="markdown instead of JSON")
|
|
1496
|
+
p.add_argument(
|
|
1497
|
+
"--sizes",
|
|
1498
|
+
dest="sizes",
|
|
1499
|
+
action="store_true",
|
|
1500
|
+
default=None,
|
|
1501
|
+
help="show per-writer byte counts before the drafts are sealed",
|
|
1502
|
+
)
|
|
1503
|
+
p.add_argument(
|
|
1504
|
+
"--no-sizes", dest="sizes", action="store_false", help="always withhold them"
|
|
1505
|
+
)
|
|
1506
|
+
p = sub.add_parser(
|
|
1507
|
+
"anonymize",
|
|
1508
|
+
help="shuffle collected drafts under sealed letters",
|
|
1509
|
+
description=anonymize.__doc__,
|
|
1510
|
+
)
|
|
1511
|
+
p.add_argument("experiment")
|
|
1512
|
+
p.add_argument(
|
|
1513
|
+
"--output", default=None, help="default: the plan's first declared output"
|
|
1514
|
+
)
|
|
1515
|
+
p.add_argument(
|
|
1516
|
+
"--include-partial",
|
|
1517
|
+
action="store_true",
|
|
1518
|
+
help="shuffle in writers whose turn did not complete",
|
|
1519
|
+
)
|
|
1520
|
+
p.add_argument("--seed", type=int, default=None)
|
|
1521
|
+
|
|
1522
|
+
p = sub.add_parser("collect", help="bank a writer's draft and receipt")
|
|
1523
|
+
p.add_argument("experiment")
|
|
1524
|
+
p.add_argument("writer")
|
|
1525
|
+
p.add_argument(
|
|
1526
|
+
"--external",
|
|
1527
|
+
help="JSON file with session_id, model, provenance, optional transcript",
|
|
1528
|
+
)
|
|
1529
|
+
p = sub.add_parser("manual", help="run a writer interactively in a terminal")
|
|
1530
|
+
p.add_argument("experiment")
|
|
1531
|
+
p.add_argument("writer")
|
|
1532
|
+
p = sub.add_parser(
|
|
1533
|
+
"resume",
|
|
1534
|
+
help="continue a writer a provider quota stopped",
|
|
1535
|
+
description="Continue a writer a provider quota limit stopped, in its "
|
|
1536
|
+
"own session. Not a retry: the session keeps the reading, the "
|
|
1537
|
+
"reasoning and the draft, and the writer is told only that it was "
|
|
1538
|
+
"interrupted.",
|
|
1539
|
+
)
|
|
1540
|
+
p.add_argument("experiment")
|
|
1541
|
+
p.add_argument("writer")
|
|
1542
|
+
p.add_argument("--timeout", type=float, default=None)
|
|
1543
|
+
p = sub.add_parser(
|
|
1544
|
+
"watch",
|
|
1545
|
+
help="follow the writers as they work, without their words",
|
|
1546
|
+
description="Follow the writers as they work: who started and "
|
|
1547
|
+
"finished, and which files each read and wrote. Their words and draft "
|
|
1548
|
+
"sizes are withheld, because either identifies a draft before the "
|
|
1549
|
+
"drafts are sealed.",
|
|
1550
|
+
)
|
|
1551
|
+
p.add_argument("experiment")
|
|
1552
|
+
p.add_argument(
|
|
1553
|
+
"--words",
|
|
1554
|
+
action="store_true",
|
|
1555
|
+
help="show what the writers say and how much they wrote, deliberately",
|
|
1556
|
+
)
|
|
1557
|
+
p.add_argument("--poll", type=float, default=1.0)
|
|
1558
|
+
p.add_argument("--width", type=int, default=None)
|
|
1559
|
+
p = sub.add_parser("retry", help="stage a fresh writer in place of a failed one")
|
|
1560
|
+
p.add_argument("experiment")
|
|
1561
|
+
p.add_argument("writer")
|
|
1562
|
+
p.add_argument("new_id")
|
|
1563
|
+
args = parser.parse_args(argv)
|
|
1564
|
+
try:
|
|
1565
|
+
if args.action == "prepare":
|
|
1566
|
+
p = prepare(args.plan, args.destination)
|
|
1567
|
+
result = {
|
|
1568
|
+
"experiment": args.destination,
|
|
1569
|
+
"writers": len(p["writers"]),
|
|
1570
|
+
"supplied_bytes": p["supplied_bytes"],
|
|
1571
|
+
}
|
|
1572
|
+
elif args.action == "run":
|
|
1573
|
+
# Each writer's progress goes to stderr as it happens, so the
|
|
1574
|
+
# command's own log is not silent until the last writer finishes.
|
|
1575
|
+
with feed.narrating(Feed(args.experiment, history="none", show=NARRATION)):
|
|
1576
|
+
result = run(args.experiment, args.writers, args.jobs, args.timeout)
|
|
1577
|
+
print(json.dumps(result, indent=2))
|
|
1578
|
+
return int(any(v != "completed" for v in result.values()))
|
|
1579
|
+
elif args.action == "watch":
|
|
1580
|
+
following = Feed(
|
|
1581
|
+
args.experiment,
|
|
1582
|
+
show=feed.Show() if args.words else WITHHELD,
|
|
1583
|
+
width=args.width,
|
|
1584
|
+
)
|
|
1585
|
+
print("\n".join(following.header()), file=sys.stderr)
|
|
1586
|
+
try:
|
|
1587
|
+
feed.watch(following, poll=args.poll)
|
|
1588
|
+
except KeyboardInterrupt:
|
|
1589
|
+
pass
|
|
1590
|
+
return 0
|
|
1591
|
+
elif args.action == "roster":
|
|
1592
|
+
result = roster(check=args.check, plan=args.plan)
|
|
1593
|
+
print(json.dumps(result, indent=2, ensure_ascii=False))
|
|
1594
|
+
return int(bool(result.get("unavailable")))
|
|
1595
|
+
elif args.action == "inspect":
|
|
1596
|
+
result = inspect(args.experiment, full=args.full)
|
|
1597
|
+
elif args.action == "report":
|
|
1598
|
+
data = report(args.experiment, sizes=args.sizes)
|
|
1599
|
+
print(report_table(data) if args.table else json.dumps(data, indent=2))
|
|
1600
|
+
return 0
|
|
1601
|
+
elif args.action == "anonymize":
|
|
1602
|
+
result = anonymize(
|
|
1603
|
+
args.experiment,
|
|
1604
|
+
args.output,
|
|
1605
|
+
seed=args.seed,
|
|
1606
|
+
include_partial=args.include_partial,
|
|
1607
|
+
)
|
|
1608
|
+
elif args.action == "collect":
|
|
1609
|
+
result = collect(
|
|
1610
|
+
args.experiment,
|
|
1611
|
+
args.writer,
|
|
1612
|
+
external=read_json(args.external) if args.external else None,
|
|
1613
|
+
)
|
|
1614
|
+
elif args.action == "manual":
|
|
1615
|
+
result = run_writer(args.experiment, args.writer, manual=True)
|
|
1616
|
+
elif args.action == "resume":
|
|
1617
|
+
with feed.narrating(Feed(args.experiment, history="none", show=NARRATION)):
|
|
1618
|
+
result = resume(args.experiment, args.writer, timeout=args.timeout)
|
|
1619
|
+
else:
|
|
1620
|
+
result = retry(args.experiment, args.writer, args.new_id)
|
|
1621
|
+
print(json.dumps(result, indent=2, ensure_ascii=False))
|
|
1622
|
+
return 0
|
|
1623
|
+
except (ValueError, RuntimeError, OSError, subprocess.SubprocessError) as exc:
|
|
1624
|
+
print(f"workshop: {exc}", file=sys.stderr)
|
|
1625
|
+
return 1
|
|
1626
|
+
|
|
1627
|
+
|
|
1628
|
+
if __name__ == "__main__":
|
|
1629
|
+
raise SystemExit(main())
|