taskproof 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- taskproof/__init__.py +23 -0
- taskproof/__main__.py +8 -0
- taskproof/adapters/__init__.py +101 -0
- taskproof/adapters/base.py +228 -0
- taskproof/adapters/claude.py +46 -0
- taskproof/adapters/codex.py +140 -0
- taskproof/adapters/custom.py +61 -0
- taskproof/adapters/gemini.py +45 -0
- taskproof/adapters/opencode.py +46 -0
- taskproof/api/__init__.py +1 -0
- taskproof/api/server.py +822 -0
- taskproof/cli.py +1400 -0
- taskproof/concurrency.py +329 -0
- taskproof/dispatch.py +2289 -0
- taskproof/errors.py +55 -0
- taskproof/ledger.py +185 -0
- taskproof/models.py +155 -0
- taskproof/registry.py +1585 -0
- taskproof/schemas/result.json +58 -0
- taskproof/sql/schema.sql +70 -0
- taskproof/storage.py +372 -0
- taskproof/verify.py +615 -0
- taskproof/web/assets/MatrixView-BM8nu9s2.css +446 -0
- taskproof/web/assets/MatrixView-BM8nu9s2.css.gz +0 -0
- taskproof/web/assets/MatrixView-BMsawnxh.js +475 -0
- taskproof/web/assets/MatrixView-BMsawnxh.js.gz +0 -0
- taskproof/web/assets/ProjectsView-BcXcx5Gw.js +302 -0
- taskproof/web/assets/ProjectsView-BcXcx5Gw.js.gz +0 -0
- taskproof/web/assets/ProjectsView-CqNFL-fi.css +23 -0
- taskproof/web/assets/SettingsView-D_-waf0s.js +585 -0
- taskproof/web/assets/SettingsView-D_-waf0s.js.gz +0 -0
- taskproof/web/assets/SettingsView-aqTS3ubB.css +19 -0
- taskproof/web/assets/TaskDrawer-6JoGTH0s.js +597 -0
- taskproof/web/assets/TaskDrawer-6JoGTH0s.js.gz +0 -0
- taskproof/web/assets/TaskDrawer-N299VKzd.css +465 -0
- taskproof/web/assets/TaskDrawer-N299VKzd.css.gz +0 -0
- taskproof/web/assets/TasksView-BWhSHCHi.js +114 -0
- taskproof/web/assets/TasksView-BWhSHCHi.js.gz +0 -0
- taskproof/web/assets/TasksView-Bn08731V.css +30 -0
- taskproof/web/assets/icon-Dw-ohsdc.png +0 -0
- taskproof/web/assets/index-B-PLdDbR.js +75421 -0
- taskproof/web/assets/index-B-PLdDbR.js.gz +0 -0
- taskproof/web/assets/index-NKj7kbYl.css +1234 -0
- taskproof/web/assets/index-NKj7kbYl.css.gz +0 -0
- taskproof/web/index.html +23 -0
- taskproof-0.3.0.dist-info/METADATA +470 -0
- taskproof-0.3.0.dist-info/RECORD +51 -0
- taskproof-0.3.0.dist-info/WHEEL +5 -0
- taskproof-0.3.0.dist-info/entry_points.txt +2 -0
- taskproof-0.3.0.dist-info/licenses/LICENSE +201 -0
- taskproof-0.3.0.dist-info/top_level.txt +1 -0
taskproof/dispatch.py
ADDED
|
@@ -0,0 +1,2289 @@
|
|
|
1
|
+
"""The dispatch pipeline.
|
|
2
|
+
|
|
3
|
+
resolve project -> take concurrency slots -> run adapter -> parse result
|
|
4
|
+
-> check forbidden paths -> run acceptance -> record -> release slots
|
|
5
|
+
|
|
6
|
+
Every step appends to the audit stream, so a crashed run is reconstructable
|
|
7
|
+
from `events-*.jsonl` alone.
|
|
8
|
+
|
|
9
|
+
What this module deliberately does NOT do: decide whether the agent's claim is
|
|
10
|
+
true. The acceptance command in `verify.py` decides that.
|
|
11
|
+
|
|
12
|
+
Slot handling
|
|
13
|
+
-------------
|
|
14
|
+
`concurrency.acquire()` returns a list of :class:`concurrency.Lease` objects.
|
|
15
|
+
A ``Lease`` is a ``str`` (the scope name) that *also* carries the owner
|
|
16
|
+
``token``; `concurrency.release()` walks that list through ``_iter_leases`` and
|
|
17
|
+
only deletes a row whose token matches this process. The value returned by
|
|
18
|
+
``acquire`` is therefore kept verbatim and handed straight back to ``release``
|
|
19
|
+
inside a ``finally`` — passing it through anything lossy (e.g. a bare scope
|
|
20
|
+
string) would strip the proof of ownership and silently leak the slot.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import json
|
|
24
|
+
import os
|
|
25
|
+
import shutil
|
|
26
|
+
import signal
|
|
27
|
+
import subprocess
|
|
28
|
+
import threading
|
|
29
|
+
import time
|
|
30
|
+
from datetime import datetime
|
|
31
|
+
from typing import Optional
|
|
32
|
+
|
|
33
|
+
from . import concurrency, registry, storage, verify
|
|
34
|
+
from .adapters import get as get_adapter
|
|
35
|
+
from .adapters.base import ResultParseError
|
|
36
|
+
from .errors import EXIT_VERIFY, AdapterError, ConcurrencyError, UsageError, VerifyError
|
|
37
|
+
from .models import (
|
|
38
|
+
STATUS_BLOCKED,
|
|
39
|
+
STATUS_CANCELLED,
|
|
40
|
+
STATUS_DONE,
|
|
41
|
+
STATUS_FAILED,
|
|
42
|
+
STATUS_RUNNING,
|
|
43
|
+
STATUS_TIMEOUT,
|
|
44
|
+
STATUS_VERIFYING,
|
|
45
|
+
TERMINAL_STATUSES,
|
|
46
|
+
Task,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
#: Seconds to wait after SIGTERM before escalating to SIGKILL for a task tree.
|
|
50
|
+
#: Mirrors the grace period `_terminate_tree` uses on the adapter-spawn path.
|
|
51
|
+
TERMINATE_GRACE_SECONDS = 5.0
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class TaskNotFoundError(UsageError):
|
|
55
|
+
"""No task with that id. The API maps this to HTTP 404."""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class TaskStateError(UsageError):
|
|
59
|
+
"""The task is not in a state that allows the action. The API maps this
|
|
60
|
+
to HTTP 409 (state conflict), the CLI to its usual usage exit code."""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
#: Every status the dashboard reports on, so a status with no rows still shows
|
|
64
|
+
#: up as 0 instead of disappearing from the summary.
|
|
65
|
+
_ALL_STATUSES = (
|
|
66
|
+
STATUS_RUNNING,
|
|
67
|
+
STATUS_VERIFYING,
|
|
68
|
+
STATUS_DONE,
|
|
69
|
+
STATUS_FAILED,
|
|
70
|
+
STATUS_BLOCKED,
|
|
71
|
+
STATUS_TIMEOUT,
|
|
72
|
+
STATUS_CANCELLED,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
#: Written to `tasks.verify_cmd` when acceptance did not run. Deliberately NOT a
|
|
76
|
+
#: command: a skipped check must never be readable as a pass.
|
|
77
|
+
VERIFY_SKIPPED = "SKIPPED"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
#: Ceiling on the number of paths a single `changes` event carries. The audit
|
|
81
|
+
#: stream is read by humans (`show`) as well as machines, so a card that sweeps
|
|
82
|
+
#: thousands of files must not bloat the payload. The cut is not silent: the
|
|
83
|
+
#: unchanged `count` still reports the true total and `truncated` marks the cap.
|
|
84
|
+
MAX_RECORDED_CHANGES = 200
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def change_surface(changed, changed_before) -> dict:
|
|
88
|
+
"""Payload for the `changes` event: what this run left dirty, and what was
|
|
89
|
+
already dirty before it started.
|
|
90
|
+
|
|
91
|
+
`changed` is ``verify.diff_changed_files(...)`` — the paths the run actually
|
|
92
|
+
touched (created / removed / status-shifted). `changed_before` is the pre-run
|
|
93
|
+
``git status`` snapshot the forbidden-path gate already took, so no extra git
|
|
94
|
+
call is made here.
|
|
95
|
+
|
|
96
|
+
``paths`` is the union of the two, sorted: the full surface a reader would
|
|
97
|
+
see by running ``git status`` themselves. ``preexisting`` is the intersection
|
|
98
|
+
with the before-snapshot — the paths that were ALREADY dirty, which is what
|
|
99
|
+
inflates ``files_changed`` above this card's own work. Marking them is the
|
|
100
|
+
whole point: without it a reader cannot tell 「本卡改的」 from 「本来就在那儿的」.
|
|
101
|
+
"""
|
|
102
|
+
preexisting = set(changed_before or {})
|
|
103
|
+
surface = sorted(set(changed) | preexisting)
|
|
104
|
+
return {
|
|
105
|
+
"paths": surface[:MAX_RECORDED_CHANGES],
|
|
106
|
+
"count": len(surface),
|
|
107
|
+
"preexisting": sorted(preexisting),
|
|
108
|
+
"truncated": len(surface) > MAX_RECORDED_CHANGES,
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def prepare_workspace(workspace: str) -> None:
|
|
113
|
+
"""Create the workspace, database and audit directory if missing.
|
|
114
|
+
|
|
115
|
+
A brand-new workspace gets a registry holding the `[defaults]` table and no
|
|
116
|
+
projects: `register` is how entries arrive. Seeding an example project here
|
|
117
|
+
meant the dashboard listed a placeholder repository until someone deleted it.
|
|
118
|
+
"""
|
|
119
|
+
os.makedirs(workspace, exist_ok=True)
|
|
120
|
+
# The audit stream (`events-<month>.jsonl`) and per-task logs live beside the
|
|
121
|
+
# database, i.e. in the workspace itself; `logs/` keeps the logs tidy.
|
|
122
|
+
os.makedirs(os.path.join(workspace, "logs"), exist_ok=True)
|
|
123
|
+
|
|
124
|
+
conn = storage.connect(storage.db_path(workspace))
|
|
125
|
+
try:
|
|
126
|
+
storage.migrate(conn)
|
|
127
|
+
finally:
|
|
128
|
+
conn.close()
|
|
129
|
+
|
|
130
|
+
reg_path = registry.workspace_registry_path(workspace)
|
|
131
|
+
if not os.path.exists(reg_path):
|
|
132
|
+
with open(reg_path, "w", encoding="utf-8") as handle:
|
|
133
|
+
handle.write(registry.SAMPLE_REGISTRY)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _cap_aware_refusal(exc: ConcurrencyError, setting) -> ConcurrencyError:
|
|
137
|
+
"""Point a GLOBAL-cap refusal at its value, its source, and its ways out.
|
|
138
|
+
|
|
139
|
+
A same-group refusal is returned untouched: the honest hint there stays
|
|
140
|
+
"wait your turn". ``concurrency`` already phrases the two cases distinctly
|
|
141
|
+
("global cap reached" vs "group '<g>' is busy") — the same signal the REST
|
|
142
|
+
layer turns into its ``reason`` field — so the reroute needs no change to
|
|
143
|
+
the concurrency module, and both the CLI ``run`` and the REST create paths
|
|
144
|
+
get it because both go through this module.
|
|
145
|
+
|
|
146
|
+
For a global-cap refusal we append the *effective* cap and where it came
|
|
147
|
+
from (card 42: ``上限 3,来源:自动探测 14 核 ÷ 4``) and name the exits —
|
|
148
|
+
``run --cap N`` / ``taskproof config --concurrency N`` — so the operator is
|
|
149
|
+
never told "no" without a next step.
|
|
150
|
+
"""
|
|
151
|
+
if str(exc).startswith("global cap"):
|
|
152
|
+
return ConcurrencyError(
|
|
153
|
+
f"{exc}(上限 {setting.value},来源:{concurrency.source_label(setting)})",
|
|
154
|
+
hint=(
|
|
155
|
+
"临时抬高(run --cap N)/"
|
|
156
|
+
"改配置(taskproof config --concurrency N)"
|
|
157
|
+
),
|
|
158
|
+
)
|
|
159
|
+
return exc
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def dispatch(
|
|
163
|
+
workspace: str,
|
|
164
|
+
project_key: str,
|
|
165
|
+
brief: str,
|
|
166
|
+
*,
|
|
167
|
+
adapter: str = "codex",
|
|
168
|
+
model: Optional[str] = None,
|
|
169
|
+
reasoning: Optional[str] = None,
|
|
170
|
+
read_only: bool = False,
|
|
171
|
+
worktree: bool = False,
|
|
172
|
+
skip_verify: bool = False,
|
|
173
|
+
timeout: Optional[int] = None,
|
|
174
|
+
idle_timeout: Optional[int] = None,
|
|
175
|
+
rerun_of: Optional[str] = None,
|
|
176
|
+
cap: Optional[int] = None,
|
|
177
|
+
) -> str:
|
|
178
|
+
"""Run one task end to end and return its id.
|
|
179
|
+
|
|
180
|
+
``rerun_of`` names the terminal card this run is a fresh copy of; when set,
|
|
181
|
+
the new card gets a ``rerun`` event carrying ``rerun_of`` and the old card
|
|
182
|
+
gets one carrying ``rerun_as``, both written before the adapter runs so the
|
|
183
|
+
link survives even a failed rerun. ``taskproof rerun`` is the only caller.
|
|
184
|
+
|
|
185
|
+
Raises the typed errors in `errors.py` so the CLI can map them to exit codes:
|
|
186
|
+
ConcurrencyError -> 75, AdapterError -> 70, VerifyError -> 71, RegistryError -> 2
|
|
187
|
+
|
|
188
|
+
Rules that must survive any refactor:
|
|
189
|
+
* the acceptance command runs in the PROJECT workdir, never the workspace
|
|
190
|
+
* `skip_verify` records verify as SKIPPED, never as passed
|
|
191
|
+
* slots are released on every exit path, including exceptions
|
|
192
|
+
* only processes this function spawned are ever terminated
|
|
193
|
+
"""
|
|
194
|
+
# ② prepare + connect (creates the db and, on a fresh workspace, the sample
|
|
195
|
+
# registry we are about to read).
|
|
196
|
+
prepare_workspace(workspace)
|
|
197
|
+
|
|
198
|
+
# ① resolve the taskgroup. A missing/unknown key is a RegistryError (exit 2).
|
|
199
|
+
reg = registry.load(registry.workspace_registry_path(workspace))
|
|
200
|
+
taskgroup = reg.require(project_key)
|
|
201
|
+
|
|
202
|
+
# The effective per-run timeout and where it came from (card 81): a
|
|
203
|
+
# `--timeout` wins over the lane's own `timeout`, which wins over
|
|
204
|
+
# `[defaults] timeout`, which wins over the built-in fallback.
|
|
205
|
+
timeout_setting = registry.resolve_timeout(taskgroup, reg.defaults, timeout)
|
|
206
|
+
effective_timeout = timeout_setting.value
|
|
207
|
+
|
|
208
|
+
# The effective no-progress watchdog budget and where it came from (card 83):
|
|
209
|
+
# a `--idle-timeout` wins over the lane's own `idle_timeout`, which wins over
|
|
210
|
+
# the built-in fallback (`0` == off, the shipped default).
|
|
211
|
+
idle_setting = registry.resolve_idle_timeout(taskgroup, idle_timeout)
|
|
212
|
+
effective_idle_timeout = idle_setting.value
|
|
213
|
+
|
|
214
|
+
# The effective global cap for this dispatch, with its source (card 42). A
|
|
215
|
+
# `--cap N` is a one-off override: it rides the event stream but is never
|
|
216
|
+
# written to the registry.
|
|
217
|
+
setting = concurrency.resolve(reg.defaults, cap)
|
|
218
|
+
|
|
219
|
+
# Resolve the adapter object now (a bare constructor, no side effects): a bad
|
|
220
|
+
# adapter name is a UsageError raised before any task row or slot exists, so
|
|
221
|
+
# a typo can never leave a task stuck in `running`.
|
|
222
|
+
adapter_obj = get_adapter(
|
|
223
|
+
adapter, model=model, reasoning=reasoning, timeout=effective_timeout
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
conn = storage.connect(storage.db_path(workspace))
|
|
227
|
+
try:
|
|
228
|
+
storage.migrate(conn)
|
|
229
|
+
|
|
230
|
+
# ② allocate the task id before claiming a slot, so the claim row can
|
|
231
|
+
# name the task it is protecting.
|
|
232
|
+
prefix_date = datetime.now().strftime("%Y%m%d")
|
|
233
|
+
task_id = storage.next_task_id(conn, prefix_date=prefix_date)
|
|
234
|
+
|
|
235
|
+
# ③ claim the group slot + a slot under the global cap. A refusal is a
|
|
236
|
+
# ConcurrencyError (exit 75) and no task row is written.
|
|
237
|
+
ttl = max(1, int(effective_timeout)) + 60
|
|
238
|
+
try:
|
|
239
|
+
scopes = concurrency.acquire(
|
|
240
|
+
conn, task_id, taskgroup.group, cap=setting.value, ttl=ttl
|
|
241
|
+
)
|
|
242
|
+
except ConcurrencyError as exc:
|
|
243
|
+
raise _cap_aware_refusal(exc, setting) from None
|
|
244
|
+
|
|
245
|
+
# ⑨ Whatever happens below — success, a recorded failure, or a raised
|
|
246
|
+
# error — the exact credential from ③ is released. `release` verifies the
|
|
247
|
+
# ownership token, so this can only ever drop our own claim.
|
|
248
|
+
try:
|
|
249
|
+
# ④ the task row + the `started` event.
|
|
250
|
+
task = Task(
|
|
251
|
+
id=task_id,
|
|
252
|
+
project=taskgroup.id,
|
|
253
|
+
group=taskgroup.group,
|
|
254
|
+
brief=brief,
|
|
255
|
+
status=STATUS_RUNNING,
|
|
256
|
+
adapter=adapter,
|
|
257
|
+
model=model,
|
|
258
|
+
reasoning=reasoning,
|
|
259
|
+
pid=os.getpid(),
|
|
260
|
+
created_at=storage.now_iso(),
|
|
261
|
+
started_at=storage.now_iso(),
|
|
262
|
+
)
|
|
263
|
+
storage.insert_task(conn, task)
|
|
264
|
+
storage.append_event(
|
|
265
|
+
conn,
|
|
266
|
+
task_id,
|
|
267
|
+
"started",
|
|
268
|
+
{
|
|
269
|
+
"project": taskgroup.id,
|
|
270
|
+
"group": taskgroup.group,
|
|
271
|
+
"adapter": adapter,
|
|
272
|
+
"read_only": read_only,
|
|
273
|
+
"worktree": worktree,
|
|
274
|
+
"skip_verify": skip_verify,
|
|
275
|
+
# The hard timeout this run enforces, and where it came
|
|
276
|
+
# from (card 81). The value has no column, so it rides in
|
|
277
|
+
# the event: `rerun` reads it back to reuse the same flag
|
|
278
|
+
# (See `_run_options`). `timeout_source` is shaped like the
|
|
279
|
+
# `cap_source` beside it — cli / lane / toml / default.
|
|
280
|
+
"timeout": effective_timeout,
|
|
281
|
+
"timeout_source": timeout_setting.source,
|
|
282
|
+
# The no-progress watchdog budget this run enforces and
|
|
283
|
+
# where it came from (card 83). `idle_timeout_source` is
|
|
284
|
+
# shaped like `timeout_source` beside it — cli / lane /
|
|
285
|
+
# default — and `0` means the watchdog is off for this run.
|
|
286
|
+
"idle_timeout": effective_idle_timeout,
|
|
287
|
+
"idle_timeout_source": idle_setting.source,
|
|
288
|
+
# The effective global cap *and its source* (card 42), so the
|
|
289
|
+
# event stream can explain why a card was admitted — e.g. a
|
|
290
|
+
# one-off `--cap 5` that let a fourth card run.
|
|
291
|
+
"cap": setting.value,
|
|
292
|
+
"cap_source": setting.source,
|
|
293
|
+
},
|
|
294
|
+
)
|
|
295
|
+
|
|
296
|
+
if rerun_of:
|
|
297
|
+
_link_rerun(conn, task_id, rerun_of)
|
|
298
|
+
|
|
299
|
+
_execute_claimed(
|
|
300
|
+
conn,
|
|
301
|
+
task_id=task_id,
|
|
302
|
+
taskgroup=taskgroup,
|
|
303
|
+
adapter_obj=adapter_obj,
|
|
304
|
+
adapter_name=adapter,
|
|
305
|
+
brief=brief,
|
|
306
|
+
workspace=workspace,
|
|
307
|
+
read_only=read_only,
|
|
308
|
+
worktree=worktree,
|
|
309
|
+
skip_verify=skip_verify,
|
|
310
|
+
timeout=effective_timeout,
|
|
311
|
+
idle_timeout=effective_idle_timeout,
|
|
312
|
+
)
|
|
313
|
+
return task_id
|
|
314
|
+
finally:
|
|
315
|
+
concurrency.release(conn, scopes)
|
|
316
|
+
finally:
|
|
317
|
+
conn.close()
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def _execute_claimed(
|
|
321
|
+
conn,
|
|
322
|
+
*,
|
|
323
|
+
task_id: str,
|
|
324
|
+
taskgroup,
|
|
325
|
+
adapter_obj,
|
|
326
|
+
adapter_name: str,
|
|
327
|
+
brief: str,
|
|
328
|
+
workspace: str,
|
|
329
|
+
read_only: bool,
|
|
330
|
+
worktree: bool,
|
|
331
|
+
skip_verify: bool,
|
|
332
|
+
timeout: int,
|
|
333
|
+
idle_timeout: int,
|
|
334
|
+
) -> None:
|
|
335
|
+
"""Steps 5-8 for a task that is already `running` and holds a slot.
|
|
336
|
+
|
|
337
|
+
The row is already `running` and holds a slot. Owns the whole worktree
|
|
338
|
+
lifecycle (create -> execute -> keep-if-changed / remove-if-clean) and
|
|
339
|
+
raises whatever `_execute` raises; it never touches the concurrency claim,
|
|
340
|
+
which stays the caller's.
|
|
341
|
+
"""
|
|
342
|
+
worktree_path = None
|
|
343
|
+
run_dir = taskgroup.path
|
|
344
|
+
try:
|
|
345
|
+
# A lane that declares `workspace = "worktree"` wins over the one-time
|
|
346
|
+
# `--worktree` flag: it already IS the long-lived checkout, so creating a
|
|
347
|
+
# second one-off worktree would be wrong. The workspace is created or
|
|
348
|
+
# reused here -- the first moment a card actually has a slot and is about
|
|
349
|
+
# to run, never merely by editing the registry.
|
|
350
|
+
if taskgroup.workspace == "worktree":
|
|
351
|
+
_, run_dir = _ensure_lane_workspace(
|
|
352
|
+
conn, task_id, taskgroup, worktree_flag=worktree
|
|
353
|
+
)
|
|
354
|
+
elif worktree:
|
|
355
|
+
if not _is_git_repo(taskgroup.path):
|
|
356
|
+
raise UsageError(
|
|
357
|
+
f"--worktree needs a git repository, but {taskgroup.path} "
|
|
358
|
+
"is not one",
|
|
359
|
+
hint="run without --worktree, or point the registry at a git checkout",
|
|
360
|
+
)
|
|
361
|
+
worktree_path = _create_worktree(taskgroup.path, task_id)
|
|
362
|
+
run_dir = worktree_path
|
|
363
|
+
storage.append_event(conn, task_id, "worktree_created", {"path": run_dir})
|
|
364
|
+
|
|
365
|
+
# Land the run directory on the row the moment it is final -- after a
|
|
366
|
+
# worktree has resolved its path, before the adapter spawns. Without it
|
|
367
|
+
# the row carried NULL workdir for the whole run and the live
|
|
368
|
+
# `files_changed_live` probe had nothing to look at.
|
|
369
|
+
storage.update_task(conn, task_id, workdir=run_dir)
|
|
370
|
+
|
|
371
|
+
_execute(
|
|
372
|
+
conn,
|
|
373
|
+
task_id=task_id,
|
|
374
|
+
taskgroup=taskgroup,
|
|
375
|
+
adapter_obj=adapter_obj,
|
|
376
|
+
adapter_name=adapter_name,
|
|
377
|
+
brief=brief,
|
|
378
|
+
run_dir=run_dir,
|
|
379
|
+
workspace=workspace,
|
|
380
|
+
read_only=read_only,
|
|
381
|
+
skip_verify=skip_verify,
|
|
382
|
+
timeout=timeout,
|
|
383
|
+
idle_timeout=idle_timeout,
|
|
384
|
+
)
|
|
385
|
+
finally:
|
|
386
|
+
# Decide the worktree's fate on every exit path. A run that left changes
|
|
387
|
+
# behind KEEPS the checkout and announces it -- silently discarding
|
|
388
|
+
# uncommitted work was the whole bug this replaced. A clean run removes
|
|
389
|
+
# it as before; a cleanup failure is recorded rather than raised (it
|
|
390
|
+
# would mask the real outcome).
|
|
391
|
+
if worktree_path is not None:
|
|
392
|
+
changed = _worktree_change_count(worktree_path)
|
|
393
|
+
if changed:
|
|
394
|
+
print(
|
|
395
|
+
f"worktree kept: {worktree_path} ({changed} files changed)",
|
|
396
|
+
flush=True,
|
|
397
|
+
)
|
|
398
|
+
storage.append_event(
|
|
399
|
+
conn,
|
|
400
|
+
task_id,
|
|
401
|
+
"worktree",
|
|
402
|
+
{
|
|
403
|
+
"path": worktree_path,
|
|
404
|
+
"changed": changed,
|
|
405
|
+
# The source repo, so `rm` can unregister the checkout
|
|
406
|
+
# without a registry lookup.
|
|
407
|
+
"repo": taskgroup.path,
|
|
408
|
+
},
|
|
409
|
+
)
|
|
410
|
+
else:
|
|
411
|
+
error = _remove_worktree(taskgroup.path, worktree_path)
|
|
412
|
+
if error:
|
|
413
|
+
storage.append_event(
|
|
414
|
+
conn,
|
|
415
|
+
task_id,
|
|
416
|
+
"worktree_cleanup_failed",
|
|
417
|
+
{"path": worktree_path, "error": error},
|
|
418
|
+
)
|
|
419
|
+
else:
|
|
420
|
+
storage.append_event(
|
|
421
|
+
conn, task_id, "worktree_removed", {"path": worktree_path}
|
|
422
|
+
)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _link_rerun(conn, new_id: str, old_id: str) -> None:
|
|
426
|
+
"""Record the two-way link between a rerun and the card it was copied from.
|
|
427
|
+
|
|
428
|
+
Written through `storage.append_event`, so it lands in BOTH the `events`
|
|
429
|
+
table and the JSONL audit stream — `taskproof log <id> --json` reads the
|
|
430
|
+
stream, which is why the link has to be an event and not a column. The new
|
|
431
|
+
card carries ``rerun_of``, the old card carries ``rerun_as``.
|
|
432
|
+
"""
|
|
433
|
+
storage.append_event(conn, new_id, "rerun", {"rerun_of": old_id})
|
|
434
|
+
storage.append_event(conn, old_id, "rerun", {"rerun_as": new_id})
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _run_options(conn, task_id: str) -> dict:
|
|
438
|
+
"""Recover the run flags a task carried, for `rerun` to reuse.
|
|
439
|
+
|
|
440
|
+
A card born running records read_only / worktree / skip_verify / timeout /
|
|
441
|
+
idle_timeout on its `started` event. The events are read oldest-first and
|
|
442
|
+
merged, so the newest value of each key wins.
|
|
443
|
+
|
|
444
|
+
A flag missing from every event is simply absent; `rerun` then applies the
|
|
445
|
+
documented default (read_only / worktree / skip_verify false; timeout =
|
|
446
|
+
the registry timeout; idle_timeout = the lane's budget, or off). A row
|
|
447
|
+
written by an older build that recorded no flags yields `{}` and reruns
|
|
448
|
+
with those same defaults.
|
|
449
|
+
"""
|
|
450
|
+
options: dict = {}
|
|
451
|
+
rows = conn.execute(
|
|
452
|
+
"SELECT payload FROM events WHERE task_id = ? "
|
|
453
|
+
"AND event = 'started' ORDER BY id ASC",
|
|
454
|
+
(task_id,),
|
|
455
|
+
).fetchall()
|
|
456
|
+
for row in rows:
|
|
457
|
+
text = row[0]
|
|
458
|
+
try:
|
|
459
|
+
payload = json.loads(text) if text else {}
|
|
460
|
+
except (json.JSONDecodeError, TypeError, ValueError):
|
|
461
|
+
continue
|
|
462
|
+
if not isinstance(payload, dict):
|
|
463
|
+
continue
|
|
464
|
+
for key in ("read_only", "worktree", "skip_verify", "timeout", "idle_timeout"):
|
|
465
|
+
if payload.get(key) is not None:
|
|
466
|
+
options[key] = payload[key]
|
|
467
|
+
return options
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _execute(
|
|
472
|
+
conn,
|
|
473
|
+
*,
|
|
474
|
+
task_id: str,
|
|
475
|
+
taskgroup,
|
|
476
|
+
adapter_obj,
|
|
477
|
+
adapter_name: str,
|
|
478
|
+
brief: str,
|
|
479
|
+
run_dir: str,
|
|
480
|
+
workspace: str,
|
|
481
|
+
read_only: bool,
|
|
482
|
+
skip_verify: bool,
|
|
483
|
+
timeout: int,
|
|
484
|
+
idle_timeout: int = 0,
|
|
485
|
+
) -> None:
|
|
486
|
+
"""Steps ⑤-⑧: run the adapter, gate on forbidden paths, verify, then record.
|
|
487
|
+
|
|
488
|
+
Raises AdapterError / VerifyError; always leaves the task row in a terminal
|
|
489
|
+
state (or already recorded) before it does.
|
|
490
|
+
"""
|
|
491
|
+
log_path = os.path.join(workspace, "logs", f"{task_id}.log")
|
|
492
|
+
|
|
493
|
+
# Resolve the structured-result contract for THIS taskgroup: the taskgroup
|
|
494
|
+
# setting wins, otherwise the package default. `None` means structured
|
|
495
|
+
# output is disabled — that is recorded rather than done silently, so a
|
|
496
|
+
# free-text result can always be explained after the fact.
|
|
497
|
+
schema_path = registry.result_schema_path(taskgroup)
|
|
498
|
+
if schema_path is None:
|
|
499
|
+
storage.append_event(
|
|
500
|
+
conn,
|
|
501
|
+
task_id,
|
|
502
|
+
"result_schema",
|
|
503
|
+
{
|
|
504
|
+
"enabled": False,
|
|
505
|
+
"path": None,
|
|
506
|
+
"note": "structured result disabled (result_schema = \"none\")",
|
|
507
|
+
},
|
|
508
|
+
)
|
|
509
|
+
else:
|
|
510
|
+
storage.append_event(
|
|
511
|
+
conn, task_id, "result_schema", {"enabled": True, "path": schema_path}
|
|
512
|
+
)
|
|
513
|
+
|
|
514
|
+
# ⑤ run the agent CLI, teeing its output to `log_path` as it goes.
|
|
515
|
+
#
|
|
516
|
+
# Snapshot both sides of the run before launching the adapter. The git
|
|
517
|
+
# change list is naturally a current-state snapshot, but the gate needs a
|
|
518
|
+
# difference: pre-existing dirty/untracked paths are not the adapter's
|
|
519
|
+
# work. The protected-path snapshots remain the backstop for ignored paths
|
|
520
|
+
# and `.git/`.
|
|
521
|
+
git_rules = []
|
|
522
|
+
file_rules = []
|
|
523
|
+
presence_rules = []
|
|
524
|
+
for raw in taskgroup.forbidden_paths or []:
|
|
525
|
+
# Keep the historical untyped `.git/` spelling working: it has always
|
|
526
|
+
# meant the state probe since card 30. An explicit `file:.git/` opts
|
|
527
|
+
# back into the file fingerprint signal.
|
|
528
|
+
if verify.is_git_forbidden_rule(run_dir, raw):
|
|
529
|
+
git_rules.append(raw)
|
|
530
|
+
continue
|
|
531
|
+
rule = verify.parse_forbidden_rule(raw)
|
|
532
|
+
if rule is None:
|
|
533
|
+
continue
|
|
534
|
+
if rule.kind == "git":
|
|
535
|
+
git_rules.append(raw)
|
|
536
|
+
elif rule.kind == "presence":
|
|
537
|
+
presence_rules.append(rule)
|
|
538
|
+
else:
|
|
539
|
+
file_rules.append(rule)
|
|
540
|
+
|
|
541
|
+
forbidden_before, file_before_truncated = verify.snapshot_forbidden(
|
|
542
|
+
run_dir, [rule.path for rule in file_rules]
|
|
543
|
+
)
|
|
544
|
+
presence_before, presence_before_truncated = verify.snapshot_forbidden(
|
|
545
|
+
run_dir, [rule.path for rule in presence_rules]
|
|
546
|
+
)
|
|
547
|
+
git_state_before = verify.git_state_snapshot(run_dir) if git_rules else {}
|
|
548
|
+
changed_before = verify.changed_files_snapshot(run_dir)
|
|
549
|
+
|
|
550
|
+
def _record_pgid(proc) -> None:
|
|
551
|
+
"""Persist the adapter's own group id the moment it exists.
|
|
552
|
+
|
|
553
|
+
`_run_adapter` spawns with ``start_new_session=True``, so the child leads
|
|
554
|
+
a brand-new session/process group and its pgid equals its pid. That is
|
|
555
|
+
the only group that contains the adapter and its descendants without
|
|
556
|
+
also containing this dispatcher or unrelated siblings — exactly what
|
|
557
|
+
`cancel <id>` must signal after a restart. The row's `pid` stays this
|
|
558
|
+
run process' ``os.getpid()`` (written before the adapter exists); the
|
|
559
|
+
pgid cannot be known until ``Popen`` returns, so it is recorded here,
|
|
560
|
+
immediately after the spawn and before we wait on the tree.
|
|
561
|
+
"""
|
|
562
|
+
try:
|
|
563
|
+
pgid = os.getpgid(proc.pid)
|
|
564
|
+
except (ProcessLookupError, OSError):
|
|
565
|
+
return
|
|
566
|
+
storage.update_task(conn, task_id, pgid=pgid)
|
|
567
|
+
|
|
568
|
+
try:
|
|
569
|
+
exit_code, out, err = _run_adapter(
|
|
570
|
+
adapter_obj,
|
|
571
|
+
brief=brief,
|
|
572
|
+
workdir=run_dir,
|
|
573
|
+
log_path=log_path,
|
|
574
|
+
schema_path=schema_path,
|
|
575
|
+
read_only=read_only,
|
|
576
|
+
idle_timeout=idle_timeout,
|
|
577
|
+
on_spawn=_record_pgid,
|
|
578
|
+
)
|
|
579
|
+
except VerifyError as exc:
|
|
580
|
+
if _task_cancelled(conn, task_id):
|
|
581
|
+
# A concurrent `cancel` already recorded the terminal state; do not
|
|
582
|
+
# overwrite it or append a contradictory event.
|
|
583
|
+
return
|
|
584
|
+
# timeout: `timeout` terminal state, exit code carries EXIT_VERIFY
|
|
585
|
+
# meaning. The event names WHY the clock ran out — the hard wall clock
|
|
586
|
+
# or the no-progress watchdog (card 83) — and carries the budget that
|
|
587
|
+
# fired, so a card that stopped itself early is never mistaken for one
|
|
588
|
+
# that burned the whole wall clock.
|
|
589
|
+
_finish_task(
|
|
590
|
+
conn, task_id, status=STATUS_TIMEOUT, exit_code=EXIT_VERIFY, workdir=run_dir
|
|
591
|
+
)
|
|
592
|
+
reason = getattr(exc, "reason", "wall-clock")
|
|
593
|
+
payload = {"adapter": adapter_name, "detail": str(exc), "reason": reason}
|
|
594
|
+
if reason == "idle":
|
|
595
|
+
payload["idle_timeout"] = getattr(exc, "idle_timeout", idle_timeout)
|
|
596
|
+
else:
|
|
597
|
+
payload["timeout"] = getattr(exc, "timeout", timeout)
|
|
598
|
+
storage.append_event(conn, task_id, "timeout", payload)
|
|
599
|
+
raise
|
|
600
|
+
except AdapterError as exc:
|
|
601
|
+
if _task_cancelled(conn, task_id):
|
|
602
|
+
return
|
|
603
|
+
_finish_task(conn, task_id, status=STATUS_FAILED, workdir=run_dir)
|
|
604
|
+
storage.append_event(
|
|
605
|
+
conn, task_id, "failed", {"stage": "adapter", "reason": str(exc)}
|
|
606
|
+
)
|
|
607
|
+
raise
|
|
608
|
+
|
|
609
|
+
# The adapter is gone. If a `cancel` landed while it was running (e.g. it
|
|
610
|
+
# was signalled from another process), the row is already `cancelled`: stop
|
|
611
|
+
# here so the pipeline cannot round a cancelled task up to done/failed.
|
|
612
|
+
if _task_cancelled(conn, task_id):
|
|
613
|
+
return
|
|
614
|
+
|
|
615
|
+
# Snapshot the protected paths again here — immediately after the agent
|
|
616
|
+
# exits, and BEFORE our own change detection runs. `git status` refreshes the
|
|
617
|
+
# index and takes a lock file, so snapshotting after it would attribute our
|
|
618
|
+
# own housekeeping to the agent: with a `.git/` rule that made every run fail.
|
|
619
|
+
forbidden_after, file_after_truncated = verify.snapshot_forbidden(
|
|
620
|
+
run_dir, [rule.path for rule in file_rules]
|
|
621
|
+
)
|
|
622
|
+
presence_after, presence_after_truncated = verify.snapshot_forbidden(
|
|
623
|
+
run_dir, [rule.path for rule in presence_rules]
|
|
624
|
+
)
|
|
625
|
+
# Take the git-state probe before our own `git status` snapshots: that call
|
|
626
|
+
# can refresh `.git/index`, which a state-only `.git/` rule must not read as
|
|
627
|
+
# the agent's doing (same reason the fingerprint snapshot runs first).
|
|
628
|
+
git_state_after = verify.git_state_snapshot(run_dir) if git_rules else {}
|
|
629
|
+
changed_after = verify.changed_files_snapshot(run_dir)
|
|
630
|
+
forbidden_truncated = (
|
|
631
|
+
file_before_truncated
|
|
632
|
+
or file_after_truncated
|
|
633
|
+
or presence_before_truncated
|
|
634
|
+
or presence_after_truncated
|
|
635
|
+
)
|
|
636
|
+
|
|
637
|
+
# ⑤ (cont.) parse the adapter's result. A parse failure is an adapter failure
|
|
638
|
+
# (exit 70) — it must never be rounded up to success.
|
|
639
|
+
result_path = getattr(adapter_obj, "result_path", None)
|
|
640
|
+
result = None
|
|
641
|
+
parse_error = None
|
|
642
|
+
if exit_code == 0:
|
|
643
|
+
try:
|
|
644
|
+
result = adapter_obj.parse_result(
|
|
645
|
+
exit_code=exit_code, stdout=out, stderr=err, result_path=result_path
|
|
646
|
+
)
|
|
647
|
+
except ResultParseError as exc:
|
|
648
|
+
parse_error = exc
|
|
649
|
+
|
|
650
|
+
if exit_code != 0 or result is None:
|
|
651
|
+
reason = parse_error or f"agent CLI exited {exit_code}"
|
|
652
|
+
_finish_task(
|
|
653
|
+
conn,
|
|
654
|
+
task_id,
|
|
655
|
+
status=STATUS_FAILED,
|
|
656
|
+
exit_code=exit_code,
|
|
657
|
+
workdir=run_dir,
|
|
658
|
+
result_path=result_path,
|
|
659
|
+
)
|
|
660
|
+
storage.append_event(
|
|
661
|
+
conn,
|
|
662
|
+
task_id,
|
|
663
|
+
"failed",
|
|
664
|
+
{"stage": "adapter", "exit_code": exit_code, "reason": str(reason)},
|
|
665
|
+
)
|
|
666
|
+
raise AdapterError(
|
|
667
|
+
f"adapter '{adapter_name}' failed: {reason}",
|
|
668
|
+
hint=f"see {log_path} for the full agent output",
|
|
669
|
+
)
|
|
670
|
+
|
|
671
|
+
storage.append_event(
|
|
672
|
+
conn,
|
|
673
|
+
task_id,
|
|
674
|
+
"adapter",
|
|
675
|
+
{
|
|
676
|
+
"exit_code": exit_code,
|
|
677
|
+
"degraded": bool(result.parse_degraded),
|
|
678
|
+
"summary": result.summary,
|
|
679
|
+
},
|
|
680
|
+
)
|
|
681
|
+
|
|
682
|
+
# ⑥ change + forbidden-path check, on the PROJECT workdir (or its worktree).
|
|
683
|
+
changed = verify.diff_changed_files(changed_before, changed_after)
|
|
684
|
+
files_changed = verify.detect_changes(run_dir)
|
|
685
|
+
# Record the change surface in the audit stream (schema untouched: an event,
|
|
686
|
+
# not a column). `changed` is this run's own work; `changed_before` is the
|
|
687
|
+
# pre-run snapshot, so the event also flags what was already dirty — the part
|
|
688
|
+
# of `files_changed` that is not this card's doing.
|
|
689
|
+
storage.append_event(
|
|
690
|
+
conn,
|
|
691
|
+
task_id,
|
|
692
|
+
"changes",
|
|
693
|
+
change_surface(changed, changed_before),
|
|
694
|
+
)
|
|
695
|
+
# Three independent signals:
|
|
696
|
+
# file -> git status difference + size:mtime_ns fingerprints
|
|
697
|
+
# presence -> before/after file-set membership only
|
|
698
|
+
# git-state -> HEAD/refs/stash snapshot
|
|
699
|
+
file_snapshot_changes = verify.diff_snapshots(
|
|
700
|
+
forbidden_before, forbidden_after
|
|
701
|
+
)
|
|
702
|
+
presence_changes = verify.diff_presence_snapshots(
|
|
703
|
+
presence_before, presence_after
|
|
704
|
+
)
|
|
705
|
+
git_state_changes = (
|
|
706
|
+
verify.diff_snapshots(git_state_before, git_state_after) if git_rules else []
|
|
707
|
+
)
|
|
708
|
+
|
|
709
|
+
violations = []
|
|
710
|
+
if git_state_changes:
|
|
711
|
+
for raw in git_rules:
|
|
712
|
+
label = ".git" if verify.is_git_forbidden_rule(run_dir, raw) else str(raw)
|
|
713
|
+
violations.append(
|
|
714
|
+
{"rule": label, "kind": "git-state", "changed": git_state_changes}
|
|
715
|
+
)
|
|
716
|
+
for rule in file_rules:
|
|
717
|
+
rule_paths = sorted(
|
|
718
|
+
set(verify.check_forbidden(run_dir, [rule.path], changed))
|
|
719
|
+
| set(
|
|
720
|
+
verify.check_forbidden(
|
|
721
|
+
run_dir, [rule.path], file_snapshot_changes
|
|
722
|
+
)
|
|
723
|
+
)
|
|
724
|
+
)
|
|
725
|
+
if rule_paths:
|
|
726
|
+
violations.append(
|
|
727
|
+
{"rule": rule.raw, "kind": "file", "paths": rule_paths}
|
|
728
|
+
)
|
|
729
|
+
for rule in presence_rules:
|
|
730
|
+
rule_paths = verify.check_forbidden(
|
|
731
|
+
run_dir, [rule.path], presence_changes
|
|
732
|
+
)
|
|
733
|
+
if rule_paths:
|
|
734
|
+
violations.append(
|
|
735
|
+
{"rule": rule.raw, "kind": "presence", "paths": rule_paths}
|
|
736
|
+
)
|
|
737
|
+
|
|
738
|
+
violation_paths = sorted(
|
|
739
|
+
{
|
|
740
|
+
path
|
|
741
|
+
for entry in violations
|
|
742
|
+
for path in (entry.get("changed") or entry.get("paths") or [])
|
|
743
|
+
}
|
|
744
|
+
)
|
|
745
|
+
if violations:
|
|
746
|
+
# A protected path was touched. This is a fact about the run, not a
|
|
747
|
+
# verdict that the work is bad — so record it and DO NOT short-circuit.
|
|
748
|
+
# Acceptance still runs below (⑦) and a human decides in layer 3
|
|
749
|
+
# (`accept`), not this pipeline. The offending paths go into the audit
|
|
750
|
+
# stream here, untouched, exactly as before.
|
|
751
|
+
storage.append_event(
|
|
752
|
+
conn,
|
|
753
|
+
task_id,
|
|
754
|
+
"forbidden",
|
|
755
|
+
{
|
|
756
|
+
"violations": violations,
|
|
757
|
+
# Flat, backwards-compatible list for older boards/callers.
|
|
758
|
+
"paths": violation_paths,
|
|
759
|
+
"rules": list(taskgroup.forbidden_paths),
|
|
760
|
+
# True when a protected tree was too large to walk fully, so the
|
|
761
|
+
# guarantee for that rule was partial.
|
|
762
|
+
"snapshot_truncated": forbidden_truncated,
|
|
763
|
+
},
|
|
764
|
+
)
|
|
765
|
+
|
|
766
|
+
# ⑦ acceptance, run in the PROJECT workdir — never the workspace. A run that
|
|
767
|
+
# is skipped (read-only, --no-verify, or no configured command) is recorded
|
|
768
|
+
# as SKIPPED, never as a pass.
|
|
769
|
+
skip_reason = _skip_reason(read_only, skip_verify, taskgroup.verify)
|
|
770
|
+
outcome = None
|
|
771
|
+
if skip_reason is not None:
|
|
772
|
+
verify_status = VERIFY_SKIPPED
|
|
773
|
+
verify_cmd_field = VERIFY_SKIPPED
|
|
774
|
+
verify_exit_field = None
|
|
775
|
+
ran = False
|
|
776
|
+
note = skip_reason
|
|
777
|
+
else:
|
|
778
|
+
# ⑦a Flip the card to `verifying` for the whole acceptance window, before
|
|
779
|
+
# the command starts. Acceptance can run for seconds to minutes and the
|
|
780
|
+
# phase has to be visible while it does; writing it before
|
|
781
|
+
# `run_acceptance` also means a spawn failure or a hung check is still
|
|
782
|
+
# recorded as `verifying` rather than a stale `running`.
|
|
783
|
+
storage.update_task(conn, task_id, status=STATUS_VERIFYING)
|
|
784
|
+
storage.append_event(
|
|
785
|
+
conn, task_id, "verifying", {"command": taskgroup.verify}
|
|
786
|
+
)
|
|
787
|
+
outcome = verify.run_acceptance(
|
|
788
|
+
run_dir, taskgroup.verify, timeout=timeout
|
|
789
|
+
)
|
|
790
|
+
ran = outcome.ran
|
|
791
|
+
if not outcome.ran:
|
|
792
|
+
verify_status = VERIFY_SKIPPED
|
|
793
|
+
verify_cmd_field = VERIFY_SKIPPED
|
|
794
|
+
verify_exit_field = None
|
|
795
|
+
else:
|
|
796
|
+
verify_status = "PASSED" if outcome.passed else "FAILED"
|
|
797
|
+
verify_cmd_field = taskgroup.verify
|
|
798
|
+
verify_exit_field = outcome.exit_code
|
|
799
|
+
note = outcome.note
|
|
800
|
+
|
|
801
|
+
storage.append_event(
|
|
802
|
+
conn,
|
|
803
|
+
task_id,
|
|
804
|
+
"verify",
|
|
805
|
+
{
|
|
806
|
+
"status": verify_status,
|
|
807
|
+
"ran": ran,
|
|
808
|
+
"exit_code": verify_exit_field,
|
|
809
|
+
"note": note,
|
|
810
|
+
"output_tail": outcome.output_tail if outcome is not None else "",
|
|
811
|
+
},
|
|
812
|
+
)
|
|
813
|
+
|
|
814
|
+
# ⑧ terminal row + event. Two *facts* may be in play — a protected path
|
|
815
|
+
# moved, and/or acceptance failed — and the ledger must carry both, so the
|
|
816
|
+
# rule is: a violation outranks a red acceptance (a human reviews the
|
|
817
|
+
# boundary breach), a red acceptance is a plain `failed`, and a clean green
|
|
818
|
+
# run is `done`. Every branch has already written the `verify` event above,
|
|
819
|
+
# so whichever terminal we pick here, the acceptance result is not lost.
|
|
820
|
+
common = dict(
|
|
821
|
+
exit_code=exit_code,
|
|
822
|
+
verify_cmd=verify_cmd_field,
|
|
823
|
+
verify_exit=verify_exit_field,
|
|
824
|
+
files_changed=files_changed,
|
|
825
|
+
workdir=run_dir,
|
|
826
|
+
result_path=result_path,
|
|
827
|
+
)
|
|
828
|
+
|
|
829
|
+
if violations:
|
|
830
|
+
# A protected path was touched: `blocked` (= human review), NOT a
|
|
831
|
+
# failure of the work. The exit code stays 71 (VerifyError, below) even
|
|
832
|
+
# though the terminal state is blocked — "what the task is recorded as"
|
|
833
|
+
# and "how this command ended" are deliberately different values.
|
|
834
|
+
# Layer 3's `accept <id>` is what clears a blocked card; this pipeline
|
|
835
|
+
# never auto-promotes it to `done`, even when acceptance passed.
|
|
836
|
+
_finish_task(conn, task_id, status=STATUS_BLOCKED, **common)
|
|
837
|
+
storage.append_event(
|
|
838
|
+
conn,
|
|
839
|
+
task_id,
|
|
840
|
+
"blocked",
|
|
841
|
+
{
|
|
842
|
+
"stage": "forbidden",
|
|
843
|
+
"violations": violations,
|
|
844
|
+
"paths": violation_paths,
|
|
845
|
+
# Both facts survive in the ledger: the breach and the verdict.
|
|
846
|
+
"verify": verify_status,
|
|
847
|
+
"verify_exit": verify_exit_field,
|
|
848
|
+
},
|
|
849
|
+
)
|
|
850
|
+
raise VerifyError(
|
|
851
|
+
f"forbidden path(s) changed: {', '.join(violation_paths)}",
|
|
852
|
+
hint="the task touched a path the registry protects",
|
|
853
|
+
)
|
|
854
|
+
|
|
855
|
+
if verify_status == "FAILED":
|
|
856
|
+
_finish_task(conn, task_id, status=STATUS_FAILED, **common)
|
|
857
|
+
storage.append_event(
|
|
858
|
+
conn,
|
|
859
|
+
task_id,
|
|
860
|
+
"failed",
|
|
861
|
+
{
|
|
862
|
+
"stage": "verify",
|
|
863
|
+
"verify_exit": verify_exit_field,
|
|
864
|
+
"note": note,
|
|
865
|
+
},
|
|
866
|
+
)
|
|
867
|
+
raise VerifyError(
|
|
868
|
+
f"acceptance command failed (exit {verify_exit_field})",
|
|
869
|
+
hint=f"output: {outcome.output_tail}" if outcome and outcome.output_tail else None,
|
|
870
|
+
)
|
|
871
|
+
|
|
872
|
+
_finish_task(conn, task_id, status=STATUS_DONE, **common)
|
|
873
|
+
storage.append_event(
|
|
874
|
+
conn,
|
|
875
|
+
task_id,
|
|
876
|
+
"done",
|
|
877
|
+
{
|
|
878
|
+
"verify": verify_status,
|
|
879
|
+
"files_changed": files_changed,
|
|
880
|
+
"summary": result.summary,
|
|
881
|
+
},
|
|
882
|
+
)
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
def _skip_reason(read_only: bool, skip_verify: bool, verify_cmd) -> Optional[str]:
|
|
886
|
+
"""Why acceptance is not being run, or None when it should run."""
|
|
887
|
+
if read_only:
|
|
888
|
+
return "read-only run: acceptance skipped"
|
|
889
|
+
if skip_verify:
|
|
890
|
+
return "--no-verify requested: acceptance skipped"
|
|
891
|
+
if not verify_cmd:
|
|
892
|
+
return "no acceptance command configured"
|
|
893
|
+
return None
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
def _task_cancelled(conn, task_id: str) -> bool:
|
|
897
|
+
"""True when `cancel` has already recorded this task as terminal."""
|
|
898
|
+
row = conn.execute("SELECT status FROM tasks WHERE id = ?", (task_id,)).fetchone()
|
|
899
|
+
return row is not None and row[0] == STATUS_CANCELLED
|
|
900
|
+
|
|
901
|
+
|
|
902
|
+
def _finish_task(
|
|
903
|
+
conn,
|
|
904
|
+
task_id: str,
|
|
905
|
+
*,
|
|
906
|
+
status: str,
|
|
907
|
+
exit_code=None,
|
|
908
|
+
verify_cmd=None,
|
|
909
|
+
verify_exit=None,
|
|
910
|
+
files_changed=None,
|
|
911
|
+
workdir=None,
|
|
912
|
+
result_path=None,
|
|
913
|
+
) -> None:
|
|
914
|
+
"""Patch the terminal columns of a task. Only non-None fields are written.
|
|
915
|
+
|
|
916
|
+
A task that `cancel` already marked terminal is left alone: the in-flight
|
|
917
|
+
dispatcher must never overwrite `cancelled` with a later done/failed state.
|
|
918
|
+
"""
|
|
919
|
+
if _task_cancelled(conn, task_id):
|
|
920
|
+
return
|
|
921
|
+
fields = {"status": status, "finished_at": storage.now_iso()}
|
|
922
|
+
if exit_code is not None:
|
|
923
|
+
fields["exit_code"] = exit_code
|
|
924
|
+
if verify_cmd is not None:
|
|
925
|
+
fields["verify_cmd"] = verify_cmd
|
|
926
|
+
if verify_exit is not None:
|
|
927
|
+
fields["verify_exit"] = verify_exit
|
|
928
|
+
if files_changed is not None:
|
|
929
|
+
fields["files_changed"] = files_changed
|
|
930
|
+
if workdir is not None:
|
|
931
|
+
fields["workdir"] = workdir
|
|
932
|
+
if result_path is not None:
|
|
933
|
+
fields["result_path"] = result_path
|
|
934
|
+
storage.update_task(conn, task_id, **fields)
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
# ---------------------------------------------------------------------------
|
|
938
|
+
# Adapter execution
|
|
939
|
+
# ---------------------------------------------------------------------------
|
|
940
|
+
|
|
941
|
+
|
|
942
|
+
def _terminate_tree(proc) -> None:
|
|
943
|
+
"""SIGTERM (then SIGKILL) only the process group this function started.
|
|
944
|
+
|
|
945
|
+
`_run_adapter` spawns with `start_new_session=True`, so the child leads its
|
|
946
|
+
own process group; signalling that group reaches the agent and its children
|
|
947
|
+
but nothing else. If the group id ever equaled ours we fall back to
|
|
948
|
+
signalling the child alone, so we can never take down our own process.
|
|
949
|
+
"""
|
|
950
|
+
try:
|
|
951
|
+
pgid = os.getpgid(proc.pid)
|
|
952
|
+
except (ProcessLookupError, OSError):
|
|
953
|
+
return
|
|
954
|
+
|
|
955
|
+
own_group = os.getpgid(0)
|
|
956
|
+
if pgid == own_group:
|
|
957
|
+
# Should not happen with start_new_session; never risk our own group.
|
|
958
|
+
try:
|
|
959
|
+
proc.terminate()
|
|
960
|
+
except (ProcessLookupError, OSError):
|
|
961
|
+
pass
|
|
962
|
+
try:
|
|
963
|
+
proc.wait(timeout=5)
|
|
964
|
+
except subprocess.TimeoutExpired:
|
|
965
|
+
try:
|
|
966
|
+
proc.kill()
|
|
967
|
+
except (ProcessLookupError, OSError):
|
|
968
|
+
pass
|
|
969
|
+
return
|
|
970
|
+
|
|
971
|
+
try:
|
|
972
|
+
os.killpg(pgid, signal.SIGTERM)
|
|
973
|
+
except (ProcessLookupError, OSError):
|
|
974
|
+
return
|
|
975
|
+
try:
|
|
976
|
+
proc.wait(timeout=5)
|
|
977
|
+
return
|
|
978
|
+
except subprocess.TimeoutExpired:
|
|
979
|
+
pass
|
|
980
|
+
try:
|
|
981
|
+
os.killpg(pgid, signal.SIGKILL)
|
|
982
|
+
except (ProcessLookupError, OSError):
|
|
983
|
+
pass
|
|
984
|
+
try:
|
|
985
|
+
proc.wait(timeout=5)
|
|
986
|
+
except subprocess.TimeoutExpired:
|
|
987
|
+
pass
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
def _wait_group_gone(pgid: int, timeout: float) -> bool:
|
|
991
|
+
"""Poll a process group until it is empty, or `timeout` elapses.
|
|
992
|
+
|
|
993
|
+
`os.killpg(pgid, 0)` is a pure liveness probe: it raises `ProcessLookupError`
|
|
994
|
+
once no process is left in the group. We cannot `wait()` for a group we may
|
|
995
|
+
not be the parent of, so polling is how the recorded-group path honours the
|
|
996
|
+
same SIGTERM -> grace -> SIGKILL timing as `_terminate_tree`.
|
|
997
|
+
"""
|
|
998
|
+
deadline = time.monotonic() + timeout
|
|
999
|
+
while True:
|
|
1000
|
+
try:
|
|
1001
|
+
os.killpg(pgid, 0)
|
|
1002
|
+
except ProcessLookupError:
|
|
1003
|
+
return True
|
|
1004
|
+
except OSError:
|
|
1005
|
+
# EPERM etc.: the group still exists as far as we can tell.
|
|
1006
|
+
pass
|
|
1007
|
+
if time.monotonic() >= deadline:
|
|
1008
|
+
return False
|
|
1009
|
+
time.sleep(0.05)
|
|
1010
|
+
|
|
1011
|
+
|
|
1012
|
+
def _terminate_recorded_group(pgid) -> None:
|
|
1013
|
+
"""SIGTERM -> 5s -> SIGKILL exactly the group recorded for a task.
|
|
1014
|
+
|
|
1015
|
+
This is the after-a-restart counterpart of `_terminate_tree`: it takes the
|
|
1016
|
+
pgid persisted on the task row (the adapter's own new session — see
|
|
1017
|
+
`_execute._record_pgid`) instead of a live `Popen` handle. The discipline is
|
|
1018
|
+
identical — signal only that one group, never `pkill`, never a name/brief
|
|
1019
|
+
match — and the `pgid == our own group` guard is kept so this can never take
|
|
1020
|
+
down the caller. Where `_terminate_tree` falls back to signalling the single
|
|
1021
|
+
child it owns, this path has no child handle to fall back to, so it signals
|
|
1022
|
+
nothing rather than risk our own process group.
|
|
1023
|
+
"""
|
|
1024
|
+
if pgid is None:
|
|
1025
|
+
return
|
|
1026
|
+
try:
|
|
1027
|
+
own_group = os.getpgid(0)
|
|
1028
|
+
except OSError:
|
|
1029
|
+
own_group = None
|
|
1030
|
+
if own_group is not None and pgid == own_group:
|
|
1031
|
+
return
|
|
1032
|
+
try:
|
|
1033
|
+
os.killpg(pgid, signal.SIGTERM)
|
|
1034
|
+
except (ProcessLookupError, OSError):
|
|
1035
|
+
return
|
|
1036
|
+
if _wait_group_gone(pgid, TERMINATE_GRACE_SECONDS):
|
|
1037
|
+
return
|
|
1038
|
+
try:
|
|
1039
|
+
os.killpg(pgid, signal.SIGKILL)
|
|
1040
|
+
except (ProcessLookupError, OSError):
|
|
1041
|
+
return
|
|
1042
|
+
_wait_group_gone(pgid, TERMINATE_GRACE_SECONDS)
|
|
1043
|
+
|
|
1044
|
+
|
|
1045
|
+
#: Loopback hosts that must never be sent to a forward proxy: a local service
|
|
1046
|
+
#: (e.g. a CC Switch endpoint on http://127.0.0.1:15721/v1) is not reachable
|
|
1047
|
+
#: through a proxy.
|
|
1048
|
+
_LOCALHOST_NO_PROXY = ("127.0.0.1", "localhost", "::1")
|
|
1049
|
+
|
|
1050
|
+
#: Proxy variables whose presence means the child could otherwise be routed.
|
|
1051
|
+
_PROXY_ENV_NAMES = (
|
|
1052
|
+
"http_proxy", "https_proxy", "all_proxy",
|
|
1053
|
+
"HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY",
|
|
1054
|
+
)
|
|
1055
|
+
|
|
1056
|
+
|
|
1057
|
+
def _ensure_localhost_no_proxy(env: dict) -> dict:
|
|
1058
|
+
"""Make loopback hosts bypass any proxy, without clobbering the user's.
|
|
1059
|
+
|
|
1060
|
+
Defence: when the parent routes through a proxy (any of http_proxy /
|
|
1061
|
+
https_proxy / all_proxy), a request to a local service must still go direct,
|
|
1062
|
+
or it is shipped off to the proxy and fails. `no_proxy` and `NO_PROXY` are
|
|
1063
|
+
extended with 127.0.0.1, localhost and ::1; existing entries are kept and
|
|
1064
|
+
merged — never overwritten — and no other proxy semantics are changed.
|
|
1065
|
+
"""
|
|
1066
|
+
if not any(env.get(name) for name in _PROXY_ENV_NAMES):
|
|
1067
|
+
return env
|
|
1068
|
+
for name in ("no_proxy", "NO_PROXY"):
|
|
1069
|
+
entries = [
|
|
1070
|
+
item.strip() for item in (env.get(name) or "").split(",") if item.strip()
|
|
1071
|
+
]
|
|
1072
|
+
seen = {item.lower() for item in entries}
|
|
1073
|
+
for host in _LOCALHOST_NO_PROXY:
|
|
1074
|
+
if host.lower() not in seen:
|
|
1075
|
+
entries.append(host)
|
|
1076
|
+
seen.add(host.lower())
|
|
1077
|
+
env[name] = ",".join(entries)
|
|
1078
|
+
return env
|
|
1079
|
+
|
|
1080
|
+
|
|
1081
|
+
#: Sentinel meaning "no work-tree baseline has been taken yet this wait".
|
|
1082
|
+
_UNSET = object()
|
|
1083
|
+
|
|
1084
|
+
|
|
1085
|
+
def _log_bytes(log_path: str) -> int:
|
|
1086
|
+
"""The current size of the run's log, or 0 if it is not there yet.
|
|
1087
|
+
|
|
1088
|
+
The cheapest possible liveness probe: one ``os.stat``. When it grows the
|
|
1089
|
+
card is doing something visible and the watchdog never has to look at git.
|
|
1090
|
+
"""
|
|
1091
|
+
try:
|
|
1092
|
+
return os.stat(log_path).st_size
|
|
1093
|
+
except OSError:
|
|
1094
|
+
return 0
|
|
1095
|
+
|
|
1096
|
+
|
|
1097
|
+
def _tree_digest(workdir: str):
|
|
1098
|
+
"""A cheap "is the working tree still moving?" summary, or ``None``.
|
|
1099
|
+
|
|
1100
|
+
Reuses the gate's own probes (card 83 must not invent a second one):
|
|
1101
|
+
``verify.changed_files_snapshot`` gives the porcelain path -> status map,
|
|
1102
|
+
and ``verify._fingerprint`` gives each path a ``size:mtime_ns`` string. The
|
|
1103
|
+
digest is the component-wise MAX of size and mtime over those paths.
|
|
1104
|
+
|
|
1105
|
+
A bare porcelain diff cannot see a rewrite of an already-dirty file — the
|
|
1106
|
+
two status columns are identical — but that rewrite always pushes the
|
|
1107
|
+
file's ``mtime_ns`` above every earlier changed path, so the max still
|
|
1108
|
+
moves. ``None`` means "this directory is not a git work tree", and the
|
|
1109
|
+
caller then watches the log alone instead of erroring.
|
|
1110
|
+
"""
|
|
1111
|
+
snapshot = verify.changed_files_snapshot(workdir)
|
|
1112
|
+
if snapshot is None:
|
|
1113
|
+
return None
|
|
1114
|
+
max_size = None
|
|
1115
|
+
max_mtime = None
|
|
1116
|
+
for rel in snapshot:
|
|
1117
|
+
fingerprint = verify._fingerprint(os.path.join(workdir, rel))
|
|
1118
|
+
try:
|
|
1119
|
+
size_text, mtime_text = fingerprint.split(":", 1)
|
|
1120
|
+
size, mtime = int(size_text), int(mtime_text)
|
|
1121
|
+
except (ValueError, AttributeError):
|
|
1122
|
+
# `_fingerprint` returns "gone" for a vanished path; ignore it.
|
|
1123
|
+
continue
|
|
1124
|
+
if max_size is None or size > max_size:
|
|
1125
|
+
max_size = size
|
|
1126
|
+
if max_mtime is None or mtime > max_mtime:
|
|
1127
|
+
max_mtime = mtime
|
|
1128
|
+
return (max_size, max_mtime)
|
|
1129
|
+
|
|
1130
|
+
|
|
1131
|
+
def _idle_watch(proc, *, log_path, workdir, wall_clock, idle_timeout):
|
|
1132
|
+
"""Wait for `proc`, stopping early when the card goes quiet.
|
|
1133
|
+
|
|
1134
|
+
"Quiet" means BOTH the log file and the working tree have shown no change
|
|
1135
|
+
for `idle_timeout` seconds. Returns ``None`` when the child exits on its
|
|
1136
|
+
own, otherwise the reason the wait gave up: ``"idle"`` or ``"wall-clock"``.
|
|
1137
|
+
|
|
1138
|
+
It never terminates the process — the caller does — so the idle and the
|
|
1139
|
+
hard wall-clock endings converge on ONE code path (a single ``timeout``
|
|
1140
|
+
terminal state). Cost is deliberately asymmetric: the cheap ``os.stat`` of
|
|
1141
|
+
the log runs every tick, but the (potentially costly on a huge repo) ``git
|
|
1142
|
+
status`` digest runs ONLY on a tick where the log was silent, and a
|
|
1143
|
+
non-git ``workdir`` is noticed once and then never probed again. The tick
|
|
1144
|
+
length is ``max(1, min(30, idle_timeout // 2))``.
|
|
1145
|
+
"""
|
|
1146
|
+
poll = max(1, min(30, idle_timeout // 2))
|
|
1147
|
+
start = time.monotonic()
|
|
1148
|
+
last_progress = start
|
|
1149
|
+
last_log = _log_bytes(log_path)
|
|
1150
|
+
tree_live = True
|
|
1151
|
+
last_tree = _UNSET
|
|
1152
|
+
|
|
1153
|
+
while True:
|
|
1154
|
+
remaining = wall_clock - (time.monotonic() - start)
|
|
1155
|
+
if remaining <= 0:
|
|
1156
|
+
return "wall-clock"
|
|
1157
|
+
try:
|
|
1158
|
+
proc.wait(timeout=min(poll, remaining))
|
|
1159
|
+
return None # the child exited on its own
|
|
1160
|
+
except subprocess.TimeoutExpired:
|
|
1161
|
+
pass
|
|
1162
|
+
|
|
1163
|
+
now = time.monotonic()
|
|
1164
|
+
# Cheap path first: the log grew, so the card is alive — never touch git.
|
|
1165
|
+
size = _log_bytes(log_path)
|
|
1166
|
+
if size != last_log:
|
|
1167
|
+
last_log = size
|
|
1168
|
+
last_progress = now
|
|
1169
|
+
continue
|
|
1170
|
+
|
|
1171
|
+
# The log is silent. Only now pay for the tree probe.
|
|
1172
|
+
if tree_live:
|
|
1173
|
+
tree = _tree_digest(workdir)
|
|
1174
|
+
if tree is None:
|
|
1175
|
+
# Not a git work tree: degrade to the log as the only signal.
|
|
1176
|
+
tree_live = False
|
|
1177
|
+
elif last_tree is _UNSET:
|
|
1178
|
+
last_tree = tree # first silent tick just sets the baseline
|
|
1179
|
+
elif tree != last_tree:
|
|
1180
|
+
last_tree = tree
|
|
1181
|
+
last_progress = now
|
|
1182
|
+
continue
|
|
1183
|
+
|
|
1184
|
+
idle_deadline = last_progress + idle_timeout
|
|
1185
|
+
wall_deadline = start + wall_clock
|
|
1186
|
+
if now >= idle_deadline or now >= wall_deadline:
|
|
1187
|
+
# Whichever budget ran out first wins; a tie is the wall clock's.
|
|
1188
|
+
return "idle" if idle_deadline < wall_deadline else "wall-clock"
|
|
1189
|
+
|
|
1190
|
+
|
|
1191
|
+
def _run_adapter(adapter_obj, *, brief: str, workdir: str, log_path: str,
|
|
1192
|
+
schema_path: Optional[str] = None, read_only: bool = False,
|
|
1193
|
+
idle_timeout: int = 0, on_spawn=None):
|
|
1194
|
+
"""Spawn the agent CLI, tee output to `log_path`, return (exit_code, out, err).
|
|
1195
|
+
|
|
1196
|
+
* argv is a list and the shell is never used.
|
|
1197
|
+
* stdout/stderr are streamed to ``log_path`` line by line as the child runs,
|
|
1198
|
+
so the board can watch progress instead of waiting for the process to exit.
|
|
1199
|
+
* ``stdin`` is ``/dev/null`` so an agent that reads input can never hang.
|
|
1200
|
+
* `adapter_obj.timeout` is enforced; on expiry the process group this call
|
|
1201
|
+
created is terminated (SIGTERM then SIGKILL) and a `VerifyError` is raised
|
|
1202
|
+
(exit code 71 semantics).
|
|
1203
|
+
* when ``idle_timeout`` > 0 a no-progress watchdog waits alongside the wall
|
|
1204
|
+
clock (card 83): the card is stopped only when BOTH its log and its tree
|
|
1205
|
+
have been quiet for that long. ``idle_timeout == 0`` (the shipped default)
|
|
1206
|
+
keeps the plain ``proc.wait(timeout=...)`` with no polling at all.
|
|
1207
|
+
* a wall-clock or idle stop raises the SAME ``VerifyError`` — a single
|
|
1208
|
+
``timeout`` terminal state — but the exception carries ``reason`` /
|
|
1209
|
+
``idle_timeout`` / ``timeout`` so the caller can record why it fired.
|
|
1210
|
+
* ``on_spawn(proc)`` (optional) runs immediately after the child exists and
|
|
1211
|
+
before we wait on it — the hook the dispatcher uses to persist the child's
|
|
1212
|
+
process-group id for a later ``cancel``.
|
|
1213
|
+
"""
|
|
1214
|
+
try:
|
|
1215
|
+
argv = adapter_obj.build_command(
|
|
1216
|
+
brief=brief, workdir=workdir, schema_path=schema_path, read_only=read_only
|
|
1217
|
+
)
|
|
1218
|
+
except (ValueError, TypeError) as exc:
|
|
1219
|
+
raise AdapterError(
|
|
1220
|
+
f"adapter '{adapter_obj.name}': could not build a command: {exc}"
|
|
1221
|
+
)
|
|
1222
|
+
if not argv:
|
|
1223
|
+
raise AdapterError(f"adapter '{adapter_obj.name}': built an empty command")
|
|
1224
|
+
|
|
1225
|
+
env = {**os.environ, **(adapter_obj.env or {})}
|
|
1226
|
+
env = _ensure_localhost_no_proxy(env)
|
|
1227
|
+
timeout = adapter_obj.timeout
|
|
1228
|
+
|
|
1229
|
+
os.makedirs(os.path.dirname(os.path.abspath(log_path)), exist_ok=True)
|
|
1230
|
+
out_chunks = []
|
|
1231
|
+
err_chunks = []
|
|
1232
|
+
write_lock = threading.Lock()
|
|
1233
|
+
|
|
1234
|
+
sink = open(log_path, "w", encoding="utf-8")
|
|
1235
|
+
try:
|
|
1236
|
+
try:
|
|
1237
|
+
proc = subprocess.Popen(
|
|
1238
|
+
argv,
|
|
1239
|
+
cwd=workdir,
|
|
1240
|
+
stdin=subprocess.DEVNULL,
|
|
1241
|
+
stdout=subprocess.PIPE,
|
|
1242
|
+
stderr=subprocess.PIPE,
|
|
1243
|
+
text=True,
|
|
1244
|
+
bufsize=1,
|
|
1245
|
+
env=env,
|
|
1246
|
+
start_new_session=True,
|
|
1247
|
+
)
|
|
1248
|
+
except OSError as exc:
|
|
1249
|
+
raise AdapterError(
|
|
1250
|
+
f"adapter '{adapter_obj.name}': could not start '{argv[0]}': {exc}"
|
|
1251
|
+
)
|
|
1252
|
+
|
|
1253
|
+
if on_spawn is not None:
|
|
1254
|
+
# Record the identity before we block on the child, so a cancel from
|
|
1255
|
+
# another process can reach the tree even mid-run.
|
|
1256
|
+
on_spawn(proc)
|
|
1257
|
+
|
|
1258
|
+
def pump(stream, chunks):
|
|
1259
|
+
try:
|
|
1260
|
+
for line in stream:
|
|
1261
|
+
with write_lock:
|
|
1262
|
+
sink.write(line)
|
|
1263
|
+
sink.flush()
|
|
1264
|
+
chunks.append(line)
|
|
1265
|
+
finally:
|
|
1266
|
+
try:
|
|
1267
|
+
stream.close()
|
|
1268
|
+
except Exception:
|
|
1269
|
+
pass
|
|
1270
|
+
|
|
1271
|
+
readers = [
|
|
1272
|
+
threading.Thread(target=pump, args=(proc.stdout, out_chunks), daemon=True),
|
|
1273
|
+
threading.Thread(target=pump, args=(proc.stderr, err_chunks), daemon=True),
|
|
1274
|
+
]
|
|
1275
|
+
for reader in readers:
|
|
1276
|
+
reader.start()
|
|
1277
|
+
|
|
1278
|
+
timed_out = False
|
|
1279
|
+
timeout_reason = None
|
|
1280
|
+
if idle_timeout and idle_timeout > 0:
|
|
1281
|
+
# The watchdog and the wall clock race; the FIRST to fire wins.
|
|
1282
|
+
# `_idle_watch` returns the reason (or None when the child left on
|
|
1283
|
+
# its own). Either termination below is the SAME path the plain
|
|
1284
|
+
# wall-clock wait uses, so both end in one `timeout` state.
|
|
1285
|
+
timeout_reason = _idle_watch(
|
|
1286
|
+
proc,
|
|
1287
|
+
log_path=log_path,
|
|
1288
|
+
workdir=workdir,
|
|
1289
|
+
wall_clock=timeout,
|
|
1290
|
+
idle_timeout=idle_timeout,
|
|
1291
|
+
)
|
|
1292
|
+
timed_out = timeout_reason is not None
|
|
1293
|
+
if timed_out:
|
|
1294
|
+
_terminate_tree(proc)
|
|
1295
|
+
exit_code = proc.wait()
|
|
1296
|
+
else:
|
|
1297
|
+
# Off (the shipped default): wait on the wall clock exactly as
|
|
1298
|
+
# before — no polling, no extra probes.
|
|
1299
|
+
try:
|
|
1300
|
+
exit_code = proc.wait(timeout=timeout)
|
|
1301
|
+
except subprocess.TimeoutExpired:
|
|
1302
|
+
timed_out = True
|
|
1303
|
+
timeout_reason = "wall-clock"
|
|
1304
|
+
_terminate_tree(proc)
|
|
1305
|
+
exit_code = proc.wait()
|
|
1306
|
+
for reader in readers:
|
|
1307
|
+
reader.join()
|
|
1308
|
+
finally:
|
|
1309
|
+
sink.close()
|
|
1310
|
+
|
|
1311
|
+
out = "".join(out_chunks)
|
|
1312
|
+
err = "".join(err_chunks)
|
|
1313
|
+
if timed_out:
|
|
1314
|
+
if timeout_reason == "idle":
|
|
1315
|
+
message = (
|
|
1316
|
+
f"adapter '{adapter_obj.name}': idle timeout after {idle_timeout}s "
|
|
1317
|
+
"(log and work tree both quiet; process tree terminated)"
|
|
1318
|
+
)
|
|
1319
|
+
else:
|
|
1320
|
+
message = (
|
|
1321
|
+
f"adapter '{adapter_obj.name}': timeout after {timeout}s "
|
|
1322
|
+
"(process tree terminated)"
|
|
1323
|
+
)
|
|
1324
|
+
exc = VerifyError(message, hint=f"see {log_path}")
|
|
1325
|
+
exc.reason = timeout_reason
|
|
1326
|
+
exc.idle_timeout = idle_timeout
|
|
1327
|
+
exc.timeout = timeout
|
|
1328
|
+
raise exc
|
|
1329
|
+
return exit_code, out, err
|
|
1330
|
+
|
|
1331
|
+
|
|
1332
|
+
# ---------------------------------------------------------------------------
|
|
1333
|
+
# git worktrees
|
|
1334
|
+
# ---------------------------------------------------------------------------
|
|
1335
|
+
|
|
1336
|
+
|
|
1337
|
+
def _is_git_repo(path: str) -> bool:
|
|
1338
|
+
"""Read-only probe: is `path` inside a git work tree?"""
|
|
1339
|
+
try:
|
|
1340
|
+
proc = subprocess.run(
|
|
1341
|
+
["git", "rev-parse", "--is-inside-work-tree"],
|
|
1342
|
+
cwd=path,
|
|
1343
|
+
stdout=subprocess.PIPE,
|
|
1344
|
+
stderr=subprocess.DEVNULL,
|
|
1345
|
+
text=True,
|
|
1346
|
+
)
|
|
1347
|
+
except (OSError, subprocess.SubprocessError):
|
|
1348
|
+
return False
|
|
1349
|
+
return proc.returncode == 0 and proc.stdout.strip() == "true"
|
|
1350
|
+
|
|
1351
|
+
|
|
1352
|
+
def _worktree_target_path(repo_path: str, task_id: str) -> str:
|
|
1353
|
+
"""The sibling directory a run's checkout lives in.
|
|
1354
|
+
|
|
1355
|
+
The convention is ``<repo parent>/<repo name>-wt-<task id>`` -- BESIDE the
|
|
1356
|
+
repo, never inside it, so a kept checkout can never pollute the project's
|
|
1357
|
+
own ``git status``. It is never reused: a task id is unique, and if the exact
|
|
1358
|
+
name is somehow already taken (e.g. the same id run twice) a numeric suffix
|
|
1359
|
+
keeps the two runs from colliding.
|
|
1360
|
+
"""
|
|
1361
|
+
repo_abs = os.path.abspath(repo_path)
|
|
1362
|
+
base = os.path.join(
|
|
1363
|
+
os.path.dirname(repo_abs), f"{os.path.basename(repo_abs)}-wt-{task_id}"
|
|
1364
|
+
)
|
|
1365
|
+
candidate = base
|
|
1366
|
+
n = 2
|
|
1367
|
+
while os.path.lexists(candidate):
|
|
1368
|
+
candidate = f"{base}-{n}"
|
|
1369
|
+
n += 1
|
|
1370
|
+
return candidate
|
|
1371
|
+
|
|
1372
|
+
|
|
1373
|
+
def _create_worktree(repo_path: str, task_id: str) -> str:
|
|
1374
|
+
"""`git worktree add --detach` a fresh checkout; return its path.
|
|
1375
|
+
|
|
1376
|
+
The checkout is created directly at the finally-visible path
|
|
1377
|
+
(:func:`_worktree_target_path`), so the ``worktree_created`` event, the kept
|
|
1378
|
+
location and the printed line all name the same directory.
|
|
1379
|
+
"""
|
|
1380
|
+
path = _worktree_target_path(repo_path, task_id)
|
|
1381
|
+
proc = subprocess.run(
|
|
1382
|
+
["git", "worktree", "add", "--detach", path],
|
|
1383
|
+
cwd=repo_path,
|
|
1384
|
+
stdout=subprocess.PIPE,
|
|
1385
|
+
stderr=subprocess.STDOUT,
|
|
1386
|
+
text=True,
|
|
1387
|
+
)
|
|
1388
|
+
if proc.returncode != 0:
|
|
1389
|
+
shutil.rmtree(path, ignore_errors=True)
|
|
1390
|
+
raise UsageError(
|
|
1391
|
+
"could not create a git worktree for this project",
|
|
1392
|
+
hint=(proc.stdout or "").strip() or "git worktree add failed",
|
|
1393
|
+
)
|
|
1394
|
+
return path
|
|
1395
|
+
|
|
1396
|
+
|
|
1397
|
+
def _worktree_change_count(path: str) -> int:
|
|
1398
|
+
"""How many files ``git status --porcelain`` reports in a worktree.
|
|
1399
|
+
|
|
1400
|
+
Read-only, and reuses the very probe `verify` uses for a task's
|
|
1401
|
+
``files_changed`` field, so the count the message prints matches the audit.
|
|
1402
|
+
"""
|
|
1403
|
+
return verify.detect_changes(path) or 0
|
|
1404
|
+
|
|
1405
|
+
|
|
1406
|
+
def _remove_worktree(repo_path: str, path: str) -> Optional[str]:
|
|
1407
|
+
"""Best-effort `git worktree remove`; returns an error string on failure."""
|
|
1408
|
+
error = None
|
|
1409
|
+
try:
|
|
1410
|
+
proc = subprocess.run(
|
|
1411
|
+
["git", "worktree", "remove", "--force", path],
|
|
1412
|
+
cwd=repo_path,
|
|
1413
|
+
stdout=subprocess.PIPE,
|
|
1414
|
+
stderr=subprocess.STDOUT,
|
|
1415
|
+
text=True,
|
|
1416
|
+
)
|
|
1417
|
+
if proc.returncode != 0:
|
|
1418
|
+
error = (proc.stdout or "").strip() or (
|
|
1419
|
+
f"git worktree remove exited {proc.returncode}"
|
|
1420
|
+
)
|
|
1421
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
1422
|
+
error = str(exc)
|
|
1423
|
+
|
|
1424
|
+
# The checkout dir always goes, even when git refused to unregister it.
|
|
1425
|
+
shutil.rmtree(path, ignore_errors=True)
|
|
1426
|
+
if error is not None:
|
|
1427
|
+
try:
|
|
1428
|
+
subprocess.run(
|
|
1429
|
+
["git", "worktree", "prune"],
|
|
1430
|
+
cwd=repo_path,
|
|
1431
|
+
stdout=subprocess.DEVNULL,
|
|
1432
|
+
stderr=subprocess.DEVNULL,
|
|
1433
|
+
)
|
|
1434
|
+
except (OSError, subprocess.SubprocessError):
|
|
1435
|
+
pass
|
|
1436
|
+
return error
|
|
1437
|
+
|
|
1438
|
+
|
|
1439
|
+
# ---------------------------------------------------------------------------
|
|
1440
|
+
# Taskgroup workspaces
|
|
1441
|
+
#
|
|
1442
|
+
# A lane may declare `workspace = "worktree"`: instead of running in the main
|
|
1443
|
+
# checkout, taskproof lazily creates ONE long-lived git worktree beside the
|
|
1444
|
+
# repo and reuses it for every card on that lane. It is never removed
|
|
1445
|
+
# automatically -- `taskproof workspace-rm <lane>` is the only way out. This is
|
|
1446
|
+
# deliberately a per-lane opt-in; the default (`workspace = "none"`) keeps the
|
|
1447
|
+
# historical in-place behaviour.
|
|
1448
|
+
# ---------------------------------------------------------------------------
|
|
1449
|
+
|
|
1450
|
+
#: Build file -> dependency directory to symlink from the main checkout into a
|
|
1451
|
+
#: lane workspace. Mirrors `registry.infer_verify`: a short, opinionated list,
|
|
1452
|
+
#: not a general-purpose detector. Python virtual environments are deliberately
|
|
1453
|
+
#: NOT here: `pyvenv.cfg` and the console-script shebangs under `bin/` record
|
|
1454
|
+
#: absolute paths, so a symlinked venv would resolve back to the main tree (or
|
|
1455
|
+
#: break outright) instead of isolating the run.
|
|
1456
|
+
_DEPENDENCY_LINKS = (
|
|
1457
|
+
("package.json", "node_modules"),
|
|
1458
|
+
("Cargo.toml", "target"),
|
|
1459
|
+
)
|
|
1460
|
+
|
|
1461
|
+
|
|
1462
|
+
def _git_toplevel(path: str) -> Optional[str]:
|
|
1463
|
+
"""Absolute git repository root containing ``path``, or None.
|
|
1464
|
+
|
|
1465
|
+
Read-only probe. A ``path`` that is not inside a work tree (or where git is
|
|
1466
|
+
unavailable) yields None rather than raising, so callers can decide.
|
|
1467
|
+
"""
|
|
1468
|
+
try:
|
|
1469
|
+
proc = subprocess.run(
|
|
1470
|
+
["git", "rev-parse", "--show-toplevel"],
|
|
1471
|
+
cwd=path,
|
|
1472
|
+
stdout=subprocess.PIPE,
|
|
1473
|
+
stderr=subprocess.DEVNULL,
|
|
1474
|
+
text=True,
|
|
1475
|
+
)
|
|
1476
|
+
except (OSError, subprocess.SubprocessError):
|
|
1477
|
+
return None
|
|
1478
|
+
if proc.returncode != 0:
|
|
1479
|
+
return None
|
|
1480
|
+
root = proc.stdout.strip()
|
|
1481
|
+
return root or None
|
|
1482
|
+
|
|
1483
|
+
|
|
1484
|
+
def _git_worktree_paths(repo_root: str) -> list:
|
|
1485
|
+
"""Every checkout path git has registered for ``repo_root`` (realpath'd)."""
|
|
1486
|
+
try:
|
|
1487
|
+
proc = subprocess.run(
|
|
1488
|
+
["git", "worktree", "list", "--porcelain"],
|
|
1489
|
+
cwd=repo_root,
|
|
1490
|
+
stdout=subprocess.PIPE,
|
|
1491
|
+
stderr=subprocess.DEVNULL,
|
|
1492
|
+
text=True,
|
|
1493
|
+
)
|
|
1494
|
+
except (OSError, subprocess.SubprocessError):
|
|
1495
|
+
return []
|
|
1496
|
+
if proc.returncode != 0:
|
|
1497
|
+
return []
|
|
1498
|
+
paths = []
|
|
1499
|
+
for line in proc.stdout.splitlines():
|
|
1500
|
+
if line.startswith("worktree "):
|
|
1501
|
+
paths.append(os.path.realpath(line[len("worktree "):].strip()))
|
|
1502
|
+
return paths
|
|
1503
|
+
|
|
1504
|
+
|
|
1505
|
+
def _lane_workspace_path(lane, repo_root: str) -> str:
|
|
1506
|
+
"""The sibling directory a lane's long-lived workspace lives in.
|
|
1507
|
+
|
|
1508
|
+
``<repo root name>-ws-<lane id>`` BESIDE the repo root, never inside it, so
|
|
1509
|
+
a workspace can never pollute the main checkout's ``git status``.
|
|
1510
|
+
"""
|
|
1511
|
+
repo_abs = os.path.abspath(repo_root)
|
|
1512
|
+
return os.path.join(
|
|
1513
|
+
os.path.dirname(repo_abs), f"{os.path.basename(repo_abs)}-ws-{lane.id}"
|
|
1514
|
+
)
|
|
1515
|
+
|
|
1516
|
+
|
|
1517
|
+
def _lane_run_dir(lane, repo_root: str, workspace_root: str) -> str:
|
|
1518
|
+
"""Where inside the workspace this lane's cards run.
|
|
1519
|
+
|
|
1520
|
+
A ``path`` under the repo root maps to the same relative location inside the
|
|
1521
|
+
workspace; a ``path`` that IS the repo root (or outside it, e.g. its own
|
|
1522
|
+
independent clone) runs at the workspace root.
|
|
1523
|
+
"""
|
|
1524
|
+
repo = os.path.realpath(repo_root)
|
|
1525
|
+
lane_path = os.path.realpath(lane.path)
|
|
1526
|
+
if lane_path == repo:
|
|
1527
|
+
return workspace_root
|
|
1528
|
+
try:
|
|
1529
|
+
common = os.path.commonpath([repo, lane_path])
|
|
1530
|
+
except ValueError: # different drives on Windows
|
|
1531
|
+
return workspace_root
|
|
1532
|
+
if common == repo:
|
|
1533
|
+
rel = os.path.relpath(lane_path, repo)
|
|
1534
|
+
if rel and rel != ".":
|
|
1535
|
+
return os.path.join(workspace_root, rel)
|
|
1536
|
+
return workspace_root
|
|
1537
|
+
|
|
1538
|
+
|
|
1539
|
+
def _probe_dependency_links(lane, repo_root: Optional[str] = None) -> list:
|
|
1540
|
+
"""Dependency directories to symlink in, inferred from build files.
|
|
1541
|
+
|
|
1542
|
+
Detection mirrors ``registry.infer_verify`` (a marker file in the lane path),
|
|
1543
|
+
but the returned path is workspace-root-relative -- the same base an explicit
|
|
1544
|
+
``link`` uses -- so a subdirectory lane links ``<lane rel>/node_modules``
|
|
1545
|
+
rather than the repository root's copy.
|
|
1546
|
+
"""
|
|
1547
|
+
prefix = ""
|
|
1548
|
+
if repo_root:
|
|
1549
|
+
rel = os.path.relpath(os.path.realpath(lane.path), os.path.realpath(repo_root))
|
|
1550
|
+
if rel not in (".", ""):
|
|
1551
|
+
prefix = rel
|
|
1552
|
+
found = []
|
|
1553
|
+
for marker, link in _DEPENDENCY_LINKS:
|
|
1554
|
+
if os.path.isfile(os.path.join(lane.path, marker)):
|
|
1555
|
+
found.append(os.path.join(prefix, link) if prefix else link)
|
|
1556
|
+
return found
|
|
1557
|
+
|
|
1558
|
+
|
|
1559
|
+
def _lane_links(lane, repo_root: Optional[str] = None) -> list:
|
|
1560
|
+
"""The relative paths to symlink from the main tree into the workspace.
|
|
1561
|
+
|
|
1562
|
+
An explicit ``link`` wins; otherwise the build-file probe supplies the
|
|
1563
|
+
defaults (``package.json`` -> ``node_modules``, ``Cargo.toml`` -> ``target``).
|
|
1564
|
+
"""
|
|
1565
|
+
if lane.link:
|
|
1566
|
+
return list(lane.link)
|
|
1567
|
+
return _probe_dependency_links(lane, repo_root)
|
|
1568
|
+
|
|
1569
|
+
|
|
1570
|
+
def _ensure_links(repo_root: str, workspace_root: str, links) -> list:
|
|
1571
|
+
"""Idempotently symlink ``links`` from the main tree into the workspace.
|
|
1572
|
+
|
|
1573
|
+
Each target is ``<workspace>/<rel>`` pointing at ``<repo root>/<rel>`` --
|
|
1574
|
+
the main checkout's copy, so dependencies are shared rather than reinstalled.
|
|
1575
|
+
An existing target (symlink or real directory) is left untouched. Returns the
|
|
1576
|
+
links this call actually created; no package manager is ever invoked.
|
|
1577
|
+
"""
|
|
1578
|
+
created = []
|
|
1579
|
+
for rel in links:
|
|
1580
|
+
target = os.path.join(workspace_root, rel)
|
|
1581
|
+
if os.path.lexists(target):
|
|
1582
|
+
continue
|
|
1583
|
+
parent = os.path.dirname(target)
|
|
1584
|
+
if parent:
|
|
1585
|
+
os.makedirs(parent, exist_ok=True)
|
|
1586
|
+
os.symlink(os.path.join(repo_root, rel), target)
|
|
1587
|
+
created.append(rel)
|
|
1588
|
+
return created
|
|
1589
|
+
|
|
1590
|
+
|
|
1591
|
+
def _worktree_add(repo_root: str, path: str, lane) -> None:
|
|
1592
|
+
"""``git worktree add --detach`` a fresh, long-lived lane workspace."""
|
|
1593
|
+
proc = subprocess.run(
|
|
1594
|
+
["git", "worktree", "add", "--detach", path],
|
|
1595
|
+
cwd=repo_root,
|
|
1596
|
+
stdout=subprocess.PIPE,
|
|
1597
|
+
stderr=subprocess.STDOUT,
|
|
1598
|
+
text=True,
|
|
1599
|
+
)
|
|
1600
|
+
if proc.returncode != 0:
|
|
1601
|
+
shutil.rmtree(path, ignore_errors=True)
|
|
1602
|
+
raise UsageError(
|
|
1603
|
+
f"could not create a git worktree for lane '{lane.id}'",
|
|
1604
|
+
hint=(proc.stdout or "").strip()
|
|
1605
|
+
or "git worktree add failed (is the repo clean?)",
|
|
1606
|
+
)
|
|
1607
|
+
|
|
1608
|
+
|
|
1609
|
+
def _ensure_lane_workspace(conn, task_id: str, lane, *, worktree_flag: bool = False):
|
|
1610
|
+
"""Create-or-reuse ``lane``'s long-lived workspace; return (root, run_dir).
|
|
1611
|
+
|
|
1612
|
+
Called only once a card actually has a slot and is about to run, so simply
|
|
1613
|
+
editing the registry never builds anything. A workspace is created once and
|
|
1614
|
+
reused forever after; a directory that exists but is not a worktree of the
|
|
1615
|
+
same repo is a hard error with the `workspace-rm` remedy. ``worktree_flag``
|
|
1616
|
+
is the one-time ``--worktree`` argument: when set it is ignored, because the
|
|
1617
|
+
lane workspace already is the long-lived checkout.
|
|
1618
|
+
"""
|
|
1619
|
+
if not _is_git_repo(lane.path):
|
|
1620
|
+
raise UsageError(
|
|
1621
|
+
f"workspace = \"worktree\" needs a git repository, but {lane.path} "
|
|
1622
|
+
"is not one",
|
|
1623
|
+
hint='set workspace = "none" for this lane, or point it at a git checkout',
|
|
1624
|
+
)
|
|
1625
|
+
repo_root = _git_toplevel(lane.path)
|
|
1626
|
+
if not repo_root:
|
|
1627
|
+
raise UsageError(
|
|
1628
|
+
f"could not find a git repository root for lane '{lane.id}' "
|
|
1629
|
+
f"(path {lane.path})",
|
|
1630
|
+
)
|
|
1631
|
+
repo_root = os.path.realpath(repo_root)
|
|
1632
|
+
workspace_root = _lane_workspace_path(lane, repo_root)
|
|
1633
|
+
|
|
1634
|
+
created = False
|
|
1635
|
+
if os.path.lexists(workspace_root):
|
|
1636
|
+
if os.path.realpath(workspace_root) not in _git_worktree_paths(repo_root):
|
|
1637
|
+
raise UsageError(
|
|
1638
|
+
f"{workspace_root} already exists but is not a worktree of "
|
|
1639
|
+
f"{repo_root}",
|
|
1640
|
+
hint=f"taskproof workspace-rm {lane.id}",
|
|
1641
|
+
)
|
|
1642
|
+
else:
|
|
1643
|
+
_worktree_add(repo_root, workspace_root, lane)
|
|
1644
|
+
created = True
|
|
1645
|
+
|
|
1646
|
+
links = _ensure_links(repo_root, workspace_root, _lane_links(lane, repo_root))
|
|
1647
|
+
run_dir = _lane_run_dir(lane, repo_root, workspace_root)
|
|
1648
|
+
|
|
1649
|
+
payload = {"path": workspace_root, "repo": repo_root, "links": links}
|
|
1650
|
+
if worktree_flag:
|
|
1651
|
+
payload["worktree_flag_ignored"] = True
|
|
1652
|
+
payload["note"] = (
|
|
1653
|
+
"using the lane workspace; --worktree is ignored while "
|
|
1654
|
+
'workspace = "worktree"'
|
|
1655
|
+
)
|
|
1656
|
+
if created:
|
|
1657
|
+
storage.append_event(conn, task_id, "workspace_created", payload)
|
|
1658
|
+
else:
|
|
1659
|
+
payload["reused"] = True
|
|
1660
|
+
storage.append_event(conn, task_id, "workspace", payload)
|
|
1661
|
+
return workspace_root, run_dir
|
|
1662
|
+
|
|
1663
|
+
|
|
1664
|
+
def _worktree_changes_list(path: str, *, links=()) -> list:
|
|
1665
|
+
"""``git status --porcelain`` lines for a workspace (empty on any failure).
|
|
1666
|
+
|
|
1667
|
+
``links`` are the dependency symlinks taskproof itself created; they are
|
|
1668
|
+
excluded so a freshly-built workspace counts as clean instead of being
|
|
1669
|
+
flagged as dirty (an untracked ``node_modules`` symlink in a repo that does
|
|
1670
|
+
not gitignore it is taskproof's plumbing, not the adapter's work).
|
|
1671
|
+
|
|
1672
|
+
``--untracked-files=all`` is essential, not cosmetic: by default git
|
|
1673
|
+
collapses a wholly-untracked directory to ``?? desktop/``, which would hide
|
|
1674
|
+
a link like ``desktop/node_modules`` behind its parent and defeat the
|
|
1675
|
+
exclusion (and equally hide the adapter's own new files).
|
|
1676
|
+
"""
|
|
1677
|
+
try:
|
|
1678
|
+
proc = subprocess.run(
|
|
1679
|
+
["git", "status", "--porcelain", "--untracked-files=all"],
|
|
1680
|
+
cwd=path,
|
|
1681
|
+
stdout=subprocess.PIPE,
|
|
1682
|
+
stderr=subprocess.DEVNULL,
|
|
1683
|
+
text=True,
|
|
1684
|
+
)
|
|
1685
|
+
except (OSError, subprocess.SubprocessError):
|
|
1686
|
+
return []
|
|
1687
|
+
if proc.returncode != 0:
|
|
1688
|
+
return []
|
|
1689
|
+
excluded = [item.rstrip("/") for item in links if item]
|
|
1690
|
+
result = []
|
|
1691
|
+
for line in proc.stdout.splitlines():
|
|
1692
|
+
if not line.strip():
|
|
1693
|
+
continue
|
|
1694
|
+
candidate = line[3:].strip().strip('"') if len(line) > 3 else ""
|
|
1695
|
+
if any(
|
|
1696
|
+
candidate == item or candidate.startswith(item + "/")
|
|
1697
|
+
for item in excluded
|
|
1698
|
+
):
|
|
1699
|
+
continue
|
|
1700
|
+
result.append(line)
|
|
1701
|
+
return result
|
|
1702
|
+
|
|
1703
|
+
|
|
1704
|
+
def _unmerged_commits(path: str, repo_root: Optional[str] = None) -> list:
|
|
1705
|
+
"""Commits in a workspace that the main checkout has not merged.
|
|
1706
|
+
|
|
1707
|
+
A lane workspace is created ``--detach``, so any commit the adapter makes
|
|
1708
|
+
lives off-branch: exactly the "not merged anywhere" work an operator must be
|
|
1709
|
+
warned about before deleting. The exclusion base is the source repo's HEAD,
|
|
1710
|
+
so a clean, freshly-checked-out workspace lists none while a commit the
|
|
1711
|
+
adapter made (even on its own branch) is listed. ``--not --all`` cannot be
|
|
1712
|
+
used: git's ``--all`` includes HEAD itself, which would hide every commit.
|
|
1713
|
+
"""
|
|
1714
|
+
args = ["git", "log", "--oneline", "--no-decorate", "HEAD"]
|
|
1715
|
+
base = None
|
|
1716
|
+
if repo_root:
|
|
1717
|
+
try:
|
|
1718
|
+
probe = subprocess.run(
|
|
1719
|
+
["git", "rev-parse", "HEAD"],
|
|
1720
|
+
cwd=repo_root,
|
|
1721
|
+
stdout=subprocess.PIPE,
|
|
1722
|
+
stderr=subprocess.DEVNULL,
|
|
1723
|
+
text=True,
|
|
1724
|
+
)
|
|
1725
|
+
if probe.returncode == 0 and probe.stdout.strip():
|
|
1726
|
+
base = probe.stdout.strip()
|
|
1727
|
+
except (OSError, subprocess.SubprocessError):
|
|
1728
|
+
base = None
|
|
1729
|
+
if base:
|
|
1730
|
+
args += ["--not", base]
|
|
1731
|
+
else:
|
|
1732
|
+
args += ["--not", "--branches", "--tags", "--remotes"]
|
|
1733
|
+
try:
|
|
1734
|
+
proc = subprocess.run(
|
|
1735
|
+
args,
|
|
1736
|
+
cwd=path,
|
|
1737
|
+
stdout=subprocess.PIPE,
|
|
1738
|
+
stderr=subprocess.DEVNULL,
|
|
1739
|
+
text=True,
|
|
1740
|
+
)
|
|
1741
|
+
except (OSError, subprocess.SubprocessError):
|
|
1742
|
+
return []
|
|
1743
|
+
if proc.returncode != 0:
|
|
1744
|
+
return []
|
|
1745
|
+
return [line for line in proc.stdout.splitlines() if line.strip()]
|
|
1746
|
+
|
|
1747
|
+
|
|
1748
|
+
def _lane_workspace_status(lane) -> dict:
|
|
1749
|
+
"""Read-only evidence about one lane's long-lived workspace."""
|
|
1750
|
+
repo_root = _git_toplevel(lane.path) if os.path.isdir(lane.path) else None
|
|
1751
|
+
repo_root = os.path.realpath(repo_root) if repo_root else None
|
|
1752
|
+
ws_path = _lane_workspace_path(lane, repo_root) if repo_root else None
|
|
1753
|
+
exists = bool(ws_path) and os.path.isdir(ws_path)
|
|
1754
|
+
links = _lane_links(lane, repo_root)
|
|
1755
|
+
changed_files = _worktree_changes_list(ws_path, links=links) if exists else []
|
|
1756
|
+
unmerged = _unmerged_commits(ws_path, repo_root) if exists else []
|
|
1757
|
+
return {
|
|
1758
|
+
"id": lane.id,
|
|
1759
|
+
"project": lane.project,
|
|
1760
|
+
"path": lane.path,
|
|
1761
|
+
"repo": repo_root,
|
|
1762
|
+
"workspace": ws_path,
|
|
1763
|
+
"exists": exists,
|
|
1764
|
+
"changed_files": changed_files,
|
|
1765
|
+
"changed": len(changed_files),
|
|
1766
|
+
"unmerged_commits": unmerged,
|
|
1767
|
+
"unmerged": len(unmerged),
|
|
1768
|
+
"links": links,
|
|
1769
|
+
}
|
|
1770
|
+
|
|
1771
|
+
|
|
1772
|
+
def lane_workspace_status(workspace: str, lane_key: str) -> dict:
|
|
1773
|
+
"""Status of one lane's long-lived workspace (`taskproof workspace-rm`)."""
|
|
1774
|
+
reg = registry.load(registry.workspace_registry_path(workspace))
|
|
1775
|
+
return _lane_workspace_status(reg.require(lane_key))
|
|
1776
|
+
|
|
1777
|
+
|
|
1778
|
+
def list_workspaces(workspace: str) -> list:
|
|
1779
|
+
"""Every lane that declares ``workspace = "worktree"``, with its status."""
|
|
1780
|
+
reg = registry.load(registry.workspace_registry_path(workspace))
|
|
1781
|
+
return [
|
|
1782
|
+
_lane_workspace_status(lane)
|
|
1783
|
+
for lane in reg.taskgroups
|
|
1784
|
+
if lane.workspace == "worktree"
|
|
1785
|
+
]
|
|
1786
|
+
|
|
1787
|
+
|
|
1788
|
+
def remove_workspace(workspace: str, lane_key: str, *, force: bool = False) -> dict:
|
|
1789
|
+
"""Delete a lane's long-lived workspace, refusing when it holds work.
|
|
1790
|
+
|
|
1791
|
+
Uncommitted changes or commits that exist on no ref make this refuse unless
|
|
1792
|
+
``force`` is set; the refusal names the files/commits so nothing is deleted
|
|
1793
|
+
blind. Either way the git worktree is unregistered first
|
|
1794
|
+
(``git worktree remove --force``) with a directory fallback, and a
|
|
1795
|
+
``workspace_removed`` event records the outcome.
|
|
1796
|
+
"""
|
|
1797
|
+
reg = registry.load(registry.workspace_registry_path(workspace))
|
|
1798
|
+
lane = reg.require(lane_key)
|
|
1799
|
+
if lane.workspace != "worktree":
|
|
1800
|
+
raise UsageError(
|
|
1801
|
+
f"lane '{lane.id}' does not declare a workspace "
|
|
1802
|
+
'(workspace = "worktree")',
|
|
1803
|
+
hint="nothing to remove; see `taskproof workspaces`",
|
|
1804
|
+
)
|
|
1805
|
+
repo_root = _git_toplevel(lane.path) if os.path.isdir(lane.path) else None
|
|
1806
|
+
if not repo_root:
|
|
1807
|
+
raise UsageError(
|
|
1808
|
+
f"lane '{lane.id}' path {lane.path} is not a git repository",
|
|
1809
|
+
)
|
|
1810
|
+
repo_root = os.path.realpath(repo_root)
|
|
1811
|
+
ws_path = _lane_workspace_path(lane, repo_root)
|
|
1812
|
+
|
|
1813
|
+
if not os.path.lexists(ws_path):
|
|
1814
|
+
return {
|
|
1815
|
+
"path": ws_path,
|
|
1816
|
+
"repo": repo_root,
|
|
1817
|
+
"removed": False,
|
|
1818
|
+
"forced": False,
|
|
1819
|
+
"changed_files": [],
|
|
1820
|
+
"unmerged_commits": [],
|
|
1821
|
+
}
|
|
1822
|
+
|
|
1823
|
+
changed = _worktree_changes_list(ws_path, links=_lane_links(lane, repo_root))
|
|
1824
|
+
unmerged = _unmerged_commits(ws_path, repo_root)
|
|
1825
|
+
dirty = bool(changed or unmerged)
|
|
1826
|
+
if dirty and not force:
|
|
1827
|
+
detail = "; ".join(
|
|
1828
|
+
part
|
|
1829
|
+
for part in (
|
|
1830
|
+
"uncommitted: " + ", ".join(changed) if changed else "",
|
|
1831
|
+
"unmerged: " + ", ".join(unmerged) if unmerged else "",
|
|
1832
|
+
)
|
|
1833
|
+
if part
|
|
1834
|
+
)
|
|
1835
|
+
raise UsageError(
|
|
1836
|
+
f"workspace {ws_path} still holds work ({detail}); refusing to "
|
|
1837
|
+
"remove",
|
|
1838
|
+
hint="re-run with --force to delete it anyway",
|
|
1839
|
+
)
|
|
1840
|
+
|
|
1841
|
+
error = _remove_worktree(repo_root, ws_path)
|
|
1842
|
+
result = {
|
|
1843
|
+
"path": ws_path,
|
|
1844
|
+
"repo": repo_root,
|
|
1845
|
+
"removed": True,
|
|
1846
|
+
"forced": bool(force and dirty),
|
|
1847
|
+
"changed_files": changed,
|
|
1848
|
+
"unmerged_commits": unmerged,
|
|
1849
|
+
}
|
|
1850
|
+
if error:
|
|
1851
|
+
result["error"] = error
|
|
1852
|
+
|
|
1853
|
+
conn = storage.connect(storage.db_path(workspace))
|
|
1854
|
+
try:
|
|
1855
|
+
storage.migrate(conn)
|
|
1856
|
+
payload = {"path": ws_path, "repo": repo_root}
|
|
1857
|
+
if force and dirty:
|
|
1858
|
+
payload["forced"] = True
|
|
1859
|
+
payload["note"] = "removed while dirty / with unmerged commits"
|
|
1860
|
+
storage.append_event(conn, None, "workspace_removed", payload)
|
|
1861
|
+
finally:
|
|
1862
|
+
conn.close()
|
|
1863
|
+
return result
|
|
1864
|
+
|
|
1865
|
+
|
|
1866
|
+
# ---------------------------------------------------------------------------
|
|
1867
|
+
# Queries
|
|
1868
|
+
# ---------------------------------------------------------------------------
|
|
1869
|
+
|
|
1870
|
+
|
|
1871
|
+
def verify_task(workspace: str, task_id: str) -> verify.VerifyOutcome:
|
|
1872
|
+
"""Re-run acceptance for an existing task (`taskproof verify <id>`)."""
|
|
1873
|
+
conn = storage.connect(storage.db_path(workspace))
|
|
1874
|
+
try:
|
|
1875
|
+
storage.migrate(conn)
|
|
1876
|
+
row = storage.get_task(conn, task_id)
|
|
1877
|
+
if row is None:
|
|
1878
|
+
raise UsageError(f"no such task: {task_id}")
|
|
1879
|
+
taskgroup = registry.load(
|
|
1880
|
+
registry.workspace_registry_path(workspace)
|
|
1881
|
+
).require(row["project"])
|
|
1882
|
+
workdir = row["workdir"] or taskgroup.path
|
|
1883
|
+
|
|
1884
|
+
outcome = verify.run_acceptance(workdir, taskgroup.verify)
|
|
1885
|
+
status = VERIFY_SKIPPED if not outcome.ran else (
|
|
1886
|
+
"PASSED" if outcome.passed else "FAILED"
|
|
1887
|
+
)
|
|
1888
|
+
fields = {"verify_cmd": taskgroup.verify if outcome.ran else VERIFY_SKIPPED}
|
|
1889
|
+
if outcome.exit_code is not None:
|
|
1890
|
+
fields["verify_exit"] = outcome.exit_code
|
|
1891
|
+
if outcome.files_changed is not None:
|
|
1892
|
+
fields["files_changed"] = outcome.files_changed
|
|
1893
|
+
storage.update_task(conn, task_id, **fields)
|
|
1894
|
+
storage.append_event(
|
|
1895
|
+
conn,
|
|
1896
|
+
task_id,
|
|
1897
|
+
"verify",
|
|
1898
|
+
{
|
|
1899
|
+
"status": status,
|
|
1900
|
+
"ran": outcome.ran,
|
|
1901
|
+
"exit_code": outcome.exit_code,
|
|
1902
|
+
"note": outcome.note,
|
|
1903
|
+
"violations": outcome.violations,
|
|
1904
|
+
},
|
|
1905
|
+
)
|
|
1906
|
+
return outcome
|
|
1907
|
+
finally:
|
|
1908
|
+
conn.close()
|
|
1909
|
+
|
|
1910
|
+
|
|
1911
|
+
def task_detail(workspace: str, task_id: str) -> dict:
|
|
1912
|
+
"""Task row plus its (JSON-decoded) event stream, for `show` and the board."""
|
|
1913
|
+
conn = storage.connect(storage.db_path(workspace))
|
|
1914
|
+
try:
|
|
1915
|
+
storage.migrate(conn)
|
|
1916
|
+
row = storage.get_task(conn, task_id)
|
|
1917
|
+
task = dict(row) if row is not None else None
|
|
1918
|
+
|
|
1919
|
+
events = []
|
|
1920
|
+
for event in storage.list_events(conn, task_id):
|
|
1921
|
+
item = dict(event)
|
|
1922
|
+
raw = item.get("payload")
|
|
1923
|
+
if raw is not None:
|
|
1924
|
+
try:
|
|
1925
|
+
item["payload"] = json.loads(raw)
|
|
1926
|
+
except (json.JSONDecodeError, TypeError, ValueError):
|
|
1927
|
+
pass
|
|
1928
|
+
events.append(item)
|
|
1929
|
+
return {"task": task, "events": events}
|
|
1930
|
+
finally:
|
|
1931
|
+
conn.close()
|
|
1932
|
+
|
|
1933
|
+
|
|
1934
|
+
# ---------------------------------------------------------------------------
|
|
1935
|
+
# Control: cancel / remove
|
|
1936
|
+
# ---------------------------------------------------------------------------
|
|
1937
|
+
|
|
1938
|
+
|
|
1939
|
+
def cancel_task(workspace: str, task_id: str) -> dict:
|
|
1940
|
+
"""Stop a task and record the terminal state `cancelled`.
|
|
1941
|
+
|
|
1942
|
+
* `running` / `verifying`: signal the recorded process group (SIGTERM, then
|
|
1943
|
+
SIGKILL after a grace period).
|
|
1944
|
+
* The workdir is NEVER touched — matching "a failed dispatch does not roll
|
|
1945
|
+
back its workspace".
|
|
1946
|
+
* A task already in a terminal state is a readable error, never a silent
|
|
1947
|
+
no-op.
|
|
1948
|
+
|
|
1949
|
+
Returns the updated task row.
|
|
1950
|
+
"""
|
|
1951
|
+
conn = storage.connect(storage.db_path(workspace))
|
|
1952
|
+
try:
|
|
1953
|
+
storage.migrate(conn)
|
|
1954
|
+
row = storage.get_task(conn, task_id)
|
|
1955
|
+
if row is None:
|
|
1956
|
+
raise TaskNotFoundError(f"no such task: {task_id}")
|
|
1957
|
+
status = row["status"]
|
|
1958
|
+
if status in TERMINAL_STATUSES:
|
|
1959
|
+
raise TaskStateError(
|
|
1960
|
+
f"task {task_id} is already {status}; nothing to cancel"
|
|
1961
|
+
)
|
|
1962
|
+
if status not in (STATUS_RUNNING, STATUS_VERIFYING):
|
|
1963
|
+
raise TaskStateError(
|
|
1964
|
+
f"task {task_id} cannot be cancelled from state {status}"
|
|
1965
|
+
)
|
|
1966
|
+
|
|
1967
|
+
pgid = row["pgid"]
|
|
1968
|
+
pid = row["pid"]
|
|
1969
|
+
# Record the terminal state BEFORE signalling. That way the in-flight
|
|
1970
|
+
# dispatcher observes `cancelled` (via `_task_cancelled`) before its
|
|
1971
|
+
# adapter dies, so it returns without appending a contradictory
|
|
1972
|
+
# `failed`/`timeout` event for a task the user deliberately stopped.
|
|
1973
|
+
storage.update_task(
|
|
1974
|
+
conn,
|
|
1975
|
+
task_id,
|
|
1976
|
+
status=STATUS_CANCELLED,
|
|
1977
|
+
finished_at=storage.now_iso(),
|
|
1978
|
+
)
|
|
1979
|
+
storage.append_event(
|
|
1980
|
+
conn,
|
|
1981
|
+
task_id,
|
|
1982
|
+
"cancelled",
|
|
1983
|
+
{"pid": pid, "pgid": pgid, "was": status},
|
|
1984
|
+
)
|
|
1985
|
+
if status in (STATUS_RUNNING, STATUS_VERIFYING) and pgid is not None:
|
|
1986
|
+
# Only the group the adapter created; unrelated processes are safe.
|
|
1987
|
+
_terminate_recorded_group(pgid)
|
|
1988
|
+
return dict(storage.get_task(conn, task_id))
|
|
1989
|
+
finally:
|
|
1990
|
+
conn.close()
|
|
1991
|
+
|
|
1992
|
+
|
|
1993
|
+
#: `verify` labels carried by an `accepted` event. They describe the acceptance
|
|
1994
|
+
#: result the human accepted the breach *alongside*, not a fresh verdict.
|
|
1995
|
+
_VERIFY_PASSED = "passed"
|
|
1996
|
+
_VERIFY_FAILED = "failed"
|
|
1997
|
+
_VERIFY_SKIPPED = "skipped"
|
|
1998
|
+
|
|
1999
|
+
|
|
2000
|
+
def _violation_summary(conn, task_id: str) -> dict:
|
|
2001
|
+
"""The breach an `accept` confirms: the rules and the paths they hit.
|
|
2002
|
+
|
|
2003
|
+
Read from the card's own `forbidden` / `blocked` events, so the `accepted`
|
|
2004
|
+
event carries the facts the human actually looked at rather than a
|
|
2005
|
+
re-derivation from the registry.
|
|
2006
|
+
"""
|
|
2007
|
+
summary = {"rules": [], "paths": [], "violations": []}
|
|
2008
|
+
for name in ("forbidden", "blocked"):
|
|
2009
|
+
row = conn.execute(
|
|
2010
|
+
"SELECT payload FROM events WHERE task_id = ? AND event = ? "
|
|
2011
|
+
"ORDER BY id DESC LIMIT 1",
|
|
2012
|
+
(task_id, name),
|
|
2013
|
+
).fetchone()
|
|
2014
|
+
if row is None or row[0] is None:
|
|
2015
|
+
continue
|
|
2016
|
+
try:
|
|
2017
|
+
payload = json.loads(row[0])
|
|
2018
|
+
except (json.JSONDecodeError, TypeError, ValueError):
|
|
2019
|
+
continue
|
|
2020
|
+
if not isinstance(payload, dict):
|
|
2021
|
+
continue
|
|
2022
|
+
if not summary["violations"] and isinstance(payload.get("violations"), list):
|
|
2023
|
+
summary["violations"] = payload["violations"]
|
|
2024
|
+
if not summary["rules"] and isinstance(payload.get("rules"), list):
|
|
2025
|
+
summary["rules"] = list(payload["rules"])
|
|
2026
|
+
if not summary["paths"] and isinstance(payload.get("paths"), list):
|
|
2027
|
+
summary["paths"] = list(payload["paths"])
|
|
2028
|
+
# A `blocked` event may carry only the per-violation detail; derive the rule
|
|
2029
|
+
# list from it so `rules` is never empty when a breach was recorded.
|
|
2030
|
+
if not summary["rules"]:
|
|
2031
|
+
summary["rules"] = sorted(
|
|
2032
|
+
{
|
|
2033
|
+
entry.get("rule")
|
|
2034
|
+
for entry in summary["violations"]
|
|
2035
|
+
if isinstance(entry, dict) and entry.get("rule")
|
|
2036
|
+
}
|
|
2037
|
+
)
|
|
2038
|
+
return summary
|
|
2039
|
+
|
|
2040
|
+
|
|
2041
|
+
#: How much of the agent's own last summary the `accepted` event carries. The
|
|
2042
|
+
#: summary is context for a human sign-off, never a verdict — a tail is enough.
|
|
2043
|
+
_SUMMARY_TAIL_CHARS = 500
|
|
2044
|
+
|
|
2045
|
+
|
|
2046
|
+
def _last_summary(conn, task_id: str) -> Optional[str]:
|
|
2047
|
+
"""The agent's own last summary, newest first, or None when there is none.
|
|
2048
|
+
|
|
2049
|
+
`adapter` carries the agent's claim right after it exits; a clean `done`
|
|
2050
|
+
carries it again. A failed *acceptance* records no summary (only the
|
|
2051
|
+
verdict), so this is best-effort context, not a required field.
|
|
2052
|
+
"""
|
|
2053
|
+
for name in ("adapter", "done"):
|
|
2054
|
+
row = conn.execute(
|
|
2055
|
+
"SELECT payload FROM events WHERE task_id = ? AND event = ? "
|
|
2056
|
+
"ORDER BY id DESC LIMIT 1",
|
|
2057
|
+
(task_id, name),
|
|
2058
|
+
).fetchone()
|
|
2059
|
+
if row is None or row[0] is None:
|
|
2060
|
+
continue
|
|
2061
|
+
try:
|
|
2062
|
+
payload = json.loads(row[0])
|
|
2063
|
+
except (json.JSONDecodeError, TypeError, ValueError):
|
|
2064
|
+
continue
|
|
2065
|
+
if isinstance(payload, dict):
|
|
2066
|
+
text = payload.get("summary")
|
|
2067
|
+
if isinstance(text, str) and text.strip():
|
|
2068
|
+
return text
|
|
2069
|
+
return None
|
|
2070
|
+
|
|
2071
|
+
|
|
2072
|
+
def accept_task(
|
|
2073
|
+
workspace: str, task_id: str, *, by: str = "cli", note: Optional[str] = None
|
|
2074
|
+
) -> dict:
|
|
2075
|
+
"""Clear a `blocked` or `failed` card — the human disposition of one run.
|
|
2076
|
+
|
|
2077
|
+
`blocked` (a boundary breach) is accepted with or without a note, exactly as
|
|
2078
|
+
it always has been: a green acceptance earns `done`, a red or skipped one
|
|
2079
|
+
earns `failed`. `failed` (the adapter failed, or acceptance ran and went red)
|
|
2080
|
+
may also be accepted, but ONLY with a non-empty ``note``: closing a failed
|
|
2081
|
+
card is a human sign-off, and the reason must survive in the ledger.
|
|
2082
|
+
|
|
2083
|
+
`accept` is a human **registration**, not a verdict: it never re-runs
|
|
2084
|
+
acceptance and never rewrites `verify_*`. Whatever the run recorded when it
|
|
2085
|
+
stopped stays authoritative; a failed card is promoted to `done` only because
|
|
2086
|
+
a human judged the recorded failure a false red, and that judgement is kept
|
|
2087
|
+
in the `accepted` event alongside the evidence the human saw.
|
|
2088
|
+
|
|
2089
|
+
Every other status (running / verifying / done / timeout /
|
|
2090
|
+
cancelled) is refused with a readable `TaskStateError`, so `accept` can never
|
|
2091
|
+
silently re-close a card. The card's `verify_*` columns and `finished_at` are
|
|
2092
|
+
left untouched — the run's own facts do not change because a human later
|
|
2093
|
+
signed off. Returns the updated task row.
|
|
2094
|
+
"""
|
|
2095
|
+
note_text = note if isinstance(note, str) else None
|
|
2096
|
+
note_present = bool(note_text and note_text.strip())
|
|
2097
|
+
conn = storage.connect(storage.db_path(workspace))
|
|
2098
|
+
try:
|
|
2099
|
+
storage.migrate(conn)
|
|
2100
|
+
row = storage.get_task(conn, task_id)
|
|
2101
|
+
if row is None:
|
|
2102
|
+
raise TaskNotFoundError(f"no such task: {task_id}")
|
|
2103
|
+
status = row["status"]
|
|
2104
|
+
if status not in (STATUS_BLOCKED, STATUS_FAILED):
|
|
2105
|
+
raise TaskStateError(
|
|
2106
|
+
f"task {task_id} is {status}; "
|
|
2107
|
+
"only a blocked or failed task can be accepted"
|
|
2108
|
+
)
|
|
2109
|
+
if status == STATUS_FAILED and not note_present:
|
|
2110
|
+
# A failed card is closed by human judgement, not by a fresh verdict,
|
|
2111
|
+
# so the reason MUST be on the record. Refuse before touching the row.
|
|
2112
|
+
raise UsageError(
|
|
2113
|
+
"accepting a failed task requires a note: "
|
|
2114
|
+
"人工收尾必须留说明",
|
|
2115
|
+
hint=(
|
|
2116
|
+
"accept is a human registration, not a verdict; pass "
|
|
2117
|
+
'--note "why this failure is being cleared"'
|
|
2118
|
+
),
|
|
2119
|
+
)
|
|
2120
|
+
|
|
2121
|
+
verify_exit = row["verify_exit"]
|
|
2122
|
+
if verify_exit == 0:
|
|
2123
|
+
verify_label = _VERIFY_PASSED
|
|
2124
|
+
elif verify_exit is None:
|
|
2125
|
+
verify_label = _VERIFY_SKIPPED
|
|
2126
|
+
else:
|
|
2127
|
+
verify_label = _VERIFY_FAILED
|
|
2128
|
+
if status == STATUS_FAILED:
|
|
2129
|
+
# A human signed the red off as a false negative: the recorded facts
|
|
2130
|
+
# stay, the terminal state becomes `done`.
|
|
2131
|
+
new_status = STATUS_DONE
|
|
2132
|
+
else:
|
|
2133
|
+
# `blocked` keeps following its acceptance result, exactly as before.
|
|
2134
|
+
new_status = STATUS_DONE if verify_exit == 0 else STATUS_FAILED
|
|
2135
|
+
|
|
2136
|
+
payload = {
|
|
2137
|
+
"by": by,
|
|
2138
|
+
"verify": verify_label,
|
|
2139
|
+
"verify_exit": verify_exit,
|
|
2140
|
+
"from": status,
|
|
2141
|
+
"to": new_status,
|
|
2142
|
+
# The evidence the human actually saw when signing off.
|
|
2143
|
+
"files_changed": row["files_changed"],
|
|
2144
|
+
# The note is stored verbatim (None for a blocked card with no note).
|
|
2145
|
+
"note": note_text,
|
|
2146
|
+
}
|
|
2147
|
+
if status == STATUS_BLOCKED:
|
|
2148
|
+
summary = _violation_summary(conn, task_id)
|
|
2149
|
+
payload["rules"] = summary["rules"]
|
|
2150
|
+
payload["paths"] = summary["paths"]
|
|
2151
|
+
payload["violations"] = summary["violations"]
|
|
2152
|
+
else:
|
|
2153
|
+
agent_summary = _last_summary(conn, task_id)
|
|
2154
|
+
if agent_summary:
|
|
2155
|
+
payload["summary"] = agent_summary[:_SUMMARY_TAIL_CHARS]
|
|
2156
|
+
|
|
2157
|
+
storage.update_task(conn, task_id, status=new_status)
|
|
2158
|
+
storage.append_event(conn, task_id, "accepted", payload)
|
|
2159
|
+
return dict(storage.get_task(conn, task_id))
|
|
2160
|
+
finally:
|
|
2161
|
+
conn.close()
|
|
2162
|
+
|
|
2163
|
+
|
|
2164
|
+
def rerun_task(workspace: str, task_id: str) -> str:
|
|
2165
|
+
"""Start a fresh card from a terminal one, linked both ways in the ledger.
|
|
2166
|
+
|
|
2167
|
+
The new card reuses the old card's brief and run flags (adapter / model /
|
|
2168
|
+
reasoning / timeout / idle_timeout / read_only / worktree / skip_verify),
|
|
2169
|
+
recovered from its row and its events by :func:`_run_options`. A flag the
|
|
2170
|
+
ledger never captured falls back to the documented default (see
|
|
2171
|
+
`_run_options`).
|
|
2172
|
+
|
|
2173
|
+
Only a terminal card may be rerun, `blocked` included; a non-terminal card
|
|
2174
|
+
is refused so two attempts of the same work can never overlap. The new card
|
|
2175
|
+
goes down the normal ``dispatch`` path and gets ``rerun_of`` while the old
|
|
2176
|
+
card gets ``rerun_as``. Returns the new task id.
|
|
2177
|
+
"""
|
|
2178
|
+
conn = storage.connect(storage.db_path(workspace))
|
|
2179
|
+
try:
|
|
2180
|
+
storage.migrate(conn)
|
|
2181
|
+
row = storage.get_task(conn, task_id)
|
|
2182
|
+
if row is None:
|
|
2183
|
+
raise TaskNotFoundError(f"no such task: {task_id}")
|
|
2184
|
+
if row["status"] not in TERMINAL_STATUSES:
|
|
2185
|
+
raise TaskStateError(
|
|
2186
|
+
f"task {task_id} is {row['status']}; "
|
|
2187
|
+
"only a terminal task can be rerun"
|
|
2188
|
+
)
|
|
2189
|
+
options = _run_options(conn, task_id)
|
|
2190
|
+
old = dict(row)
|
|
2191
|
+
finally:
|
|
2192
|
+
conn.close()
|
|
2193
|
+
|
|
2194
|
+
return dispatch(
|
|
2195
|
+
workspace,
|
|
2196
|
+
old["project"],
|
|
2197
|
+
old["brief"],
|
|
2198
|
+
adapter=old["adapter"] or "codex",
|
|
2199
|
+
model=old["model"],
|
|
2200
|
+
reasoning=old["reasoning"],
|
|
2201
|
+
read_only=bool(options.get("read_only")),
|
|
2202
|
+
worktree=bool(options.get("worktree")),
|
|
2203
|
+
skip_verify=bool(options.get("skip_verify")),
|
|
2204
|
+
timeout=options.get("timeout"),
|
|
2205
|
+
idle_timeout=options.get("idle_timeout"),
|
|
2206
|
+
rerun_of=task_id,
|
|
2207
|
+
)
|
|
2208
|
+
|
|
2209
|
+
|
|
2210
|
+
def _kept_worktrees(conn, task_id: str):
|
|
2211
|
+
"""Every kept-worktree payload for a task, newest first.
|
|
2212
|
+
|
|
2213
|
+
A run that left changes behind appends one ``worktree`` event carrying its
|
|
2214
|
+
``path`` and ``changed`` count; `rm` reads those back (before deleting the
|
|
2215
|
+
events) so it can clean up the directories too.
|
|
2216
|
+
"""
|
|
2217
|
+
found = []
|
|
2218
|
+
for row in conn.execute(
|
|
2219
|
+
"SELECT payload FROM events WHERE task_id = ? AND event = 'worktree' "
|
|
2220
|
+
"ORDER BY id DESC",
|
|
2221
|
+
(task_id,),
|
|
2222
|
+
):
|
|
2223
|
+
text = row[0]
|
|
2224
|
+
try:
|
|
2225
|
+
payload = json.loads(text) if text else {}
|
|
2226
|
+
except (json.JSONDecodeError, TypeError, ValueError):
|
|
2227
|
+
payload = {}
|
|
2228
|
+
if isinstance(payload, dict) and payload.get("path"):
|
|
2229
|
+
found.append(payload)
|
|
2230
|
+
return found
|
|
2231
|
+
|
|
2232
|
+
|
|
2233
|
+
def _discard_kept_worktree(payload: dict) -> None:
|
|
2234
|
+
"""Delete a worktree a run kept, as `rm <id>` requires.
|
|
2235
|
+
|
|
2236
|
+
The tradeoff: `rm` owns the whole task, so the unrecoverable work it kept on
|
|
2237
|
+
disk goes with it. Unregister via git when the source repo is known (best
|
|
2238
|
+
effort), then remove the directory either way.
|
|
2239
|
+
"""
|
|
2240
|
+
path = payload.get("path")
|
|
2241
|
+
if not path:
|
|
2242
|
+
return
|
|
2243
|
+
repo = payload.get("repo")
|
|
2244
|
+
if repo and os.path.isdir(repo):
|
|
2245
|
+
_remove_worktree(repo, path)
|
|
2246
|
+
else:
|
|
2247
|
+
shutil.rmtree(path, ignore_errors=True)
|
|
2248
|
+
|
|
2249
|
+
|
|
2250
|
+
def remove_task(workspace: str, task_id: str) -> str:
|
|
2251
|
+
"""Delete a terminal task's row, its `events` rows, and any kept worktree.
|
|
2252
|
+
|
|
2253
|
+
Non-terminal tasks are refused with a hint to cancel first. The append-only
|
|
2254
|
+
JSONL audit stream is intentionally left in place (see `storage.delete_task`).
|
|
2255
|
+
Returns the removed id.
|
|
2256
|
+
"""
|
|
2257
|
+
conn = storage.connect(storage.db_path(workspace))
|
|
2258
|
+
try:
|
|
2259
|
+
storage.migrate(conn)
|
|
2260
|
+
row = storage.get_task(conn, task_id)
|
|
2261
|
+
if row is None:
|
|
2262
|
+
raise TaskNotFoundError(f"no such task: {task_id}")
|
|
2263
|
+
if row["status"] not in TERMINAL_STATUSES:
|
|
2264
|
+
raise TaskStateError(
|
|
2265
|
+
f"task {task_id} is {row['status']}; cancel it before removing",
|
|
2266
|
+
hint=f"taskproof cancel {task_id}",
|
|
2267
|
+
)
|
|
2268
|
+
# A kept worktree is part of the task `rm` is discarding; snapshot the
|
|
2269
|
+
# paths before `delete_task` wipes the events that name them.
|
|
2270
|
+
kept_worktrees = _kept_worktrees(conn, task_id)
|
|
2271
|
+
storage.delete_task(conn, task_id)
|
|
2272
|
+
for payload in kept_worktrees:
|
|
2273
|
+
_discard_kept_worktree(payload)
|
|
2274
|
+
return task_id
|
|
2275
|
+
finally:
|
|
2276
|
+
conn.close()
|
|
2277
|
+
|
|
2278
|
+
|
|
2279
|
+
|
|
2280
|
+
def summary_counts(workspace: str, conn) -> dict:
|
|
2281
|
+
"""Counts per status — the one query the dashboard leans on.
|
|
2282
|
+
|
|
2283
|
+
Every known status is present (0 when it has no rows) so the board can draw
|
|
2284
|
+
a stable set of columns without special-casing a missing bucket.
|
|
2285
|
+
"""
|
|
2286
|
+
counts = {status: 0 for status in _ALL_STATUSES}
|
|
2287
|
+
for row in conn.execute("SELECT status, COUNT(*) FROM tasks GROUP BY status"):
|
|
2288
|
+
counts[row[0]] = int(row[1])
|
|
2289
|
+
return counts
|