ctrlrelay 0.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ctrlrelay/__init__.py +33 -0
- ctrlrelay/bridge/__init__.py +21 -0
- ctrlrelay/bridge/__main__.py +89 -0
- ctrlrelay/bridge/protocol.py +75 -0
- ctrlrelay/bridge/server.py +740 -0
- ctrlrelay/bridge/telegram_handler.py +136 -0
- ctrlrelay/cli.py +3138 -0
- ctrlrelay/core/__init__.py +54 -0
- ctrlrelay/core/audit.py +257 -0
- ctrlrelay/core/checkpoint.py +155 -0
- ctrlrelay/core/config.py +778 -0
- ctrlrelay/core/dispatcher.py +372 -0
- ctrlrelay/core/github.py +432 -0
- ctrlrelay/core/network.py +277 -0
- ctrlrelay/core/obs.py +118 -0
- ctrlrelay/core/poller.py +920 -0
- ctrlrelay/core/pr_verifier.py +230 -0
- ctrlrelay/core/pr_watcher.py +121 -0
- ctrlrelay/core/question_expiry.py +193 -0
- ctrlrelay/core/scheduler.py +337 -0
- ctrlrelay/core/state.py +642 -0
- ctrlrelay/core/worktree.py +893 -0
- ctrlrelay/dashboard/__init__.py +5 -0
- ctrlrelay/dashboard/client.py +159 -0
- ctrlrelay/gh_protocol.py +47 -0
- ctrlrelay/install.py +203 -0
- ctrlrelay/personalization/__init__.py +26 -0
- ctrlrelay/personalization/manager.py +1183 -0
- ctrlrelay/personalization/paths.py +143 -0
- ctrlrelay/pipelines/__init__.py +15 -0
- ctrlrelay/pipelines/base.py +50 -0
- ctrlrelay/pipelines/dev.py +1367 -0
- ctrlrelay/pipelines/post_merge.py +411 -0
- ctrlrelay/pipelines/secops.py +1018 -0
- ctrlrelay/pipelines/task.py +713 -0
- ctrlrelay/setup.py +617 -0
- ctrlrelay/templates/__init__.py +0 -0
- ctrlrelay/templates/global-CLAUDE.md.snippet +61 -0
- ctrlrelay/templates/launchd/__init__.py +0 -0
- ctrlrelay/templates/launchd/bridge.plist.template +37 -0
- ctrlrelay/templates/launchd/poller.plist.template +44 -0
- ctrlrelay/templates/systemd/__init__.py +0 -0
- ctrlrelay/templates/systemd/bridge.service.template +20 -0
- ctrlrelay/templates/systemd/poller.service.template +25 -0
- ctrlrelay/transports/__init__.py +34 -0
- ctrlrelay/transports/base.py +74 -0
- ctrlrelay/transports/file_mock.py +103 -0
- ctrlrelay/transports/socket_client.py +298 -0
- ctrlrelay-0.10.0.dist-info/METADATA +254 -0
- ctrlrelay-0.10.0.dist-info/RECORD +53 -0
- ctrlrelay-0.10.0.dist-info/WHEEL +4 -0
- ctrlrelay-0.10.0.dist-info/entry_points.txt +2 -0
- ctrlrelay-0.10.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,1367 @@
|
|
|
1
|
+
"""Dev pipeline for issue-to-PR workflow."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import time
|
|
7
|
+
import uuid
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from ctrlrelay.core.checkpoint import CheckpointStatus
|
|
13
|
+
from ctrlrelay.core.dispatcher import AgentAdapter, SessionResult
|
|
14
|
+
from ctrlrelay.core.github import GitHubCLI
|
|
15
|
+
from ctrlrelay.core.obs import get_logger, hash_text, log_event
|
|
16
|
+
from ctrlrelay.core.pr_verifier import PRVerifier, VerificationResult
|
|
17
|
+
from ctrlrelay.core.state import StateDB
|
|
18
|
+
from ctrlrelay.core.worktree import WorktreeManager
|
|
19
|
+
from ctrlrelay.dashboard.client import DashboardClient, EventPayload
|
|
20
|
+
from ctrlrelay.pipelines.base import PipelineContext, PipelineResult
|
|
21
|
+
from ctrlrelay.transports.base import Transport
|
|
22
|
+
|
|
23
|
+
DEFAULT_MAX_FIX_ATTEMPTS = 3
|
|
24
|
+
DEFAULT_MAX_BLOCKED_ROUNDS = 5
|
|
25
|
+
|
|
26
|
+
# Reacquire policy for the repo lock after the unlocked verification phase.
|
|
27
|
+
# Verification releases the lock because CI polling doesn't touch git; when
|
|
28
|
+
# we need to run `request_fix` (which spawns claude + writes worktree) we
|
|
29
|
+
# must get it back. Contention here is NOT necessarily brief — a peer
|
|
30
|
+
# session may be running its own full claude pass on another issue in the
|
|
31
|
+
# same repo, which can take tens of minutes. A short retry budget would
|
|
32
|
+
# spuriously fail healthy same-repo parallelism; bound the wait at roughly
|
|
33
|
+
# one hour, long enough to outlast a normal peer run and short enough to
|
|
34
|
+
# escape a stuck/SIGKILL'd peer whose lock row is lingering.
|
|
35
|
+
# Tests monkey-patch these to speed contention scenarios without changing
|
|
36
|
+
# the call sites.
|
|
37
|
+
_REACQUIRE_LOCK_ATTEMPTS = 720
|
|
38
|
+
_REACQUIRE_LOCK_SLEEP_SECONDS = 5.0
|
|
39
|
+
# Log progress periodically so an operator watching can see we're alive
|
|
40
|
+
# and waiting, not hung. At 5s cadence this fires ~every 5 minutes.
|
|
41
|
+
_REACQUIRE_PROGRESS_LOG_EVERY = 60
|
|
42
|
+
|
|
43
|
+
# Post-verify CLEANUP (worktree rm, branch delete) is best-effort — if
|
|
44
|
+
# a peer grabs the lock while we're polling CI and we can't get it back
|
|
45
|
+
# fast, skipping cleanup is fine (next run / operator reclaims). Don't
|
|
46
|
+
# use the fix-path budget here: run_poll_loop awaits handlers serially,
|
|
47
|
+
# so waiting an hour for cleanup would stall the whole poll cycle AND
|
|
48
|
+
# delay the merge-watcher spawn that happens after run_dev_issue
|
|
49
|
+
# returns. Three quick retries cover a fleeting hiccup; anything longer
|
|
50
|
+
# belongs elsewhere.
|
|
51
|
+
_REACQUIRE_CLEANUP_ATTEMPTS = 3
|
|
52
|
+
_REACQUIRE_CLEANUP_SLEEP_SECONDS = 1.0
|
|
53
|
+
|
|
54
|
+
# Release-lock retry policy for transient SQLite errors. The final
|
|
55
|
+
# release() call in run_dev_issue's outer finally is the last chance
|
|
56
|
+
# to clean up; if that raises and we silently returned, the repo_locks
|
|
57
|
+
# row would linger forever. Short retry loop handles the common
|
|
58
|
+
# transient case; exhausted retries emit a loud wedged event so ops
|
|
59
|
+
# can reclaim manually.
|
|
60
|
+
_RELEASE_RETRY_ATTEMPTS = 3
|
|
61
|
+
_RELEASE_RETRY_SLEEP_SECONDS = 0.5
|
|
62
|
+
|
|
63
|
+
# Two distinct error messages so callers can tell which phase of the
|
|
64
|
+
# pipeline hit contention:
|
|
65
|
+
# INITIAL — nothing started, safe to retry from scratch (cli un-marks
|
|
66
|
+
# the issue so the next poll picks it up).
|
|
67
|
+
# DURING_VERIFY — a PR already exists; re-running from scratch would
|
|
68
|
+
# launch a duplicate dev pass against the open PR.
|
|
69
|
+
# cli leaves the issue marked seen and only spawns the
|
|
70
|
+
# PR merge watcher so the existing PR still auto-closes
|
|
71
|
+
# on merge.
|
|
72
|
+
_LOCK_CONTENDED_ERROR = "Repository locked by another session"
|
|
73
|
+
_LOCK_CONTENDED_DURING_VERIFY_ERROR = (
|
|
74
|
+
"Repo lock reacquire contended during PR verification"
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class _RepoLockHandle:
|
|
79
|
+
"""Tracks whether this session currently holds the repo lock and
|
|
80
|
+
allows release/reacquire around phases that don't need exclusive git
|
|
81
|
+
access (PR CI verification is pure `gh` polling — see issue #29).
|
|
82
|
+
The ``finally`` block at the end of ``run_dev_issue`` calls
|
|
83
|
+
``release`` regardless; calling it twice is a no-op because
|
|
84
|
+
``StateDB.release_lock`` is idempotent (deletes rows, rowcount=0 the
|
|
85
|
+
second time). Tracking ``held`` locally lets the pipeline decide
|
|
86
|
+
whether worktree/branch cleanup is safe to attempt on error paths."""
|
|
87
|
+
|
|
88
|
+
__slots__ = ("state_db", "repo", "session_id", "held")
|
|
89
|
+
|
|
90
|
+
def __init__(self, state_db: StateDB, repo: str, session_id: str) -> None:
|
|
91
|
+
self.state_db = state_db
|
|
92
|
+
self.repo = repo
|
|
93
|
+
self.session_id = session_id
|
|
94
|
+
# Starts False; run_dev_issue wraps its initial acquire call and
|
|
95
|
+
# flips this to True on success.
|
|
96
|
+
self.held = False
|
|
97
|
+
|
|
98
|
+
def release(self) -> None:
|
|
99
|
+
"""Release the lock if we hold it. Idempotent and safe to call
|
|
100
|
+
from a ``finally`` block that may run before acquire ever
|
|
101
|
+
succeeded.
|
|
102
|
+
|
|
103
|
+
Retries the DELETE a few times on transient failures (SQLite
|
|
104
|
+
``database is locked``, etc.). The outer ``finally`` is the
|
|
105
|
+
last-chance call — if THAT raised and we silently returned,
|
|
106
|
+
the ``repo_locks`` row would linger forever, wedging every
|
|
107
|
+
future acquire on this repo until manual DB cleanup.
|
|
108
|
+
|
|
109
|
+
After exhausting retries we emit a ``dev.lock.release_wedged``
|
|
110
|
+
critical-level event naming the repo + session so an operator
|
|
111
|
+
can reclaim the lock row. held is left True so an outer
|
|
112
|
+
retry (if any) still has a chance.
|
|
113
|
+
|
|
114
|
+
The inner retry's exception is swallowed only after the
|
|
115
|
+
wedged event fires — release must not leak exceptions from a
|
|
116
|
+
finally path where it would mask the caller's original error."""
|
|
117
|
+
if not self.held:
|
|
118
|
+
return
|
|
119
|
+
last_exc: Exception | None = None
|
|
120
|
+
for attempt in range(1, _RELEASE_RETRY_ATTEMPTS + 1):
|
|
121
|
+
try:
|
|
122
|
+
self.state_db.release_lock(self.repo, self.session_id)
|
|
123
|
+
except Exception as e:
|
|
124
|
+
last_exc = e
|
|
125
|
+
log_event(
|
|
126
|
+
_logger,
|
|
127
|
+
"dev.lock.release_failed",
|
|
128
|
+
session_id=self.session_id,
|
|
129
|
+
repo=self.repo,
|
|
130
|
+
attempt=attempt,
|
|
131
|
+
max_attempts=_RELEASE_RETRY_ATTEMPTS,
|
|
132
|
+
reason=type(e).__name__,
|
|
133
|
+
error=str(e)[:200],
|
|
134
|
+
)
|
|
135
|
+
if attempt < _RELEASE_RETRY_ATTEMPTS:
|
|
136
|
+
time.sleep(_RELEASE_RETRY_SLEEP_SECONDS)
|
|
137
|
+
continue
|
|
138
|
+
self.held = False
|
|
139
|
+
return
|
|
140
|
+
# All retries exhausted. The row is presumed still present;
|
|
141
|
+
# alert loudly so it doesn't silently wedge the repo.
|
|
142
|
+
log_event(
|
|
143
|
+
_logger,
|
|
144
|
+
"dev.lock.release_wedged",
|
|
145
|
+
session_id=self.session_id,
|
|
146
|
+
repo=self.repo,
|
|
147
|
+
reason=type(last_exc).__name__ if last_exc else "unknown",
|
|
148
|
+
error=str(last_exc)[:200] if last_exc else "",
|
|
149
|
+
operator_action=(
|
|
150
|
+
"DELETE FROM repo_locks WHERE repo = ? AND session_id = ? "
|
|
151
|
+
"manually, or restart the poller after confirming the "
|
|
152
|
+
"session is not live."
|
|
153
|
+
),
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
async def reacquire(
|
|
157
|
+
self,
|
|
158
|
+
*,
|
|
159
|
+
attempts: int | None = None,
|
|
160
|
+
sleep_seconds: float | None = None,
|
|
161
|
+
) -> bool:
|
|
162
|
+
"""Try to re-grab the lock after a release. Returns True on
|
|
163
|
+
success, False if contention persists past the retry budget.
|
|
164
|
+
|
|
165
|
+
Retry budget: ``attempts`` INSERT tries separated by
|
|
166
|
+
``sleep_seconds`` of sleep. Default 720×5s ≈ 1 hour. That's
|
|
167
|
+
long enough to outlast a peer running its own full dev pass
|
|
168
|
+
on another issue in the same repo (claude + CI can take
|
|
169
|
+
tens of minutes), and short enough to escape a stuck or
|
|
170
|
+
SIGKILL'd peer whose lock row never got cleaned up.
|
|
171
|
+
|
|
172
|
+
Defaults resolve at call time (``None`` sentinel) so tests can
|
|
173
|
+
monkey-patch the module constants to speed up contention
|
|
174
|
+
scenarios without threading kwargs through every caller.
|
|
175
|
+
"""
|
|
176
|
+
if self.held:
|
|
177
|
+
return True
|
|
178
|
+
effective_attempts = (
|
|
179
|
+
_REACQUIRE_LOCK_ATTEMPTS if attempts is None else attempts
|
|
180
|
+
)
|
|
181
|
+
effective_sleep = (
|
|
182
|
+
_REACQUIRE_LOCK_SLEEP_SECONDS
|
|
183
|
+
if sleep_seconds is None
|
|
184
|
+
else sleep_seconds
|
|
185
|
+
)
|
|
186
|
+
for attempt in range(max(1, effective_attempts)):
|
|
187
|
+
if self.state_db.acquire_lock(self.repo, self.session_id):
|
|
188
|
+
self.held = True
|
|
189
|
+
return True
|
|
190
|
+
# Surface a heartbeat every N attempts so an operator
|
|
191
|
+
# tailing logs during a long peer hold can tell the
|
|
192
|
+
# session is waiting, not wedged.
|
|
193
|
+
if attempt > 0 and attempt % _REACQUIRE_PROGRESS_LOG_EVERY == 0:
|
|
194
|
+
log_event(
|
|
195
|
+
_logger,
|
|
196
|
+
"dev.lock.reacquire_waiting",
|
|
197
|
+
session_id=self.session_id,
|
|
198
|
+
repo=self.repo,
|
|
199
|
+
attempt=attempt,
|
|
200
|
+
max_attempts=effective_attempts,
|
|
201
|
+
)
|
|
202
|
+
if attempt < effective_attempts - 1:
|
|
203
|
+
await asyncio.sleep(effective_sleep)
|
|
204
|
+
return False
|
|
205
|
+
|
|
206
|
+
AGENT_CLAIM_MARKER = "<!-- ctrlrelay:claimed -->"
|
|
207
|
+
AGENT_CLAIM_COMMENT = (
|
|
208
|
+
"CTRLRelay is working on this issue. A PR will be opened for review.\n\n"
|
|
209
|
+
f"{AGENT_CLAIM_MARKER}"
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
# Discussion budget for the prompt. Newest comments are kept because a
|
|
213
|
+
# redirect is far likelier to be the latest word than the first reply,
|
|
214
|
+
# and each body is clipped so one pasted log dump can't crowd out the
|
|
215
|
+
# rest of the thread.
|
|
216
|
+
MAX_PROMPT_COMMENTS = 20
|
|
217
|
+
MAX_PROMPT_COMMENT_CHARS = 2000
|
|
218
|
+
|
|
219
|
+
_logger = get_logger("pipeline.dev")
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def format_issue_comments(
|
|
223
|
+
comments: list[dict[str, Any]] | None,
|
|
224
|
+
*,
|
|
225
|
+
max_comments: int = MAX_PROMPT_COMMENTS,
|
|
226
|
+
max_chars: int = MAX_PROMPT_COMMENT_CHARS,
|
|
227
|
+
) -> str:
|
|
228
|
+
"""Render the issue discussion for the dev prompt.
|
|
229
|
+
|
|
230
|
+
The body states the problem as it was first filed. The thread is
|
|
231
|
+
where an operator narrows, redirects or outright vetoes that plan,
|
|
232
|
+
and nobody edits the body afterwards. A vulnq issue was filed asking
|
|
233
|
+
for a client fix and then redirected in a comment to price up
|
|
234
|
+
dropping the data source instead; the prompt carried only the body,
|
|
235
|
+
so the agent spent a full session building the plan that had already
|
|
236
|
+
been rejected in writing.
|
|
237
|
+
|
|
238
|
+
Drops our own claim comment — it is our noise, not the operator's.
|
|
239
|
+
Keeps the newest ``max_comments`` but renders them oldest-first so
|
|
240
|
+
the thread still reads in order.
|
|
241
|
+
"""
|
|
242
|
+
if not comments:
|
|
243
|
+
return "_No comments._"
|
|
244
|
+
|
|
245
|
+
kept = [
|
|
246
|
+
c for c in comments
|
|
247
|
+
if AGENT_CLAIM_MARKER not in (c.get("body") or "")
|
|
248
|
+
]
|
|
249
|
+
if not kept:
|
|
250
|
+
return "_No comments._"
|
|
251
|
+
|
|
252
|
+
dropped = len(kept) - max_comments
|
|
253
|
+
if dropped > 0:
|
|
254
|
+
kept = kept[-max_comments:]
|
|
255
|
+
|
|
256
|
+
blocks: list[str] = []
|
|
257
|
+
if dropped > 0:
|
|
258
|
+
blocks.append(
|
|
259
|
+
f"_({dropped} earlier comment(s) omitted; read the issue on "
|
|
260
|
+
f"GitHub if the thread below references them.)_"
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
for c in kept:
|
|
264
|
+
author = (c.get("author") or {}).get("login") or "unknown"
|
|
265
|
+
created = c.get("createdAt") or ""
|
|
266
|
+
body = (c.get("body") or "").strip() or "_(empty)_"
|
|
267
|
+
if len(body) > max_chars:
|
|
268
|
+
body = body[:max_chars] + "\n_[...truncated]_"
|
|
269
|
+
blocks.append(f"--- @{author} ({created}) ---\n{body}")
|
|
270
|
+
|
|
271
|
+
return "\n\n".join(blocks)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _question_for_persist(session_id: str, result: PipelineResult) -> str:
|
|
275
|
+
"""Return a non-empty question string for pending_resumes storage.
|
|
276
|
+
|
|
277
|
+
Mirrors the synthesized prompt the in-process BLOCKED loop uses
|
|
278
|
+
when ``result.question`` is empty — without this, a BLOCKED exit
|
|
279
|
+
with no question text would skip persistence and the session
|
|
280
|
+
would become unresumable via Telegram."""
|
|
281
|
+
q = (result.question or "").strip()
|
|
282
|
+
if q:
|
|
283
|
+
return q
|
|
284
|
+
return (
|
|
285
|
+
f"Session {session_id} is blocked but did not include a "
|
|
286
|
+
"question. Reply with guidance to resume."
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
@dataclass
|
|
291
|
+
class DevPipeline:
|
|
292
|
+
"""Dev pipeline for implementing issues and opening PRs."""
|
|
293
|
+
|
|
294
|
+
dispatcher: AgentAdapter
|
|
295
|
+
github: GitHubCLI
|
|
296
|
+
worktree: WorktreeManager
|
|
297
|
+
dashboard: DashboardClient | None
|
|
298
|
+
state_db: StateDB
|
|
299
|
+
transport: Transport | None
|
|
300
|
+
|
|
301
|
+
name: str = "dev"
|
|
302
|
+
|
|
303
|
+
async def run(self, ctx: PipelineContext) -> PipelineResult:
|
|
304
|
+
"""Run dev pipeline on a single issue."""
|
|
305
|
+
prompt = self._build_prompt(
|
|
306
|
+
ctx.repo, ctx.issue_number, ctx.extra,
|
|
307
|
+
session_id=ctx.session_id,
|
|
308
|
+
state_file=ctx.state_file,
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
result = await self._spawn(ctx, prompt, resume=False)
|
|
312
|
+
return self._session_to_result(result)
|
|
313
|
+
|
|
314
|
+
async def resume(self, ctx: PipelineContext, answer: str) -> PipelineResult:
|
|
315
|
+
"""Resume blocked dev session with user answer."""
|
|
316
|
+
prompt = f"User answered: {answer}\n\nContinue from where you left off."
|
|
317
|
+
|
|
318
|
+
# Resume uses Claude's own session UUID (captured on the first spawn
|
|
319
|
+
# and persisted to state_db). Passing our composite id here makes
|
|
320
|
+
# `claude --resume` reject the call on v2.0.x+ CLI builds.
|
|
321
|
+
resume_uuid = self.state_db.get_agent_session_id(ctx.session_id)
|
|
322
|
+
|
|
323
|
+
log_event(
|
|
324
|
+
_logger,
|
|
325
|
+
"dev.session.resumed",
|
|
326
|
+
session_id=ctx.session_id,
|
|
327
|
+
repo=ctx.repo,
|
|
328
|
+
issue_number=ctx.issue_number,
|
|
329
|
+
pipeline=self.name,
|
|
330
|
+
resume_session_id=resume_uuid,
|
|
331
|
+
answer_length=len(answer),
|
|
332
|
+
answer_hash=hash_text(answer),
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
result = await self._spawn(ctx, prompt, resume=True)
|
|
336
|
+
return self._session_to_result(result)
|
|
337
|
+
|
|
338
|
+
async def request_fix(
|
|
339
|
+
self, ctx: PipelineContext, fix_instructions: str
|
|
340
|
+
) -> PipelineResult:
|
|
341
|
+
"""Resume the session with a fix request (failing CI or merge conflict)."""
|
|
342
|
+
result = await self._spawn(ctx, fix_instructions, resume=True)
|
|
343
|
+
return self._session_to_result(result)
|
|
344
|
+
|
|
345
|
+
async def _spawn(
|
|
346
|
+
self,
|
|
347
|
+
ctx: PipelineContext,
|
|
348
|
+
prompt: str,
|
|
349
|
+
*,
|
|
350
|
+
resume: bool,
|
|
351
|
+
) -> SessionResult:
|
|
352
|
+
"""Centralized dispatcher call: looks up the agent UUID on resume,
|
|
353
|
+
persists any newly-captured UUID to state_db."""
|
|
354
|
+
resume_uuid: str | None = None
|
|
355
|
+
if resume:
|
|
356
|
+
# If state_db has no stored UUID (session predates the capture, or
|
|
357
|
+
# first spawn didn't emit JSON) we fall back to a fresh session —
|
|
358
|
+
# better than hard-failing with `claude --resume <composite-id>`.
|
|
359
|
+
resume_uuid = self.state_db.get_agent_session_id(ctx.session_id)
|
|
360
|
+
|
|
361
|
+
result = await self.dispatcher.spawn_session(
|
|
362
|
+
session_id=ctx.session_id,
|
|
363
|
+
prompt=prompt,
|
|
364
|
+
working_dir=ctx.worktree_path,
|
|
365
|
+
state_file=ctx.state_file,
|
|
366
|
+
resume_session_id=resume_uuid,
|
|
367
|
+
repo=ctx.repo,
|
|
368
|
+
issue_number=ctx.issue_number,
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
if result.agent_session_id:
|
|
372
|
+
try:
|
|
373
|
+
self.state_db.set_agent_session_id(
|
|
374
|
+
ctx.session_id, result.agent_session_id
|
|
375
|
+
)
|
|
376
|
+
except Exception:
|
|
377
|
+
# state_db write failures must not mask the session result.
|
|
378
|
+
pass
|
|
379
|
+
|
|
380
|
+
return result
|
|
381
|
+
|
|
382
|
+
def _build_prompt(
|
|
383
|
+
self,
|
|
384
|
+
repo: str,
|
|
385
|
+
issue_number: int | None,
|
|
386
|
+
extra: dict[str, Any],
|
|
387
|
+
session_id: str = "",
|
|
388
|
+
state_file: Path | None = None,
|
|
389
|
+
) -> str:
|
|
390
|
+
"""Build the dev pipeline prompt."""
|
|
391
|
+
issue_title = extra.get("issue_title", "")
|
|
392
|
+
issue_body = extra.get("issue_body", "")
|
|
393
|
+
issue_comments = extra.get("issue_comments") or "_No comments._"
|
|
394
|
+
branch_name = extra.get("branch_name", "")
|
|
395
|
+
ci_wait_timeout_seconds = extra.get("ci_wait_timeout_seconds", 600)
|
|
396
|
+
state_file_path = str(state_file) if state_file else "/tmp/state.json"
|
|
397
|
+
|
|
398
|
+
return f"""You are working on issue #{issue_number} in repository {repo}.
|
|
399
|
+
|
|
400
|
+
**Issue Title:** {issue_title}
|
|
401
|
+
|
|
402
|
+
**Issue Body:**
|
|
403
|
+
{issue_body}
|
|
404
|
+
|
|
405
|
+
**Discussion:**
|
|
406
|
+
{issue_comments}
|
|
407
|
+
|
|
408
|
+
The body is the issue as first filed; the discussion is newer. Where a
|
|
409
|
+
comment narrows, redirects or rejects what the body asks for, the comment
|
|
410
|
+
wins — nobody goes back and edits the body after changing their mind in
|
|
411
|
+
the thread. If the discussion contradicts the body in a way you cannot
|
|
412
|
+
reconcile, signal BLOCKED and ask rather than picking one.
|
|
413
|
+
|
|
414
|
+
**Branch:** {branch_name}
|
|
415
|
+
|
|
416
|
+
Execute the following workflow:
|
|
417
|
+
|
|
418
|
+
1. Validate the issue still applies to the current codebase
|
|
419
|
+
2. If anything is unclear, signal BLOCKED (see below) to ask for clarification
|
|
420
|
+
3. Plan and implement the fix using TDD
|
|
421
|
+
4. Push the branch and open a PR that references the issue
|
|
422
|
+
5. Before signaling DONE, verify the PR is mergeable:
|
|
423
|
+
- Wait for CI by running EXACTLY this command (do NOT improvise bash):
|
|
424
|
+
`ctrlrelay ci wait --pr <PR> --repo {repo} --timeout {ci_wait_timeout_seconds}`
|
|
425
|
+
Exit codes: 0 = all checks passed, 1 = a check failed (investigate,
|
|
426
|
+
fix, push, then re-run the wait), 2 = hard timeout while CI is still
|
|
427
|
+
pending (treat as acceptable — hand off and let the orchestrator
|
|
428
|
+
re-verify). Do NOT write your own `until` / `while` loops around
|
|
429
|
+
`gh pr checks`; those have been miswritten in the past (inverted
|
|
430
|
+
semantics, pipes swallowing exit codes) and burned the whole session
|
|
431
|
+
timeout on PRs that were already green.
|
|
432
|
+
- Run `gh pr view <PR> --json mergeable,mergeStateStatus` — if `mergeable`
|
|
433
|
+
is `CONFLICTING` or `mergeStateStatus` is `DIRTY`, rebase onto the base
|
|
434
|
+
branch, resolve conflicts, and push again.
|
|
435
|
+
6. Signal DONE only when the PR is green AND conflict-free, with the PR URL.
|
|
436
|
+
|
|
437
|
+
Do NOT merge the PR - wait for human review.
|
|
438
|
+
|
|
439
|
+
The orchestrator re-verifies CI and mergeability after you hand off; if it
|
|
440
|
+
finds the PR broken it will resume this session asking you to fix it.
|
|
441
|
+
|
|
442
|
+
## Signaling Completion
|
|
443
|
+
|
|
444
|
+
**CRITICAL**: Before exiting, you MUST write a checkpoint file to signal completion.
|
|
445
|
+
|
|
446
|
+
STATE_FILE: {state_file_path}
|
|
447
|
+
SESSION_ID: {session_id}
|
|
448
|
+
|
|
449
|
+
**DONE** (PR opened AND verified green + conflict-free):
|
|
450
|
+
```bash
|
|
451
|
+
mkdir -p "$(dirname '{state_file_path}')"
|
|
452
|
+
printf '{{"version":"1","status":"DONE","session_id":"{session_id}",'\
|
|
453
|
+
'"timestamp":"%s","summary":"PR opened",'\
|
|
454
|
+
'"outputs":{{"pr_url":"%s","pr_number":%d}}}}' \\
|
|
455
|
+
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" "<PR_URL>" <PR_NUM> > '{state_file_path}'
|
|
456
|
+
```
|
|
457
|
+
|
|
458
|
+
**BLOCKED** (need input):
|
|
459
|
+
```bash
|
|
460
|
+
mkdir -p "$(dirname '{state_file_path}')"
|
|
461
|
+
printf '{{"version":"1","status":"BLOCKED_NEEDS_INPUT",'\
|
|
462
|
+
'"session_id":"{session_id}","timestamp":"%s","question":"%s"}}' \\
|
|
463
|
+
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" "<QUESTION>" > '{state_file_path}'
|
|
464
|
+
```
|
|
465
|
+
|
|
466
|
+
**FAILED**:
|
|
467
|
+
```bash
|
|
468
|
+
mkdir -p "$(dirname '{state_file_path}')"
|
|
469
|
+
printf '{{"version":"1","status":"FAILED","session_id":"{session_id}",'\
|
|
470
|
+
'"timestamp":"%s","error":"%s"}}' \\
|
|
471
|
+
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" "<ERROR>" > '{state_file_path}'
|
|
472
|
+
```"""
|
|
473
|
+
|
|
474
|
+
def _session_to_result(self, result: SessionResult) -> PipelineResult:
|
|
475
|
+
"""Convert SessionResult to PipelineResult."""
|
|
476
|
+
if result.state is None:
|
|
477
|
+
return PipelineResult(
|
|
478
|
+
success=False,
|
|
479
|
+
session_id=result.session_id,
|
|
480
|
+
summary="No checkpoint state returned",
|
|
481
|
+
error=result.stderr or "Unknown error",
|
|
482
|
+
)
|
|
483
|
+
|
|
484
|
+
if result.state.status == CheckpointStatus.DONE:
|
|
485
|
+
return PipelineResult(
|
|
486
|
+
success=True,
|
|
487
|
+
session_id=result.session_id,
|
|
488
|
+
summary=result.state.summary or "Completed",
|
|
489
|
+
outputs=result.state.outputs,
|
|
490
|
+
)
|
|
491
|
+
|
|
492
|
+
if result.state.status == CheckpointStatus.BLOCKED_NEEDS_INPUT:
|
|
493
|
+
return PipelineResult(
|
|
494
|
+
success=False,
|
|
495
|
+
session_id=result.session_id,
|
|
496
|
+
summary="Blocked on user input",
|
|
497
|
+
blocked=True,
|
|
498
|
+
question=result.state.question,
|
|
499
|
+
)
|
|
500
|
+
|
|
501
|
+
return PipelineResult(
|
|
502
|
+
success=False,
|
|
503
|
+
session_id=result.session_id,
|
|
504
|
+
summary="Failed",
|
|
505
|
+
error=result.state.error,
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _build_fix_prompt(pr_number: int, verification: VerificationResult) -> str:
|
|
510
|
+
"""Build a resume-prompt asking Claude to fix CI failures or merge conflicts."""
|
|
511
|
+
if verification.mergeable == "CONFLICTING":
|
|
512
|
+
return (
|
|
513
|
+
f"PR #{pr_number} has merge conflicts with the base branch "
|
|
514
|
+
f"(mergeStateStatus={verification.merge_state_status}). "
|
|
515
|
+
"Rebase the branch onto the base, resolve the conflicts, push the "
|
|
516
|
+
"updated branch, then signal DONE with the same "
|
|
517
|
+
f'outputs (pr_url, pr_number={pr_number}) once the PR reports '
|
|
518
|
+
"MERGEABLE. Signal FAILED if conflicts cannot be resolved."
|
|
519
|
+
)
|
|
520
|
+
|
|
521
|
+
if verification.merge_state_status == "BEHIND":
|
|
522
|
+
return (
|
|
523
|
+
f"PR #{pr_number} is behind the base branch and base-branch "
|
|
524
|
+
"protection requires it to be up-to-date before merge "
|
|
525
|
+
"(mergeStateStatus=BEHIND). Rebase the branch onto the base and "
|
|
526
|
+
"force-push, then signal DONE with the same "
|
|
527
|
+
f"outputs (pr_url, pr_number={pr_number}) once the PR reports "
|
|
528
|
+
"mergeStateStatus=CLEAN."
|
|
529
|
+
)
|
|
530
|
+
|
|
531
|
+
if verification.failing_checks:
|
|
532
|
+
names = ", ".join(
|
|
533
|
+
f"{c.get('name', '?')}={c.get('state') or c.get('bucket')}"
|
|
534
|
+
for c in verification.failing_checks
|
|
535
|
+
)
|
|
536
|
+
return (
|
|
537
|
+
f"PR #{pr_number} has failing or incomplete CI checks: {names}. "
|
|
538
|
+
"Investigate the failures (fetch logs via `gh run view` as needed), "
|
|
539
|
+
"fix the underlying issues, commit and push, then wait for CI to go "
|
|
540
|
+
"green. Call checkpoint.done() with the same outputs once all checks "
|
|
541
|
+
"pass and the PR is MERGEABLE."
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
return (
|
|
545
|
+
f"PR #{pr_number} is not ready to hand off: {verification.reason}. "
|
|
546
|
+
"Investigate, push any required fixes, and call checkpoint.done() once "
|
|
547
|
+
"CI is green and the PR is MERGEABLE."
|
|
548
|
+
)
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
async def _verify_and_fix_pr(
|
|
552
|
+
*,
|
|
553
|
+
pipeline: DevPipeline,
|
|
554
|
+
ctx: PipelineContext,
|
|
555
|
+
result: PipelineResult,
|
|
556
|
+
verifier: PRVerifier,
|
|
557
|
+
max_attempts: int,
|
|
558
|
+
lock_handle: _RepoLockHandle | None = None,
|
|
559
|
+
) -> PipelineResult:
|
|
560
|
+
"""Loop: verify CI+mergeability, ask Claude to fix, re-verify.
|
|
561
|
+
|
|
562
|
+
Lock discipline (issue #29): the verify phase is pure `gh` polling
|
|
563
|
+
and holds no git state, so callers pass a ``lock_handle`` whose
|
|
564
|
+
repo lock we release before each ``verifier.verify`` call and
|
|
565
|
+
reacquire before each ``request_fix`` call (which spawns claude
|
|
566
|
+
and writes to the worktree). This lets peer sessions targeting
|
|
567
|
+
the same repo run their own git-op phases while we wait on CI.
|
|
568
|
+
|
|
569
|
+
If reacquire fails after the retry budget, we surface a clear
|
|
570
|
+
failed result rather than silently running ``request_fix``
|
|
571
|
+
without the lock — another session may be mutating the shared
|
|
572
|
+
bare repo and our claude process could corrupt its state.
|
|
573
|
+
Cancellation during the unlocked phase is safe: the lock_handle
|
|
574
|
+
tracks ``held=False`` so the outer ``finally`` no-ops its release.
|
|
575
|
+
"""
|
|
576
|
+
pr_number_raw = result.outputs.get("pr_number")
|
|
577
|
+
if pr_number_raw is None:
|
|
578
|
+
return result
|
|
579
|
+
pr_number = int(pr_number_raw)
|
|
580
|
+
|
|
581
|
+
async def _run_verify() -> VerificationResult:
|
|
582
|
+
"""Release the lock around the pure-gh polling window, then
|
|
583
|
+
hand the lock back to the caller on return. Reacquire is
|
|
584
|
+
deferred to just before the fix path so peer sessions get the
|
|
585
|
+
widest possible window."""
|
|
586
|
+
if lock_handle is not None:
|
|
587
|
+
lock_handle.release()
|
|
588
|
+
try:
|
|
589
|
+
return await verifier.verify(ctx.repo, pr_number)
|
|
590
|
+
except asyncio.CancelledError:
|
|
591
|
+
# Intentionally leave held=False so the outer finally's
|
|
592
|
+
# release_lock is a no-op; the row is already gone. Re-raise
|
|
593
|
+
# to let the cancellation propagate.
|
|
594
|
+
raise
|
|
595
|
+
|
|
596
|
+
verification = await _run_verify()
|
|
597
|
+
# If CI is simply slow (timed_out) we hand the PR off rather than asking
|
|
598
|
+
# Claude to "fix" something that isn't broken.
|
|
599
|
+
if verification.timed_out:
|
|
600
|
+
return result
|
|
601
|
+
attempts = 0
|
|
602
|
+
while not verification.ready and attempts < max_attempts:
|
|
603
|
+
# Reacquire before request_fix — claude spawns inside the
|
|
604
|
+
# worktree and pushes to the shared bare repo. If we can't
|
|
605
|
+
# get the lock back in a reasonable window, bail cleanly
|
|
606
|
+
# rather than racing another session.
|
|
607
|
+
if lock_handle is not None and not lock_handle.held:
|
|
608
|
+
reacquired = await lock_handle.reacquire()
|
|
609
|
+
if not reacquired:
|
|
610
|
+
log_event(
|
|
611
|
+
_logger,
|
|
612
|
+
"dev.verify.lock_reacquire_contended",
|
|
613
|
+
session_id=result.session_id,
|
|
614
|
+
repo=ctx.repo,
|
|
615
|
+
pr_number=pr_number,
|
|
616
|
+
attempts=_REACQUIRE_LOCK_ATTEMPTS,
|
|
617
|
+
)
|
|
618
|
+
return PipelineResult(
|
|
619
|
+
success=False,
|
|
620
|
+
session_id=result.session_id,
|
|
621
|
+
summary=(
|
|
622
|
+
f"PR #{pr_number} verification found work to do "
|
|
623
|
+
"but could not re-acquire repo lock after "
|
|
624
|
+
f"{_REACQUIRE_LOCK_ATTEMPTS} attempts"
|
|
625
|
+
),
|
|
626
|
+
error=_LOCK_CONTENDED_DURING_VERIFY_ERROR,
|
|
627
|
+
outputs=result.outputs,
|
|
628
|
+
)
|
|
629
|
+
|
|
630
|
+
fix_prompt = _build_fix_prompt(pr_number, verification)
|
|
631
|
+
fix_result = await pipeline.request_fix(ctx, fix_prompt)
|
|
632
|
+
attempts += 1
|
|
633
|
+
|
|
634
|
+
if not fix_result.success:
|
|
635
|
+
# Preserve the original PR info in outputs so callers can still find it.
|
|
636
|
+
merged_outputs = dict(result.outputs)
|
|
637
|
+
merged_outputs.update(fix_result.outputs)
|
|
638
|
+
return PipelineResult(
|
|
639
|
+
success=False,
|
|
640
|
+
session_id=fix_result.session_id,
|
|
641
|
+
summary=fix_result.summary,
|
|
642
|
+
blocked=fix_result.blocked,
|
|
643
|
+
question=fix_result.question,
|
|
644
|
+
error=fix_result.error,
|
|
645
|
+
outputs=merged_outputs,
|
|
646
|
+
)
|
|
647
|
+
|
|
648
|
+
result = fix_result
|
|
649
|
+
verification = await _run_verify()
|
|
650
|
+
if verification.timed_out:
|
|
651
|
+
# Same rule after a fix round: slow CI isn't a Claude task.
|
|
652
|
+
return result
|
|
653
|
+
|
|
654
|
+
if verification.ready:
|
|
655
|
+
return result
|
|
656
|
+
|
|
657
|
+
return PipelineResult(
|
|
658
|
+
success=False,
|
|
659
|
+
session_id=result.session_id,
|
|
660
|
+
summary=(
|
|
661
|
+
f"PR #{pr_number} not ready after {max_attempts} fix attempt(s): "
|
|
662
|
+
f"{verification.reason}"
|
|
663
|
+
),
|
|
664
|
+
error=verification.reason,
|
|
665
|
+
outputs=result.outputs,
|
|
666
|
+
)
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
async def run_dev_issue(
|
|
670
|
+
repo: str,
|
|
671
|
+
issue_number: int,
|
|
672
|
+
branch_template: str,
|
|
673
|
+
dispatcher: AgentAdapter,
|
|
674
|
+
github: GitHubCLI,
|
|
675
|
+
worktree: WorktreeManager,
|
|
676
|
+
dashboard: DashboardClient | None,
|
|
677
|
+
state_db: StateDB,
|
|
678
|
+
transport: Transport | None,
|
|
679
|
+
contexts_dir: Path,
|
|
680
|
+
max_fix_attempts: int = DEFAULT_MAX_FIX_ATTEMPTS,
|
|
681
|
+
max_blocked_rounds: int = DEFAULT_MAX_BLOCKED_ROUNDS,
|
|
682
|
+
pr_verifier: PRVerifier | None = None,
|
|
683
|
+
ci_wait_timeout_seconds: int = 600,
|
|
684
|
+
) -> PipelineResult:
|
|
685
|
+
"""Run dev pipeline for a single issue."""
|
|
686
|
+
session_id = f"dev-{repo.replace('/', '-')}-{issue_number}-{uuid.uuid4().hex[:8]}"
|
|
687
|
+
branch_name = branch_template.replace("{n}", str(issue_number))
|
|
688
|
+
|
|
689
|
+
lock = _RepoLockHandle(state_db, repo, session_id)
|
|
690
|
+
if not state_db.acquire_lock(repo, session_id):
|
|
691
|
+
return PipelineResult(
|
|
692
|
+
success=False,
|
|
693
|
+
session_id=session_id,
|
|
694
|
+
summary=f"Could not acquire lock for {repo}",
|
|
695
|
+
error="Repository locked by another session",
|
|
696
|
+
)
|
|
697
|
+
lock.held = True
|
|
698
|
+
|
|
699
|
+
worktree_path: Path | None = None
|
|
700
|
+
# Ownership signal: True when THIS session created the branch (either
|
|
701
|
+
# from default or via the stale-merged delete+recreate path). False
|
|
702
|
+
# when we reused an existing branch. Stays False until
|
|
703
|
+
# create_worktree_with_new_branch returns so the exception-path
|
|
704
|
+
# cleanup can't accidentally delete a pre-existing branch belonging
|
|
705
|
+
# to another session. Issue #51: a pre-call snapshot of
|
|
706
|
+
# branch_exists_locally goes stale the moment the worktree helper
|
|
707
|
+
# detects a stale fully-merged local branch, deletes it, and
|
|
708
|
+
# recreates it — so the flag has to come out of the helper itself.
|
|
709
|
+
created_fresh = False
|
|
710
|
+
# Pre-call snapshot of branch_exists_locally, used ONLY by the
|
|
711
|
+
# exception-cleanup path to handle partial failures of
|
|
712
|
+
# ``git worktree add -b`` that register the branch ref before the
|
|
713
|
+
# directory setup crashes — in that case the helper raises before
|
|
714
|
+
# returning created_fresh=True, so the snapshot is our only
|
|
715
|
+
# signal that the leftover ref belongs to this session. Pessimistic
|
|
716
|
+
# default (True) so we never delete a branch on a path that
|
|
717
|
+
# didn't even get far enough to probe. The #51 fix (stale snapshot
|
|
718
|
+
# after delete+recreate) doesn't apply here because the
|
|
719
|
+
# delete+recreate path runs entirely INSIDE the helper and only
|
|
720
|
+
# takes effect on successful return, where ``created_fresh``
|
|
721
|
+
# wins over this snapshot.
|
|
722
|
+
branch_existed_before = True
|
|
723
|
+
|
|
724
|
+
try:
|
|
725
|
+
# Get issue details
|
|
726
|
+
issue = await github.get_issue(repo, issue_number)
|
|
727
|
+
|
|
728
|
+
# Post claim comment so collaborators can see the agent picked it up.
|
|
729
|
+
# Skip if a previous attempt already left the marker — keeps it idempotent
|
|
730
|
+
# across retries / resumed sessions.
|
|
731
|
+
existing_comments = issue.get("comments") or []
|
|
732
|
+
already_claimed = any(
|
|
733
|
+
AGENT_CLAIM_MARKER in (c.get("body") or "")
|
|
734
|
+
for c in existing_comments
|
|
735
|
+
)
|
|
736
|
+
if not already_claimed:
|
|
737
|
+
await github.comment_on_issue(
|
|
738
|
+
repo=repo,
|
|
739
|
+
issue_number=issue_number,
|
|
740
|
+
body=AGENT_CLAIM_COMMENT,
|
|
741
|
+
)
|
|
742
|
+
|
|
743
|
+
# Issue #51: ownership is signaled by ``created_fresh`` returned
|
|
744
|
+
# from create_worktree_with_new_branch. True when this call
|
|
745
|
+
# created the branch from default OR went through the
|
|
746
|
+
# stale-merged delete+recreate path — either way THIS session
|
|
747
|
+
# owns the resulting ref and cleanup on failure should delete it.
|
|
748
|
+
#
|
|
749
|
+
# Issue #52: pass the GitHubCLI so the reuse path can refuse a
|
|
750
|
+
# branch that still backs an open PR (prior DONE session whose
|
|
751
|
+
# PR is unmerged, or any external source). Without this, we'd
|
|
752
|
+
# silently check out the reviewer's already-reviewed branch and
|
|
753
|
+
# either hijack the PR or hit "A pull request already exists"
|
|
754
|
+
# on `gh pr create` at the end.
|
|
755
|
+
await worktree.ensure_bare_repo(repo)
|
|
756
|
+
# Snapshot for the exception-path cleanup only (see
|
|
757
|
+
# ``branch_existed_before`` comment at the top of this function).
|
|
758
|
+
# We still need this because ``git worktree add -b`` can register
|
|
759
|
+
# the branch ref before the directory step crashes, and the
|
|
760
|
+
# helper raises before it can return ``created_fresh=True``.
|
|
761
|
+
branch_existed_before = await worktree.branch_exists_locally(
|
|
762
|
+
repo, branch_name,
|
|
763
|
+
)
|
|
764
|
+
worktree_path, created_fresh = await worktree.create_worktree_with_new_branch(
|
|
765
|
+
repo=repo,
|
|
766
|
+
session_id=session_id,
|
|
767
|
+
new_branch=branch_name,
|
|
768
|
+
github=github,
|
|
769
|
+
)
|
|
770
|
+
|
|
771
|
+
# Symlink context
|
|
772
|
+
context_path = contexts_dir / repo.replace("/", "-") / "CLAUDE.md"
|
|
773
|
+
if context_path.exists():
|
|
774
|
+
worktree.symlink_context(worktree_path, context_path)
|
|
775
|
+
|
|
776
|
+
# Setup state file
|
|
777
|
+
state_file = worktree_path / ".ctrlrelay" / "state.json"
|
|
778
|
+
state_file.parent.mkdir(parents=True, exist_ok=True)
|
|
779
|
+
|
|
780
|
+
ctx = PipelineContext(
|
|
781
|
+
session_id=session_id,
|
|
782
|
+
repo=repo,
|
|
783
|
+
worktree_path=worktree_path,
|
|
784
|
+
context_path=context_path,
|
|
785
|
+
state_file=state_file,
|
|
786
|
+
issue_number=issue_number,
|
|
787
|
+
extra={
|
|
788
|
+
"issue_title": issue.get("title", ""),
|
|
789
|
+
"issue_body": issue.get("body", ""),
|
|
790
|
+
"issue_comments": format_issue_comments(issue.get("comments")),
|
|
791
|
+
"branch_name": branch_name,
|
|
792
|
+
"ci_wait_timeout_seconds": ci_wait_timeout_seconds,
|
|
793
|
+
},
|
|
794
|
+
)
|
|
795
|
+
|
|
796
|
+
# Record session
|
|
797
|
+
state_db.execute(
|
|
798
|
+
"""INSERT INTO sessions
|
|
799
|
+
(id, pipeline, repo, worktree_path, status, started_at, issue_number)
|
|
800
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
|
801
|
+
(
|
|
802
|
+
session_id, "dev", repo, str(worktree_path),
|
|
803
|
+
"running", int(time.time()), issue_number,
|
|
804
|
+
),
|
|
805
|
+
)
|
|
806
|
+
state_db.commit()
|
|
807
|
+
|
|
808
|
+
# Run pipeline
|
|
809
|
+
pipeline = DevPipeline(
|
|
810
|
+
dispatcher=dispatcher,
|
|
811
|
+
github=github,
|
|
812
|
+
worktree=worktree,
|
|
813
|
+
dashboard=dashboard,
|
|
814
|
+
state_db=state_db,
|
|
815
|
+
transport=transport,
|
|
816
|
+
)
|
|
817
|
+
result = await pipeline.run(ctx)
|
|
818
|
+
|
|
819
|
+
# BLOCKED loop: if Claude needs input, post the question to the
|
|
820
|
+
# transport (Telegram), wait for the operator's reply, and resume
|
|
821
|
+
# the session. Loop until Claude signals DONE/FAILED or we run out
|
|
822
|
+
# of trips. Each round-trip is capped at the transport's own
|
|
823
|
+
# timeout (see Transport.ask signature — default 300s); the total
|
|
824
|
+
# is bounded by max_blocked_rounds.
|
|
825
|
+
rounds = 0
|
|
826
|
+
while (
|
|
827
|
+
result.blocked
|
|
828
|
+
and transport is not None
|
|
829
|
+
and rounds < max_blocked_rounds
|
|
830
|
+
):
|
|
831
|
+
question = (result.question or "").strip() or (
|
|
832
|
+
f"Session {session_id} is blocked but did not include a "
|
|
833
|
+
"question. Reply with guidance to resume."
|
|
834
|
+
)
|
|
835
|
+
try:
|
|
836
|
+
answer = await transport.ask(
|
|
837
|
+
question,
|
|
838
|
+
session_id=session_id,
|
|
839
|
+
repo=repo,
|
|
840
|
+
issue_number=issue_number,
|
|
841
|
+
)
|
|
842
|
+
except Exception as e:
|
|
843
|
+
# Transport failed (bridge down, timeout, etc.). The
|
|
844
|
+
# session IS still blocked — we just couldn't reach the
|
|
845
|
+
# operator. Preserve blocked=True so the outer
|
|
846
|
+
# persistence-on-blocked branch fires and writes a
|
|
847
|
+
# pending_resumes row. Operator replying later via
|
|
848
|
+
# Telegram then routes through the sweeper. Setting
|
|
849
|
+
# blocked=False here (the prior behavior) wedged the
|
|
850
|
+
# session permanently — codex P1.
|
|
851
|
+
result = PipelineResult(
|
|
852
|
+
success=False,
|
|
853
|
+
blocked=True,
|
|
854
|
+
session_id=session_id,
|
|
855
|
+
summary=f"Blocked session deferred (transport): {e}",
|
|
856
|
+
error=str(e),
|
|
857
|
+
question=question,
|
|
858
|
+
outputs=result.outputs,
|
|
859
|
+
)
|
|
860
|
+
break
|
|
861
|
+
rounds += 1
|
|
862
|
+
result = await pipeline.resume(ctx, answer)
|
|
863
|
+
|
|
864
|
+
# Verify PR is green & conflict-free before handing off. Resume the
|
|
865
|
+
# session with a fix request if either is broken, up to max_fix_attempts.
|
|
866
|
+
#
|
|
867
|
+
# Issue #29: the lock handle is passed in so the verifier releases
|
|
868
|
+
# the repo lock during its CI-polling window (can be up to 30 min)
|
|
869
|
+
# and reacquires before any request_fix. Peer sessions targeting
|
|
870
|
+
# the same repo can now run their own git-op phases while we wait.
|
|
871
|
+
if result.success and result.outputs.get("pr_number") is not None:
|
|
872
|
+
verifier = pr_verifier or PRVerifier(github=github)
|
|
873
|
+
result = await _verify_and_fix_pr(
|
|
874
|
+
pipeline=pipeline,
|
|
875
|
+
ctx=ctx,
|
|
876
|
+
result=result,
|
|
877
|
+
verifier=verifier,
|
|
878
|
+
max_attempts=max_fix_attempts,
|
|
879
|
+
lock_handle=lock,
|
|
880
|
+
)
|
|
881
|
+
|
|
882
|
+
# Reacquire the lock for the post-verify cleanup phase (remove
|
|
883
|
+
# worktree / delete branch). Usually SHORT budget (cleanup is
|
|
884
|
+
# best-effort, run_poll_loop awaits handlers serially).
|
|
885
|
+
#
|
|
886
|
+
# EXCEPT for the verify-contention error: that path signals
|
|
887
|
+
# "PR needs another fix round that we couldn't enter," cli
|
|
888
|
+
# will un-mark the issue for retry, and a fresh run would
|
|
889
|
+
# fail at create_worktree if this one left the worktree
|
|
890
|
+
# registered. Retry success DEPENDS on cleanup here, so use
|
|
891
|
+
# the full fix budget — accepting a possible long wait in
|
|
892
|
+
# this rare double-contention case beats deterministically
|
|
893
|
+
# wedging the issue.
|
|
894
|
+
if not lock.held:
|
|
895
|
+
if result.error == _LOCK_CONTENDED_DURING_VERIFY_ERROR:
|
|
896
|
+
reacquired = await lock.reacquire() # full fix budget
|
|
897
|
+
else:
|
|
898
|
+
reacquired = await lock.reacquire(
|
|
899
|
+
attempts=_REACQUIRE_CLEANUP_ATTEMPTS,
|
|
900
|
+
sleep_seconds=_REACQUIRE_CLEANUP_SLEEP_SECONDS,
|
|
901
|
+
)
|
|
902
|
+
if not reacquired:
|
|
903
|
+
log_event(
|
|
904
|
+
_logger,
|
|
905
|
+
"dev.cleanup.lock_reacquire_contended",
|
|
906
|
+
session_id=session_id,
|
|
907
|
+
repo=repo,
|
|
908
|
+
issue_number=issue_number,
|
|
909
|
+
)
|
|
910
|
+
# Signal to cli.handle_issue that the worktree was NOT
|
|
911
|
+
# torn down so the caller can skip any "unmark for
|
|
912
|
+
# retry" logic. A fresh dev run against this issue
|
|
913
|
+
# would fail at `create_worktree_with_new_branch`
|
|
914
|
+
# because the abandoned worktree is still registered
|
|
915
|
+
# on the bare repo. Requires operator cleanup (or the
|
|
916
|
+
# next poller restart, which prunes stale worktrees).
|
|
917
|
+
merged_outputs = dict(result.outputs)
|
|
918
|
+
merged_outputs["cleanup_deferred"] = True
|
|
919
|
+
result = PipelineResult(
|
|
920
|
+
success=result.success,
|
|
921
|
+
session_id=result.session_id,
|
|
922
|
+
summary=result.summary,
|
|
923
|
+
blocked=result.blocked,
|
|
924
|
+
question=result.question,
|
|
925
|
+
error=result.error,
|
|
926
|
+
outputs=merged_outputs,
|
|
927
|
+
)
|
|
928
|
+
|
|
929
|
+
# Update session status
|
|
930
|
+
status = "done" if result.success else ("blocked" if result.blocked else "failed")
|
|
931
|
+
state_db.execute(
|
|
932
|
+
"UPDATE sessions SET status = ?, summary = ?, ended_at = ? WHERE id = ?",
|
|
933
|
+
(status, result.summary, int(time.time()), session_id),
|
|
934
|
+
)
|
|
935
|
+
state_db.commit()
|
|
936
|
+
|
|
937
|
+
# Persist BLOCKED so a Telegram reply arriving AFTER this session
|
|
938
|
+
# exits (max_blocked_rounds exhausted, or transport unavailable
|
|
939
|
+
# during the session) still routes to a resume. The per-minute
|
|
940
|
+
# pending_resume_sweeper drains these rows by calling
|
|
941
|
+
# resume_dev_from_pending below.
|
|
942
|
+
if result.blocked:
|
|
943
|
+
try:
|
|
944
|
+
state_db.add_pending_resume(
|
|
945
|
+
session_id=session_id,
|
|
946
|
+
pipeline="dev",
|
|
947
|
+
repo=repo,
|
|
948
|
+
question=_question_for_persist(session_id, result),
|
|
949
|
+
)
|
|
950
|
+
except Exception as e:
|
|
951
|
+
log_event(
|
|
952
|
+
_logger,
|
|
953
|
+
"dev.pending_resume.insert_failed",
|
|
954
|
+
session_id=session_id,
|
|
955
|
+
repo=repo,
|
|
956
|
+
issue_number=issue_number,
|
|
957
|
+
error_type=type(e).__name__,
|
|
958
|
+
error=str(e)[:200],
|
|
959
|
+
)
|
|
960
|
+
|
|
961
|
+
# Push event to dashboard
|
|
962
|
+
if dashboard and result.success:
|
|
963
|
+
await dashboard.push_event(EventPayload(
|
|
964
|
+
level="info",
|
|
965
|
+
pipeline="dev",
|
|
966
|
+
repo=repo,
|
|
967
|
+
message=result.summary,
|
|
968
|
+
session_id=session_id,
|
|
969
|
+
details={
|
|
970
|
+
"issue_number": issue_number,
|
|
971
|
+
"pr_number": result.outputs.get("pr_number"),
|
|
972
|
+
},
|
|
973
|
+
))
|
|
974
|
+
|
|
975
|
+
# Cleanup rules:
|
|
976
|
+
# DONE -> remove worktree, keep branch (the open PR references it)
|
|
977
|
+
# BLOCKED -> keep both (user may resume the session)
|
|
978
|
+
# FAILED -> remove worktree AND delete branch so the next retry can
|
|
979
|
+
# re-create `fix/issue-<n>` cleanly — BUT only if this
|
|
980
|
+
# session created the branch (``created_fresh`` — see
|
|
981
|
+
# issue #51) and it was never pushed (no recoverable
|
|
982
|
+
# work on origin).
|
|
983
|
+
#
|
|
984
|
+
# Cleanup mutates the shared bare repo (worktree metadata, branch
|
|
985
|
+
# refs) so it runs only when we hold the lock. If reacquire above
|
|
986
|
+
# failed, skip it and let the next retry / operator reclaim — the
|
|
987
|
+
# verified result is already final either way.
|
|
988
|
+
if lock.held:
|
|
989
|
+
if result.success:
|
|
990
|
+
worktree.remove_context_symlink(worktree_path)
|
|
991
|
+
await worktree.remove_worktree(repo, session_id)
|
|
992
|
+
elif not result.blocked:
|
|
993
|
+
worktree.remove_context_symlink(worktree_path)
|
|
994
|
+
await worktree.remove_worktree(repo, session_id)
|
|
995
|
+
if (
|
|
996
|
+
created_fresh
|
|
997
|
+
and not await worktree.branch_exists_on_remote(
|
|
998
|
+
repo, branch_name
|
|
999
|
+
)
|
|
1000
|
+
):
|
|
1001
|
+
await worktree.delete_branch(repo, branch_name)
|
|
1002
|
+
|
|
1003
|
+
return result
|
|
1004
|
+
|
|
1005
|
+
except Exception as e:
|
|
1006
|
+
state_db.execute(
|
|
1007
|
+
"UPDATE sessions SET status = ?, summary = ?, ended_at = ? WHERE id = ?",
|
|
1008
|
+
("failed", f"Error: {e}", int(time.time()), session_id),
|
|
1009
|
+
)
|
|
1010
|
+
state_db.commit()
|
|
1011
|
+
|
|
1012
|
+
# Best-effort cleanup so a retry isn't blocked by leftover state.
|
|
1013
|
+
# Only touch the branch if THIS session created it (issue #51 —
|
|
1014
|
+
# ``created_fresh``) AND origin has no copy (no recoverable work
|
|
1015
|
+
# to orphan). Covers partial failures of `git worktree add -b`
|
|
1016
|
+
# that create the ref before the directory setup crashes, and
|
|
1017
|
+
# the stale-merged delete+recreate path where the branch now on
|
|
1018
|
+
# disk was put there by this very call. If we're mid-verify the
|
|
1019
|
+
# lock may be released — try to get it back for the cleanup.
|
|
1020
|
+
# Uses the SHORT cleanup budget (not the fix-path one): a
|
|
1021
|
+
# transient exception inside verifier.verify raising through
|
|
1022
|
+
# while a peer holds the lock would otherwise stall the serial
|
|
1023
|
+
# poll cycle for the full hour-long fix budget just to report
|
|
1024
|
+
# the failure.
|
|
1025
|
+
if not lock.held:
|
|
1026
|
+
try:
|
|
1027
|
+
await lock.reacquire(
|
|
1028
|
+
attempts=_REACQUIRE_CLEANUP_ATTEMPTS,
|
|
1029
|
+
sleep_seconds=_REACQUIRE_CLEANUP_SLEEP_SECONDS,
|
|
1030
|
+
)
|
|
1031
|
+
except Exception:
|
|
1032
|
+
pass
|
|
1033
|
+
if lock.held:
|
|
1034
|
+
if worktree_path is not None:
|
|
1035
|
+
try:
|
|
1036
|
+
worktree.remove_context_symlink(worktree_path)
|
|
1037
|
+
except Exception:
|
|
1038
|
+
pass
|
|
1039
|
+
# Always attempt remove_worktree + prune — handles the case where
|
|
1040
|
+
# `git worktree add -b` registered worktree metadata before the dir
|
|
1041
|
+
# step failed, leaving worktree_path unassigned but metadata in the
|
|
1042
|
+
# bare repo that would prevent the branch from being deleted.
|
|
1043
|
+
try:
|
|
1044
|
+
await worktree.remove_worktree(repo, session_id)
|
|
1045
|
+
except Exception:
|
|
1046
|
+
pass
|
|
1047
|
+
# Exception path ownership: three signals, ordered by
|
|
1048
|
+
# reliability.
|
|
1049
|
+
# 1. created_fresh covers the normal return-and-fail case
|
|
1050
|
+
# (issue #51).
|
|
1051
|
+
# 2. StaleRecreatePartialFailureError signals the helper
|
|
1052
|
+
# destroyed the old ref before `git worktree add -b`,
|
|
1053
|
+
# so anything on disk with this branch name is ours
|
|
1054
|
+
# regardless of what the pre-call snapshot said. The
|
|
1055
|
+
# signal is per-call (exception attribute) so
|
|
1056
|
+
# concurrent sessions on the shared WorktreeManager
|
|
1057
|
+
# can't clobber each other's ownership state.
|
|
1058
|
+
# 3. Pre-call snapshot (`not branch_existed_before`) is
|
|
1059
|
+
# the fallback for partial failures of the plain
|
|
1060
|
+
# fresh-branch path where the helper raises before
|
|
1061
|
+
# setting created_fresh.
|
|
1062
|
+
from ctrlrelay.core.worktree import StaleRecreatePartialFailureError
|
|
1063
|
+
stale_recreate_attempted = isinstance(
|
|
1064
|
+
e, StaleRecreatePartialFailureError,
|
|
1065
|
+
)
|
|
1066
|
+
we_own_branch = (
|
|
1067
|
+
created_fresh
|
|
1068
|
+
or stale_recreate_attempted
|
|
1069
|
+
or not branch_existed_before
|
|
1070
|
+
)
|
|
1071
|
+
if we_own_branch:
|
|
1072
|
+
try:
|
|
1073
|
+
has_remote = await worktree.branch_exists_on_remote(
|
|
1074
|
+
repo, branch_name
|
|
1075
|
+
)
|
|
1076
|
+
except Exception:
|
|
1077
|
+
has_remote = True
|
|
1078
|
+
if not has_remote:
|
|
1079
|
+
try:
|
|
1080
|
+
await worktree.delete_branch(repo, branch_name)
|
|
1081
|
+
except Exception:
|
|
1082
|
+
pass
|
|
1083
|
+
|
|
1084
|
+
return PipelineResult(
|
|
1085
|
+
success=False,
|
|
1086
|
+
session_id=session_id,
|
|
1087
|
+
summary=f"Error processing issue #{issue_number}",
|
|
1088
|
+
error=str(e),
|
|
1089
|
+
)
|
|
1090
|
+
|
|
1091
|
+
finally:
|
|
1092
|
+
# Safe even if we already released inside `_verify_and_fix_pr` —
|
|
1093
|
+
# the handle's `release` is idempotent (no-op when held=False)
|
|
1094
|
+
# and StateDB.release_lock is itself idempotent (DELETE with
|
|
1095
|
+
# rowcount=0 is a valid outcome). This also covers
|
|
1096
|
+
# CancelledError: if cancellation arrives during the unlocked
|
|
1097
|
+
# verify window, held is already False, so no stray DELETE runs.
|
|
1098
|
+
lock.release()
|
|
1099
|
+
|
|
1100
|
+
|
|
1101
|
+
async def resume_dev_from_pending(
|
|
1102
|
+
session_id: str,
|
|
1103
|
+
repo: str,
|
|
1104
|
+
answer: str,
|
|
1105
|
+
branch_template: str,
|
|
1106
|
+
dispatcher: AgentAdapter,
|
|
1107
|
+
github: GitHubCLI,
|
|
1108
|
+
worktree: WorktreeManager,
|
|
1109
|
+
dashboard: DashboardClient | None,
|
|
1110
|
+
state_db: StateDB,
|
|
1111
|
+
transport: Transport | None,
|
|
1112
|
+
contexts_dir: Path,
|
|
1113
|
+
max_blocked_rounds: int = DEFAULT_MAX_BLOCKED_ROUNDS,
|
|
1114
|
+
max_fix_attempts: int = DEFAULT_MAX_FIX_ATTEMPTS,
|
|
1115
|
+
pr_verifier: PRVerifier | None = None,
|
|
1116
|
+
) -> PipelineResult:
|
|
1117
|
+
"""Resume a BLOCKED dev session using an answer that arrived via
|
|
1118
|
+
Telegram after ``run_dev_issue`` exited.
|
|
1119
|
+
|
|
1120
|
+
Unlike secops, the dev pipeline keeps its worktree + branch alive on
|
|
1121
|
+
BLOCKED exits (so resume doesn't have to re-clone/re-apply). We try
|
|
1122
|
+
to reuse the stored worktree_path first; if it's missing (manual
|
|
1123
|
+
cleanup, daemon restart lost the mount, etc.) we fall back to
|
|
1124
|
+
``create_worktree_with_new_branch`` which handles existing-branch
|
|
1125
|
+
reuse.
|
|
1126
|
+
|
|
1127
|
+
Caller is responsible for flipping the pending_resumes row
|
|
1128
|
+
``resumed_at`` after this returns — except in the lock-contention
|
|
1129
|
+
case, which is retryable; we surface it via
|
|
1130
|
+
``error == "Repository locked by another session"``.
|
|
1131
|
+
"""
|
|
1132
|
+
session_row = state_db.get_session_row(session_id)
|
|
1133
|
+
if session_row is None:
|
|
1134
|
+
return PipelineResult(
|
|
1135
|
+
success=False,
|
|
1136
|
+
session_id=session_id,
|
|
1137
|
+
summary=(
|
|
1138
|
+
f"Cannot resume dev session {session_id}: row missing "
|
|
1139
|
+
"from sessions table"
|
|
1140
|
+
),
|
|
1141
|
+
error="session_row_missing",
|
|
1142
|
+
)
|
|
1143
|
+
issue_number = session_row.get("issue_number")
|
|
1144
|
+
if issue_number is None:
|
|
1145
|
+
return PipelineResult(
|
|
1146
|
+
success=False,
|
|
1147
|
+
session_id=session_id,
|
|
1148
|
+
summary=(
|
|
1149
|
+
f"Cannot resume dev session {session_id}: issue_number "
|
|
1150
|
+
"missing on sessions row"
|
|
1151
|
+
),
|
|
1152
|
+
error="session_issue_number_missing",
|
|
1153
|
+
)
|
|
1154
|
+
|
|
1155
|
+
branch_name = branch_template.replace("{n}", str(issue_number))
|
|
1156
|
+
|
|
1157
|
+
lock = _RepoLockHandle(state_db, repo, session_id)
|
|
1158
|
+
if not state_db.acquire_lock(repo, session_id):
|
|
1159
|
+
return PipelineResult(
|
|
1160
|
+
success=False,
|
|
1161
|
+
session_id=session_id,
|
|
1162
|
+
summary=f"Could not acquire lock for {repo} to resume dev",
|
|
1163
|
+
error="Repository locked by another session",
|
|
1164
|
+
)
|
|
1165
|
+
lock.held = True
|
|
1166
|
+
|
|
1167
|
+
worktree_path: Path | None = None
|
|
1168
|
+
|
|
1169
|
+
try:
|
|
1170
|
+
await worktree.ensure_bare_repo(repo)
|
|
1171
|
+
|
|
1172
|
+
# Prefer re-using the existing worktree left alive by the BLOCKED
|
|
1173
|
+
# cleanup rule ("BLOCKED → keep both"). That preserves the
|
|
1174
|
+
# session's in-progress commits / uncommitted work without
|
|
1175
|
+
# another fetch round-trip.
|
|
1176
|
+
stored_worktree = session_row.get("worktree_path")
|
|
1177
|
+
if stored_worktree:
|
|
1178
|
+
candidate = Path(stored_worktree)
|
|
1179
|
+
if candidate.exists() and (candidate / ".git").exists():
|
|
1180
|
+
worktree_path = candidate
|
|
1181
|
+
|
|
1182
|
+
if worktree_path is None:
|
|
1183
|
+
# Resume path: the branch should already exist (BLOCKED
|
|
1184
|
+
# sessions preserve both worktree and branch). We discard
|
|
1185
|
+
# ``created_fresh`` here — cleanup semantics don't apply to
|
|
1186
|
+
# resume, which never deletes the branch on exit; the
|
|
1187
|
+
# original run_dev_issue is the single owner of that
|
|
1188
|
+
# decision. Don't pass ``github`` either: an open PR for a
|
|
1189
|
+
# mid-BLOCKED session is possible (claude opened the PR and
|
|
1190
|
+
# blocked on a review question) and refusing reuse would
|
|
1191
|
+
# wedge the resume path.
|
|
1192
|
+
worktree_path, _ = await worktree.create_worktree_with_new_branch(
|
|
1193
|
+
repo=repo,
|
|
1194
|
+
session_id=session_id,
|
|
1195
|
+
new_branch=branch_name,
|
|
1196
|
+
)
|
|
1197
|
+
|
|
1198
|
+
context_path = contexts_dir / repo.replace("/", "-") / "CLAUDE.md"
|
|
1199
|
+
if context_path.exists():
|
|
1200
|
+
worktree.symlink_context(worktree_path, context_path)
|
|
1201
|
+
|
|
1202
|
+
state_file = worktree_path / ".ctrlrelay" / "state.json"
|
|
1203
|
+
state_file.parent.mkdir(parents=True, exist_ok=True)
|
|
1204
|
+
|
|
1205
|
+
# Fetch issue context again so the resumed prompt has the same
|
|
1206
|
+
# metadata the first run did — avoids the agent hallucinating
|
|
1207
|
+
# details from stale state.
|
|
1208
|
+
issue = await github.get_issue(repo, int(issue_number))
|
|
1209
|
+
|
|
1210
|
+
ctx = PipelineContext(
|
|
1211
|
+
session_id=session_id,
|
|
1212
|
+
repo=repo,
|
|
1213
|
+
worktree_path=worktree_path,
|
|
1214
|
+
context_path=context_path,
|
|
1215
|
+
state_file=state_file,
|
|
1216
|
+
issue_number=int(issue_number),
|
|
1217
|
+
extra={
|
|
1218
|
+
"issue_title": issue.get("title", ""),
|
|
1219
|
+
"issue_body": issue.get("body", ""),
|
|
1220
|
+
"issue_comments": format_issue_comments(issue.get("comments")),
|
|
1221
|
+
"branch_name": branch_name,
|
|
1222
|
+
},
|
|
1223
|
+
)
|
|
1224
|
+
|
|
1225
|
+
state_db.execute(
|
|
1226
|
+
"UPDATE sessions SET status = ?, ended_at = NULL WHERE id = ?",
|
|
1227
|
+
("running", session_id),
|
|
1228
|
+
)
|
|
1229
|
+
state_db.commit()
|
|
1230
|
+
|
|
1231
|
+
pipeline = DevPipeline(
|
|
1232
|
+
dispatcher=dispatcher,
|
|
1233
|
+
github=github,
|
|
1234
|
+
worktree=worktree,
|
|
1235
|
+
dashboard=dashboard,
|
|
1236
|
+
state_db=state_db,
|
|
1237
|
+
transport=transport,
|
|
1238
|
+
)
|
|
1239
|
+
|
|
1240
|
+
# Feed the operator's answer as the resume prompt. If the agent
|
|
1241
|
+
# re-blocks (common for multi-step decisions), the same
|
|
1242
|
+
# in-process BLOCKED loop that run_dev_issue uses kicks in here
|
|
1243
|
+
# — transport.ask() waits on the live socket — so the operator
|
|
1244
|
+
# can have a back-and-forth exchange on the resume path too.
|
|
1245
|
+
result = await pipeline.resume(ctx, answer)
|
|
1246
|
+
|
|
1247
|
+
rounds = 0
|
|
1248
|
+
while (
|
|
1249
|
+
result.blocked
|
|
1250
|
+
and transport is not None
|
|
1251
|
+
and rounds < max_blocked_rounds
|
|
1252
|
+
):
|
|
1253
|
+
question = (result.question or "").strip() or (
|
|
1254
|
+
f"Session {session_id} is blocked but did not include a "
|
|
1255
|
+
"question. Reply with guidance to resume."
|
|
1256
|
+
)
|
|
1257
|
+
try:
|
|
1258
|
+
next_answer = await transport.ask(
|
|
1259
|
+
question,
|
|
1260
|
+
session_id=session_id,
|
|
1261
|
+
repo=repo,
|
|
1262
|
+
issue_number=int(issue_number),
|
|
1263
|
+
)
|
|
1264
|
+
except Exception as e:
|
|
1265
|
+
# Keep blocked=True so outer persistence branch fires
|
|
1266
|
+
# and the sweeper can pick this up again when the
|
|
1267
|
+
# operator's next Telegram reply arrives. Same
|
|
1268
|
+
# reasoning as run_dev_issue's loop — transport
|
|
1269
|
+
# failure is not the same as a resolved session.
|
|
1270
|
+
result = PipelineResult(
|
|
1271
|
+
success=False,
|
|
1272
|
+
blocked=True,
|
|
1273
|
+
session_id=session_id,
|
|
1274
|
+
summary=(
|
|
1275
|
+
f"Blocked session deferred during resume "
|
|
1276
|
+
f"(transport): {e}"
|
|
1277
|
+
),
|
|
1278
|
+
error=str(e),
|
|
1279
|
+
question=question,
|
|
1280
|
+
outputs=result.outputs,
|
|
1281
|
+
)
|
|
1282
|
+
break
|
|
1283
|
+
rounds += 1
|
|
1284
|
+
result = await pipeline.resume(ctx, next_answer)
|
|
1285
|
+
|
|
1286
|
+
# Verify PR once the agent says DONE, matching run_dev_issue.
|
|
1287
|
+
# The lock handle is threaded through so the verifier releases
|
|
1288
|
+
# the repo lock during its CI-polling window (issue #29).
|
|
1289
|
+
if result.success and result.outputs.get("pr_number") is not None:
|
|
1290
|
+
verifier = pr_verifier or PRVerifier(github=github)
|
|
1291
|
+
result = await _verify_and_fix_pr(
|
|
1292
|
+
pipeline=pipeline,
|
|
1293
|
+
ctx=ctx,
|
|
1294
|
+
result=result,
|
|
1295
|
+
verifier=verifier,
|
|
1296
|
+
max_attempts=max_fix_attempts,
|
|
1297
|
+
lock_handle=lock,
|
|
1298
|
+
)
|
|
1299
|
+
|
|
1300
|
+
status = "done" if result.success else (
|
|
1301
|
+
"blocked" if result.blocked else "failed"
|
|
1302
|
+
)
|
|
1303
|
+
state_db.execute(
|
|
1304
|
+
"UPDATE sessions SET status = ?, summary = ?, ended_at = ? "
|
|
1305
|
+
"WHERE id = ?",
|
|
1306
|
+
(status, result.summary, int(time.time()), session_id),
|
|
1307
|
+
)
|
|
1308
|
+
state_db.commit()
|
|
1309
|
+
|
|
1310
|
+
if result.blocked:
|
|
1311
|
+
try:
|
|
1312
|
+
state_db.add_pending_resume(
|
|
1313
|
+
session_id=session_id,
|
|
1314
|
+
pipeline="dev",
|
|
1315
|
+
repo=repo,
|
|
1316
|
+
question=_question_for_persist(session_id, result),
|
|
1317
|
+
)
|
|
1318
|
+
except Exception as e:
|
|
1319
|
+
log_event(
|
|
1320
|
+
_logger,
|
|
1321
|
+
"dev.pending_resume.reblock_insert_failed",
|
|
1322
|
+
session_id=session_id,
|
|
1323
|
+
repo=repo,
|
|
1324
|
+
issue_number=issue_number,
|
|
1325
|
+
error_type=type(e).__name__,
|
|
1326
|
+
error=str(e)[:200],
|
|
1327
|
+
)
|
|
1328
|
+
|
|
1329
|
+
return result
|
|
1330
|
+
|
|
1331
|
+
except Exception as e:
|
|
1332
|
+
state_db.execute(
|
|
1333
|
+
"UPDATE sessions SET status = ?, summary = ?, ended_at = ? "
|
|
1334
|
+
"WHERE id = ?",
|
|
1335
|
+
("failed", f"Resume error: {e}", int(time.time()), session_id),
|
|
1336
|
+
)
|
|
1337
|
+
state_db.commit()
|
|
1338
|
+
return PipelineResult(
|
|
1339
|
+
success=False,
|
|
1340
|
+
session_id=session_id,
|
|
1341
|
+
summary=f"Error resuming dev session {session_id} on {repo}",
|
|
1342
|
+
error=str(e),
|
|
1343
|
+
)
|
|
1344
|
+
|
|
1345
|
+
finally:
|
|
1346
|
+
# Intentionally leave the worktree in place — matches
|
|
1347
|
+
# run_dev_issue's "BLOCKED → keep both" contract and makes the
|
|
1348
|
+
# worktree available for a third resume if the second one
|
|
1349
|
+
# re-blocks. A follow-up can add teardown for the DONE/FAILED
|
|
1350
|
+
# outcomes (small disk leak, not a correctness issue; operator
|
|
1351
|
+
# can run `git worktree prune` in the bare repo to reclaim).
|
|
1352
|
+
#
|
|
1353
|
+
# ``lock.release`` is idempotent, so this is a no-op if the
|
|
1354
|
+
# verify phase already released and never reacquired (CI-only
|
|
1355
|
+
# path with no fix attempts + no cleanup work). Matches issue
|
|
1356
|
+
# #29's release-during-verify semantics.
|
|
1357
|
+
try:
|
|
1358
|
+
lock.release()
|
|
1359
|
+
except Exception as lock_exc:
|
|
1360
|
+
log_event(
|
|
1361
|
+
_logger,
|
|
1362
|
+
"dev.resume.cleanup.lock_release_failed",
|
|
1363
|
+
session_id=session_id,
|
|
1364
|
+
repo=repo,
|
|
1365
|
+
error_type=type(lock_exc).__name__,
|
|
1366
|
+
error=str(lock_exc)[:200],
|
|
1367
|
+
)
|