ctrlrelay 0.10.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. ctrlrelay/__init__.py +33 -0
  2. ctrlrelay/bridge/__init__.py +21 -0
  3. ctrlrelay/bridge/__main__.py +89 -0
  4. ctrlrelay/bridge/protocol.py +75 -0
  5. ctrlrelay/bridge/server.py +740 -0
  6. ctrlrelay/bridge/telegram_handler.py +136 -0
  7. ctrlrelay/cli.py +3138 -0
  8. ctrlrelay/core/__init__.py +54 -0
  9. ctrlrelay/core/audit.py +257 -0
  10. ctrlrelay/core/checkpoint.py +155 -0
  11. ctrlrelay/core/config.py +778 -0
  12. ctrlrelay/core/dispatcher.py +372 -0
  13. ctrlrelay/core/github.py +432 -0
  14. ctrlrelay/core/network.py +277 -0
  15. ctrlrelay/core/obs.py +118 -0
  16. ctrlrelay/core/poller.py +920 -0
  17. ctrlrelay/core/pr_verifier.py +230 -0
  18. ctrlrelay/core/pr_watcher.py +121 -0
  19. ctrlrelay/core/question_expiry.py +193 -0
  20. ctrlrelay/core/scheduler.py +337 -0
  21. ctrlrelay/core/state.py +642 -0
  22. ctrlrelay/core/worktree.py +893 -0
  23. ctrlrelay/dashboard/__init__.py +5 -0
  24. ctrlrelay/dashboard/client.py +159 -0
  25. ctrlrelay/gh_protocol.py +47 -0
  26. ctrlrelay/install.py +203 -0
  27. ctrlrelay/personalization/__init__.py +26 -0
  28. ctrlrelay/personalization/manager.py +1183 -0
  29. ctrlrelay/personalization/paths.py +143 -0
  30. ctrlrelay/pipelines/__init__.py +15 -0
  31. ctrlrelay/pipelines/base.py +50 -0
  32. ctrlrelay/pipelines/dev.py +1367 -0
  33. ctrlrelay/pipelines/post_merge.py +411 -0
  34. ctrlrelay/pipelines/secops.py +1018 -0
  35. ctrlrelay/pipelines/task.py +713 -0
  36. ctrlrelay/setup.py +617 -0
  37. ctrlrelay/templates/__init__.py +0 -0
  38. ctrlrelay/templates/global-CLAUDE.md.snippet +61 -0
  39. ctrlrelay/templates/launchd/__init__.py +0 -0
  40. ctrlrelay/templates/launchd/bridge.plist.template +37 -0
  41. ctrlrelay/templates/launchd/poller.plist.template +44 -0
  42. ctrlrelay/templates/systemd/__init__.py +0 -0
  43. ctrlrelay/templates/systemd/bridge.service.template +20 -0
  44. ctrlrelay/templates/systemd/poller.service.template +25 -0
  45. ctrlrelay/transports/__init__.py +34 -0
  46. ctrlrelay/transports/base.py +74 -0
  47. ctrlrelay/transports/file_mock.py +103 -0
  48. ctrlrelay/transports/socket_client.py +298 -0
  49. ctrlrelay-0.10.0.dist-info/METADATA +254 -0
  50. ctrlrelay-0.10.0.dist-info/RECORD +53 -0
  51. ctrlrelay-0.10.0.dist-info/WHEEL +4 -0
  52. ctrlrelay-0.10.0.dist-info/entry_points.txt +2 -0
  53. ctrlrelay-0.10.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,1367 @@
1
+ """Dev pipeline for issue-to-PR workflow."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import time
7
+ import uuid
8
+ from dataclasses import dataclass
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ from ctrlrelay.core.checkpoint import CheckpointStatus
13
+ from ctrlrelay.core.dispatcher import AgentAdapter, SessionResult
14
+ from ctrlrelay.core.github import GitHubCLI
15
+ from ctrlrelay.core.obs import get_logger, hash_text, log_event
16
+ from ctrlrelay.core.pr_verifier import PRVerifier, VerificationResult
17
+ from ctrlrelay.core.state import StateDB
18
+ from ctrlrelay.core.worktree import WorktreeManager
19
+ from ctrlrelay.dashboard.client import DashboardClient, EventPayload
20
+ from ctrlrelay.pipelines.base import PipelineContext, PipelineResult
21
+ from ctrlrelay.transports.base import Transport
22
+
23
+ DEFAULT_MAX_FIX_ATTEMPTS = 3
24
+ DEFAULT_MAX_BLOCKED_ROUNDS = 5
25
+
26
+ # Reacquire policy for the repo lock after the unlocked verification phase.
27
+ # Verification releases the lock because CI polling doesn't touch git; when
28
+ # we need to run `request_fix` (which spawns claude + writes worktree) we
29
+ # must get it back. Contention here is NOT necessarily brief — a peer
30
+ # session may be running its own full claude pass on another issue in the
31
+ # same repo, which can take tens of minutes. A short retry budget would
32
+ # spuriously fail healthy same-repo parallelism; bound the wait at roughly
33
+ # one hour, long enough to outlast a normal peer run and short enough to
34
+ # escape a stuck/SIGKILL'd peer whose lock row is lingering.
35
+ # Tests monkey-patch these to speed contention scenarios without changing
36
+ # the call sites.
37
+ _REACQUIRE_LOCK_ATTEMPTS = 720
38
+ _REACQUIRE_LOCK_SLEEP_SECONDS = 5.0
39
+ # Log progress periodically so an operator watching can see we're alive
40
+ # and waiting, not hung. At 5s cadence this fires ~every 5 minutes.
41
+ _REACQUIRE_PROGRESS_LOG_EVERY = 60
42
+
43
+ # Post-verify CLEANUP (worktree rm, branch delete) is best-effort — if
44
+ # a peer grabs the lock while we're polling CI and we can't get it back
45
+ # fast, skipping cleanup is fine (next run / operator reclaims). Don't
46
+ # use the fix-path budget here: run_poll_loop awaits handlers serially,
47
+ # so waiting an hour for cleanup would stall the whole poll cycle AND
48
+ # delay the merge-watcher spawn that happens after run_dev_issue
49
+ # returns. Three quick retries cover a fleeting hiccup; anything longer
50
+ # belongs elsewhere.
51
+ _REACQUIRE_CLEANUP_ATTEMPTS = 3
52
+ _REACQUIRE_CLEANUP_SLEEP_SECONDS = 1.0
53
+
54
+ # Release-lock retry policy for transient SQLite errors. The final
55
+ # release() call in run_dev_issue's outer finally is the last chance
56
+ # to clean up; if that raises and we silently returned, the repo_locks
57
+ # row would linger forever. Short retry loop handles the common
58
+ # transient case; exhausted retries emit a loud wedged event so ops
59
+ # can reclaim manually.
60
+ _RELEASE_RETRY_ATTEMPTS = 3
61
+ _RELEASE_RETRY_SLEEP_SECONDS = 0.5
62
+
63
+ # Two distinct error messages so callers can tell which phase of the
64
+ # pipeline hit contention:
65
+ # INITIAL — nothing started, safe to retry from scratch (cli un-marks
66
+ # the issue so the next poll picks it up).
67
+ # DURING_VERIFY — a PR already exists; re-running from scratch would
68
+ # launch a duplicate dev pass against the open PR.
69
+ # cli leaves the issue marked seen and only spawns the
70
+ # PR merge watcher so the existing PR still auto-closes
71
+ # on merge.
72
+ _LOCK_CONTENDED_ERROR = "Repository locked by another session"
73
+ _LOCK_CONTENDED_DURING_VERIFY_ERROR = (
74
+ "Repo lock reacquire contended during PR verification"
75
+ )
76
+
77
+
78
+ class _RepoLockHandle:
79
+ """Tracks whether this session currently holds the repo lock and
80
+ allows release/reacquire around phases that don't need exclusive git
81
+ access (PR CI verification is pure `gh` polling — see issue #29).
82
+ The ``finally`` block at the end of ``run_dev_issue`` calls
83
+ ``release`` regardless; calling it twice is a no-op because
84
+ ``StateDB.release_lock`` is idempotent (deletes rows, rowcount=0 the
85
+ second time). Tracking ``held`` locally lets the pipeline decide
86
+ whether worktree/branch cleanup is safe to attempt on error paths."""
87
+
88
+ __slots__ = ("state_db", "repo", "session_id", "held")
89
+
90
+ def __init__(self, state_db: StateDB, repo: str, session_id: str) -> None:
91
+ self.state_db = state_db
92
+ self.repo = repo
93
+ self.session_id = session_id
94
+ # Starts False; run_dev_issue wraps its initial acquire call and
95
+ # flips this to True on success.
96
+ self.held = False
97
+
98
+ def release(self) -> None:
99
+ """Release the lock if we hold it. Idempotent and safe to call
100
+ from a ``finally`` block that may run before acquire ever
101
+ succeeded.
102
+
103
+ Retries the DELETE a few times on transient failures (SQLite
104
+ ``database is locked``, etc.). The outer ``finally`` is the
105
+ last-chance call — if THAT raised and we silently returned,
106
+ the ``repo_locks`` row would linger forever, wedging every
107
+ future acquire on this repo until manual DB cleanup.
108
+
109
+ After exhausting retries we emit a ``dev.lock.release_wedged``
110
+ critical-level event naming the repo + session so an operator
111
+ can reclaim the lock row. held is left True so an outer
112
+ retry (if any) still has a chance.
113
+
114
+ The inner retry's exception is swallowed only after the
115
+ wedged event fires — release must not leak exceptions from a
116
+ finally path where it would mask the caller's original error."""
117
+ if not self.held:
118
+ return
119
+ last_exc: Exception | None = None
120
+ for attempt in range(1, _RELEASE_RETRY_ATTEMPTS + 1):
121
+ try:
122
+ self.state_db.release_lock(self.repo, self.session_id)
123
+ except Exception as e:
124
+ last_exc = e
125
+ log_event(
126
+ _logger,
127
+ "dev.lock.release_failed",
128
+ session_id=self.session_id,
129
+ repo=self.repo,
130
+ attempt=attempt,
131
+ max_attempts=_RELEASE_RETRY_ATTEMPTS,
132
+ reason=type(e).__name__,
133
+ error=str(e)[:200],
134
+ )
135
+ if attempt < _RELEASE_RETRY_ATTEMPTS:
136
+ time.sleep(_RELEASE_RETRY_SLEEP_SECONDS)
137
+ continue
138
+ self.held = False
139
+ return
140
+ # All retries exhausted. The row is presumed still present;
141
+ # alert loudly so it doesn't silently wedge the repo.
142
+ log_event(
143
+ _logger,
144
+ "dev.lock.release_wedged",
145
+ session_id=self.session_id,
146
+ repo=self.repo,
147
+ reason=type(last_exc).__name__ if last_exc else "unknown",
148
+ error=str(last_exc)[:200] if last_exc else "",
149
+ operator_action=(
150
+ "DELETE FROM repo_locks WHERE repo = ? AND session_id = ? "
151
+ "manually, or restart the poller after confirming the "
152
+ "session is not live."
153
+ ),
154
+ )
155
+
156
+ async def reacquire(
157
+ self,
158
+ *,
159
+ attempts: int | None = None,
160
+ sleep_seconds: float | None = None,
161
+ ) -> bool:
162
+ """Try to re-grab the lock after a release. Returns True on
163
+ success, False if contention persists past the retry budget.
164
+
165
+ Retry budget: ``attempts`` INSERT tries separated by
166
+ ``sleep_seconds`` of sleep. Default 720×5s ≈ 1 hour. That's
167
+ long enough to outlast a peer running its own full dev pass
168
+ on another issue in the same repo (claude + CI can take
169
+ tens of minutes), and short enough to escape a stuck or
170
+ SIGKILL'd peer whose lock row never got cleaned up.
171
+
172
+ Defaults resolve at call time (``None`` sentinel) so tests can
173
+ monkey-patch the module constants to speed up contention
174
+ scenarios without threading kwargs through every caller.
175
+ """
176
+ if self.held:
177
+ return True
178
+ effective_attempts = (
179
+ _REACQUIRE_LOCK_ATTEMPTS if attempts is None else attempts
180
+ )
181
+ effective_sleep = (
182
+ _REACQUIRE_LOCK_SLEEP_SECONDS
183
+ if sleep_seconds is None
184
+ else sleep_seconds
185
+ )
186
+ for attempt in range(max(1, effective_attempts)):
187
+ if self.state_db.acquire_lock(self.repo, self.session_id):
188
+ self.held = True
189
+ return True
190
+ # Surface a heartbeat every N attempts so an operator
191
+ # tailing logs during a long peer hold can tell the
192
+ # session is waiting, not wedged.
193
+ if attempt > 0 and attempt % _REACQUIRE_PROGRESS_LOG_EVERY == 0:
194
+ log_event(
195
+ _logger,
196
+ "dev.lock.reacquire_waiting",
197
+ session_id=self.session_id,
198
+ repo=self.repo,
199
+ attempt=attempt,
200
+ max_attempts=effective_attempts,
201
+ )
202
+ if attempt < effective_attempts - 1:
203
+ await asyncio.sleep(effective_sleep)
204
+ return False
205
+
206
+ AGENT_CLAIM_MARKER = "<!-- ctrlrelay:claimed -->"
207
+ AGENT_CLAIM_COMMENT = (
208
+ "CTRLRelay is working on this issue. A PR will be opened for review.\n\n"
209
+ f"{AGENT_CLAIM_MARKER}"
210
+ )
211
+
212
+ # Discussion budget for the prompt. Newest comments are kept because a
213
+ # redirect is far likelier to be the latest word than the first reply,
214
+ # and each body is clipped so one pasted log dump can't crowd out the
215
+ # rest of the thread.
216
+ MAX_PROMPT_COMMENTS = 20
217
+ MAX_PROMPT_COMMENT_CHARS = 2000
218
+
219
+ _logger = get_logger("pipeline.dev")
220
+
221
+
222
+ def format_issue_comments(
223
+ comments: list[dict[str, Any]] | None,
224
+ *,
225
+ max_comments: int = MAX_PROMPT_COMMENTS,
226
+ max_chars: int = MAX_PROMPT_COMMENT_CHARS,
227
+ ) -> str:
228
+ """Render the issue discussion for the dev prompt.
229
+
230
+ The body states the problem as it was first filed. The thread is
231
+ where an operator narrows, redirects or outright vetoes that plan,
232
+ and nobody edits the body afterwards. A vulnq issue was filed asking
233
+ for a client fix and then redirected in a comment to price up
234
+ dropping the data source instead; the prompt carried only the body,
235
+ so the agent spent a full session building the plan that had already
236
+ been rejected in writing.
237
+
238
+ Drops our own claim comment — it is our noise, not the operator's.
239
+ Keeps the newest ``max_comments`` but renders them oldest-first so
240
+ the thread still reads in order.
241
+ """
242
+ if not comments:
243
+ return "_No comments._"
244
+
245
+ kept = [
246
+ c for c in comments
247
+ if AGENT_CLAIM_MARKER not in (c.get("body") or "")
248
+ ]
249
+ if not kept:
250
+ return "_No comments._"
251
+
252
+ dropped = len(kept) - max_comments
253
+ if dropped > 0:
254
+ kept = kept[-max_comments:]
255
+
256
+ blocks: list[str] = []
257
+ if dropped > 0:
258
+ blocks.append(
259
+ f"_({dropped} earlier comment(s) omitted; read the issue on "
260
+ f"GitHub if the thread below references them.)_"
261
+ )
262
+
263
+ for c in kept:
264
+ author = (c.get("author") or {}).get("login") or "unknown"
265
+ created = c.get("createdAt") or ""
266
+ body = (c.get("body") or "").strip() or "_(empty)_"
267
+ if len(body) > max_chars:
268
+ body = body[:max_chars] + "\n_[...truncated]_"
269
+ blocks.append(f"--- @{author} ({created}) ---\n{body}")
270
+
271
+ return "\n\n".join(blocks)
272
+
273
+
274
+ def _question_for_persist(session_id: str, result: PipelineResult) -> str:
275
+ """Return a non-empty question string for pending_resumes storage.
276
+
277
+ Mirrors the synthesized prompt the in-process BLOCKED loop uses
278
+ when ``result.question`` is empty — without this, a BLOCKED exit
279
+ with no question text would skip persistence and the session
280
+ would become unresumable via Telegram."""
281
+ q = (result.question or "").strip()
282
+ if q:
283
+ return q
284
+ return (
285
+ f"Session {session_id} is blocked but did not include a "
286
+ "question. Reply with guidance to resume."
287
+ )
288
+
289
+
290
+ @dataclass
291
+ class DevPipeline:
292
+ """Dev pipeline for implementing issues and opening PRs."""
293
+
294
+ dispatcher: AgentAdapter
295
+ github: GitHubCLI
296
+ worktree: WorktreeManager
297
+ dashboard: DashboardClient | None
298
+ state_db: StateDB
299
+ transport: Transport | None
300
+
301
+ name: str = "dev"
302
+
303
+ async def run(self, ctx: PipelineContext) -> PipelineResult:
304
+ """Run dev pipeline on a single issue."""
305
+ prompt = self._build_prompt(
306
+ ctx.repo, ctx.issue_number, ctx.extra,
307
+ session_id=ctx.session_id,
308
+ state_file=ctx.state_file,
309
+ )
310
+
311
+ result = await self._spawn(ctx, prompt, resume=False)
312
+ return self._session_to_result(result)
313
+
314
+ async def resume(self, ctx: PipelineContext, answer: str) -> PipelineResult:
315
+ """Resume blocked dev session with user answer."""
316
+ prompt = f"User answered: {answer}\n\nContinue from where you left off."
317
+
318
+ # Resume uses Claude's own session UUID (captured on the first spawn
319
+ # and persisted to state_db). Passing our composite id here makes
320
+ # `claude --resume` reject the call on v2.0.x+ CLI builds.
321
+ resume_uuid = self.state_db.get_agent_session_id(ctx.session_id)
322
+
323
+ log_event(
324
+ _logger,
325
+ "dev.session.resumed",
326
+ session_id=ctx.session_id,
327
+ repo=ctx.repo,
328
+ issue_number=ctx.issue_number,
329
+ pipeline=self.name,
330
+ resume_session_id=resume_uuid,
331
+ answer_length=len(answer),
332
+ answer_hash=hash_text(answer),
333
+ )
334
+
335
+ result = await self._spawn(ctx, prompt, resume=True)
336
+ return self._session_to_result(result)
337
+
338
+ async def request_fix(
339
+ self, ctx: PipelineContext, fix_instructions: str
340
+ ) -> PipelineResult:
341
+ """Resume the session with a fix request (failing CI or merge conflict)."""
342
+ result = await self._spawn(ctx, fix_instructions, resume=True)
343
+ return self._session_to_result(result)
344
+
345
+ async def _spawn(
346
+ self,
347
+ ctx: PipelineContext,
348
+ prompt: str,
349
+ *,
350
+ resume: bool,
351
+ ) -> SessionResult:
352
+ """Centralized dispatcher call: looks up the agent UUID on resume,
353
+ persists any newly-captured UUID to state_db."""
354
+ resume_uuid: str | None = None
355
+ if resume:
356
+ # If state_db has no stored UUID (session predates the capture, or
357
+ # first spawn didn't emit JSON) we fall back to a fresh session —
358
+ # better than hard-failing with `claude --resume <composite-id>`.
359
+ resume_uuid = self.state_db.get_agent_session_id(ctx.session_id)
360
+
361
+ result = await self.dispatcher.spawn_session(
362
+ session_id=ctx.session_id,
363
+ prompt=prompt,
364
+ working_dir=ctx.worktree_path,
365
+ state_file=ctx.state_file,
366
+ resume_session_id=resume_uuid,
367
+ repo=ctx.repo,
368
+ issue_number=ctx.issue_number,
369
+ )
370
+
371
+ if result.agent_session_id:
372
+ try:
373
+ self.state_db.set_agent_session_id(
374
+ ctx.session_id, result.agent_session_id
375
+ )
376
+ except Exception:
377
+ # state_db write failures must not mask the session result.
378
+ pass
379
+
380
+ return result
381
+
382
+ def _build_prompt(
383
+ self,
384
+ repo: str,
385
+ issue_number: int | None,
386
+ extra: dict[str, Any],
387
+ session_id: str = "",
388
+ state_file: Path | None = None,
389
+ ) -> str:
390
+ """Build the dev pipeline prompt."""
391
+ issue_title = extra.get("issue_title", "")
392
+ issue_body = extra.get("issue_body", "")
393
+ issue_comments = extra.get("issue_comments") or "_No comments._"
394
+ branch_name = extra.get("branch_name", "")
395
+ ci_wait_timeout_seconds = extra.get("ci_wait_timeout_seconds", 600)
396
+ state_file_path = str(state_file) if state_file else "/tmp/state.json"
397
+
398
+ return f"""You are working on issue #{issue_number} in repository {repo}.
399
+
400
+ **Issue Title:** {issue_title}
401
+
402
+ **Issue Body:**
403
+ {issue_body}
404
+
405
+ **Discussion:**
406
+ {issue_comments}
407
+
408
+ The body is the issue as first filed; the discussion is newer. Where a
409
+ comment narrows, redirects or rejects what the body asks for, the comment
410
+ wins — nobody goes back and edits the body after changing their mind in
411
+ the thread. If the discussion contradicts the body in a way you cannot
412
+ reconcile, signal BLOCKED and ask rather than picking one.
413
+
414
+ **Branch:** {branch_name}
415
+
416
+ Execute the following workflow:
417
+
418
+ 1. Validate the issue still applies to the current codebase
419
+ 2. If anything is unclear, signal BLOCKED (see below) to ask for clarification
420
+ 3. Plan and implement the fix using TDD
421
+ 4. Push the branch and open a PR that references the issue
422
+ 5. Before signaling DONE, verify the PR is mergeable:
423
+ - Wait for CI by running EXACTLY this command (do NOT improvise bash):
424
+ `ctrlrelay ci wait --pr <PR> --repo {repo} --timeout {ci_wait_timeout_seconds}`
425
+ Exit codes: 0 = all checks passed, 1 = a check failed (investigate,
426
+ fix, push, then re-run the wait), 2 = hard timeout while CI is still
427
+ pending (treat as acceptable — hand off and let the orchestrator
428
+ re-verify). Do NOT write your own `until` / `while` loops around
429
+ `gh pr checks`; those have been miswritten in the past (inverted
430
+ semantics, pipes swallowing exit codes) and burned the whole session
431
+ timeout on PRs that were already green.
432
+ - Run `gh pr view <PR> --json mergeable,mergeStateStatus` — if `mergeable`
433
+ is `CONFLICTING` or `mergeStateStatus` is `DIRTY`, rebase onto the base
434
+ branch, resolve conflicts, and push again.
435
+ 6. Signal DONE only when the PR is green AND conflict-free, with the PR URL.
436
+
437
+ Do NOT merge the PR - wait for human review.
438
+
439
+ The orchestrator re-verifies CI and mergeability after you hand off; if it
440
+ finds the PR broken it will resume this session asking you to fix it.
441
+
442
+ ## Signaling Completion
443
+
444
+ **CRITICAL**: Before exiting, you MUST write a checkpoint file to signal completion.
445
+
446
+ STATE_FILE: {state_file_path}
447
+ SESSION_ID: {session_id}
448
+
449
+ **DONE** (PR opened AND verified green + conflict-free):
450
+ ```bash
451
+ mkdir -p "$(dirname '{state_file_path}')"
452
+ printf '{{"version":"1","status":"DONE","session_id":"{session_id}",'\
453
+ '"timestamp":"%s","summary":"PR opened",'\
454
+ '"outputs":{{"pr_url":"%s","pr_number":%d}}}}' \\
455
+ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "<PR_URL>" <PR_NUM> > '{state_file_path}'
456
+ ```
457
+
458
+ **BLOCKED** (need input):
459
+ ```bash
460
+ mkdir -p "$(dirname '{state_file_path}')"
461
+ printf '{{"version":"1","status":"BLOCKED_NEEDS_INPUT",'\
462
+ '"session_id":"{session_id}","timestamp":"%s","question":"%s"}}' \\
463
+ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "<QUESTION>" > '{state_file_path}'
464
+ ```
465
+
466
+ **FAILED**:
467
+ ```bash
468
+ mkdir -p "$(dirname '{state_file_path}')"
469
+ printf '{{"version":"1","status":"FAILED","session_id":"{session_id}",'\
470
+ '"timestamp":"%s","error":"%s"}}' \\
471
+ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "<ERROR>" > '{state_file_path}'
472
+ ```"""
473
+
474
+ def _session_to_result(self, result: SessionResult) -> PipelineResult:
475
+ """Convert SessionResult to PipelineResult."""
476
+ if result.state is None:
477
+ return PipelineResult(
478
+ success=False,
479
+ session_id=result.session_id,
480
+ summary="No checkpoint state returned",
481
+ error=result.stderr or "Unknown error",
482
+ )
483
+
484
+ if result.state.status == CheckpointStatus.DONE:
485
+ return PipelineResult(
486
+ success=True,
487
+ session_id=result.session_id,
488
+ summary=result.state.summary or "Completed",
489
+ outputs=result.state.outputs,
490
+ )
491
+
492
+ if result.state.status == CheckpointStatus.BLOCKED_NEEDS_INPUT:
493
+ return PipelineResult(
494
+ success=False,
495
+ session_id=result.session_id,
496
+ summary="Blocked on user input",
497
+ blocked=True,
498
+ question=result.state.question,
499
+ )
500
+
501
+ return PipelineResult(
502
+ success=False,
503
+ session_id=result.session_id,
504
+ summary="Failed",
505
+ error=result.state.error,
506
+ )
507
+
508
+
509
+ def _build_fix_prompt(pr_number: int, verification: VerificationResult) -> str:
510
+ """Build a resume-prompt asking Claude to fix CI failures or merge conflicts."""
511
+ if verification.mergeable == "CONFLICTING":
512
+ return (
513
+ f"PR #{pr_number} has merge conflicts with the base branch "
514
+ f"(mergeStateStatus={verification.merge_state_status}). "
515
+ "Rebase the branch onto the base, resolve the conflicts, push the "
516
+ "updated branch, then signal DONE with the same "
517
+ f'outputs (pr_url, pr_number={pr_number}) once the PR reports '
518
+ "MERGEABLE. Signal FAILED if conflicts cannot be resolved."
519
+ )
520
+
521
+ if verification.merge_state_status == "BEHIND":
522
+ return (
523
+ f"PR #{pr_number} is behind the base branch and base-branch "
524
+ "protection requires it to be up-to-date before merge "
525
+ "(mergeStateStatus=BEHIND). Rebase the branch onto the base and "
526
+ "force-push, then signal DONE with the same "
527
+ f"outputs (pr_url, pr_number={pr_number}) once the PR reports "
528
+ "mergeStateStatus=CLEAN."
529
+ )
530
+
531
+ if verification.failing_checks:
532
+ names = ", ".join(
533
+ f"{c.get('name', '?')}={c.get('state') or c.get('bucket')}"
534
+ for c in verification.failing_checks
535
+ )
536
+ return (
537
+ f"PR #{pr_number} has failing or incomplete CI checks: {names}. "
538
+ "Investigate the failures (fetch logs via `gh run view` as needed), "
539
+ "fix the underlying issues, commit and push, then wait for CI to go "
540
+ "green. Call checkpoint.done() with the same outputs once all checks "
541
+ "pass and the PR is MERGEABLE."
542
+ )
543
+
544
+ return (
545
+ f"PR #{pr_number} is not ready to hand off: {verification.reason}. "
546
+ "Investigate, push any required fixes, and call checkpoint.done() once "
547
+ "CI is green and the PR is MERGEABLE."
548
+ )
549
+
550
+
551
+ async def _verify_and_fix_pr(
552
+ *,
553
+ pipeline: DevPipeline,
554
+ ctx: PipelineContext,
555
+ result: PipelineResult,
556
+ verifier: PRVerifier,
557
+ max_attempts: int,
558
+ lock_handle: _RepoLockHandle | None = None,
559
+ ) -> PipelineResult:
560
+ """Loop: verify CI+mergeability, ask Claude to fix, re-verify.
561
+
562
+ Lock discipline (issue #29): the verify phase is pure `gh` polling
563
+ and holds no git state, so callers pass a ``lock_handle`` whose
564
+ repo lock we release before each ``verifier.verify`` call and
565
+ reacquire before each ``request_fix`` call (which spawns claude
566
+ and writes to the worktree). This lets peer sessions targeting
567
+ the same repo run their own git-op phases while we wait on CI.
568
+
569
+ If reacquire fails after the retry budget, we surface a clear
570
+ failed result rather than silently running ``request_fix``
571
+ without the lock — another session may be mutating the shared
572
+ bare repo and our claude process could corrupt its state.
573
+ Cancellation during the unlocked phase is safe: the lock_handle
574
+ tracks ``held=False`` so the outer ``finally`` no-ops its release.
575
+ """
576
+ pr_number_raw = result.outputs.get("pr_number")
577
+ if pr_number_raw is None:
578
+ return result
579
+ pr_number = int(pr_number_raw)
580
+
581
+ async def _run_verify() -> VerificationResult:
582
+ """Release the lock around the pure-gh polling window, then
583
+ hand the lock back to the caller on return. Reacquire is
584
+ deferred to just before the fix path so peer sessions get the
585
+ widest possible window."""
586
+ if lock_handle is not None:
587
+ lock_handle.release()
588
+ try:
589
+ return await verifier.verify(ctx.repo, pr_number)
590
+ except asyncio.CancelledError:
591
+ # Intentionally leave held=False so the outer finally's
592
+ # release_lock is a no-op; the row is already gone. Re-raise
593
+ # to let the cancellation propagate.
594
+ raise
595
+
596
+ verification = await _run_verify()
597
+ # If CI is simply slow (timed_out) we hand the PR off rather than asking
598
+ # Claude to "fix" something that isn't broken.
599
+ if verification.timed_out:
600
+ return result
601
+ attempts = 0
602
+ while not verification.ready and attempts < max_attempts:
603
+ # Reacquire before request_fix — claude spawns inside the
604
+ # worktree and pushes to the shared bare repo. If we can't
605
+ # get the lock back in a reasonable window, bail cleanly
606
+ # rather than racing another session.
607
+ if lock_handle is not None and not lock_handle.held:
608
+ reacquired = await lock_handle.reacquire()
609
+ if not reacquired:
610
+ log_event(
611
+ _logger,
612
+ "dev.verify.lock_reacquire_contended",
613
+ session_id=result.session_id,
614
+ repo=ctx.repo,
615
+ pr_number=pr_number,
616
+ attempts=_REACQUIRE_LOCK_ATTEMPTS,
617
+ )
618
+ return PipelineResult(
619
+ success=False,
620
+ session_id=result.session_id,
621
+ summary=(
622
+ f"PR #{pr_number} verification found work to do "
623
+ "but could not re-acquire repo lock after "
624
+ f"{_REACQUIRE_LOCK_ATTEMPTS} attempts"
625
+ ),
626
+ error=_LOCK_CONTENDED_DURING_VERIFY_ERROR,
627
+ outputs=result.outputs,
628
+ )
629
+
630
+ fix_prompt = _build_fix_prompt(pr_number, verification)
631
+ fix_result = await pipeline.request_fix(ctx, fix_prompt)
632
+ attempts += 1
633
+
634
+ if not fix_result.success:
635
+ # Preserve the original PR info in outputs so callers can still find it.
636
+ merged_outputs = dict(result.outputs)
637
+ merged_outputs.update(fix_result.outputs)
638
+ return PipelineResult(
639
+ success=False,
640
+ session_id=fix_result.session_id,
641
+ summary=fix_result.summary,
642
+ blocked=fix_result.blocked,
643
+ question=fix_result.question,
644
+ error=fix_result.error,
645
+ outputs=merged_outputs,
646
+ )
647
+
648
+ result = fix_result
649
+ verification = await _run_verify()
650
+ if verification.timed_out:
651
+ # Same rule after a fix round: slow CI isn't a Claude task.
652
+ return result
653
+
654
+ if verification.ready:
655
+ return result
656
+
657
+ return PipelineResult(
658
+ success=False,
659
+ session_id=result.session_id,
660
+ summary=(
661
+ f"PR #{pr_number} not ready after {max_attempts} fix attempt(s): "
662
+ f"{verification.reason}"
663
+ ),
664
+ error=verification.reason,
665
+ outputs=result.outputs,
666
+ )
667
+
668
+
669
+ async def run_dev_issue(
670
+ repo: str,
671
+ issue_number: int,
672
+ branch_template: str,
673
+ dispatcher: AgentAdapter,
674
+ github: GitHubCLI,
675
+ worktree: WorktreeManager,
676
+ dashboard: DashboardClient | None,
677
+ state_db: StateDB,
678
+ transport: Transport | None,
679
+ contexts_dir: Path,
680
+ max_fix_attempts: int = DEFAULT_MAX_FIX_ATTEMPTS,
681
+ max_blocked_rounds: int = DEFAULT_MAX_BLOCKED_ROUNDS,
682
+ pr_verifier: PRVerifier | None = None,
683
+ ci_wait_timeout_seconds: int = 600,
684
+ ) -> PipelineResult:
685
+ """Run dev pipeline for a single issue."""
686
+ session_id = f"dev-{repo.replace('/', '-')}-{issue_number}-{uuid.uuid4().hex[:8]}"
687
+ branch_name = branch_template.replace("{n}", str(issue_number))
688
+
689
+ lock = _RepoLockHandle(state_db, repo, session_id)
690
+ if not state_db.acquire_lock(repo, session_id):
691
+ return PipelineResult(
692
+ success=False,
693
+ session_id=session_id,
694
+ summary=f"Could not acquire lock for {repo}",
695
+ error="Repository locked by another session",
696
+ )
697
+ lock.held = True
698
+
699
+ worktree_path: Path | None = None
700
+ # Ownership signal: True when THIS session created the branch (either
701
+ # from default or via the stale-merged delete+recreate path). False
702
+ # when we reused an existing branch. Stays False until
703
+ # create_worktree_with_new_branch returns so the exception-path
704
+ # cleanup can't accidentally delete a pre-existing branch belonging
705
+ # to another session. Issue #51: a pre-call snapshot of
706
+ # branch_exists_locally goes stale the moment the worktree helper
707
+ # detects a stale fully-merged local branch, deletes it, and
708
+ # recreates it — so the flag has to come out of the helper itself.
709
+ created_fresh = False
710
+ # Pre-call snapshot of branch_exists_locally, used ONLY by the
711
+ # exception-cleanup path to handle partial failures of
712
+ # ``git worktree add -b`` that register the branch ref before the
713
+ # directory setup crashes — in that case the helper raises before
714
+ # returning created_fresh=True, so the snapshot is our only
715
+ # signal that the leftover ref belongs to this session. Pessimistic
716
+ # default (True) so we never delete a branch on a path that
717
+ # didn't even get far enough to probe. The #51 fix (stale snapshot
718
+ # after delete+recreate) doesn't apply here because the
719
+ # delete+recreate path runs entirely INSIDE the helper and only
720
+ # takes effect on successful return, where ``created_fresh``
721
+ # wins over this snapshot.
722
+ branch_existed_before = True
723
+
724
+ try:
725
+ # Get issue details
726
+ issue = await github.get_issue(repo, issue_number)
727
+
728
+ # Post claim comment so collaborators can see the agent picked it up.
729
+ # Skip if a previous attempt already left the marker — keeps it idempotent
730
+ # across retries / resumed sessions.
731
+ existing_comments = issue.get("comments") or []
732
+ already_claimed = any(
733
+ AGENT_CLAIM_MARKER in (c.get("body") or "")
734
+ for c in existing_comments
735
+ )
736
+ if not already_claimed:
737
+ await github.comment_on_issue(
738
+ repo=repo,
739
+ issue_number=issue_number,
740
+ body=AGENT_CLAIM_COMMENT,
741
+ )
742
+
743
+ # Issue #51: ownership is signaled by ``created_fresh`` returned
744
+ # from create_worktree_with_new_branch. True when this call
745
+ # created the branch from default OR went through the
746
+ # stale-merged delete+recreate path — either way THIS session
747
+ # owns the resulting ref and cleanup on failure should delete it.
748
+ #
749
+ # Issue #52: pass the GitHubCLI so the reuse path can refuse a
750
+ # branch that still backs an open PR (prior DONE session whose
751
+ # PR is unmerged, or any external source). Without this, we'd
752
+ # silently check out the reviewer's already-reviewed branch and
753
+ # either hijack the PR or hit "A pull request already exists"
754
+ # on `gh pr create` at the end.
755
+ await worktree.ensure_bare_repo(repo)
756
+ # Snapshot for the exception-path cleanup only (see
757
+ # ``branch_existed_before`` comment at the top of this function).
758
+ # We still need this because ``git worktree add -b`` can register
759
+ # the branch ref before the directory step crashes, and the
760
+ # helper raises before it can return ``created_fresh=True``.
761
+ branch_existed_before = await worktree.branch_exists_locally(
762
+ repo, branch_name,
763
+ )
764
+ worktree_path, created_fresh = await worktree.create_worktree_with_new_branch(
765
+ repo=repo,
766
+ session_id=session_id,
767
+ new_branch=branch_name,
768
+ github=github,
769
+ )
770
+
771
+ # Symlink context
772
+ context_path = contexts_dir / repo.replace("/", "-") / "CLAUDE.md"
773
+ if context_path.exists():
774
+ worktree.symlink_context(worktree_path, context_path)
775
+
776
+ # Setup state file
777
+ state_file = worktree_path / ".ctrlrelay" / "state.json"
778
+ state_file.parent.mkdir(parents=True, exist_ok=True)
779
+
780
+ ctx = PipelineContext(
781
+ session_id=session_id,
782
+ repo=repo,
783
+ worktree_path=worktree_path,
784
+ context_path=context_path,
785
+ state_file=state_file,
786
+ issue_number=issue_number,
787
+ extra={
788
+ "issue_title": issue.get("title", ""),
789
+ "issue_body": issue.get("body", ""),
790
+ "issue_comments": format_issue_comments(issue.get("comments")),
791
+ "branch_name": branch_name,
792
+ "ci_wait_timeout_seconds": ci_wait_timeout_seconds,
793
+ },
794
+ )
795
+
796
+ # Record session
797
+ state_db.execute(
798
+ """INSERT INTO sessions
799
+ (id, pipeline, repo, worktree_path, status, started_at, issue_number)
800
+ VALUES (?, ?, ?, ?, ?, ?, ?)""",
801
+ (
802
+ session_id, "dev", repo, str(worktree_path),
803
+ "running", int(time.time()), issue_number,
804
+ ),
805
+ )
806
+ state_db.commit()
807
+
808
+ # Run pipeline
809
+ pipeline = DevPipeline(
810
+ dispatcher=dispatcher,
811
+ github=github,
812
+ worktree=worktree,
813
+ dashboard=dashboard,
814
+ state_db=state_db,
815
+ transport=transport,
816
+ )
817
+ result = await pipeline.run(ctx)
818
+
819
+ # BLOCKED loop: if Claude needs input, post the question to the
820
+ # transport (Telegram), wait for the operator's reply, and resume
821
+ # the session. Loop until Claude signals DONE/FAILED or we run out
822
+ # of trips. Each round-trip is capped at the transport's own
823
+ # timeout (see Transport.ask signature — default 300s); the total
824
+ # is bounded by max_blocked_rounds.
825
+ rounds = 0
826
+ while (
827
+ result.blocked
828
+ and transport is not None
829
+ and rounds < max_blocked_rounds
830
+ ):
831
+ question = (result.question or "").strip() or (
832
+ f"Session {session_id} is blocked but did not include a "
833
+ "question. Reply with guidance to resume."
834
+ )
835
+ try:
836
+ answer = await transport.ask(
837
+ question,
838
+ session_id=session_id,
839
+ repo=repo,
840
+ issue_number=issue_number,
841
+ )
842
+ except Exception as e:
843
+ # Transport failed (bridge down, timeout, etc.). The
844
+ # session IS still blocked — we just couldn't reach the
845
+ # operator. Preserve blocked=True so the outer
846
+ # persistence-on-blocked branch fires and writes a
847
+ # pending_resumes row. Operator replying later via
848
+ # Telegram then routes through the sweeper. Setting
849
+ # blocked=False here (the prior behavior) wedged the
850
+ # session permanently — codex P1.
851
+ result = PipelineResult(
852
+ success=False,
853
+ blocked=True,
854
+ session_id=session_id,
855
+ summary=f"Blocked session deferred (transport): {e}",
856
+ error=str(e),
857
+ question=question,
858
+ outputs=result.outputs,
859
+ )
860
+ break
861
+ rounds += 1
862
+ result = await pipeline.resume(ctx, answer)
863
+
864
+ # Verify PR is green & conflict-free before handing off. Resume the
865
+ # session with a fix request if either is broken, up to max_fix_attempts.
866
+ #
867
+ # Issue #29: the lock handle is passed in so the verifier releases
868
+ # the repo lock during its CI-polling window (can be up to 30 min)
869
+ # and reacquires before any request_fix. Peer sessions targeting
870
+ # the same repo can now run their own git-op phases while we wait.
871
+ if result.success and result.outputs.get("pr_number") is not None:
872
+ verifier = pr_verifier or PRVerifier(github=github)
873
+ result = await _verify_and_fix_pr(
874
+ pipeline=pipeline,
875
+ ctx=ctx,
876
+ result=result,
877
+ verifier=verifier,
878
+ max_attempts=max_fix_attempts,
879
+ lock_handle=lock,
880
+ )
881
+
882
+ # Reacquire the lock for the post-verify cleanup phase (remove
883
+ # worktree / delete branch). Usually SHORT budget (cleanup is
884
+ # best-effort, run_poll_loop awaits handlers serially).
885
+ #
886
+ # EXCEPT for the verify-contention error: that path signals
887
+ # "PR needs another fix round that we couldn't enter," cli
888
+ # will un-mark the issue for retry, and a fresh run would
889
+ # fail at create_worktree if this one left the worktree
890
+ # registered. Retry success DEPENDS on cleanup here, so use
891
+ # the full fix budget — accepting a possible long wait in
892
+ # this rare double-contention case beats deterministically
893
+ # wedging the issue.
894
+ if not lock.held:
895
+ if result.error == _LOCK_CONTENDED_DURING_VERIFY_ERROR:
896
+ reacquired = await lock.reacquire() # full fix budget
897
+ else:
898
+ reacquired = await lock.reacquire(
899
+ attempts=_REACQUIRE_CLEANUP_ATTEMPTS,
900
+ sleep_seconds=_REACQUIRE_CLEANUP_SLEEP_SECONDS,
901
+ )
902
+ if not reacquired:
903
+ log_event(
904
+ _logger,
905
+ "dev.cleanup.lock_reacquire_contended",
906
+ session_id=session_id,
907
+ repo=repo,
908
+ issue_number=issue_number,
909
+ )
910
+ # Signal to cli.handle_issue that the worktree was NOT
911
+ # torn down so the caller can skip any "unmark for
912
+ # retry" logic. A fresh dev run against this issue
913
+ # would fail at `create_worktree_with_new_branch`
914
+ # because the abandoned worktree is still registered
915
+ # on the bare repo. Requires operator cleanup (or the
916
+ # next poller restart, which prunes stale worktrees).
917
+ merged_outputs = dict(result.outputs)
918
+ merged_outputs["cleanup_deferred"] = True
919
+ result = PipelineResult(
920
+ success=result.success,
921
+ session_id=result.session_id,
922
+ summary=result.summary,
923
+ blocked=result.blocked,
924
+ question=result.question,
925
+ error=result.error,
926
+ outputs=merged_outputs,
927
+ )
928
+
929
+ # Update session status
930
+ status = "done" if result.success else ("blocked" if result.blocked else "failed")
931
+ state_db.execute(
932
+ "UPDATE sessions SET status = ?, summary = ?, ended_at = ? WHERE id = ?",
933
+ (status, result.summary, int(time.time()), session_id),
934
+ )
935
+ state_db.commit()
936
+
937
+ # Persist BLOCKED so a Telegram reply arriving AFTER this session
938
+ # exits (max_blocked_rounds exhausted, or transport unavailable
939
+ # during the session) still routes to a resume. The per-minute
940
+ # pending_resume_sweeper drains these rows by calling
941
+ # resume_dev_from_pending below.
942
+ if result.blocked:
943
+ try:
944
+ state_db.add_pending_resume(
945
+ session_id=session_id,
946
+ pipeline="dev",
947
+ repo=repo,
948
+ question=_question_for_persist(session_id, result),
949
+ )
950
+ except Exception as e:
951
+ log_event(
952
+ _logger,
953
+ "dev.pending_resume.insert_failed",
954
+ session_id=session_id,
955
+ repo=repo,
956
+ issue_number=issue_number,
957
+ error_type=type(e).__name__,
958
+ error=str(e)[:200],
959
+ )
960
+
961
+ # Push event to dashboard
962
+ if dashboard and result.success:
963
+ await dashboard.push_event(EventPayload(
964
+ level="info",
965
+ pipeline="dev",
966
+ repo=repo,
967
+ message=result.summary,
968
+ session_id=session_id,
969
+ details={
970
+ "issue_number": issue_number,
971
+ "pr_number": result.outputs.get("pr_number"),
972
+ },
973
+ ))
974
+
975
+ # Cleanup rules:
976
+ # DONE -> remove worktree, keep branch (the open PR references it)
977
+ # BLOCKED -> keep both (user may resume the session)
978
+ # FAILED -> remove worktree AND delete branch so the next retry can
979
+ # re-create `fix/issue-<n>` cleanly — BUT only if this
980
+ # session created the branch (``created_fresh`` — see
981
+ # issue #51) and it was never pushed (no recoverable
982
+ # work on origin).
983
+ #
984
+ # Cleanup mutates the shared bare repo (worktree metadata, branch
985
+ # refs) so it runs only when we hold the lock. If reacquire above
986
+ # failed, skip it and let the next retry / operator reclaim — the
987
+ # verified result is already final either way.
988
+ if lock.held:
989
+ if result.success:
990
+ worktree.remove_context_symlink(worktree_path)
991
+ await worktree.remove_worktree(repo, session_id)
992
+ elif not result.blocked:
993
+ worktree.remove_context_symlink(worktree_path)
994
+ await worktree.remove_worktree(repo, session_id)
995
+ if (
996
+ created_fresh
997
+ and not await worktree.branch_exists_on_remote(
998
+ repo, branch_name
999
+ )
1000
+ ):
1001
+ await worktree.delete_branch(repo, branch_name)
1002
+
1003
+ return result
1004
+
1005
+ except Exception as e:
1006
+ state_db.execute(
1007
+ "UPDATE sessions SET status = ?, summary = ?, ended_at = ? WHERE id = ?",
1008
+ ("failed", f"Error: {e}", int(time.time()), session_id),
1009
+ )
1010
+ state_db.commit()
1011
+
1012
+ # Best-effort cleanup so a retry isn't blocked by leftover state.
1013
+ # Only touch the branch if THIS session created it (issue #51 —
1014
+ # ``created_fresh``) AND origin has no copy (no recoverable work
1015
+ # to orphan). Covers partial failures of `git worktree add -b`
1016
+ # that create the ref before the directory setup crashes, and
1017
+ # the stale-merged delete+recreate path where the branch now on
1018
+ # disk was put there by this very call. If we're mid-verify the
1019
+ # lock may be released — try to get it back for the cleanup.
1020
+ # Uses the SHORT cleanup budget (not the fix-path one): a
1021
+ # transient exception inside verifier.verify raising through
1022
+ # while a peer holds the lock would otherwise stall the serial
1023
+ # poll cycle for the full hour-long fix budget just to report
1024
+ # the failure.
1025
+ if not lock.held:
1026
+ try:
1027
+ await lock.reacquire(
1028
+ attempts=_REACQUIRE_CLEANUP_ATTEMPTS,
1029
+ sleep_seconds=_REACQUIRE_CLEANUP_SLEEP_SECONDS,
1030
+ )
1031
+ except Exception:
1032
+ pass
1033
+ if lock.held:
1034
+ if worktree_path is not None:
1035
+ try:
1036
+ worktree.remove_context_symlink(worktree_path)
1037
+ except Exception:
1038
+ pass
1039
+ # Always attempt remove_worktree + prune — handles the case where
1040
+ # `git worktree add -b` registered worktree metadata before the dir
1041
+ # step failed, leaving worktree_path unassigned but metadata in the
1042
+ # bare repo that would prevent the branch from being deleted.
1043
+ try:
1044
+ await worktree.remove_worktree(repo, session_id)
1045
+ except Exception:
1046
+ pass
1047
+ # Exception path ownership: three signals, ordered by
1048
+ # reliability.
1049
+ # 1. created_fresh covers the normal return-and-fail case
1050
+ # (issue #51).
1051
+ # 2. StaleRecreatePartialFailureError signals the helper
1052
+ # destroyed the old ref before `git worktree add -b`,
1053
+ # so anything on disk with this branch name is ours
1054
+ # regardless of what the pre-call snapshot said. The
1055
+ # signal is per-call (exception attribute) so
1056
+ # concurrent sessions on the shared WorktreeManager
1057
+ # can't clobber each other's ownership state.
1058
+ # 3. Pre-call snapshot (`not branch_existed_before`) is
1059
+ # the fallback for partial failures of the plain
1060
+ # fresh-branch path where the helper raises before
1061
+ # setting created_fresh.
1062
+ from ctrlrelay.core.worktree import StaleRecreatePartialFailureError
1063
+ stale_recreate_attempted = isinstance(
1064
+ e, StaleRecreatePartialFailureError,
1065
+ )
1066
+ we_own_branch = (
1067
+ created_fresh
1068
+ or stale_recreate_attempted
1069
+ or not branch_existed_before
1070
+ )
1071
+ if we_own_branch:
1072
+ try:
1073
+ has_remote = await worktree.branch_exists_on_remote(
1074
+ repo, branch_name
1075
+ )
1076
+ except Exception:
1077
+ has_remote = True
1078
+ if not has_remote:
1079
+ try:
1080
+ await worktree.delete_branch(repo, branch_name)
1081
+ except Exception:
1082
+ pass
1083
+
1084
+ return PipelineResult(
1085
+ success=False,
1086
+ session_id=session_id,
1087
+ summary=f"Error processing issue #{issue_number}",
1088
+ error=str(e),
1089
+ )
1090
+
1091
+ finally:
1092
+ # Safe even if we already released inside `_verify_and_fix_pr` —
1093
+ # the handle's `release` is idempotent (no-op when held=False)
1094
+ # and StateDB.release_lock is itself idempotent (DELETE with
1095
+ # rowcount=0 is a valid outcome). This also covers
1096
+ # CancelledError: if cancellation arrives during the unlocked
1097
+ # verify window, held is already False, so no stray DELETE runs.
1098
+ lock.release()
1099
+
1100
+
1101
+ async def resume_dev_from_pending(
1102
+ session_id: str,
1103
+ repo: str,
1104
+ answer: str,
1105
+ branch_template: str,
1106
+ dispatcher: AgentAdapter,
1107
+ github: GitHubCLI,
1108
+ worktree: WorktreeManager,
1109
+ dashboard: DashboardClient | None,
1110
+ state_db: StateDB,
1111
+ transport: Transport | None,
1112
+ contexts_dir: Path,
1113
+ max_blocked_rounds: int = DEFAULT_MAX_BLOCKED_ROUNDS,
1114
+ max_fix_attempts: int = DEFAULT_MAX_FIX_ATTEMPTS,
1115
+ pr_verifier: PRVerifier | None = None,
1116
+ ) -> PipelineResult:
1117
+ """Resume a BLOCKED dev session using an answer that arrived via
1118
+ Telegram after ``run_dev_issue`` exited.
1119
+
1120
+ Unlike secops, the dev pipeline keeps its worktree + branch alive on
1121
+ BLOCKED exits (so resume doesn't have to re-clone/re-apply). We try
1122
+ to reuse the stored worktree_path first; if it's missing (manual
1123
+ cleanup, daemon restart lost the mount, etc.) we fall back to
1124
+ ``create_worktree_with_new_branch`` which handles existing-branch
1125
+ reuse.
1126
+
1127
+ Caller is responsible for flipping the pending_resumes row
1128
+ ``resumed_at`` after this returns — except in the lock-contention
1129
+ case, which is retryable; we surface it via
1130
+ ``error == "Repository locked by another session"``.
1131
+ """
1132
+ session_row = state_db.get_session_row(session_id)
1133
+ if session_row is None:
1134
+ return PipelineResult(
1135
+ success=False,
1136
+ session_id=session_id,
1137
+ summary=(
1138
+ f"Cannot resume dev session {session_id}: row missing "
1139
+ "from sessions table"
1140
+ ),
1141
+ error="session_row_missing",
1142
+ )
1143
+ issue_number = session_row.get("issue_number")
1144
+ if issue_number is None:
1145
+ return PipelineResult(
1146
+ success=False,
1147
+ session_id=session_id,
1148
+ summary=(
1149
+ f"Cannot resume dev session {session_id}: issue_number "
1150
+ "missing on sessions row"
1151
+ ),
1152
+ error="session_issue_number_missing",
1153
+ )
1154
+
1155
+ branch_name = branch_template.replace("{n}", str(issue_number))
1156
+
1157
+ lock = _RepoLockHandle(state_db, repo, session_id)
1158
+ if not state_db.acquire_lock(repo, session_id):
1159
+ return PipelineResult(
1160
+ success=False,
1161
+ session_id=session_id,
1162
+ summary=f"Could not acquire lock for {repo} to resume dev",
1163
+ error="Repository locked by another session",
1164
+ )
1165
+ lock.held = True
1166
+
1167
+ worktree_path: Path | None = None
1168
+
1169
+ try:
1170
+ await worktree.ensure_bare_repo(repo)
1171
+
1172
+ # Prefer re-using the existing worktree left alive by the BLOCKED
1173
+ # cleanup rule ("BLOCKED → keep both"). That preserves the
1174
+ # session's in-progress commits / uncommitted work without
1175
+ # another fetch round-trip.
1176
+ stored_worktree = session_row.get("worktree_path")
1177
+ if stored_worktree:
1178
+ candidate = Path(stored_worktree)
1179
+ if candidate.exists() and (candidate / ".git").exists():
1180
+ worktree_path = candidate
1181
+
1182
+ if worktree_path is None:
1183
+ # Resume path: the branch should already exist (BLOCKED
1184
+ # sessions preserve both worktree and branch). We discard
1185
+ # ``created_fresh`` here — cleanup semantics don't apply to
1186
+ # resume, which never deletes the branch on exit; the
1187
+ # original run_dev_issue is the single owner of that
1188
+ # decision. Don't pass ``github`` either: an open PR for a
1189
+ # mid-BLOCKED session is possible (claude opened the PR and
1190
+ # blocked on a review question) and refusing reuse would
1191
+ # wedge the resume path.
1192
+ worktree_path, _ = await worktree.create_worktree_with_new_branch(
1193
+ repo=repo,
1194
+ session_id=session_id,
1195
+ new_branch=branch_name,
1196
+ )
1197
+
1198
+ context_path = contexts_dir / repo.replace("/", "-") / "CLAUDE.md"
1199
+ if context_path.exists():
1200
+ worktree.symlink_context(worktree_path, context_path)
1201
+
1202
+ state_file = worktree_path / ".ctrlrelay" / "state.json"
1203
+ state_file.parent.mkdir(parents=True, exist_ok=True)
1204
+
1205
+ # Fetch issue context again so the resumed prompt has the same
1206
+ # metadata the first run did — avoids the agent hallucinating
1207
+ # details from stale state.
1208
+ issue = await github.get_issue(repo, int(issue_number))
1209
+
1210
+ ctx = PipelineContext(
1211
+ session_id=session_id,
1212
+ repo=repo,
1213
+ worktree_path=worktree_path,
1214
+ context_path=context_path,
1215
+ state_file=state_file,
1216
+ issue_number=int(issue_number),
1217
+ extra={
1218
+ "issue_title": issue.get("title", ""),
1219
+ "issue_body": issue.get("body", ""),
1220
+ "issue_comments": format_issue_comments(issue.get("comments")),
1221
+ "branch_name": branch_name,
1222
+ },
1223
+ )
1224
+
1225
+ state_db.execute(
1226
+ "UPDATE sessions SET status = ?, ended_at = NULL WHERE id = ?",
1227
+ ("running", session_id),
1228
+ )
1229
+ state_db.commit()
1230
+
1231
+ pipeline = DevPipeline(
1232
+ dispatcher=dispatcher,
1233
+ github=github,
1234
+ worktree=worktree,
1235
+ dashboard=dashboard,
1236
+ state_db=state_db,
1237
+ transport=transport,
1238
+ )
1239
+
1240
+ # Feed the operator's answer as the resume prompt. If the agent
1241
+ # re-blocks (common for multi-step decisions), the same
1242
+ # in-process BLOCKED loop that run_dev_issue uses kicks in here
1243
+ # — transport.ask() waits on the live socket — so the operator
1244
+ # can have a back-and-forth exchange on the resume path too.
1245
+ result = await pipeline.resume(ctx, answer)
1246
+
1247
+ rounds = 0
1248
+ while (
1249
+ result.blocked
1250
+ and transport is not None
1251
+ and rounds < max_blocked_rounds
1252
+ ):
1253
+ question = (result.question or "").strip() or (
1254
+ f"Session {session_id} is blocked but did not include a "
1255
+ "question. Reply with guidance to resume."
1256
+ )
1257
+ try:
1258
+ next_answer = await transport.ask(
1259
+ question,
1260
+ session_id=session_id,
1261
+ repo=repo,
1262
+ issue_number=int(issue_number),
1263
+ )
1264
+ except Exception as e:
1265
+ # Keep blocked=True so outer persistence branch fires
1266
+ # and the sweeper can pick this up again when the
1267
+ # operator's next Telegram reply arrives. Same
1268
+ # reasoning as run_dev_issue's loop — transport
1269
+ # failure is not the same as a resolved session.
1270
+ result = PipelineResult(
1271
+ success=False,
1272
+ blocked=True,
1273
+ session_id=session_id,
1274
+ summary=(
1275
+ f"Blocked session deferred during resume "
1276
+ f"(transport): {e}"
1277
+ ),
1278
+ error=str(e),
1279
+ question=question,
1280
+ outputs=result.outputs,
1281
+ )
1282
+ break
1283
+ rounds += 1
1284
+ result = await pipeline.resume(ctx, next_answer)
1285
+
1286
+ # Verify PR once the agent says DONE, matching run_dev_issue.
1287
+ # The lock handle is threaded through so the verifier releases
1288
+ # the repo lock during its CI-polling window (issue #29).
1289
+ if result.success and result.outputs.get("pr_number") is not None:
1290
+ verifier = pr_verifier or PRVerifier(github=github)
1291
+ result = await _verify_and_fix_pr(
1292
+ pipeline=pipeline,
1293
+ ctx=ctx,
1294
+ result=result,
1295
+ verifier=verifier,
1296
+ max_attempts=max_fix_attempts,
1297
+ lock_handle=lock,
1298
+ )
1299
+
1300
+ status = "done" if result.success else (
1301
+ "blocked" if result.blocked else "failed"
1302
+ )
1303
+ state_db.execute(
1304
+ "UPDATE sessions SET status = ?, summary = ?, ended_at = ? "
1305
+ "WHERE id = ?",
1306
+ (status, result.summary, int(time.time()), session_id),
1307
+ )
1308
+ state_db.commit()
1309
+
1310
+ if result.blocked:
1311
+ try:
1312
+ state_db.add_pending_resume(
1313
+ session_id=session_id,
1314
+ pipeline="dev",
1315
+ repo=repo,
1316
+ question=_question_for_persist(session_id, result),
1317
+ )
1318
+ except Exception as e:
1319
+ log_event(
1320
+ _logger,
1321
+ "dev.pending_resume.reblock_insert_failed",
1322
+ session_id=session_id,
1323
+ repo=repo,
1324
+ issue_number=issue_number,
1325
+ error_type=type(e).__name__,
1326
+ error=str(e)[:200],
1327
+ )
1328
+
1329
+ return result
1330
+
1331
+ except Exception as e:
1332
+ state_db.execute(
1333
+ "UPDATE sessions SET status = ?, summary = ?, ended_at = ? "
1334
+ "WHERE id = ?",
1335
+ ("failed", f"Resume error: {e}", int(time.time()), session_id),
1336
+ )
1337
+ state_db.commit()
1338
+ return PipelineResult(
1339
+ success=False,
1340
+ session_id=session_id,
1341
+ summary=f"Error resuming dev session {session_id} on {repo}",
1342
+ error=str(e),
1343
+ )
1344
+
1345
+ finally:
1346
+ # Intentionally leave the worktree in place — matches
1347
+ # run_dev_issue's "BLOCKED → keep both" contract and makes the
1348
+ # worktree available for a third resume if the second one
1349
+ # re-blocks. A follow-up can add teardown for the DONE/FAILED
1350
+ # outcomes (small disk leak, not a correctness issue; operator
1351
+ # can run `git worktree prune` in the bare repo to reclaim).
1352
+ #
1353
+ # ``lock.release`` is idempotent, so this is a no-op if the
1354
+ # verify phase already released and never reacquired (CI-only
1355
+ # path with no fix attempts + no cleanup work). Matches issue
1356
+ # #29's release-during-verify semantics.
1357
+ try:
1358
+ lock.release()
1359
+ except Exception as lock_exc:
1360
+ log_event(
1361
+ _logger,
1362
+ "dev.resume.cleanup.lock_release_failed",
1363
+ session_id=session_id,
1364
+ repo=repo,
1365
+ error_type=type(lock_exc).__name__,
1366
+ error=str(lock_exc)[:200],
1367
+ )