ctrlrelay 0.1.5__py3-none-any.whl

ctrlrelay/pipelines/post_merge.py

@@ -0,0 +1,279 @@
+"""Post-merge handling for dev pipeline."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Awaitable, Callable
+
+from ctrlrelay.core.github import GitHubCLI
+from ctrlrelay.core.obs import get_logger, log_event
+from ctrlrelay.core.pr_watcher import PRWatcher
+from ctrlrelay.transports.base import Transport
+
+_logger = get_logger("pipelines.post_merge")
+
+# Default merge-watch window. Real PR review cycles commonly exceed a
+# day (weekends, time-zone splits, larger changes), and a silent watch
+# timeout means the linked issue never auto-closes and no merge
+# notification fires. 7 days is the sweet spot: covers typical review
+# lag without keeping a zombie task alive for months on an abandoned PR.
+# Operators can still override via an explicit `timeout=` argument.
+DEFAULT_PR_WATCH_TIMEOUT = 7 * 24 * 60 * 60
+
+# After a merge is detected, handle_merge can fail transiently on
+# `gh issue close` or the transport.send (network blip, rate limit).
+# Without retry the whole automation is wasted — PR merged, issue
+# stays open forever. Retry with modest backoff; if all attempts fail,
+# the last failure is logged via dev.pr.watch_failed.
+_HANDLE_MERGE_RETRY_ATTEMPTS = 5
+_HANDLE_MERGE_RETRY_BASE_DELAY = 5  # seconds; doubled each attempt
+
+
+async def handle_merge(
+    repo: str,
+    pr_number: int,
+    issue_number: int,
+    github: GitHubCLI,
+    transport: Transport | None = None,
+) -> None:
+    """Handle post-merge actions."""
+    await github.close_issue(
+        repo,
+        issue_number,
+        f"Closed by PR #{pr_number}",
+    )
+
+    if transport:
+        await transport.send(
+            f"Issue #{issue_number} closed after PR #{pr_number} merged in {repo}"
+        )
+
+
+async def watch_and_handle_merge(
+    repo: str,
+    pr_number: int,
+    issue_number: int,
+    github: GitHubCLI,
+    transport: Transport | None = None,
+    poll_interval: int = 60,
+    timeout: int = DEFAULT_PR_WATCH_TIMEOUT,
+) -> bool:
+    """Watch for PR merge and handle post-merge actions."""
+    watcher = PRWatcher(github=github, poll_interval=poll_interval)
+
+    merged = await watcher.wait_for_merge(repo, pr_number, timeout=timeout)
+
+    if merged:
+        await handle_merge(
+            repo=repo,
+            pr_number=pr_number,
+            issue_number=issue_number,
+            github=github,
+            transport=transport,
+        )
+        return True
+
+    return False
+
+
+async def _handle_merge_with_retry(
+    *,
+    repo: str,
+    pr_number: int,
+    issue_number: int,
+    github: GitHubCLI,
+    transport_factory: Callable[[], Awaitable[Transport | None]] | None,
+    session_id: str | None,
+) -> None:
+    """Run the post-merge close + notify steps with per-step idempotent
+    retry, rebuilding the transport on each attempt so a bridge restart
+    mid-retry doesn't permanently break the notification.
+
+    Idempotency: ``github.close_issue`` is called at most once even
+    across retries — if it succeeds but the notification step later
+    fails, subsequent attempts skip the close so we don't post
+    duplicate "Closed by PR #N" comments.
+
+    Transport rebuild: ``transport_factory`` is invoked INSIDE each
+    attempt. If the bridge socket drops between the watcher start and
+    a merge several days later, or during a retry, the next attempt
+    gets a fresh connection.
+
+    On final exhaustion, raises the last exception; pr_watch_task's
+    outer handler logs it as dev.pr.watch_failed.
+    """
+    # Split the post-merge work into three independent steps with
+    # their own idempotency flags. close_issue() in the github layer
+    # is NOT atomic — it posts a comment then runs `gh issue close`.
+    # If the close fails after the comment succeeded, a retry that
+    # called close_issue again would post a DUPLICATE comment.
+    # Tracking each sub-step separately ensures every side-effect
+    # runs at most once across retries.
+    comment_posted = False
+    issue_closed = False
+    notification_sent = False
+    last_exc: Exception | None = None
+    delay = _HANDLE_MERGE_RETRY_BASE_DELAY
+    comment = f"Closed by PR #{pr_number}"
+    notification = (
+        f"Issue #{issue_number} closed after PR #{pr_number} merged in {repo}"
+    )
+
+    for attempt in range(1, _HANDLE_MERGE_RETRY_ATTEMPTS + 1):
+        transport: Transport | None = None
+        transport_build_error: Exception | None = None
+        if not notification_sent and transport_factory is not None:
+            try:
+                transport = await transport_factory()
+            except Exception as e:
+                # Factory raised — bridge likely transiently unavailable.
+                # Record the error so we can retry on the next attempt
+                # instead of silently dropping the notification.
+                transport_build_error = e
+                log_event(
+                    _logger, "dev.pr.watch_transport_failed",
+                    session_id=session_id, repo=repo,
+                    issue_number=issue_number, pr_number=pr_number,
+                    attempt=attempt,
+                    reason=type(e).__name__, error=str(e)[:200],
+                )
+                transport = None
+            # Factory returned None legitimately (e.g. non-Telegram
+            # config or socket intentionally absent) → no notification
+            # channel is configured; do NOT retry. `transport_build_error`
+            # remains None, so the "done" check below returns cleanly.
+
+        try:
+            if not comment_posted:
+                await github.comment_on_issue(repo, issue_number, comment)
+                comment_posted = True
+            if not issue_closed:
+                await github._run_gh(
+                    "issue", "close", str(issue_number), "--repo", repo,
+                )
+                issue_closed = True
+            if not notification_sent and transport is not None:
+                await transport.send(notification)
+                notification_sent = True
+
+            # Done when either: we sent the notification this round, OR
+            # the factory returned None meaning no channel was ever
+            # configured. Fall through to the retry path ONLY if the
+            # factory itself raised — that's a transient failure worth
+            # retrying so a brief bridge outage doesn't drop the
+            # notification permanently.
+            if notification_sent or transport_build_error is None:
+                return
+            raise transport_build_error
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:
+            last_exc = e
+            log_event(
+                _logger, "dev.pr.handle_merge_retry",
+                session_id=session_id, repo=repo,
+                issue_number=issue_number, pr_number=pr_number,
+                attempt=attempt,
+                max_attempts=_HANDLE_MERGE_RETRY_ATTEMPTS,
+                comment_posted=comment_posted,
+                issue_closed=issue_closed,
+                notification_sent=notification_sent,
+                reason=type(e).__name__, error=str(e)[:200],
+            )
+            if attempt >= _HANDLE_MERGE_RETRY_ATTEMPTS:
+                break
+            await asyncio.sleep(delay)
+            delay = min(delay * 2, 300)
+        finally:
+            if transport is not None:
+                try:
+                    await transport.close()
+                except Exception:
+                    pass
+    assert last_exc is not None
+    raise last_exc
+
+
+async def pr_watch_task(
+    *,
+    repo: str,
+    issue_number: int,
+    pr_url: str,
+    pr_number: int,
+    session_id: str | None,
+    github: GitHubCLI,
+    transport_factory: Callable[[], Awaitable[Transport | None]] | None = None,
+    poll_interval: int = 60,
+    timeout: int = DEFAULT_PR_WATCH_TIMEOUT,
+) -> dict[str, Any]:
+    """Background-safe wrapper around the merge watcher that emits
+    structured ``dev.pr.*`` events and manages a transport LAZILY —
+    the transport is only created AFTER the merge is detected, so a
+    bridge that's down or restarted during the multi-day watch window
+    doesn't leave us with a stale connection at notification time.
+
+    ``transport_factory`` is an async callable returning a connected
+    transport (or None). Called once, right before ``handle_merge``;
+    its result is closed immediately after the merge handler runs.
+    A raising factory is logged and skipped (merge detection proceeds
+    without the notification channel rather than aborting the close).
+
+    Returns a dict describing the outcome:
+      {"merged": bool, "timed_out": bool, "cancelled": bool, "failed": str|None}
+    """
+    outcome: dict[str, Any] = {
+        "merged": False, "timed_out": False,
+        "cancelled": False, "failed": None,
+    }
+
+    log_event(
+        _logger, "dev.pr.watching",
+        session_id=session_id, repo=repo, issue_number=issue_number,
+        pr_number=pr_number, pr_url=pr_url,
+    )
+    try:
+        watcher = PRWatcher(github=github, poll_interval=poll_interval)
+        merged = await watcher.wait_for_merge(
+            repo=repo, pr_number=pr_number, timeout=timeout,
+        )
+        # Record merge detection BEFORE running the post-merge handler,
+        # so an exhausted retry loop (e.g. permanent bridge outage) still
+        # leaves `outcome["merged"] = True`. The merge really did happen;
+        # only the cleanup chain is degraded, and `outcome["failed"]`
+        # already surfaces that via the except branch below.
+        outcome["merged"] = merged
+        outcome["timed_out"] = not merged
+        log_event(
+            _logger,
+            "dev.pr.merged" if merged else "dev.pr.watch_timeout",
+            session_id=session_id, repo=repo, issue_number=issue_number,
+            pr_number=pr_number,
+        )
+        if merged:
+            # Defer transport construction to inside the retry loop so
+            # each attempt gets a fresh connection — a bridge restart
+            # during the watch or between retries doesn't leave us
+            # hitting a dead socket forever.
+            await _handle_merge_with_retry(
+                repo=repo, pr_number=pr_number,
+                issue_number=issue_number, github=github,
+                transport_factory=transport_factory,
+                session_id=session_id,
+            )
+    except asyncio.CancelledError:
+        outcome["cancelled"] = True
+        log_event(
+            _logger, "dev.pr.watch_cancelled",
+            session_id=session_id, repo=repo, issue_number=issue_number,
+            pr_number=pr_number,
+        )
+        raise
+    except Exception as e:
+        outcome["failed"] = type(e).__name__
+        log_event(
+            _logger, "dev.pr.watch_failed",
+            session_id=session_id, repo=repo, issue_number=issue_number,
+            pr_number=pr_number,
+            reason=type(e).__name__, error=str(e)[:200],
+        )
+    return outcome

ctrlrelay/pipelines/secops.py

@@ -0,0 +1,379 @@
+"""Secops pipeline for security triage across repos."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from ctrlrelay.core.checkpoint import CheckpointStatus
+from ctrlrelay.core.dispatcher import AgentAdapter, SessionResult
+from ctrlrelay.core.github import GitHubCLI
+from ctrlrelay.core.obs import get_logger, hash_text, log_event
+from ctrlrelay.core.state import StateDB
+from ctrlrelay.core.worktree import WorktreeManager
+from ctrlrelay.dashboard.client import DashboardClient, EventPayload
+from ctrlrelay.pipelines.base import PipelineContext, PipelineResult
+from ctrlrelay.transports.base import Transport
+
+_logger = get_logger("pipeline.secops")
+
+
+@dataclass
+class SecopsPipeline:
+    """Security operations pipeline for daily triage."""
+
+    dispatcher: AgentAdapter
+    github: GitHubCLI
+    worktree: WorktreeManager
+    dashboard: DashboardClient | None
+    state_db: StateDB
+    transport: Transport | None
+
+    name: str = "secops"
+
+    async def run(self, ctx: PipelineContext) -> PipelineResult:
+        """Run secops on a single repo."""
+        prompt = self._build_prompt(
+            ctx.repo,
+            session_id=ctx.session_id,
+            state_file=ctx.state_file,
+        )
+
+        result = await self.dispatcher.spawn_session(
+            session_id=ctx.session_id,
+            prompt=prompt,
+            working_dir=ctx.worktree_path,
+            state_file=ctx.state_file,
+        )
+
+        return self._session_to_result(result)
+
+    async def resume(self, ctx: PipelineContext, answer: str) -> PipelineResult:
+        """Resume blocked secops with user answer."""
+        prompt = f"User answered: {answer}\n\nContinue from where you left off."
+
+        log_event(
+            _logger,
+            "dev.session.resumed",
+            session_id=ctx.session_id,
+            repo=ctx.repo,
+            issue_number=ctx.issue_number,
+            pipeline=self.name,
+            resume_session_id=ctx.session_id,
+            answer_length=len(answer),
+            answer_hash=hash_text(answer),
+        )
+
+        result = await self.dispatcher.spawn_session(
+            session_id=ctx.session_id,
+            prompt=prompt,
+            working_dir=ctx.worktree_path,
+            state_file=ctx.state_file,
+            resume_session_id=ctx.session_id,
+        )
+
+        return self._session_to_result(result)
+
+    def _build_prompt(
+        self,
+        repo: str,
+        session_id: str = "",
+        state_file: Path | None = None,
+    ) -> str:
+        """Build the secops prompt."""
+        state_file_path = str(state_file) if state_file else "/tmp/state.json"
+
+        return f"""Execute security operations for repository {repo}.
+
+1. Check Dependabot alerts:
+   `gh api repos/{repo}/dependabot/alerts --jq '.[] | select(.state=="open")'`
+2. Check security PRs:
+   `gh pr list --repo {repo} --author "app/dependabot" --json number,title`
+3. For each alert or PR:
+   - Review the severity and impact
+   - If patch/minor update with passing CI, merge the PR
+   - If major or unclear, signal BLOCKED to ask for guidance
+4. Summarize actions taken
+
+## Signaling Completion
+
+**CRITICAL**: Before exiting, you MUST write a checkpoint file to signal completion.
+
+STATE_FILE: {state_file_path}
+SESSION_ID: {session_id}
+
+**DONE** (completed):
+```bash
+mkdir -p "$(dirname '{state_file_path}')"
+printf '{{"version":"1","status":"DONE","session_id":"{session_id}",'\
+'"timestamp":"%s","summary":"%s"}}' \\
+  "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "<SUMMARY>" > '{state_file_path}'
+```
+
+**BLOCKED** (need input):
+```bash
+mkdir -p "$(dirname '{state_file_path}')"
+printf '{{"version":"1","status":"BLOCKED_NEEDS_INPUT",'\
+'"session_id":"{session_id}","timestamp":"%s","question":"%s"}}' \\
+  "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "<QUESTION>" > '{state_file_path}'
+```
+
+**FAILED**:
+```bash
+mkdir -p "$(dirname '{state_file_path}')"
+printf '{{"version":"1","status":"FAILED","session_id":"{session_id}",'\
+'"timestamp":"%s","error":"%s"}}' \\
+  "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "<ERROR>" > '{state_file_path}'
+```"""
+
+    def _session_to_result(self, result: SessionResult) -> PipelineResult:
+        """Convert SessionResult to PipelineResult."""
+        if result.state is None:
+            return PipelineResult(
+                success=False,
+                session_id=result.session_id,
+                summary="No checkpoint state returned",
+                error=result.stderr or "Unknown error",
+            )
+
+        if result.state.status == CheckpointStatus.DONE:
+            return PipelineResult(
+                success=True,
+                session_id=result.session_id,
+                summary=result.state.summary or "Completed",
+                outputs=result.state.outputs,
+            )
+
+        if result.state.status == CheckpointStatus.BLOCKED_NEEDS_INPUT:
+            return PipelineResult(
+                success=False,
+                session_id=result.session_id,
+                summary="Blocked on user input",
+                blocked=True,
+                question=result.state.question,
+            )
+
+        return PipelineResult(
+            success=False,
+            session_id=result.session_id,
+            summary="Failed",
+            error=result.state.error,
+        )
+
+
+async def run_secops_all(
+    repos: list[Any],
+    dispatcher: AgentAdapter,
+    github: GitHubCLI,
+    worktree: WorktreeManager,
+    dashboard: DashboardClient | None,
+    state_db: StateDB,
+    transport: Transport | None,
+    contexts_dir: Path,
+) -> list[PipelineResult]:
+    """Run secops pipeline on all configured repos."""
+    results = []
+
+    pipeline = SecopsPipeline(
+        dispatcher=dispatcher,
+        github=github,
+        worktree=worktree,
+        dashboard=dashboard,
+        state_db=state_db,
+        transport=transport,
+    )
+
+    for repo_config in repos:
+        repo = repo_config.name
+        session_id = f"secops-{repo.replace('/', '-')}-{uuid.uuid4().hex[:8]}"
+
+        if not state_db.acquire_lock(repo, session_id):
+            results.append(PipelineResult(
+                success=False,
+                session_id=session_id,
+                summary=f"Could not acquire lock for {repo}",
+                error="Repository locked by another session",
+            ))
+            continue
+
+        worktree_path: Path | None = None
+        session_row_inserted = False
+        session_final_state_written = False
+        try:
+            await worktree.ensure_bare_repo(repo)
+            worktree_path = await worktree.create_worktree(repo, session_id)
+
+            context_path = contexts_dir / repo.replace("/", "-") / "CLAUDE.md"
+            if context_path.exists():
+                worktree.symlink_context(worktree_path, context_path)
+
+            state_file = worktree_path / ".ctrlrelay" / "state.json"
+            state_file.parent.mkdir(parents=True, exist_ok=True)
+
+            ctx = PipelineContext(
+                session_id=session_id,
+                repo=repo,
+                worktree_path=worktree_path,
+                context_path=context_path,
+                state_file=state_file,
+            )
+
+            state_db.execute(
+                """INSERT INTO sessions (id, pipeline, repo, worktree_path, status, started_at)
+                   VALUES (?, ?, ?, ?, ?, ?)""",
+                (session_id, "secops", repo, str(worktree_path), "running", int(time.time())),
+            )
+            state_db.commit()
+            session_row_inserted = True
+
+            result = await pipeline.run(ctx)
+            results.append(result)
+
+            status = "done" if result.success else ("blocked" if result.blocked else "failed")
+            state_db.execute(
+                "UPDATE sessions SET status = ?, summary = ?, ended_at = ? WHERE id = ?",
+                (status, result.summary, int(time.time()), session_id),
+            )
+            state_db.commit()
+            session_final_state_written = True
+
+            if dashboard and result.success:
+                await dashboard.push_event(EventPayload(
+                    level="info",
+                    pipeline="secops",
+                    repo=repo,
+                    message=result.summary,
+                    session_id=session_id,
+                ))
+
+        except asyncio.CancelledError:
+            # Scheduled secops interrupted mid-run (SIGTERM during a
+            # scheduler.shutdown). Mark the session as cancelled so later
+            # inspection doesn't see a phantom "running" row — but ONLY
+            # if we hadn't already written a final state. Without this
+            # guard, a cancel landing during the post-run dashboard push
+            # would clobber a successful "done" status with "cancelled"
+            # even though the work completed.
+            if session_row_inserted and not session_final_state_written:
+                try:
+                    state_db.execute(
+                        "UPDATE sessions SET status = ?, summary = ?, "
+                        "ended_at = ? WHERE id = ?",
+                        (
+                            "cancelled",
+                            "Cancelled during shutdown",
+                            int(time.time()),
+                            session_id,
+                        ),
+                    )
+                    state_db.commit()
+                except Exception:
+                    pass
+            raise
+
+        except Exception as e:
+            if session_row_inserted:
+                state_db.execute(
+                    "UPDATE sessions SET status = ?, summary = ?, "
+                    "ended_at = ? WHERE id = ?",
+                    ("failed", f"Error: {e}", int(time.time()), session_id),
+                )
+                state_db.commit()
+            results.append(PipelineResult(
+                success=False,
+                session_id=session_id,
+                summary=f"Error processing {repo}",
+                error=str(e),
+            ))
+
+        finally:
+            # Hold the per-repo lock THROUGH worktree cleanup so a
+            # concurrent dev/secops run can't acquire the same repo and
+            # race `git worktree prune` on the shared bare clone.
+            # Cleanup is bounded by an asyncio.wait_for timeout so a
+            # truly-stuck operation can't pin the lock forever — once
+            # the timeout fires we release anyway, preferring "possibly
+            # leaked git admin state (recoverable via `git worktree
+            # prune`)" over "repo locked across daemon restart".
+            #
+            # No asyncio.shield here: `_run_git` kills its subprocess
+            # synchronously on CancelledError so unshielded cleanup
+            # unwinds fast and without leaking a stray git. Previously
+            # the shield made the outer task cancel immediately while
+            # the inner cleanup kept running untracked — leaking state.
+            if worktree_path is not None:
+                try:
+                    worktree.remove_context_symlink(worktree_path)
+                except Exception as cleanup_exc:
+                    log_event(
+                        _logger,
+                        "secops.cleanup.symlink_failed",
+                        session_id=session_id,
+                        repo=repo,
+                        error_type=type(cleanup_exc).__name__,
+                        error=str(cleanup_exc)[:200],
+                    )
+                try:
+                    # Timeout matches Scheduler's cancel budget (150s) — a
+                    # full worktree prune can take up to 120s per
+                    # WorktreeManager._run_git ceiling.
+                    await asyncio.wait_for(
+                        worktree.remove_worktree(repo, session_id),
+                        timeout=130.0,
+                    )
+                except asyncio.TimeoutError:
+                    log_event(
+                        _logger,
+                        "secops.cleanup.worktree_timeout",
+                        session_id=session_id,
+                        repo=repo,
+                        worktree_path=str(worktree_path),
+                    )
+                except asyncio.CancelledError:
+                    log_event(
+                        _logger,
+                        "secops.cleanup.worktree_cancelled_mid_shutdown",
+                        session_id=session_id,
+                        repo=repo,
+                        worktree_path=str(worktree_path),
+                    )
+                    # Release the lock before propagating — leaving it
+                    # held across a daemon restart is worse than the
+                    # small race with a future scheduled run (the dev
+                    # handler retries on lock conflict anyway).
+                    try:
+                        state_db.release_lock(repo, session_id)
+                    except Exception:
+                        pass
+                    raise
+                except Exception as cleanup_exc:
+                    log_event(
+                        _logger,
+                        "secops.cleanup.worktree_failed",
+                        session_id=session_id,
+                        repo=repo,
+                        worktree_path=str(worktree_path),
+                        error_type=type(cleanup_exc).__name__,
+                        error=str(cleanup_exc)[:200],
+                    )
+
+            # Release the lock AFTER cleanup attempt so new runs don't
+            # race the prune. In the cancel path above we released early
+            # and re-raised; here we handle the non-cancel paths.
+            try:
+                state_db.release_lock(repo, session_id)
+            except Exception as lock_exc:
+                log_event(
+                    _logger,
+                    "secops.cleanup.lock_release_failed",
+                    session_id=session_id,
+                    repo=repo,
+                    error_type=type(lock_exc).__name__,
+                    error=str(lock_exc)[:200],
+                )
+
+    return results

ctrlrelay/transports/__init__.py

@@ -0,0 +1,33 @@
+"""Transport abstraction for orchestrator communication."""
+
+from ctrlrelay.core.config import TransportConfig, TransportType
+from ctrlrelay.transports.base import Transport, TransportError
+from ctrlrelay.transports.file_mock import FileMockTransport
+from ctrlrelay.transports.socket_client import SocketTransport
+
+
+def get_transport(config: TransportConfig) -> Transport:
+    """Create transport instance from config."""
+    if config.type == TransportType.FILE_MOCK:
+        assert config.file_mock is not None
+        return FileMockTransport(
+            inbox=config.file_mock.inbox,
+            outbox=config.file_mock.outbox,
+        )
+
+    if config.type == TransportType.TELEGRAM:
+        assert config.telegram is not None
+        return SocketTransport(
+            socket_path=config.telegram.socket_path,
+        )
+
+    raise TransportError(f"Unknown transport type: {config.type}")
+
+
+__all__ = [
+    "FileMockTransport",
+    "SocketTransport",
+    "Transport",
+    "TransportError",
+    "get_transport",
+]