ctrlrelay 0.10.0__py3-none-any.whl

ctrlrelay/pipelines/secops.py

@@ -0,0 +1,1018 @@
+"""Secops pipeline for security triage across repos."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from ctrlrelay.core.checkpoint import CheckpointStatus
+from ctrlrelay.core.dispatcher import AgentAdapter, SessionResult
+from ctrlrelay.core.github import GitHubCLI
+from ctrlrelay.core.obs import get_logger, hash_text, log_event
+from ctrlrelay.core.question_expiry import extract_referenced_numbers
+from ctrlrelay.core.state import StateDB
+from ctrlrelay.core.worktree import WorktreeManager
+from ctrlrelay.dashboard.client import DashboardClient, EventPayload
+from ctrlrelay.pipelines.base import PipelineContext, PipelineResult
+from ctrlrelay.transports.base import Transport
+
+_logger = get_logger("pipeline.secops")
+
+DEFAULT_MAX_BLOCKED_ROUNDS = 5
+
+# 30-day rolling window of operator decisions injected into each
+# sweep's prompt. Long enough to cover weekly cadences and a vacation;
+# short enough that a stale "approve" from months ago can't auto-merge
+# a PR whose risk profile may have shifted (new transitive deps,
+# changed maintainer, etc.).
+DECISION_RECALL_SECONDS = 30 * 86400
+
+# Shared with the question-expiry sweep, which parses the same "#N"
+# citations out of the same questions to decide whether a question's
+# subject has already been resolved elsewhere. One regex, one meaning.
+_extract_pr_numbers = extract_referenced_numbers
+
+
+def _record_decisions_from_answer(
+    state_db: StateDB,
+    *,
+    repo: str,
+    question: str,
+    answer: str,
+    session_id: str,
+) -> None:
+    """Best-effort persistence of an operator's verbatim Telegram
+    answer against every PR# mentioned in the question. Failure here
+    is logged but does not break the pipeline — the answer has
+    already been acted on by the agent; losing the persistence layer
+    just means the next sweep will re-ask, which is the pre-fix
+    status quo."""
+    pr_numbers = _extract_pr_numbers(question)
+    if not pr_numbers:
+        return
+    for pr_num in pr_numbers:
+        try:
+            state_db.record_automation_decision(
+                repo=repo,
+                operation="dependabot_pr",
+                item_id=f"#{pr_num}",
+                decision=answer,
+                decided_by="operator",
+                context=(question or "")[:500],
+            )
+        except Exception as e:
+            log_event(
+                _logger,
+                "secops.decision.record_failed",
+                session_id=session_id,
+                repo=repo,
+                pr_number=pr_num,
+                error_type=type(e).__name__,
+                error=str(e)[:200],
+            )
+
+
+def _question_for_persist(session_id: str, result: PipelineResult) -> str:
+    """Return a non-empty question string for pending_resumes storage.
+
+    When a BLOCKED result includes no question text (rare — the prompt
+    is supposed to make the agent write one), fall back to a synthesized
+    placeholder so the row is still persisted. Without this, the empty
+    question branch would silently skip the persist and the session
+    would become unresumable via Telegram."""
+    q = (result.question or "").strip()
+    if q:
+        return q
+    return (
+        f"Session {session_id} is blocked but did not include a "
+        "question. Reply with guidance to resume."
+    )
+
+
+@dataclass
+class SecopsPipeline:
+    """Security operations pipeline for daily triage."""
+
+    dispatcher: AgentAdapter
+    github: GitHubCLI
+    worktree: WorktreeManager
+    dashboard: DashboardClient | None
+    state_db: StateDB
+    transport: Transport | None
+
+    name: str = "secops"
+
+    async def run(self, ctx: PipelineContext) -> PipelineResult:
+        """Run secops on a single repo."""
+        prompt = self._build_prompt(
+            ctx.repo,
+            session_id=ctx.session_id,
+            state_file=ctx.state_file,
+            automation=ctx.extra.get("automation"),
+            prior_decisions=ctx.extra.get("prior_decisions"),
+        )
+
+        result = await self._spawn(ctx, prompt, resume=False)
+        return self._session_to_result(result)
+
+    async def resume(self, ctx: PipelineContext, answer: str) -> PipelineResult:
+        """Resume blocked secops with user answer."""
+        prompt = f"User answered: {answer}\n\nContinue from where you left off."
+
+        # Resume uses Claude's own session UUID (captured on the first spawn
+        # and persisted to state_db). Our composite id is not a UUID and will
+        # be rejected by `claude --resume` on v2.0.x+.
+        resume_uuid = self.state_db.get_agent_session_id(ctx.session_id)
+
+        log_event(
+            _logger,
+            "dev.session.resumed",
+            session_id=ctx.session_id,
+            repo=ctx.repo,
+            issue_number=ctx.issue_number,
+            pipeline=self.name,
+            resume_session_id=resume_uuid,
+            answer_length=len(answer),
+            answer_hash=hash_text(answer),
+        )
+
+        result = await self._spawn(ctx, prompt, resume=True)
+        return self._session_to_result(result)
+
+    async def _spawn(
+        self,
+        ctx: PipelineContext,
+        prompt: str,
+        *,
+        resume: bool,
+    ) -> SessionResult:
+        """Centralized dispatcher call: looks up the agent UUID on resume,
+        persists any newly-captured UUID to state_db."""
+        resume_uuid: str | None = None
+        if resume:
+            resume_uuid = self.state_db.get_agent_session_id(ctx.session_id)
+
+        result = await self.dispatcher.spawn_session(
+            session_id=ctx.session_id,
+            prompt=prompt,
+            working_dir=ctx.worktree_path,
+            state_file=ctx.state_file,
+            resume_session_id=resume_uuid,
+            repo=ctx.repo,
+            issue_number=ctx.issue_number,
+        )
+
+        if result.agent_session_id:
+            try:
+                self.state_db.set_agent_session_id(
+                    ctx.session_id, result.agent_session_id
+                )
+            except Exception:
+                pass
+
+        return result
+
+    def _build_prompt(
+        self,
+        repo: str,
+        session_id: str = "",
+        state_file: Path | None = None,
+        automation: Any = None,
+        prior_decisions: list[dict[str, Any]] | None = None,
+    ) -> str:
+        """Build the secops prompt.
+
+        ``automation`` is an optional ``AutomationConfig`` carrying the
+        per-repo Dependabot policy (auto/ask/never per severity tier).
+        When provided, the prompt explicitly tells the agent which tiers
+        auto-merge vs ask vs never-merge. When ``None``, falls back to
+        the default policy (patch=auto, minor=ask, never-major) so
+        existing callers and the schema defaults stay coherent.
+
+        ``prior_decisions`` is a list of recent automation_decisions
+        rows for this repo. When present, they're rendered as a
+        "Prior operator decisions" block so the agent can act on a
+        past answer instead of asking again about the same PR.
+        """
+        state_file_path = str(state_file) if state_file else "/tmp/state.json"
+
+        # Translate per-repo policy into prompt directives. Use defaults
+        # from AutomationConfig if no policy was passed.
+        patch_pol = (
+            automation.dependabot_patch.value
+            if automation is not None and hasattr(automation, "dependabot_patch")
+            else "auto"
+        )
+        minor_pol = (
+            automation.dependabot_minor.value
+            if automation is not None and hasattr(automation, "dependabot_minor")
+            else "ask"
+        )
+        major_pol = (
+            automation.dependabot_major.value
+            if automation is not None and hasattr(automation, "dependabot_major")
+            else "never"
+        )
+
+        def _policy_directive(tier: str, policy: str) -> str:
+            if policy == "auto":
+                return (
+                    f"     - {tier}: AUTO-MERGE with passing CI. Squash + "
+                    "delete branch."
+                )
+            if policy == "ask":
+                return (
+                    f"     - {tier}: ASK — signal BLOCKED with a one-line "
+                    f"summary so the operator decides."
+                )
+            return (
+                f"     - {tier}: NEVER — do not merge. Leave the PR open "
+                "for manual review."
+            )
+
+        dependabot_policy_block = "\n".join(
+            [
+                _policy_directive("patch updates", patch_pol),
+                _policy_directive("minor updates", minor_pol),
+                _policy_directive("major updates", major_pol),
+            ]
+        )
+
+        # Render the rolling window of prior operator decisions for this
+        # repo. The agent reads the verbatim Telegram answer and decides
+        # whether the prior call still applies — we don't try to
+        # normalize "yes"/"approve"/"merge it" to a canonical decision
+        # because that classification is exactly what the agent does
+        # best, and getting it wrong silently auto-merges PRs.
+        #
+        # We render the stored ``context`` (question snippet captured
+        # at decision time) alongside item_id + answer so the agent
+        # can compare prior bump/CI state with the PR it's looking at
+        # right now. Without context, the "only re-ASK if circumstances
+        # materially changed" rule below is unenforceable — the agent
+        # can't detect a force-pushed PR that swapped the version bump
+        # under the same PR number.
+        if prior_decisions:
+            lines = ["## Prior operator decisions (last 30 days)", ""]
+            for d in prior_decisions:
+                ts = d.get("decided_at") or 0
+                date = (
+                    time.strftime("%Y-%m-%d", time.gmtime(int(ts)))
+                    if ts else "unknown-date"
+                )
+                item = d.get("item_id") or "?"
+                ans = (d.get("decision") or "").strip().replace("\n", " ")
+                # Cap each answer so a pathological Telegram reply
+                # doesn't bloat the prompt past the model's context.
+                if len(ans) > 160:
+                    ans = ans[:157] + "..."
+                ctx = (d.get("context") or "").strip().replace("\n", " ")
+                # Same cap rationale as answers; the prompt has many
+                # entries and at-scale this is the bigger context risk.
+                if len(ctx) > 240:
+                    ctx = ctx[:237] + "..."
+                if ctx:
+                    lines.append(
+                        f'- {date} {item}: operator said "{ans}" '
+                        f"(prior question: {ctx})"
+                    )
+                else:
+                    lines.append(
+                        f'- {date} {item}: operator said "{ans}"'
+                    )
+            lines.append("")
+            lines.append(
+                "**Use these to avoid re-asking.** If a Dependabot PR "
+                "matches an item above and circumstances are unchanged "
+                "(same package, version range still applies, CI state "
+                "still the same — compare the current PR against the "
+                "prior-question snippet above), act per the prior "
+                "decision without signalling BLOCKED. Only re-ASK if "
+                "the situation has materially changed (different "
+                "version bump, CI flipped red, new conflicts) — and "
+                "quote what changed in the question."
+            )
+            prior_decisions_block = "\n".join(lines) + "\n\n"
+        else:
+            prior_decisions_block = ""
+
+        return f"""Execute security operations for repository {repo}.
+
+1. Check Dependabot alerts:
+   `gh api repos/{repo}/dependabot/alerts --jq '.[] | select(.state=="open")'`
+2. Check Dependabot-authored security PRs:
+   `gh pr list --repo {repo} --author "app/dependabot" --json number,title`
+3. Determine the trusted operator's GitHub login (the identity the
+   agent itself runs as). The auto-merge carve-out below applies ONLY
+   to PRs authored by this exact login — never to other contributors,
+   apps, or bots even if their PR looks safe:
+   `OPERATOR=$(gh api user --jq '.login')`
+4. Check open PRs authored by the trusted operator that enable
+   Dependabot itself. These are prerequisites for future Dependabot
+   work and block the security pipeline if left open:
+   `gh pr list --repo {repo} --state open --author "$OPERATOR" --json number,title,files`
+   For each, inspect files: `gh pr view <NUM> --repo {repo} --json files`
+5. For each alert or PR:
+   - **Dependabot PRs**: classify the update as patch/minor/major and
+     apply the per-repo policy below. If the severity is unclear, signal
+     BLOCKED to ask for guidance.
+
+{dependabot_policy_block}
+
+{prior_decisions_block}**Edge cases that override the policy ladder above:**
+
+- **No CI configured** (repo has no `.github/workflows/*.yml`): the
+  "passing CI" gate cannot be evaluated, so an "auto" policy cannot
+  fire safely. Treat ALL open Dependabot PRs as ASK regardless of
+  their tier — signal BLOCKED ONCE for the whole repo with a single
+  consolidated question naming every open PR, e.g.: "Repo has no CI;
+  approve merging: #N (pkg X -> Y, patch), #M (pkg A -> B, minor)?"
+  Do not ask one BLOCKED per PR — operators get one consolidated
+  question per repo per sweep.
+
+- **CI failing on a check UNRELATED to the PR's own package** (e.g.
+  pip-audit fails on a transitive CVE in `idna` while the PR bumps
+  `boto3`): do NOT silently report this and move on — that's how
+  patch PRs end up stuck for weeks. Signal BLOCKED with a one-line
+  question naming the underlying issue: "Patch PRs blocked by
+  pre-existing pip-audit failure on idna CVE-2026-... in lockfile.
+  Fix idna >= 3.15 first, then re-evaluate #117 and #119?" Include
+  the failing check name and a one-line root-cause hint extracted
+  from `gh run view --log-failed`.
+
+**Scope discipline:**
+
+You have exactly three legitimate exits for each open PR:
+  1. Merge it (per policy or per prior decision).
+  2. Leave it open (per policy or per prior decision).
+  3. Signal BLOCKED with a one-line policy question.
+
+Do NOT propose alternative actions outside this ladder. In
+particular, do NOT offer to: batch multiple Dependabot PRs into a
+single review PR, open a tracking issue for the operator, perform a
+manual code review of the bump, or any other freelance option. If
+the right action isn't clear from policy + prior decisions + edge
+cases above, ASK with a one-line question — don't invent.
+
+Every BLOCKED question must contain ONLY items where the operator's
+answer changes what you do next. Never pad a question with items you
+have already decided — no "confirm no action wanted", no "left open
+per policy" tails, no FYI clauses. Those belong in the step-6
+summary. If nothing in the repo needs a decision, do not signal
+BLOCKED at all: finish DONE and say what you left alone.
+
+   - **PR authored by `$OPERATOR` touching ONLY `.github/dependabot.yml`**
+     (additive ecosystem entries, no other files changed): MAYBE
+     auto-merge — but only after diff validation. These are the
+     "enable Dependabot" prerequisites the operator opened.
+
+     Required validation BEFORE merging:
+     a. **Author check**: confirm `author.login` exactly equals
+        `$OPERATOR`. Collaborators, GitHub apps, or external
+        contributors can craft a `dependabot.yml`-only PR to slip
+        past review — those go to BLOCKED, not auto-merge.
+     b. **Diff check**: pull the diff and confirm it is PURELY
+        ADDITIVE — no `-` lines (deletions or modifications of
+        existing stanzas), only `+` lines that introduce new
+        `package-ecosystem` blocks. Run:
+        `gh pr diff <NUM> --repo {repo}`
+        If any line in the diff begins with `-` (other than `---`
+        file headers), the PR modifies or removes existing config:
+        signal BLOCKED with a one-line summary like "operator
+        dependabot.yml PR #N modifies existing config — needs
+        review". Do NOT auto-merge.
+     c. **CI check**: confirm all status checks pass (matches the
+        Dependabot PR rule above). Run:
+        `gh pr checks <NUM> --repo {repo}`
+        Any failing or pending check -> BLOCKED. A PR that adds
+        invalid dependabot YAML can pass author+diff but fail
+        repo validation; without this gate the agent could merge
+        broken config and break Dependabot for the repo.
+
+     If (a), (b), AND (c) all pass: merge with squash, delete the
+     branch. If any one fails: BLOCKED.
+   - **Any other PR from `$OPERATOR`** (touches code, tests, configs
+     other than dependabot.yml): signal BLOCKED for operator approval.
+     Never auto-merge code changes, even from the trusted operator.
+   - **PRs from anyone else** (collaborators, contributors, other bots):
+     out of scope — see the PR scope fence below. Do not fetch them, do
+     not evaluate them, do not name them, do not signal BLOCKED on them.
+6. Summarize actions taken. Report only what you acted on or decided
+   under the queries in steps 1-4. Do not report counts or descriptions
+   of PRs outside that scope.
+
+**PR scope fence:**
+
+The ONLY open PRs in scope are the ones returned by the two author-
+filtered queries above: `--author "app/dependabot"` (step 2) and
+`--author "$OPERATOR"` (step 4). That is the complete universe of PRs
+this pipeline may look at.
+
+Never run an unfiltered `gh pr list` (or any other broad PR/issue
+search) to discover PRs beyond those two queries. A third party's open
+PR is not this pipeline's business: it cannot be merged here, it needs
+no decision here, and surfacing it only produces noise. If you catch
+yourself about to write a phrase like "no open PRs at all" or "N open
+PRs total", you have already stepped outside the fence — you can only
+speak to the author-filtered results.
+
+## Signaling Completion
+
+**CRITICAL**: Before exiting, you MUST write a checkpoint file to signal completion.
+
+STATE_FILE: {state_file_path}
+SESSION_ID: {session_id}
+
+**DONE** (completed):
+```bash
+mkdir -p "$(dirname '{state_file_path}')"
+printf '{{"version":"1","status":"DONE","session_id":"{session_id}",'\
+'"timestamp":"%s","summary":"%s"}}' \\
+  "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "<SUMMARY>" > '{state_file_path}'
+```
+
+**BLOCKED** (need input):
+```bash
+mkdir -p "$(dirname '{state_file_path}')"
+printf '{{"version":"1","status":"BLOCKED_NEEDS_INPUT",'\
+'"session_id":"{session_id}","timestamp":"%s","question":"%s"}}' \\
+  "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "<QUESTION>" > '{state_file_path}'
+```
+
+**FAILED**:
+```bash
+mkdir -p "$(dirname '{state_file_path}')"
+printf '{{"version":"1","status":"FAILED","session_id":"{session_id}",'\
+'"timestamp":"%s","error":"%s"}}' \\
+  "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "<ERROR>" > '{state_file_path}'
+```"""
+
+    def _session_to_result(self, result: SessionResult) -> PipelineResult:
+        """Convert SessionResult to PipelineResult."""
+        if result.state is None:
+            return PipelineResult(
+                success=False,
+                session_id=result.session_id,
+                summary="No checkpoint state returned",
+                error=result.stderr or "Unknown error",
+            )
+
+        if result.state.status == CheckpointStatus.DONE:
+            return PipelineResult(
+                success=True,
+                session_id=result.session_id,
+                summary=result.state.summary or "Completed",
+                outputs=result.state.outputs,
+            )
+
+        if result.state.status == CheckpointStatus.BLOCKED_NEEDS_INPUT:
+            return PipelineResult(
+                success=False,
+                session_id=result.session_id,
+                summary="Blocked on user input",
+                blocked=True,
+                question=result.state.question,
+            )
+
+        return PipelineResult(
+            success=False,
+            session_id=result.session_id,
+            summary="Failed",
+            error=result.state.error,
+        )
+
+
+async def run_secops_all(
+    repos: list[Any],
+    dispatcher: AgentAdapter,
+    github: GitHubCLI,
+    worktree: WorktreeManager,
+    dashboard: DashboardClient | None,
+    state_db: StateDB,
+    transport: Transport | None,
+    contexts_dir: Path,
+    max_blocked_rounds: int = DEFAULT_MAX_BLOCKED_ROUNDS,
+) -> list[PipelineResult]:
+    """Run secops pipeline on all configured repos."""
+    results = []
+
+    pipeline = SecopsPipeline(
+        dispatcher=dispatcher,
+        github=github,
+        worktree=worktree,
+        dashboard=dashboard,
+        state_db=state_db,
+        transport=transport,
+    )
+
+    for repo_config in repos:
+        repo = repo_config.name
+        session_id = f"secops-{repo.replace('/', '-')}-{uuid.uuid4().hex[:8]}"
+
+        if not state_db.acquire_lock(repo, session_id):
+            results.append(PipelineResult(
+                success=False,
+                session_id=session_id,
+                summary=f"Could not acquire lock for {repo}",
+                error="Repository locked by another session",
+            ))
+            continue
+
+        worktree_path: Path | None = None
+        session_row_inserted = False
+        session_final_state_written = False
+        try:
+            await worktree.ensure_bare_repo(repo)
+            worktree_path = await worktree.create_worktree(repo, session_id)
+
+            context_path = contexts_dir / repo.replace("/", "-") / "CLAUDE.md"
+            if context_path.exists():
+                worktree.symlink_context(worktree_path, context_path)
+
+            state_file = worktree_path / ".ctrlrelay" / "state.json"
+            state_file.parent.mkdir(parents=True, exist_ok=True)
+
+            # Pull the 30-day rolling window of prior operator decisions
+            # so the prompt can include them. Best-effort: if the lookup
+            # fails for any reason, fall through with an empty list —
+            # losing this just means the next sweep re-asks, which is the
+            # pre-fix status quo, not a regression.
+            try:
+                prior_decisions = state_db.list_recent_automation_decisions(
+                    repo,
+                    operation="dependabot_pr",
+                    since_ts=int(time.time()) - DECISION_RECALL_SECONDS,
+                )
+            except Exception as e:
+                log_event(
+                    _logger,
+                    "secops.prior_decisions.lookup_failed",
+                    session_id=session_id,
+                    repo=repo,
+                    error_type=type(e).__name__,
+                    error=str(e)[:200],
+                )
+                prior_decisions = []
+
+            ctx = PipelineContext(
+                session_id=session_id,
+                repo=repo,
+                worktree_path=worktree_path,
+                context_path=context_path,
+                state_file=state_file,
+                extra={
+                    "automation": getattr(repo_config, "automation", None),
+                    "prior_decisions": prior_decisions,
+                },
+            )
+
+            state_db.execute(
+                """INSERT INTO sessions (id, pipeline, repo, worktree_path, status, started_at)
+                   VALUES (?, ?, ?, ?, ?, ?)""",
+                (session_id, "secops", repo, str(worktree_path), "running", int(time.time())),
+            )
+            state_db.commit()
+            session_row_inserted = True
+
+            result = await pipeline.run(ctx)
+
+            # BLOCKED loop: when the agent ends BLOCKED_NEEDS_INPUT, push
+            # the question to the operator via the configured transport
+            # (Telegram), wait for an answer, and resume. Mirrors the
+            # dev/task pipelines — without this, secops blocked sessions
+            # silently land as "blocked" in the DB without ever reaching
+            # the operator. transport.ask() failure preserves blocked=True
+            # so the persistence-on-blocked branch below still fires and
+            # writes a pending_resumes row that a later out-of-band reply
+            # could resume via the sweeper.
+            rounds = 0
+            while (
+                result.blocked
+                and transport is not None
+                and rounds < max_blocked_rounds
+            ):
+                question = _question_for_persist(session_id, result)
+                try:
+                    answer = await transport.ask(
+                        question,
+                        session_id=session_id,
+                        repo=repo,
+                    )
+                except Exception as e:
+                    log_event(
+                        _logger,
+                        "secops.transport.ask_failed",
+                        session_id=session_id,
+                        repo=repo,
+                        error_type=type(e).__name__,
+                        error=str(e)[:200],
+                    )
+                    result = PipelineResult(
+                        success=False,
+                        blocked=True,
+                        session_id=session_id,
+                        summary=f"Blocked session deferred (transport): {e}",
+                        error=str(e),
+                        question=question,
+                        outputs=result.outputs,
+                    )
+                    break
+                _record_decisions_from_answer(
+                    state_db,
+                    repo=repo,
+                    question=question,
+                    answer=answer,
+                    session_id=session_id,
+                )
+                rounds += 1
+                result = await pipeline.resume(ctx, answer)
+
+            results.append(result)
+
+            status = "done" if result.success else ("blocked" if result.blocked else "failed")
+            state_db.execute(
+                "UPDATE sessions SET status = ?, summary = ?, ended_at = ? WHERE id = ?",
+                (status, result.summary, int(time.time()), session_id),
+            )
+            state_db.commit()
+            session_final_state_written = True
+
+            # Persist BLOCKED so a Telegram reply arriving after this
+            # session tears down can still be routed to a resume. Without
+            # this, the bridge's in-memory `_pending_questions` dies with
+            # the ASK socket and the operator's answer lands in a void.
+            if result.blocked:
+                try:
+                    state_db.add_pending_resume(
+                        session_id=session_id,
+                        pipeline="secops",
+                        repo=repo,
+                        question=_question_for_persist(session_id, result),
+                    )
+                except Exception as e:
+                    log_event(
+                        _logger,
+                        "secops.pending_resume.insert_failed",
+                        session_id=session_id,
+                        repo=repo,
+                        error_type=type(e).__name__,
+                        error=str(e)[:200],
+                    )
+
+            if dashboard and result.success:
+                await dashboard.push_event(EventPayload(
+                    level="info",
+                    pipeline="secops",
+                    repo=repo,
+                    message=result.summary,
+                    session_id=session_id,
+                ))
+
+        except asyncio.CancelledError:
+            # Scheduled secops interrupted mid-run (SIGTERM during a
+            # scheduler.shutdown). Mark the session as cancelled so later
+            # inspection doesn't see a phantom "running" row — but ONLY
+            # if we hadn't already written a final state. Without this
+            # guard, a cancel landing during the post-run dashboard push
+            # would clobber a successful "done" status with "cancelled"
+            # even though the work completed.
+            if session_row_inserted and not session_final_state_written:
+                try:
+                    state_db.execute(
+                        "UPDATE sessions SET status = ?, summary = ?, "
+                        "ended_at = ? WHERE id = ?",
+                        (
+                            "cancelled",
+                            "Cancelled during shutdown",
+                            int(time.time()),
+                            session_id,
+                        ),
+                    )
+                    state_db.commit()
+                except Exception:
+                    pass
+            raise
+
+        except Exception as e:
+            # Never let a repo fail silently. Two independent holes put
+            # the cause beyond reach for the operator:
+            #   1. No log_event here, so nothing reached poller.log.
+            #   2. The DB row is written only when the failure lands
+            #      AFTER the INSERT — anything raised by ensure_bare_repo
+            #      or create_worktree (both network-bound, both before
+            #      it) left no row at all.
+            # With an exception whose str() is empty (a bare
+            # asyncio.TimeoutError) the Telegram alert then fell back to
+            # the generic summary and named only the repo. Log first, so
+            # the cause survives regardless of which branch runs next.
+            detail = str(e) or type(e).__name__
+            # This is the last-resort handler for the whole repo, and
+            # every repo after this one in the sweep depends on it
+            # returning normally. A raising logger here would abort the
+            # remaining repos AND destroy the original exception —
+            # trading one repo's failure for the entire pass. Losing the
+            # log line is the strictly better failure.
+            try:
+                log_event(
+                    _logger,
+                    "secops.repo.failed",
+                    session_id=session_id,
+                    repo=repo,
+                    session_row_inserted=session_row_inserted,
+                    error_type=type(e).__name__,
+                    error=detail[:200],
+                )
+            except Exception:
+                pass
+            if session_row_inserted:
+                state_db.execute(
+                    "UPDATE sessions SET status = ?, summary = ?, "
+                    "ended_at = ? WHERE id = ?",
+                    ("failed", f"Error: {detail}", int(time.time()), session_id),
+                )
+                state_db.commit()
+            results.append(PipelineResult(
+                success=False,
+                session_id=session_id,
+                summary=f"Error processing {repo}: {type(e).__name__}: {detail}",
+                # `str(e)` alone can be empty; the notifier picks
+                # `result.error or result.summary`, so an empty error
+                # silently degraded the alert to the bare summary.
+                error=f"{type(e).__name__}: {detail}",
+            ))
+
+        finally:
+            # Hold the per-repo lock THROUGH worktree cleanup so a
+            # concurrent dev/secops run can't acquire the same repo and
+            # race `git worktree prune` on the shared bare clone.
+            # Cleanup is bounded by an asyncio.wait_for timeout so a
+            # truly-stuck operation can't pin the lock forever — once
+            # the timeout fires we release anyway, preferring "possibly
+            # leaked git admin state (recoverable via `git worktree
+            # prune`)" over "repo locked across daemon restart".
+            #
+            # No asyncio.shield here: `_run_git` kills its subprocess
+            # synchronously on CancelledError so unshielded cleanup
+            # unwinds fast and without leaking a stray git. Previously
+            # the shield made the outer task cancel immediately while
+            # the inner cleanup kept running untracked — leaking state.
+            if worktree_path is not None:
+                try:
+                    worktree.remove_context_symlink(worktree_path)
+                except Exception as cleanup_exc:
+                    log_event(
+                        _logger,
+                        "secops.cleanup.symlink_failed",
+                        session_id=session_id,
+                        repo=repo,
+                        error_type=type(cleanup_exc).__name__,
+                        error=str(cleanup_exc)[:200],
+                    )
+                try:
+                    # Timeout matches Scheduler's cancel budget (150s) — a
+                    # full worktree prune can take up to 120s per
+                    # WorktreeManager._run_git ceiling.
+                    await asyncio.wait_for(
+                        worktree.remove_worktree(repo, session_id),
+                        timeout=130.0,
+                    )
+                except asyncio.TimeoutError:
+                    log_event(
+                        _logger,
+                        "secops.cleanup.worktree_timeout",
+                        session_id=session_id,
+                        repo=repo,
+                        worktree_path=str(worktree_path),
+                    )
+                except asyncio.CancelledError:
+                    log_event(
+                        _logger,
+                        "secops.cleanup.worktree_cancelled_mid_shutdown",
+                        session_id=session_id,
+                        repo=repo,
+                        worktree_path=str(worktree_path),
+                    )
+                    # Release the lock before propagating — leaving it
+                    # held across a daemon restart is worse than the
+                    # small race with a future scheduled run (the dev
+                    # handler retries on lock conflict anyway).
+                    try:
+                        state_db.release_lock(repo, session_id)
+                    except Exception:
+                        pass
+                    raise
+                except Exception as cleanup_exc:
+                    log_event(
+                        _logger,
+                        "secops.cleanup.worktree_failed",
+                        session_id=session_id,
+                        repo=repo,
+                        worktree_path=str(worktree_path),
+                        error_type=type(cleanup_exc).__name__,
+                        error=str(cleanup_exc)[:200],
+                    )
+
+            # Release the lock AFTER cleanup attempt so new runs don't
+            # race the prune. In the cancel path above we released early
+            # and re-raised; here we handle the non-cancel paths.
+            try:
+                state_db.release_lock(repo, session_id)
+            except Exception as lock_exc:
+                log_event(
+                    _logger,
+                    "secops.cleanup.lock_release_failed",
+                    session_id=session_id,
+                    repo=repo,
+                    error_type=type(lock_exc).__name__,
+                    error=str(lock_exc)[:200],
+                )
+
+    return results
+
+
+async def resume_secops_from_pending(
+    session_id: str,
+    repo: str,
+    answer: str,
+    dispatcher: AgentAdapter,
+    github: GitHubCLI,
+    worktree: WorktreeManager,
+    dashboard: DashboardClient | None,
+    state_db: StateDB,
+    transport: Transport | None,
+    contexts_dir: Path,
+    automation: Any = None,
+    question: str | None = None,
+) -> PipelineResult:
+    """Resume a BLOCKED secops session using an answer that arrived via
+    Telegram after the original session had already torn down.
+
+    Mirrors ``run_secops_all``'s per-repo lifecycle (lock → worktree →
+    resume → cleanup) but targets a single session that's already got a
+    row in ``sessions`` (status='blocked') and a ``pending_resumes`` row
+    ready for ``mark_pending_resume_resumed``. Caller is responsible for
+    flipping the pending_resumes row after this returns.
+
+    ``question`` is the original BLOCKED question (carried in the
+    pending_resumes row). When supplied, the operator answer is
+    persisted against any PR# the question mentioned so the next
+    sweep skips re-asking. Optional for backwards compat with older
+    callers that pre-date the persistence work.
+    """
+    if question:
+        _record_decisions_from_answer(
+            state_db,
+            repo=repo,
+            question=question,
+            answer=answer,
+            session_id=session_id,
+        )
+    pipeline = SecopsPipeline(
+        dispatcher=dispatcher,
+        github=github,
+        worktree=worktree,
+        dashboard=dashboard,
+        state_db=state_db,
+        transport=transport,
+    )
+
+    if not state_db.acquire_lock(repo, session_id):
+        return PipelineResult(
+            success=False,
+            session_id=session_id,
+            summary=f"Could not acquire lock for {repo} to resume",
+            error="Repository locked by another session",
+        )
+
+    worktree_path: Path | None = None
+    try:
+        await worktree.ensure_bare_repo(repo)
+        worktree_path = await worktree.create_worktree(repo, session_id)
+
+        context_path = contexts_dir / repo.replace("/", "-") / "CLAUDE.md"
+        if context_path.exists():
+            worktree.symlink_context(worktree_path, context_path)
+
+        state_file = worktree_path / ".ctrlrelay" / "state.json"
+        state_file.parent.mkdir(parents=True, exist_ok=True)
+
+        ctx = PipelineContext(
+            session_id=session_id,
+            repo=repo,
+            worktree_path=worktree_path,
+            context_path=context_path,
+            state_file=state_file,
+            extra={"automation": automation},
+        )
+
+        # Flip the session back to running so an observer sees progress,
+        # not a stale 'blocked' row while the resume is in flight.
+        state_db.execute(
+            "UPDATE sessions SET status = ?, ended_at = NULL WHERE id = ?",
+            ("running", session_id),
+        )
+        state_db.commit()
+
+        result = await pipeline.resume(ctx, answer)
+
+        status = "done" if result.success else (
+            "blocked" if result.blocked else "failed"
+        )
+        state_db.execute(
+            "UPDATE sessions SET status = ?, summary = ?, ended_at = ? "
+            "WHERE id = ?",
+            (status, result.summary, int(time.time()), session_id),
+        )
+        state_db.commit()
+
+        # If the resume re-blocked (operator answer was ambiguous, agent
+        # needs more), refresh the pending_resumes row so a new reply
+        # routes to this same session again.
+        if result.blocked:
+            try:
+                state_db.add_pending_resume(
+                    session_id=session_id,
+                    pipeline="secops",
+                    repo=repo,
+                    question=_question_for_persist(session_id, result),
+                )
+            except Exception as e:
+                log_event(
+                    _logger,
+                    "secops.pending_resume.reblock_insert_failed",
+                    session_id=session_id,
+                    repo=repo,
+                    error_type=type(e).__name__,
+                    error=str(e)[:200],
+                )
+
+        return result
+
+    except Exception as e:
+        state_db.execute(
+            "UPDATE sessions SET status = ?, summary = ?, ended_at = ? "
+            "WHERE id = ?",
+            ("failed", f"Resume error: {e}", int(time.time()), session_id),
+        )
+        state_db.commit()
+        return PipelineResult(
+            success=False,
+            session_id=session_id,
+            summary=f"Error resuming {repo}",
+            error=str(e),
+        )
+
+    finally:
+        # Same cleanup pattern as run_secops_all: symlink → worktree →
+        # lock, bounded timeouts, log-don't-crash.
+        if worktree_path is not None:
+            try:
+                worktree.remove_context_symlink(worktree_path)
+            except Exception as cleanup_exc:
+                log_event(
+                    _logger,
+                    "secops.resume.cleanup.symlink_failed",
+                    session_id=session_id,
+                    repo=repo,
+                    error_type=type(cleanup_exc).__name__,
+                    error=str(cleanup_exc)[:200],
+                )
+            try:
+                await asyncio.wait_for(
+                    worktree.remove_worktree(repo, session_id),
+                    timeout=130.0,
+                )
+            except asyncio.TimeoutError:
+                log_event(
+                    _logger,
+                    "secops.resume.cleanup.worktree_timeout",
+                    session_id=session_id,
+                    repo=repo,
+                    worktree_path=str(worktree_path),
+                )
+            except Exception as cleanup_exc:
+                log_event(
+                    _logger,
+                    "secops.resume.cleanup.worktree_failed",
+                    session_id=session_id,
+                    repo=repo,
+                    worktree_path=str(worktree_path),
+                    error_type=type(cleanup_exc).__name__,
+                    error=str(cleanup_exc)[:200],
+                )
+
+        try:
+            state_db.release_lock(repo, session_id)
+        except Exception as lock_exc:
+            log_event(
+                _logger,
+                "secops.resume.cleanup.lock_release_failed",
+                session_id=session_id,
+                repo=repo,
+                error_type=type(lock_exc).__name__,
+                error=str(lock_exc)[:200],
+            )