codna 0.2.73__tar.gz → 0.2.75__tar.gz

{codna-0.2.73 → codna-0.2.75}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codna
-Version: 0.2.73
+Version: 0.2.75
 Summary: Local-first repo intelligence and agent fixing for Codna.
 Author: Thyn
 Maintainer: Thyn

{codna-0.2.73 → codna-0.2.75}/codna/__init__.py

@@ -5,4 +5,4 @@ This package is the `codna` CLI. Users never touch "algenta" directly — the en
 is an internal dependency reached over HTTP.
 """
 
-__version__ = "0.2.73"
+__version__ = "0.2.75"

{codna-0.2.73 → codna-0.2.75}/codna/webhook.py

@@ -29,6 +29,7 @@ import json
 import os
 import re
 import sys
+import threading
 from dataclasses import dataclass
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
@@ -668,13 +669,14 @@ def default_queue_path() -> str:
 
 # --- self-hostable server: verify + enqueue (thin ingress) ----------------------------
 def _log_delivery(event: str, payload: Mapping[str, Any], delivery_id: str | None, outcome: str,
-                  job: WebhookJob | None = None) -> None:
+                  job: WebhookJob | None = None, **fields: Any) -> None:
     """One stderr JSON line per verified delivery: what arrived and what became of it.
 
     Until now a delivery that classified as `ignored` or collapsed as a `duplicate` left no trace
     anywhere -- on 2026-09-17 a `@codna fix` reply produced neither a job nor a log line and the
     only way to tell "never delivered" from "delivered and dropped" was to guess. Same shape as the
     worker's lines so one grep covers ingress and processing (`"service": "codna-webhook"`).
+    ``fields`` are extra keys a path wants on record (the Check Run the ingress opened, for one).
     """
     repo = (payload.get("repository") or {}).get("full_name")
     line = {
@@ -688,6 +690,7 @@ def _log_delivery(event: str, payload: Mapping[str, Any], delivery_id: str | Non
     }
     if job is not None:
         line.update({"kind": job.kind, "reason": job.reason, "pr_number": job.pr_number})
+    line.update(fields)
     print(json.dumps(line, sort_keys=True), file=sys.stderr, flush=True)
 
 
@@ -797,7 +800,35 @@ class _Handler(BaseHTTPRequestHandler):
             _log_delivery(event, payload, delivery_id, "accepted", job)
             self._reply(202, {"status": "accepted", "kind": job.kind, "repo": job.repo_full_name})
             return
-        fresh = queue.enqueue(job, delivery_id=delivery_id)
+        # The review's Check Run is opened `queued` HERE, before the row exists, so the tenant sees
+        # it the moment the delivery is accepted rather than when a worker slot frees (measured
+        # 2026-09-20: p95 23 s invisible during a 21-review fan-out). The row is born owning the
+        # run; the worker updates it. webhook_queued_check has the whole lifecycle; the service
+        # switches this on for the Postgres backend only (webhook_service.serve).
+        queued_check = bool(getattr(self.server, "queued_check_runs", False))
+        pre = None
+        late_gate: threading.Event | None = None
+        if queued_check:
+            from . import webhook_queued_check
+
+            late_gate = threading.Event()
+            pre = webhook_queued_check.precreate(
+                job, queue=queue, delivery_id=delivery_id, github=self._github(),
+                app_id=os.environ.get("GITHUB_APP_ID"), private_key=os.environ.get("GITHUB_APP_PRIVATE_KEY"),
+                late_gate=late_gate)
+        try:
+            if pre is not None:
+                fresh = queue.enqueue(job, delivery_id=delivery_id, check_run_id=pre.check_run_id)
+            else:
+                fresh = queue.enqueue(job, delivery_id=delivery_id)
+        finally:
+            if late_gate is not None:
+                late_gate.set()  # a create that answers after the budget may now adopt onto this row
+        check_run: dict[str, Any] = {}
+        if pre is not None:
+            bound = webhook_queued_check.bind_or_close(pre, job=job, queue=queue, delivery_id=delivery_id,
+                                                      fresh=fresh, github=self._github())
+            check_run = {"check_run_id": pre.check_run_id, "check_run": bound}
         # A new head retires the work the old one left behind. enqueue() already collapsed the
         # WAITING rows; this stops the ones already RUNNING, whose verdict would land on a commit
         # that no longer exists (thyn-ai/mojo-kernels#32, 2026-09-19: two reviews ran to completion
@@ -809,13 +840,41 @@ class _Handler(BaseHTTPRequestHandler):
                 superseded = webhook_control.supersede_running(job, queue=queue)
             except Exception:  # noqa: BLE001 -- a delivery must be accepted even if cancelling fails
                 superseded = 0
-        _log_delivery(event, payload, delivery_id, "accepted" if fresh else "duplicate", job)
+            if queued_check and job.pr_number is not None:
+                # ... and the WAITING rows enqueue() just retired had Check Runs of their own, opened
+                # queued by this ingress: complete them neutral ("superseded by <sha>") the way the
+                # running ones are, off the request thread. The reaper sweeps whatever this misses.
+                # The pre-create's token is scoped to this repository and may complete those runs;
+                # without one (a `fix` retiring a review's row) a token is minted per row.
+                token_for = webhook_queued_check.token_for(
+                    github=self._github(), app_id=os.environ.get("GITHUB_APP_ID"),
+                    private_key=os.environ.get("GITHUB_APP_PRIVATE_KEY"))
+                if pre is not None:
+                    def token_for(_row: Any, _token: str = pre.token) -> str:  # one token, one repository
+                        return _token
+                try:
+                    webhook_queued_check.settle_retired_in_background(
+                        queue, github=self._github(), repo=job.repo_full_name, pr_number=job.pr_number,
+                        token_for=token_for)
+                except Exception:  # noqa: BLE001 -- the reaper's sweep is the backstop
+                    pass
+        _log_delivery(event, payload, delivery_id, "accepted" if fresh else "duplicate", job, **check_run)
         self._reply(
             202 if fresh else 200,
             {"status": "accepted" if fresh else "duplicate", "kind": job.kind,
-             "repo": job.repo_full_name, "reason": job.reason, "superseded_running": superseded},
+             "repo": job.repo_full_name, "reason": job.reason, "superseded_running": superseded, **check_run},
         )
 
+    def _github(self) -> Any:
+        """The GitHub client the ingress opens and closes Check Runs with: an object the service
+        (or a test) hung on the server, else the real module."""
+        client = getattr(self.server, "github", None)
+        if client is not None:
+            return client
+        from . import webhook_github
+
+        return webhook_github
+
 
 def require_webhook_secret(environ: Mapping[str, str] | None = None) -> str:
     """Return the configured webhook secret or fail before opening a listener."""

{codna-0.2.73 → codna-0.2.75}/codna/webhook_backend.py

@@ -122,7 +122,11 @@ class ShadowQueue:
     def shadow(self) -> Any:
         return self._shadow
 
-    def enqueue(self, job: WebhookJob, *, delivery_id: str | None, priority: int = 0, unless_pending: bool = False) -> bool:
+    def enqueue(self, job: WebhookJob, *, delivery_id: str | None, priority: int = 0, unless_pending: bool = False,
+                check_run_id: int | None = None) -> bool:
+        # check_run_id is accepted for signature parity with the Postgres queue and ignored: the
+        # ingress never pre-creates a Check Run on this backend (SQLite runs the job and creates
+        # the run at claim), so a caller can never hand one in.
         fresh = self._primary.enqueue(job, delivery_id=delivery_id, priority=priority, unless_pending=unless_pending)
         if fresh:
             try:
@@ -216,20 +220,26 @@ class SpoolingQueue:
         return sum(counts.get(k, 0) for k in ("queued", "retry", "running"))
 
     # -- enqueue: Postgres first, the file when it cannot answer ------------------------------------
-    def enqueue(self, job: WebhookJob, *, delivery_id: str | None, priority: int = 0, unless_pending: bool = False) -> bool:
+    def enqueue(self, job: WebhookJob, *, delivery_id: str | None, priority: int = 0, unless_pending: bool = False,
+                check_run_id: int | None = None) -> bool:
         # The late path (a Postgres write that answers after the timeout) retires the SPOOL copy,
         # so it must not look before that copy exists: it waits on this gate, which the timeout
         # branch below sets once its spool write is done (success or failure). Ordered, not timed.
         spool_written = threading.Event()
         try:
             fresh = _call_with_timeout(
-                lambda: self._primary.enqueue(job, delivery_id=delivery_id, priority=priority, unless_pending=unless_pending),
+                lambda: self._primary.enqueue(job, delivery_id=delivery_id, priority=priority, unless_pending=unless_pending,
+                                              check_run_id=check_run_id),
                 self._enqueue_timeout_s,
                 on_late=lambda accepted: self._retire_spooled(delivery_id) if accepted else None,
                 late_gate=spool_written)
         except Exception as exc:  # noqa: BLE001 -- the file takes it; GitHub still gets its 202
             self._mark_down()
             try:
+                # The spool row does NOT carry the pre-created Check Run: the file has no owner for
+                # it (no reaper sweeps the spool), and the ingress, finding the run bound to no
+                # Postgres row, closes it at once (webhook_queued_check.bind_or_close). The worker
+                # that eventually claims the forwarded row creates the run then, as it always has.
                 fresh = self._spool.enqueue(job, delivery_id=delivery_id, priority=priority, unless_pending=unless_pending)
             finally:
                 spool_written.set()

{codna-0.2.73 → codna-0.2.75}/codna/webhook_github.py

@@ -22,8 +22,15 @@ _API = "https://api.github.com"
 # round-trip happens once, not per job.
 _WORKFLOWS_GRANTED: dict[str, bool] = {}
 
+# installation id -> whether the App has the `statuses` permission there (Commit statuses: read),
+# learned the same way from the first review-token request per process. Requesting a permission the
+# installation never granted makes GitHub refuse the WHOLE mint (422), so an installation without it
+# gets a review token without `statuses` -- the review then sees check runs but no commit statuses
+# (a Vercel deployment reports as one), and the operator log says so once.
+_STATUSES_GRANTED: dict[str, bool] = {}
 
-def token_permissions_for(kind: str, *, workflows: bool = False) -> dict[str, str]:
+
+def token_permissions_for(kind: str, *, workflows: bool = False, statuses: bool = False) -> dict[str, str]:
     """Least-privilege installation-token permissions for a job kind.
 
     fix needs to push a branch + open a PR; secure only reads code + code-scanning results.
@@ -32,6 +39,11 @@ def token_permissions_for(kind: str, *, workflows: bool = False) -> dict[str, st
     .github/workflows files differ from the default branch -- which is every PR branch that
     predates a workflow change on main (thyn-ai/algenta#1003, 2026-09-17), even though the fix
     commit itself never touches a workflow.
+
+    ``statuses=True`` adds `statuses: read` to a review token: the PR head's ``statusCheckRollup``
+    (review_github.failing_required_checks) lists commit statuses -- a Vercel deployment reports as
+    one -- only to a token that may read them; without it the review sees check runs alone and
+    approved thyn-ai/codna-site#57 with no word about its FAILED required `Vercel` status.
     """
     if kind == "fix":
         perms = {"contents": "write", "pull_requests": "write", "checks": "write", "issues": "write"}
@@ -42,7 +54,10 @@ def token_permissions_for(kind: str, *, workflows: bool = False) -> dict[str, st
         return {"contents": "read", "security_events": "read", "checks": "write", "issues": "write"}
     if kind == "review":
         # Read code, write PR review comments + a check. No contents:write — review never pushes.
-        return {"contents": "read", "pull_requests": "write", "checks": "write"}
+        perms = {"contents": "read", "pull_requests": "write", "checks": "write"}
+        if statuses:
+            perms["statuses"] = "read"
+        return perms
     if kind == "queue":
         # Read the PR head + its check runs, write ONE check on the merge-group commit.
         return {"pull_requests": "read", "checks": "write"}
@@ -57,10 +72,12 @@ def token_permissions_for(kind: str, *, workflows: bool = False) -> dict[str, st
     raise WebhookError("unknown_job_kind", f"no token scope for job kind {kind!r}")
 
 
-def scoped_token_request(repo_full_name: str, kind: str, *, workflows: bool = False) -> dict[str, Any]:
+def scoped_token_request(repo_full_name: str, kind: str, *, workflows: bool = False,
+                         statuses: bool = False) -> dict[str, Any]:
     """Body for POST /app/installations/{id}/access_tokens — scoped to the one repo + minimal perms."""
     repo_name = repo_full_name.split("/", 1)[-1]
-    return {"repositories": [repo_name], "permissions": token_permissions_for(kind, workflows=workflows)}
+    return {"repositories": [repo_name],
+            "permissions": token_permissions_for(kind, workflows=workflows, statuses=statuses)}
 
 
 def normalize_private_key_pem(private_key_pem: str) -> str:
@@ -125,22 +142,39 @@ def installation_token(
     except Exception as exc:  # noqa: BLE001
         raise WebhookError("github_app_auth_unavailable", f"cannot sign App JWT: {exc}") from exc
 
-    def _mint(with_workflows: bool):
+    def _mint(*, with_workflows: bool, with_statuses: bool):
         return httpx.post(
             f"{_API}/app/installations/{installation_id}/access_tokens",
             headers={"Authorization": f"Bearer {jwt}", "Accept": "application/vnd.github+json"},
-            json=scoped_token_request(repo_full_name, kind, workflows=with_workflows),
+            json=scoped_token_request(repo_full_name, kind, workflows=with_workflows, statuses=with_statuses),
             timeout=30.0, follow_redirects=True,
         )
 
-    want_workflows = kind == "fix" and _WORKFLOWS_GRANTED.get(str(installation_id), True)
-    resp = _mint(want_workflows)
-    if want_workflows and resp.status_code == 422 and "permission" in resp.text.lower():
-        # The App has not been granted `workflows` on this installation: remember, mint without it.
-        _WORKFLOWS_GRANTED[str(installation_id)] = False
-        resp = _mint(False)
-    elif want_workflows and resp.status_code < 300:
-        _WORKFLOWS_GRANTED[str(installation_id)] = True
+    # The one OPTIONAL permission a kind asks for -- `workflows` (fix) or `statuses` (review) -- is
+    # requested until this installation says it never granted it; GitHub answers 422 for the whole
+    # mint then, so the token is minted once more without it and the answer is remembered per process.
+    inst = str(installation_id)
+    want_workflows = kind == "fix" and _WORKFLOWS_GRANTED.get(inst, True)
+    want_statuses = kind == "review" and _STATUSES_GRANTED.get(inst, True)
+    resp = _mint(with_workflows=want_workflows, with_statuses=want_statuses)
+    if (want_workflows or want_statuses) and resp.status_code == 422 and "permission" in resp.text.lower():
+        if want_workflows:
+            # The App has not been granted `workflows` on this installation: remember, mint without it.
+            _WORKFLOWS_GRANTED[inst] = False
+        if want_statuses:
+            # Not granted `Commit statuses: read` here (multi-tenant: each installation grants the
+            # App's new permissions on its own schedule). The review runs on check runs alone; a
+            # failed commit status such as Vercel cannot be named until the installation grants it.
+            _STATUSES_GRANTED[inst] = False
+            _warn("statuses_permission_missing", installation_id=inst, repo=repo_full_name, kind=kind,
+                  effect="commit statuses (e.g. Vercel) are invisible to the review's red-head check",
+                  remedy="accept the App's 'Commit statuses: read' permission on this installation")
+        resp = _mint(with_workflows=False, with_statuses=False)
+    elif (want_workflows or want_statuses) and resp.status_code < 300:
+        if want_workflows:
+            _WORKFLOWS_GRANTED[inst] = True
+        if want_statuses:
+            _STATUSES_GRANTED[inst] = True
     if resp.status_code >= 300:
         raise WebhookError("installation_token_failed", f"installation token: {resp.status_code} {resp.text[:200]}")
     token = resp.json().get("token")
@@ -158,8 +192,14 @@ def check_run_payload(name: str, head_sha: str, *, status: str, conclusion: str
     return body
 
 
-def create_check_run(repo_full_name: str, token: str, *, name: str, head_sha: str, summary: str) -> int | None:
-    """Create an in_progress Check Run so status shows in the PR UI. Returns its id."""
+def create_check_run(repo_full_name: str, token: str, *, name: str, head_sha: str, summary: str,
+                     status: str = "in_progress") -> int | None:
+    """Create a Check Run so status shows in the PR UI. Returns its id.
+
+    ``in_progress`` by default (a worker that is about to run the job). The ingress creates a
+    review's run ``queued`` the moment the delivery is accepted, so the check is visible while
+    the job waits for a worker slot; ``start_check_run`` flips it when a worker claims the row.
+    """
     try:
         import httpx
     except Exception as exc:  # noqa: BLE001
@@ -167,7 +207,7 @@ def create_check_run(repo_full_name: str, token: str, *, name: str, head_sha: st
     resp = httpx.post(
         f"{_API}/repos/{repo_full_name}/check-runs",
         headers={"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"},
-        json=check_run_payload(name, head_sha, status="in_progress", conclusion=None, summary=summary),
+        json=check_run_payload(name, head_sha, status=status, conclusion=None, summary=summary),
         timeout=30.0, follow_redirects=True,
     )
     if resp.status_code >= 300:
@@ -178,6 +218,50 @@ def create_check_run(repo_full_name: str, token: str, *, name: str, head_sha: st
     return check_run_id
 
 
+def start_check_run(repo_full_name: str, token: str, check_run_id: int, *, name: str, summary: str) -> None:
+    """Flip a ``queued`` Check Run to ``in_progress``: the worker that claimed its job is running it.
+
+    ``started_at`` is set to now, so the run's duration in the UI is the job's own, not the time
+    it waited in the queue (GitHub Actions stamps its job runs the same way)."""
+    try:
+        import httpx
+    except Exception as exc:  # noqa: BLE001
+        raise WebhookError("check_run_unavailable", f"Check Run update needs httpx: {exc}") from exc
+    import time as _time
+
+    body = check_run_payload(name, "", status="in_progress", conclusion=None, summary=summary)
+    body.pop("head_sha", None)  # a PATCH never re-anchors the run
+    body["started_at"] = _time.strftime("%Y-%m-%dT%H:%M:%SZ", _time.gmtime())
+    resp = httpx.patch(
+        f"{_API}/repos/{repo_full_name}/check-runs/{check_run_id}",
+        headers={"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"},
+        json=body, timeout=30.0, follow_redirects=True,
+    )
+    if resp.status_code >= 300:
+        raise WebhookError("check_run_update_failed", f"start check run: {resp.status_code} {resp.text[:200]}")
+
+
+def check_run_status(repo_full_name: str, token: str, check_run_id: int) -> str | None:
+    """``queued`` / ``in_progress`` / ``completed`` for one Check Run, or None when it cannot be read
+    (not ours, gone, or a transport error). A retry asks this before reusing the run its row
+    carries: a run the previous attempt already completed is not reopened, a fresh one replaces it."""
+    try:
+        import httpx
+    except Exception:  # noqa: BLE001
+        return None
+    try:
+        resp = httpx.get(
+            f"{_API}/repos/{repo_full_name}/check-runs/{int(check_run_id)}",
+            headers={"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"},
+            timeout=20.0, follow_redirects=True,
+        )
+    except Exception:  # noqa: BLE001
+        return None
+    data = _check_run_json(resp)
+    status = data.get("status") if data else None
+    return str(status) if status else None
+
+
 def update_check_run(repo_full_name: str, token: str, check_run_id: int, *, conclusion: str, summary: str,
                      name: str = "Codna") -> None:
     try:
@@ -271,8 +355,9 @@ def complete_stale_check_runs(
     repo_full_name: str, token: str, *, head_sha: str, name: str, summary: str,
     conclusion: str = "cancelled",
 ) -> int:
-    """Complete (as ``conclusion``, default ``cancelled``) every still-``in_progress`` Check Run
-    named ``name`` on ``head_sha``. Returns how many were closed. Best-effort hygiene: never raises.
+    """Complete (as ``conclusion``, default ``cancelled``) every still-open (``queued`` or
+    ``in_progress``) Check Run named ``name`` on ``head_sha``. Returns how many were closed.
+    Best-effort hygiene: never raises.
 
     A run this worker opened and then lost -- killed mid-job by a redeploy, or a crash before
     update_check_run -- stays "in progress" on that commit forever, because only the App that
@@ -282,39 +367,42 @@ def complete_stale_check_runs(
     predecessor instead of stacking a second spinner next to it -- and, as ``neutral``, on the
     event SHA of a review whose pull request head had already moved when the job started
     (thyn-ai/codna#569): a run left ``in_progress`` on a commit nobody is merging still strands
-    automation that waits for "all checks complete".
+    automation that waits for "all checks complete". ``queued`` runs are swept too: the ingress
+    opens a review's run in that state, and one whose row never reached a worker is as stranded
+    as an ``in_progress`` one. The listing endpoint filters on one status per request.
     """
     try:
         import httpx
     except Exception:  # noqa: BLE001
         return 0
     headers = {"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"}
-    try:
-        resp = httpx.get(
-            f"{_API}/repos/{repo_full_name}/commits/{head_sha}/check-runs",
-            headers=headers, params={"check_name": name, "status": "in_progress", "per_page": 50},
-            timeout=30.0, follow_redirects=True,
-        )
-    except Exception:  # noqa: BLE001
-        return 0
-    if resp.status_code >= 300:
-        return 0
     closed = 0
-    try:
-        runs = (resp.json() or {}).get("check_runs") or []
-    except Exception:  # noqa: BLE001
-        return 0
-    for run in runs:
-        run_id = run.get("id")
-        if not run_id:
-            continue
+    for status in ("in_progress", "queued"):
         try:
-            update_check_run(
-                repo_full_name, token, run_id, conclusion=conclusion, summary=summary, name=name
+            resp = httpx.get(
+                f"{_API}/repos/{repo_full_name}/commits/{head_sha}/check-runs",
+                headers=headers, params={"check_name": name, "status": status, "per_page": 50},
+                timeout=30.0, follow_redirects=True,
             )
-            closed += 1
-        except Exception:  # noqa: BLE001 — someone else's run, already completed, or a transport error
+        except Exception:  # noqa: BLE001
             continue
+        if resp.status_code >= 300:
+            continue
+        try:
+            runs = (resp.json() or {}).get("check_runs") or []
+        except Exception:  # noqa: BLE001
+            continue
+        for run in runs:
+            run_id = run.get("id")
+            if not run_id:
+                continue
+            try:
+                update_check_run(
+                    repo_full_name, token, run_id, conclusion=conclusion, summary=summary, name=name
+                )
+                closed += 1
+            except Exception:  # noqa: BLE001 — someone else's run, already completed, or a transport error
+                continue
     return closed
 
 

{codna-0.2.73 → codna-0.2.75}/codna/webhook_lease.py

@@ -15,7 +15,11 @@ Two threads, both stdlib, both unit-tested with fakes:
   completed with the dead-letter conclusion, comment-fix thread replied to), never a silent drop
   and never a red check from our own outage; (3) once Postgres has been healthy again for
   ``outage_retry_after_s``, runs dead rows whose failure text names an outage one more time;
-  (4) forgets workers whose heartbeat is long gone.
+  (4) forgets workers whose heartbeat is long gone; (5) completes the Check Runs of rows that were
+  retired before any worker touched them -- the ``queued`` run the ingress opens at enqueue, when
+  a newer head superseded the row, an operator cancelled it, a rollback exported it, or it failed
+  before the worker could report (``webhook_queued_check.settle_retired``; the ingress does the
+  same for the pull request it just retired rows on, this is the backstop).
 """
 from __future__ import annotations
 
@@ -26,7 +30,7 @@ import threading
 import time
 from typing import Any, Callable
 
-from . import webhook_github, webhook_pg_ops, webhook_retry
+from . import webhook_github, webhook_pg_ops, webhook_queued_check, webhook_retry
 from .webhook import WebhookJob, check_run_name
 from .webhook_pg_queue import MAX_RESUMES, PostgresQueue
 
@@ -302,12 +306,21 @@ class Reaper:
                 retried += 1
         return retried
 
+    # -- (5) the runs of rows retired before a worker saw them ----------------------------------------
+    def close_retired_check_runs(self) -> int:
+        """A ``queued`` Check Run the ingress opened belongs to a row; when that row ends without a
+        worker (superseded while waiting, cancelled by an operator, exported by a rollback, failed
+        before reporting) nothing else completes the run. Exactly once per row (``posted.check_closed``)."""
+        return webhook_queued_check.settle_retired(self._queue, github=self._github, token_for=self._token,
+                                                   retrigger=RETRIGGER)
+
     # -- the loop --------------------------------------------------------------------------------------
     def tick(self) -> dict[str, int]:
         self.ticks += 1
         out = self.reap_expired()
         out["dead_letters_notified"] = self.notify_dead_letters()
         out["outage_retries"] = self.retry_after_outage()
+        out["retired_checks_closed"] = self.close_retired_check_runs()
         out["workers_pruned"] = webhook_pg_ops.prune_dead_workers(self._queue)
         self.last_outcome = out
         if any(out.values()):

{codna-0.2.73 → codna-0.2.75}/codna/webhook_pg_ops.py

@@ -99,10 +99,16 @@ def retry_now(queue: PostgresQueue, row_id: int, *, actor: str) -> dict[str, Any
             return {"id": row_id, "outcome": "not_found"}
         if row["status"] not in ("dead", "failed", "cancelled", "queued"):
             return {"id": row_id, "outcome": "not_retryable", "status": row["status"]}
+        # The row keeps its Check Run id: the worker that claims it asks GitHub whether that run is
+        # still open (webhook_queued_check.open_check_run) -- it continues one the sweep has not
+        # reached yet, and opens a fresh one when the verdict, the dead-letter notice or the sweep
+        # already completed it. Forgetting the id here would leave a still-open run with nothing
+        # pointing at it. `check_closed` is cleared so a later retirement of THIS row sweeps whatever
+        # run it then carries.
         conn.execute(
             f"UPDATE {_q(queue)}.jobs SET status='queued', attempts=0, max_attempts=%(budget)s, not_before=NULL, "
             f"finished_at=NULL, owner=NULL, lease_expires_at=NULL, cancel_requested=NULL, "
-            f"posted = posted - 'dead_letter' WHERE id=%(id)s",
+            f"posted = posted - 'dead_letter' - 'check_closed' WHERE id=%(id)s",
             {"budget": queue.policy_for(row["kind"]).budget, "id": row_id})
         queue.event(row_id, "retry_now", actor=actor, detail={"from_status": row["status"]}, conn=conn)
     return {"id": row_id, "outcome": "queued"}
@@ -280,6 +286,35 @@ def unposted_dead(queue: PostgresQueue, *, limit: int = 50) -> list[dict[str, An
     return [dict(r) for r in rows]
 
 
+# Terminal states a row can reach WITHOUT a worker ever closing the Check Run it owns: retired while
+# waiting (a newer head, an operator's cancel, a rollback export) or failed before the worker could
+# report (a token that would not mint, orphaned by a restart on its last attempt).
+RETIRED_STATUSES = ("superseded", "cancelled", "exported", "failed")
+
+
+def retired_check_runs(queue: PostgresQueue, *, repo: str | None = None, pr_number: int | None = None,
+                       since_s: float = 7 * 24 * 3600, limit: int = 100) -> list[dict[str, Any]]:
+    """Rows in a ``RETIRED_STATUSES`` state that still carry a Check Run id and have not had that run
+    confirmed closed (``posted.check_closed`` unset), oldest first. The ingress asks for one pull
+    request right after its enqueue retired the older heads; the reaper asks for everything as the
+    backstop. The window bounds the one-time backfill over rows that predate the sweep."""
+    clauses = ["status = ANY(%(statuses)s)", "check_run_id IS NOT NULL", "NOT (posted ? 'check_closed')",
+               f"finished_at >= {_NOW} - make_interval(secs => %(since)s)"]
+    params: dict[str, Any] = {"statuses": list(RETIRED_STATUSES), "since": float(since_s), "limit": int(limit),
+                              **_now_param(queue)}
+    if repo:
+        clauses.append("repo = %(repo)s")
+        params["repo"] = repo
+    if pr_number is not None:
+        clauses.append("pr_number = %(pr)s")
+        params["pr"] = int(pr_number)
+    with queue.connection() as conn:
+        rows = conn.execute(
+            f"SELECT id, kind, repo, ref, pr_number, installation_id, status, check_run_id, last_error, result, finished_at "
+            f"FROM {_q(queue)}.jobs WHERE {' AND '.join(clauses)} ORDER BY finished_at LIMIT %(limit)s", params).fetchall()
+    return [dict(r) for r in rows]
+
+
 def dead_after_outage(queue: PostgresQueue, *, since_s: float = 6 * 3600, limit: int = 200) -> list[dict[str, Any]]:
     """Dead rows from the last ``since_s`` whose failure text names an outage -- the candidates the
     post-outage sweep runs again (never those already retried by an operator)."""

{codna-0.2.73 → codna-0.2.75}/codna/webhook_pg_queue.py

@@ -234,7 +234,12 @@ class PostgresQueue:
         the reaper queueing the current head of a reaped job) -- any job of this kind for exactly
         this head that is waiting, running OR already finished: a head that has its review does not
         get a second one because an older head's job noticed, as it ended, that the head had moved
-        (the older job is normally cancelled first; this closes the window in which it is not)."""
+        (the older job is normally cancelled first; this closes the window in which it is not).
+
+        ``check_run_id`` is the Check Run the ingress already opened ``queued`` for this job (see
+        ``webhook_queued_check``); the row is born owning it, ``check_started_at`` is stamped now
+        (the SLO clock: the tenant can see the check from this moment), and the worker that claims
+        the row updates that run instead of creating one."""
         if job.kind not in KINDS:
             raise ValueError(f"unknown job kind {job.kind!r}")
         if delivery_id is None:
@@ -265,10 +270,12 @@ class PostgresQueue:
                 row = conn.execute(
                     f"INSERT INTO {self._t('jobs')} (delivery_id, idempotency_key, kind, repo, ref, installation_id, "
                     f"issue_number, pr_number, context, reason, priority, attempts, max_attempts, resumes, not_before, "
-                    f"check_run_id, created_at, src_sqlite_id) "
+                    f"check_run_id, check_started_at, created_at, src_sqlite_id) "
                     f"VALUES (%(delivery_id)s, %(key)s, %(kind)s, %(repo)s, %(ref)s, %(installation_id)s, %(issue_number)s, "
                     f"%(pr_number)s, %(context)s::jsonb, %(reason)s, %(priority)s, %(attempts)s, %(max_attempts)s, "
-                    f"%(resumes)s, %(not_before)s, %(check_run_id)s, COALESCE(%(created_at)s::timestamptz, {_NOW}), %(src)s) "
+                    f"%(resumes)s, %(not_before)s, %(check_run_id)s, "
+                    f"CASE WHEN %(check_run_id)s::bigint IS NULL THEN NULL ELSE {_NOW} END, "
+                    f"COALESCE(%(created_at)s::timestamptz, {_NOW}), %(src)s) "
                     f"ON CONFLICT DO NOTHING RETURNING id",
                     {
                         "delivery_id": delivery_id, "key": idempotency_key(job), "kind": job.kind,
@@ -286,7 +293,8 @@ class PostgresQueue:
                 new_id = int(row["id"])
                 self.event(new_id, "enqueued", conn=conn,
                            detail={"delivery_id": delivery_id, "kind": job.kind, "reason": job.reason,
-                                   "priority": int(priority), "imported_from_sqlite": src_sqlite_id is not None})
+                                   "priority": int(priority), "imported_from_sqlite": src_sqlite_id is not None,
+                                   "check_run_id": check_run_id})
                 if job.kind in SUPERSEDABLE_KINDS and job.pr_number is not None and job.ref:
                     # A new head of this pull request retires the WAITING older heads in the same
                     # transaction as its own insert -- the SQLite queue's rule (webhook_queue
@@ -334,6 +342,59 @@ class PostgresQueue:
         with self._pool.connection() as conn:
             return self._pending_within(conn, repo=repo, pr_number=pr_number, ref=ref, kind=kind)
 
+    # --- the Check Run the ingress opens at enqueue (webhook_queued_check) -----------------------
+    def has_delivery(self, delivery_id: str | None) -> bool:
+        """Whether a row for this ``delivery_id`` already exists, whatever its state. The ingress
+        asks before opening a queued Check Run for a delivery: a redelivery of an already-processed
+        GUID collapses in ``enqueue`` and must not leave a second run on the commit."""
+        if not delivery_id:
+            return False
+        with self._pool.connection() as conn:
+            row = conn.execute(f"SELECT 1 FROM {self._t('jobs')} WHERE delivery_id = %s LIMIT 1", (delivery_id,)).fetchone()
+        return row is not None
+
+    def check_run_bound(self, delivery_id: str | None) -> int | None:
+        """The Check Run id the row for this delivery carries, or None (no row, or none bound). The
+        ingress reads it right after ``enqueue`` to learn whether the run it opened is owned by a
+        row -- anything else is an orphan it must close at once."""
+        if not delivery_id:
+            return None
+        with self._pool.connection() as conn:
+            row = conn.execute(f"SELECT check_run_id FROM {self._t('jobs')} WHERE delivery_id = %s", (delivery_id,)).fetchone()
+        return int(row["check_run_id"]) if row and row.get("check_run_id") else None
+
+    def adopt_check_run(self, job: WebhookJob, check_run_id: int, *, delivery_id: str | None = None) -> bool:
+        """Hand an already-open Check Run to the WAITING row for exactly this head (or, with
+        ``delivery_id``, to this delivery's own waiting row) when that row carries none yet. True when
+        a row took it: the worker that claims it will update this run instead of creating one.
+
+        Two callers: the ingress whose insert collapsed onto a live row for the same head (a
+        second delivery for one commit) -- if that row was written without a run (by a worker's or
+        the reaper's requeue, or an older ingress) it adopts this one, and nothing is orphaned; and
+        the late path of a pre-create that outran the ingress's time budget, which adopts onto the
+        delivery's own row if no worker has claimed it. Conditional on ``status = 'queued'`` and
+        ``check_run_id IS NULL``, under the row lock the UPDATE takes, so a claim that lands first
+        wins and the caller closes the run instead."""
+        if not check_run_id:
+            return False
+        if delivery_id:
+            where, params = "delivery_id = %(d)s", {"d": delivery_id}
+        else:
+            if not job.ref:
+                return False
+            where = "repo = %(repo)s AND kind = %(kind)s AND pr_number IS NOT DISTINCT FROM %(pr)s AND ref = %(ref)s"
+            params = {"repo": job.repo_full_name, "kind": job.kind, "pr": job.pr_number, "ref": job.ref}
+        with self._pool.connection() as conn, conn.transaction():
+            row = conn.execute(
+                f"UPDATE {self._t('jobs')} SET check_run_id = %(cid)s, check_started_at = COALESCE(check_started_at, {_NOW}) "
+                f"WHERE {where} AND status = 'queued' AND check_run_id IS NULL RETURNING id",
+                {**params, "cid": int(check_run_id), "now": self._now()},
+            ).fetchone()
+            if row is None:
+                return False
+            self.event(int(row["id"]), "check_run_adopted", conn=conn, detail={"check_run_id": int(check_run_id)})
+        return True
+
     # --- claim + lease -------------------------------------------------------------------------
     def claim(self, *, reserve_priority: bool = False, classes: Iterable[int] | None = None,
               lease_s: float | None = None) -> QueuedJob | None:
@@ -363,7 +424,8 @@ class PostgresQueue:
             self.event(int(row["id"]), "claimed", conn=conn,
                        detail={"owner": thread_owner, "attempt": int(row["attempts"]), "lease_s": lease})
         qjob = QueuedJob(row_id=int(row["id"]), delivery_id=row["delivery_id"], attempts=int(row["attempts"]),
-                         job=_job_from_row(row))
+                         job=_job_from_row(row),
+                         check_run_id=int(row["check_run_id"]) if row.get("check_run_id") else None)
         self._start_lease(qjob.row_id, thread_owner)
         return qjob