codna 0.2.48__tar.gz → 0.2.50__tar.gz

{codna-0.2.48 → codna-0.2.50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codna
-Version: 0.2.48
+Version: 0.2.50
 Summary: Local-first repo intelligence and agent fixing for Codna.
 Author: Thyn
 Maintainer: Thyn

{codna-0.2.48 → codna-0.2.50}/codna/__init__.py

@@ -5,4 +5,4 @@ This package is the `codna` CLI. Users never touch "algenta" directly — the en
 is an internal dependency reached over HTTP.
 """
 
-__version__ = "0.2.48"
+__version__ = "0.2.50"

{codna-0.2.48 → codna-0.2.50}/codna/webhook.py

@@ -820,16 +820,21 @@ def serve(host: str = "0.0.0.0", port: int = 8080) -> None:
 
     from .webhook_procs import prepare_scratch_root
     from .webhook_queue import WebhookQueue
+    from .webhook_resume import RunningJobRegistry, registry_dir_for
     from .webhook_worker import WorkerPool
 
     prepare_scratch_root()  # before anything touches tempfile: job scratch lives on the volume
 
-    queue = WebhookQueue(default_queue_path())
+    queue_path = default_queue_path()
+    queue = WebhookQueue(queue_path)
     pool = WorkerPool(
         queue,
         concurrency=int(os.environ.get("CODNA_WEBHOOK_CONCURRENCY", "2")),
         app_id=os.environ.get("GITHUB_APP_ID"),
         private_key=os.environ.get("GITHUB_APP_PRIVATE_KEY"),
+        # Jobs in flight, beside the queue on the same volume: a redeploy mid-job is resumed once on
+        # the next boot (or its Check Run closed honestly) instead of spinning forever -- webhook_resume.
+        registry=RunningJobRegistry(registry_dir_for(queue_path)),
     )
     pool.start()
     httpd = ThreadingHTTPServer((host, port), _Handler)

{codna-0.2.48 → codna-0.2.50}/codna/webhook_github.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 import json
 import os
 import re
+import sys
 from typing import Any, Mapping
 
 from .webhook import WebhookError
@@ -193,6 +194,79 @@ def update_check_run(repo_full_name: str, token: str, check_run_id: int, *, conc
         raise WebhookError("check_run_update_failed", f"update check run: {resp.status_code} {resp.text[:200]}")
 
 
+def _warn(event: str, **fields: Any) -> None:
+    """One JSON line on stderr, the worker's log shape, for a condition an operator must see."""
+    payload: dict[str, Any] = {"service": "codna-webhook-worker", "level": "warning", "event": event}
+    payload.update(fields)
+    print(json.dumps(payload, sort_keys=True), file=sys.stderr, flush=True)
+
+
+def _check_run_json(resp: Any) -> dict[str, Any] | None:
+    """The run a 2xx answer carries, else None (error status, or a body that is not the run)."""
+    if resp.status_code >= 300:
+        return None
+    try:
+        data = resp.json()
+    except Exception:  # noqa: BLE001
+        return None
+    return data if isinstance(data, dict) else None
+
+
+def complete_check_run_if_open(repo_full_name: str, token: str, check_run_id: int, *, conclusion: str,
+                               summary: str, name: str) -> bool:
+    """Complete ONE known Check Run, but only while it is still open. Returns True if it did.
+
+    The restart reconciler (webhook_resume) knows the exact id of the run an interrupted job
+    opened. `codna review --post` completes that same run with its findings, and the kill can
+    land right after -- so the run is read first and left alone once it is ``completed``.
+
+    What that guarantee is worth, honestly. GitHub has no conditional update for Check Runs (no
+    If-Match, no status-guarded PATCH), and a PATCH by the owning App on an already-completed run
+    answers 200 and replaces its output. So a completion that lands between the read and the
+    write here is overwritten, and afterwards nothing on the API tells: the run simply carries
+    this summary. The window is kept as small as the API allows -- one request on an already-open
+    connection: read, then write on the same client, nothing in between. A completion that lands
+    right AFTER the write is visible: the run is read back once more and, when its output is not
+    what was written, a ``check_run_clobbered`` warning names the run and its suite so the race is
+    at least on record. Who could be writing at that moment: only a ``codna`` child of the
+    previous process that outlived it (``WorkerPool.stop`` joins its threads but never signals
+    the job's process group), and on the hosted app the machine replacement kills that child
+    before this process boots. Never raises (best-effort hygiene, same contract as
+    complete_stale_check_runs)."""
+    try:
+        import httpx
+    except Exception:  # noqa: BLE001
+        return False
+    headers = {"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"}
+    url = f"{_API}/repos/{repo_full_name}/check-runs/{check_run_id}"
+    body = check_run_payload(name, "", status="completed", conclusion=conclusion, summary=summary)
+    try:
+        with httpx.Client(headers=headers, timeout=30.0, follow_redirects=True) as client:
+            before = _check_run_json(client.get(url))
+            if before is None or before.get("status") == "completed":
+                return False  # missing, not ours to read, or already completed: its findings stay
+            if client.patch(url, json=body).status_code >= 300:
+                return False  # completed by someone else in between (a 422), or not ours
+            try:
+                after = _check_run_json(client.get(url))
+            except Exception:  # noqa: BLE001 -- the read-back is diagnostics only
+                after = None
+    except Exception:  # noqa: BLE001 -- transport error at any step
+        return False
+    if after is not None and after.get("status") == "completed":
+        output = after.get("output") or {}
+        if (output.get("title"), output.get("summary")) != (name, summary):
+            # Someone completed this run right after this write (its content won, this
+            # cancellation note is gone): the interrupted job's own CLI, still alive somewhere.
+            _warn("check_run_clobbered", repo=repo_full_name, check_run_id=int(check_run_id),
+                  check_suite_id=(after.get("check_suite") or {}).get("id"),
+                  head_sha=after.get("head_sha"), name=name,
+                  observed_title=str(output.get("title") or "")[:200],
+                  observed_summary=str(output.get("summary") or "")[:200],
+                  observed_conclusion=after.get("conclusion"), written_summary=summary[:200])
+    return True
+
+
 def complete_stale_check_runs(
     repo_full_name: str, token: str, *, head_sha: str, name: str, summary: str
 ) -> int:

{codna-0.2.48 → codna-0.2.50}/codna/webhook_queue.py

@@ -73,6 +73,11 @@ class WebhookQueue:
             conn.executescript(_SCHEMA)
             self._migrate(conn)
 
+    @property
+    def path(self) -> str:
+        """The database file; the restart registry (webhook_resume) lives in a sibling directory."""
+        return self._path
+
     @staticmethod
     def _migrate(conn: sqlite3.Connection) -> None:
         """Add any columns introduced after the table first shipped (idempotent)."""
@@ -211,6 +216,13 @@ class WebhookQueue:
             ).rowcount
         return {"requeued": requeued, "failed": failed}
 
+    def row_state(self, row_id: int) -> tuple[str, int] | None:
+        """``(status, attempts)`` of one row, or None if it does not exist. The restart reconciler
+        reads this right after recover_stale to decide whether a registered job runs again."""
+        with self._conn() as conn:
+            row = conn.execute("SELECT status, attempts FROM jobs WHERE id=?", (row_id,)).fetchone()
+        return None if row is None else (str(row["status"]), int(row["attempts"]))
+
     def counts(self) -> dict[str, int]:
         with self._conn() as conn:
             rows = conn.execute("SELECT status, COUNT(*) c FROM jobs GROUP BY status").fetchall()

codna-0.2.50/codna/webhook_resume.py

@@ -0,0 +1,334 @@
+"""Resume, or honestly close, the jobs a webhook restart interrupted.
+
+The hosted webhook is one Fly machine that every merge to ``main`` redeploys. A job running at
+that moment is killed: ``WorkerPool.stop`` drains for the grace window, the process exits, and
+the queue row stays ``running`` with its Check Run ``in_progress``. On the next boot
+``WebhookQueue.recover_stale`` requeues the row -- but the Check Run's id lived only in a local
+variable of ``process_job``, so the ONLY thing that could ever complete it was the retry reaching
+``complete_stale_check_runs`` after token minting and the account bridge. When the retry died
+before that point (or the row was already on its last attempt) the run spun forever, and since
+``codna review`` is a required check the pull request could not merge until someone pushed again.
+Live: thyn-ai/algenta#1056 head 13b07d1b, check run 105937057683 started 2026-09-19T17:27:51Z,
+machine replaced 17:28:49Z-17:29:27Z, still ``in_progress`` at 17:44Z with one row gone
+``failed`` and no second run.
+
+Two pieces, both stdlib, both unit-tested with fakes:
+
+* :class:`RunningJobRegistry` -- one small JSON file per claimed job, written on claim, updated
+  with the Check Run id the moment it exists, removed on finish, on the volume BESIDE the queue
+  (``<queue dir>/running/``) so it survives SIGTERM and SIGKILL alike. ``stop()`` marks whatever
+  is still registered as interrupted by ``sigterm`` so the next boot can say why.
+* :func:`reconcile_interrupted` -- at startup, after ``recover_stale``, every registered job is
+  resolved exactly once: resumed (its stale run completed as ``cancelled`` and a fresh attempt
+  replaces it), superseded (the PR head moved: the stale head is not re-run), or cancelled with
+  the re-trigger comment (out of attempts, or already resumed once). Nothing this App opened is
+  left ``in_progress``.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+import tempfile
+import threading
+from dataclasses import asdict, dataclass, replace
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from . import webhook_github
+from .webhook import check_run_name
+from .webhook_queue import QueuedJob, WebhookQueue
+
+# A job is brought back automatically after ONE restart. A second interruption of the same job
+# (deploys 6 minutes apart on 2026-09-19 restarted the machine twice) is not re-run again: the
+# check is completed as cancelled with the comment that re-triggers it, so a job that cannot
+# finish between deploys never loops, and the queue's own attempt cap still bounds everything.
+_MAX_RESUMES = 1
+REGISTRY_DIRNAME = "running"
+
+_RETRIGGER = {
+    "review": "Comment `@codna review` on the pull request to run it again.",
+    "fix": "Reply `@codna fix` on the finding (or re-apply the `codna-fix` label) to run it again.",
+    "secure": "Re-apply the `codna-secure` label to run it again.",
+    "queue": "Re-queue the pull request to run it again.",
+}
+
+
+def registry_dir_for(queue_path: str | Path) -> Path:
+    """Where the registry lives: beside the durable queue (``/data/running`` on the hosted app)."""
+    return Path(queue_path).parent / REGISTRY_DIRNAME
+
+
+def _utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+@dataclass(frozen=True)
+class RunningJob:
+    """What the next process needs to know about a job this one was running."""
+
+    row_id: int
+    kind: str
+    repo: str
+    ref: str | None
+    pr_number: int | None
+    installation_id: int | None
+    issue_number: int | None
+    reason: str
+    delivery_id: str | None
+    attempt: int
+    started_at: str
+    check_run_id: int | None = None
+    resumes: int = 0
+    interrupted_by: str | None = None
+    interrupted_at: str | None = None
+
+
+class RunningJobRegistry:
+    """Per-job JSON files in one directory. Every method is best-effort and never raises: the
+    registry exists to make restarts honest, it must never take a live job down.
+
+    Three threads write it: the worker thread (``start`` on claim, ``record_check_run`` right
+    after ``create_check_run``, ``finish`` when the job ends), the main thread (``mark_interrupted``
+    from ``WorkerPool.stop`` once the drain grace runs out) and the reconcile thread
+    (``mark_resumed``). Every read-modify-write holds one lock, so a stamp landing while the run
+    id is being recorded cannot make either update disappear: the surviving file always carries
+    both. Each write stages through its own unique temp file, so two writers can never share a
+    staging path even outside the lock."""
+
+    def __init__(self, root: str | Path) -> None:
+        self._root = Path(root)
+        self._lock = threading.Lock()  # serializes every read-modify-write of an entry
+        try:
+            self._root.mkdir(parents=True, exist_ok=True)
+        except OSError as exc:
+            _log("registry_unavailable", error=f"{type(exc).__name__}: {exc}")
+
+    @property
+    def root(self) -> Path:
+        return self._root
+
+    def _path(self, row_id: int) -> Path:
+        return self._root / f"{int(row_id)}.json"
+
+    def _write(self, entry: RunningJob) -> None:
+        """Caller holds ``self._lock``. The staging file is unique per write (``mkstemp``): a
+        fixed ``{row_id}.json.tmp`` let two writers truncate and rename the same file."""
+        path = self._path(entry.row_id)
+        tmp: str | None = None
+        try:
+            fd, tmp = tempfile.mkstemp(dir=self._root, prefix=f"{int(entry.row_id)}.", suffix=".tmp")
+            with os.fdopen(fd, "w", encoding="utf-8") as handle:
+                handle.write(json.dumps(asdict(entry), sort_keys=True))
+            os.replace(tmp, path)  # atomic: a crash mid-write leaves the old entry, never half a file
+        except OSError as exc:
+            if tmp is not None:
+                try:
+                    os.unlink(tmp)
+                except OSError:
+                    pass
+            _log("registry_write_failed", row_id=entry.row_id, error=f"{type(exc).__name__}: {exc}")
+
+    def get(self, row_id: int) -> RunningJob | None:
+        return self._read(self._path(row_id))
+
+    @staticmethod
+    def _read(path: Path) -> RunningJob | None:
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            return RunningJob(**data)
+        except (OSError, ValueError, TypeError):
+            return None  # missing, being written, or from an incompatible version: skip, don't crash
+
+    def entries(self) -> list[RunningJob]:
+        try:
+            paths = sorted(p for p in self._root.glob("*.json") if p.suffix == ".json")
+        except OSError:
+            return []
+        found = [self._read(p) for p in paths]
+        return sorted((e for e in found if e is not None), key=lambda e: e.row_id)
+
+    def start(self, qjob: QueuedJob) -> RunningJob:
+        """Record a claimed job. The resume count of an earlier entry for the same row (the
+        attempt this one replaces) is carried forward, so the cap counts restarts, not attempts."""
+        job = qjob.job
+        with self._lock:
+            prior = self.get(qjob.row_id)
+            entry = RunningJob(
+                row_id=qjob.row_id, kind=job.kind, repo=job.repo_full_name, ref=job.ref,
+                pr_number=job.pr_number, installation_id=job.installation_id,
+                issue_number=job.issue_number, reason=job.reason, delivery_id=qjob.delivery_id,
+                attempt=qjob.attempts, started_at=_utc_now(),
+                resumes=prior.resumes if prior is not None else 0,
+            )
+            self._write(entry)
+        return entry
+
+    def record_check_run(self, row_id: int, check_run_id: int | None) -> None:
+        """The job's Check Run exists now: from here on a restart must complete it."""
+        if not check_run_id:
+            return
+        with self._lock:
+            entry = self.get(row_id)
+            if entry is not None:
+                self._write(replace(entry, check_run_id=int(check_run_id)))
+
+    def finish(self, row_id: int) -> None:
+        with self._lock:
+            try:
+                self._path(row_id).unlink()
+            except FileNotFoundError:
+                pass
+            except OSError as exc:
+                _log("registry_clear_failed", row_id=row_id, error=f"{type(exc).__name__}: {exc}")
+
+    def mark_resumed(self, row_id: int) -> None:
+        with self._lock:
+            entry = self.get(row_id)
+            if entry is not None:
+                self._write(replace(entry, resumes=entry.resumes + 1, check_run_id=None,
+                                    interrupted_by=None, interrupted_at=None))
+
+    def mark_interrupted(self, reason: str) -> int:
+        """Stamp every job still registered (the drain grace ran out on it) with why it is about
+        to die. Returns how many. Called from ``WorkerPool.stop`` on SIGTERM. Holds the lock for
+        the whole pass: a worker recording its run id, or finishing, in the middle of it lands
+        strictly before or strictly after, never inside a stale snapshot."""
+        stamped = 0
+        with self._lock:
+            for entry in self.entries():
+                self._write(replace(entry, interrupted_by=reason, interrupted_at=_utc_now()))
+                stamped += 1
+        return stamped
+
+
+def _log(event: str, **fields: Any) -> None:
+    payload: dict[str, Any] = {"service": "codna-webhook-worker", "event": event}
+    payload.update(fields)
+    print(json.dumps(payload, sort_keys=True), file=sys.stderr, flush=True)
+
+
+def _token_for(entry: RunningJob, *, app_id: str | None, private_key: str | None, github: Any,
+               environ: Any) -> str | None:
+    """The same scope the job itself ran with (it created the run, so it may complete it)."""
+    if entry.installation_id and app_id and private_key:
+        try:
+            return github.installation_token(app_id, private_key, entry.installation_id,
+                                             repo_full_name=entry.repo, kind=entry.kind)
+        except Exception as exc:  # noqa: BLE001 -- minting may be exactly what a fresh boot cannot do yet
+            _log("job_resume_token_failed", row_id=entry.row_id, error=type(exc).__name__)
+            return None
+    return environ.get("GITHUB_TOKEN") or None
+
+
+def _close(entry: RunningJob, token: str, github: Any, *, conclusion: str, summary: str) -> int | None:
+    """Complete the job's own Check Run if it is still open. Returns the id of the run this call
+    completed, or None when there was nothing to close (no run recorded, already completed by the
+    job's own CLI -- its findings stay -- or a transport error). The residual race between the
+    read and the write lives in ``webhook_github.complete_check_run_if_open``."""
+    if not entry.check_run_id:
+        return None
+    try:
+        closed = bool(github.complete_check_run_if_open(
+            entry.repo, token, entry.check_run_id, conclusion=conclusion, summary=summary,
+            name=check_run_name(entry.kind),
+        ))
+    except Exception as exc:  # noqa: BLE001 -- a transport error must not stop the other entries
+        _log("job_resume_close_failed", row_id=entry.row_id, error=type(exc).__name__)
+        return None
+    return int(entry.check_run_id) if closed else None
+
+
+def _reconcile_one(entry: RunningJob, queue: WebhookQueue, registry: RunningJobRegistry, *,
+                   app_id: str | None, private_key: str | None, github: Any,
+                   environ: Any) -> tuple[str, int | None]:
+    """One entry's verdict and the id of the Check Run this boot actually completed (None when
+    none was): ``mark_resumed`` clears the id from the entry, so it is captured here, not read
+    back later."""
+    state = queue.row_state(entry.row_id)
+    if state is None or state[0] == "done":
+        registry.finish(entry.row_id)  # finished right before the kill, or the row is gone
+        return "finished", None
+    status, attempts = state
+    token = _token_for(entry, app_id=app_id, private_key=private_key, github=github, environ=environ)
+    if token is None:
+        return "skipped", None  # entry kept: the next boot tries again; the retry's own hygiene still applies
+    name = check_run_name(entry.kind)
+    cause = "a redeploy" if entry.interrupted_by == "sigterm" else "a service restart"
+    hint = _RETRIGGER.get(entry.kind, "Trigger it again to run it.")
+    if status == "failed":
+        # recover_stale found no attempts left (or the last attempt failed on its own): terminal.
+        closed = _close(entry, token, github, conclusion="cancelled",
+                        summary=f"{name}: interrupted by {cause} with no attempts left. {hint}")
+        registry.finish(entry.row_id)
+        return "cancelled", closed
+    if entry.resumes >= _MAX_RESUMES:
+        queue.complete(entry.row_id, status="failed", retry=False,
+                       result={"error": "interrupted_after_resume", "attempt": attempts})
+        closed = _close(entry, token, github, conclusion="cancelled",
+                        summary=f"{name}: interrupted by {cause} again after one automatic resume; "
+                                f"not re-run automatically. {hint}")
+        registry.finish(entry.row_id)
+        return "cancelled", closed
+    if entry.pr_number and entry.ref:
+        head = github.pull_request_head_sha(entry.repo, token, entry.pr_number)
+        if head and head != entry.ref:
+            queue.complete(entry.row_id, status="done", retry=False,
+                           result={"summary": f"superseded: PR head moved to {head[:8]} across a restart"})
+            closed = _close(entry, token, github, conclusion="cancelled",
+                            summary=f"{name}: interrupted by {cause}; the pull request head has since "
+                                    f"moved to {head[:8]}, which gets its own run. This commit was not re-run.")
+            registry.finish(entry.row_id)
+            return "superseded", closed
+    closed = _close(entry, token, github, conclusion="cancelled",
+                    summary=f"{name}: interrupted by {cause}; re-queued automatically as attempt {attempts + 1}. "
+                            f"A new `{name}` run replaces this one. If none appears: {hint}")
+    registry.mark_resumed(entry.row_id)  # the row is already 'retry' (recover_stale) -- it runs again
+    return "resumed", closed
+
+
+def reconcile_interrupted(queue: WebhookQueue, registry: RunningJobRegistry, *,
+                          app_id: str | None, private_key: str | None,
+                          github: Any = None, environ: Any = None) -> dict[str, int]:
+    """Resolve every job the previous process left registered. Call AFTER ``queue.recover_stale``
+    and BEFORE any worker claims: the decisions here (run again / do not) are made on rows that
+    recover_stale has just put back in ``retry``. Returns counts per verdict. Never raises."""
+    github = github or webhook_github
+    environ = os.environ if environ is None else environ
+    outcome = {"resumed": 0, "superseded": 0, "cancelled": 0, "finished": 0, "skipped": 0}
+    for entry in registry.entries():
+        closed: int | None = None
+        try:
+            verdict, closed = _reconcile_one(entry, queue, registry, app_id=app_id,
+                                             private_key=private_key, github=github, environ=environ)
+        except Exception as exc:  # noqa: BLE001 -- one bad entry must not strand the others
+            _log("job_resume_error", row_id=entry.row_id, error=f"{type(exc).__name__}: {str(exc)[:200]}")
+            verdict = "skipped"
+        outcome[verdict] += 1
+        # check_run_id: the run the job had opened. closed_check_run_id: the one THIS boot completed
+        # (None: none recorded, the job's own CLI had already completed it, or the close failed).
+        _log("job_resume", row_id=entry.row_id, kind=entry.kind, repo=entry.repo, ref=entry.ref,
+             check_run_id=entry.check_run_id, closed_check_run_id=closed,
+             interrupted_by=entry.interrupted_by, resumes=entry.resumes, verdict=verdict)
+    return outcome
+
+
+def reconcile_in_background(queue: WebhookQueue, registry: RunningJobRegistry, *,
+                            app_id: str | None, private_key: str | None,
+                            done: threading.Event) -> threading.Thread:
+    """Run :func:`reconcile_interrupted` on its own thread and set ``done`` when it has finished
+    (or failed). Off the startup path because each GitHub call may take up to its 30 s timeout
+    and ``/health`` must answer at once after a deploy; the pool holds every claim on ``done``."""
+
+    def _run() -> None:
+        try:
+            reconcile_interrupted(queue, registry, app_id=app_id, private_key=private_key)
+        except Exception as exc:  # noqa: BLE001 -- reconciliation must never keep the pool from starting
+            _log("job_resume_error", error=f"{type(exc).__name__}: {str(exc)[:200]}")
+        finally:
+            done.set()
+
+    thread = threading.Thread(target=_run, name="codna-webhook-reconcile", daemon=True)
+    thread.start()
+    return thread

{codna-0.2.48 → codna-0.2.50}/codna/webhook_worker.py

@@ -44,6 +44,7 @@ from .webhook import (
     webhook_marker,
 )
 from .webhook_queue import _MAX_ATTEMPTS, QueuedJob, WebhookQueue
+from .webhook_resume import RunningJobRegistry, reconcile_in_background
 from .webhook_summaries import (  # noqa: F401 -- re-exported: callers and tests address these via this module
     _CLI_SUMMARY,
     _NON_RETRYABLE_CODES,
@@ -415,6 +416,7 @@ def process_job(
         [int | None], tuple[str | None, str | None]
     ] = webhook_metering.resolve_provider_credentials,
     resolve_fix_enabled: Callable[[int | None], bool] = webhook_metering.resolve_fix_enabled,
+    on_check_run: Callable[[int], None] | None = None,
 ) -> JobResult:
     """Resolve the org credential (fail closed), mint a GitHub token, open a Check Run, run, report."""
     job = qjob.job
@@ -615,6 +617,8 @@ def process_job(
             job.repo_full_name, token, name=check_run_name(job.kind), head_sha=job.ref,
             summary=f"{check_run_name(job.kind)} started ({job.reason})",
         )
+        if check_id and on_check_run:
+            on_check_run(check_id)  # restart registry: a killed job's run is completed on the next boot
     if job.kind == "fix" and job.reason == "check_suite_failure" and token and job.ref:
         # Before spending a metered run on a red check suite: is the failure still current, and is
         # it a code defect at all? A `bun install` tarball error or a runner that died is fixed by a
@@ -780,8 +784,11 @@ class WorkerPool:
     def __init__(self, queue: WebhookQueue, *, concurrency: int = 2, poll_interval: float = 1.0,
                  app_id: str | None = None, private_key: str | None = None,
                  drain_timeout: float | None = None,
-                 process: Callable[..., JobResult] | None = None) -> None:
+                 process: Callable[..., JobResult] | None = None,
+                 registry: RunningJobRegistry | None = None) -> None:
         self._queue = queue
+        self._registry = registry  # None: no restart bookkeeping (bare pools in tests / embedders)
+        self._reconciled = threading.Event()  # start(): claims wait until interrupted jobs are resolved
         self._concurrency = max(1, concurrency)
         self._poll = poll_interval
         self._app_id = app_id
@@ -795,13 +802,16 @@ class WorkerPool:
         self._busy_since: dict[str, float] = {}  # thread name -> monotonic start of its current job
 
     def _default_process(self, qjob: QueuedJob) -> JobResult:
-        return process_job(qjob, app_id=self._app_id, private_key=self._private_key)
+        record = (lambda cid: self._registry.record_check_run(qjob.row_id, cid)) if self._registry else None
+        return process_job(qjob, app_id=self._app_id, private_key=self._private_key, on_check_run=record)
 
     def _loop(self) -> None:
         # MC-predictive admission (admission_control): thread count is only the ceiling on CLAIMERS;
         # how many fixes actually run at once is decided per-fix against the observed duration
         # distribution, so a fix that would breach the SLA under current contention waits instead.
         admitter = get_admitter()
+        while not self._stop.is_set() and not self._reconciled.wait(self._poll):
+            pass  # start() is still deciding which requeued rows run again: claim nothing yet
         while not self._stop.is_set():
             # ADMIT BEFORE CLAIMING. A claimed row is status='running', which claim() excludes
             # (webhook_queue.py:122, inside BEGIN IMMEDIATE) — so parking a claimed job while waiting
@@ -845,11 +855,15 @@ class WorkerPool:
         me = threading.current_thread().name
         with self._busy_lock:
             self._busy_since[me] = time.monotonic()
+        if self._registry:
+            self._registry.start(qjob)  # on the volume: a restart from here on finds this job
         try:
             self._run_claimed_inner(qjob)
         finally:
             with self._busy_lock:
                 self._busy_since.pop(me, None)
+            if self._registry:
+                self._registry.finish(qjob.row_id)
 
     def _run_claimed_inner(self, qjob: QueuedJob) -> None:
         """Process one claimed job and mark its queue row terminal. A worker must never die on one
@@ -912,6 +926,11 @@ class WorkerPool:
 
     def start(self) -> None:
         self._queue.recover_stale()
+        if self._registry is None:
+            self._reconciled.set()
+        else:  # resolve every job the previous process left registered; _loop claims only after this
+            reconcile_in_background(self._queue, self._registry, app_id=self._app_id,
+                                    private_key=self._private_key, done=self._reconciled)
         for i in range(self._concurrency):
             t = threading.Thread(target=self._loop, name=f"codna-webhook-worker-{i}", daemon=True)
             t.start()
@@ -959,3 +978,5 @@ class WorkerPool:
         deadline = time.monotonic() + max(0.0, grace)
         for t in self._threads:
             t.join(timeout=max(0.0, deadline - time.monotonic()))
+        if self._registry:  # whatever is still registered outlived the grace: say so for the next boot
+            self._registry.mark_interrupted("sigterm")

{codna-0.2.48 → codna-0.2.50}/codna.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codna
-Version: 0.2.48
+Version: 0.2.50
 Summary: Local-first repo intelligence and agent fixing for Codna.
 Author: Thyn
 Maintainer: Thyn

{codna-0.2.48 → codna-0.2.50}/codna.egg-info/SOURCES.txt

@@ -79,6 +79,7 @@ codna/webhook_github.py
 codna/webhook_metering.py
 codna/webhook_procs.py
 codna/webhook_queue.py
+codna/webhook_resume.py
 codna/webhook_summaries.py
 codna/webhook_worker.py
 codna/worker.py
@@ -198,6 +199,7 @@ tests/test_webhook_github.py
 tests/test_webhook_github_ci.py
 tests/test_webhook_metering.py
 tests/test_webhook_queue.py
+tests/test_webhook_resume.py
 tests/test_webhook_worker.py
 tests/test_webhook_worker_feedback_dispatch.py
 tests/test_wheel_guard.py

{codna-0.2.48 → codna-0.2.50}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "codna"
-version = "0.2.48"
+version = "0.2.50"
 description = "Local-first repo intelligence and agent fixing for Codna."
 readme = "README.md"
 requires-python = ">=3.10"