codna 0.2.58__tar.gz → 0.2.59__tar.gz

{codna-0.2.58 → codna-0.2.59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codna
-Version: 0.2.58
+Version: 0.2.59
 Summary: Local-first repo intelligence and agent fixing for Codna.
 Author: Thyn
 Maintainer: Thyn

{codna-0.2.58 → codna-0.2.59}/codna/__init__.py

@@ -5,4 +5,4 @@ This package is the `codna` CLI. Users never touch "algenta" directly — the en
 is an internal dependency reached over HTTP.
 """
 
-__version__ = "0.2.58"
+__version__ = "0.2.59"

{codna-0.2.58 → codna-0.2.59}/codna/webhook_lease.py

@@ -210,7 +210,7 @@ class Reaper:
                             WebhookJob(kind, row["repo"], ref=head, pr_number=int(row["pr_number"]),
                                        installation_id=row.get("installation_id"),
                                        reason=f"reaper_requeue_current_head_after_{row.get('reason') or 'interrupt'}"[:120]),
-                            delivery_id=f"reaper-{rid}-{head}", priority=1)
+                            delivery_id=f"reaper-{rid}-{head}", priority=1, unless_pending=True)
                     return "superseded"
                 return "skipped"
         hold = webhook_retry.backoff_s(kind, 1)

{codna-0.2.58 → codna-0.2.59}/codna/webhook_pg_queue.py

@@ -34,6 +34,7 @@ from typing import Any, Callable, Iterable, Mapping
 
 from . import webhook_retry
 from .webhook import WebhookJob, webhook_marker
+from .webhook_control import SUPERSEDABLE_KINDS
 from .webhook_pg_schema import DEFAULT_SCHEMA, KINDS, connect, lock_key, migrate, validate_schema_name
 from .webhook_queue import QueuedJob
 
@@ -47,6 +48,8 @@ _JOB_COLUMNS = ("id, delivery_id, kind, repo, ref, installation_id, issue_number
                 "check_run_id, check_started_at, posted, cancel_requested, last_error, result, created_at, "
                 "claimed_at, finished_at, src_sqlite_id")
 _JOB_COLUMNS_J = ", ".join("j." + c.strip() for c in _JOB_COLUMNS.split(","))  # qualified, for UPDATE ... FROM
+# The kinds a newer head of the same pull request retires (webhook_queue uses the same set).
+_SUPERSEDABLE_SQL = ",".join(f"'{k}'" for k in sorted(SUPERSEDABLE_KINDS))
 
 
 def _log(event: str, **fields: Any) -> None:
@@ -211,8 +214,11 @@ class PostgresQueue:
         (a redelivery), a queued/running review or fix for the SAME head (``jobs_one_live_head``:
         a new GUID for a head already covered), a CI-failure fix for a (repo, ref) that already
         had one in the last 24 h (the rule ported line for line from the SQLite queue), or -- with
-        ``unless_pending`` (the worker re-queueing a head that moved under a review, codna#569) --
-        any waiting/running job of this kind for exactly this head, whatever the kind."""
+        ``unless_pending`` (the worker re-queueing a head that moved under a review, codna#569, and
+        the reaper queueing the current head of a reaped job) -- any job of this kind for exactly
+        this head that is waiting, running OR already finished: a head that has its review does not
+        get a second one because an older head's job noticed, as it ended, that the head had moved
+        (the older job is normally cancelled first; this closes the window in which it is not)."""
         if job.kind not in KINDS:
             raise ValueError(f"unknown job kind {job.kind!r}")
         if delivery_id is None:
@@ -228,33 +234,58 @@ class PostgresQueue:
                 ).fetchone()
                 if dup is not None:
                     return False
-            if unless_pending and job.ref and self._pending_within(conn, repo=job.repo_full_name, pr_number=job.pr_number,
-                                                                    ref=job.ref, kind=job.kind):
-                return False
-            row = conn.execute(
-                f"INSERT INTO {self._t('jobs')} (delivery_id, idempotency_key, kind, repo, ref, installation_id, "
-                f"issue_number, pr_number, context, reason, priority, attempts, max_attempts, resumes, not_before, "
-                f"check_run_id, created_at, src_sqlite_id) "
-                f"VALUES (%(delivery_id)s, %(key)s, %(kind)s, %(repo)s, %(ref)s, %(installation_id)s, %(issue_number)s, "
-                f"%(pr_number)s, %(context)s::jsonb, %(reason)s, %(priority)s, %(attempts)s, %(max_attempts)s, "
-                f"%(resumes)s, %(not_before)s, %(check_run_id)s, COALESCE(%(created_at)s::timestamptz, {_NOW}), %(src)s) "
-                f"ON CONFLICT DO NOTHING RETURNING id",
-                {
-                    "delivery_id": delivery_id, "key": idempotency_key(job), "kind": job.kind,
-                    "repo": job.repo_full_name, "ref": job.ref, "installation_id": job.installation_id,
-                    "issue_number": job.issue_number, "pr_number": job.pr_number,
-                    "context": json.dumps(job.context) if job.context is not None else None,
-                    "reason": job.reason or "", "priority": int(priority), "attempts": int(attempts),
-                    "max_attempts": self.policy_for(job.kind).budget, "resumes": int(resumes),
-                    "not_before": not_before, "check_run_id": check_run_id, "created_at": created_at,
-                    "now": now, "src": src_sqlite_id,
-                },
-            ).fetchone()
-            if row is None:
-                return False
-            self.event(int(row["id"]), "enqueued", conn=conn,
-                       detail={"delivery_id": delivery_id, "kind": job.kind, "reason": job.reason,
-                               "priority": int(priority), "imported_from_sqlite": src_sqlite_id is not None})
+            with conn.transaction():
+                if unless_pending and job.ref:
+                    # Two requeuers can race for one head (two reapers servicing two reaped jobs of
+                    # the same pull request; a mover and a reaper). For queued/running rows the
+                    # jobs_one_live_head index already decides the race; a `done` row has no index,
+                    # so the check and the insert are serialized per head: same-head requeues take
+                    # this lock in turn, everything else is untouched.
+                    conn.execute("SELECT pg_advisory_xact_lock(hashtext(%s))",
+                                 (lock_key(self._schema, f"head:{job.repo_full_name}|{job.pr_number}|{job.ref}|{job.kind}"),))
+                    if self._covered_within(conn, repo=job.repo_full_name, pr_number=job.pr_number, ref=job.ref,
+                                            kind=job.kind, now=now):
+                        return False
+                row = conn.execute(
+                    f"INSERT INTO {self._t('jobs')} (delivery_id, idempotency_key, kind, repo, ref, installation_id, "
+                    f"issue_number, pr_number, context, reason, priority, attempts, max_attempts, resumes, not_before, "
+                    f"check_run_id, created_at, src_sqlite_id) "
+                    f"VALUES (%(delivery_id)s, %(key)s, %(kind)s, %(repo)s, %(ref)s, %(installation_id)s, %(issue_number)s, "
+                    f"%(pr_number)s, %(context)s::jsonb, %(reason)s, %(priority)s, %(attempts)s, %(max_attempts)s, "
+                    f"%(resumes)s, %(not_before)s, %(check_run_id)s, COALESCE(%(created_at)s::timestamptz, {_NOW}), %(src)s) "
+                    f"ON CONFLICT DO NOTHING RETURNING id",
+                    {
+                        "delivery_id": delivery_id, "key": idempotency_key(job), "kind": job.kind,
+                        "repo": job.repo_full_name, "ref": job.ref, "installation_id": job.installation_id,
+                        "issue_number": job.issue_number, "pr_number": job.pr_number,
+                        "context": json.dumps(job.context) if job.context is not None else None,
+                        "reason": job.reason or "", "priority": int(priority), "attempts": int(attempts),
+                        "max_attempts": self.policy_for(job.kind).budget, "resumes": int(resumes),
+                        "not_before": not_before, "check_run_id": check_run_id, "created_at": created_at,
+                        "now": now, "src": src_sqlite_id,
+                    },
+                ).fetchone()
+                if row is None:
+                    return False
+                new_id = int(row["id"])
+                self.event(new_id, "enqueued", conn=conn,
+                           detail={"delivery_id": delivery_id, "kind": job.kind, "reason": job.reason,
+                                   "priority": int(priority), "imported_from_sqlite": src_sqlite_id is not None})
+                if job.kind in SUPERSEDABLE_KINDS and job.pr_number is not None and job.ref:
+                    # A new head of this pull request retires the WAITING older heads in the same
+                    # transaction as its own insert -- the SQLite queue's rule (webhook_queue
+                    # ``_supersede_within``), which this backend lacked: with the queue saturated,
+                    # push A then push B had A's job run later, read the moved head, review B, and
+                    # B's own job review B again (two `codna review` runs on one commit, measured
+                    # 2026-09-20). The RUNNING older heads are flagged ``cancel_requested`` so the
+                    # worker holding them -- on ANY machine, not only the ingress's own process --
+                    # stops at its next heartbeat (webhook_service wires the lost lease into the
+                    # same cancellation the ingress applies in-process). The row just written is
+                    # excluded: a redelivery of the SAME head must not retire its predecessor.
+                    self._supersede_within(conn, repo=job.repo_full_name, pr_number=job.pr_number,
+                                           new_ref=job.ref, exclude_row_id=new_id, now=now)
+                    self._request_cancel_within(conn, repo=job.repo_full_name, pr_number=job.pr_number,
+                                                new_ref=job.ref, exclude_row_id=new_id)
         return True
 
     def _pending_within(self, conn: Any, *, repo: str, pr_number: int | None, ref: str | None, kind: str) -> bool:
@@ -263,6 +294,21 @@ class PostgresQueue:
             f"AND kind = %s AND status IN ('queued', 'running') LIMIT 1", (repo, pr_number, ref, kind)).fetchone()
         return row is not None
 
+    def _covered_within(self, conn: Any, *, repo: str, pr_number: int | None, ref: str | None, kind: str,
+                        now: datetime | None) -> bool:
+        """``_pending_within`` plus rows that ran to a verdict (``done``) within the last 24 hours --
+        what ``enqueue``'s ``unless_pending`` asks. The window this guards is minutes wide (an older
+        head's job outliving the new head's review by at most the job timeout plus a lease), and
+        the bound keeps an old verdict from silencing a requeue of a head force-pushed back months
+        later; 24 h is the check_suite_failure dedup rule's window. A ``failed``/``dead``/
+        ``cancelled`` row does not cover the head."""
+        row = conn.execute(
+            f"SELECT 1 FROM {self._t('jobs')} WHERE repo = %(repo)s AND pr_number IS NOT DISTINCT FROM %(pr)s AND ref = %(ref)s "
+            f"AND kind = %(kind)s AND (status IN ('queued', 'running') "
+            f"OR (status = 'done' AND finished_at >= {_NOW} - interval '24 hours')) LIMIT 1",
+            {"repo": repo, "pr": pr_number, "ref": ref, "kind": kind, "now": now}).fetchone()
+        return row is not None
+
     def has_pending(self, *, repo: str, pr_number: int | None, ref: str | None, kind: str) -> bool:
         """Whether a job of ``kind`` for exactly this pull request head is already waiting or
         running -- the SQLite queue's question, answered from the shared table. ``ref`` None never
@@ -390,6 +436,12 @@ class PostgresQueue:
             row = conn.execute(f"SELECT cancel_requested FROM {self._t('jobs')} WHERE id = %s", (row_id,)).fetchone()
         return (row or {}).get("cancel_requested") if row else None
 
+    def set_lease_lost_handler(self, callback: Callable[[int, str], None] | None) -> None:
+        """Install (or clear) what the lease keeper calls when a lease can no longer be renewed:
+        the row was cancelled from outside (a newer head, an operator) or reaped. The service wires
+        the worker's own cancellation here once the queue exists (``webhook_service.build_service``)."""
+        self._on_lease_lost = callback
+
     def lease_lost_reason(self, row_id: int) -> str | None:
         """What the heartbeat recorded for a lease this process no longer holds, else None."""
         with self._leases_lock:
@@ -583,17 +635,41 @@ class PostgresQueue:
         if not repo or pr_number is None or not new_ref:
             return []
         with self._pool.connection() as conn, conn.transaction():
-            rows = conn.execute(
-                f"UPDATE {self._t('jobs')} SET status='superseded', finished_at={_NOW}, "
-                f"result=%(r)s::jsonb, last_error='superseded' WHERE repo=%(repo)s AND pr_number=%(pr)s "
-                f"AND kind IN ('review','fix') AND status='queued' AND ref IS NOT NULL AND ref <> %(ref)s "
-                f"AND id <> %(ex)s RETURNING id",
-                {"now": self._now(), "r": json.dumps({"summary": f"superseded by {new_ref[:8]}"}),
-                 "repo": repo, "pr": pr_number, "ref": new_ref, "ex": -1 if exclude_row_id is None else exclude_row_id},
-            ).fetchall()
-            ids = [int(r["id"]) for r in rows]
-            for rid in ids:
-                self.event(rid, "superseded", conn=conn, detail={"new_ref": new_ref})
+            return self._supersede_within(conn, repo=repo, pr_number=pr_number, new_ref=new_ref,
+                                          exclude_row_id=exclude_row_id, now=self._now())
+
+    def _supersede_within(self, conn: Any, *, repo: str, pr_number: int, new_ref: str,
+                          exclude_row_id: int | None, now: datetime | None) -> list[int]:
+        """The retirement itself, inside a transaction the CALLER owns -- so ``enqueue`` makes the
+        insert and the retirement of the heads it replaces one atomic step."""
+        rows = conn.execute(
+            f"UPDATE {self._t('jobs')} SET status='superseded', finished_at={_NOW}, "
+            f"result=%(r)s::jsonb, last_error='superseded' WHERE repo=%(repo)s AND pr_number=%(pr)s "
+            f"AND kind IN ({_SUPERSEDABLE_SQL}) AND status='queued' AND ref IS NOT NULL AND ref <> %(ref)s "
+            f"AND id <> %(ex)s RETURNING id",
+            {"now": now, "r": json.dumps({"summary": f"superseded by {new_ref[:8]}"}),
+             "repo": repo, "pr": pr_number, "ref": new_ref, "ex": -1 if exclude_row_id is None else exclude_row_id},
+        ).fetchall()
+        ids = [int(r["id"]) for r in rows]
+        for rid in ids:
+            self.event(rid, "superseded", conn=conn, detail={"new_ref": new_ref})
+        return ids
+
+    def _request_cancel_within(self, conn: Any, *, repo: str, pr_number: int, new_ref: str,
+                               exclude_row_id: int) -> list[int]:
+        """Ask the worker running an older head of this pull request to stop: ``cancel_requested``
+        fails its next heartbeat (``heartbeat``), and the lease keeper's callback cancels the job
+        where it runs. Rows already asked keep their first reason; the reaper finishes whichever
+        row its worker does not (``webhook_lease.Reaper._reap_one``)."""
+        rows = conn.execute(
+            f"UPDATE {self._t('jobs')} SET cancel_requested = %(why)s WHERE repo=%(repo)s AND pr_number=%(pr)s "
+            f"AND kind IN ({_SUPERSEDABLE_SQL}) AND status='running' AND ref IS NOT NULL AND ref <> %(ref)s "
+            f"AND cancel_requested IS NULL AND id <> %(ex)s RETURNING id",
+            {"why": f"superseded by {new_ref[:8]}", "repo": repo, "pr": pr_number, "ref": new_ref, "ex": exclude_row_id},
+        ).fetchall()
+        ids = [int(r["id"]) for r in rows]
+        for rid in ids:
+            self.event(rid, "cancel_requested", conn=conn, detail={"reason": "superseded", "new_ref": new_ref})
         return ids
 
     # --- migration to and from the SQLite file --------------------------------------------------

{codna-0.2.58 → codna-0.2.59}/codna/webhook_service.py

@@ -290,6 +290,16 @@ def build_service(*, environ: Any = None, queue: Any = None, pool_factory: Calla
     from .webhook_ops import _pg
 
     pg = _pg(queue)
+    if pg is not None and backend == "postgres" and role in ("all", "worker"):
+        # A lease this process can no longer renew means the row was taken from it: cancelled from
+        # outside (a newer head arriving at ANY ingress, an operator's `cancel`) or reaped after a
+        # missed heartbeat. The job still running here is stopped exactly as an in-process
+        # supersede stops it -- JobHandle.cancel + settle: runtime torn down, Check Run closed
+        # neutral, the row written once -- so a superseded review does not run on to post on a
+        # commit nobody is merging, and a cancelled fix never opens its pull request. Without this
+        # the ingress's ``supersede_running`` reached only its own process (measured 2026-09-20 on
+        # the ingress + worker-pool shape: the old head's review ran to completion).
+        pg.set_lease_lost_handler(_cancel_local_job(queue, github or _default_github()))
     concurrency = webhook_control.default_concurrency(e)  # memory-derived, CODNA_WEBHOOK_CONCURRENCY wins when set
     pool = None
     registration = None
@@ -339,6 +349,23 @@ def _default_github() -> Any:
     return webhook_github
 
 
+def _cancel_local_job(queue: Any, github: Any) -> Callable[[int, str], None]:
+    """The lease keeper's callback: cancel the job this process runs for a row it no longer holds.
+    ``reason`` is the row's ``cancel_requested`` text (``superseded by <sha>``, an operator's
+    reason) or ``lease_lost`` (reaped: another worker owns it now). A handle already cancelled --
+    the ingress's own in-process supersede got there first -- is left to that path."""
+    from . import webhook_control
+
+    def _on_lease_lost(row_id: int, reason: str) -> None:
+        handle = webhook_control.get_control().get(row_id)
+        if handle is None or handle.cancelled:
+            return
+        _log("job_cancelled_by_queue", row_id=row_id, kind=handle.kind, repo=handle.repo, reason=reason)
+        webhook_control.cancel_jobs([handle], reason="cancel_requested", summary=reason, github=github, queue=queue)
+
+    return _on_lease_lost
+
+
 def _process_with_job_context(*, pool_default: Callable[..., Any]) -> Callable[..., Any]:
     """Wrap the pool's per-job entry so the queue row id is visible to ``_job_env`` (thread-local)
     while the job runs: the CLI's review body then carries ``codna:review:run=<id>``."""

{codna-0.2.58 → codna-0.2.59}/codna.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codna
-Version: 0.2.58
+Version: 0.2.59
 Summary: Local-first repo intelligence and agent fixing for Codna.
 Author: Thyn
 Maintainer: Thyn

{codna-0.2.58 → codna-0.2.59}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "codna"
-version = "0.2.58"
+version = "0.2.59"
 description = "Local-first repo intelligence and agent fixing for Codna."
 readme = "README.md"
 requires-python = ">=3.10"

{codna-0.2.58 → codna-0.2.59}/tests/test_fly_toml.py

@@ -83,6 +83,11 @@ def test_worker_pool_runs_the_postgres_backend_as_a_worker(worker):
 
 def test_worker_kill_timeout_is_at_flys_maximum_and_a_drained_worker_stays_stopped(worker):
     assert _seconds(worker["kill_timeout"]) == 300  # Fly's maximum: a longer fix must be DRAINED, never just stopped
+    # The pool's own drain grace must fill that window (its code default, 25 s, is the INGRESS
+    # machine's 30 s kill_timeout): measured 2026-09-20, a `fly machine stop` with the default
+    # exited 25 s into a 120 s review and handed it to the reaper.
+    drain_s = int(worker["env"]["CODNA_WEBHOOK_DRAIN_S"])
+    assert 240 <= drain_s <= _seconds(worker["kill_timeout"]) - 5
     [restart] = worker["restart"]
     assert restart["policy"] == "on-failure"  # exit 0 after a drain = the machine stays stopped (the pool shrinks)
 

{codna-0.2.58 → codna-0.2.59}/tests/test_webhook_pg_queue.py

@@ -282,6 +282,142 @@ def test_heartbeat_renews_only_the_owners_live_lease(make_pg_queue):
     assert other.heartbeat(claimed.row_id, owner="someone-else:1:t") is False
 
 
+def test_a_new_head_on_enqueue_retires_waiting_older_heads_and_flags_the_running_one(make_pg_queue):
+    """The SQLite queue's rule, on this backend too: push A then push B on one pull request must
+    leave ONE live job for it. Measured without this (2026-09-20, load proof): A's job ran later,
+    read the moved head, reviewed B -- and B's own job reviewed B again, two `codna review` runs
+    on one commit. The waiting older heads are retired in the insert's own transaction; the
+    running one is asked to stop (``cancel_requested``), which its worker learns at its next beat."""
+    q = make_pg_queue()
+    other = make_pg_queue(tenant_default_cap=3)
+    assert q.enqueue(_job(ref="A", pr=7), delivery_id="dA")
+    running = q.claim()
+    assert running is not None and running.job.ref == "A"
+    assert q.enqueue(_job(ref="B", pr=7), delivery_id="dB")                                           # waits behind A
+    assert q.enqueue(_job(kind="fix", ref="B", pr=7, reason="review_comment_codna_fix"), delivery_id="dBfix")
+    assert q.enqueue(_job(ref="B", pr=7), delivery_id="dB-redelivered") is False                     # same head, new GUID: nothing retired
+    assert q.enqueue(_job(kind="queue", ref="G", pr=7, reason="merge_group_checks_requested"), delivery_id="dG")  # a group commit is never stale
+    assert q.enqueue(_job(ref=None, pr=7, reason="comment_codna_review"), delivery_id="dNoRef")     # resolves its head when it runs
+    assert q.enqueue(_job(ref="A", pr=8, installation=43), delivery_id="dA8")                       # another pull request
+    assert q.enqueue(_job(ref="C", pr=7), delivery_id="dC")
+    by = {r["delivery_id"]: r for r in q.recent(limit=20)}
+    assert by["dB"]["status"] == "superseded" and by["dBfix"]["status"] == "superseded"
+    assert by["dB"]["result"] == {"summary": "superseded by C"} and by["dBfix"]["last_error"] == "superseded"
+    assert {by[d]["status"] for d in ("dG", "dNoRef", "dA8", "dC")} == {"queued"}
+    # The running row was asked to stop when B arrived and keeps that first reason (a later head
+    # does not overwrite it); its worker learns at its next beat.
+    assert by["dA"]["status"] == "running" and q.job(running.row_id)["cancel_requested"] == "superseded by B"
+    assert q.heartbeat(running.row_id) is False
+    assert [e["event"] for e in q.events(running.row_id)][-1] == "cancel_requested"
+    assert [e["event"] for e in q.events(by["dB"]["id"])][-1] == "superseded"
+    claimed = [c.delivery_id for c in (other.claim() for _ in range(5)) if c is not None]
+    assert set(claimed) == {"dA8", "dNoRef", "dC"}                                                    # tenant 42: A + two more = its cap of 3
+    assert not {"dB", "dBfix"} & set(claimed)
+
+
+def test_the_worker_holding_a_superseded_head_learns_it_through_its_lease(make_pg_queue):
+    """Cross-machine: the push lands at an ingress, the review runs on a worker elsewhere. The
+    worker's lease keeper reports the row's reason, which webhook_service turns into the same
+    cancellation the ingress applies to its own process."""
+    ingress = make_pg_queue(owner="ingress:1")
+    lost = []
+    worker = make_pg_queue(owner="worker-b:1", heartbeat_s=0.05, lease_s=5,
+                           on_lease_lost=lambda rid, why: lost.append((rid, why)))
+    assert ingress.enqueue(_job(ref="A", pr=7), delivery_id="dA")
+    claimed = worker.claim()
+    assert ingress.enqueue(_job(ref="B", pr=7), delivery_id="dB")
+    pause = threading.Event()
+    for _ in range(200):
+        if lost:
+            break
+        pause.wait(0.05)
+    assert lost == [(claimed.row_id, "superseded by B")]
+    assert worker.lease_lost_reason(claimed.row_id) == "superseded by B"
+
+
+def test_unless_pending_also_skips_a_head_that_already_has_its_verdict(make_pg_queue):
+    """The window the supersede leaves open: an older head's job ends normally before its worker
+    learns it was superseded, sees the head moved, and re-queues the new head -- which was already
+    reviewed. ``unless_pending`` treats a finished (``done``) row as covering the head; a plain
+    enqueue (a `@codna review` re-request) still queues it."""
+    clock = _Clock()
+    q = make_pg_queue(clock=clock)
+    assert q.enqueue(_job(ref="B", pr=7), delivery_id="dB")
+    claimed = q.claim()
+    q.complete(claimed.row_id, status="done", result={"summary": "reviewed"})
+    assert q.enqueue(_job(ref="B", pr=7, reason="head_moved_during_review"), delivery_id="head-moved-1-B", unless_pending=True) is False
+    clock.advance(23 * 3600)
+    assert q.enqueue(_job(ref="B", pr=7, reason="head_moved_during_review"), delivery_id="head-moved-1-B-later", unless_pending=True) is False
+    clock.advance(2 * 3600)                                                                    # a verdict older than 24 h covers nothing
+    assert q.enqueue(_job(ref="B", pr=7, reason="head_moved_during_review"), delivery_id="head-moved-1-B-old", unless_pending=True) is True
+    q.complete(q.claim().row_id, status="done", result={"summary": "reviewed again"})
+    assert q.enqueue(_job(ref="B", pr=7, reason="head_moved_during_review"), delivery_id="head-moved-1-B-again", unless_pending=True) is False
+    assert q.enqueue(_job(ref="F", pr=9), delivery_id="dF")                                    # a head whose only row FAILED
+    failed = q.claim()
+    assert failed is not None and failed.delivery_id == "dF"
+    q.complete(failed.row_id, status="failed", result={"error": "cli_error"}, retry=False)
+    assert q.job(failed.row_id)["status"] == "failed"
+    assert q.enqueue(_job(ref="F", pr=9), delivery_id="head-moved-2-F", unless_pending=True) is True   # not covered
+    assert q.enqueue(_job(ref="B", pr=7, reason="comment_codna_review"), delivery_id="dB-again") is True
+    assert q.counts() == {"done": 2, "failed": 1, "queued": 2}
+
+
+def test_racing_requeues_for_one_reviewed_head_insert_at_most_one_row(make_pg_queue):
+    """Two requeuers (two reapers, a mover and a reaper) asking ``unless_pending`` for the same
+    head at the same time: the check runs inside the insert's transaction under a per-head
+    advisory lock, so only one can insert -- and none when the head already has its verdict."""
+    q = make_pg_queue(pool_max=12)
+    assert q.enqueue(_job(ref="B", pr=7), delivery_id="dB")
+    q.complete(q.claim().row_id, status="done", result={"summary": "reviewed"})
+    results, barrier = [], threading.Barrier(8)
+
+    def _requeue(i):
+        barrier.wait()
+        results.append(q.enqueue(_job(ref="B", pr=7, reason="head_moved_during_review"), delivery_id=f"requeue-{i}", unless_pending=True))
+
+    threads = [threading.Thread(target=_requeue, args=(i,)) for i in range(8)]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join(timeout=30)
+    assert results == [False] * 8 and q.counts() == {"done": 1}
+    # and for a head with NO verdict yet, exactly one of the racers inserts
+    results.clear()
+    barrier = threading.Barrier(8)
+    threads = [threading.Thread(target=lambda i=i: (barrier.wait(), results.append(
+        q.enqueue(_job(ref="C", pr=7, reason="head_moved_during_review"), delivery_id=f"requeue-C-{i}", unless_pending=True))))
+        for i in range(8)]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join(timeout=30)
+    assert results.count(True) == 1 and q.counts() == {"done": 1, "queued": 1}
+
+
+def test_the_reaper_does_not_queue_a_current_head_that_was_already_reviewed(make_pg_queue):
+    """Same window, reaper side: a worker dies while finishing an older head after the current head
+    was already reviewed. The stale row is cancelled and its Check Run closed; the current head is
+    NOT queued a second time."""
+    clock = _Clock()
+    ingress = make_pg_queue(owner="ingress:1", clock=clock, lease_s=10)
+    worker = make_pg_queue(owner="worker-a:1", clock=clock, lease_s=10)
+    ingress.enqueue(_job(ref="newhead", pr=7), delivery_id="dNew")
+    reviewed = worker.claim()
+    worker.complete(reviewed.row_id, status="done", result={"summary": "reviewed newhead"})
+    ingress.enqueue(_job(ref="oldhead", pr=7, reason="late_delivery"), delivery_id="dOld")   # a late event for the older head
+    stale = worker.claim()
+    worker.record_check_run(stale.row_id, 77)
+    worker.abandon_leases()
+    clock.advance(11)
+    github = _GitHub(head="newhead", open_runs={77})
+    reaper = Reaper(ingress, app_id="APP", private_key="pem", github=github, environ={}, leader=_Held())
+    assert reaper.reap_expired()["superseded"] == 1
+    assert ingress.job(stale.row_id)["status"] == "cancelled"
+    assert [c[1:3] for c in github.of("complete")] == [(77, "cancelled")]
+    assert worker.claim() is None                                   # newhead already has its verdict: nothing queued
+    assert ingress.counts() == {"done": 1, "cancelled": 1}
+
+
 def test_cancel_requested_fails_the_heartbeat_and_the_lost_lease_is_reported(make_pg_queue):
     lost = []
     q = make_pg_queue(heartbeat_s=0.05, lease_s=5, on_lease_lost=lambda rid, why: lost.append((rid, why)))

{codna-0.2.58 → codna-0.2.59}/tests/test_webhook_service.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 
 import json
 import threading
+import time
 from http.server import ThreadingHTTPServer
 from urllib.request import Request, urlopen
 from urllib.error import HTTPError
@@ -239,3 +240,39 @@ def test_argparse_exposes_the_new_subcommands():
     assert ns.action_name == "queue" and ns.direction == "export"
     assert parser.parse_args(["webhook", "migrate"]).action_name == "migrate"
     assert parser.parse_args(["webhook"]).action == "serve"                    # bare `codna webhook` still serves
+
+
+def test_a_lost_lease_cancels_the_job_this_process_runs(make_pg_queue, tmp_path, monkeypatch):
+    """The Postgres backend's cross-machine cancellation, wired end to end: a row cancelled from
+    outside (here a newer head enqueued as another ingress would) fails the worker's next heartbeat,
+    and build_service's lease-lost handler stops the local job exactly as an in-process supersede
+    does -- handle cancelled, row written once with the reason, the next claim is the new head."""
+    from codna import webhook_control
+    from codna.webhook import WebhookJob
+
+    pg = make_pg_queue(heartbeat_s=0.05, lease_s=5)
+    monkeypatch.setenv("CODNA_WEBHOOK_QUEUE", str(tmp_path / "queue.db"))
+    monkeypatch.setenv("CODNA_RUNTIME_ROOT", str(tmp_path / "rt"))
+    monkeypatch.setenv("TMPDIR", str(tmp_path / "tmp"))
+    monkeypatch.setenv("CODNA_WEBHOOK_QUEUE_BACKEND", "postgres")
+    monkeypatch.setenv("CODNA_WEBHOOK_ROLE", "worker")
+    monkeypatch.delenv("FLY_API_TOKEN", raising=False)
+    build_service(queue=pg, pool_factory=_Pool)                     # the injected queue gets the handler too
+    assert pg.enqueue(WebhookJob("review", "acme/app", ref="old", pr_number=7, installation_id=42, reason="t"), delivery_id="d1")
+    claimed = pg.claim()
+    handle = webhook_control.get_control().register(row_id=claimed.row_id, kind="review", repo="acme/app",
+                                                    pr_number=7, ref="old", deadline_s=600)
+    try:
+        assert pg.enqueue(WebhookJob("review", "acme/app", ref="new", pr_number=7, installation_id=42, reason="t"), delivery_id="d2")
+        for _ in range(200):
+            if handle.cancelled and pg.row_state(claimed.row_id)[0] != "running":
+                break
+            time.sleep(0.05)
+        assert handle.cancelled and handle.cancel_detail == "superseded by new"
+        assert pg.row_state(claimed.row_id)[0] == "done"
+        assert pg.job(claimed.row_id)["result"] == {"summary": "superseded by new", "cancelled": "cancel_requested"}
+        nxt = pg.claim()
+        assert nxt is not None and nxt.job.ref == "new"
+    finally:
+        webhook_control.get_control().release(claimed.row_id)
+        pg.set_lease_lost_handler(None)