code-coordinator 0.5.492__tar.gz → 0.5.493__tar.gz

{code_coordinator-0.5.492 → code_coordinator-0.5.493}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.492
+Version: 0.5.493
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.492 → code_coordinator-0.5.493}/code_coordinator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.492
+Version: 0.5.493
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.492 → code_coordinator-0.5.493}/code_coordinator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.5.492",
+  "tag": "0.5.493",
   "distance": 0,
-  "node": "gf825ecb0009197eb0b97ba8d836676c545ec6a00",
+  "node": "g21581c4b4d4c5107d3a7da2aefbfdfbd93aeee1e",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-09-18"

{code_coordinator-0.5.492 → code_coordinator-0.5.493}/coord/db.py

@@ -666,6 +666,129 @@ def rollback_after_driver_error(conn: Any | None, exc: BaseException) -> None:
         pass
 
 
+def undo_pending_write(conn: Any | None, exc: BaseException, *, undo: Callable[[], Any]) -> None:
+    """Run *undo* — a compensating statement touching only THIS write's own
+    row — after *exc*, the #3382 sibling of
+    :func:`rollback_after_driver_error` for a writer whose own
+    ``conn.commit()`` call, not just the statement before it, can raise.
+
+    :func:`rollback_after_driver_error` deliberately no-ops for a plain
+    SQLite error (see its docstring): that is correct when the failure is a
+    STATEMENT that never got to write anything — SQLite's ``SQLITE_BUSY``,
+    hit while acquiring the lock a write needs, leaves nothing applied, so
+    simply re-running the whole write closure from scratch (what
+    :func:`retry_on_locked` does) is exactly as good as a rollback.
+
+    That reasoning does not hold when it is ``conn.commit()`` itself that
+    raises. WAL-checkpoint contention can make ``COMMIT`` hit
+    ``SQLITE_BUSY`` *after* the statement(s) before it already applied
+    inside the still-open transaction — the write is neither committed nor
+    undone, so it sits there, genuinely pending, on ``coord.db``'s shared
+    process-wide connection, ready for a completely unrelated handler's next
+    unrelated ``commit()`` to durably persist it later. #3382: this is
+    exactly how a ``review_claims`` row survived two callers who each saw
+    their own ``/review-claim`` POST fail with a 503 — neither call's own
+    retry-free write had anything guarding its ``commit()``, so the pending
+    insert outlived both failed calls and was swept up by a later,
+    completely unrelated write on the same shared connection.
+
+    A bare retry of the closure does not fix this the way it fixes a failed
+    statement, either: an ``INSERT OR IGNORE`` re-run against its own
+    still-pending prior insert reads back as a conflict (``rowcount == 0``),
+    so the retry silently misreports "someone else already holds this
+    claim" about a claim only this process's own earlier, abandoned attempt
+    took.
+
+    **Why a compensating statement, and not a transaction-level undo of any
+    kind (#3382 review rounds 1 and 2).** Two earlier shapes of this
+    function were both rejected, for the same underlying reason:
+
+    * ``conn.rollback()`` (round 1). On ``coord.db``'s shared, process-wide
+      SQLite singleton (see this module's own docstring), that discards
+      *every* other thread's not-yet-committed statement too — this process
+      dispatches other ``_*_local`` writers through ``run_in_threadpool`` on
+      real OS worker threads (the daemon's periodic tick, the ``/board``
+      build, ``/assignment/*`` handlers), and Python's ``sqlite3`` joins ANY
+      thread's statement issued on this one connection between commits into
+      the SAME open transaction. An unrelated writer that already got
+      ``sql.execute``/``sql.insert_ignore``'s success back (nothing in this
+      tree commits internally — see ``coord/sql.py``) then silently loses it
+      when its own later ``conn.commit()`` no-ops or raises ``cannot commit
+      - no transaction is active`` — this issue's own second logged symptom.
+
+    * ``SAVEPOINT`` + ``ROLLBACK TO SAVEPOINT`` (round 2), whether the
+      savepoint name is a shared literal or made unique per call. Unique
+      names fix only the narrower half of the problem (two concurrent calls
+      resolving each other's marker, since SQLite binds a name to the most
+      recently established savepoint of that name). They do **not** fix the
+      half that matters: ``ROLLBACK TO SAVEPOINT`` reverts the database to
+      its state *just after that savepoint was established*, so it undoes
+      every statement issued since — including another thread's, which this
+      writer has no business undoing — and destroys that thread's own
+      savepoints along the way. Verified against a real ``sqlite3``
+      connection (Python 3.12, SQLite 3.45): with two distinctly-named
+      savepoints opened back to back and one row inserted after each,
+      ``ROLLBACK TO`` the OUTER (uniquely-named) savepoint removes BOTH
+      rows. A transaction-scoped undo simply cannot express "mine only" on a
+      connection several threads write through.
+
+    So the undo is row-scoped instead: *undo* issues one statement that
+    names this write's own row and nothing else (delete the row this call
+    inserted, identified by its own ``claimed_at`` stamp; re-insert the row
+    this call deleted, with the ``claimed_at`` it read back first). It is
+    exact regardless of how other threads' statements interleave with this
+    one — before the failure or after it — because it never refers to their
+    rows.
+
+    It also degrades better than a transaction-level undo when an unrelated
+    thread's ``commit()`` lands *between* this write's statement and this
+    recovery: a savepoint would already have been destroyed by that commit
+    (``COMMIT`` releases all savepoints), leaving ``ROLLBACK TO`` to fail
+    and this write's row durably committed and leaked — precisely the
+    incident. The compensating statement instead stays pending and is
+    flushed by whichever ``commit()`` reaches it next (this closure's own
+    retry, or another writer's), so the net effect on this row is still
+    nothing.
+
+    Call this from the ``except sql.driver_errors():`` block of a write
+    closure that both executes a statement *and* calls ``conn.commit()``
+    itself, and **only when that closure's own statement actually applied**
+    (``cursor.rowcount``) — a statement that failed applied nothing, so
+    there is nothing to compensate, the same "a failed STATEMENT applies
+    nothing" reasoning :func:`rollback_after_driver_error` already
+    documents. Today's callers are the ``review_claims``/``smoke_claims``
+    claim and release writes in ``coord/state.py`` (#3113/#3333), the two
+    sites where "did this claim land" is an exclusive-ownership decision the
+    rest of the system relies on, unlike most of this module's writers where
+    a duplicate-on-retry UPDATE/DELETE is harmless.
+
+    Unlike :func:`rollback_after_driver_error`, this does not inspect *exc*
+    for a SQLSTATE: the plain-SQLite case above is exactly what it exists to
+    cover. On Postgres it is a harmless no-op in practice — a failed
+    ``COMMIT`` there aborts the whole transaction (so *undo* raises
+    ``InFailedSqlTransaction``, suppressed below) and
+    :func:`retry_on_locked`'s own :func:`rollback_after_driver_error` call
+    has already undone this write for real; ``get_connection()`` also hands
+    each THREAD its own connection there (see this module's docstring), so
+    no other thread's statement was ever at risk.
+
+    Same suppression rule as :func:`rollback_after_driver_error`: a ``None``
+    *conn* is a no-op, and a failure of *undo* itself is swallowed so it
+    never masks *exc*, the exception actually worth seeing. If the
+    compensating statement cannot run either (sustained contention
+    outlasting even this), this write's own row is left pending for the next
+    writer's eventual commit/rollback to resolve — the same residual,
+    already-accepted risk :func:`rollback_after_driver_error`'s own
+    suppressed-failure path carries today, not a new one.
+    """
+    if conn is None:
+        return
+    try:
+        undo()
+    except Exception:  # noqa: BLE001 — never mask the caught driver error
+        pass
+
+
 # #2538: a handful of consecutive short retries — long enough to ride out a
 # concurrent writer (the daemon's own passive tick, another `coord merge`/
 # `coord notify` invocation) that only holds the DB for a moment, short

{code_coordinator-0.5.492 → code_coordinator-0.5.493}/coord/state.py

@@ -52,6 +52,7 @@ from coord.db import (
     is_lock_contention_error,
     retry_on_locked,
     rollback_after_driver_error,
+    undo_pending_write,
 )
 from coord.models import (
     WORK_LIKE_TYPES,
@@ -2629,15 +2630,63 @@ def _claim_review_dispatch_local(of_assignment_id: str) -> bool:
     """Local-DB write for :func:`claim_review_dispatch`.
 
     Called directly by the daemon endpoint so it never re-routes back over
-    HTTP — mirrors every other ``_*_local`` write in this module.
+    HTTP — mirrors every other ``_*_local`` write in this module. The
+    ``/review-claim`` endpoint itself calls this synchronously, not via
+    ``run_in_threadpool`` — the two review-claim endpoints can't race each
+    other's writes — but ``coord.db.get_connection()``'s connection is the
+    same process-wide SQLite singleton every OTHER ``_*_local`` writer this
+    process dispatches through ``run_in_threadpool`` on a real OS worker
+    thread uses too (#3382 review).
+
+    #3382: wrapped in :func:`coord.db.retry_on_locked`, unlike this
+    function's original shape which had no lock-contention handling at all
+    — a `database is locked` collision raised straight out to the caller as
+    a 503 with nothing retried. On top of the retry, when the `INSERT OR
+    IGNORE` applied but the `conn.commit()` after it raised, the closure
+    undoes its own just-inserted row via
+    :func:`coord.db.undo_pending_write` before re-raising — see that
+    function's docstring for why the undo is a compensating statement
+    naming THIS row rather than any transaction-level rollback (neither a
+    bare ``conn.rollback()`` nor ``ROLLBACK TO SAVEPOINT`` can express
+    "mine only" on a connection several threads write through). Without it,
+    a `conn.commit()` failure leaves a just-applied insert pending on the
+    shared connection for a wholly unrelated handler's next commit to
+    durably persist later — the incident this issue reports, a
+    `review_claims` row held by neither of the two calls that raced for it,
+    wedging vimcode#1086's review for ~13.5h.
     """
-    conn = get_connection()
-    cur = sql.insert_ignore(
-        conn, "review_claims", ["of_assignment_id", "claimed_at"],
-        (of_assignment_id, time.time()),
-    )
-    conn.commit()
-    return (cur.rowcount or 0) > 0
+
+    def _write() -> int:
+        conn = get_connection()
+        claimed_at = time.time()
+        inserted = 0
+        try:
+            cur = sql.insert_ignore(
+                conn, "review_claims", ["of_assignment_id", "claimed_at"],
+                (of_assignment_id, claimed_at),
+            )
+            inserted = cur.rowcount or 0
+            conn.commit()
+        except sql.driver_errors() as exc:  # #2784: was sqlite3.OperationalError only
+            if inserted:
+                # Only this call's own row: `claimed_at` is the stamp THIS
+                # attempt generated, so a claim another process legitimately
+                # holds (different stamp) is never touched. `inserted == 0`
+                # means the insert either lost the race or never applied at
+                # all — nothing of ours to compensate for.
+                undo_pending_write(
+                    conn, exc,
+                    undo=lambda: sql.execute(
+                        conn,
+                        "DELETE FROM review_claims "
+                        "WHERE of_assignment_id=? AND claimed_at=?",
+                        (of_assignment_id, claimed_at),
+                    ),
+                )
+            raise
+        return inserted
+
+    return retry_on_locked(_write) > 0
 
 
 def has_review_claim(of_assignment_id: str) -> bool:
@@ -2730,11 +2779,48 @@ def _release_review_dispatch_claim_local(of_assignment_id: str) -> None:
     Called directly by the daemon endpoint so it never re-routes back over
     HTTP, and by ``coord.issue_store._update_local_state`` (which always runs
     against whatever DB is local to that process — the canonical one, when
-    that process is the daemon or a non-thin-client host).
+    that process is the daemon or a non-thin-client host, and which CAN run
+    on a ``run_in_threadpool`` worker thread — #3382 review).
+
+    #3382: same `retry_on_locked` + undo-this-row-on-commit-failure
+    treatment as :func:`_claim_review_dispatch_local` — see that function's
+    docstring and :func:`coord.db.undo_pending_write`'s for why the undo is
+    a compensating statement rather than any transaction-level rollback.
+    This write had neither before, and a lock collision here left a stray
+    `DELETE` pending on the shared connection for the same reason the claim
+    write did — so the undo here is the mirror image: put back the ONE row
+    this call deleted, with the `claimed_at` it read back before deleting,
+    leaving every other row (and every other thread's pending statement)
+    exactly as it found them.
     """
-    conn = get_connection()
-    sql.execute(conn, "DELETE FROM review_claims WHERE of_assignment_id=?", (of_assignment_id,))
-    conn.commit()
+
+    def _write() -> None:
+        conn = get_connection()
+        row = sql.execute(
+            conn,
+            "SELECT claimed_at FROM review_claims WHERE of_assignment_id=?",
+            (of_assignment_id,),
+        ).fetchone()
+        claimed_at = row[0] if row is not None else None
+        deleted = 0
+        try:
+            cur = sql.execute(
+                conn, "DELETE FROM review_claims WHERE of_assignment_id=?", (of_assignment_id,)
+            )
+            deleted = cur.rowcount or 0
+            conn.commit()
+        except sql.driver_errors() as exc:  # #2784: was sqlite3.OperationalError only
+            if deleted and claimed_at is not None:
+                undo_pending_write(
+                    conn, exc,
+                    undo=lambda: sql.insert_ignore(
+                        conn, "review_claims", ["of_assignment_id", "claimed_at"],
+                        (of_assignment_id, claimed_at),
+                    ),
+                )
+            raise
+
+    retry_on_locked(_write)
 
 
 def release_review_claim_if_row_is_review(assignment_id: str) -> None:
@@ -2843,16 +2929,43 @@ def _claim_smoke_dispatch_local(work_assignment_id: str, capability_partition: s
     """Local-DB write for :func:`claim_smoke_dispatch`.
 
     Called directly by the daemon endpoint so it never re-routes back over
-    HTTP — mirrors :func:`_claim_review_dispatch_local`.
+    HTTP — mirrors :func:`_claim_review_dispatch_local`, including its
+    #3382 `retry_on_locked` + row-scoped
+    :func:`coord.db.undo_pending_write` treatment: this write is the
+    same "INSERT OR IGNORE, then commit" exclusive-claim shape, so it was
+    exposed to the identical 503-after-successful-write class that issue
+    reports for the review claim table, and to the same review-flagged risk
+    of a transaction-level rollback discarding an unrelated writer's own
+    pending statement on the shared connection.
     """
-    conn = get_connection()
-    cur = sql.insert_ignore(
-        conn, "smoke_claims",
-        ["work_assignment_id", "capability_partition", "claimed_at"],
-        (work_assignment_id, capability_partition, time.time()),
-    )
-    conn.commit()
-    return (cur.rowcount or 0) > 0
+
+    def _write() -> int:
+        conn = get_connection()
+        claimed_at = time.time()
+        inserted = 0
+        try:
+            cur = sql.insert_ignore(
+                conn, "smoke_claims",
+                ["work_assignment_id", "capability_partition", "claimed_at"],
+                (work_assignment_id, capability_partition, claimed_at),
+            )
+            inserted = cur.rowcount or 0
+            conn.commit()
+        except sql.driver_errors() as exc:  # #2784: was sqlite3.OperationalError only
+            if inserted:
+                undo_pending_write(
+                    conn, exc,
+                    undo=lambda: sql.execute(
+                        conn,
+                        "DELETE FROM smoke_claims WHERE work_assignment_id=? "
+                        "AND capability_partition=? AND claimed_at=?",
+                        (work_assignment_id, capability_partition, claimed_at),
+                    ),
+                )
+            raise
+        return inserted
+
+    return retry_on_locked(_write) > 0
 
 
 def release_smoke_dispatch_claim(work_assignment_id: str, capability_partition: str) -> None:
@@ -2887,15 +3000,44 @@ def _release_smoke_dispatch_claim_local(
 
     Called directly by the daemon endpoint so it never re-routes back over
     HTTP, and by :func:`release_smoke_claim_if_row_is_smoke_leg` (which
-    always runs against whatever DB is local to that process).
+    always runs against whatever DB is local to that process, and CAN run
+    on a ``run_in_threadpool`` worker thread — #3382 review).
+
+    #3382: same `retry_on_locked` + undo-this-row-on-commit-failure
+    treatment as :func:`_release_review_dispatch_claim_local`.
     """
-    conn = get_connection()
-    sql.execute(
-        conn,
-        "DELETE FROM smoke_claims WHERE work_assignment_id=? AND capability_partition=?",
-        (work_assignment_id, capability_partition),
-    )
-    conn.commit()
+
+    def _write() -> None:
+        conn = get_connection()
+        row = sql.execute(
+            conn,
+            "SELECT claimed_at FROM smoke_claims "
+            "WHERE work_assignment_id=? AND capability_partition=?",
+            (work_assignment_id, capability_partition),
+        ).fetchone()
+        claimed_at = row[0] if row is not None else None
+        deleted = 0
+        try:
+            cur = sql.execute(
+                conn,
+                "DELETE FROM smoke_claims WHERE work_assignment_id=? AND capability_partition=?",
+                (work_assignment_id, capability_partition),
+            )
+            deleted = cur.rowcount or 0
+            conn.commit()
+        except sql.driver_errors() as exc:  # #2784: was sqlite3.OperationalError only
+            if deleted and claimed_at is not None:
+                undo_pending_write(
+                    conn, exc,
+                    undo=lambda: sql.insert_ignore(
+                        conn, "smoke_claims",
+                        ["work_assignment_id", "capability_partition", "claimed_at"],
+                        (work_assignment_id, capability_partition, claimed_at),
+                    ),
+                )
+            raise
+
+    retry_on_locked(_write)
 
 
 def release_smoke_claim_if_row_is_smoke_leg(assignment_id: str) -> None:

{code_coordinator-0.5.492 → code_coordinator-0.5.493}/tests/test_db.py

@@ -2835,6 +2835,124 @@ class TestRollbackAfterDriverError:
         assert conn.rollbacks == 1
 
 
+class _ExecuteAlwaysFailsConn:
+    """Wraps a real connection: `cursor().execute()` always raises — stands
+    in for a connection where even the compensating statement
+    `undo_pending_write` issues hits sustained contention, for
+    `TestUndoPendingWrite.test_a_failing_undo_never_masks_the_caught_error`.
+    Deliberately does NOT delegate `cursor()` to the real connection (unlike
+    `tests/test_state.py`'s `_CommitFlakyConn`, which only fakes
+    `commit()`) — every statement this wrapper's cursor is asked to run
+    fails, exactly like `_RollbackRecorder(fail=True)` did for the old
+    unconditional-`conn.rollback()` shape this class replaces."""
+
+    __module__ = "sqlite3"  # coord.sql.detect_dialect keys off this
+
+    class _FailingCursor:
+        def execute(self, *_a, **_kw):  # noqa: ANN002,ANN003,ANN201
+            raise sqlite3.OperationalError("rollback itself failed")
+
+    def cursor(self):  # noqa: ANN201
+        return self._FailingCursor()
+
+
+class TestUndoPendingWrite:
+    """#3382 review round 2: `db.undo_pending_write` runs a compensating
+    statement naming THIS write's own row, not any transaction-level undo —
+    neither `conn.rollback()` (round 1) nor `SAVEPOINT` +
+    `ROLLBACK TO SAVEPOINT` (round 2) can express "mine only" on
+    `coord.db`'s shared, process-wide SQLite singleton, which every other
+    `_*_local` writer dispatched through `run_in_threadpool` also writes
+    through. See the function's own docstring for the full incident."""
+
+    def _seed_unrelated_pending_write(self, conn: sqlite3.Connection, key: str) -> None:
+        """A write pending on *conn*, uncommitted — mirrors another thread's
+        own in-flight statement on the shared singleton."""
+        conn.execute(f"INSERT INTO board_meta (key, value) VALUES ('{key}', 'pending')")  # noqa: S608
+
+    def test_undoes_only_this_writes_own_row(
+        self, isolated_conn: sqlite3.Connection,
+    ) -> None:
+        """Both halves of the hazard in one test: an unrelated writer's
+        statement pending BEFORE this write's own (what round 1's bare
+        `conn.rollback()` discarded) *and* one landing AFTER it (what a
+        `ROLLBACK TO SAVEPOINT` would discard, uniquely-named or not — see
+        `test_a_savepoint_scoped_undo_would_discard_a_later_write` below)
+        must both survive, while this write's own row goes."""
+        self._seed_unrelated_pending_write(isolated_conn, "before")
+        isolated_conn.execute(
+            "INSERT INTO board_meta (key, value) VALUES ('ours', 'should-vanish')"
+        )
+        self._seed_unrelated_pending_write(isolated_conn, "after")
+
+        db_mod.undo_pending_write(
+            isolated_conn,
+            sqlite3.OperationalError("database is locked"),
+            undo=lambda: sql.execute(
+                isolated_conn, "DELETE FROM board_meta WHERE key=?", ("ours",)
+            ),
+        )
+
+        # Both unrelated pending writes are untouched and still committable
+        # by whoever's writes they actually are.
+        isolated_conn.commit()
+        rows = {
+            r["key"]: r["value"]
+            for r in isolated_conn.execute("SELECT key, value FROM board_meta").fetchall()
+        }
+        assert rows.get("before") == "pending"
+        assert rows.get("after") == "pending"
+        assert "ours" not in rows
+
+    def test_a_savepoint_scoped_undo_would_discard_a_later_write(
+        self, isolated_conn: sqlite3.Connection,
+    ) -> None:
+        """Pins the premise of the design decision above, against the real
+        driver rather than by assertion: `ROLLBACK TO SAVEPOINT` reverts the
+        database to its state just after that savepoint, so it takes every
+        statement issued since with it — including another thread's, and
+        even when each thread uses its OWN distinctly-named savepoint (the
+        round-2 review's suggested fix, which closes only the narrower
+        name-collision half). If SQLite ever changed this, the comment
+        trail in `db.undo_pending_write` would be wrong and this test is how
+        we would find out."""
+        sql.execute(isolated_conn, "SAVEPOINT sp_thread_a")
+        sql.execute(isolated_conn, "SAVEPOINT sp_thread_b")  # unique name, still nested
+        isolated_conn.execute("INSERT INTO board_meta (key, value) VALUES ('a', 'x')")
+        isolated_conn.execute("INSERT INTO board_meta (key, value) VALUES ('b', 'x')")
+
+        sql.execute(isolated_conn, "ROLLBACK TO SAVEPOINT sp_thread_a")
+        sql.execute(isolated_conn, "RELEASE SAVEPOINT sp_thread_a")
+        isolated_conn.commit()
+
+        keys = {
+            r["key"] for r in isolated_conn.execute("SELECT key FROM board_meta").fetchall()
+        }
+        assert "a" not in keys
+        assert "b" not in keys, (
+            "thread B's row survived — SQLite's ROLLBACK TO semantics changed, "
+            "and db.undo_pending_write's rationale needs revisiting"
+        )
+
+    def test_none_connection_is_a_no_op(self) -> None:
+        called: list[int] = []
+        db_mod.undo_pending_write(
+            None,
+            sqlite3.OperationalError("database is locked"),
+            undo=lambda: called.append(1),
+        )  # must not raise
+        assert called == []
+
+    def test_a_failing_undo_never_masks_the_caught_error(self) -> None:
+        conn = _ExecuteAlwaysFailsConn()
+
+        db_mod.undo_pending_write(
+            conn,
+            sqlite3.OperationalError("database is locked"),
+            undo=lambda: sql.execute(conn, "DELETE FROM board_meta WHERE key=?", ("ours",)),
+        )  # must not raise, even though the compensating statement itself fails
+
+
 class TestBoardConnectionIfOpen:
     """`retry_on_locked`'s default connection resolution — it must never
     *open* one from inside an error handler."""