code-coordinator 0.5.239__tar.gz → 0.5.240__tar.gz

{code_coordinator-0.5.239 → code_coordinator-0.5.240}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.239
+Version: 0.5.240
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.239 → code_coordinator-0.5.240}/code_coordinator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.239
+Version: 0.5.240
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.239 → code_coordinator-0.5.240}/code_coordinator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.5.239",
+  "tag": "0.5.240",
   "distance": 0,
-  "node": "g6bf7406e8a26af4949bd4dac5a486ef69a92af43",
+  "node": "gbfdc750bff3b8352eace6d4ce6ab5dde9b39771a",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-08-23"

{code_coordinator-0.5.239 → code_coordinator-0.5.240}/coord/audit.py

@@ -22,16 +22,24 @@ row is best-effort observability on top of it.
 
 from __future__ import annotations
 
+import atexit
 import json
 import logging
+import sqlite3
+import threading
 import time
 from typing import Any
 
-from coord.db import get_connection
+from coord.db import get_connection, is_lock_contention_error, retry_on_locked
 
 _log = logging.getLogger(__name__)
 
-__all__ = ["record_audit", "query_audit_log"]
+__all__ = [
+    "record_audit",
+    "query_audit_log",
+    "audit_lock_contention_losses",
+    "flush_lock_contention_summary",
+]
 
 # Valid values are documented in the issue but not enforced here — callers
 # are all internal (coord.state / coord.issue_store), and rejecting an
@@ -94,7 +102,19 @@ def record_audit(
             details=details,
         )
     except Exception as exc:  # noqa: BLE001 — audit logging must never break the caller
-        _log.warning("record_audit: best-effort write failed: %s", exc)
+        if is_lock_contention_error(exc):
+            # #2597: `_record_audit_unsafe` already retried this write
+            # several times (`retry_on_locked`) before this was reached —
+            # sustained contention, not a momentary collision. Still
+            # best-effort (never raises into the caller), but a WARNING per
+            # occurrence is exactly the flood #2597 measured (1,804/day on
+            # dellserver, one indistinguishable line each) — count it
+            # instead and let `_flush_lock_contention_summary` report the
+            # aggregate once per run.
+            _record_lock_contention_loss()
+            _log.debug("record_audit: write lost to lock contention: %s", exc)
+        else:
+            _log.warning("record_audit: best-effort write failed: %s", exc)
 
 
 def _record_audit_unsafe(
@@ -115,27 +135,139 @@ def _record_audit_unsafe(
     try/except wrapper is the ONLY thing between this and the caller —
     keeps the swallow-and-log behavior in one obvious place."""
     conn = get_connection()
-    conn.execute(
-        """INSERT INTO audit_log (
-            ts, tier, category, event_type, actor,
-            repo, issue, assignment_id, machine, summary, details_json
-        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
-        (
-            ts if ts is not None else time.time(),
-            tier,
-            category,
-            event_type,
-            actor,
-            repo,
-            issue,
-            assignment_id,
-            machine,
-            summary,
-            json.dumps(details) if details is not None else None,
-        ),
-    )
-    conn.commit()
-    _maybe_trim(conn)
+
+    def _write() -> None:
+        conn.execute(
+            """INSERT INTO audit_log (
+                ts, tier, category, event_type, actor,
+                repo, issue, assignment_id, machine, summary, details_json
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+            (
+                ts if ts is not None else time.time(),
+                tier,
+                category,
+                event_type,
+                actor,
+                repo,
+                issue,
+                assignment_id,
+                machine,
+                summary,
+                json.dumps(details) if details is not None else None,
+            ),
+        )
+        conn.commit()
+
+    # #2597: this table is the coordinator's primary diagnostic surface —
+    # "coord audit" — and 1,804 rows/24h were measured lost to ordinary,
+    # momentary lock contention (a concurrent writer holding the DB for a
+    # beat) on dellserver, indistinguishable from a real gap in the trail.
+    # Give this write the same short, backed-off retry every other
+    # load-bearing write in this codebase already gets before falling
+    # through to record_audit's best-effort swallow.
+    retry_on_locked(_write)
+    # #2597-review: the trim below is opportunistic housekeeping over rows
+    # that are already durably committed above — isolate its own retry/
+    # failure from the insert's. Left unguarded, a `DELETE` that hits lock
+    # contention here would propagate out to `record_audit`'s except clause
+    # and get classified (by `is_lock_contention_error` alone, with no way
+    # to tell "the row itself didn't write" apart from "the row wrote fine
+    # but the trim afterward didn't run") as a *lost audit write* — over-
+    # counting genuine data loss for a row that made it into audit_log just
+    # fine. Give the trim the same retry budget as any other write, but
+    # keep a trim-only failure from ever touching that counter.
+    try:
+        retry_on_locked(lambda: _maybe_trim(conn))
+    except sqlite3.OperationalError as exc:
+        if not is_lock_contention_error(exc):
+            raise
+        _log.debug(
+            "record_audit: audit_log trim skipped due to lock contention "
+            "(row itself was already written): %s", exc,
+        )
+
+
+# ── Lock-contention loss counter (#2597) ────────────────────────────────────
+#
+# `retry_on_locked` above turns most lock collisions into a successful write
+# a few hundred milliseconds later, so a write reaching `record_audit`'s
+# `except` clause with `is_lock_contention_error(exc)` true means sustained
+# contention outlasted that whole retry budget — rare, but not impossible
+# under a genuinely busy DB. Logging each occurrence at WARNING is exactly
+# the flood #2597 measured (1,804 identical lines/day) — count them instead
+# and report ONE aggregated warning per process (`atexit`, so it fires for a
+# short-lived CLI invocation like `coord merge` and, if it ever shuts down
+# cleanly, the long-running daemon too) rather than leaving the loss
+# invisible or drowning the log in duplicates.
+
+_lock_contention_losses = 0
+_summary_flush_registered = False
+# #2597-review: `coord serve` calls `record_audit` from multiple
+# threads/greenlets (the tick loop's `run_in_threadpool` calls, concurrent
+# request handlers) — a bare `+= 1` on a module global can lose increments
+# under concurrent contention (read-modify-write race). The counter is
+# purely diagnostic (never gates a decision), but the whole point of it is
+# to make the loss rate trustworthy, so it should not itself be lossy.
+_lock_contention_lock = threading.Lock()
+
+
+def _record_lock_contention_loss() -> None:
+    """Bump the run-scoped lost-write counter and arrange for the aggregate
+    to be reported once this process exits (see
+    :func:`_flush_lock_contention_summary`)."""
+    global _lock_contention_losses, _summary_flush_registered
+    with _lock_contention_lock:
+        _lock_contention_losses += 1
+        register = not _summary_flush_registered
+        _summary_flush_registered = True
+    if register:
+        atexit.register(_flush_lock_contention_summary)
+
+
+def _flush_lock_contention_summary() -> None:
+    """Emit ONE aggregated warning for this run's audit writes lost to
+    sustained lock contention, if any, and reset the counter.
+
+    Registered via :func:`atexit.register` the first time a write is lost
+    (see :func:`_record_lock_contention_loss`) rather than called from a
+    fixed call site, so it fires for every entry point — `coord merge`,
+    `coord notify`, the daemon — without each one needing its own hook.
+    Also callable directly (tests; a caller that wants to flush mid-run
+    without waiting for process exit).
+    """
+    global _lock_contention_losses
+    with _lock_contention_lock:
+        pending = _lock_contention_losses
+        _lock_contention_losses = 0
+    if pending:
+        _log.warning("audit: %d writes lost to lock contention", pending)
+
+
+def audit_lock_contention_losses() -> int:
+    """Current count of audit writes lost to sustained lock contention this
+    run, not yet flushed by :func:`_flush_lock_contention_summary`.
+
+    Exposed for tests and diagnostics — advisory only, like the rest of the
+    audit log; nothing gates dispatch/review/merge decisions on this value.
+    """
+    with _lock_contention_lock:
+        return _lock_contention_losses
+
+
+def flush_lock_contention_summary() -> None:
+    """Public entry point for :func:`_flush_lock_contention_summary`.
+
+    #2597-review: the `atexit` registration above is the right default for
+    a short-lived CLI invocation (`coord merge`, `coord notify`) — it always
+    fires exactly once, at exit. It is a much weaker guarantee for a
+    long-running process (`coord serve`), which the code's own comment
+    already conceded only flushes "if it ever shuts down cleanly" — for a
+    daemon that runs for days between restarts, losses can accumulate for
+    its entire uptime with zero visibility. Call this on a periodic cadence
+    (e.g. once per daemon tick) to close that gap; it is a no-op cost-wise
+    when there is nothing pending (see :func:`_flush_lock_contention_summary`).
+    """
+    _flush_lock_contention_summary()
 
 
 def _maybe_trim(conn) -> None:

{code_coordinator-0.5.239 → code_coordinator-0.5.240}/coord/auto_loop.py

@@ -42,6 +42,7 @@ from dataclasses import dataclass
 import httpx
 
 from coord.config import Config
+from coord.db import is_lock_contention_error
 from coord.dispatch import AGENT_PORT
 from coord import github_ops
 from coord.models import SEALED_PATH_AUTHOR_TYPES, Assignment, Board
@@ -1228,7 +1229,7 @@ def _dispatch_fix(
             repo_github=repo.github,
         )
     except sqlite3.OperationalError as exc:
-        if "database is locked" not in str(exc).lower():
+        if not is_lock_contention_error(exc):
             # Not the transient contention this guard exists for (#2538) —
             # a schema mismatch or malformed statement is a real bug and
             # must not be swallowed as an ordinary declined dispatch.

{code_coordinator-0.5.239 → code_coordinator-0.5.240}/coord/commands/merge.py

@@ -4,6 +4,7 @@ coord/cli.py (#747)."""
 
 from __future__ import annotations
 
+import sqlite3
 import sys
 from pathlib import Path
 
@@ -11,6 +12,7 @@ import click
 
 
 from coord.commands._common import _CONFIG_OPTION, _load_config
+from coord.db import is_lock_contention_error, retry_on_locked
 from coord.models import WORK_LIKE_TYPES, effective_issue_number
 
 
@@ -2128,6 +2130,14 @@ def merge(
     open_by_repo, known_by_repo = _load_issue_states()
 
     auto_enqueued: list[str] = []
+    # #2597-review: assignments whose enqueue write is still locked after
+    # `retry_on_locked` exhausts its budget. Collected here instead of
+    # raised in place — one contended assignment must not abort the scan
+    # for every other assignment still waiting in this batch (that's the
+    # exact #1353 regression this file already fixed once, resurrected for
+    # lock contention specifically). The whole run still fails loudly: once
+    # every assignment has had its turn, a non-zero exit is raised below.
+    lock_contention_failures: list[tuple[str, sqlite3.OperationalError]] = []
     # Per-repo cache of branches that still exist on origin.  Lets us skip
     # re-enqueuing done-work whose branch was already merged-and-deleted — the
     # dominant merge-queue clog source.  A done assignment for a closed issue
@@ -2273,11 +2283,17 @@ def merge(
                 # longer matches.  Dedup by (repo_github, branch) is
                 # preserved — refresh_entry_assignment is a no-op when the
                 # entry is already correctly keyed.
-                if mq.refresh_entry_assignment(
+                # #2597: wrapped in retry_on_locked — this is a real DB write
+                # (merge_queue.save_queue), and unlike the gh-round-trip
+                # failures this try/except otherwise isolates (#1353), a
+                # `database is locked` collision here is pure transient
+                # contention with a concurrent writer, not a reason to give
+                # up on this assignment.
+                if retry_on_locked(lambda: mq.refresh_entry_assignment(
                     a,
                     repo_github=repo_cfg.github,
                     target_branch=target_branch,
-                ):
+                )):
                     auto_enqueued.append(
                         f"  auto-enqueued: {a.repo_name} #{a.issue_number} "
                         f"({a.branch} → {target_branch}){gate_note}"
@@ -2295,6 +2311,34 @@ def merge(
                         f"{target_branch}) — "
                         f"{mq.describe_merge_gate_failures(gate_failures)}"
                     )
+            except sqlite3.OperationalError as exc:
+                if is_lock_contention_error(exc):
+                    # #2597: `retry_on_locked` above already gave the write
+                    # several backed-off attempts — reaching here means the
+                    # DB is still locked after that budget, i.e. sustained
+                    # contention, not a momentary collision. Dropping a
+                    # mergeable assignment out of the scan on that basis is
+                    # the exact correctness bug #2597 reported (a skipped
+                    # entry here is silent — nothing re-scans it until the
+                    # next `coord merge` invocation). This is a hard
+                    # failure, not the soft "skipped" outcome below — but
+                    # (per review) it must not abort the rest of the batch
+                    # either. Record it and keep scanning; a clearly-flagged
+                    # line goes into the summary and the run still fails
+                    # loudly (non-zero exit, real exception) once every
+                    # other assignment has had its turn.
+                    lock_contention_failures.append((a.assignment_id, exc))
+                    auto_enqueued.append(
+                        f"  LOCK CONTENTION: {a.repo_name} #{a.issue_number} "
+                        f"(assignment {a.assignment_id}) — auto-enqueue write "
+                        f"still locked after exhausting retries: {exc!r}"
+                    )
+                    continue
+                auto_enqueued.append(
+                    f"  skipped: {a.repo_name} #{a.issue_number} "
+                    f"(assignment {a.assignment_id}) — auto-enqueue scan "
+                    f"failed, skipping this assignment: {exc!r}"
+                )
             except Exception as e:  # noqa: BLE001
                 auto_enqueued.append(
                     f"  skipped: {a.repo_name} #{a.issue_number} "
@@ -2304,6 +2348,20 @@ def merge(
     for line in auto_enqueued:
         click.echo(line)
 
+    if lock_contention_failures:
+        # #2597-review: every assignment in the batch has now had its turn
+        # — including the ones after the first contended write, and their
+        # results (if any) are already printed above. Surface the
+        # contention loudly now: a real `sqlite3.OperationalError`
+        # propagating out of this command (not a swallowed summary line)
+        # so `coord merge` exits non-zero and the failure is impossible to
+        # miss, same as any other genuinely unrecoverable write failure.
+        assignment_ids = ", ".join(aid for aid, _exc in lock_contention_failures)
+        raise sqlite3.OperationalError(
+            "database is locked — auto-enqueue write(s) still locked after "
+            f"exhausting retries for assignment(s): {assignment_ids}"
+        ) from lock_contention_failures[-1][1]
+
     # #1477: re-test any parked CONFLICT entry against GitHub's own
     # mergeability computation before the pending scan below — a branch
     # repaired by a conflict-fix worker (or by hand) since the last tick

{code_coordinator-0.5.239 → code_coordinator-0.5.240}/coord/db.py

@@ -113,6 +113,29 @@ _LOCK_RETRY_ATTEMPTS = 5
 _LOCK_RETRY_BASE_DELAY_S = 0.1
 
 
+def is_lock_contention_error(exc: BaseException) -> bool:
+    """True when *exc* is transient SQLite lock/busy contention rather than
+    a real bug (#2597).
+
+    Centralizes a check that used to be duplicated — and drifting — between
+    this module's own :func:`retry_on_locked` and ``coord.auto_loop``'s
+    ``except`` block: only the ``"database is locked"`` substring was
+    matched anywhere, missing both SQLite's other lock-collision message
+    (``"database table is locked"``, raised for a table-level lock rather
+    than the whole-database one) and the underlying ``SQLITE_BUSY`` result
+    code, which a driver could in principle surface through a differently
+    worded message. Checking the code as well as the text means a future
+    SQLite/driver wording change can't silently turn this back into a
+    "raise instead of retry" bug the way the single-substring version could.
+    """
+    if not isinstance(exc, sqlite3.OperationalError):
+        return False
+    message = str(exc).lower()
+    if "database is locked" in message or "database table is locked" in message:
+        return True
+    return getattr(exc, "sqlite_errorcode", None) == sqlite3.SQLITE_BUSY
+
+
 def retry_on_locked(
     write: Callable[[], _T],
     *,
@@ -120,8 +143,8 @@ def retry_on_locked(
     base_delay: float = _LOCK_RETRY_BASE_DELAY_S,
 ) -> _T:
     """Run *write* (a zero-arg callable performing one or more SQLite writes),
-    retrying with exponential backoff when SQLite raises ``database is
-    locked`` (#2538).
+    retrying with exponential backoff when SQLite raises lock/busy
+    contention (:func:`is_lock_contention_error`, #2538/#2597).
 
     That error is transient contention — a concurrent writer holding the DB
     at the exact moment this call tries to write, not a real failure — and a
@@ -135,7 +158,7 @@ def retry_on_locked(
     Any other ``OperationalError`` (schema drift, a malformed statement, …)
     is re-raised immediately without retrying — those are not transient, and
     retrying would only delay surfacing a real bug.  After *attempts*
-    consecutive ``database is locked`` collisions, re-raises the last
+    consecutive lock-contention collisions, re-raises the last
     ``sqlite3.OperationalError`` so the caller can decide how to degrade
     (see ``coord.state._record_dispatched_assignment_local``, whose caller —
     ``coord.auto_loop._dispatch_fix`` — treats it as a declined dispatch
@@ -146,7 +169,7 @@ def retry_on_locked(
         try:
             return write()
         except sqlite3.OperationalError as exc:
-            if "database is locked" not in str(exc).lower() or attempt >= attempts:
+            if not is_lock_contention_error(exc) or attempt >= attempts:
                 raise
             time.sleep(delay)
             delay *= 2

{code_coordinator-0.5.239 → code_coordinator-0.5.240}/coord/serve_app.py

@@ -6944,6 +6944,9 @@ def build_app(store: CoordStore, config: Config, *, token: str | None = None) ->
         async def _tick_loop() -> None:
             nonlocal last_housekeeping, last_merge_reconcile, last_worktree_clean, last_wal_checkpoint
             nonlocal last_notify_drain, last_notifier, last_portal_sync
+            from coord.audit import (  # noqa: PLC0415
+                flush_lock_contention_summary as _flush_audit_lock_contention_summary,
+            )
             from coord.reconcile import reconcile_completed_assignments  # noqa: PLC0415
             from coord import merge_queue as _mq  # noqa: PLC0415
 
@@ -6956,6 +6959,22 @@ def build_app(store: CoordStore, config: Config, *, token: str | None = None) ->
                 # for a daemon-side hand-edit to take effect (faster than
                 # waiting on a `/board` request).
                 _refresh_config()
+                # #2597-review: flush any pending "audit: N writes lost to
+                # lock contention" warning on this tick's cadence rather
+                # than only at process exit. `coord.audit`'s aggregate
+                # counter is atexit-registered, which is the right default
+                # for a short-lived CLI (`coord merge`, `coord notify`) but
+                # means this daemon — arguably the busiest audit writer of
+                # all (this tick loop, the drive-queue tick, up to 4 drives,
+                # 3 agents) — can accumulate losses for its entire uptime
+                # with zero visibility unless it happens to shut down
+                # cleanly. Cheap (in-memory counter + a log call only when
+                # non-zero) and run inline rather than via
+                # `run_in_threadpool` — no I/O, so no reason to hop threads.
+                try:
+                    _flush_audit_lock_contention_summary()
+                except Exception:  # noqa: BLE001 — diagnostics must never crash the daemon
+                    log.warning("audit lock-contention flush failed", exc_info=True)
                 # Step 1: reconcile (independent try/except so a failure here
                 # does not prevent the enqueue step below).
                 try:

{code_coordinator-0.5.239 → code_coordinator-0.5.240}/coord/state.py

@@ -2551,12 +2551,22 @@ def _update_assignment_cost_local(assignment_id: str, cost_usd: float) -> None:
     if not assignment_id:
         return
     conn = get_connection()
-    conn.execute(
-        "UPDATE assignments SET cost_usd=? WHERE assignment_id=? "
-        "AND (cost_usd IS NULL OR cost_usd < ?)",
-        (cost_usd, assignment_id, cost_usd),
-    )
-    conn.commit()
+
+    # #2597: this write previously had no lock-contention protection at
+    # all, so a momentary collision with a concurrent writer (the daemon's
+    # own tick, another `coord merge`/`coord notify`) raised straight out to
+    # `coord.notify._capture_cost`, which logs and swallows it — silently
+    # understating per-assignment spend rather than losing the write to a
+    # collision that a short retry would have ridden out.
+    def _write() -> None:
+        conn.execute(
+            "UPDATE assignments SET cost_usd=? WHERE assignment_id=? "
+            "AND (cost_usd IS NULL OR cost_usd < ?)",
+            (cost_usd, assignment_id, cost_usd),
+        )
+        conn.commit()
+
+    retry_on_locked(_write)
 
 
 def update_assignment_branch(assignment_id: str, branch: str) -> None:
@@ -2872,7 +2882,8 @@ def _update_assignment_tokens_local(
     if total <= 0:
         return
     conn = get_connection()
-    try:
+
+    def _write() -> None:
         conn.execute(
             "UPDATE assignments SET "
             "input_tokens=?, output_tokens=?, "
@@ -2886,8 +2897,16 @@ def _update_assignment_tokens_local(
             ),
         )
         conn.commit()
+
+    try:
+        # #2597: retry momentary lock contention (same rationale as
+        # `_update_assignment_cost_local` above) before falling through to
+        # this function's existing best-effort swallow, which stays in
+        # place for the case that guard was originally written for — a
+        # genuinely missing column on a pre-migration DB or test fixture —
+        # and now also covers a lock that outlasts the retry budget.
+        retry_on_locked(_write)
     except sqlite3.OperationalError:
-        # Column may not exist yet (pre-migration DB or test fixtures).
         pass