alissa-tools-github-revloop 0.16.7__tar.gz → 0.16.9__tar.gz

{alissa_tools_github_revloop-0.16.7/src/main/alissa_tools_github_revloop.egg-info → alissa_tools_github_revloop-0.16.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alissa-tools-github-revloop
-Version: 0.16.7
+Version: 0.16.9
 Summary: ALISSA-TOOLS-GITHUB-REVLOOP
 Home-page: https://alissa.app
 Author: Fahera

{alissa_tools_github_revloop-0.16.7 → alissa_tools_github_revloop-0.16.9}/src/main/alissa/tools/github/revloop/__main__.py

@@ -19,7 +19,7 @@ from .config import (
     resolve_config_path,
 )
 from .ghclient import IdentityMismatch
-from .loop import ReviewWatcher
+from .loop import LedgerUnwritable, ReviewWatcher
 from .proc import CommandError
 
 log = logging.getLogger(__name__)
@@ -189,6 +189,16 @@ def main(argv: list[str] | None = None) -> int:
         datefmt="%H:%M:%S",
     )
 
+    # STARTUP and STEADY STATE are separated on purpose (issue #62). The
+    # handlers below label a FileNotFoundError / ValueError "config error" and
+    # exit 2, which is right for the startup phase -- a missing config file or
+    # an unparseable value cannot be fixed by trying again. It was fatally
+    # wrong for the poll loop, where the same classes mean a transient
+    # subprocess ENOENT or a bad response: `run_forever` now firewalls those
+    # per iteration and never lets them reach here at all. The one-shot modes
+    # (`--pr`, `--once`) still surface a failure to their caller, as a
+    # one-shot must -- including a refused pass, which leaves through the
+    # LedgerUnwritable handler below with exit 1 rather than looking clean.
     try:
         config = resolve_config(args)
         log.info("workspace: %s", config.workspace_root)
@@ -207,6 +217,20 @@ def main(argv: list[str] | None = None) -> int:
             watcher.poll_once()
         else:
             watcher.run_forever()
+    except LedgerUnwritable as exc:
+        # Only reachable from `--once`: run_forever handles its own refusals and
+        # keeps polling. A one-shot REPORTS rather than retries, so this must
+        # not look like a clean pass to `... --once && echo ok` or to a health
+        # probe. Exit 1 ("the environment failed"), not the 2 reserved for
+        # "your config is wrong" -- a config error tells you to edit a file, an
+        # unwritable ledger tells you to look at the volume mount.
+        print(
+            f"ledger error: {exc} is not writable — no decisions were taken. "
+            "The daemon refuses to spawn, escalate, grant or post what it "
+            "cannot record; fix the volume mount or its ownership.",
+            file=sys.stderr,
+        )
+        return 1
     except IdentityMismatch as exc:
         print(f"identity error: {exc}", file=sys.stderr)
         return 2

{alissa_tools_github_revloop-0.16.7 → alissa_tools_github_revloop-0.16.9}/src/main/alissa/tools/github/revloop/loop.py

@@ -16,7 +16,7 @@ import secrets
 import sqlite3
 import time
 from collections import Counter
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum
 from pathlib import Path
 
@@ -99,6 +99,152 @@ MAX_VERDICT_POST_ATTEMPTS = 5
 # day, per stuck PR, with the operator paged exactly once.
 MAX_VERDICT_POST_BACKOFF_SECONDS = 60 * 60
 
+# -- the poll-failure firewall (issue #62) ------------------------------------
+#
+# 2026-07-29 killed the Railway daemon three times in one day: a subprocess
+# ENOENT on the `alissa` CLI (an image-layer file that vanished mid-run) and a
+# readonly-sqlite snapshot write both escaped `poll_once`, and __main__'s
+# startup-shaped handlers turned each into `exit 2`. One bad poll must never end
+# the daemon: the steady state's contract is degraded-but-alive with a loud
+# signal, never a silent exit. Startup config errors keep the fast exit -- those
+# a restart genuinely cannot fix.
+
+# Ceiling on the doubling backoff a failing poll applies. Matches the
+# rate-limit branch's cap: a daemon that has been failing for a quarter of an
+# hour gains nothing from polling more often than every 15 minutes, and a
+# recovered substrate is picked up within one window.
+POLL_BACKOFF_CAP_SECONDS = 900
+
+# How long one exception CLASS must fire on every consecutive poll before the
+# firewall stops calling it transient and escalates to a page-worthy ERROR.
+# Half an hour is several backoff windows -- long past anything a container
+# blip explains -- and still inside the window an operator can act on the same
+# day. A different class arriving resets the streak: that is a different
+# condition, not a continuation of this one.
+POLL_ESCALATE_SECONDS = 30 * 60
+
+# Streak limiting for the firewall's log line: the first few failures of a
+# streak are logged in full, then one in every POLL_FAILURE_LOG_EVERY, so a
+# substrate outage costs a handful of lines an hour instead of one per poll.
+# The escalation crossing and the recovery line are logged unconditionally --
+# both are state changes, and suppressing either would hide the very transition
+# the log exists to show.
+POLL_FAILURE_LOG_HEAD = 3
+POLL_FAILURE_LOG_EVERY = 10
+
+# Poll failures whose message is the whole diagnosis: CommandError already
+# carries the command and its stderr, so a traceback adds noise. Anything else
+# reaching the firewall is by definition unanticipated -- log where it came
+# from.
+EXPECTED_POLL_FAILURES = (CommandError,)
+
+
+class LedgerUnwritable(RuntimeError):
+    """Raised by `poll_once` when the ledger gate refuses the pass.
+
+    A dedicated signal rather than an empty result, because the two callers
+    have to tell "refused" apart from "polled, nothing to do" and a bare `[]`
+    cannot (PR #63 round-2 major and one of its minors, both closed by this):
+
+    * `run_forever` must leave the firewall's failure streak COMPLETELY alone.
+      A refused pass is not evidence that anything cleared -- it is evidence
+      the daemon did not look -- so counting it as a success printed a false
+      recovery line and re-armed the escalation clock for a fault that was
+      still failing. That is the same power the RateLimited branch was stripped
+      of in round 2, for the same reason.
+    * `--once` must exit non-zero. A one-shot reports rather than retries, and
+      a health probe or `... --once && echo ok` reading a refused pass as a
+      clean one is the one failure mode it cannot survive.
+    """
+
+
+@dataclass
+class Streak:
+    """A run of consecutive identical outcomes, with the log policy attached:
+    how many, how long, whether it has been escalated, and whether THIS one is
+    worth a line.
+
+    Extracted so the two callers that need "streak, limit, escalate, recover"
+    -- the poll firewall and the ledger gate -- cannot disagree about it (PR #63
+    round-2 nit; the gate's hand-rolled copy dropped the crossing bypass, which
+    made the parameters table's "the escalation crossing is never suppressed"
+    false at any poll interval other than 60s).
+
+    Deliberately a value object taking `now` from its caller rather than reading
+    the clock: the escalation rule is a statement about elapsed time, and it has
+    to be testable without sleeping through it.
+    """
+
+    count: int = 0
+    first_at: float = 0.0
+    escalated: bool = False
+
+    def record(self, now: float) -> tuple[bool, bool]:
+        """Fold one occurrence in. Returns (log_this_one, escalated_just_now).
+
+        The crossing BYPASSES the streak limit: it is a state change, and
+        suppressing it would hide the one transition the log exists to show.
+        """
+        if self.count == 0:
+            self.first_at, self.escalated = now, False
+        self.count += 1
+        crossing = not self.escalated and now - self.first_at >= POLL_ESCALATE_SECONDS
+        if crossing:
+            self.escalated = True
+        should_log = (
+            crossing
+            or self.count <= POLL_FAILURE_LOG_HEAD
+            or self.count % POLL_FAILURE_LOG_EVERY == 0
+        )
+        return should_log, crossing
+
+    def held(self, now: float) -> float:
+        """Seconds since the streak began. Zero when there is no streak."""
+        return 0.0 if self.count == 0 else now - self.first_at
+
+    def clear(self) -> None:
+        self.count, self.first_at, self.escalated = 0, 0.0, False
+
+    def resolve(self, now: float) -> tuple[int, float] | None:
+        """End the streak. Returns (occurrences, seconds), or None if there was
+        no streak -- the caller logs the recovery, which is the only evidence in
+        the log that a degraded daemon came back on its own."""
+        if self.count == 0:
+            return None
+        ended = (self.count, now - self.first_at)
+        self.clear()
+        return ended
+
+
+@dataclass
+class PollFailures:
+    """The firewall's memory of the current run of consecutive poll failures.
+
+    All the counting, limiting, escalation and recovery lives in `Streak`; what
+    is genuinely this class's own is the KEY. A streak is identified by the
+    exception CLASS, and a different class arriving mid-outage starts a new one
+    (re-arming escalation) because it is a different fault -- reporting "ENOENT
+    has been failing for 40 minutes" when the last 30 were sqlite errors would
+    be a lie the operator acts on.
+    """
+
+    kind: str | None = None
+    streak: Streak = field(default_factory=Streak)
+
+    def record(self, exc: BaseException, now: float) -> tuple[bool, bool]:
+        """Fold one failure in. Returns (log_this_one, escalated_just_now)."""
+        kind = type(exc).__name__
+        if kind != self.kind:
+            self.kind = kind
+            self.streak.clear()
+        return self.streak.record(now)
+
+    def resolve(self, now: float) -> tuple[int, float] | None:
+        """Clear the streak on a successful poll."""
+        self.kind = None
+        return self.streak.resolve(now)
+
+
 # The GitHub review states the CI gate can produce for a round it refused to
 # approve: a red head lands as CHANGES_REQUESTED, a rollup that never concluded
 # as COMMENTED. Read by _convergence_reason, which must not converge on an
@@ -181,8 +327,9 @@ CHECKS_UNSETTLED_LEAD = (
     "concluded.**\n\n"
     "{detail}\n\n"
     "This round's verdict was held for {waited} min waiting for the head's "
-    "checks to settle ({bound} min bound) and they did not, so it is recorded "
-    "as a comment: an approve would claim a head this loop never saw go green. "
+    "checks to settle ({bound} min bound){total_note} and they did not, so it is "
+    "recorded as a comment: an approve would claim a head this loop never saw go "
+    "green. "
     "Nothing about the review itself changed — the verdict below is the round's "
     "own.\n\n"
     "Submitting this review consumes the pending review request, so the daemon "
@@ -221,6 +368,13 @@ CHECKS_UNSETTLED_PAGE = (
     "No label was touched and no further round is queued."
 )
 
+# Appended to `{bound}` above only when the hold was PROMOTED -- an unreadable
+# wait that became a genuine pending one restarts the clock, so the bound the
+# operator configured applies per condition and the round can be held up to
+# twice it. Saying "held 30 min (30 min bound)" after 60 real minutes is the
+# report being wrong about the one number an operator tunes.
+CHECKS_TOTAL_HELD = ", {total} min in total across both waits,"
+
 # The `{detail}` above, per reason the rollup did not settle.
 CHECKS_STILL_RUNNING = "Still running at the bound: {names}."
 CHECKS_UNREADABLE = (
@@ -801,6 +955,14 @@ class ReviewWatcher:
         # rollup (two API calls) on every poll, forever, for every PR with an
         # owed approve. In-memory for the same reason _dry_run_drift is.
         self._dry_run_rollups: dict[tuple[str, int, int, str], str] = {}
+        # Consecutive passes refused by the ledger gate in poll_once, and when
+        # the refusal began -- the same streak-limit-then-escalate shape the
+        # poll firewall uses, for the same reason: a read-only volume refuses
+        # every pass, and one line per poll would bury the condition it is
+        # reporting. In memory because it describes THIS process's degraded
+        # state, and because the only durable place to put it is the ledger
+        # that cannot be written.
+        self._ledger_streak = Streak()
 
     # -- per-PR decision ---------------------------------------------------
 
@@ -1165,22 +1327,29 @@ class ReviewWatcher:
         # never "restart whenever the state changes" -- a reader flapping between
         # the two would then push the bound out forever, which is precisely the
         # unbounded hold this bound exists to prevent.
-        held_at, held_state = self.state.checks_hold(pr.full_name, pr.number, round_)
-        promoted = held_state == CHECKS_UNKNOWN and rollup.state == CHECKS_PENDING
-        if held_at is None or promoted:
-            held_at = self.state.record_checks_hold(
+        hold = self.state.checks_hold(pr.full_name, pr.number, round_)
+        promoted = hold.condition == CHECKS_UNKNOWN and rollup.state == CHECKS_PENDING
+        if hold.since is None or promoted:
+            self.state.record_checks_hold(
                 pr.full_name, pr.number, round_, rollup.state
             )
-        waited = max(time.time() - held_at, 0)
+            hold = self.state.checks_hold(pr.full_name, pr.number, round_)
+        # Two numbers, both reported: `waited` is the wait THIS condition has had
+        # and is what the bound applies to; `held` is how long the round has been
+        # held at all. They differ by up to a full bound once a hold has been
+        # promoted, so a report that shows only the first tells an operator who
+        # set 30 minutes that a 60-minute hold waited 30.
+        waited = max(time.time() - (hold.since or time.time()), 0)
+        held = max(time.time() - (hold.first_at or time.time()), 0)
         bound = self.config.checks_wait_seconds
         if waited < bound:
             # One line per poll, one activity note per held round; see
             # checks_hold_kind.
             log.info(
-                "%s round %d: holding its %s — CI rollup at %s is %s (%dm of the "
-                "%dm bound waited)",
+                "%s round %d: holding its %s — CI rollup at %s is %s (%dm on this "
+                "condition of the %dm bound; %dm held in total)",
                 pr.slug, round_, VERDICT_APPROVE, judged[:8], rollup.summary,
-                waited // 60, bound // 60,
+                waited // 60, bound // 60, held // 60,
             )
             self._note_checks_hold(pr, round_, judged, rollup)
             return ChecksGate(
@@ -1188,7 +1357,8 @@ class ReviewWatcher:
                     Action.AWAITING_POST,
                     f"round {round_} holds its {VERDICT_APPROVE} — the CI rollup "
                     f"at {judged[:8]} is {rollup.summary}; "
-                    f"{int((bound - waited) // 60)}m of the wait bound left",
+                    f"{int((bound - waited) // 60)}m of the wait bound left "
+                    f"({int(held // 60)}m held in total)",
                     round_,
                     task_ref=task.ref,
                 ),
@@ -1196,10 +1366,10 @@ class ReviewWatcher:
             )
 
         log.warning(
-            "%s round %d: the CI rollup at %s is still %s after %dm — recording "
-            "the %s envelope as a %s review, never an APPROVE on an unverified "
-            "head",
-            pr.slug, round_, judged[:8], rollup.summary, waited // 60,
+            "%s round %d: the CI rollup at %s is still %s after %dm on this "
+            "condition (%dm held in total) — recording the %s envelope as a %s "
+            "review, never an APPROVE on an unverified head",
+            pr.slug, round_, judged[:8], rollup.summary, waited // 60, held // 60,
             VERDICT_APPROVE, EVENT_COMMENT,
         )
         detail = (
@@ -1216,6 +1386,11 @@ class ReviewWatcher:
                 detail=detail,
                 waited=int(waited // 60),
                 bound=int(bound // 60),
+                total_note=(
+                    CHECKS_TOTAL_HELD.format(total=int(held // 60))
+                    if hold.promoted
+                    else ""
+                ),
             ),
             state=rollup.state,
             detail=detail,
@@ -2748,6 +2923,42 @@ class ReviewWatcher:
     # -- polling -----------------------------------------------------------
 
     def poll_once(self) -> list[tuple[str, Decision]]:
+        # THE LEDGER GATE (issue #62, PR #63 round-1 blocker). Nothing below
+        # may run when the ledger cannot record what it does.
+        #
+        # Keeping the correctness writes strict aborts the pass that fails, but
+        # the firewall in run_forever hands the loop straight back here -- and
+        # by then the side effect is already taken. Concretely, over a
+        # read-only volume: _spawn enqueues a reviewer session, record_spawn
+        # raises, the pass dies, and the next pass finds no spawn row (the
+        # in-flight check is a READ of the row that never landed), so it
+        # enqueues another one. Every poll. Each a live agent that submits a
+        # real review and burns the round budget. Before the firewall existed
+        # the daemon died after one such duplicate -- bad, but bounded.
+        #
+        # So the gate is above everything, including the reap sweep (a kill is
+        # a side effect and record_reap is a correctness write). The pass takes
+        # no decisions and returns empty; the loop stays alive and keeps
+        # probing. One race remains and is deliberate: a volume that flips
+        # read-only BETWEEN this probe and record_spawn costs one duplicate,
+        # which is the pre-firewall blast radius, and every pass after it is
+        # gated. Closing it would mean recording before enqueuing, which trades
+        # this for an orphan row that wedges the round for a full stale window
+        # on any enqueue failure.
+        # DRY-RUN IS EXEMPT, and vacuously so: it already suppresses every side
+        # effect AND every correctness write (`_spawn` skips record_spawn, the
+        # reaper logs instead of killing, the drift/cap-out/deferral paths
+        # return before both their comment and their record). Its only ledger
+        # write is the snapshot, which this module classifies as best-effort
+        # telemetry and which _write_snapshot writes in dry-run deliberately.
+        # So the gate would protect nothing there and cost the operator the one
+        # tool that answers "what would you do right now" -- asked, precisely,
+        # during the substrate incident this whole change is about.
+        if not self.config.dry_run and not self.state.writable():
+            self._note_ledger_unwritable()
+            raise LedgerUnwritable(str(self.config.state_db))
+        self._note_ledger_writable()
+
         # Sweep BEFORE evaluating: a full worker is exactly when a fresh spawn
         # needs the slot a finished session is squatting on. Deliberately not
         # inside the per-request loop below — the sweep must reach sessions
@@ -2785,6 +2996,48 @@ class ReviewWatcher:
         )
         return results
 
+    def _note_ledger_unwritable(self) -> None:
+        """Report a pass refused because the ledger cannot record it.
+
+        Shares `Streak` with the poll firewall, so the streak limit, the
+        escalation window and -- the part the hand-rolled copy got wrong -- the
+        crossing's bypass of that limit are one implementation. A daemon that
+        is up, polling, and deciding NOTHING is the most misleading state it
+        can be in, so once the condition outlasts POLL_ESCALATE_SECONDS every
+        logged line says so at page-worthy level.
+        """
+        now = time.monotonic()
+        should_log, crossing = self._ledger_streak.record(now)
+        if not should_log:
+            return
+        log.log(
+            logging.ERROR if self._ledger_streak.escalated else logging.WARNING,
+            "ledger at %s cannot be written — skipping this pass entirely "
+            "(%d consecutive, %.0f min)%s: the daemon will not spawn, escalate, "
+            "grant or post what it cannot record. It is alive and re-probing "
+            "every poll; no review will be queued until the volume is writable.",
+            self.config.state_db,
+            self._ledger_streak.count,
+            self._ledger_streak.held(now) / 60,
+            " — this is no longer transient" if crossing else "",
+        )
+
+    def _note_ledger_writable(self) -> None:
+        """Announce that the gate has re-opened. Unconditional, like the
+        firewall's recovery line: the operator's last word on a degraded
+        daemon must not be the degradation."""
+        ended = self._ledger_streak.resolve(time.monotonic())
+        if ended is None:
+            return
+        skipped, seconds = ended
+        log.info(
+            "ledger at %s is writable again after %d skipped pass(es) over "
+            "%.0fs — resuming normal decisions",
+            self.config.state_db,
+            skipped,
+            seconds,
+        )
+
     def _stage_record(self, slug: str, decision: Decision) -> dict:
         """One per-item entry of a poll snapshot's compact JSON: the PR
         reference (the slug and the number parsed from it), the current stage
@@ -2863,7 +3116,16 @@ class ReviewWatcher:
     def run_forever(self) -> None:
         # preflight() is the caller's responsibility -- the CLI runs it once for
         # every mode, so calling it here too would double every check.
+        #
+        # Every exception a poll can raise is caught HERE (issue #62). The
+        # daemon's steady state has no fatal errors: a transient subprocess
+        # ENOENT, a parse error, a readonly ledger -- any of them is one bad
+        # poll, and poll N+1 may well succeed. Only KeyboardInterrupt and
+        # SystemExit pass through (neither is an `Exception`), and only startup
+        # -- resolve_config, before this loop is ever entered -- still exits
+        # fast.
         backoff = self.config.poll_interval
+        failures = PollFailures()
         while True:
             # The sleep lives INSIDE the KeyboardInterrupt guard: with a 60s
             # poll interval (up to 900s backing off) the loop spends nearly
@@ -2873,13 +3135,91 @@ class ReviewWatcher:
                 try:
                     self.poll_once()
                     backoff = self.config.poll_interval
+                    self._note_poll_recovered(failures)
+                except LedgerUnwritable:
+                    # Already logged, streak-limited and escalating, by the
+                    # gate itself. What matters HERE is what is NOT done: the
+                    # firewall's `failures` is left completely untouched, so a
+                    # fault that is still failing keeps counting toward its own
+                    # page instead of having the clock re-armed by a pass the
+                    # daemon never took. The backoff DOES reset to the poll
+                    # interval, deliberately: probing at cadence is the point
+                    # of the gate, and inheriting a failing streak's 15-minute
+                    # backoff would leave a healed volume unnoticed that long.
+                    backoff = self.config.poll_interval
                 except RateLimited as exc:
-                    backoff = min(backoff * 2, 900)
+                    # Not a failure of the daemon: GitHub is telling it to slow
+                    # down, and it does. It does NOT count toward the firewall's
+                    # streak -- and, just as deliberately, it does not END one
+                    # either (PR #63 round-1 major). A rate limit is not
+                    # evidence that a substrate fault cleared: `review_requests`
+                    # is the first GitHub call in the pass, so it can pre-empt
+                    # the failing call site entirely, and resolving the streak
+                    # here let a busy hour re-arm the escalation clock forever
+                    # and cancel a page the DoD requires. The streak is left
+                    # exactly as it was; only a genuinely successful poll ends
+                    # one, and that path logs the recovery.
+                    backoff = min(backoff * 2, POLL_BACKOFF_CAP_SECONDS)
                     log.warning("rate limited (%s) — backing off %ds", exc, backoff)
-                except CommandError as exc:
-                    backoff = min(backoff * 2, 900)
-                    log.error("poll failed: %s — retrying in %ds", exc, backoff)
+                except Exception as exc:
+                    backoff = min(backoff * 2, POLL_BACKOFF_CAP_SECONDS)
+                    self._note_poll_failure(failures, exc, backoff)
                 time.sleep(backoff)
             except KeyboardInterrupt:
                 log.info("stopping")
                 return
+
+    def _note_poll_failure(
+        self, failures: PollFailures, exc: Exception, backoff: int
+    ) -> None:
+        """Log one firewalled poll failure, streak-limited and escalating.
+
+        WARNING while the fault still looks transient, ERROR once the same
+        class has fired on every poll for POLL_ESCALATE_SECONDS -- the daemon
+        is alive either way, so the log level is the only thing that can tell
+        an operator "this one is not healing".
+        """
+        should_log, crossing = failures.record(exc, time.monotonic())
+        if not should_log:
+            return
+        # A class the firewall did not anticipate gets its traceback; the ones
+        # that carry their own diagnosis do not (see EXPECTED_POLL_FAILURES).
+        traced = not isinstance(exc, EXPECTED_POLL_FAILURES)
+        if crossing:
+            log.error(
+                "poll has failed with %s on every attempt for %.0f min "
+                "(%d consecutive failures, latest: %s) — the daemon is alive and "
+                "still retrying every %ds, but this is no longer transient",
+                failures.kind,
+                failures.streak.held(time.monotonic()) / 60,
+                failures.streak.count,
+                exc,
+                backoff,
+                exc_info=traced,
+            )
+            return
+        log.log(
+            logging.ERROR if failures.streak.escalated else logging.WARNING,
+            "poll failed (%s: %s) — failure %d of this streak; retrying in %ds",
+            failures.kind,
+            exc,
+            failures.streak.count,
+            backoff,
+            exc_info=traced,
+        )
+
+    @staticmethod
+    def _note_poll_recovered(failures: PollFailures) -> None:
+        """Announce that a failing streak ended. Unconditional: the recovery is
+        the counterpart of the escalation, and a streak that healed silently
+        leaves an operator reading the last ERROR as the current state."""
+        ended = failures.resolve(time.monotonic())
+        if ended is None:
+            return
+        count, seconds = ended
+        log.info(
+            "poll recovered after %d consecutive failure(s) over %.0fs — "
+            "resuming normal polling",
+            count,
+            seconds,
+        )

{alissa_tools_github_revloop-0.16.7 → alissa_tools_github_revloop-0.16.9}/src/main/alissa/tools/github/revloop/state.py

@@ -26,11 +26,15 @@ it, alongside the untouched legacy ledgers.
 from __future__ import annotations
 
 import json
+import logging
 import sqlite3
 import time
-from collections.abc import Iterable
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
 from pathlib import Path
 
+log = logging.getLogger(__name__)
+
 # Poll-snapshot retention: the newest N rows are kept, older ones pruned on
 # every write. Fixed, not a config key -- `poll_snapshots` is an observation
 # buffer for a future console sidecar, and a bounded ring is all it needs (it
@@ -38,6 +42,13 @@ from pathlib import Path
 # observable buffer size, so it is pinned by a test.
 SNAPSHOT_RETENTION = 1000
 
+# Streak limiting for the best-effort telemetry writer's WARN (issue #62), on
+# the same rule the poll firewall uses: the first few failures in full, then
+# one in ten. A ledger that has gone read-only fails on every single poll, and
+# the warning is worth nothing if it drowns the decisions around it.
+TELEMETRY_LOG_HEAD = 3
+TELEMETRY_LOG_EVERY = 10
+
 # Shared between SCHEMA and the migration so the two can never drift.
 _SPAWNS_TABLE = """
 CREATE TABLE IF NOT EXISTS spawns (
@@ -108,10 +119,9 @@ CREATE TABLE IF NOT EXISTS verdict_posts (
     -- then released rather than held open forever; see loop._abandon_verdict.
     abandoned_at  INTEGER,
     -- When this round's APPROVE was FIRST held back because the judged head's
-    -- CI rollup had not settled. The wait bound is measured from here, so a
-    -- rollup that never concludes degrades the verdict to a comment at a fixed
-    -- distance from the first observation rather than from whenever the last
-    -- poll happened to land. NULL means the gate never held this round.
+    -- CI rollup had not settled -- and it is never overwritten afterwards, so
+    -- "how long has this round really been held?" always has an answer. NULL
+    -- means the gate never held this round.
     checks_held_at INTEGER,
     -- WHICH unsettled condition that stamp belongs to ('pending' -- checks are
     -- genuinely running -- or 'unknown' -- the rollup could not be read). The
@@ -121,6 +131,13 @@ CREATE TABLE IF NOT EXISTS verdict_posts (
     -- 'pending' one. The policy lives in loop._gate_on_checks; this column is
     -- what lets it be decided from the ledger instead of from memory.
     checks_held_state TEXT,
+    -- When that promotion happened: the stamp the bound is measured from once
+    -- the wait is on checks that are genuinely running. Separate from
+    -- `checks_held_at` rather than replacing it, because the two answer
+    -- different questions and a report that conflates them says a promoted hold
+    -- waited one bound when it waited two. NULL until (and unless) the promotion
+    -- happens; the bound then reads `checks_pending_at or checks_held_at`.
+    checks_pending_at INTEGER,
     review_url    TEXT,
     last_error    TEXT,
     PRIMARY KEY (repo, number, round)
@@ -163,10 +180,42 @@ _ADDED_COLUMNS = {
     "verdict_posts": (
         ("checks_held_at", "INTEGER"),
         ("checks_held_state", "TEXT"),
+        ("checks_pending_at", "INTEGER"),
     ),
 }
 
 
+@dataclass(frozen=True)
+class ChecksHold:
+    """One round's CI hold, as the ledger remembers it.
+
+    Two stamps, because the gate has two honest numbers to report and they can
+    differ by a whole wait bound:
+
+    * `first_at` -- when the round was first held at all, on whatever condition;
+    * `pending_at` -- when an unreadable hold was promoted to a genuinely
+      pending one, which restarts the clock the bound is measured from (see
+      loop._gate_on_checks for why exactly once).
+
+    `since` is the one the bound uses. `first_at` is the one an operator means by
+    "how long has this been held?", and reporting only `since` after a promotion
+    understates it by up to the full bound.
+    """
+
+    first_at: int | None = None
+    condition: str | None = None
+    pending_at: int | None = None
+
+    @property
+    def since(self) -> int | None:
+        """The stamp the wait bound is measured from."""
+        return self.pending_at or self.first_at
+
+    @property
+    def promoted(self) -> bool:
+        return self.pending_at is not None
+
+
 class State:
     def __init__(self, path: Path, *, read_only: bool = False):
         """Open the ledger. The daemon opens it read-write (creating the file,
@@ -192,6 +241,16 @@ class State:
         for any path an operator can type.
         """
         path = Path(path).expanduser()
+        # Kept so a best-effort telemetry write can RECONNECT after a failure
+        # (see _reconnect): the daemon's ledger lives on a platform volume, and
+        # a remount leaves the open handle pointing at a file descriptor that
+        # is gone while the path is perfectly good again.
+        self._path = path
+        self._read_only = read_only
+        # Consecutive failures of the best-effort writer, for streak-limited
+        # logging and for firing the one reconnect attempt on the FIRST failure
+        # of a streak rather than on every write.
+        self._telemetry_failures = 0
         if read_only:
             uri = Path(path).absolute().as_uri() + "?mode=ro"
             self._db = sqlite3.connect(uri, uri=True)
@@ -265,6 +324,167 @@ class State:
             return False  # fresh database, nothing to migrate
         return [r["name"] for r in info if r["pk"]] != ["session"]
 
+    # -- best-effort writes (issue #62) ------------------------------------
+    #
+    # CLASSIFICATION. Every write in this class is one of two kinds, and only
+    # one of them may ever be swallowed:
+    #
+    # * TELEMETRY -- `record_snapshot`, and nothing else. `poll_snapshots` is an
+    #   observation buffer: the daemon never reads it back to make a decision
+    #   (only `read_snapshots`, for the console, does), so a row lost to a
+    #   read-only volume costs one missing datapoint on a dashboard. On
+    #   2026-07-29 it cost the whole daemon instead -- the sqlite exception
+    #   escaped `poll_once` and killed the process mid-poll.
+    #
+    # * CORRECTNESS -- every other write here (`record_spawn`, `record_reap`,
+    #   `record_ping`, `record_escalation`, `record_grant`, the `verdict_posts`
+    #   writes, `age_out_spawn`). Each is a dedupe key or an in-flight marker
+    #   for an action the daemon TAKES: swallowing one does not lose a
+    #   datapoint, it re-spawns a reviewer round, re-pages an operator, or
+    #   re-grants a cap. Those stay strict and raise.
+    #
+    # STRICTNESS IS NOT, BY ITSELF, THE PROTECTION -- and the first draft of
+    # this change claimed it was (PR #63 round-1 blocker). Raising aborts the
+    # pass that failed; it says nothing about the next one. The poll firewall
+    # then hands the loop straight back to the same code path, and the side
+    # effect the write was meant to dedupe has ALREADY been taken -- so a
+    # read-only volume turned "enqueue a reviewer, fail to record it" into a
+    # fresh reviewer session every poll, indefinitely, where before it merely
+    # killed the daemon after one. What actually protects the side effect is
+    # `writable()` above, checked by `loop.poll_once` before the pass takes any
+    # decision at all: the daemon does not take an action it cannot record.
+    # Strictness is what makes an unrecordable action VISIBLE; the gate is what
+    # makes it not repeat.
+
+    @staticmethod
+    def _write_probe(db: sqlite3.Connection) -> bool:
+        """Can this connection actually write? A no-op header write, which
+        exercises exactly the path a real write needs, changes nothing, and
+        costs one page. Read-only-ness is the thing being detected, so it
+        cannot be answered by inspecting the file's mode: sqlite decides it at
+        open time and a handle can be read-only over a writable file (and, for
+        one recoverable moment, the reverse)."""
+        try:
+            version = db.execute("PRAGMA user_version").fetchone()[0]
+            db.execute(f"PRAGMA user_version = {int(version)}")
+            db.commit()
+        except sqlite3.DatabaseError:
+            return False
+        return True
+
+    def writable(self) -> bool:
+        """Whether the ledger can accept a write RIGHT NOW.
+
+        The daemon asks this before it takes any action it would have to
+        record (issue #62, round-1 blocker). Keeping a correctness write strict
+        aborts the pass that fails, but the poll firewall hands the loop
+        straight back to the same code path -- so without this gate a read-only
+        volume turns "enqueue a reviewer, then fail to record it" into a fresh
+        reviewer session every poll, forever, each one a live agent. The
+        invariant the gate buys is simple: the daemon does not take an action
+        it cannot record.
+
+        A failing probe retries through `_reconnect`, whose candidate is
+        write-probed before adoption -- so a stale handle over a live file
+        heals here too, and only a genuinely unwritable ledger answers False.
+        A read-only `State` (the console's) is never writable by construction.
+        """
+        if self._read_only:
+            return False
+        return self._write_probe(self._db) or self._reconnect()
+
+    def _reconnect(self) -> bool:
+        """Swap in a fresh connection, but ONLY if the fresh one is better.
+        True when the swap happened -- which, because the candidate is
+        write-probed, is also proof that the ledger is writable.
+
+        Deliberately raw: it re-establishes the connection and NOTHING else --
+        no schema script, no migration. The reconnect exists for the
+        stale-handle-after-remount case, where the database on disk is the one
+        this process already migrated; re-running DDL through a path that only
+        a failed telemetry write reaches would be a far larger act than the
+        failure justifies.
+
+        The candidate is WRITE-PROBED before it is adopted, and the old
+        connection is kept when the probe fails, because a blind reconnect
+        makes the read-only case permanently worse rather than better: sqlite
+        decides read-only-ness when it OPENS the file, so a handle opened while
+        the volume was read-only stays read-only for the rest of its life even
+        after the volume comes back -- while the handle opened before the fault
+        heals by itself the moment writes are possible again. Replacing the
+        healable handle with a poisoned one would trade a transient outage for
+        a permanent one.
+
+        WRITE MODE ONLY. A read-only `State` is the console's, it must never
+        write, and there is nothing a reconnect could improve for it -- so it
+        returns False rather than swapping one equivalent handle for another.
+        That also keeps the contract absolute: a True from here always means a
+        candidate passed the write probe, which is what `writable()` relies on.
+        """
+        if self._read_only:
+            return False
+        candidate: sqlite3.Connection | None = None
+        try:
+            candidate = sqlite3.connect(str(self._path))
+            candidate.row_factory = sqlite3.Row
+        except sqlite3.Error as exc:
+            log.debug("state: reconnect to %s declined: %s", self._path, exc)
+            return False
+        if not self._write_probe(candidate):
+            log.debug("state: reconnect to %s declined (candidate cannot write)", self._path)
+            try:
+                candidate.close()
+            except sqlite3.Error:
+                pass
+            return False
+        try:
+            self._db.close()
+        except sqlite3.Error:
+            pass  # already broken; the point was to replace it
+        self._db = candidate
+        return True
+
+    def _write_telemetry(self, write: "Callable[[], None]", what: str) -> bool:
+        """Run a TELEMETRY write, absorbing any database error. True on success.
+
+        One reconnect attempt on the FIRST failure of a streak (not on every
+        one: a database that is read-only stays read-only, and reconnecting per
+        poll would add a file open to every pass for nothing), then a
+        streak-limited WARN and back to polling.
+        """
+        try:
+            write()
+        except sqlite3.DatabaseError as exc:
+            first = self._telemetry_failures == 0
+            if first and self._reconnect():
+                try:
+                    write()
+                except sqlite3.DatabaseError as retry_exc:
+                    exc = retry_exc
+                else:
+                    log.info(
+                        "state: %s succeeded after reconnecting to %s",
+                        what, self._path,
+                    )
+                    return True
+            self._telemetry_failures += 1
+            n = self._telemetry_failures
+            if n <= TELEMETRY_LOG_HEAD or n % TELEMETRY_LOG_EVERY == 0:
+                log.warning(
+                    "state: %s failed (%s: %s) — failure %d of this streak; "
+                    "telemetry is best-effort, the loop keeps polling",
+                    what, type(exc).__name__, exc, n,
+                )
+            return False
+        if self._telemetry_failures:
+            log.info(
+                "state: %s succeeded after %d failed attempt(s) — telemetry "
+                "is persisting again",
+                what, self._telemetry_failures,
+            )
+            self._telemetry_failures = 0
+        return True
+
     def close(self) -> None:
         self._db.close()
 
@@ -509,38 +729,59 @@ class State:
         row = self.get_verdict_post(repo, number, round_)
         return int(row["attempts"]) if row else 0
 
-    def checks_hold(
-        self, repo: str, number: int, round_: int
-    ) -> "tuple[int | None, str | None]":
-        """When this round's approve was first held on CI, and on WHICH
-        condition -- (None, None) if it has never been held.
+    def checks_hold(self, repo: str, number: int, round_: int) -> "ChecksHold":
+        """This round's CI hold: when it began, what it is waiting on, and when
+        that became a genuine `pending` -- an all-None ChecksHold if it has never
+        been held.
 
         A read, deliberately: whether an existing stamp still applies is a
         policy question about CI (see loop._gate_on_checks), and this table's job
-        is to remember the answer, not to make it.
+        is to remember the answer, not to make it. It remembers BOTH stamps
+        because the two answer different questions -- `since` bounds the wait,
+        `first_at` is how long the round has really been held -- and a report
+        that conflates them tells an operator a promoted hold waited 30 minutes
+        when it waited 60.
         """
         row = self.get_verdict_post(repo, number, round_)
         if row is None or not row["checks_held_at"]:
-            return None, None
-        return int(row["checks_held_at"]), (row["checks_held_state"] or None)
+            return ChecksHold()
+        pending_at = row["checks_pending_at"]
+        return ChecksHold(
+            first_at=int(row["checks_held_at"]),
+            condition=(row["checks_held_state"] or None),
+            pending_at=int(pending_at) if pending_at else None,
+        )
 
     def record_checks_hold(
         self, repo: str, number: int, round_: int, condition: str
     ) -> int:
-        """Start (or restart) the hold clock for this round; return the stamp.
+        """Record that this round is held on `condition`; return the stamp the
+        bound is measured from.
 
         The caller decides WHEN to call this -- once when the hold begins, and at
         most once more when an unreadable hold is promoted to a genuinely pending
         one, because the bound is defined against the first observation of the
-        condition actually being waited on. Unconditional here so that policy
-        stays in one place instead of being half-expressed as a WHERE clause.
+        condition actually being waited on (loop._gate_on_checks owns that rule).
+
+        The promotion NO LONGER overwrites `checks_held_at`: it fills
+        `checks_pending_at` instead, so the ledger keeps when the round was first
+        held as well as when its current wait started. Nothing about the bound
+        changes; what changes is that the daemon can now say both numbers out
+        loud, which the operator-facing report needs.
         """
         now = int(time.time())
-        self._db.execute(
-            "UPDATE verdict_posts SET checks_held_at = ?, checks_held_state = ? "
-            "WHERE repo=? AND number=? AND round=?",
-            (now, condition, repo, number, round_),
-        )
+        if self.checks_hold(repo, number, round_).first_at is None:
+            self._db.execute(
+                "UPDATE verdict_posts SET checks_held_at = ?, checks_held_state = ? "
+                "WHERE repo=? AND number=? AND round=?",
+                (now, condition, repo, number, round_),
+            )
+        else:
+            self._db.execute(
+                "UPDATE verdict_posts SET checks_pending_at = ?, "
+                "checks_held_state = ? WHERE repo=? AND number=? AND round=?",
+                (now, condition, repo, number, round_),
+            )
         self._db.commit()
         return now
 
@@ -603,7 +844,7 @@ class State:
         return self._read_rows(
             "SELECT repo, number, round, first_seen_at, head_sha, attempts, "
             "last_attempt_at, posted_at, abandoned_at, checks_held_at, "
-            "checks_held_state, review_url, last_error "
+            "checks_held_state, checks_pending_at, review_url, last_error "
             "FROM verdict_posts "
             "ORDER BY first_seen_at DESC, number DESC",
             limit,
@@ -629,7 +870,7 @@ class State:
         awaiting_post: int = 0,
         abandoned: int = 0,
         stages: list[dict],
-    ) -> None:
+    ) -> bool:
         """Append one poll-pass observation, then prune to the newest
         SNAPSHOT_RETENTION rows. `ts` is stamped here (wall-clock seconds,
         like every other row in this ledger); `stages` is the compact
@@ -638,7 +879,56 @@ class State:
         dry-run included -- and pruned on write, so the table is
         self-bounding. The count kwargs default to 0 so a caller need only
         pass the ones a given pass produced.
+
+        BEST-EFFORT, and the only write in this class that is (issue #62): a
+        snapshot observes the pass, it is not something the daemon has to
+        remember, so a database error here is absorbed, reported once per
+        streak-limited window, and the loop keeps polling. Returns whether the
+        row landed, for a caller that wants to say so; nothing in the daemon
+        depends on it.
         """
+        return self._write_telemetry(
+            lambda: self._insert_snapshot(
+                duration_ms=duration_ms,
+                candidates=candidates,
+                spawned=spawned,
+                stale_reenqueued=stale_reenqueued,
+                in_flight=in_flight,
+                deferred=deferred,
+                converged=converged,
+                capped=capped,
+                escalated=escalated,
+                skipped=skipped,
+                reaped=reaped,
+                posted=posted,
+                awaiting_post=awaiting_post,
+                abandoned=abandoned,
+                stages=stages,
+            ),
+            "poll snapshot",
+        )
+
+    def _insert_snapshot(
+        self,
+        *,
+        duration_ms: int,
+        candidates: int,
+        spawned: int,
+        stale_reenqueued: int,
+        in_flight: int,
+        deferred: int,
+        converged: int,
+        capped: int,
+        escalated: int,
+        skipped: int,
+        reaped: int,
+        posted: int,
+        awaiting_post: int,
+        abandoned: int,
+        stages: list[dict],
+    ) -> None:
+        """The snapshot INSERT + prune itself, strict. Split out so the
+        best-effort wrapper can RETRY it verbatim after a reconnect."""
         self._db.execute(
             "INSERT INTO poll_snapshots "
             "(ts, duration_ms, candidates, spawned, stale_reenqueued, "

alissa_tools_github_revloop-0.16.9/src/main/alissa/tools/github/revloop/version

@@ -0,0 +1 @@
+0.16.9

{alissa_tools_github_revloop-0.16.7 → alissa_tools_github_revloop-0.16.9/src/main/alissa_tools_github_revloop.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alissa-tools-github-revloop
-Version: 0.16.7
+Version: 0.16.9
 Summary: ALISSA-TOOLS-GITHUB-REVLOOP
 Home-page: https://alissa.app
 Author: Fahera

alissa_tools_github_revloop-0.16.7/src/main/alissa/tools/github/revloop/version

@@ -1 +0,0 @@
-0.16.7