switchroom 0.19.19 → 0.19.23

package/vendor/hindsight-memory/scripts/drain_pending.py

@@ -3,9 +3,25 @@
 
 SessionStart calls into ``drain()`` to retry any retain payloads that
 ``session_end.py`` queued on failure (#1071). Each entry is retried up
-to ``MAX_ATTEMPTS`` (5) times; after that it's renamed to ``.dead`` so
-the queue no longer drains it but the operator can still inspect via
-``switchroom doctor``.
+to ``MAX_ATTEMPTS`` (5) times; after that a **permanently** failing entry
+(a 4xx that a re-POST cannot fix — see ``pending.is_permanent_failure``)
+is retired into the ``pending-dead/`` archive so the queue no longer
+drains it but the operator can still inspect via ``switchroom doctor``.
+The marker deliberately does NOT stay in the live queue directory: it is
+the only remaining copy of that memory, and leaving it among the live
+entries put it in the path of every janitor that sweeps that
+directory. An entry failing on anything
+else — a 5xx, a timeout, a connection error — stays queued past the
+attempt budget: a transient upstream is never evidence that the memory
+is unsaveable, and retiring it would lose content the user believes was
+saved.
+
+An entry past the budget is DEMOTED rather than retired (``_drain_order``
+and ``_over_budget``): it sorts behind everything still inside its budget
+and abstains from the stall guard. That is what keeps "never destroy a
+memory" from degrading into "never drain anything" — the drain is
+sequential and oldest-first, so without the demotion three chronically
+failing entries sit at the head and end every run at zero progress.
 
 Boundaries
 ----------
@@ -36,14 +52,35 @@ accumulated backlog. Worse, they CREATE one: the per-entry timeout is
 clamped to the remaining hook budget (1-8s) while ``_retry_one`` posts
 synchronously and a real retain takes 30-90s, so **the server commits the
 document and the client always gives up before the ack**. The entry is
-never deleted and is re-posted on every session start, forever, until it
-hits ``MAX_ATTEMPTS`` and goes ``.dead``. The queue depth was a symptom
+never deleted and is re-posted on every session start. The queue depth was a symptom
 of that loop, not of lost memory: a full sweep of 5,751 queued entries on
 this fleet (2026-07-25) found **4,048 (70.4%) already existed as
 documents**, 3,815 of them with facts extracted.
 
-``--backlog`` is therefore a two-phase, out-of-hook replay:
-
+``--backlog`` is therefore a three-phase, out-of-hook replay:
+
+* **Phase 0 — collapse duplicates (free, no network).** Queued entries
+  sharing ``(bank_id, part_position, sha256(content))`` are the same
+  memory; the redundant copies are archived so the phases below never pay
+  for one memory twice. Measured 2026-07-26: 1,060 queued files across 11
+  agents fell into ~368 distinct groups — ~65% of the queue was duplicate,
+  with one group repeated 32 times. At ~168 s per phase-2 extraction that
+  one group alone was 90 minutes of LLM lane time for a single memory.
+* **Phase 0b — relocate legacy ``.dead`` markers (free, no network).**
+  Markers written by an older build into the live queue directory are moved
+  into ``pending-dead/``. ``mark_dead`` no longer produces such a marker, so
+  after this phase has run once the live queue holds only live entries and no
+  janitor glob over it can match a memory. Note the CONDITION: phases 0b and
+  0c run in BACKLOG mode only (``drain_backlog``, and not under
+  ``--dry-run``). The SessionStart ``drain()`` never calls them, so on a host
+  where the backlog drain has not run, legacy markers are still sitting in
+  the queue directory.
+* **Phase 0c — re-split over-bound entries (free, no network).** An entry
+  whose content exceeds ``retain_content_limit()`` needs more sequential
+  extraction calls than fit the client deadline, so it can never be drained
+  as-is; splitting it makes every part drainable. Measured 2026-07-26: 18 of
+  211 queued entries exceeded 100,000 chars, the largest 744,546. Backlog
+  mode only, same as 0b.
 * **Phase 1 — reconcile (free).** GET the document. If it exists, the
   memory is already durable; retire the queue entry without a POST. No
   LLM work, no cost, idempotent, resumable at any point. Only for
@@ -73,7 +110,7 @@ reports the upstream is already slow.
 Standalone usage::
 
     python3 drain_pending.py               # bounded in-hook drain
-    python3 drain_pending.py --backlog     # two-phase backlog replay
+    python3 drain_pending.py --backlog     # three-phase backlog replay
     python3 drain_pending.py --backlog --phase reconcile   # free pass only
     python3 drain_pending.py --backlog --dry-run
 """
@@ -94,12 +131,16 @@ from lib.config import debug_log, load_config
 from lib.pending import (
     MAX_ATTEMPTS,
     archive_reconciled,
+    collapse_duplicates,
     is_content_derived_document_id,
+    is_permanent_failure,
     iter_entries,
     mark_dead,
+    resplit_over_bound_entries,
+    sweep_legacy_dead_markers,
     update_attempt,
 )
-from lib.retain_split import retain_client_deadline
+from lib.retain_split import retain_client_deadline, retain_content_limit
 
 
 STALL_THRESHOLD = 3
@@ -203,16 +244,30 @@ def _backlog_timeout() -> int:
     client-side timeout on a request the server then commits anyway.
 
     The default is DERIVED, not a literal: it is
-    ``retain_split.retain_client_deadline()`` (280s), the same deadline the
+    ``retain_split.retain_client_deadline()`` (310s), the same deadline the
     retain content bound is sized against. Those two must be ONE number.
     This function shipped as a bare ``180`` (#3599), and against a 180s
     deadline both halves of the retain budget break: a maximally-sized part
     is ~276s of sequential extraction, and the SERVER per-call timeout
-    derived in ``src/setup/hindsight.ts`` (#3611) is 204s — so the drain
+    derived in ``src/setup/hindsight.ts`` is larger still — so the drain
     client would abandon a request the server is still legitimately working
     on, leave the entry queued, and rebuild the re-post loop #3599 exists to
     kill, one size class up.
 
+    This is the OUTERMOST deadline of the paired-budget family, and it is the
+    one the 2026-07-26 backlog-recovery logs show as a fixed ~280.1s per-entry
+    give-up. That was not an unexplained third number: it is this default at
+    the then-current ``DEFAULT_RETAIN_CLIENT_DEADLINE_S = 280.0``, plus the
+    request's own setup overhead. It was WRONG for the same reason #3611's
+    204s was wrong — it did not cover hindsight's LiteLLM routing chain
+    (local 200s + OpenRouter fallback 90s + router margin 10s = 300s), so
+    every drained entry whose retain fell through to the fallback was
+    abandoned client-side at 280s while the server was still inside a
+    legitimate 300s budget. Raising the derivation's base to 310 fixes this
+    lane and the in-hook lane with the same number, by construction.
+    ``src/litellm/timeout-budget.ts`` is where the chain is declared, and
+    ``tests/setup/hindsight.test.ts`` fails if these drift apart again.
+
     ``HINDSIGHT_DRAIN_BACKLOG_TIMEOUT`` still overrides it outright;
     ``HINDSIGHT_RETAIN_CLIENT_DEADLINE_S`` moves the derivation and the
     content bound together.
@@ -348,6 +403,76 @@ def _document_state(entry: dict, timeout: int = 30):
         return None
 
 
+def _drain_order(entries: list[tuple[str, dict]]) -> list[tuple[str, dict]]:
+    """Oldest-first, but with budget-exhausted entries demoted to the back.
+
+    THE HEAD-OF-LINE BUG THIS EXISTS FOR. Since the permanence gate in
+    ``_record_failure``, an entry failing on anything transient stays queued
+    past ``MAX_ATTEMPTS`` indefinitely — deliberately, because a transient
+    upstream is not evidence the memory is unsaveable. But ``iter_entries()``
+    is oldest-first, and the entries that have been failing longest are by
+    construction the OLDEST, so they sit at the head of every drain. Backlog
+    concurrency defaults to 1 (``_backlog_concurrency``), so the drain is
+    sequential: three such entries in a row trip ``STALL_THRESHOLD`` and the
+    run breaks having drained nothing — and, because they never retire, it
+    breaks identically on every subsequent run. Measured on this branch before
+    this function existed: 6 queued entries, the 3 oldest raising
+    ``TimeoutError``, 4 consecutive ``drain_backlog`` runs each returning
+    ``stalled=True, drained=0`` with the queue depth still 6. The 3 healthy
+    entries behind them were never even attempted. On ``main`` the same repro
+    converges: run 0 retires the 3 heads to ``.dead`` and run 1 onward drains
+    normally. So the permanence gate, alone, traded "rarely destroys a memory"
+    for "eventually drains nothing at all".
+
+    ``.dead`` was doing double duty: it was the honesty policy AND it was the
+    queue's only un-wedging mechanism. Removing it as a policy has to leave
+    the un-wedging behind, and demotion is that — it keeps every property the
+    gate was added for (the entry is still queued, still retried, never
+    destroyed) while removing the one it broke (it can no longer starve a
+    healthy entry behind it).
+
+    The terminal condition for a permanently-unsaveable entry is therefore no
+    longer deletion but DEMOTION: ``attempt_count`` only ever climbs, so such
+    an entry crosses the budget once and stays in the back group for good,
+    where it can delay only itself. It is still reconciled for free on every
+    run — ``_reconcile_phase`` sweeps ALL entries with a sub-second presence
+    GET, in no particular order and with no stall guard — so an entry whose
+    document did land is still retired without a POST.
+
+    Ordering is a stable partition, so relative age is preserved inside each
+    group and FIFO still holds for everything that has not blown its budget.
+    """
+    fresh: list[tuple[str, dict]] = []
+    exhausted: list[tuple[str, dict]] = []
+    for path, entry in entries:
+        (exhausted if _over_budget(entry) else fresh).append((path, entry))
+    return fresh + exhausted
+
+
+def _over_budget(entry: dict) -> bool:
+    """Has this entry already burned its ``MAX_ATTEMPTS`` budget?
+
+    Such an entry is chronically failing but, since the permanence gate, is
+    never retired. It gets two demotions — last in the drain order
+    (``_drain_order``) and no vote in the stall guard (see below) — because
+    ordering alone does not close the wedge. Ordering fixes the common shape
+    (a few old poison entries in front of healthy ones), but not the shape
+    where the WHOLE queue is over budget: an upstream down for a week takes
+    every entry past 5 attempts, and when it recovers the partition is empty
+    on one side, the poisoned entries are at the head again, and the run
+    stalls before reaching the entries that would now succeed. Measured: with
+    ordering alone and all 6 entries at ``MAX_ATTEMPTS``, 4 consecutive runs
+    still returned ``stalled=True, drained=0``.
+    """
+    try:
+        return int(entry.get("attempt_count", 0)) >= MAX_ATTEMPTS
+    except (TypeError, ValueError):
+        # A hand-edited or corrupt counter must not decide ordering, and must
+        # not raise on the drain path. Treat it as fresh: the cost of guessing
+        # wrong here is one retry in the normal position, not a lost memory.
+        return False
+
+
 def _record_failure(
     config: dict,
     path: str,
@@ -358,16 +483,53 @@ def _record_failure(
     """Apply the per-entry failure policy. Returns the error class name.
 
     Shared by the sequential (SessionStart) and backlog drains so both age
-    entries toward ``.dead`` on exactly the same schedule.
+    entries toward ``.dead`` on exactly the same schedule — and, since the
+    permanence gate below, refuse to retire them on exactly the same rule.
     """
     err_class = type(e).__name__
     attempts = int(entry.get("attempt_count", 1))
-    if attempts >= MAX_ATTEMPTS:
+    # ``.dead`` retires a memory the user believes was saved, so it is gated on
+    # the failure being PERMANENT — a 4xx that re-POSTing cannot fix. A
+    # transient failure keeps its attempt counter climbing but stays queued,
+    # because an exhausted attempt budget is not evidence that the content is
+    # unpersistable.
+    #
+    # Before this gate, ANY five failures retired the entry. The dominant
+    # failure on this fleet is an HTTP 500 "Fact extraction failed … chunk 0:
+    # JSONDecodeError" — the extraction model returned an empty or non-JSON
+    # completion for one chunk on that sampling run. The identical content
+    # succeeds on a later attempt, so five unlucky samples were destroying
+    # memories that were never unsaveable. See ``pending.is_permanent_failure``.
+    #
+    # WHAT BOUNDS THIS, precisely — because an unbounded queue of undying
+    # entries would be a worse outcome than the bug this gate fixes.
+    #
+    # DISK is bounded by the queue's MAX_ENTRIES / MAX_BYTES caps, which shed
+    # the oldest entries into ``pending-evicted/`` (an archive, not a delete).
+    # Note what that bound is NOT: ``_evict_to_fit`` is called only from
+    # ``enqueue`` (lib/pending.py:933), so it fires on new writes, never on
+    # drain — and it "bounds" the queue by shedding memory unsaved, which is
+    # the very outcome this gate exists to avoid. It is a backstop, not the
+    # answer.
+    #
+    # PROGRESS is bounded by ``_drain_order`` / ``_over_budget``. An entry that
+    # can never be persisted no longer terminates by being destroyed; it
+    # terminates by being DEMOTED — sorted behind every entry still inside its
+    # budget, and stripped of its vote in the stall guard. ``attempt_count``
+    # only ever climbs, so the crossing happens once and is permanent. That is
+    # the real terminal condition, and it is what keeps a poisoned entry from
+    # starving the queue behind it. Without it, three such entries ended every
+    # drain at zero progress, permanently (tests/test_pending_wedge.py).
+    #
+    # The attempt counter itself was never a bound on either, and ``switchroom
+    # doctor`` still surfaces queue depth so the operator sees a growing tail.
+    if attempts >= MAX_ATTEMPTS and is_permanent_failure(e):
         marker = mark_dead(path, entry)
         summary["dead"] += 1
         print(
             f"[Hindsight] drain_pending: entry exceeded {MAX_ATTEMPTS} "
-            f"attempts, marking dead at {marker} (last error: {err_class}: {e})",
+            f"attempts on a permanent failure, marking dead at {marker} "
+            f"(last error: {err_class}: {e})",
             file=sys.stderr,
         )
     else:
@@ -375,7 +537,12 @@ def _record_failure(
         summary["retried"] += 1
         debug_log(
             config,
-            f"drain_pending: retry {attempts}/{MAX_ATTEMPTS} failed for {path} ({err_class}: {e})",
+            # ``attempts`` can now exceed MAX_ATTEMPTS — a transient failure
+            # keeps the entry queued past the budget instead of retiring it —
+            # so print the budget as a threshold, not as a fraction that would
+            # render the nonsense "retry 7/5".
+            f"drain_pending: retry {attempts} (budget {MAX_ATTEMPTS}) failed "
+            f"for {path} ({err_class}: {e})",
         )
     return err_class
 
@@ -393,17 +560,24 @@ def _new_summary() -> dict:
         # so it must not be counted as drained/reconciled, which would
         # report a retire that did not happen.
         "archive_failed": 0,
+        # Redundant copies retired by `pending.collapse_duplicates` before
+        # any network work. Backlog mode only — see `_drain_backlog_impl`.
+        "collapsed": 0,
+        "dead_relocated": 0,
+        "resplit": 0,
+        "resplit_parts": 0,
         "stalled": False,
         "budget_exceeded": False,
     }
 
 
 def drain_backlog(config: dict | None = None, **kw) -> dict:
-    """Two-phase backlog replay, off the SessionStart budget entirely.
+    """Three-phase backlog replay, off the SessionStart budget entirely.
 
     See the module docstring. Summary shape is ``drain()``'s plus
-    ``reconciled`` (already durable — no POST issued) and ``unknown``
-    (presence could not be established; left queued).
+    ``reconciled`` (already durable — no POST issued), ``unknown``
+    (presence could not be established; left queued) and ``collapsed``
+    (redundant duplicate copies archived before any network work).
     """
     return drain(config, backlog=True, **kw)
 
@@ -428,6 +602,10 @@ def drain(
          "unknown": int,   # presence unknown, left queued
          "archive_failed": int,  # durable, but the archive was unwritable
                                  # so the entry is STILL QUEUED
+         "collapsed": int, # duplicate copies archived (backlog mode only)
+         "dead_relocated": int,  # legacy .dead markers moved out of the queue dir
+         "resplit":   int, # over-bound entries split into drainable parts
+         "resplit_parts": int,  # parts those entries became
          "stalled": bool,  # stall guard tripped
          "budget_exceeded": bool}
     """
@@ -440,7 +618,11 @@ def drain(
 
     summary = _new_summary()
 
-    entries = iter_entries()
+    # Budget-exhausted entries go last so they cannot starve the healthy ones
+    # behind them — see ``_drain_order``. Matters even more here than in the
+    # backlog drain: the in-hook budget is ~4s, so a single entry at the head
+    # that always burns its clamped timeout consumes the entire run.
+    entries = _drain_order(iter_entries())
     if not entries:
         debug_log(config, "drain_pending: queue empty")
         return summary
@@ -496,14 +678,30 @@ def drain(
             # allowance must not be handed out twice (#3599 review F2).
             _retry_one(entry, timeout=_clamp(timeout, budget, started))
         except Exception as e:
+            # Read the budget state BEFORE _record_failure: update_attempt
+            # mutates `entry["attempt_count"]` in place (lib/pending.py:1020),
+            # so asking afterwards would count the entry that just CROSSED the
+            # budget on this very failure as an abstainer, silently weakening
+            # the stall guard for ordinary entries.
+            abstains = _over_budget(entry)
             err_class = _record_failure(config, path, entry, e, summary)
-            if err_class == last_error_class:
-                consecutive_failures += 1
-            else:
-                consecutive_failures = 1
-                last_error_class = err_class
+            # STALL GUARD ABSTENTION — see ``_over_budget``. The guard exists
+            # to detect a broken UPSTREAM and stop hammering it. A chronically
+            # failing entry that has already burned its attempt budget is
+            # evidence about that ENTRY, not about the upstream, and letting it
+            # vote is what wedges the queue: it never retires, so it fails
+            # identically on every run, and three of them end every run before
+            # the healthy entries behind them are ever reached. It abstains —
+            # neither incrementing the counter nor resetting it, so a genuine
+            # upstream outage is still caught by the fresh entries around it.
+            if not abstains:
+                if err_class == last_error_class:
+                    consecutive_failures += 1
+                else:
+                    consecutive_failures = 1
+                    last_error_class = err_class
 
-            if consecutive_failures >= STALL_THRESHOLD:
+            if not abstains and consecutive_failures >= STALL_THRESHOLD:
                 summary["stalled"] = True
                 print(
                     f"[Hindsight] drain_pending: {consecutive_failures} consecutive "
@@ -626,9 +824,65 @@ def _wait_for_upstream(backoff_ms: int, started: float, budget: float) -> bool:
 def _drain_backlog_impl(
     config: dict, phase: str = "both", dry_run: bool = False
 ) -> dict:
-    """Concurrent-capable, long-budget, two-phase backlog replay."""
+    """Concurrent-capable, long-budget, three-phase backlog replay."""
     summary = _new_summary()
 
+    # PHASE 0 — collapse duplicates (local, free, no network at all).
+    #
+    # Runs FIRST because every later phase is per-entry: a GET in phase 1
+    # and, worse, a ~168 s LLM-backed extraction in phase 2. On the measured
+    # 2026-07-26 fleet backlog ~65% of queued files were byte-identical
+    # copies of another queued file (top group 32x), so skipping this pass
+    # means paying phase 2 up to 32 times over for one memory.
+    #
+    # DELIBERATELY NOT run by the in-hook drain. It reads every queued entry
+    # to recompute identity from content — bounded, but not instant — and
+    # the SessionStart drain's whole contract is a hard wall-clock ceiling
+    # on hook latency. New duplicates cannot accumulate there anyway:
+    # `pending.enqueue`'s filename-keyed guard stops those at the producer.
+    if not dry_run:
+        summary["collapsed"] = collapse_duplicates()
+        if summary["collapsed"]:
+            _blog(
+                f"phase 0: collapsed {summary['collapsed']} duplicate entries "
+                f"(byte-identical content already queued under another entry); "
+                f"archived, not deleted"
+            )
+
+        # PHASE 0b — relocate any legacy `.dead` markers out of the live
+        # queue directory. Free, local, and an UPGRADE step: markers written
+        # by an older build sit in the directory external janitors sweep, and
+        # a marker is the only remaining copy of its memory.
+        moved = sweep_legacy_dead_markers()
+        if moved:
+            summary["dead_relocated"] = moved
+            _blog(
+                f"phase 0b: relocated {moved} legacy .dead marker(s) out of "
+                f"the live queue directory; the queue now holds only live "
+                f"entries, so no janitor glob over it can match a memory"
+            )
+
+        # PHASE 0c — re-split entries the drain provably CANNOT retain.
+        #
+        # An entry over `retain_content_limit()` needs more sequential
+        # extraction calls than fit the deadline, so every POST is guaranteed
+        # waste; if the server rejects the body as a 4xx it is classified
+        # permanent and the memory goes `.dead`. Splitting it makes every
+        # part drainable, which is the difference between a lost memory and a
+        # slow one. Runs after the duplicate collapse so a duplicated
+        # over-bound entry is split ONCE, not once per copy.
+        entries_split, parts_written = resplit_over_bound_entries()
+        if entries_split:
+            summary["resplit"] = entries_split
+            summary["resplit_parts"] = parts_written
+            _blog(
+                f"phase 0c: re-split {entries_split} entr"
+                f"{'y' if entries_split == 1 else 'ies'} over the "
+                f"{retain_content_limit()}-char retain bound into "
+                f"{parts_written} drainable part(s); the originals are "
+                f"archived, not deleted"
+            )
+
     if phase in ("reconcile", "both"):
         _reconcile_phase(config, summary, dry_run)
     if phase == "reconcile":
@@ -641,7 +895,10 @@ def _drain_backlog_impl(
     backoff_ms = _p95_backoff_ms()
     started = time.monotonic()
 
-    entries = iter_entries()
+    # See ``_drain_order``: without this, three budget-exhausted entries at the
+    # head trip the stall guard on every run forever and the drain never makes
+    # progress again.
+    entries = _drain_order(iter_entries())
     if not entries:
         _blog("phase 2: nothing left to retain")
         return summary
@@ -728,15 +985,22 @@ def _drain_backlog_impl(
             # and breaking immediately after the tripping entry makes the
             # two paths bump exactly the same number of entries.
             err_class = type(err).__name__
-            if err_class == last_error_class:
-                consecutive_failures += 1
-            else:
-                consecutive_failures = 1
-                last_error_class = err_class
+            # STALL GUARD ABSTENTION for entries past their attempt budget —
+            # see ``_over_budget`` and the matching branch in the sequential
+            # drain. Evaluated here, before ``_record_failure``, which is both
+            # where the sequential drain evaluates it and necessary anyway
+            # because ``update_attempt`` mutates ``attempt_count`` in place.
+            abstains = _over_budget(entry)
+            if not abstains:
+                if err_class == last_error_class:
+                    consecutive_failures += 1
+                else:
+                    consecutive_failures = 1
+                    last_error_class = err_class
 
             _record_failure(config, path, entry, err, summary)
 
-            if consecutive_failures >= STALL_THRESHOLD:
+            if not abstains and consecutive_failures >= STALL_THRESHOLD:
                 summary["stalled"] = True
                 _blog(
                     f"{consecutive_failures} consecutive failures with "
@@ -767,7 +1031,7 @@ def _parse_args(argv: list[str] | None):
     ap.add_argument(
         "--backlog",
         action="store_true",
-        help="two-phase backlog replay, off the SessionStart budget",
+        help="three-phase backlog replay, off the SessionStart budget",
     )
     ap.add_argument(
         "--phase",
@@ -799,6 +1063,9 @@ def main(argv: list[str] | None = None) -> int:
     if any(
         summary[k]
         for k in (
+            "collapsed",
+            "dead_relocated",
+            "resplit",
             "drained",
             "retried",
             "dead",
@@ -810,6 +1077,9 @@ def main(argv: list[str] | None = None) -> int:
         print(
             f"[Hindsight] drain_pending{'(backlog)' if args.backlog else ''}: "
             f"drained={summary['drained']} reconciled={summary['reconciled']} "
+            f"collapsed={summary['collapsed']} "
+            f"dead_relocated={summary['dead_relocated']} "
+            f"resplit={summary['resplit']}(+{summary['resplit_parts']} parts) "
             f"retried={summary['retried']} dead={summary['dead']} "
             f"unknown={summary['unknown']} "
             f"archive_failed={summary['archive_failed']} "