switchroom 0.19.19 → 0.19.23

package/vendor/hindsight-memory/scripts/lib/retain_split.py

@@ -68,7 +68,8 @@ from typing import Optional
 # The bound — derived, not chosen
 # ---------------------------------------------------------------------------
 #
-#   max_content_chars = retain_chunk_size × floor(client_deadline / chunk_latency)
+#   max_content_chars = retain_chunk_size
+#                       × floor(client_deadline × deadline_safety / chunk_latency)
 #
 # Each input is a measured or read property of the deployment, not a taste
 # call, and each is env-overridable so the bound tracks the deployment:
@@ -83,7 +84,7 @@ from typing import Optional
 #                               completion-token bucket; the runaway bucket is
 #                               a separate defect, fixed by capping
 #                               HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS).
-#   client_deadline     280 s  — the deadline of the DURABILITY path (the
+#   client_deadline     310 s  — the deadline of the DURABILITY path (the
 #                               out-of-hook backlog drain, #3599), deliberately
 #                               not the live Stop hook's 15s. The live path is
 #                               allowed to miss its deadline: it enqueues to
@@ -96,20 +97,60 @@ from typing import Optional
 #                               literal, and `src/setup/hindsight.ts`
 #                               (`HINDSIGHT_RETAIN_CLIENT_DEADLINE_S`, #3611)
 #                               mirrors it as the client half of that PR's
-#                               `server per-call timeout < client deadline`
-#                               assertion — which its derived 204s server
-#                               timeout satisfies against 280 and would NOT
-#                               against #3599's original 180s literal.
+#                               `hindsight per-call timeout < client deadline`
+#                               assertion. A test in `tests/setup/hindsight.test.ts`
+#                               enforces that the two stay equal, so the mirror
+#                               is checked, not merely asserted in prose.
+#                               WAS 280.0. Raised to 310 when the retain
+#                               per-call timeout became derived from the litellm
+#                               routing chain (`local 200 + fallback 90 +
+#                               margin 10 = 300`) instead of from the token
+#                               budget alone: #3611's 204s could not cover that
+#                               chain, so the retain OpenRouter fallback hop had
+#                               4s of headroom and could never complete. 310 is
+#                               that 300 plus the same one margin, so the plugin
+#                               always outlives hindsight by construction. It is
+#                               NOT a hand-set number on either side — change
+#                               `src/litellm/timeout-budget.ts` and both move.
 #
-#   floor(280 / 18.4) = 15 chunks  →  15 × 3000 = 45,000 chars
+#   deadline_safety    0.7    — the FRACTION of the deadline a maximally-sized
+#                               part is allowed to consume. See below; this
+#                               input did not exist until #3693 and was
+#                               effectively 1.0.
 #
-# Sanity check against the same backlog: every entry at or below 60,000 chars
-# drained successfully inside the 280s deadline (observed per-entry times
-# 0.3s–153.4s at concurrency 3), so 45,000 sits inside demonstrated-good
-# territory with margin for a slower model or a busier box.
+#   floor(310 × 0.7 / 18.4) = 11 chunks  →  11 × 3000 = 33,000 chars
+#
+# WHY THERE IS A SAFETY FRACTION AT ALL (#3693). Without it the bound was
+# `floor(deadline / latency)`, which sizes a maximally-sized part to consume
+# ~100% of the deadline BY CONSTRUCTION: 16 × 18.4 = 294.4s against a 310s
+# deadline is 15.6s — 5% — of headroom for the POST/response, server queueing,
+# and any chunk slower than the n=752 MEAN the latency input is. Half the
+# extraction calls are slower than the mean; a part sized to the mean therefore
+# misses the deadline roughly half the time. `drain_pending._backlog_timeout()`
+# defaults to the SAME deadline, so the drainer inherits the same zero margin,
+# times the entry out, bumps its attempt count, and after MAX_ATTEMPTS ages it
+# to `.dead`. That is not a theory: on this fleet every `.dead` marker measured
+# on 2026-07-26 held content of 16,076–44,568 chars — every one of them UNDER
+# the then-current 45,000 bound. The bound was manufacturing dead memories.
+#
+# 0.7 is not a taste call either: it is the utilisation at which the observed
+# per-chunk latency distribution fits, and it independently matches the 30,000
+# char limit the host-side stopgap converged on empirically before this landed.
+# At 0.7 a maximally-sized part is 11 × 18.4 = 202.4s against 310s, leaving
+# 107.6s — enough for a chunk distribution ~50% worse than its own mean.
+#
+# The trade is more parts per logical memory (33,000 rather than 48,000 chars
+# each). That costs nothing in total LLM work — the chunk count across the
+# whole memory is unchanged, only its grouping — and each part now finishes.
+# A part that finishes is worth strictly more than a larger part that does not.
 DEFAULT_RETAIN_CHUNK_SIZE = 3000
 DEFAULT_RETAIN_CHUNK_LATENCY_S = 18.4
-DEFAULT_RETAIN_CLIENT_DEADLINE_S = 280.0
+DEFAULT_RETAIN_CLIENT_DEADLINE_S = 310.0
+
+# Fraction of the client deadline a maximally-sized part may consume.
+# Clamped to (0, 1]: a value above 1.0 would size parts to overrun the
+# deadline outright, which is the defect this input exists to prevent.
+DEFAULT_RETAIN_DEADLINE_SAFETY = 0.7
 
 # Absolute floor: one chunk. A bound below one chunk would split every
 # transcript into extraction-sized confetti and is never the right answer.
@@ -140,6 +181,18 @@ def retain_client_deadline() -> float:
     return _env_number("HINDSIGHT_RETAIN_CLIENT_DEADLINE_S", DEFAULT_RETAIN_CLIENT_DEADLINE_S)
 
 
+def retain_deadline_safety() -> float:
+    """Fraction of the client deadline a maximally-sized part may consume.
+
+    Clamped to ``(0, 1]``. A value above 1.0 is not honoured: it would size
+    parts to overrun the deadline by construction, which is exactly the defect
+    (#3693) this input exists to prevent, so it is treated as "no margin at
+    all" — 1.0 — rather than as a licence to go further.
+    """
+    value = _env_number("HINDSIGHT_RETAIN_DEADLINE_SAFETY", DEFAULT_RETAIN_DEADLINE_SAFETY)
+    return min(1.0, value)
+
+
 def retain_content_limit() -> int:
     """Max chars of retain content that can complete inside the client deadline.
 
@@ -159,7 +212,12 @@ def retain_content_limit() -> int:
     latency = _env_number("HINDSIGHT_RETAIN_CHUNK_LATENCY_S", DEFAULT_RETAIN_CHUNK_LATENCY_S)
     deadline = retain_client_deadline()
 
-    chunks = int(deadline // latency)
+    # A FRACTION of the deadline, not all of it (#3693). Sizing to the whole
+    # deadline leaves a maximally-sized part no headroom for the POST itself,
+    # server queueing, or a chunk slower than the MEAN `latency` is measured
+    # as — so the part times out, the drain bumps its attempt count, and it
+    # ages to `.dead`. See the derivation block at the top of this module.
+    chunks = int((deadline * retain_deadline_safety()) // latency)
     if chunks < 1:
         chunks = 1
     return max(MIN_RETAIN_CONTENT_CHARS, chunk_size * chunks)

package/vendor/hindsight-memory/scripts/recall.py

@@ -1296,6 +1296,56 @@ def _combine_context(base, nudge) -> str:
     return "\n\n".join(parts)
 
 
+def degraded_recall_notice(bank_id, bank_timings) -> str:
+    """Switchroom #3619 — return the degraded-recall disclosure for this turn,
+    or "" when the agent's own bank answered.
+
+    Until now a recall whose own bank timed out was indistinguishable, from the
+    agent's side, from a bank that genuinely held nothing relevant: both
+    produced an empty block and silence. That ambiguity is what let a measured
+    ~90% own-bank timeout rate run for weeks unnoticed while every agent's
+    CLAUDE.md asserted recall "auto-fires on every inbound message" — the agent
+    had no way to know it was answering from an empty context, so it never said
+    so and the operator never saw it.
+
+    Only the agent's OWN bank warrants the notice: additional banks (a shared
+    profile bank, say) are supplementary, and a side-bank timeout does not mean
+    the agent lost its own memory. Matching is by `bank_id`, never by position
+    in `bank_timings` — the fan-out order is not stable.
+
+    Kept to a single short line on purpose: this fires on an already-degraded
+    turn, and a verbose block would spend the very budget the degradation is
+    starving. The caller must keep it OUT of the cached context (see
+    `_combine_context`) — it is per-turn state and would otherwise replay on a
+    later healthy cache hit.
+    """
+    if not bank_id or not bank_timings:
+        return ""
+    own = next(
+        (
+            bt
+            for bt in bank_timings
+            if isinstance(bt, dict) and bt.get("bank_id") == bank_id
+        ),
+        None,
+    )
+    if not own:
+        return ""
+    if own.get("timed_out"):
+        reason = "timed out"
+    elif own.get("errored"):
+        reason = "was unreachable"
+    else:
+        return ""
+    return (
+        f"[Hindsight] Memory recall was DEGRADED this turn: your own bank "
+        f"('{bank_id}') {reason}, so the memories below (if any) are "
+        f"incomplete and may be missing entirely. Treat an absence of "
+        f"relevant memory as UNKNOWN, not as 'nothing was remembered' — "
+        f"say so rather than asserting there is no prior context."
+    )
+
+
 def main():
     config = load_config()
 
@@ -2129,13 +2179,22 @@ def main():
         "transcript_fallback_truncated": transcript_fallback_telemetry["truncated"],
     })
 
-    # If neither block has content, there's nothing to inject — exit
+    # Switchroom #3619 — DEGRADED-RECALL DISCLOSURE. See
+    # `degraded_recall_notice` for why this exists and why only the agent's
+    # OWN bank counts.
+    degraded_block = degraded_recall_notice(bank_id, bank_timings)
+
+    # If no block has content, there's nothing to inject — exit
     # silently to avoid emitting an empty hookSpecificOutput. #2848: unless
     # the directive-capture nudge fired, in which case emit the nudge alone
     # (a correction with no memories/directives still needs the reminder).
+    # #3619: a degraded own-bank read is likewise worth emitting alone — that
+    # is precisely the turn on which the agent must not assume it remembers.
     if not directives_block and not memories_block and not transcript_fallback_block:
-        if nudge_block:
-            _emit_cached_context(nudge_block)
+        if degraded_block or nudge_block:
+            _emit_cached_context(
+                "\n\n".join([b for b in (degraded_block, nudge_block) if b])
+            )
         return
 
     # Compose final context. Directives block goes ABOVE memories so the
@@ -2143,6 +2202,12 @@ def main():
     # transcript fallback (#3369) goes LAST — it is the lowest-confidence
     # signal (raw transcript, not synthesized fact) and only present when
     # memories_block is empty by construction.
+    #
+    # #3619's degraded notice is deliberately NOT part of context_message: like
+    # the #2848 nudge it is per-turn state, and this string is what gets cached
+    # and written to LAST_RECALL_STATE. Caching it would replay "recall was
+    # DEGRADED" on later healthy cache hits; it is prepended at emit time
+    # instead, so a cache hit re-derives the turn's real condition.
     parts = []
     if directives_block:
         parts.append(directives_block)
@@ -2177,11 +2242,15 @@ def main():
 
     # Output JSON for Claude Code hook system. #2848: append the
     # directive-capture nudge (if it fired) at emit time — it's kept out of
-    # the cached / last-recall context above so it can't go stale.
+    # the cached / last-recall context above so it can't go stale. #3619: the
+    # degraded-recall notice is prepended for the same reason, and goes FIRST
+    # because it changes how everything after it should be read.
     output = {
         "hookSpecificOutput": {
             "hookEventName": "UserPromptSubmit",
-            "additionalContext": _combine_context(context_message, nudge_block),
+            "additionalContext": _combine_context(
+                _combine_context(degraded_block, context_message), nudge_block
+            ),
         }
     }
     json.dump(output, sys.stdout)