switchroom 0.19.19 → 0.19.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/dist/auth-broker/index.js +53 -0
  2. package/dist/cli/switchroom.js +2444 -1264
  3. package/dist/host-control/main.js +54 -1
  4. package/dist/vault/approvals/kernel-server.js +53 -0
  5. package/dist/vault/broker/server.js +53 -0
  6. package/package.json +4 -2
  7. package/skills/switchroom-release/SKILL.md +103 -20
  8. package/telegram-plugin/card-format.ts +92 -3
  9. package/telegram-plugin/dist/gateway/gateway.js +769 -172
  10. package/telegram-plugin/edit-flood-fuse.ts +477 -0
  11. package/telegram-plugin/format.ts +19 -7
  12. package/telegram-plugin/gateway/boot-sweep-gate.ts +164 -0
  13. package/telegram-plugin/gateway/callback-query-handlers.ts +454 -81
  14. package/telegram-plugin/gateway/gateway.ts +66 -56
  15. package/telegram-plugin/gateway/inbound-interceptors.ts +27 -4
  16. package/telegram-plugin/gateway/narrative-lane.ts +49 -3
  17. package/telegram-plugin/gateway/status-pin-api.ts +145 -0
  18. package/telegram-plugin/hooks/subagent-tracker-posttool.mjs +325 -45
  19. package/telegram-plugin/retry-api-call.ts +15 -2
  20. package/telegram-plugin/send-gate.ts +1 -1
  21. package/telegram-plugin/status-no-truncate.ts +64 -1
  22. package/telegram-plugin/status-pin-driver.ts +50 -27
  23. package/telegram-plugin/status-pin.ts +43 -5
  24. package/telegram-plugin/tests/activity-card-send-gate.test.ts +275 -0
  25. package/telegram-plugin/tests/activity-card-wiring.test.ts +16 -7
  26. package/telegram-plugin/tests/boot-pin-sweep-wiring.test.ts +101 -0
  27. package/telegram-plugin/tests/boot-sweep-gate.test.ts +293 -0
  28. package/telegram-plugin/tests/boot-version-string.test.ts +0 -0
  29. package/telegram-plugin/tests/edit-flood-fuse.test.ts +431 -0
  30. package/telegram-plugin/tests/pinned-card-collapse.test.ts +356 -0
  31. package/telegram-plugin/tests/status-pin-api.test.ts +178 -0
  32. package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +94 -11
  33. package/telegram-plugin/tests/status-pin.test.ts +106 -5
  34. package/telegram-plugin/tests/subagent-tracker-hooks.test.ts +631 -1
  35. package/telegram-plugin/tests/tool-activity-summary.test.ts +19 -10
  36. package/telegram-plugin/tests/vault-approval-posture.test.ts +6 -1
  37. package/telegram-plugin/tests/vault-passphrase-retry.test.ts +666 -0
  38. package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +42 -21
  39. package/telegram-plugin/tests/worker-feed-coalesce.test.ts +233 -1
  40. package/telegram-plugin/tool-activity-summary.ts +85 -13
  41. package/telegram-plugin/worker-activity-feed.ts +5 -1
  42. package/vendor/hindsight-memory/scripts/drain_pending.py +193 -25
  43. package/vendor/hindsight-memory/scripts/lib/pending.py +84 -5
  44. package/vendor/hindsight-memory/scripts/lib/retain_split.py +21 -10
  45. package/vendor/hindsight-memory/scripts/recall.py +74 -5
  46. package/vendor/hindsight-memory/scripts/tests/test_pending_drops.py +158 -4
  47. package/vendor/hindsight-memory/scripts/tests/test_pending_failure_class.py +105 -0
  48. package/vendor/hindsight-memory/scripts/tests/test_pending_wedge.py +300 -0
  49. package/vendor/hindsight-memory/scripts/tests/test_recall_degraded_notice.py +365 -0
  50. package/vendor/hindsight-memory/scripts/tests/test_recall_envelope_strip_telemetry.py +12 -4
  51. package/vendor/hindsight-memory/scripts/tests/test_recall_transcript_fallback.py +27 -2
  52. package/vendor/hindsight-memory/scripts/tests/test_retain_split.py +19 -11
  53. package/vendor/hindsight-memory/tests/test_drain_pending.py +28 -2
@@ -3,9 +3,21 @@
3
3
 
4
4
  SessionStart calls into ``drain()`` to retry any retain payloads that
5
5
  ``session_end.py`` queued on failure (#1071). Each entry is retried up
6
- to ``MAX_ATTEMPTS`` (5) times; after that it's renamed to ``.dead`` so
7
- the queue no longer drains it but the operator can still inspect via
8
- ``switchroom doctor``.
6
+ to ``MAX_ATTEMPTS`` (5) times; after that a **permanently** failing entry
7
+ (a 4xx that a re-POST cannot fix — see ``pending.is_permanent_failure``)
8
+ is renamed to ``.dead`` so the queue no longer drains it but the operator
9
+ can still inspect via ``switchroom doctor``. An entry failing on anything
10
+ else — a 5xx, a timeout, a connection error — stays queued past the
11
+ attempt budget: a transient upstream is never evidence that the memory
12
+ is unsaveable, and retiring it would lose content the user believes was
13
+ saved.
14
+
15
+ An entry past the budget is DEMOTED rather than retired (``_drain_order``
16
+ and ``_over_budget``): it sorts behind everything still inside its budget
17
+ and abstains from the stall guard. That is what keeps "never destroy a
18
+ memory" from degrading into "never drain anything" — the drain is
19
+ sequential and oldest-first, so without the demotion three chronically
20
+ failing entries sit at the head and end every run at zero progress.
9
21
 
10
22
  Boundaries
11
23
  ----------
@@ -36,8 +48,7 @@ accumulated backlog. Worse, they CREATE one: the per-entry timeout is
36
48
  clamped to the remaining hook budget (1-8s) while ``_retry_one`` posts
37
49
  synchronously and a real retain takes 30-90s, so **the server commits the
38
50
  document and the client always gives up before the ack**. The entry is
39
- never deleted and is re-posted on every session start, forever, until it
40
- hits ``MAX_ATTEMPTS`` and goes ``.dead``. The queue depth was a symptom
51
+ never deleted and is re-posted on every session start. The queue depth was a symptom
41
52
  of that loop, not of lost memory: a full sweep of 5,751 queued entries on
42
53
  this fleet (2026-07-25) found **4,048 (70.4%) already existed as
43
54
  documents**, 3,815 of them with facts extracted.
@@ -95,6 +106,7 @@ from lib.pending import (
95
106
  MAX_ATTEMPTS,
96
107
  archive_reconciled,
97
108
  is_content_derived_document_id,
109
+ is_permanent_failure,
98
110
  iter_entries,
99
111
  mark_dead,
100
112
  update_attempt,
@@ -203,16 +215,30 @@ def _backlog_timeout() -> int:
203
215
  client-side timeout on a request the server then commits anyway.
204
216
 
205
217
  The default is DERIVED, not a literal: it is
206
- ``retain_split.retain_client_deadline()`` (280s), the same deadline the
218
+ ``retain_split.retain_client_deadline()`` (310s), the same deadline the
207
219
  retain content bound is sized against. Those two must be ONE number.
208
220
  This function shipped as a bare ``180`` (#3599), and against a 180s
209
221
  deadline both halves of the retain budget break: a maximally-sized part
210
222
  is ~276s of sequential extraction, and the SERVER per-call timeout
211
- derived in ``src/setup/hindsight.ts`` (#3611) is 204s — so the drain
223
+ derived in ``src/setup/hindsight.ts`` is larger still — so the drain
212
224
  client would abandon a request the server is still legitimately working
213
225
  on, leave the entry queued, and rebuild the re-post loop #3599 exists to
214
226
  kill, one size class up.
215
227
 
228
+ This is the OUTERMOST deadline of the paired-budget family, and it is the
229
+ one the 2026-07-26 backlog-recovery logs show as a fixed ~280.1s per-entry
230
+ give-up. That was not an unexplained third number: it is this default at
231
+ the then-current ``DEFAULT_RETAIN_CLIENT_DEADLINE_S = 280.0``, plus the
232
+ request's own setup overhead. It was WRONG for the same reason #3611's
233
+ 204s was wrong — it did not cover hindsight's LiteLLM routing chain
234
+ (local 200s + OpenRouter fallback 90s + router margin 10s = 300s), so
235
+ every drained entry whose retain fell through to the fallback was
236
+ abandoned client-side at 280s while the server was still inside a
237
+ legitimate 300s budget. Raising the derivation's base to 310 fixes this
238
+ lane and the in-hook lane with the same number, by construction.
239
+ ``src/litellm/timeout-budget.ts`` is where the chain is declared, and
240
+ ``tests/setup/hindsight.test.ts`` fails if these drift apart again.
241
+
216
242
  ``HINDSIGHT_DRAIN_BACKLOG_TIMEOUT`` still overrides it outright;
217
243
  ``HINDSIGHT_RETAIN_CLIENT_DEADLINE_S`` moves the derivation and the
218
244
  content bound together.
@@ -348,6 +374,76 @@ def _document_state(entry: dict, timeout: int = 30):
348
374
  return None
349
375
 
350
376
 
377
+ def _drain_order(entries: list[tuple[str, dict]]) -> list[tuple[str, dict]]:
378
+ """Oldest-first, but with budget-exhausted entries demoted to the back.
379
+
380
+ THE HEAD-OF-LINE BUG THIS EXISTS FOR. Since the permanence gate in
381
+ ``_record_failure``, an entry failing on anything transient stays queued
382
+ past ``MAX_ATTEMPTS`` indefinitely — deliberately, because a transient
383
+ upstream is not evidence the memory is unsaveable. But ``iter_entries()``
384
+ is oldest-first, and the entries that have been failing longest are by
385
+ construction the OLDEST, so they sit at the head of every drain. Backlog
386
+ concurrency defaults to 1 (``_backlog_concurrency``), so the drain is
387
+ sequential: three such entries in a row trip ``STALL_THRESHOLD`` and the
388
+ run breaks having drained nothing — and, because they never retire, it
389
+ breaks identically on every subsequent run. Measured on this branch before
390
+ this function existed: 6 queued entries, the 3 oldest raising
391
+ ``TimeoutError``, 4 consecutive ``drain_backlog`` runs each returning
392
+ ``stalled=True, drained=0`` with the queue depth still 6. The 3 healthy
393
+ entries behind them were never even attempted. On ``main`` the same repro
394
+ converges: run 0 retires the 3 heads to ``.dead`` and run 1 onward drains
395
+ normally. So the permanence gate, alone, traded "rarely destroys a memory"
396
+ for "eventually drains nothing at all".
397
+
398
+ ``.dead`` was doing double duty: it was the honesty policy AND it was the
399
+ queue's only un-wedging mechanism. Removing it as a policy has to leave
400
+ the un-wedging behind, and demotion is that — it keeps every property the
401
+ gate was added for (the entry is still queued, still retried, never
402
+ destroyed) while removing the one it broke (it can no longer starve a
403
+ healthy entry behind it).
404
+
405
+ The terminal condition for a permanently-unsaveable entry is therefore no
406
+ longer deletion but DEMOTION: ``attempt_count`` only ever climbs, so such
407
+ an entry crosses the budget once and stays in the back group for good,
408
+ where it can delay only itself. It is still reconciled for free on every
409
+ run — ``_reconcile_phase`` sweeps ALL entries with a sub-second presence
410
+ GET, in no particular order and with no stall guard — so an entry whose
411
+ document did land is still retired without a POST.
412
+
413
+ Ordering is a stable partition, so relative age is preserved inside each
414
+ group and FIFO still holds for everything that has not blown its budget.
415
+ """
416
+ fresh: list[tuple[str, dict]] = []
417
+ exhausted: list[tuple[str, dict]] = []
418
+ for path, entry in entries:
419
+ (exhausted if _over_budget(entry) else fresh).append((path, entry))
420
+ return fresh + exhausted
421
+
422
+
423
+ def _over_budget(entry: dict) -> bool:
424
+ """Has this entry already burned its ``MAX_ATTEMPTS`` budget?
425
+
426
+ Such an entry is chronically failing but, since the permanence gate, is
427
+ never retired. It gets two demotions — last in the drain order
428
+ (``_drain_order``) and no vote in the stall guard (see below) — because
429
+ ordering alone does not close the wedge. Ordering fixes the common shape
430
+ (a few old poison entries in front of healthy ones), but not the shape
431
+ where the WHOLE queue is over budget: an upstream down for a week takes
432
+ every entry past 5 attempts, and when it recovers the partition is empty
433
+ on one side, the poisoned entries are at the head again, and the run
434
+ stalls before reaching the entries that would now succeed. Measured: with
435
+ ordering alone and all 6 entries at ``MAX_ATTEMPTS``, 4 consecutive runs
436
+ still returned ``stalled=True, drained=0``.
437
+ """
438
+ try:
439
+ return int(entry.get("attempt_count", 0)) >= MAX_ATTEMPTS
440
+ except (TypeError, ValueError):
441
+ # A hand-edited or corrupt counter must not decide ordering, and must
442
+ # not raise on the drain path. Treat it as fresh: the cost of guessing
443
+ # wrong here is one retry in the normal position, not a lost memory.
444
+ return False
445
+
446
+
351
447
  def _record_failure(
352
448
  config: dict,
353
449
  path: str,
@@ -358,16 +454,53 @@ def _record_failure(
358
454
  """Apply the per-entry failure policy. Returns the error class name.
359
455
 
360
456
  Shared by the sequential (SessionStart) and backlog drains so both age
361
- entries toward ``.dead`` on exactly the same schedule.
457
+ entries toward ``.dead`` on exactly the same schedule — and, since the
458
+ permanence gate below, refuse to retire them on exactly the same rule.
362
459
  """
363
460
  err_class = type(e).__name__
364
461
  attempts = int(entry.get("attempt_count", 1))
365
- if attempts >= MAX_ATTEMPTS:
462
+ # ``.dead`` retires a memory the user believes was saved, so it is gated on
463
+ # the failure being PERMANENT — a 4xx that re-POSTing cannot fix. A
464
+ # transient failure keeps its attempt counter climbing but stays queued,
465
+ # because an exhausted attempt budget is not evidence that the content is
466
+ # unpersistable.
467
+ #
468
+ # Before this gate, ANY five failures retired the entry. The dominant
469
+ # failure on this fleet is an HTTP 500 "Fact extraction failed … chunk 0:
470
+ # JSONDecodeError" — the extraction model returned an empty or non-JSON
471
+ # completion for one chunk on that sampling run. The identical content
472
+ # succeeds on a later attempt, so five unlucky samples were destroying
473
+ # memories that were never unsaveable. See ``pending.is_permanent_failure``.
474
+ #
475
+ # WHAT BOUNDS THIS, precisely — because an unbounded queue of undying
476
+ # entries would be a worse outcome than the bug this gate fixes.
477
+ #
478
+ # DISK is bounded by the queue's MAX_ENTRIES / MAX_BYTES caps, which shed
479
+ # the oldest entries into ``pending-evicted/`` (an archive, not a delete).
480
+ # Note what that bound is NOT: ``_evict_to_fit`` is called only from
481
+ # ``enqueue`` (lib/pending.py:933), so it fires on new writes, never on
482
+ # drain — and it "bounds" the queue by shedding memory unsaved, which is
483
+ # the very outcome this gate exists to avoid. It is a backstop, not the
484
+ # answer.
485
+ #
486
+ # PROGRESS is bounded by ``_drain_order`` / ``_over_budget``. An entry that
487
+ # can never be persisted no longer terminates by being destroyed; it
488
+ # terminates by being DEMOTED — sorted behind every entry still inside its
489
+ # budget, and stripped of its vote in the stall guard. ``attempt_count``
490
+ # only ever climbs, so the crossing happens once and is permanent. That is
491
+ # the real terminal condition, and it is what keeps a poisoned entry from
492
+ # starving the queue behind it. Without it, three such entries ended every
493
+ # drain at zero progress, permanently (tests/test_pending_wedge.py).
494
+ #
495
+ # The attempt counter itself was never a bound on either, and ``switchroom
496
+ # doctor`` still surfaces queue depth so the operator sees a growing tail.
497
+ if attempts >= MAX_ATTEMPTS and is_permanent_failure(e):
366
498
  marker = mark_dead(path, entry)
367
499
  summary["dead"] += 1
368
500
  print(
369
501
  f"[Hindsight] drain_pending: entry exceeded {MAX_ATTEMPTS} "
370
- f"attempts, marking dead at {marker} (last error: {err_class}: {e})",
502
+ f"attempts on a permanent failure, marking dead at {marker} "
503
+ f"(last error: {err_class}: {e})",
371
504
  file=sys.stderr,
372
505
  )
373
506
  else:
@@ -375,7 +508,12 @@ def _record_failure(
375
508
  summary["retried"] += 1
376
509
  debug_log(
377
510
  config,
378
- f"drain_pending: retry {attempts}/{MAX_ATTEMPTS} failed for {path} ({err_class}: {e})",
511
+ # ``attempts`` can now exceed MAX_ATTEMPTS — a transient failure
512
+ # keeps the entry queued past the budget instead of retiring it —
513
+ # so print the budget as a threshold, not as a fraction that would
514
+ # render the nonsense "retry 7/5".
515
+ f"drain_pending: retry {attempts} (budget {MAX_ATTEMPTS}) failed "
516
+ f"for {path} ({err_class}: {e})",
379
517
  )
380
518
  return err_class
381
519
 
@@ -440,7 +578,11 @@ def drain(
440
578
 
441
579
  summary = _new_summary()
442
580
 
443
- entries = iter_entries()
581
+ # Budget-exhausted entries go last so they cannot starve the healthy ones
582
+ # behind them — see ``_drain_order``. Matters even more here than in the
583
+ # backlog drain: the in-hook budget is ~4s, so a single entry at the head
584
+ # that always burns its clamped timeout consumes the entire run.
585
+ entries = _drain_order(iter_entries())
444
586
  if not entries:
445
587
  debug_log(config, "drain_pending: queue empty")
446
588
  return summary
@@ -496,14 +638,30 @@ def drain(
496
638
  # allowance must not be handed out twice (#3599 review F2).
497
639
  _retry_one(entry, timeout=_clamp(timeout, budget, started))
498
640
  except Exception as e:
641
+ # Read the budget state BEFORE _record_failure: update_attempt
642
+ # mutates `entry["attempt_count"]` in place (lib/pending.py:1020),
643
+ # so asking afterwards would count the entry that just CROSSED the
644
+ # budget on this very failure as an abstainer, silently weakening
645
+ # the stall guard for ordinary entries.
646
+ abstains = _over_budget(entry)
499
647
  err_class = _record_failure(config, path, entry, e, summary)
500
- if err_class == last_error_class:
501
- consecutive_failures += 1
502
- else:
503
- consecutive_failures = 1
504
- last_error_class = err_class
648
+ # STALL GUARD ABSTENTION — see ``_over_budget``. The guard exists
649
+ # to detect a broken UPSTREAM and stop hammering it. A chronically
650
+ # failing entry that has already burned its attempt budget is
651
+ # evidence about that ENTRY, not about the upstream, and letting it
652
+ # vote is what wedges the queue: it never retires, so it fails
653
+ # identically on every run, and three of them end every run before
654
+ # the healthy entries behind them are ever reached. It abstains —
655
+ # neither incrementing the counter nor resetting it, so a genuine
656
+ # upstream outage is still caught by the fresh entries around it.
657
+ if not abstains:
658
+ if err_class == last_error_class:
659
+ consecutive_failures += 1
660
+ else:
661
+ consecutive_failures = 1
662
+ last_error_class = err_class
505
663
 
506
- if consecutive_failures >= STALL_THRESHOLD:
664
+ if not abstains and consecutive_failures >= STALL_THRESHOLD:
507
665
  summary["stalled"] = True
508
666
  print(
509
667
  f"[Hindsight] drain_pending: {consecutive_failures} consecutive "
@@ -641,7 +799,10 @@ def _drain_backlog_impl(
641
799
  backoff_ms = _p95_backoff_ms()
642
800
  started = time.monotonic()
643
801
 
644
- entries = iter_entries()
802
+ # See ``_drain_order``: without this, three budget-exhausted entries at the
803
+ # head trip the stall guard on every run forever and the drain never makes
804
+ # progress again.
805
+ entries = _drain_order(iter_entries())
645
806
  if not entries:
646
807
  _blog("phase 2: nothing left to retain")
647
808
  return summary
@@ -728,15 +889,22 @@ def _drain_backlog_impl(
728
889
  # and breaking immediately after the tripping entry makes the
729
890
  # two paths bump exactly the same number of entries.
730
891
  err_class = type(err).__name__
731
- if err_class == last_error_class:
732
- consecutive_failures += 1
733
- else:
734
- consecutive_failures = 1
735
- last_error_class = err_class
892
+ # STALL GUARD ABSTENTION for entries past their attempt budget —
893
+ # see ``_over_budget`` and the matching branch in the sequential
894
+ # drain. Evaluated here, before ``_record_failure``, which is both
895
+ # where the sequential drain evaluates it and necessary anyway
896
+ # because ``update_attempt`` mutates ``attempt_count`` in place.
897
+ abstains = _over_budget(entry)
898
+ if not abstains:
899
+ if err_class == last_error_class:
900
+ consecutive_failures += 1
901
+ else:
902
+ consecutive_failures = 1
903
+ last_error_class = err_class
736
904
 
737
905
  _record_failure(config, path, entry, err, summary)
738
906
 
739
- if consecutive_failures >= STALL_THRESHOLD:
907
+ if not abstains and consecutive_failures >= STALL_THRESHOLD:
740
908
  summary["stalled"] = True
741
909
  _blog(
742
910
  f"{consecutive_failures} consecutive failures with "
@@ -10,7 +10,10 @@ entry into the bounded ``pending-reconciled/`` archive (the drain never
10
10
  deletes — see "Retiring an entry" below for that promise and its one
11
11
  bound, a full disk), failure bumps an attempt
12
12
  counter (up to MAX_ATTEMPTS) and leaves the entry for the run after
13
- that.
13
+ that. Exhausting MAX_ATTEMPTS retires the entry to ``.dead`` only when
14
+ the failure is PERMANENT (``is_permanent_failure`` — a 4xx a re-POST
15
+ cannot fix). A transient failure never retires the memory, however many
16
+ attempts it has burned.
14
17
 
15
18
  Layout
16
19
  ------
@@ -144,11 +147,19 @@ oversized payload into one entry PER PART (``lib/retain_split``) instead
144
147
  of writing a single giant entry. This is load-bearing, not tidiness. The
145
148
  daemon runs one sequential extraction LLM call per ``retain_chunk_size``
146
149
  chars, so an entry above the derived content bound cannot complete inside
147
- ANY client deadline — including the 280s out-of-hook backlog deadline that
150
+ ANY client deadline — including the out-of-hook backlog deadline that
148
151
  ``drain_pending._backlog_timeout()`` now takes from the same derivation.
149
- Such an entry fails every drain, burns its ``MAX_ATTEMPTS``, and is
150
- renamed ``.dead``: the mechanism that stranded 154 of the 629 entries in
151
- the 2026-07-25 fleet backlog. Note this is orthogonal to the re-post loop
152
+ Such an entry fails every drain and burns its ``MAX_ATTEMPTS``: the
153
+ mechanism that stranded 154 of the 629 entries in the 2026-07-25 fleet
154
+ backlog. (It is no longer renamed ``.dead`` for that — a client-side
155
+ timeout is not a permanent failure — but it still never drains until it
156
+ is split, so splitting remains the fix. Such an entry is therefore
157
+ IMMORTAL, and ``enqueue`` is the only caller of the splitter, so one
158
+ queued before #3610 is never split in place. That is precisely the shape
159
+ ``drain_pending._drain_order`` / ``_over_budget`` exist to contain: it is
160
+ demoted behind every entry still inside its attempt budget and abstains
161
+ from the stall guard, so it can delay only itself rather than wedging the
162
+ drain.) Note this is orthogonal to the re-post loop
152
163
  #3599 fixed — a presence GET retires an oversized entry for free when the
153
164
  document IS already durable; splitting is what makes the entry drainable
154
165
  when it is NOT. Part document_ids are deterministic, so a part already
@@ -1027,6 +1038,74 @@ def update_attempt(path: str, entry: dict, error: BaseException) -> bool:
1027
1038
  return False
1028
1039
 
1029
1040
 
1041
+ #: HTTP statuses that are 4xx but describe a TRANSIENT condition, so they
1042
+ #: must be read as retryable despite the 4xx class.
1043
+ _RETRYABLE_4XX = frozenset({408, 425, 429})
1044
+
1045
+
1046
+ def is_permanent_failure(error: BaseException) -> bool:
1047
+ """True when ``error`` can never succeed on a later identical retry.
1048
+
1049
+ This is the gate on ``mark_dead`` (see ``drain_pending._record_failure``).
1050
+ Getting it wrong in the permissive direction costs a re-POST — which is an
1051
+ upsert, so it costs time. Getting it wrong in the strict direction costs
1052
+ the USER'S MEMORY. So the rule is deliberately asymmetric: an error is
1053
+ permanent only when we can positively identify it as a client-side defect
1054
+ in the request itself. Everything we cannot classify is retryable.
1055
+
1056
+ PERMANENT — a 4xx other than 408/425/429. The server understood us and
1057
+ rejected the request: a malformed payload, an unknown bank, an oversized
1058
+ body, a bad token. Re-POSTing the identical bytes reproduces it exactly,
1059
+ so attempts are pure waste and ``.dead`` is the honest outcome.
1060
+
1061
+ RETRYABLE — everything else. Notably 5xx, which is what a failed
1062
+ fact-extraction surfaces as::
1063
+
1064
+ HTTP 500 ...: {"detail": "Fact extraction failed: 1/1 chunks failed.
1065
+ First failures: chunk 0: JSONDecodeError: Expecting value: line 1
1066
+ column 1 (char 0)"}
1067
+
1068
+ That 500 means the extraction model returned an empty or non-JSON
1069
+ completion for one chunk (measured 2026-07-26: Ollama returning
1070
+ ``content: ""`` with all-zero usage, and gpt-oss-20b emitting a numbered
1071
+ prose list instead of the JSON schema). It is a property of one sampling
1072
+ run, NOT of the queued content — the very same entry succeeds on a later
1073
+ attempt. Counting it toward ``MAX_ATTEMPTS`` is what turned a flaky model
1074
+ into permanently lost memories: five unlucky samples and a real memory
1075
+ went ``.dead``.
1076
+
1077
+ Timeouts, connection resets, DNS failures and anything unrecognised are
1078
+ retryable for the same reason — none of them is evidence that the content
1079
+ can never be persisted.
1080
+
1081
+ The fleet bears this out. A census of every ``.dead`` marker on this host
1082
+ (2026-07-26, 129 markers across 10 agents) found the retiring error was
1083
+ ``TimeoutError`` 128 times and ``URLError`` once. **Not one was a 4xx.**
1084
+ Every permanently-lost memory here was lost to a transient failure, so
1085
+ this gate would have kept all 129 queued and drainable. Two were retired
1086
+ at 00:56Z that same morning — this was live, not historical.
1087
+
1088
+ ``client.HindsightClient._request`` re-raises ``urllib`` HTTP failures as
1089
+ ``RuntimeError(f"HTTP {code} from {url}: {body}")`` with the original
1090
+ ``HTTPError`` chained on ``__cause__``, so the status is read from the
1091
+ cause when present and parsed out of the message otherwise (the message
1092
+ form is what a de-chained/re-serialised error leaves behind).
1093
+ """
1094
+ code = getattr(error, "code", None)
1095
+ cause = getattr(error, "__cause__", None)
1096
+ if not isinstance(code, int) and cause is not None:
1097
+ code = getattr(cause, "code", None)
1098
+ if not isinstance(code, int):
1099
+ text = str(error)
1100
+ if text.startswith("HTTP "):
1101
+ head = text[5:].split(" ", 1)[0]
1102
+ if head.isdigit():
1103
+ code = int(head)
1104
+ if not isinstance(code, int):
1105
+ return False
1106
+ return 400 <= code < 500 and code not in _RETRYABLE_4XX
1107
+
1108
+
1030
1109
  def mark_dead(path: str, entry: dict) -> Optional[str]:
1031
1110
  """Convert an entry that exceeded ``MAX_ATTEMPTS`` into a permanent
1032
1111
  failure marker at ``<path>.dead`` so the queue no longer drains it
@@ -83,7 +83,7 @@ from typing import Optional
83
83
  # completion-token bucket; the runaway bucket is
84
84
  # a separate defect, fixed by capping
85
85
  # HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS).
86
- # client_deadline 280 s — the deadline of the DURABILITY path (the
86
+ # client_deadline 310 s — the deadline of the DURABILITY path (the
87
87
  # out-of-hook backlog drain, #3599), deliberately
88
88
  # not the live Stop hook's 15s. The live path is
89
89
  # allowed to miss its deadline: it enqueues to
@@ -96,20 +96,31 @@ from typing import Optional
96
96
  # literal, and `src/setup/hindsight.ts`
97
97
  # (`HINDSIGHT_RETAIN_CLIENT_DEADLINE_S`, #3611)
98
98
  # mirrors it as the client half of that PR's
99
- # `server per-call timeout < client deadline`
100
- # assertion — which its derived 204s server
101
- # timeout satisfies against 280 and would NOT
102
- # against #3599's original 180s literal.
99
+ # `hindsight per-call timeout < client deadline`
100
+ # assertion. A test in `tests/setup/hindsight.test.ts`
101
+ # enforces that the two stay equal, so the mirror
102
+ # is checked, not merely asserted in prose.
103
+ # WAS 280.0. Raised to 310 when the retain
104
+ # per-call timeout became derived from the litellm
105
+ # routing chain (`local 200 + fallback 90 +
106
+ # margin 10 = 300`) instead of from the token
107
+ # budget alone: #3611's 204s could not cover that
108
+ # chain, so the retain OpenRouter fallback hop had
109
+ # 4s of headroom and could never complete. 310 is
110
+ # that 300 plus the same one margin, so the plugin
111
+ # always outlives hindsight by construction. It is
112
+ # NOT a hand-set number on either side — change
113
+ # `src/litellm/timeout-budget.ts` and both move.
103
114
  #
104
- # floor(280 / 18.4) = 15 chunks → 15 × 3000 = 45,000 chars
115
+ # floor(310 / 18.4) = 16 chunks → 16 × 3000 = 48,000 chars
105
116
  #
106
117
  # Sanity check against the same backlog: every entry at or below 60,000 chars
107
- # drained successfully inside the 280s deadline (observed per-entry times
108
- # 0.3s–153.4s at concurrency 3), so 45,000 sits inside demonstrated-good
109
- # territory with margin for a slower model or a busier box.
118
+ # drained successfully inside the (then 280s) deadline (observed per-entry
119
+ # times 0.3s–153.4s at concurrency 3), so 48,000 still sits inside
120
+ # demonstrated-good territory with margin for a slower model or a busier box.
110
121
  DEFAULT_RETAIN_CHUNK_SIZE = 3000
111
122
  DEFAULT_RETAIN_CHUNK_LATENCY_S = 18.4
112
- DEFAULT_RETAIN_CLIENT_DEADLINE_S = 280.0
123
+ DEFAULT_RETAIN_CLIENT_DEADLINE_S = 310.0
113
124
 
114
125
  # Absolute floor: one chunk. A bound below one chunk would split every
115
126
  # transcript into extraction-sized confetti and is never the right answer.
@@ -1296,6 +1296,56 @@ def _combine_context(base, nudge) -> str:
1296
1296
  return "\n\n".join(parts)
1297
1297
 
1298
1298
 
1299
+ def degraded_recall_notice(bank_id, bank_timings) -> str:
1300
+ """Switchroom #3619 — return the degraded-recall disclosure for this turn,
1301
+ or "" when the agent's own bank answered.
1302
+
1303
+ Until now a recall whose own bank timed out was indistinguishable, from the
1304
+ agent's side, from a bank that genuinely held nothing relevant: both
1305
+ produced an empty block and silence. That ambiguity is what let a measured
1306
+ ~90% own-bank timeout rate run for weeks unnoticed while every agent's
1307
+ CLAUDE.md asserted recall "auto-fires on every inbound message" — the agent
1308
+ had no way to know it was answering from an empty context, so it never said
1309
+ so and the operator never saw it.
1310
+
1311
+ Only the agent's OWN bank warrants the notice: additional banks (a shared
1312
+ profile bank, say) are supplementary, and a side-bank timeout does not mean
1313
+ the agent lost its own memory. Matching is by `bank_id`, never by position
1314
+ in `bank_timings` — the fan-out order is not stable.
1315
+
1316
+ Kept to a single short line on purpose: this fires on an already-degraded
1317
+ turn, and a verbose block would spend the very budget the degradation is
1318
+ starving. The caller must keep it OUT of the cached context (see
1319
+ `_combine_context`) — it is per-turn state and would otherwise replay on a
1320
+ later healthy cache hit.
1321
+ """
1322
+ if not bank_id or not bank_timings:
1323
+ return ""
1324
+ own = next(
1325
+ (
1326
+ bt
1327
+ for bt in bank_timings
1328
+ if isinstance(bt, dict) and bt.get("bank_id") == bank_id
1329
+ ),
1330
+ None,
1331
+ )
1332
+ if not own:
1333
+ return ""
1334
+ if own.get("timed_out"):
1335
+ reason = "timed out"
1336
+ elif own.get("errored"):
1337
+ reason = "was unreachable"
1338
+ else:
1339
+ return ""
1340
+ return (
1341
+ f"[Hindsight] Memory recall was DEGRADED this turn: your own bank "
1342
+ f"('{bank_id}') {reason}, so the memories below (if any) are "
1343
+ f"incomplete and may be missing entirely. Treat an absence of "
1344
+ f"relevant memory as UNKNOWN, not as 'nothing was remembered' — "
1345
+ f"say so rather than asserting there is no prior context."
1346
+ )
1347
+
1348
+
1299
1349
  def main():
1300
1350
  config = load_config()
1301
1351
 
@@ -2129,13 +2179,22 @@ def main():
2129
2179
  "transcript_fallback_truncated": transcript_fallback_telemetry["truncated"],
2130
2180
  })
2131
2181
 
2132
- # If neither block has content, there's nothing to inject — exit
2182
+ # Switchroom #3619 — DEGRADED-RECALL DISCLOSURE. See
2183
+ # `degraded_recall_notice` for why this exists and why only the agent's
2184
+ # OWN bank counts.
2185
+ degraded_block = degraded_recall_notice(bank_id, bank_timings)
2186
+
2187
+ # If no block has content, there's nothing to inject — exit
2133
2188
  # silently to avoid emitting an empty hookSpecificOutput. #2848: unless
2134
2189
  # the directive-capture nudge fired, in which case emit the nudge alone
2135
2190
  # (a correction with no memories/directives still needs the reminder).
2191
+ # #3619: a degraded own-bank read is likewise worth emitting alone — that
2192
+ # is precisely the turn on which the agent must not assume it remembers.
2136
2193
  if not directives_block and not memories_block and not transcript_fallback_block:
2137
- if nudge_block:
2138
- _emit_cached_context(nudge_block)
2194
+ if degraded_block or nudge_block:
2195
+ _emit_cached_context(
2196
+ "\n\n".join([b for b in (degraded_block, nudge_block) if b])
2197
+ )
2139
2198
  return
2140
2199
 
2141
2200
  # Compose final context. Directives block goes ABOVE memories so the
@@ -2143,6 +2202,12 @@ def main():
2143
2202
  # transcript fallback (#3369) goes LAST — it is the lowest-confidence
2144
2203
  # signal (raw transcript, not synthesized fact) and only present when
2145
2204
  # memories_block is empty by construction.
2205
+ #
2206
+ # #3619's degraded notice is deliberately NOT part of context_message: like
2207
+ # the #2848 nudge it is per-turn state, and this string is what gets cached
2208
+ # and written to LAST_RECALL_STATE. Caching it would replay "recall was
2209
+ # DEGRADED" on later healthy cache hits; it is prepended at emit time
2210
+ # instead, so a cache hit re-derives the turn's real condition.
2146
2211
  parts = []
2147
2212
  if directives_block:
2148
2213
  parts.append(directives_block)
@@ -2177,11 +2242,15 @@ def main():
2177
2242
 
2178
2243
  # Output JSON for Claude Code hook system. #2848: append the
2179
2244
  # directive-capture nudge (if it fired) at emit time — it's kept out of
2180
- # the cached / last-recall context above so it can't go stale.
2245
+ # the cached / last-recall context above so it can't go stale. #3619: the
2246
+ # degraded-recall notice is prepended for the same reason, and goes FIRST
2247
+ # because it changes how everything after it should be read.
2181
2248
  output = {
2182
2249
  "hookSpecificOutput": {
2183
2250
  "hookEventName": "UserPromptSubmit",
2184
- "additionalContext": _combine_context(context_message, nudge_block),
2251
+ "additionalContext": _combine_context(
2252
+ _combine_context(degraded_block, context_message), nudge_block
2253
+ ),
2185
2254
  }
2186
2255
  }
2187
2256
  json.dump(output, sys.stdout)