switchroom 0.19.19 → 0.19.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-broker/index.js +53 -0
- package/dist/cli/switchroom.js +2444 -1264
- package/dist/host-control/main.js +54 -1
- package/dist/vault/approvals/kernel-server.js +53 -0
- package/dist/vault/broker/server.js +53 -0
- package/package.json +4 -2
- package/skills/switchroom-release/SKILL.md +103 -20
- package/telegram-plugin/card-format.ts +92 -3
- package/telegram-plugin/dist/gateway/gateway.js +769 -172
- package/telegram-plugin/edit-flood-fuse.ts +477 -0
- package/telegram-plugin/format.ts +19 -7
- package/telegram-plugin/gateway/boot-sweep-gate.ts +164 -0
- package/telegram-plugin/gateway/callback-query-handlers.ts +454 -81
- package/telegram-plugin/gateway/gateway.ts +66 -56
- package/telegram-plugin/gateway/inbound-interceptors.ts +27 -4
- package/telegram-plugin/gateway/narrative-lane.ts +49 -3
- package/telegram-plugin/gateway/status-pin-api.ts +145 -0
- package/telegram-plugin/hooks/subagent-tracker-posttool.mjs +325 -45
- package/telegram-plugin/retry-api-call.ts +15 -2
- package/telegram-plugin/send-gate.ts +1 -1
- package/telegram-plugin/status-no-truncate.ts +64 -1
- package/telegram-plugin/status-pin-driver.ts +50 -27
- package/telegram-plugin/status-pin.ts +43 -5
- package/telegram-plugin/tests/activity-card-send-gate.test.ts +275 -0
- package/telegram-plugin/tests/activity-card-wiring.test.ts +16 -7
- package/telegram-plugin/tests/boot-pin-sweep-wiring.test.ts +101 -0
- package/telegram-plugin/tests/boot-sweep-gate.test.ts +293 -0
- package/telegram-plugin/tests/boot-version-string.test.ts +0 -0
- package/telegram-plugin/tests/edit-flood-fuse.test.ts +431 -0
- package/telegram-plugin/tests/pinned-card-collapse.test.ts +356 -0
- package/telegram-plugin/tests/status-pin-api.test.ts +178 -0
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +94 -11
- package/telegram-plugin/tests/status-pin.test.ts +106 -5
- package/telegram-plugin/tests/subagent-tracker-hooks.test.ts +631 -1
- package/telegram-plugin/tests/tool-activity-summary.test.ts +19 -10
- package/telegram-plugin/tests/vault-approval-posture.test.ts +6 -1
- package/telegram-plugin/tests/vault-passphrase-retry.test.ts +666 -0
- package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +42 -21
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +233 -1
- package/telegram-plugin/tool-activity-summary.ts +85 -13
- package/telegram-plugin/worker-activity-feed.ts +5 -1
- package/vendor/hindsight-memory/scripts/drain_pending.py +193 -25
- package/vendor/hindsight-memory/scripts/lib/pending.py +84 -5
- package/vendor/hindsight-memory/scripts/lib/retain_split.py +21 -10
- package/vendor/hindsight-memory/scripts/recall.py +74 -5
- package/vendor/hindsight-memory/scripts/tests/test_pending_drops.py +158 -4
- package/vendor/hindsight-memory/scripts/tests/test_pending_failure_class.py +105 -0
- package/vendor/hindsight-memory/scripts/tests/test_pending_wedge.py +300 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_degraded_notice.py +365 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_envelope_strip_telemetry.py +12 -4
- package/vendor/hindsight-memory/scripts/tests/test_recall_transcript_fallback.py +27 -2
- package/vendor/hindsight-memory/scripts/tests/test_retain_split.py +19 -11
- package/vendor/hindsight-memory/tests/test_drain_pending.py +28 -2
|
@@ -3,9 +3,21 @@
|
|
|
3
3
|
|
|
4
4
|
SessionStart calls into ``drain()`` to retry any retain payloads that
|
|
5
5
|
``session_end.py`` queued on failure (#1071). Each entry is retried up
|
|
6
|
-
to ``MAX_ATTEMPTS`` (5) times; after that
|
|
7
|
-
|
|
8
|
-
``
|
|
6
|
+
to ``MAX_ATTEMPTS`` (5) times; after that a **permanently** failing entry
|
|
7
|
+
(a 4xx that a re-POST cannot fix — see ``pending.is_permanent_failure``)
|
|
8
|
+
is renamed to ``.dead`` so the queue no longer drains it but the operator
|
|
9
|
+
can still inspect via ``switchroom doctor``. An entry failing on anything
|
|
10
|
+
else — a 5xx, a timeout, a connection error — stays queued past the
|
|
11
|
+
attempt budget: a transient upstream is never evidence that the memory
|
|
12
|
+
is unsaveable, and retiring it would lose content the user believes was
|
|
13
|
+
saved.
|
|
14
|
+
|
|
15
|
+
An entry past the budget is DEMOTED rather than retired (``_drain_order``
|
|
16
|
+
and ``_over_budget``): it sorts behind everything still inside its budget
|
|
17
|
+
and abstains from the stall guard. That is what keeps "never destroy a
|
|
18
|
+
memory" from degrading into "never drain anything" — the drain is
|
|
19
|
+
sequential and oldest-first, so without the demotion three chronically
|
|
20
|
+
failing entries sit at the head and end every run at zero progress.
|
|
9
21
|
|
|
10
22
|
Boundaries
|
|
11
23
|
----------
|
|
@@ -36,8 +48,7 @@ accumulated backlog. Worse, they CREATE one: the per-entry timeout is
|
|
|
36
48
|
clamped to the remaining hook budget (1-8s) while ``_retry_one`` posts
|
|
37
49
|
synchronously and a real retain takes 30-90s, so **the server commits the
|
|
38
50
|
document and the client always gives up before the ack**. The entry is
|
|
39
|
-
never deleted and is re-posted on every session start
|
|
40
|
-
hits ``MAX_ATTEMPTS`` and goes ``.dead``. The queue depth was a symptom
|
|
51
|
+
never deleted and is re-posted on every session start. The queue depth was a symptom
|
|
41
52
|
of that loop, not of lost memory: a full sweep of 5,751 queued entries on
|
|
42
53
|
this fleet (2026-07-25) found **4,048 (70.4%) already existed as
|
|
43
54
|
documents**, 3,815 of them with facts extracted.
|
|
@@ -95,6 +106,7 @@ from lib.pending import (
|
|
|
95
106
|
MAX_ATTEMPTS,
|
|
96
107
|
archive_reconciled,
|
|
97
108
|
is_content_derived_document_id,
|
|
109
|
+
is_permanent_failure,
|
|
98
110
|
iter_entries,
|
|
99
111
|
mark_dead,
|
|
100
112
|
update_attempt,
|
|
@@ -203,16 +215,30 @@ def _backlog_timeout() -> int:
|
|
|
203
215
|
client-side timeout on a request the server then commits anyway.
|
|
204
216
|
|
|
205
217
|
The default is DERIVED, not a literal: it is
|
|
206
|
-
``retain_split.retain_client_deadline()`` (
|
|
218
|
+
``retain_split.retain_client_deadline()`` (310s), the same deadline the
|
|
207
219
|
retain content bound is sized against. Those two must be ONE number.
|
|
208
220
|
This function shipped as a bare ``180`` (#3599), and against a 180s
|
|
209
221
|
deadline both halves of the retain budget break: a maximally-sized part
|
|
210
222
|
is ~276s of sequential extraction, and the SERVER per-call timeout
|
|
211
|
-
derived in ``src/setup/hindsight.ts``
|
|
223
|
+
derived in ``src/setup/hindsight.ts`` is larger still — so the drain
|
|
212
224
|
client would abandon a request the server is still legitimately working
|
|
213
225
|
on, leave the entry queued, and rebuild the re-post loop #3599 exists to
|
|
214
226
|
kill, one size class up.
|
|
215
227
|
|
|
228
|
+
This is the OUTERMOST deadline of the paired-budget family, and it is the
|
|
229
|
+
one the 2026-07-26 backlog-recovery logs show as a fixed ~280.1s per-entry
|
|
230
|
+
give-up. That was not an unexplained third number: it is this default at
|
|
231
|
+
the then-current ``DEFAULT_RETAIN_CLIENT_DEADLINE_S = 280.0``, plus the
|
|
232
|
+
request's own setup overhead. It was WRONG for the same reason #3611's
|
|
233
|
+
204s was wrong — it did not cover hindsight's LiteLLM routing chain
|
|
234
|
+
(local 200s + OpenRouter fallback 90s + router margin 10s = 300s), so
|
|
235
|
+
every drained entry whose retain fell through to the fallback was
|
|
236
|
+
abandoned client-side at 280s while the server was still inside a
|
|
237
|
+
legitimate 300s budget. Raising the derivation's base to 310 fixes this
|
|
238
|
+
lane and the in-hook lane with the same number, by construction.
|
|
239
|
+
``src/litellm/timeout-budget.ts`` is where the chain is declared, and
|
|
240
|
+
``tests/setup/hindsight.test.ts`` fails if these drift apart again.
|
|
241
|
+
|
|
216
242
|
``HINDSIGHT_DRAIN_BACKLOG_TIMEOUT`` still overrides it outright;
|
|
217
243
|
``HINDSIGHT_RETAIN_CLIENT_DEADLINE_S`` moves the derivation and the
|
|
218
244
|
content bound together.
|
|
@@ -348,6 +374,76 @@ def _document_state(entry: dict, timeout: int = 30):
|
|
|
348
374
|
return None
|
|
349
375
|
|
|
350
376
|
|
|
377
|
+
def _drain_order(entries: list[tuple[str, dict]]) -> list[tuple[str, dict]]:
|
|
378
|
+
"""Oldest-first, but with budget-exhausted entries demoted to the back.
|
|
379
|
+
|
|
380
|
+
THE HEAD-OF-LINE BUG THIS EXISTS FOR. Since the permanence gate in
|
|
381
|
+
``_record_failure``, an entry failing on anything transient stays queued
|
|
382
|
+
past ``MAX_ATTEMPTS`` indefinitely — deliberately, because a transient
|
|
383
|
+
upstream is not evidence the memory is unsaveable. But ``iter_entries()``
|
|
384
|
+
is oldest-first, and the entries that have been failing longest are by
|
|
385
|
+
construction the OLDEST, so they sit at the head of every drain. Backlog
|
|
386
|
+
concurrency defaults to 1 (``_backlog_concurrency``), so the drain is
|
|
387
|
+
sequential: three such entries in a row trip ``STALL_THRESHOLD`` and the
|
|
388
|
+
run breaks having drained nothing — and, because they never retire, it
|
|
389
|
+
breaks identically on every subsequent run. Measured on this branch before
|
|
390
|
+
this function existed: 6 queued entries, the 3 oldest raising
|
|
391
|
+
``TimeoutError``, 4 consecutive ``drain_backlog`` runs each returning
|
|
392
|
+
``stalled=True, drained=0`` with the queue depth still 6. The 3 healthy
|
|
393
|
+
entries behind them were never even attempted. On ``main`` the same repro
|
|
394
|
+
converges: run 0 retires the 3 heads to ``.dead`` and run 1 onward drains
|
|
395
|
+
normally. So the permanence gate, alone, traded "rarely destroys a memory"
|
|
396
|
+
for "eventually drains nothing at all".
|
|
397
|
+
|
|
398
|
+
``.dead`` was doing double duty: it was the honesty policy AND it was the
|
|
399
|
+
queue's only un-wedging mechanism. Removing it as a policy has to leave
|
|
400
|
+
the un-wedging behind, and demotion is that — it keeps every property the
|
|
401
|
+
gate was added for (the entry is still queued, still retried, never
|
|
402
|
+
destroyed) while removing the one it broke (it can no longer starve a
|
|
403
|
+
healthy entry behind it).
|
|
404
|
+
|
|
405
|
+
The terminal condition for a permanently-unsaveable entry is therefore no
|
|
406
|
+
longer deletion but DEMOTION: ``attempt_count`` only ever climbs, so such
|
|
407
|
+
an entry crosses the budget once and stays in the back group for good,
|
|
408
|
+
where it can delay only itself. It is still reconciled for free on every
|
|
409
|
+
run — ``_reconcile_phase`` sweeps ALL entries with a sub-second presence
|
|
410
|
+
GET, in no particular order and with no stall guard — so an entry whose
|
|
411
|
+
document did land is still retired without a POST.
|
|
412
|
+
|
|
413
|
+
Ordering is a stable partition, so relative age is preserved inside each
|
|
414
|
+
group and FIFO still holds for everything that has not blown its budget.
|
|
415
|
+
"""
|
|
416
|
+
fresh: list[tuple[str, dict]] = []
|
|
417
|
+
exhausted: list[tuple[str, dict]] = []
|
|
418
|
+
for path, entry in entries:
|
|
419
|
+
(exhausted if _over_budget(entry) else fresh).append((path, entry))
|
|
420
|
+
return fresh + exhausted
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def _over_budget(entry: dict) -> bool:
|
|
424
|
+
"""Has this entry already burned its ``MAX_ATTEMPTS`` budget?
|
|
425
|
+
|
|
426
|
+
Such an entry is chronically failing but, since the permanence gate, is
|
|
427
|
+
never retired. It gets two demotions — last in the drain order
|
|
428
|
+
(``_drain_order``) and no vote in the stall guard (see below) — because
|
|
429
|
+
ordering alone does not close the wedge. Ordering fixes the common shape
|
|
430
|
+
(a few old poison entries in front of healthy ones), but not the shape
|
|
431
|
+
where the WHOLE queue is over budget: an upstream down for a week takes
|
|
432
|
+
every entry past 5 attempts, and when it recovers the partition is empty
|
|
433
|
+
on one side, the poisoned entries are at the head again, and the run
|
|
434
|
+
stalls before reaching the entries that would now succeed. Measured: with
|
|
435
|
+
ordering alone and all 6 entries at ``MAX_ATTEMPTS``, 4 consecutive runs
|
|
436
|
+
still returned ``stalled=True, drained=0``.
|
|
437
|
+
"""
|
|
438
|
+
try:
|
|
439
|
+
return int(entry.get("attempt_count", 0)) >= MAX_ATTEMPTS
|
|
440
|
+
except (TypeError, ValueError):
|
|
441
|
+
# A hand-edited or corrupt counter must not decide ordering, and must
|
|
442
|
+
# not raise on the drain path. Treat it as fresh: the cost of guessing
|
|
443
|
+
# wrong here is one retry in the normal position, not a lost memory.
|
|
444
|
+
return False
|
|
445
|
+
|
|
446
|
+
|
|
351
447
|
def _record_failure(
|
|
352
448
|
config: dict,
|
|
353
449
|
path: str,
|
|
@@ -358,16 +454,53 @@ def _record_failure(
|
|
|
358
454
|
"""Apply the per-entry failure policy. Returns the error class name.
|
|
359
455
|
|
|
360
456
|
Shared by the sequential (SessionStart) and backlog drains so both age
|
|
361
|
-
entries toward ``.dead`` on exactly the same schedule
|
|
457
|
+
entries toward ``.dead`` on exactly the same schedule — and, since the
|
|
458
|
+
permanence gate below, refuse to retire them on exactly the same rule.
|
|
362
459
|
"""
|
|
363
460
|
err_class = type(e).__name__
|
|
364
461
|
attempts = int(entry.get("attempt_count", 1))
|
|
365
|
-
|
|
462
|
+
# ``.dead`` retires a memory the user believes was saved, so it is gated on
|
|
463
|
+
# the failure being PERMANENT — a 4xx that re-POSTing cannot fix. A
|
|
464
|
+
# transient failure keeps its attempt counter climbing but stays queued,
|
|
465
|
+
# because an exhausted attempt budget is not evidence that the content is
|
|
466
|
+
# unpersistable.
|
|
467
|
+
#
|
|
468
|
+
# Before this gate, ANY five failures retired the entry. The dominant
|
|
469
|
+
# failure on this fleet is an HTTP 500 "Fact extraction failed … chunk 0:
|
|
470
|
+
# JSONDecodeError" — the extraction model returned an empty or non-JSON
|
|
471
|
+
# completion for one chunk on that sampling run. The identical content
|
|
472
|
+
# succeeds on a later attempt, so five unlucky samples were destroying
|
|
473
|
+
# memories that were never unsaveable. See ``pending.is_permanent_failure``.
|
|
474
|
+
#
|
|
475
|
+
# WHAT BOUNDS THIS, precisely — because an unbounded queue of undying
|
|
476
|
+
# entries would be a worse outcome than the bug this gate fixes.
|
|
477
|
+
#
|
|
478
|
+
# DISK is bounded by the queue's MAX_ENTRIES / MAX_BYTES caps, which shed
|
|
479
|
+
# the oldest entries into ``pending-evicted/`` (an archive, not a delete).
|
|
480
|
+
# Note what that bound is NOT: ``_evict_to_fit`` is called only from
|
|
481
|
+
# ``enqueue`` (lib/pending.py:933), so it fires on new writes, never on
|
|
482
|
+
# drain — and it "bounds" the queue by shedding memory unsaved, which is
|
|
483
|
+
# the very outcome this gate exists to avoid. It is a backstop, not the
|
|
484
|
+
# answer.
|
|
485
|
+
#
|
|
486
|
+
# PROGRESS is bounded by ``_drain_order`` / ``_over_budget``. An entry that
|
|
487
|
+
# can never be persisted no longer terminates by being destroyed; it
|
|
488
|
+
# terminates by being DEMOTED — sorted behind every entry still inside its
|
|
489
|
+
# budget, and stripped of its vote in the stall guard. ``attempt_count``
|
|
490
|
+
# only ever climbs, so the crossing happens once and is permanent. That is
|
|
491
|
+
# the real terminal condition, and it is what keeps a poisoned entry from
|
|
492
|
+
# starving the queue behind it. Without it, three such entries ended every
|
|
493
|
+
# drain at zero progress, permanently (tests/test_pending_wedge.py).
|
|
494
|
+
#
|
|
495
|
+
# The attempt counter itself was never a bound on either, and ``switchroom
|
|
496
|
+
# doctor`` still surfaces queue depth so the operator sees a growing tail.
|
|
497
|
+
if attempts >= MAX_ATTEMPTS and is_permanent_failure(e):
|
|
366
498
|
marker = mark_dead(path, entry)
|
|
367
499
|
summary["dead"] += 1
|
|
368
500
|
print(
|
|
369
501
|
f"[Hindsight] drain_pending: entry exceeded {MAX_ATTEMPTS} "
|
|
370
|
-
f"attempts, marking dead at {marker}
|
|
502
|
+
f"attempts on a permanent failure, marking dead at {marker} "
|
|
503
|
+
f"(last error: {err_class}: {e})",
|
|
371
504
|
file=sys.stderr,
|
|
372
505
|
)
|
|
373
506
|
else:
|
|
@@ -375,7 +508,12 @@ def _record_failure(
|
|
|
375
508
|
summary["retried"] += 1
|
|
376
509
|
debug_log(
|
|
377
510
|
config,
|
|
378
|
-
|
|
511
|
+
# ``attempts`` can now exceed MAX_ATTEMPTS — a transient failure
|
|
512
|
+
# keeps the entry queued past the budget instead of retiring it —
|
|
513
|
+
# so print the budget as a threshold, not as a fraction that would
|
|
514
|
+
# render the nonsense "retry 7/5".
|
|
515
|
+
f"drain_pending: retry {attempts} (budget {MAX_ATTEMPTS}) failed "
|
|
516
|
+
f"for {path} ({err_class}: {e})",
|
|
379
517
|
)
|
|
380
518
|
return err_class
|
|
381
519
|
|
|
@@ -440,7 +578,11 @@ def drain(
|
|
|
440
578
|
|
|
441
579
|
summary = _new_summary()
|
|
442
580
|
|
|
443
|
-
entries
|
|
581
|
+
# Budget-exhausted entries go last so they cannot starve the healthy ones
|
|
582
|
+
# behind them — see ``_drain_order``. Matters even more here than in the
|
|
583
|
+
# backlog drain: the in-hook budget is ~4s, so a single entry at the head
|
|
584
|
+
# that always burns its clamped timeout consumes the entire run.
|
|
585
|
+
entries = _drain_order(iter_entries())
|
|
444
586
|
if not entries:
|
|
445
587
|
debug_log(config, "drain_pending: queue empty")
|
|
446
588
|
return summary
|
|
@@ -496,14 +638,30 @@ def drain(
|
|
|
496
638
|
# allowance must not be handed out twice (#3599 review F2).
|
|
497
639
|
_retry_one(entry, timeout=_clamp(timeout, budget, started))
|
|
498
640
|
except Exception as e:
|
|
641
|
+
# Read the budget state BEFORE _record_failure: update_attempt
|
|
642
|
+
# mutates `entry["attempt_count"]` in place (lib/pending.py:1020),
|
|
643
|
+
# so asking afterwards would count the entry that just CROSSED the
|
|
644
|
+
# budget on this very failure as an abstainer, silently weakening
|
|
645
|
+
# the stall guard for ordinary entries.
|
|
646
|
+
abstains = _over_budget(entry)
|
|
499
647
|
err_class = _record_failure(config, path, entry, e, summary)
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
648
|
+
# STALL GUARD ABSTENTION — see ``_over_budget``. The guard exists
|
|
649
|
+
# to detect a broken UPSTREAM and stop hammering it. A chronically
|
|
650
|
+
# failing entry that has already burned its attempt budget is
|
|
651
|
+
# evidence about that ENTRY, not about the upstream, and letting it
|
|
652
|
+
# vote is what wedges the queue: it never retires, so it fails
|
|
653
|
+
# identically on every run, and three of them end every run before
|
|
654
|
+
# the healthy entries behind them are ever reached. It abstains —
|
|
655
|
+
# neither incrementing the counter nor resetting it, so a genuine
|
|
656
|
+
# upstream outage is still caught by the fresh entries around it.
|
|
657
|
+
if not abstains:
|
|
658
|
+
if err_class == last_error_class:
|
|
659
|
+
consecutive_failures += 1
|
|
660
|
+
else:
|
|
661
|
+
consecutive_failures = 1
|
|
662
|
+
last_error_class = err_class
|
|
505
663
|
|
|
506
|
-
if consecutive_failures >= STALL_THRESHOLD:
|
|
664
|
+
if not abstains and consecutive_failures >= STALL_THRESHOLD:
|
|
507
665
|
summary["stalled"] = True
|
|
508
666
|
print(
|
|
509
667
|
f"[Hindsight] drain_pending: {consecutive_failures} consecutive "
|
|
@@ -641,7 +799,10 @@ def _drain_backlog_impl(
|
|
|
641
799
|
backoff_ms = _p95_backoff_ms()
|
|
642
800
|
started = time.monotonic()
|
|
643
801
|
|
|
644
|
-
entries
|
|
802
|
+
# See ``_drain_order``: without this, three budget-exhausted entries at the
|
|
803
|
+
# head trip the stall guard on every run forever and the drain never makes
|
|
804
|
+
# progress again.
|
|
805
|
+
entries = _drain_order(iter_entries())
|
|
645
806
|
if not entries:
|
|
646
807
|
_blog("phase 2: nothing left to retain")
|
|
647
808
|
return summary
|
|
@@ -728,15 +889,22 @@ def _drain_backlog_impl(
|
|
|
728
889
|
# and breaking immediately after the tripping entry makes the
|
|
729
890
|
# two paths bump exactly the same number of entries.
|
|
730
891
|
err_class = type(err).__name__
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
892
|
+
# STALL GUARD ABSTENTION for entries past their attempt budget —
|
|
893
|
+
# see ``_over_budget`` and the matching branch in the sequential
|
|
894
|
+
# drain. Evaluated here, before ``_record_failure``, which is both
|
|
895
|
+
# where the sequential drain evaluates it and necessary anyway
|
|
896
|
+
# because ``update_attempt`` mutates ``attempt_count`` in place.
|
|
897
|
+
abstains = _over_budget(entry)
|
|
898
|
+
if not abstains:
|
|
899
|
+
if err_class == last_error_class:
|
|
900
|
+
consecutive_failures += 1
|
|
901
|
+
else:
|
|
902
|
+
consecutive_failures = 1
|
|
903
|
+
last_error_class = err_class
|
|
736
904
|
|
|
737
905
|
_record_failure(config, path, entry, err, summary)
|
|
738
906
|
|
|
739
|
-
if consecutive_failures >= STALL_THRESHOLD:
|
|
907
|
+
if not abstains and consecutive_failures >= STALL_THRESHOLD:
|
|
740
908
|
summary["stalled"] = True
|
|
741
909
|
_blog(
|
|
742
910
|
f"{consecutive_failures} consecutive failures with "
|
|
@@ -10,7 +10,10 @@ entry into the bounded ``pending-reconciled/`` archive (the drain never
|
|
|
10
10
|
deletes — see "Retiring an entry" below for that promise and its one
|
|
11
11
|
bound, a full disk), failure bumps an attempt
|
|
12
12
|
counter (up to MAX_ATTEMPTS) and leaves the entry for the run after
|
|
13
|
-
that.
|
|
13
|
+
that. Exhausting MAX_ATTEMPTS retires the entry to ``.dead`` only when
|
|
14
|
+
the failure is PERMANENT (``is_permanent_failure`` — a 4xx a re-POST
|
|
15
|
+
cannot fix). A transient failure never retires the memory, however many
|
|
16
|
+
attempts it has burned.
|
|
14
17
|
|
|
15
18
|
Layout
|
|
16
19
|
------
|
|
@@ -144,11 +147,19 @@ oversized payload into one entry PER PART (``lib/retain_split``) instead
|
|
|
144
147
|
of writing a single giant entry. This is load-bearing, not tidiness. The
|
|
145
148
|
daemon runs one sequential extraction LLM call per ``retain_chunk_size``
|
|
146
149
|
chars, so an entry above the derived content bound cannot complete inside
|
|
147
|
-
ANY client deadline — including the
|
|
150
|
+
ANY client deadline — including the out-of-hook backlog deadline that
|
|
148
151
|
``drain_pending._backlog_timeout()`` now takes from the same derivation.
|
|
149
|
-
Such an entry fails every drain
|
|
150
|
-
|
|
151
|
-
|
|
152
|
+
Such an entry fails every drain and burns its ``MAX_ATTEMPTS``: the
|
|
153
|
+
mechanism that stranded 154 of the 629 entries in the 2026-07-25 fleet
|
|
154
|
+
backlog. (It is no longer renamed ``.dead`` for that — a client-side
|
|
155
|
+
timeout is not a permanent failure — but it still never drains until it
|
|
156
|
+
is split, so splitting remains the fix. Such an entry is therefore
|
|
157
|
+
IMMORTAL, and ``enqueue`` is the only caller of the splitter, so one
|
|
158
|
+
queued before #3610 is never split in place. That is precisely the shape
|
|
159
|
+
``drain_pending._drain_order`` / ``_over_budget`` exist to contain: it is
|
|
160
|
+
demoted behind every entry still inside its attempt budget and abstains
|
|
161
|
+
from the stall guard, so it can delay only itself rather than wedging the
|
|
162
|
+
drain.) Note this is orthogonal to the re-post loop
|
|
152
163
|
#3599 fixed — a presence GET retires an oversized entry for free when the
|
|
153
164
|
document IS already durable; splitting is what makes the entry drainable
|
|
154
165
|
when it is NOT. Part document_ids are deterministic, so a part already
|
|
@@ -1027,6 +1038,74 @@ def update_attempt(path: str, entry: dict, error: BaseException) -> bool:
|
|
|
1027
1038
|
return False
|
|
1028
1039
|
|
|
1029
1040
|
|
|
1041
|
+
#: HTTP statuses that are 4xx but describe a TRANSIENT condition, so they
|
|
1042
|
+
#: must be read as retryable despite the 4xx class.
|
|
1043
|
+
_RETRYABLE_4XX = frozenset({408, 425, 429})
|
|
1044
|
+
|
|
1045
|
+
|
|
1046
|
+
def is_permanent_failure(error: BaseException) -> bool:
|
|
1047
|
+
"""True when ``error`` can never succeed on a later identical retry.
|
|
1048
|
+
|
|
1049
|
+
This is the gate on ``mark_dead`` (see ``drain_pending._record_failure``).
|
|
1050
|
+
Getting it wrong in the permissive direction costs a re-POST — which is an
|
|
1051
|
+
upsert, so it costs time. Getting it wrong in the strict direction costs
|
|
1052
|
+
the USER'S MEMORY. So the rule is deliberately asymmetric: an error is
|
|
1053
|
+
permanent only when we can positively identify it as a client-side defect
|
|
1054
|
+
in the request itself. Everything we cannot classify is retryable.
|
|
1055
|
+
|
|
1056
|
+
PERMANENT — a 4xx other than 408/425/429. The server understood us and
|
|
1057
|
+
rejected the request: a malformed payload, an unknown bank, an oversized
|
|
1058
|
+
body, a bad token. Re-POSTing the identical bytes reproduces it exactly,
|
|
1059
|
+
so attempts are pure waste and ``.dead`` is the honest outcome.
|
|
1060
|
+
|
|
1061
|
+
RETRYABLE — everything else. Notably 5xx, which is what a failed
|
|
1062
|
+
fact-extraction surfaces as::
|
|
1063
|
+
|
|
1064
|
+
HTTP 500 ...: {"detail": "Fact extraction failed: 1/1 chunks failed.
|
|
1065
|
+
First failures: chunk 0: JSONDecodeError: Expecting value: line 1
|
|
1066
|
+
column 1 (char 0)"}
|
|
1067
|
+
|
|
1068
|
+
That 500 means the extraction model returned an empty or non-JSON
|
|
1069
|
+
completion for one chunk (measured 2026-07-26: Ollama returning
|
|
1070
|
+
``content: ""`` with all-zero usage, and gpt-oss-20b emitting a numbered
|
|
1071
|
+
prose list instead of the JSON schema). It is a property of one sampling
|
|
1072
|
+
run, NOT of the queued content — the very same entry succeeds on a later
|
|
1073
|
+
attempt. Counting it toward ``MAX_ATTEMPTS`` is what turned a flaky model
|
|
1074
|
+
into permanently lost memories: five unlucky samples and a real memory
|
|
1075
|
+
went ``.dead``.
|
|
1076
|
+
|
|
1077
|
+
Timeouts, connection resets, DNS failures and anything unrecognised are
|
|
1078
|
+
retryable for the same reason — none of them is evidence that the content
|
|
1079
|
+
can never be persisted.
|
|
1080
|
+
|
|
1081
|
+
The fleet bears this out. A census of every ``.dead`` marker on this host
|
|
1082
|
+
(2026-07-26, 129 markers across 10 agents) found the retiring error was
|
|
1083
|
+
``TimeoutError`` 128 times and ``URLError`` once. **Not one was a 4xx.**
|
|
1084
|
+
Every permanently-lost memory here was lost to a transient failure, so
|
|
1085
|
+
this gate would have kept all 129 queued and drainable. Two were retired
|
|
1086
|
+
at 00:56Z that same morning — this was live, not historical.
|
|
1087
|
+
|
|
1088
|
+
``client.HindsightClient._request`` re-raises ``urllib`` HTTP failures as
|
|
1089
|
+
``RuntimeError(f"HTTP {code} from {url}: {body}")`` with the original
|
|
1090
|
+
``HTTPError`` chained on ``__cause__``, so the status is read from the
|
|
1091
|
+
cause when present and parsed out of the message otherwise (the message
|
|
1092
|
+
form is what a de-chained/re-serialised error leaves behind).
|
|
1093
|
+
"""
|
|
1094
|
+
code = getattr(error, "code", None)
|
|
1095
|
+
cause = getattr(error, "__cause__", None)
|
|
1096
|
+
if not isinstance(code, int) and cause is not None:
|
|
1097
|
+
code = getattr(cause, "code", None)
|
|
1098
|
+
if not isinstance(code, int):
|
|
1099
|
+
text = str(error)
|
|
1100
|
+
if text.startswith("HTTP "):
|
|
1101
|
+
head = text[5:].split(" ", 1)[0]
|
|
1102
|
+
if head.isdigit():
|
|
1103
|
+
code = int(head)
|
|
1104
|
+
if not isinstance(code, int):
|
|
1105
|
+
return False
|
|
1106
|
+
return 400 <= code < 500 and code not in _RETRYABLE_4XX
|
|
1107
|
+
|
|
1108
|
+
|
|
1030
1109
|
def mark_dead(path: str, entry: dict) -> Optional[str]:
|
|
1031
1110
|
"""Convert an entry that exceeded ``MAX_ATTEMPTS`` into a permanent
|
|
1032
1111
|
failure marker at ``<path>.dead`` so the queue no longer drains it
|
|
@@ -83,7 +83,7 @@ from typing import Optional
|
|
|
83
83
|
# completion-token bucket; the runaway bucket is
|
|
84
84
|
# a separate defect, fixed by capping
|
|
85
85
|
# HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS).
|
|
86
|
-
# client_deadline
|
|
86
|
+
# client_deadline 310 s — the deadline of the DURABILITY path (the
|
|
87
87
|
# out-of-hook backlog drain, #3599), deliberately
|
|
88
88
|
# not the live Stop hook's 15s. The live path is
|
|
89
89
|
# allowed to miss its deadline: it enqueues to
|
|
@@ -96,20 +96,31 @@ from typing import Optional
|
|
|
96
96
|
# literal, and `src/setup/hindsight.ts`
|
|
97
97
|
# (`HINDSIGHT_RETAIN_CLIENT_DEADLINE_S`, #3611)
|
|
98
98
|
# mirrors it as the client half of that PR's
|
|
99
|
-
# `
|
|
100
|
-
# assertion
|
|
101
|
-
#
|
|
102
|
-
#
|
|
99
|
+
# `hindsight per-call timeout < client deadline`
|
|
100
|
+
# assertion. A test in `tests/setup/hindsight.test.ts`
|
|
101
|
+
# enforces that the two stay equal, so the mirror
|
|
102
|
+
# is checked, not merely asserted in prose.
|
|
103
|
+
# WAS 280.0. Raised to 310 when the retain
|
|
104
|
+
# per-call timeout became derived from the litellm
|
|
105
|
+
# routing chain (`local 200 + fallback 90 +
|
|
106
|
+
# margin 10 = 300`) instead of from the token
|
|
107
|
+
# budget alone: #3611's 204s could not cover that
|
|
108
|
+
# chain, so the retain OpenRouter fallback hop had
|
|
109
|
+
# 4s of headroom and could never complete. 310 is
|
|
110
|
+
# that 300 plus the same one margin, so the plugin
|
|
111
|
+
# always outlives hindsight by construction. It is
|
|
112
|
+
# NOT a hand-set number on either side — change
|
|
113
|
+
# `src/litellm/timeout-budget.ts` and both move.
|
|
103
114
|
#
|
|
104
|
-
# floor(
|
|
115
|
+
# floor(310 / 18.4) = 16 chunks → 16 × 3000 = 48,000 chars
|
|
105
116
|
#
|
|
106
117
|
# Sanity check against the same backlog: every entry at or below 60,000 chars
|
|
107
|
-
# drained successfully inside the 280s deadline (observed per-entry
|
|
108
|
-
# 0.3s–153.4s at concurrency 3), so
|
|
109
|
-
# territory with margin for a slower model or a busier box.
|
|
118
|
+
# drained successfully inside the (then 280s) deadline (observed per-entry
|
|
119
|
+
# times 0.3s–153.4s at concurrency 3), so 48,000 still sits inside
|
|
120
|
+
# demonstrated-good territory with margin for a slower model or a busier box.
|
|
110
121
|
DEFAULT_RETAIN_CHUNK_SIZE = 3000
|
|
111
122
|
DEFAULT_RETAIN_CHUNK_LATENCY_S = 18.4
|
|
112
|
-
DEFAULT_RETAIN_CLIENT_DEADLINE_S =
|
|
123
|
+
DEFAULT_RETAIN_CLIENT_DEADLINE_S = 310.0
|
|
113
124
|
|
|
114
125
|
# Absolute floor: one chunk. A bound below one chunk would split every
|
|
115
126
|
# transcript into extraction-sized confetti and is never the right answer.
|
|
@@ -1296,6 +1296,56 @@ def _combine_context(base, nudge) -> str:
|
|
|
1296
1296
|
return "\n\n".join(parts)
|
|
1297
1297
|
|
|
1298
1298
|
|
|
1299
|
+
def degraded_recall_notice(bank_id, bank_timings) -> str:
|
|
1300
|
+
"""Switchroom #3619 — return the degraded-recall disclosure for this turn,
|
|
1301
|
+
or "" when the agent's own bank answered.
|
|
1302
|
+
|
|
1303
|
+
Until now a recall whose own bank timed out was indistinguishable, from the
|
|
1304
|
+
agent's side, from a bank that genuinely held nothing relevant: both
|
|
1305
|
+
produced an empty block and silence. That ambiguity is what let a measured
|
|
1306
|
+
~90% own-bank timeout rate run for weeks unnoticed while every agent's
|
|
1307
|
+
CLAUDE.md asserted recall "auto-fires on every inbound message" — the agent
|
|
1308
|
+
had no way to know it was answering from an empty context, so it never said
|
|
1309
|
+
so and the operator never saw it.
|
|
1310
|
+
|
|
1311
|
+
Only the agent's OWN bank warrants the notice: additional banks (a shared
|
|
1312
|
+
profile bank, say) are supplementary, and a side-bank timeout does not mean
|
|
1313
|
+
the agent lost its own memory. Matching is by `bank_id`, never by position
|
|
1314
|
+
in `bank_timings` — the fan-out order is not stable.
|
|
1315
|
+
|
|
1316
|
+
Kept to a single short line on purpose: this fires on an already-degraded
|
|
1317
|
+
turn, and a verbose block would spend the very budget the degradation is
|
|
1318
|
+
starving. The caller must keep it OUT of the cached context (see
|
|
1319
|
+
`_combine_context`) — it is per-turn state and would otherwise replay on a
|
|
1320
|
+
later healthy cache hit.
|
|
1321
|
+
"""
|
|
1322
|
+
if not bank_id or not bank_timings:
|
|
1323
|
+
return ""
|
|
1324
|
+
own = next(
|
|
1325
|
+
(
|
|
1326
|
+
bt
|
|
1327
|
+
for bt in bank_timings
|
|
1328
|
+
if isinstance(bt, dict) and bt.get("bank_id") == bank_id
|
|
1329
|
+
),
|
|
1330
|
+
None,
|
|
1331
|
+
)
|
|
1332
|
+
if not own:
|
|
1333
|
+
return ""
|
|
1334
|
+
if own.get("timed_out"):
|
|
1335
|
+
reason = "timed out"
|
|
1336
|
+
elif own.get("errored"):
|
|
1337
|
+
reason = "was unreachable"
|
|
1338
|
+
else:
|
|
1339
|
+
return ""
|
|
1340
|
+
return (
|
|
1341
|
+
f"[Hindsight] Memory recall was DEGRADED this turn: your own bank "
|
|
1342
|
+
f"('{bank_id}') {reason}, so the memories below (if any) are "
|
|
1343
|
+
f"incomplete and may be missing entirely. Treat an absence of "
|
|
1344
|
+
f"relevant memory as UNKNOWN, not as 'nothing was remembered' — "
|
|
1345
|
+
f"say so rather than asserting there is no prior context."
|
|
1346
|
+
)
|
|
1347
|
+
|
|
1348
|
+
|
|
1299
1349
|
def main():
|
|
1300
1350
|
config = load_config()
|
|
1301
1351
|
|
|
@@ -2129,13 +2179,22 @@ def main():
|
|
|
2129
2179
|
"transcript_fallback_truncated": transcript_fallback_telemetry["truncated"],
|
|
2130
2180
|
})
|
|
2131
2181
|
|
|
2132
|
-
#
|
|
2182
|
+
# Switchroom #3619 — DEGRADED-RECALL DISCLOSURE. See
|
|
2183
|
+
# `degraded_recall_notice` for why this exists and why only the agent's
|
|
2184
|
+
# OWN bank counts.
|
|
2185
|
+
degraded_block = degraded_recall_notice(bank_id, bank_timings)
|
|
2186
|
+
|
|
2187
|
+
# If no block has content, there's nothing to inject — exit
|
|
2133
2188
|
# silently to avoid emitting an empty hookSpecificOutput. #2848: unless
|
|
2134
2189
|
# the directive-capture nudge fired, in which case emit the nudge alone
|
|
2135
2190
|
# (a correction with no memories/directives still needs the reminder).
|
|
2191
|
+
# #3619: a degraded own-bank read is likewise worth emitting alone — that
|
|
2192
|
+
# is precisely the turn on which the agent must not assume it remembers.
|
|
2136
2193
|
if not directives_block and not memories_block and not transcript_fallback_block:
|
|
2137
|
-
if nudge_block:
|
|
2138
|
-
_emit_cached_context(
|
|
2194
|
+
if degraded_block or nudge_block:
|
|
2195
|
+
_emit_cached_context(
|
|
2196
|
+
"\n\n".join([b for b in (degraded_block, nudge_block) if b])
|
|
2197
|
+
)
|
|
2139
2198
|
return
|
|
2140
2199
|
|
|
2141
2200
|
# Compose final context. Directives block goes ABOVE memories so the
|
|
@@ -2143,6 +2202,12 @@ def main():
|
|
|
2143
2202
|
# transcript fallback (#3369) goes LAST — it is the lowest-confidence
|
|
2144
2203
|
# signal (raw transcript, not synthesized fact) and only present when
|
|
2145
2204
|
# memories_block is empty by construction.
|
|
2205
|
+
#
|
|
2206
|
+
# #3619's degraded notice is deliberately NOT part of context_message: like
|
|
2207
|
+
# the #2848 nudge it is per-turn state, and this string is what gets cached
|
|
2208
|
+
# and written to LAST_RECALL_STATE. Caching it would replay "recall was
|
|
2209
|
+
# DEGRADED" on later healthy cache hits; it is prepended at emit time
|
|
2210
|
+
# instead, so a cache hit re-derives the turn's real condition.
|
|
2146
2211
|
parts = []
|
|
2147
2212
|
if directives_block:
|
|
2148
2213
|
parts.append(directives_block)
|
|
@@ -2177,11 +2242,15 @@ def main():
|
|
|
2177
2242
|
|
|
2178
2243
|
# Output JSON for Claude Code hook system. #2848: append the
|
|
2179
2244
|
# directive-capture nudge (if it fired) at emit time — it's kept out of
|
|
2180
|
-
# the cached / last-recall context above so it can't go stale.
|
|
2245
|
+
# the cached / last-recall context above so it can't go stale. #3619: the
|
|
2246
|
+
# degraded-recall notice is prepended for the same reason, and goes FIRST
|
|
2247
|
+
# because it changes how everything after it should be read.
|
|
2181
2248
|
output = {
|
|
2182
2249
|
"hookSpecificOutput": {
|
|
2183
2250
|
"hookEventName": "UserPromptSubmit",
|
|
2184
|
-
"additionalContext": _combine_context(
|
|
2251
|
+
"additionalContext": _combine_context(
|
|
2252
|
+
_combine_context(degraded_block, context_message), nudge_block
|
|
2253
|
+
),
|
|
2185
2254
|
}
|
|
2186
2255
|
}
|
|
2187
2256
|
json.dump(output, sys.stdout)
|