switchroom 0.20.21 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-broker/index.js +1 -1
- package/dist/cli/switchroom.js +1953 -1539
- package/dist/host-control/main.js +286 -122
- package/dist/vault/approvals/kernel-server.js +1 -1
- package/dist/vault/broker/server.js +1 -1
- package/package.json +1 -1
- package/skills/switchroom-release/SKILL.md +12 -1
- package/telegram-plugin/dist/gateway/gateway.js +1405 -851
- package/telegram-plugin/gateway/always-allow-persist-queue.ts +2 -2
- package/telegram-plugin/gateway/boot-beacon.ts +9 -2
- package/telegram-plugin/gateway/gateway-heartbeat.ts +4 -3
- package/telegram-plugin/gateway/gateway.ts +42 -15
- package/telegram-plugin/gateway/inbound-router.ts +31 -6
- package/telegram-plugin/gateway/missed-approvals-store.ts +2 -2
- package/telegram-plugin/gateway/pending-card-store.ts +2 -2
- package/telegram-plugin/gateway/privacy-state.ts +2 -2
- package/telegram-plugin/gateway/scoped-grant-store.ts +2 -2
- package/telegram-plugin/gateway/system-message-observer.ts +242 -0
- package/telegram-plugin/gateway/turn-active-marker.ts +3 -4
- package/telegram-plugin/history.ts +178 -11
- package/telegram-plugin/registry/turns-schema.ts +8 -2
- package/telegram-plugin/tests/buzz-mirror.test.ts +12 -1
- package/telegram-plugin/tests/card-history-lane.test.ts +394 -0
- package/telegram-plugin/tests/system-message-observer.test.ts +216 -0
- package/vendor/hindsight-memory/scripts/drain_pending.py +88 -4
- package/vendor/hindsight-memory/scripts/lib/config.py +112 -11
- package/vendor/hindsight-memory/scripts/recall.py +11 -2
- package/vendor/hindsight-memory/scripts/reconcile_tail.py +36 -0
- package/vendor/hindsight-memory/scripts/retain.py +6 -1
- package/vendor/hindsight-memory/scripts/tests/test_config_retain_env.py +99 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_types_filter.py +81 -0
- package/vendor/hindsight-memory/scripts/tests/test_reconcile_durability.py +113 -0
- package/vendor/hindsight-memory/settings.json +2 -2
- package/vendor/hindsight-memory/tests/test_config.py +8 -3
- package/vendor/hindsight-memory/tests/test_hooks.py +10 -1
- package/vendor/hindsight-memory/tests/test_retain_context.py +69 -0
|
@@ -171,6 +171,7 @@ from concurrent.futures import ThreadPoolExecutor
|
|
|
171
171
|
|
|
172
172
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
173
173
|
|
|
174
|
+
from lib import watermark
|
|
174
175
|
from lib.client import HindsightClient
|
|
175
176
|
from lib.config import debug_log, load_config
|
|
176
177
|
from lib.pending import (
|
|
@@ -770,6 +771,72 @@ def _record_failure(
|
|
|
770
771
|
return err_class
|
|
771
772
|
|
|
772
773
|
|
|
774
|
+
def _reconcile_index(entries: list[tuple[str, dict]]) -> dict:
|
|
775
|
+
"""``session_id -> set(abs_path)`` for reconcile-sourced queue entries.
|
|
776
|
+
|
|
777
|
+
Powers the no-loss sibling guard in ``_commit_reconcile_watermark``. Only
|
|
778
|
+
entries reconcile stamped (``reconcile_session_id``, switchroom #4571) are
|
|
779
|
+
indexed; everything else is ignored, so a non-reconcile queue entry for the
|
|
780
|
+
same session never blocks — its turns are covered by the reconcile slice's
|
|
781
|
+
own document, not by it.
|
|
782
|
+
"""
|
|
783
|
+
idx: dict[str, set[str]] = {}
|
|
784
|
+
for path, entry in entries:
|
|
785
|
+
sid = entry.get("reconcile_session_id")
|
|
786
|
+
if sid:
|
|
787
|
+
idx.setdefault(sid, set()).add(os.path.abspath(path))
|
|
788
|
+
return idx
|
|
789
|
+
|
|
790
|
+
|
|
791
|
+
def _commit_reconcile_watermark(entry: dict, path: str, index: dict) -> None:
|
|
792
|
+
"""Advance the transcript watermark for a just-confirmed reconcile entry.
|
|
793
|
+
|
|
794
|
+
switchroom #4571. Called ONLY from a confirmed-durable retire branch — a
|
|
795
|
+
synchronous (``async_processing=False``) retain 200, or a presence GET that
|
|
796
|
+
returned True — so reaching here means this entry's content is durable. A
|
|
797
|
+
no-op for a non-reconcile entry (no ``reconcile_session_id``).
|
|
798
|
+
|
|
799
|
+
NO-LOSS GUARD. The watermark is a single "last contiguously-committed entry"
|
|
800
|
+
pointer, so it may only ever advance to the TRANSCRIPT TAIL, and only once
|
|
801
|
+
NO other slice of the same session is still queued:
|
|
802
|
+
|
|
803
|
+
* ``reconcile_is_tail`` False → an older turn-cap-split remainder that ends
|
|
804
|
+
mid-transcript; advancing to it would skip the newer turns after it.
|
|
805
|
+
* a live sibling in ``index`` → an unconfirmed earlier remainder OR a
|
|
806
|
+
size-split part of this same tail; advancing now could jump the watermark
|
|
807
|
+
past turns that are not yet durable.
|
|
808
|
+
|
|
809
|
+
Either way we leave the watermark and accept a redundant re-enqueue on the
|
|
810
|
+
next boot. ``watermark.commit`` is monotonic + idempotent and refuses
|
|
811
|
+
backward/compacted moves, so over-conservatism only ever costs repeat work,
|
|
812
|
+
never a lost turn — and that is the side to err on.
|
|
813
|
+
"""
|
|
814
|
+
sid = entry.get("reconcile_session_id")
|
|
815
|
+
if not sid:
|
|
816
|
+
return
|
|
817
|
+
live = index.get(sid)
|
|
818
|
+
if live is not None:
|
|
819
|
+
# This entry is now durable + archived; drop it from the live set so a
|
|
820
|
+
# sibling that drains AFTER it is no longer blocked by it.
|
|
821
|
+
live.discard(os.path.abspath(path))
|
|
822
|
+
if not entry.get("reconcile_is_tail"):
|
|
823
|
+
return
|
|
824
|
+
last_uuid = entry.get("reconcile_last_uuid")
|
|
825
|
+
if not last_uuid:
|
|
826
|
+
return
|
|
827
|
+
if live: # an earlier remainder / split part of this session is still queued
|
|
828
|
+
return
|
|
829
|
+
try:
|
|
830
|
+
watermark.commit(
|
|
831
|
+
sid,
|
|
832
|
+
last_uuid,
|
|
833
|
+
entry.get("document_id", ""),
|
|
834
|
+
ordered_uuids=entry.get("reconcile_ordered_uuids"),
|
|
835
|
+
)
|
|
836
|
+
except Exception: # pragma: no cover - a watermark write must never fail a drain
|
|
837
|
+
pass
|
|
838
|
+
|
|
839
|
+
|
|
773
840
|
def _new_summary() -> dict:
|
|
774
841
|
return {
|
|
775
842
|
"drained": 0,
|
|
@@ -892,7 +959,12 @@ def _drain_inhook_impl(config: dict, force: bool = False) -> dict:
|
|
|
892
959
|
# behind them — see ``_drain_order``. Matters even more here than in the
|
|
893
960
|
# backlog drain: the in-hook budget is ~4s, so a single entry at the head
|
|
894
961
|
# that always burns its clamped timeout consumes the entire run.
|
|
895
|
-
|
|
962
|
+
all_entries = iter_entries()
|
|
963
|
+
# Built from the FULL queue (before parking): a parked reconcile entry is
|
|
964
|
+
# still queued and unconfirmed, so it must still block a sibling's watermark
|
|
965
|
+
# advance (#4571).
|
|
966
|
+
reconcile_index = _reconcile_index(all_entries)
|
|
967
|
+
entries = _park_broken(_drain_order(all_entries), summary, force)
|
|
896
968
|
if not entries:
|
|
897
969
|
debug_log(
|
|
898
970
|
config,
|
|
@@ -938,6 +1010,7 @@ def _drain_inhook_impl(config: dict, force: bool = False) -> dict:
|
|
|
938
1010
|
if _document_state(entry, timeout=_clamp(timeout, budget, started)) is True:
|
|
939
1011
|
if archive_reconciled(path):
|
|
940
1012
|
summary["reconciled"] += 1
|
|
1013
|
+
_commit_reconcile_watermark(entry, path, reconcile_index)
|
|
941
1014
|
else:
|
|
942
1015
|
# Archive unwritable: the entry is STILL QUEUED (it is
|
|
943
1016
|
# never deleted), so calling it reconciled would be a lie.
|
|
@@ -1003,6 +1076,7 @@ def _drain_inhook_impl(config: dict, force: bool = False) -> dict:
|
|
|
1003
1076
|
# horizon costs.
|
|
1004
1077
|
if archive_reconciled(path):
|
|
1005
1078
|
summary["drained"] += 1
|
|
1079
|
+
_commit_reconcile_watermark(entry, path, reconcile_index)
|
|
1006
1080
|
else:
|
|
1007
1081
|
summary["archive_failed"] += 1
|
|
1008
1082
|
consecutive_failures = 0
|
|
@@ -1056,7 +1130,7 @@ def _phase_failed(summary: dict, phase: str, e: BaseException, cost: str) -> Non
|
|
|
1056
1130
|
)
|
|
1057
1131
|
|
|
1058
1132
|
|
|
1059
|
-
def _reconcile_phase(config: dict, summary: dict, dry_run: bool) -> None:
|
|
1133
|
+
def _reconcile_phase(config: dict, summary: dict, dry_run: bool, reconcile_index: dict) -> None:
|
|
1060
1134
|
"""PHASE 1 — free pass: drop entries whose document already exists.
|
|
1061
1135
|
|
|
1062
1136
|
This is the phase that makes backlog replay affordable. 70.4% of a
|
|
@@ -1082,8 +1156,11 @@ def _reconcile_phase(config: dict, summary: dict, dry_run: bool) -> None:
|
|
|
1082
1156
|
continue
|
|
1083
1157
|
state = _document_state(entry)
|
|
1084
1158
|
if state is True:
|
|
1085
|
-
if dry_run
|
|
1159
|
+
if dry_run:
|
|
1160
|
+
summary["reconciled"] += 1
|
|
1161
|
+
elif archive_reconciled(path):
|
|
1086
1162
|
summary["reconciled"] += 1
|
|
1163
|
+
_commit_reconcile_watermark(entry, path, reconcile_index)
|
|
1087
1164
|
else:
|
|
1088
1165
|
summary["archive_failed"] += 1
|
|
1089
1166
|
elif state is None:
|
|
@@ -1264,8 +1341,14 @@ def _drain_backlog_impl(
|
|
|
1264
1341
|
f"archived, not deleted"
|
|
1265
1342
|
)
|
|
1266
1343
|
|
|
1344
|
+
# Built AFTER the pre-drain phases (collapse / re-split may have changed the
|
|
1345
|
+
# queue) and shared across the reconcile pass and phase 2, so a sibling that
|
|
1346
|
+
# phase 1 retires is dropped from the live set before phase 2 evaluates the
|
|
1347
|
+
# tail's watermark advance (#4571).
|
|
1348
|
+
reconcile_index = _reconcile_index(iter_entries())
|
|
1349
|
+
|
|
1267
1350
|
if phase in ("reconcile", "both"):
|
|
1268
|
-
_reconcile_phase(config, summary, dry_run)
|
|
1351
|
+
_reconcile_phase(config, summary, dry_run, reconcile_index)
|
|
1269
1352
|
if phase == "reconcile":
|
|
1270
1353
|
return summary
|
|
1271
1354
|
|
|
@@ -1356,6 +1439,7 @@ def _drain_backlog_impl(
|
|
|
1356
1439
|
if _document_state(entry) is True:
|
|
1357
1440
|
if archive_reconciled(path):
|
|
1358
1441
|
summary["drained"] += 1
|
|
1442
|
+
_commit_reconcile_watermark(entry, path, reconcile_index)
|
|
1359
1443
|
else:
|
|
1360
1444
|
summary["archive_failed"] += 1
|
|
1361
1445
|
else:
|
|
@@ -48,16 +48,19 @@ DEFAULT_VOLATILE_SCOPE_PATTERNS = (
|
|
|
48
48
|
DEFAULTS = {
|
|
49
49
|
# Recall
|
|
50
50
|
"autoRecall": True,
|
|
51
|
-
# Switchroom default: "
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
#
|
|
55
|
-
#
|
|
56
|
-
#
|
|
57
|
-
#
|
|
58
|
-
#
|
|
59
|
-
#
|
|
60
|
-
"
|
|
51
|
+
# Switchroom fleet default: "mid". The budget sets candidate DEPTH — how
|
|
52
|
+
# many nodes the engine pulls across all TEMPR retrieval stages before
|
|
53
|
+
# ranking (recall_budget_fixed_low=100 / _mid=300 / _high=1000 units,
|
|
54
|
+
# hindsight_api engine/memory_engine.py). It does NOT gate the reranker:
|
|
55
|
+
# the cross-encoder runs at EVERY budget level and is bounded separately by
|
|
56
|
+
# RERANKER_MAX_CANDIDATES (HINDSIGHT_API_RERANKER_MAX_CANDIDATES, default
|
|
57
|
+
# 300), so "low" is not "vector-only, no rerank" — it is a shallower
|
|
58
|
+
# candidate pool feeding the same rerank+score pipeline. "mid" (300 nodes)
|
|
59
|
+
# is upstream's own default and the balanced point: deeper recall than
|
|
60
|
+
# "low" without "high"'s 1000-node cold-page tail. Operators who want the
|
|
61
|
+
# shallow/fast pool back set HINDSIGHT_RECALL_BUDGET=low via per-agent env
|
|
62
|
+
# or write `recallBudget: "low"` into the user config file.
|
|
63
|
+
"recallBudget": "mid",
|
|
61
64
|
"recallMaxTokens": 1024,
|
|
62
65
|
# Switchroom-local: cap on the number of memories injected into the
|
|
63
66
|
# `<hindsight_memories>` block, regardless of token budget. Plugin v0.4.0
|
|
@@ -206,7 +209,17 @@ DEFAULTS = {
|
|
|
206
209
|
"retainEveryNTurns": 10,
|
|
207
210
|
"retainOverlapTurns": 2,
|
|
208
211
|
"retainToolCalls": True,
|
|
209
|
-
|
|
212
|
+
# Switchroom — speaker-aware retain context. Resolved per-retain via
|
|
213
|
+
# build_retain_payload's _resolve_template, which fills {agent} from
|
|
214
|
+
# SWITCHROOM_AGENT_NAME and {bank_id} from the target bank. Tells the
|
|
215
|
+
# consolidation LLM who is speaking on each line so first-person agent
|
|
216
|
+
# actions ("experience") are not confused with the operator's world facts.
|
|
217
|
+
"retainContext": (
|
|
218
|
+
"Transcript of Claude Code agent '{agent}' ({bank_id}). "
|
|
219
|
+
"'assistant'/tool lines are the agent's own first-person actions "
|
|
220
|
+
"(experience); 'user' lines are the human operator speaking (their "
|
|
221
|
+
"statements are world facts)."
|
|
222
|
+
),
|
|
210
223
|
"retainTags": [],
|
|
211
224
|
"retainMetadata": {},
|
|
212
225
|
# Switchroom-local: per-row Hindsight `observation_scopes` on every retain.
|
|
@@ -396,6 +409,23 @@ ENV_OVERRIDES = {
|
|
|
396
409
|
"HINDSIGHT_AUTO_RECALL": ("autoRecall", bool),
|
|
397
410
|
"HINDSIGHT_AUTO_RETAIN": ("autoRetain", bool),
|
|
398
411
|
"HINDSIGHT_RETAIN_MODE": ("retainMode", str),
|
|
412
|
+
# Switchroom-local: auto-retain cadence knobs. These had a DEFAULTS entry and
|
|
413
|
+
# a settings.json stamp (applyHindsightSettingsOverrides) but NO env channel,
|
|
414
|
+
# so env — the TOP of the config precedence chain (DEFAULTS → settings.json →
|
|
415
|
+
# ~/.hindsight/claude-code.json → env) — could not reach them at all. That
|
|
416
|
+
# broke parity with the recall knobs and left the only override paths as a
|
|
417
|
+
# settings.json rewrite (scaffold-time) or a hand-edit that `switchroom apply`
|
|
418
|
+
# re-copies away. Adding the env keys lets `memory.retain.*` (or an agent
|
|
419
|
+
# `env:` map, or a docker-exec'd retain that does not inherit the supervised
|
|
420
|
+
# env) drive them, and makes the env value authoritative when set.
|
|
421
|
+
# `retainEveryNTurns` / `retainOverlapTurns` mirror the yaml surface
|
|
422
|
+
# (`memory.retain.every_n_turns` / `.overlap_turns`); `retainContext` /
|
|
423
|
+
# `retainTags` have no yaml surface yet but gain the same env channel as the
|
|
424
|
+
# other retain knobs for consistency and exec-path overrides.
|
|
425
|
+
"HINDSIGHT_RETAIN_EVERY_N_TURNS": ("retainEveryNTurns", int),
|
|
426
|
+
"HINDSIGHT_RETAIN_OVERLAP_TURNS": ("retainOverlapTurns", int),
|
|
427
|
+
"HINDSIGHT_RETAIN_CONTEXT": ("retainContext", str),
|
|
428
|
+
"HINDSIGHT_RETAIN_TAGS": ("retainTags", list),
|
|
399
429
|
# Switchroom-local: per-row observation scope on retains. Set by start.sh
|
|
400
430
|
# from agents.<name>.memory.observation_scopes (cascading through
|
|
401
431
|
# defaults.memory.observation_scopes) ONLY when the operator set it; unset
|
|
@@ -520,6 +550,77 @@ ENV_OVERRIDES = {
|
|
|
520
550
|
OBSERVATION_SCOPES_VALUES = ("per_tag", "combined", "all_combinations", "shared")
|
|
521
551
|
|
|
522
552
|
|
|
553
|
+
#: Switchroom-local: the fact types Hindsight's recall endpoint accepts. Sending
|
|
554
|
+
#: any other value makes the 0.9.0 engine return HTTP 422 ("Invalid fact type(s):
|
|
555
|
+
#: … Must be one of: experience, observation, world") — a validation that was
|
|
556
|
+
#: silently tolerated before vectorize-io/hindsight#3062. A 422 fails the WHOLE
|
|
557
|
+
#: recall for the turn, so an operator typo in `memory.recall.types` (e.g.
|
|
558
|
+
#: "observations", "fact") would otherwise kill memory injection on EVERY turn.
|
|
559
|
+
#: Verified against the live engine's 422 detail string and /openapi.json
|
|
560
|
+
#: RecallRequest; widening this set means widening it server-side too.
|
|
561
|
+
RECALL_FACT_TYPES = ("world", "experience", "observation")
|
|
562
|
+
|
|
563
|
+
#: The recall types used when a configured `recallTypes` filters down to empty
|
|
564
|
+
#: (mirrors the `recallTypes` DEFAULTS entry). Falling back to this — rather than
|
|
565
|
+
#: sending an empty/invalid set — keeps recall running with the shipped behaviour
|
|
566
|
+
#: instead of degrading to nothing.
|
|
567
|
+
DEFAULT_RECALL_FACT_TYPES = ("world", "experience")
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def filter_recall_types(config: dict):
|
|
571
|
+
"""Filter ``recallTypes`` to the set Hindsight's recall endpoint accepts.
|
|
572
|
+
|
|
573
|
+
Returns the value to send as the recall ``types`` argument. THIS FUNCTION
|
|
574
|
+
MUST NEVER RAISE: an invalid ``memory.recall.types`` value is a
|
|
575
|
+
misconfiguration, and both raising here and passing the bad value through
|
|
576
|
+
have the same catastrophic outcome — a 422 that fails the recall and drops
|
|
577
|
+
memory injection for the turn. This mirrors the degrade-don't-raise contract
|
|
578
|
+
of :func:`compute_observation_scopes` (a bad config degrades the FEATURE,
|
|
579
|
+
never loses the turn).
|
|
580
|
+
|
|
581
|
+
* ``None`` / unset → ``None``: omit the field entirely, letting the engine
|
|
582
|
+
apply its own default (world + experience). Byte-identical to the wire
|
|
583
|
+
body a pre-filter client sent.
|
|
584
|
+
* a list/tuple → keep the members in :data:`RECALL_FACT_TYPES` (order
|
|
585
|
+
and de-duplicated), dropping every unknown value WITH a stderr warning
|
|
586
|
+
that names it. If nothing valid survives, fall back to
|
|
587
|
+
:data:`DEFAULT_RECALL_FACT_TYPES` so recall still runs.
|
|
588
|
+
* anything else → shout and fall back to :data:`DEFAULT_RECALL_FACT_TYPES`.
|
|
589
|
+
"""
|
|
590
|
+
raw = config.get("recallTypes")
|
|
591
|
+
if raw is None:
|
|
592
|
+
return None
|
|
593
|
+
if not isinstance(raw, (list, tuple)):
|
|
594
|
+
print(
|
|
595
|
+
f"[Hindsight] recallTypes={raw!r} is not a list; falling back to "
|
|
596
|
+
f"{list(DEFAULT_RECALL_FACT_TYPES)}. Set it via `memory.recall.types` "
|
|
597
|
+
"in switchroom.yaml.",
|
|
598
|
+
file=sys.stderr,
|
|
599
|
+
)
|
|
600
|
+
return list(DEFAULT_RECALL_FACT_TYPES)
|
|
601
|
+
valid = []
|
|
602
|
+
for fact_type in raw:
|
|
603
|
+
if isinstance(fact_type, str) and fact_type in RECALL_FACT_TYPES:
|
|
604
|
+
if fact_type not in valid:
|
|
605
|
+
valid.append(fact_type)
|
|
606
|
+
else:
|
|
607
|
+
print(
|
|
608
|
+
f"[Hindsight] recallTypes entry {fact_type!r} is not a valid "
|
|
609
|
+
f"Hindsight fact type ({', '.join(RECALL_FACT_TYPES)}); dropping "
|
|
610
|
+
"it. An invalid type 422s the recall and drops memory injection "
|
|
611
|
+
"for the turn — fix it via `memory.recall.types` in switchroom.yaml.",
|
|
612
|
+
file=sys.stderr,
|
|
613
|
+
)
|
|
614
|
+
if not valid:
|
|
615
|
+
print(
|
|
616
|
+
f"[Hindsight] recallTypes={raw!r} left no valid fact types after "
|
|
617
|
+
f"filtering; falling back to {list(DEFAULT_RECALL_FACT_TYPES)}.",
|
|
618
|
+
file=sys.stderr,
|
|
619
|
+
)
|
|
620
|
+
return list(DEFAULT_RECALL_FACT_TYPES)
|
|
621
|
+
return valid
|
|
622
|
+
|
|
623
|
+
|
|
523
624
|
def classify_observation_scopes(config: dict):
|
|
524
625
|
"""Classify ``observationScopes`` WITHOUT raising: ``(value, error)``.
|
|
525
626
|
|
|
@@ -59,7 +59,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
|
59
59
|
|
|
60
60
|
from lib.bank import derive_bank_id, ensure_bank_mission
|
|
61
61
|
from lib.client import HindsightClient
|
|
62
|
-
from lib.config import debug_log, load_config
|
|
62
|
+
from lib.config import debug_log, filter_recall_types, load_config
|
|
63
63
|
from lib.content import (
|
|
64
64
|
_extract_text_content,
|
|
65
65
|
compose_recall_query,
|
|
@@ -2091,6 +2091,15 @@ def main():
|
|
|
2091
2091
|
if recall_request_timeout <= 0:
|
|
2092
2092
|
recall_request_timeout = 12.0
|
|
2093
2093
|
|
|
2094
|
+
# Fail-safe the recall `types` BEFORE any bank task runs: the 0.9.0 engine
|
|
2095
|
+
# 422s an invalid fact type (e.g. an operator typo "observations"/"fact" in
|
|
2096
|
+
# memory.recall.types), which fails the whole recall and drops memory
|
|
2097
|
+
# injection for the turn. filter_recall_types drops unknowns (shouting on
|
|
2098
|
+
# stderr) and falls back to the default set if the filter empties it — it
|
|
2099
|
+
# never raises. Computed once here so every bank in the fan-out sends the
|
|
2100
|
+
# same validated set.
|
|
2101
|
+
resolved_recall_types = filter_recall_types(config)
|
|
2102
|
+
|
|
2094
2103
|
def _make_bank_task(target_bank_id, b_tags, b_tags_match, b_tag_groups, timeout_override=None):
|
|
2095
2104
|
def _bank_task():
|
|
2096
2105
|
return client.recall(
|
|
@@ -2098,7 +2107,7 @@ def main():
|
|
|
2098
2107
|
query=search_query,
|
|
2099
2108
|
max_tokens=config.get("recallMaxTokens", 1024),
|
|
2100
2109
|
budget=config.get("recallBudget", "mid"),
|
|
2101
|
-
types=
|
|
2110
|
+
types=resolved_recall_types,
|
|
2102
2111
|
# Upstream 962140eef — optional per-bank tag filters (resolved
|
|
2103
2112
|
# above the cache check; part of the cache key).
|
|
2104
2113
|
tags=b_tags,
|
|
@@ -127,6 +127,36 @@ def _session_id_from_path(path: str) -> str:
|
|
|
127
127
|
return os.path.splitext(os.path.basename(path))[0]
|
|
128
128
|
|
|
129
129
|
|
|
130
|
+
def _annotate_reconcile_payload(payload: dict, session_id: str, built: dict) -> dict:
|
|
131
|
+
"""Stamp watermark-anchoring fields onto a QUEUED reconcile payload.
|
|
132
|
+
|
|
133
|
+
switchroom #4571 — the recurring ``pending-retains`` spike. A slice that
|
|
134
|
+
reconcile enqueues (over-budget / out-of-lookback) or defers (turn-cap
|
|
135
|
+
split) is later drained by ``drain_pending.py`` on a confirmed 200, but the
|
|
136
|
+
drain had no way to know WHICH session/uuid the entry anchored, so it never
|
|
137
|
+
advanced the transcript watermark for these paths. Reconcile's gap detection
|
|
138
|
+
is watermark-keyed, so on the next boot it re-derives the identical tail and
|
|
139
|
+
re-enqueues it — a false-alarm queue spike that recurs every boot (no memory
|
|
140
|
+
is lost: enqueue dedupe + the archive protect it, but the depth trips the
|
|
141
|
+
memory-queue watchdog). These fields let the drain advance the watermark once
|
|
142
|
+
the slice is confirmed durable, under the no-loss guard in
|
|
143
|
+
``drain_pending._commit_reconcile_watermark``.
|
|
144
|
+
|
|
145
|
+
``reconcile_is_tail`` is the load-bearing one: only a slice whose last uuid
|
|
146
|
+
is the TRANSCRIPT tail may ever anchor the watermark. An older remainder
|
|
147
|
+
(turn-cap split) ends mid-transcript, so advancing to it would skip the
|
|
148
|
+
newer turns that follow it.
|
|
149
|
+
"""
|
|
150
|
+
ordered = built.get("ordered_uuids") or []
|
|
151
|
+
last_uuid = built.get("last_uuid")
|
|
152
|
+
tail_uuid = ordered[-1] if ordered else None
|
|
153
|
+
payload["reconcile_session_id"] = session_id
|
|
154
|
+
payload["reconcile_last_uuid"] = last_uuid
|
|
155
|
+
payload["reconcile_ordered_uuids"] = ordered
|
|
156
|
+
payload["reconcile_is_tail"] = bool(last_uuid) and last_uuid == tail_uuid
|
|
157
|
+
return payload
|
|
158
|
+
|
|
159
|
+
|
|
130
160
|
def reconcile(config: dict | None = None, hook_input: dict | None = None) -> dict:
|
|
131
161
|
"""Diff watermarks vs transcript tails and recover un-committed work.
|
|
132
162
|
|
|
@@ -297,6 +327,11 @@ def _post_inline(
|
|
|
297
327
|
return "skip"
|
|
298
328
|
payload = built["payload"]
|
|
299
329
|
document_id = built["document_id"]
|
|
330
|
+
# Stamp the watermark-anchoring fields so that IF this slice falls through
|
|
331
|
+
# to the pending queue (lock busy / POST failed) the drain can advance the
|
|
332
|
+
# watermark once it lands, instead of leaving reconcile to re-enqueue it
|
|
333
|
+
# every boot (switchroom #4571).
|
|
334
|
+
_annotate_reconcile_payload(payload, session_id, built)
|
|
300
335
|
|
|
301
336
|
with inflight_lock(blocking=True) as acquired:
|
|
302
337
|
if not acquired: # pragma: no cover - blocking acquire fails open
|
|
@@ -350,6 +385,7 @@ def _enqueue_slice(config, session_id, path, all_messages, slice_messages, bank_
|
|
|
350
385
|
)
|
|
351
386
|
if built is None:
|
|
352
387
|
return False
|
|
388
|
+
_annotate_reconcile_payload(built["payload"], session_id, built)
|
|
353
389
|
queued = pending_enqueue(built["payload"], RuntimeError("reconcile deferred (bound/budget)"))
|
|
354
390
|
if queued is None:
|
|
355
391
|
debug_log(config, "reconcile_tail: pending-retains full, could not enqueue remainder")
|
|
@@ -549,6 +549,11 @@ def build_retain_payload(
|
|
|
549
549
|
"bank_id": bank_id,
|
|
550
550
|
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
551
551
|
"user_id": os.environ.get("HINDSIGHT_USER_ID", ""),
|
|
552
|
+
# Switchroom — the agent's own name, so a speaker-aware retainContext
|
|
553
|
+
# template can name whose first-person experience this transcript is.
|
|
554
|
+
# Empty outside switchroom (no SWITCHROOM_AGENT_NAME); a template that
|
|
555
|
+
# references {agent} then renders an empty slot, which is harmless.
|
|
556
|
+
"agent": os.environ.get("SWITCHROOM_AGENT_NAME", ""),
|
|
552
557
|
}
|
|
553
558
|
|
|
554
559
|
def _resolve_template(value: str) -> str:
|
|
@@ -682,7 +687,7 @@ def build_retain_payload(
|
|
|
682
687
|
"bank_id": bank_id,
|
|
683
688
|
"content": transcript,
|
|
684
689
|
"document_id": document_id,
|
|
685
|
-
"context": config.get("retainContext", "claude-code"),
|
|
690
|
+
"context": _resolve_template(config.get("retainContext", "claude-code")),
|
|
686
691
|
"metadata": metadata,
|
|
687
692
|
"tags": tags,
|
|
688
693
|
"observation_scopes": scope,
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Switchroom — the auto-retain cadence knobs must have an env channel.
|
|
2
|
+
|
|
3
|
+
`retainEveryNTurns`, `retainOverlapTurns`, `retainContext`, and `retainTags`
|
|
4
|
+
had a DEFAULTS entry and (for the cadence pair) a settings.json stamp, but NO
|
|
5
|
+
entry in `ENV_OVERRIDES`. Env is the TOP of the plugin's config precedence
|
|
6
|
+
chain (DEFAULTS -> settings.json -> ~/.hindsight/claude-code.json -> env), so
|
|
7
|
+
without an env key an operator's `HINDSIGHT_RETAIN_*` could not reach the
|
|
8
|
+
plugin at all, and a docker-exec'd retain that does not inherit the supervised
|
|
9
|
+
settings could not be steered either.
|
|
10
|
+
|
|
11
|
+
The outcome under test: `HINDSIGHT_RETAIN_EVERY_N_TURNS` (and its siblings)
|
|
12
|
+
actually OVERRIDE the resolved config value, and env wins over the shipped
|
|
13
|
+
default.
|
|
14
|
+
|
|
15
|
+
Stdlib-only.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import os
|
|
19
|
+
import sys
|
|
20
|
+
import unittest
|
|
21
|
+
from unittest import mock
|
|
22
|
+
|
|
23
|
+
SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
|
24
|
+
if SCRIPTS_DIR not in sys.path:
|
|
25
|
+
sys.path.insert(0, SCRIPTS_DIR)
|
|
26
|
+
|
|
27
|
+
from lib.config import DEFAULTS, ENV_OVERRIDES, load_config # noqa: E402
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# Every retain-cadence env var, with the config key it must reach.
|
|
31
|
+
RETAIN_ENV = {
|
|
32
|
+
"HINDSIGHT_RETAIN_EVERY_N_TURNS": "retainEveryNTurns",
|
|
33
|
+
"HINDSIGHT_RETAIN_OVERLAP_TURNS": "retainOverlapTurns",
|
|
34
|
+
"HINDSIGHT_RETAIN_CONTEXT": "retainContext",
|
|
35
|
+
"HINDSIGHT_RETAIN_TAGS": "retainTags",
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _load_with(env):
|
|
40
|
+
"""load_config() with a hermetic environment (no plugin/user settings)."""
|
|
41
|
+
with mock.patch.dict(os.environ, env, clear=True):
|
|
42
|
+
os.environ["CLAUDE_PLUGIN_ROOT"] = os.path.join(SCRIPTS_DIR, "does-not-exist")
|
|
43
|
+
os.environ["HOME"] = os.path.join(SCRIPTS_DIR, "does-not-exist")
|
|
44
|
+
return load_config()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class EveryRetainNameHasAChannel(unittest.TestCase):
|
|
48
|
+
def test_all_retain_env_names_are_wired(self):
|
|
49
|
+
missing = [name for name in RETAIN_ENV if name not in ENV_OVERRIDES]
|
|
50
|
+
self.assertEqual(missing, [], f"exported but never read: {missing}")
|
|
51
|
+
|
|
52
|
+
def test_each_name_maps_to_the_expected_config_key(self):
|
|
53
|
+
for name, key in RETAIN_ENV.items():
|
|
54
|
+
with self.subTest(name=name):
|
|
55
|
+
self.assertEqual(ENV_OVERRIDES[name][0], key)
|
|
56
|
+
|
|
57
|
+
def test_every_target_key_exists_in_defaults(self):
|
|
58
|
+
for name, key in RETAIN_ENV.items():
|
|
59
|
+
with self.subTest(name=name):
|
|
60
|
+
self.assertIn(key, DEFAULTS)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class ValuesActuallyLand(unittest.TestCase):
|
|
64
|
+
"""The outcome that matters: the loaded config carries the exported value."""
|
|
65
|
+
|
|
66
|
+
def test_every_n_turns_env_overrides_the_resolved_value(self):
|
|
67
|
+
# The headline outcome: HINDSIGHT_RETAIN_EVERY_N_TURNS wins over the
|
|
68
|
+
# shipped default, and lands as an int.
|
|
69
|
+
override = DEFAULTS["retainEveryNTurns"] + 5
|
|
70
|
+
cfg = _load_with({"HINDSIGHT_RETAIN_EVERY_N_TURNS": str(override)})
|
|
71
|
+
self.assertEqual(cfg["retainEveryNTurns"], override)
|
|
72
|
+
self.assertIsInstance(cfg["retainEveryNTurns"], int)
|
|
73
|
+
self.assertNotEqual(cfg["retainEveryNTurns"], DEFAULTS["retainEveryNTurns"])
|
|
74
|
+
|
|
75
|
+
def test_overlap_turns_env_overrides_the_resolved_value(self):
|
|
76
|
+
cfg = _load_with({"HINDSIGHT_RETAIN_OVERLAP_TURNS": "4"})
|
|
77
|
+
self.assertEqual(cfg["retainOverlapTurns"], 4)
|
|
78
|
+
|
|
79
|
+
def test_context_env_overrides_the_resolved_value(self):
|
|
80
|
+
cfg = _load_with({"HINDSIGHT_RETAIN_CONTEXT": "codex"})
|
|
81
|
+
self.assertEqual(cfg["retainContext"], "codex")
|
|
82
|
+
|
|
83
|
+
def test_tags_env_accepts_a_json_array(self):
|
|
84
|
+
cfg = _load_with({"HINDSIGHT_RETAIN_TAGS": '["source:transcript"]'})
|
|
85
|
+
self.assertEqual(cfg["retainTags"], ["source:transcript"])
|
|
86
|
+
|
|
87
|
+
def test_tags_env_accepts_a_comma_separated_list(self):
|
|
88
|
+
cfg = _load_with({"HINDSIGHT_RETAIN_TAGS": "a,b"})
|
|
89
|
+
self.assertEqual(cfg["retainTags"], ["a", "b"])
|
|
90
|
+
|
|
91
|
+
def test_no_retain_env_reproduces_the_default(self):
|
|
92
|
+
baseline = _load_with({})
|
|
93
|
+
for key in RETAIN_ENV.values():
|
|
94
|
+
with self.subTest(key=key):
|
|
95
|
+
self.assertEqual(baseline[key], DEFAULTS[key])
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
if __name__ == "__main__":
|
|
99
|
+
unittest.main()
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Switchroom — recall `types` must be fail-safe against an invalid fact type.
|
|
2
|
+
|
|
3
|
+
The 0.9.0 Hindsight engine returns HTTP 422 ("Invalid fact type(s): … Must be
|
|
4
|
+
one of: experience, observation, world") for an unknown recall `fact_type`,
|
|
5
|
+
which fails the WHOLE recall for the turn. Before this guard, recall.py sent
|
|
6
|
+
`types=config.get("recallTypes")` unvalidated, so an operator typo in
|
|
7
|
+
`memory.recall.types` (e.g. "observations", "fact") would 422 and drop memory
|
|
8
|
+
injection on every turn.
|
|
9
|
+
|
|
10
|
+
The outcome under test: a config carrying an invalid type resolves — via
|
|
11
|
+
`filter_recall_types` — to a FILTERED, valid `types` list (never a 422-bound
|
|
12
|
+
call), and never raises. Mirrors the degrade-don't-raise contract of
|
|
13
|
+
`compute_observation_scopes`.
|
|
14
|
+
|
|
15
|
+
Stdlib-only.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import os
|
|
19
|
+
import sys
|
|
20
|
+
import unittest
|
|
21
|
+
|
|
22
|
+
SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
|
23
|
+
if SCRIPTS_DIR not in sys.path:
|
|
24
|
+
sys.path.insert(0, SCRIPTS_DIR)
|
|
25
|
+
|
|
26
|
+
from lib.config import ( # noqa: E402
|
|
27
|
+
DEFAULT_RECALL_FACT_TYPES,
|
|
28
|
+
RECALL_FACT_TYPES,
|
|
29
|
+
filter_recall_types,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class FilterRecallTypes(unittest.TestCase):
|
|
34
|
+
def test_valid_set_matches_the_engine(self):
|
|
35
|
+
# Guards against client/server drift: verified against the live engine's
|
|
36
|
+
# own 422 detail string ("experience, observation, world").
|
|
37
|
+
self.assertEqual(set(RECALL_FACT_TYPES), {"world", "experience", "observation"})
|
|
38
|
+
|
|
39
|
+
def test_invalid_type_is_dropped_leaving_a_valid_list(self):
|
|
40
|
+
# The exact defect: an operator typo mixed with a valid type. The bad
|
|
41
|
+
# value is dropped; the good one survives — NOT a 422-bound call.
|
|
42
|
+
resolved = filter_recall_types({"recallTypes": ["observations", "world"]})
|
|
43
|
+
self.assertEqual(resolved, ["world"])
|
|
44
|
+
for t in resolved:
|
|
45
|
+
self.assertIn(t, RECALL_FACT_TYPES)
|
|
46
|
+
|
|
47
|
+
def test_all_invalid_falls_back_to_the_default_set(self):
|
|
48
|
+
# If nothing valid survives, recall must still RUN — fall back to the
|
|
49
|
+
# shipped default rather than send an empty/invalid set.
|
|
50
|
+
resolved = filter_recall_types({"recallTypes": ["fact", "observations"]})
|
|
51
|
+
self.assertEqual(resolved, list(DEFAULT_RECALL_FACT_TYPES))
|
|
52
|
+
|
|
53
|
+
def test_valid_types_pass_through_deduplicated_in_order(self):
|
|
54
|
+
resolved = filter_recall_types(
|
|
55
|
+
{"recallTypes": ["world", "experience", "world", "observation"]}
|
|
56
|
+
)
|
|
57
|
+
self.assertEqual(resolved, ["world", "experience", "observation"])
|
|
58
|
+
|
|
59
|
+
def test_unset_returns_none_so_the_field_is_omitted(self):
|
|
60
|
+
# None -> omit `types` entirely, letting the engine apply its own default.
|
|
61
|
+
self.assertIsNone(filter_recall_types({}))
|
|
62
|
+
self.assertIsNone(filter_recall_types({"recallTypes": None}))
|
|
63
|
+
|
|
64
|
+
def test_non_list_value_falls_back_without_raising(self):
|
|
65
|
+
# A scalar where a list was expected is a config mistake, not a crash.
|
|
66
|
+
resolved = filter_recall_types({"recallTypes": "observation"})
|
|
67
|
+
self.assertEqual(resolved, list(DEFAULT_RECALL_FACT_TYPES))
|
|
68
|
+
|
|
69
|
+
def test_non_string_members_are_dropped(self):
|
|
70
|
+
resolved = filter_recall_types({"recallTypes": [None, 42, "observation"]})
|
|
71
|
+
self.assertEqual(resolved, ["observation"])
|
|
72
|
+
|
|
73
|
+
def test_never_raises_on_pathological_input(self):
|
|
74
|
+
for bad in ({}, {"recallTypes": {}}, {"recallTypes": 0}, {"recallTypes": [[]]}):
|
|
75
|
+
with self.subTest(bad=bad):
|
|
76
|
+
# Must return a value, never propagate an exception.
|
|
77
|
+
filter_recall_types(bad)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
if __name__ == "__main__":
|
|
81
|
+
unittest.main()
|