switchroom 0.21.14 → 0.21.16

package/vendor/hindsight-memory/scripts/retain.py

@@ -366,6 +366,7 @@ def select_retain_window(
     overlap_turns: int,
     all_messages: list,
     force: bool = False,
+    since_uuid: str | None = None,
 ) -> tuple:
     """Decide which messages to retain and whether to send as a full window.
 
@@ -396,6 +397,17 @@ def select_retain_window(
     the whole session even if per-turn windowing had an edge. This costs a
     full sweep only ONCE per session (at end), not per turn.
 
+    ``since_uuid`` (memory-RFC P1) makes that final sweep INCREMENTAL: when
+    ``force=True`` and a committed watermark uuid is supplied, only the
+    transcript tail after it is retained, instead of re-storing the whole
+    session on top of the per-window documents that already carry it. The value
+    is resolved by the caller (``run_retain`` reads the watermark, wrapped) and
+    passed in — this function stays pure and IO-free. ``since_uuid=None``
+    reproduces the whole-session sweep exactly, which is what the vendor
+    full-session path and the ``retainEveryNTurns==1`` path pass. The slice
+    degrades to the whole transcript on a missing/compacted anchor
+    (``watermark.tail_after``), so a forced sweep never emits an empty slice.
+
     Durability invariant (jtbd-memory-survives-restart UAT): the window
     always extends to the END of the transcript (``slice_last_turns_by_user_boundary``
     returns ``messages[start:]``), so the turn that just completed — the one
@@ -412,8 +424,21 @@ def select_retain_window(
         window_turns = max(retain_every_n, 1) + overlap_turns
         messages_to_retain = slice_last_turns_by_user_boundary(all_messages, window_turns)
         return messages_to_retain, True
+    # Forced (SessionEnd) chunked sweep with a committed watermark uuid: retain
+    # only the transcript tail after it, instead of re-storing the whole session
+    # on top of the N per-window documents that already carry the same content
+    # (memory-RFC P1). ``since_uuid`` is None on the vendor full-session path and
+    # at retainEveryNTurns==1 (the caller withholds it there — see run_retain),
+    # which preserves today's whole-session behaviour byte-for-byte.
+    #
+    # tail_after is PURE and IO-free (the watermark READ happens in run_retain,
+    # wrapped) and degrades to the whole transcript on a missing/compacted
+    # anchor, so this branch can never emit an empty slice into the retain seam.
+    if force and since_uuid is not None:
+        return watermark.tail_after(all_messages, since_uuid), True
     # Full session: vendor full-session mode, OR a forced (SessionEnd) chunked
-    # sweep. Retain all messages, always as a full window.
+    # sweep with no watermark to slice against. Retain all messages, always as a
+    # full window.
     return list(all_messages), True
 
 
@@ -702,7 +727,7 @@ def build_retain_payload(
     }
 
 
-def run_retain(hook_input: dict, force: bool = False) -> dict:
+def run_retain(hook_input: dict, force: bool = False, delta: bool = False) -> dict:
     """Run the auto-retain flow.
 
     Returns a status dict::
@@ -717,6 +742,22 @@ def run_retain(hook_input: dict, force: bool = False) -> dict:
     process (the SessionStart drainer, see ``drain_pending.py`` and
     ``lib/pending.py``). ``status="skipped"`` is the normal early-exit
     cases (auto-retain disabled, empty transcript, throttled chunk).
+
+    ``delta`` (M4 P-RET, per-turn delta retain — called ONLY from
+    ``prefetch.py``, itself gated on the ``memoryPrefetchEnabled`` kill
+    switch, default off): bypasses ``retainEveryNTurns`` entirely (the
+    caller already decided this turn fires) and slices ONLY the
+    transcript tail after the committed watermark. Ignored when
+    ``force=True`` (SessionEnd always takes the force path below).
+
+    **Document-id safety invariant (red-team M4 Finding A, BINDING):** a
+    delta retain is ALWAYS posted under a distinct, content-derived
+    ``document_id`` — the SAME strategy the ``retainEveryNTurns>1``
+    chunked path already uses — and NEVER under the bare ``{session_id}``
+    id the full-session/n==1 path reserves. Posting a tail slice under
+    ``{session_id}`` would UPSERT-OVERWRITE the whole session's stored
+    memory with just the latest turn on every successful turn (see
+    ``retain.py`` §4.3 note below and ``tests/test_retain_delta.py``).
     """
     # /private-mode enforcement (switchroom privacy PR1) — FIRST, before
     # load_config(). An OPEN private interval means the operator is discussing
@@ -785,35 +826,109 @@ def run_retain(hook_input: dict, force: bool = False) -> dict:
     retain_every_n = max(1, config.get("retainEveryNTurns", 1))
     retain_full_window = False
     messages_to_retain = all_messages
-
-    # Respect retainEveryNTurns in both modes, unless force=True (SessionEnd final retain)
-    if retain_every_n > 1 and not force:
-        turn_count = increment_turn_count(session_id)
-        if turn_count % retain_every_n != 0:
-            next_at = ((turn_count // retain_every_n) + 1) * retain_every_n
-            debug_log(config, f"Turn {turn_count}/{retain_every_n}, skipping retain (next at turn {next_at})")
-            return {"status": "skipped", "reason": "throttled"}
-
-    # Window selection is decoupled from the throttle above — see
-    # select_retain_window() for the switchroom-divergence rationale
-    # (Phase 6b: chunked window-slicing now works at retainEveryNTurns=1).
-    overlap_turns = config.get("retainOverlapTurns", 0)
-    messages_to_retain, retain_full_window = select_retain_window(
-        retain_mode, retain_every_n, overlap_turns, all_messages, force=force
-    )
-    if retain_mode == "chunked" and not force:
-        window_turns = max(retain_every_n, 1) + overlap_turns
-        debug_log(
-            config,
-            f"Chunked retain firing (window: {window_turns} human turns, {len(messages_to_retain)} messages)",
-        )
-    elif retain_mode == "chunked" and force:
+    since_uuid = None
+    # True only for the per-turn delta path (M4 P-RET) — threads a distinct,
+    # content-derived document_id below (Finding A) instead of the
+    # retainEveryNTurns>1 / n==1 strategy.
+    delta_document_id_override = False
+
+    if delta and not force:
+        # M4 P-RET — per-turn delta retain, called ONLY from prefetch.py
+        # (itself gated on memoryPrefetchEnabled, default off). Bypasses the
+        # retainEveryNTurns throttle entirely: the caller already decided
+        # this turn fires, every turn, superseding the P5 cadence on a
+        # flipped agent (config comment mandated at the switchroom.yaml
+        # seam — see P-CFG). Slices ONLY the transcript tail after the
+        # committed watermark; the §4.3 HAZARD discipline applies here too —
+        # any watermark-read failure MUST degrade to the full window, never
+        # raise, never emit an empty slice.
+        try:
+            _wm = watermark.load(session_id)
+            since_uuid = _wm.get("last_uuid") if _wm else None
+        except Exception as e:  # noqa: BLE001 - degrade, never raise into this seam
+            print(
+                "[Hindsight] watermark read failed for delta retain; "
+                f"falling back to full-window slice: {e}",
+                file=sys.stderr,
+            )
+            since_uuid = None
+        messages_to_retain = watermark.tail_after(all_messages, since_uuid)
+        retain_full_window = True
+        delta_document_id_override = True
         debug_log(
             config,
-            f"Chunked retain, forced full-session sweep (SessionEnd): {len(all_messages)} messages",
+            f"Delta retain firing (since_uuid={since_uuid!r}, "
+            f"{len(messages_to_retain)}/{len(all_messages)} messages)",
         )
     else:
-        debug_log(config, f"Full session retain: {len(all_messages)} messages")
+        # Respect retainEveryNTurns in both modes, unless force=True (SessionEnd final retain)
+        if retain_every_n > 1 and not force:
+            turn_count = increment_turn_count(session_id)
+            if turn_count % retain_every_n != 0:
+                next_at = ((turn_count // retain_every_n) + 1) * retain_every_n
+                debug_log(config, f"Turn {turn_count}/{retain_every_n}, skipping retain (next at turn {next_at})")
+                return {"status": "skipped", "reason": "throttled"}
+
+        # Window selection is decoupled from the throttle above — see
+        # select_retain_window() for the switchroom-divergence rationale
+        # (Phase 6b: chunked window-slicing now works at retainEveryNTurns=1).
+        overlap_turns = config.get("retainOverlapTurns", 0)
+
+        # Incremental SessionEnd sweep (memory-RFC P1): a forced chunked sweep at
+        # retainEveryNTurns>1 retains only the transcript tail after the committed
+        # watermark, not the whole session on top of the N per-window documents that
+        # already carry it. The watermark READ is filesystem IO on a JSON file that
+        # could be truncated / permission-broken / absent, so it lives HERE (not in
+        # the pure select_retain_window) inside a catch-all try/except.
+        #
+        # §4.3 HAZARD: this is the one seam where a raise DELETES a turn rather than
+        # degrading it. Every failure mode below MUST degrade to since_uuid=None,
+        # which reproduces today's whole-session sweep exactly. The except catches
+        # Exception (not a named subset) on purpose — any raise here is worse than a
+        # wrong value. The n==1 and full-session paths deliberately never compute a
+        # watermark UNLESS the M4 delta mechanism is enabled for this agent
+        # (``memoryPrefetchEnabled`` — red-team Finding A2): with the mechanism off
+        # (the fleet default) the document id is {session_id} and a tail slice
+        # would TRUNCATE the stored document (§4.2), so they keep the full-session
+        # sweep. With it ON, a per-turn delta retain has already been posting
+        # content-derived slice documents and advancing this same watermark every
+        # turn, so the force sweep MUST also go incremental — otherwise it
+        # blindly re-posts the whole session under {session_id} on top of the
+        # per-turn delta documents (redundant storage, not the destructive
+        # truncation Finding A guards, but still wrong for cadence-1 agents).
+        if force and retain_mode == "chunked" and (
+            retain_every_n > 1 or config.get("memoryPrefetchEnabled")
+        ):
+            try:
+                _wm = watermark.load(session_id)
+                since_uuid = _wm.get("last_uuid") if _wm else None
+            except Exception as e:  # noqa: BLE001 - degrade, never raise into this seam
+                print(
+                    "[Hindsight] watermark read failed for incremental sweep; "
+                    f"falling back to full-session sweep: {e}",
+                    file=sys.stderr,
+                )
+                since_uuid = None
+
+        messages_to_retain, retain_full_window = select_retain_window(
+            retain_mode, retain_every_n, overlap_turns, all_messages, force=force,
+            since_uuid=since_uuid,
+        )
+        if retain_mode == "chunked" and not force:
+            window_turns = max(retain_every_n, 1) + overlap_turns
+            debug_log(
+                config,
+                f"Chunked retain firing (window: {window_turns} human turns, {len(messages_to_retain)} messages)",
+            )
+        elif retain_mode == "chunked" and force:
+            _swept = "incremental (tail after watermark)" if since_uuid is not None else "full-session"
+            debug_log(
+                config,
+                f"Chunked retain, forced {_swept} sweep (SessionEnd): "
+                f"{len(messages_to_retain)}/{len(all_messages)} messages",
+            )
+        else:
+            debug_log(config, f"Full session retain: {len(all_messages)} messages")
 
     # Resolve API URL
     def _dbg(*a):
@@ -853,7 +968,14 @@ def run_retain(hook_input: dict, force: bool = False) -> dict:
     # - Full-session mode / n==1: unchanged — a stable ``session_id`` base with
     #   a compaction chunk counter so a shrinking transcript doesn't overwrite
     #   the pre-compaction document with a shorter one.
-    if retain_mode == "chunked" and retain_every_n > 1:
+    if delta_document_id_override:
+        # M4 P-RET (red-team Finding A, BINDING): a delta retain ALWAYS gets
+        # a content-derived slice id — mirroring the retainEveryNTurns>1
+        # strategy — and must NEVER fall through to the bare {session_id}
+        # id below, which the full-session/n==1 path reserves and which a
+        # tail-slice POST would silently truncate.
+        document_id = None  # build_retain_payload computes the content-derived id
+    elif retain_mode == "chunked" and retain_every_n > 1:
         document_id = None  # build_retain_payload computes the content-derived id
     else:
         chunk_index, compacted = track_retention(session_id, len(all_messages))
@@ -953,6 +1075,23 @@ def run_retain(hook_input: dict, force: bool = False) -> dict:
 
 
 def main():
+    # M4 P-WIRE — when the prefetch mechanism is on for this agent,
+    # `prefetch.py`'s OWN Stop-hook entry already performs a delta retain
+    # every turn (see `prefetch.run_prefetch`, step 1). Running THIS Stop
+    # hook's own (non-delta) retain as well would double-retain the same
+    # turn: two independent POSTs, and at `retainEveryNTurns > 1` two
+    # independent `increment_turn_count` calls drifting the cadence
+    # throttle out of sync with itself. Skip here — flag OFF (the default,
+    # every agent today) leaves this function's behaviour byte-identical to
+    # pre-M4; this check is the ONLY new code on that path.
+    try:
+        _cfg = load_config()
+        if _cfg.get("memoryPrefetchEnabled", False):
+            debug_log(_cfg, "Stop retain: memoryPrefetchEnabled, delegating to prefetch.py")
+            return
+    except Exception:  # pragma: no cover - defensive, never block the legacy path
+        pass
+
     try:
         hook_input = json.load(sys.stdin)
     except (json.JSONDecodeError, EOFError):

package/vendor/hindsight-memory/scripts/tests/test_config_retain_tool_calls_env.py

@@ -0,0 +1,98 @@
+"""Switchroom — `retainToolCalls` must have an env channel, default unchanged.
+
+RFC memory-redesign P4. `retainToolCalls` (whether retain stores tool_use
+inputs + tool_result content) had a DEFAULTS entry of True but NO entry in
+`ENV_OVERRIDES` and no scaffold stamp, so an operator could not reach it: env
+is the TOP of the plugin's config precedence chain (DEFAULTS -> settings.json
+-> ~/.hindsight/claude-code.json -> env), and without an env key
+`HINDSIGHT_RETAIN_TOOL_CALLS` could not steer the plugin at all — including a
+docker-exec'd retain/backfill that does not inherit the supervised settings.
+
+This PR ships the SETTER ONLY. The invariants under test:
+
+  1. `HINDSIGHT_RETAIN_TOOL_CALLS` is wired in `ENV_OVERRIDES` and maps to the
+     `retainToolCalls` config key.
+  2. The shipped default is unchanged — `True` — so absent any override the
+     fleet behaves byte-identically (the load resolves True with no env set).
+  3. Setting the env `false` actually LANDS as Python `False` over the True
+     default (proving `false` is reachable, which is the whole point of the
+     knob), and `true`/`1`/`yes` resolve back to True.
+
+Stdlib-only.
+"""
+
+import os
+import sys
+import unittest
+from unittest import mock
+
+SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+if SCRIPTS_DIR not in sys.path:
+    sys.path.insert(0, SCRIPTS_DIR)
+
+from lib.config import DEFAULTS, ENV_OVERRIDES, load_config  # noqa: E402
+
+ENV_NAME = "HINDSIGHT_RETAIN_TOOL_CALLS"
+CONFIG_KEY = "retainToolCalls"
+
+
+def _load_with(env):
+    """load_config() with a hermetic environment (no plugin/user settings)."""
+    with mock.patch.dict(os.environ, env, clear=True):
+        os.environ["CLAUDE_PLUGIN_ROOT"] = os.path.join(SCRIPTS_DIR, "does-not-exist")
+        os.environ["HOME"] = os.path.join(SCRIPTS_DIR, "does-not-exist")
+        return load_config()
+
+
+class RetainToolCallsHasAChannel(unittest.TestCase):
+    def test_env_name_is_wired(self):
+        self.assertIn(
+            ENV_NAME,
+            ENV_OVERRIDES,
+            f"{ENV_NAME} exported nowhere / never read — the drift P4 closes",
+        )
+
+    def test_env_name_maps_to_the_expected_config_key(self):
+        self.assertEqual(ENV_OVERRIDES[ENV_NAME][0], CONFIG_KEY)
+
+    def test_env_name_is_typed_bool(self):
+        self.assertIs(ENV_OVERRIDES[ENV_NAME][1], bool)
+
+    def test_target_key_exists_in_defaults(self):
+        self.assertIn(CONFIG_KEY, DEFAULTS)
+
+
+class DefaultIsUnchanged(unittest.TestCase):
+    """The critical P4 invariant: default stays True, byte-identical fleet."""
+
+    def test_shipped_default_is_true(self):
+        self.assertIs(DEFAULTS[CONFIG_KEY], True)
+
+    def test_no_env_reproduces_the_true_default(self):
+        cfg = _load_with({})
+        self.assertIs(cfg[CONFIG_KEY], True)
+
+
+class ValuesActuallyLand(unittest.TestCase):
+    """The setter must be reachable: env `false` lands as Python False."""
+
+    def test_false_env_overrides_the_true_default(self):
+        cfg = _load_with({ENV_NAME: "false"})
+        self.assertIs(cfg[CONFIG_KEY], False)
+        self.assertNotEqual(cfg[CONFIG_KEY], DEFAULTS[CONFIG_KEY])
+
+    def test_zero_and_no_also_resolve_false(self):
+        for falsey in ("0", "no", "False"):
+            with self.subTest(value=falsey):
+                cfg = _load_with({ENV_NAME: falsey})
+                self.assertIs(cfg[CONFIG_KEY], False)
+
+    def test_true_env_resolves_true(self):
+        for truthy in ("true", "1", "yes", "TRUE"):
+            with self.subTest(value=truthy):
+                cfg = _load_with({ENV_NAME: truthy})
+                self.assertIs(cfg[CONFIG_KEY], True)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/vendor/hindsight-memory/scripts/tests/test_directives.py

@@ -243,6 +243,58 @@ class FormatActiveDirectivesBlockTests(unittest.TestCase):
         # The in-prompt footer is still emitted (agent-facing signal).
         self.assertIn("(+4 more, omitted)", out)
 
+    def test_overflow_footer_is_loud_and_operator_directed(self):
+        """memory-RFC P7 — the in-prompt overflow notice must be LOUD and tell
+        the agent to surface the drop to the operator this turn.
+
+        The pre-P7 block emitted only the quiet `(+N more, omitted)`
+        parenthetical, which reaches the agent but neither states that explicit
+        instructions were dropped nor directs the agent to relay it. This
+        asserts the OUTCOME of the loud in-turn channel: the rendered block, on
+        overflow, names the dropped count, the total, the cap, states the loss
+        in unmistakable terms, and instructs the agent to tell the operator.
+        A test that only checked `(+N more, omitted)` would still pass on the
+        pre-P7 code, so it would not guard this change; these assertions fail
+        without it.
+        """
+        total = MAX_DIRECTIVES + 3
+        directives = [
+            _directive(f"d{i}", f"c{i}", priority=total - i) for i in range(total)
+        ]
+        with patch("sys.stderr", new=StringIO()):
+            out = format_active_directives_block(directives)
+        # A loud, explicitly-labelled overflow banner (not a lone parenthetical).
+        self.assertIn("DIRECTIVE OVERFLOW", out)
+        # Names the loss in unmistakable terms, with count/total/cap.
+        self.assertIn("DROPPED", out)
+        self.assertIn("3 of", out)  # omitted of total
+        self.assertIn(str(total), out)
+        self.assertIn(f"MAX_DIRECTIVES={MAX_DIRECTIVES}", out)
+        # Directs the agent to surface it to the OPERATOR this turn — the
+        # in-turn operator-visible channel P7 adds.
+        self.assertIn("operator", out.lower())
+        self.assertIn("ACTION", out)
+        # The literal marker is retained for the recall_log/doctor cross-checks
+        # and the dedup parser.
+        self.assertIn("(+3 more, omitted)", out)
+
+    def test_no_overflow_banner_when_under_cap(self):
+        """The loud banner must appear ONLY on overflow — an under-cap block is
+        unchanged and carries neither the banner nor the action directive."""
+        directives = [_directive(f"d{i}", f"c{i}", priority=5 - i) for i in range(3)]
+        with patch("sys.stderr", new=StringIO()) as fake_err:
+            out = format_active_directives_block(directives)
+        self.assertNotIn("DIRECTIVE OVERFLOW", out)
+        self.assertNotIn("ACTION", out)
+        self.assertNotIn("more, omitted", out)
+        self.assertEqual(fake_err.getvalue(), "")
+
+    def test_max_directives_is_unchanged_by_p7(self):
+        """P7 is the loud-channel PR only. Raising MAX_DIRECTIVES is a separate,
+        later change (RFC §5 P7: order is non-negotiable). Pin the constant so a
+        cap bump cannot ride in on this change unnoticed."""
+        self.assertEqual(MAX_DIRECTIVES, 30)
+
     def test_count_omitted_directives_matches_the_rendered_footer(self):
         """The recall_log's `directives_omitted` number must equal what the
         block actually dropped — it is the operator-visible record of it."""