switchroom 0.21.15 → 0.21.16

package/vendor/hindsight-memory/scripts/prefetch.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+"""M4 P-PRE — async Stop-hook prefetch producer.
+
+carve-M4.md: moves recall injection off the synchronous UserPromptSubmit
+path by speculatively retaining + recalling at the END of turn N (this
+script, registered as an async Stop hook), so turn N+1's UserPromptSubmit
+(`recall.py`) can join an already-warm buffer instead of paying the full
+recall latency synchronously.
+
+Entirely gated by `memoryPrefetchEnabled` (default OFF/falsy — Fix C, the
+red-team-mandated per-agent kill switch). When off this script is a no-op:
+it reads config, sees the flag off, and exits silently before touching
+`lib.retain`, `lib.recall_buffer`, or the network.
+
+Call order (test a-integration in `tests/test_prefetch_pipeline.py`):
+  1. delta retain (`retain.run_retain(hook_input, force=False, delta=True)`)
+     — persists this turn's NEW slice using the Fix-A content-derived
+     document_id, never truncating the session document.
+  2. speculative recall (best-effort query derived from the transcript's
+     last human turn — the strongest available proxy for what turn N+1
+     will ask about).
+  3. `recall_buffer.write_buffer` then `recall_buffer.write_sentinel`
+     (STRICTLY in that order — the sentinel-ordering contract `lib/
+     recall_buffer.py` and `tests/test_recall_buffer.py` prove).
+
+Silent on stdout always (an async Stop hook's stdout is not surfaced to the
+transcript); errors go to stderr and NEVER raise past `main()` — a broken
+prefetch must degrade to "recall.py's synchronous path runs as before",
+never break the turn or leave a torn buffer (a crash between steps 2 and 3a
+just means no sentinel is written this turn, which `read_if_fresh` already
+treats as "nothing fresh" — fail-closed by construction).
+
+Skips the same junk turns `recall.py` skips (`<task-notification>` prefix)
+— a synthetic follow-up turn is not a real conversation shift worth
+prefetching for.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+
+from lib.bank import derive_bank_id
+from lib.client import HindsightClient
+from lib.config import debug_log, load_config
+from lib.content import _extract_text_content, strip_channel_envelope
+from lib.daemon import get_api_url
+from lib import recall_buffer
+import retain as retain_module
+
+
+def _last_human_prompt(transcript_path: str) -> str:
+    """Best-effort: the most recent human-authored user turn's text, used
+    as the speculative query for next turn's prefetch. Never raises —
+    returns "" on any read/parse failure (degrades to no query, which the
+    caller treats as "nothing to prefetch")."""
+    try:
+        messages = retain_module.read_transcript(transcript_path)
+    except Exception:
+        return ""
+    for msg in reversed(messages):
+        if msg.get("role") != "user":
+            continue
+        text = _extract_text_content(msg.get("content"), role="user")
+        text = strip_channel_envelope(text) if text else text
+        if text and text.strip():
+            return text.strip()
+    return ""
+
+
+def run_prefetch(hook_input: dict, config: dict) -> bool:
+    """Execute one prefetch cycle. Returns True iff a buffer+sentinel pair
+    was written. Never raises — every internal step is wrapped so a bug in
+    ANY sub-step degrades to "no buffer written this turn", never a crash
+    that could take the Stop hook (and thus the turn) down with it."""
+    session_id = hook_input.get("session_id") or "unknown"
+
+    prompt = (hook_input.get("prompt") or hook_input.get("user_prompt") or "").strip()
+    if prompt.startswith("<task-notification"):
+        debug_log(config, "Prefetch: task-notification turn, skipping")
+        return False
+
+    # Step 1 — delta retain (Fix A: content-derived document_id, never the
+    # bare {session_id} document; never truncates).
+    try:
+        retain_module.run_retain(hook_input, force=False, delta=True)
+    except Exception as exc:  # pragma: no cover - defensive
+        debug_log(config, f"Prefetch: delta retain failed, continuing without it: {exc}")
+
+    # Step 2 — speculative recall.
+    transcript_path = hook_input.get("transcript_path") or ""
+    query = _last_human_prompt(transcript_path)
+    if not query:
+        debug_log(config, "Prefetch: no usable query, nothing to prefetch")
+        return False
+
+    try:
+        bank_id = derive_bank_id(hook_input, config)
+        api_url = get_api_url(config)
+        client = HindsightClient(api_url)
+        response = client.recall(
+            bank_id,
+            query,
+            types=None,
+            timeout=config.get("memoryPrefetchTimeoutSeconds", 5),
+        )
+        results = (response or {}).get("results") or []
+    except Exception as exc:  # pragma: no cover - defensive
+        debug_log(config, f"Prefetch: recall fetch failed: {exc}")
+        return False
+
+    if not results:
+        debug_log(config, "Prefetch: no candidates, nothing to buffer")
+        return False
+
+    from lib.content import format_memories
+
+    memories_block = format_memories(results)
+    if not memories_block:
+        return False
+
+    # Step 3 — write payload THEN sentinel, strictly in that order.
+    try:
+        recall_buffer.write_buffer(session_id, memories_block, {"result_count": len(results)})
+        recall_buffer.write_sentinel(session_id)
+    except Exception as exc:  # pragma: no cover - defensive
+        debug_log(config, f"Prefetch: buffer write failed: {exc}")
+        return False
+
+    return True
+
+
+def main():
+    config = load_config()
+    if not config.get("memoryPrefetchEnabled", False):
+        # Fix C, producer side: the whole mechanism is dark by default.
+        return
+
+    try:
+        hook_input = json.load(sys.stdin)
+    except (json.JSONDecodeError, EOFError):
+        print("[Hindsight] Prefetch: failed to read hook input", file=sys.stderr)
+        return
+
+    try:
+        run_prefetch(hook_input, config)
+    except Exception as exc:  # pragma: no cover - defensive, must never break the turn
+        print(f"[Hindsight] Prefetch: unexpected error: {exc}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:  # pragma: no cover - defensive
+        print(f"[Hindsight] Unexpected error in prefetch: {e}", file=sys.stderr)
+        sys.exit(0)

package/vendor/hindsight-memory/scripts/recall.py

@@ -82,6 +82,7 @@ from lib.directives import (
 )
 from lib.gateway_ipc import extract_chat_id_from_prompt, extract_topic_from_prompt, extract_user_from_prompt, update_placeholder
 from lib.parallel_recall import run_parallel
+from lib import recall_buffer
 from lib.state import read_state, write_state
 
 # Cost of everything above, charged against the hook ceiling (see
@@ -395,6 +396,148 @@ def _emit_cached_context(context: str) -> None:
     )
 
 
+_PREFETCH_DEGRADED_NOTICE = (
+    "⏳ prefetch not ready and no prior recall is cached for this session — "
+    "proceeding without injected memory this turn."
+)
+
+
+def stale_recall_notice(memories_context: str) -> str:
+    """Wrap a PRIOR turn's cached memories-only block in an explicit
+    staleness marker for the M4 prefetch-buffer fallback path.
+
+    M4 P-REC Fix B (red-team BINDING, MUST NOT regress): `memories_context`
+    must NEVER contain a directives block — the caller is required to pass
+    `LAST_RECALL_STATE`'s `memories_context` field (directives-free by
+    construction, see the write site near `context_message`), never its
+    sibling `context` field (which bundles `directives_block`). Directives
+    stay on the synchronous, always-current fetch path only (M3's
+    directive-decoupling rule) — a stale directives block re-injected from a
+    prior turn could resurrect a rule the user rescinded moments ago.
+    """
+    if not memories_context:
+        return ""
+    return (
+        "⏳ stale (prefetch not ready this turn) — the memories below are "
+        "from the PREVIOUS turn's recall, not this turn's. Treat them as a "
+        "hint, not a confirmed current fact.\n\n" + memories_context
+    )
+
+
+def _handle_prefetch_buffer(config: dict, hook_input: dict, prompt: str) -> bool:
+    """M4 P-REC Fix C (consumer side) — join the Stop-hook producer's
+    prefetch buffer for this session instead of running recall
+    synchronously.
+
+    Gated entirely by `config.get("memoryPrefetchEnabled", False)` at the
+    caller; this function assumes the flag is already on. Returns True iff
+    it emitted an `additionalContext` payload (fresh hit, stale fallback, or
+    the explicit degraded notice) and the caller should return without
+    running the synchronous path. Returns False on a clean no-op miss (flag
+    effectively off / nothing to say) so the caller falls through.
+
+    Never raises past this function's own boundary in normal operation —
+    every internal step is wrapped so a bug here degrades to "fall through
+    to synchronous recall", never a broken turn. (The caller additionally
+    wraps this call in its own try/except as defence in depth.)
+    """
+    session_id = hook_input.get("session_id") or "unknown"
+
+    # Cold-start short-circuit (red-team MAJOR finding): if this session has
+    # NEVER produced a sentinel, polling the full cap on every single
+    # session-open turn would cost ~the poll cap on every fresh session —
+    # the opposite of M4's latency goal. Only poll when a sentinel already
+    # exists (a producer has run at least once for this session).
+    if not recall_buffer.sentinel_exists(session_id):
+        debug_log(config, "Prefetch buffer: no sentinel ever written for this session, cold-start skip")
+    else:
+        cap_ms = int(config.get("memoryPrefetchPollCapMs", 400))
+        recall_buffer.poll_for_sentinel(session_id, last_consumed_token=None, cap_ms=cap_ms)
+
+    payload, _token = recall_buffer.read_if_fresh(session_id, last_consumed_token=None)
+
+    # Directives stay on the synchronous, always-fresh path (M3 rule) even
+    # in the fast path — fetched here directly, never from the buffer.
+    directives_block = None
+    try:
+        bank_id = derive_bank_id(hook_input, config)
+        api_url = get_api_url(config)
+        client = HindsightClient(api_url)
+        directives = fetch_active_directives_cached(
+            client, bank_id, ttl_seconds=config.get("directivesCacheTtlSeconds", DIRECTIVES_CACHE_TTL_SECONDS)
+        )
+        directives_block = format_active_directives_block(directives) if directives else None
+    except Exception as exc:  # pragma: no cover - defensive, directives are best-effort here
+        debug_log(config, f"Prefetch buffer: directive fetch failed: {exc}")
+        directives_block = None
+
+    if payload is not None:
+        memories_block = payload.get("context") or ""
+        parts = [b for b in (directives_block, memories_block) if b]
+        if not parts:
+            return False
+        _emit_cached_context("\n\n".join(parts))
+        return True
+
+    # Miss: no fresh buffer. Fall back to the LAST_RECALL_STATE's
+    # directives-FREE `memories_context` field (Fix B) if one exists for
+    # this session, explicitly marked stale. Never `context` (directive-
+    # contaminated).
+    last = read_state(LAST_RECALL_STATE) or {}
+    stale_memories = last.get("memories_context") or ""
+    stale_block = stale_recall_notice(stale_memories)
+    parts = [b for b in (directives_block, stale_block) if b]
+    if parts:
+        _emit_cached_context("\n\n".join(parts))
+        return True
+
+    if directives_block:
+        _emit_cached_context(directives_block)
+        return True
+
+    # Nothing fresh, nothing stale, nothing cached — say so explicitly
+    # rather than silently emitting no context (so a degraded turn is
+    # legible, matching the #3619 degraded-disclosure precedent).
+    _emit_cached_context(_PREFETCH_DEGRADED_NOTICE)
+    return True
+
+
+def _emit_directives_only(config: dict, hook_input: dict) -> None:
+    """#4756 F2 — directive exemption for the task-notification junk gate.
+
+    The M4 P-REC junk gate skips synchronous recall on synthetic
+    `<task-notification>` turns to avoid burning latency/cost on
+    machine-generated noise. But DIRECTIVES are the one memory class that must
+    survive that gate: an agent's standing rules apply on EVERY turn, synthetic
+    or not, and suppressing them on a sub-agent-handback turn is a behavior
+    change that should not ride along with the noise gate. So the gate drops
+    the non-directive classes (observations/world/etc. — the whole `recall`
+    result set) but still fetches and injects the active directives block,
+    exactly as the prefetch fast path does (M3 directive-decoupling rule).
+
+    Emits nothing when the bank has no active directives (no empty wrapper).
+    Failure-safe: any error → emit nothing rather than break the turn, matching
+    `fetch_active_directives_cached`'s never-raise contract. Deliberately does
+    NOT run the memory `recall` HTTP call — only `list_directives` is touched.
+    """
+    try:
+        bank_id = derive_bank_id(hook_input, config)
+        api_url = get_api_url(config)
+        client = HindsightClient(api_url, config.get("hindsightApiToken"))
+        directives = fetch_active_directives_cached(
+            client,
+            bank_id,
+            ttl_seconds=config.get("directivesCacheTtlSeconds", DIRECTIVES_CACHE_TTL_SECONDS),
+        )
+        directives_block = format_active_directives_block(directives) if directives else None
+    except Exception as exc:  # pragma: no cover - defensive, directives are best-effort here
+        debug_log(config, f"Task-notification skip: directive fetch failed: {exc}")
+        return
+
+    if directives_block:
+        _emit_cached_context(directives_block)
+
+
 def _is_demoted_memory(memory) -> bool:
     """Return True if the memory has any demote-from-recall tag.
 
@@ -1971,6 +2114,38 @@ def main():
         debug_log(config, "Prompt too short for recall, skipping")
         return
 
+    # M4 P-REC junk gate: `<task-notification>` is the CLI-native envelope a
+    # sub-agent's own scheduler/harness prepends on a synthetic follow-up
+    # turn — distinct from the gateway's `<channel source=...>` wrapper
+    # (a real user message) and from `is_synthetic_inbound`. Recall on a
+    # task-notification turn burns latency/cost on a turn no human is
+    # waiting on and whose "query" is machine-generated noise, not intent.
+    # Deterministic prefix check only — never a content classifier. On by
+    # default (`recallSkipTaskNotification`); flips off for an agent that
+    # deliberately wants recall on these turns.
+    if config.get("recallSkipTaskNotification", True) and prompt.startswith("<task-notification"):
+        # #4756 F2: skip the noisy NON-directive recall, but directives are
+        # exempt — they are HARD RULES that apply on every turn, synthetic or
+        # not, so still fetch + inject the active directives block. Only the
+        # observation/world memory classes (the `recall` result set) are
+        # suppressed here; `_emit_directives_only` never touches `recall`.
+        debug_log(config, "Prompt is a task-notification envelope, skipping recall (directives exempt)")
+        _emit_directives_only(config, hook_input)
+        return
+
+    # M4 P-REC Fix C (consumer side) — the whole prefetch-buffer mechanism
+    # is gated by `memoryPrefetchEnabled`, default OFF/falsy. When on, try
+    # the buffer-join fast path first; on any hit (fresh or stale-fallback)
+    # it emits and returns, short-circuiting the synchronous recall below.
+    # A miss (disabled, cold-start, buffer absent, or any internal error)
+    # falls through to the existing synchronous path untouched.
+    if config.get("memoryPrefetchEnabled", False):
+        try:
+            if _handle_prefetch_buffer(config, hook_input, prompt):
+                return
+        except Exception as exc:  # pragma: no cover - defensive, never break recall
+            debug_log(config, f"Prefetch buffer join failed, falling back to sync recall: {exc}")
+
     # Switchroom-local: skip recall on conversational acks.
     #
     # The 5-char short-circuit catches `ok`/`yes`/`no`/`ty` but passes
@@ -3134,6 +3309,16 @@ def main():
         LAST_RECALL_STATE,
         {
             "context": context_message,
+            # M4 P-REC Fix B (red-team BINDING): a directives-FREE sibling of
+            # `context`, memories/transcript-fallback only. The stale-buffer
+            # fallback (`_handle_prefetch_buffer`) reads THIS field, never
+            # `context` — `context` bundles `directives_block` (M3's
+            # directive-decoupling rule forbids re-injecting stale directives
+            # from a prior turn's cache; directives stay on the synchronous,
+            # always-fresh fetch path only).
+            "memories_context": "\n\n".join(
+                [b for b in (memories_block, transcript_fallback_block) if b]
+            ),
             "saved_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
             "bank_id": bank_id,
             "result_count": len(results),

package/vendor/hindsight-memory/scripts/retain.py

@@ -727,7 +727,7 @@ def build_retain_payload(
     }
 
 
-def run_retain(hook_input: dict, force: bool = False) -> dict:
+def run_retain(hook_input: dict, force: bool = False, delta: bool = False) -> dict:
     """Run the auto-retain flow.
 
     Returns a status dict::
@@ -742,6 +742,22 @@ def run_retain(hook_input: dict, force: bool = False) -> dict:
     process (the SessionStart drainer, see ``drain_pending.py`` and
     ``lib/pending.py``). ``status="skipped"`` is the normal early-exit
     cases (auto-retain disabled, empty transcript, throttled chunk).
+
+    ``delta`` (M4 P-RET, per-turn delta retain — called ONLY from
+    ``prefetch.py``, itself gated on the ``memoryPrefetchEnabled`` kill
+    switch, default off): bypasses ``retainEveryNTurns`` entirely (the
+    caller already decided this turn fires) and slices ONLY the
+    transcript tail after the committed watermark. Ignored when
+    ``force=True`` (SessionEnd always takes the force path below).
+
+    **Document-id safety invariant (red-team M4 Finding A, BINDING):** a
+    delta retain is ALWAYS posted under a distinct, content-derived
+    ``document_id`` — the SAME strategy the ``retainEveryNTurns>1``
+    chunked path already uses — and NEVER under the bare ``{session_id}``
+    id the full-session/n==1 path reserves. Posting a tail slice under
+    ``{session_id}`` would UPSERT-OVERWRITE the whole session's stored
+    memory with just the latest turn on every successful turn (see
+    ``retain.py`` §4.3 note below and ``tests/test_retain_delta.py``).
     """
     # /private-mode enforcement (switchroom privacy PR1) — FIRST, before
     # load_config(). An OPEN private interval means the operator is discussing
@@ -810,66 +826,109 @@ def run_retain(hook_input: dict, force: bool = False) -> dict:
     retain_every_n = max(1, config.get("retainEveryNTurns", 1))
     retain_full_window = False
     messages_to_retain = all_messages
-
-    # Respect retainEveryNTurns in both modes, unless force=True (SessionEnd final retain)
-    if retain_every_n > 1 and not force:
-        turn_count = increment_turn_count(session_id)
-        if turn_count % retain_every_n != 0:
-            next_at = ((turn_count // retain_every_n) + 1) * retain_every_n
-            debug_log(config, f"Turn {turn_count}/{retain_every_n}, skipping retain (next at turn {next_at})")
-            return {"status": "skipped", "reason": "throttled"}
-
-    # Window selection is decoupled from the throttle above — see
-    # select_retain_window() for the switchroom-divergence rationale
-    # (Phase 6b: chunked window-slicing now works at retainEveryNTurns=1).
-    overlap_turns = config.get("retainOverlapTurns", 0)
-
-    # Incremental SessionEnd sweep (memory-RFC P1): a forced chunked sweep at
-    # retainEveryNTurns>1 retains only the transcript tail after the committed
-    # watermark, not the whole session on top of the N per-window documents that
-    # already carry it. The watermark READ is filesystem IO on a JSON file that
-    # could be truncated / permission-broken / absent, so it lives HERE (not in
-    # the pure select_retain_window) inside a catch-all try/except.
-    #
-    # §4.3 HAZARD: this is the one seam where a raise DELETES a turn rather than
-    # degrading it. Every failure mode below MUST degrade to since_uuid=None,
-    # which reproduces today's whole-session sweep exactly. The except catches
-    # Exception (not a named subset) on purpose — any raise here is worse than a
-    # wrong value. The n==1 and full-session paths deliberately never compute a
-    # watermark: on those the document id is {session_id} and a tail slice would
-    # TRUNCATE the stored document (§4.2), so they keep the full-session sweep.
     since_uuid = None
-    if force and retain_mode == "chunked" and retain_every_n > 1:
+    # True only for the per-turn delta path (M4 P-RET) — threads a distinct,
+    # content-derived document_id below (Finding A) instead of the
+    # retainEveryNTurns>1 / n==1 strategy.
+    delta_document_id_override = False
+
+    if delta and not force:
+        # M4 P-RET — per-turn delta retain, called ONLY from prefetch.py
+        # (itself gated on memoryPrefetchEnabled, default off). Bypasses the
+        # retainEveryNTurns throttle entirely: the caller already decided
+        # this turn fires, every turn, superseding the P5 cadence on a
+        # flipped agent (config comment mandated at the switchroom.yaml
+        # seam — see P-CFG). Slices ONLY the transcript tail after the
+        # committed watermark; the §4.3 HAZARD discipline applies here too —
+        # any watermark-read failure MUST degrade to the full window, never
+        # raise, never emit an empty slice.
         try:
             _wm = watermark.load(session_id)
             since_uuid = _wm.get("last_uuid") if _wm else None
         except Exception as e:  # noqa: BLE001 - degrade, never raise into this seam
             print(
-                "[Hindsight] watermark read failed for incremental sweep; "
-                f"falling back to full-session sweep: {e}",
+                "[Hindsight] watermark read failed for delta retain; "
+                f"falling back to full-window slice: {e}",
                 file=sys.stderr,
             )
             since_uuid = None
-
-    messages_to_retain, retain_full_window = select_retain_window(
-        retain_mode, retain_every_n, overlap_turns, all_messages, force=force,
-        since_uuid=since_uuid,
-    )
-    if retain_mode == "chunked" and not force:
-        window_turns = max(retain_every_n, 1) + overlap_turns
+        messages_to_retain = watermark.tail_after(all_messages, since_uuid)
+        retain_full_window = True
+        delta_document_id_override = True
         debug_log(
             config,
-            f"Chunked retain firing (window: {window_turns} human turns, {len(messages_to_retain)} messages)",
-        )
-    elif retain_mode == "chunked" and force:
-        _swept = "incremental (tail after watermark)" if since_uuid is not None else "full-session"
-        debug_log(
-            config,
-            f"Chunked retain, forced {_swept} sweep (SessionEnd): "
-            f"{len(messages_to_retain)}/{len(all_messages)} messages",
+            f"Delta retain firing (since_uuid={since_uuid!r}, "
+            f"{len(messages_to_retain)}/{len(all_messages)} messages)",
         )
     else:
-        debug_log(config, f"Full session retain: {len(all_messages)} messages")
+        # Respect retainEveryNTurns in both modes, unless force=True (SessionEnd final retain)
+        if retain_every_n > 1 and not force:
+            turn_count = increment_turn_count(session_id)
+            if turn_count % retain_every_n != 0:
+                next_at = ((turn_count // retain_every_n) + 1) * retain_every_n
+                debug_log(config, f"Turn {turn_count}/{retain_every_n}, skipping retain (next at turn {next_at})")
+                return {"status": "skipped", "reason": "throttled"}
+
+        # Window selection is decoupled from the throttle above — see
+        # select_retain_window() for the switchroom-divergence rationale
+        # (Phase 6b: chunked window-slicing now works at retainEveryNTurns=1).
+        overlap_turns = config.get("retainOverlapTurns", 0)
+
+        # Incremental SessionEnd sweep (memory-RFC P1): a forced chunked sweep at
+        # retainEveryNTurns>1 retains only the transcript tail after the committed
+        # watermark, not the whole session on top of the N per-window documents that
+        # already carry it. The watermark READ is filesystem IO on a JSON file that
+        # could be truncated / permission-broken / absent, so it lives HERE (not in
+        # the pure select_retain_window) inside a catch-all try/except.
+        #
+        # §4.3 HAZARD: this is the one seam where a raise DELETES a turn rather than
+        # degrading it. Every failure mode below MUST degrade to since_uuid=None,
+        # which reproduces today's whole-session sweep exactly. The except catches
+        # Exception (not a named subset) on purpose — any raise here is worse than a
+        # wrong value. The n==1 and full-session paths deliberately never compute a
+        # watermark UNLESS the M4 delta mechanism is enabled for this agent
+        # (``memoryPrefetchEnabled`` — red-team Finding A2): with the mechanism off
+        # (the fleet default) the document id is {session_id} and a tail slice
+        # would TRUNCATE the stored document (§4.2), so they keep the full-session
+        # sweep. With it ON, a per-turn delta retain has already been posting
+        # content-derived slice documents and advancing this same watermark every
+        # turn, so the force sweep MUST also go incremental — otherwise it
+        # blindly re-posts the whole session under {session_id} on top of the
+        # per-turn delta documents (redundant storage, not the destructive
+        # truncation Finding A guards, but still wrong for cadence-1 agents).
+        if force and retain_mode == "chunked" and (
+            retain_every_n > 1 or config.get("memoryPrefetchEnabled")
+        ):
+            try:
+                _wm = watermark.load(session_id)
+                since_uuid = _wm.get("last_uuid") if _wm else None
+            except Exception as e:  # noqa: BLE001 - degrade, never raise into this seam
+                print(
+                    "[Hindsight] watermark read failed for incremental sweep; "
+                    f"falling back to full-session sweep: {e}",
+                    file=sys.stderr,
+                )
+                since_uuid = None
+
+        messages_to_retain, retain_full_window = select_retain_window(
+            retain_mode, retain_every_n, overlap_turns, all_messages, force=force,
+            since_uuid=since_uuid,
+        )
+        if retain_mode == "chunked" and not force:
+            window_turns = max(retain_every_n, 1) + overlap_turns
+            debug_log(
+                config,
+                f"Chunked retain firing (window: {window_turns} human turns, {len(messages_to_retain)} messages)",
+            )
+        elif retain_mode == "chunked" and force:
+            _swept = "incremental (tail after watermark)" if since_uuid is not None else "full-session"
+            debug_log(
+                config,
+                f"Chunked retain, forced {_swept} sweep (SessionEnd): "
+                f"{len(messages_to_retain)}/{len(all_messages)} messages",
+            )
+        else:
+            debug_log(config, f"Full session retain: {len(all_messages)} messages")
 
     # Resolve API URL
     def _dbg(*a):
@@ -909,7 +968,14 @@ def run_retain(hook_input: dict, force: bool = False) -> dict:
     # - Full-session mode / n==1: unchanged — a stable ``session_id`` base with
     #   a compaction chunk counter so a shrinking transcript doesn't overwrite
     #   the pre-compaction document with a shorter one.
-    if retain_mode == "chunked" and retain_every_n > 1:
+    if delta_document_id_override:
+        # M4 P-RET (red-team Finding A, BINDING): a delta retain ALWAYS gets
+        # a content-derived slice id — mirroring the retainEveryNTurns>1
+        # strategy — and must NEVER fall through to the bare {session_id}
+        # id below, which the full-session/n==1 path reserves and which a
+        # tail-slice POST would silently truncate.
+        document_id = None  # build_retain_payload computes the content-derived id
+    elif retain_mode == "chunked" and retain_every_n > 1:
         document_id = None  # build_retain_payload computes the content-derived id
     else:
         chunk_index, compacted = track_retention(session_id, len(all_messages))
@@ -1009,6 +1075,23 @@ def run_retain(hook_input: dict, force: bool = False) -> dict:
 
 
 def main():
+    # M4 P-WIRE — when the prefetch mechanism is on for this agent,
+    # `prefetch.py`'s OWN Stop-hook entry already performs a delta retain
+    # every turn (see `prefetch.run_prefetch`, step 1). Running THIS Stop
+    # hook's own (non-delta) retain as well would double-retain the same
+    # turn: two independent POSTs, and at `retainEveryNTurns > 1` two
+    # independent `increment_turn_count` calls drifting the cadence
+    # throttle out of sync with itself. Skip here — flag OFF (the default,
+    # every agent today) leaves this function's behaviour byte-identical to
+    # pre-M4; this check is the ONLY new code on that path.
+    try:
+        _cfg = load_config()
+        if _cfg.get("memoryPrefetchEnabled", False):
+            debug_log(_cfg, "Stop retain: memoryPrefetchEnabled, delegating to prefetch.py")
+            return
+    except Exception:  # pragma: no cover - defensive, never block the legacy path
+        pass
+
     try:
         hook_input = json.load(sys.stdin)
     except (json.JSONDecodeError, EOFError):