switchroom 0.21.13 → 0.21.15

package/vendor/hindsight-memory/scripts/recall.py

@@ -54,6 +54,7 @@ import re  # noqa: E402
 import socket  # noqa: E402
 import sys  # noqa: E402
 import urllib.error  # noqa: E402
+from datetime import datetime  # noqa: E402
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
@@ -77,6 +78,7 @@ from lib.directives import (
     count_omitted_directives,
     fetch_active_directives_cached,
     format_active_directives_block,
+    injected_directive_ids,
 )
 from lib.gateway_ipc import extract_chat_id_from_prompt, extract_topic_from_prompt, extract_user_from_prompt, update_placeholder
 from lib.parallel_recall import run_parallel
@@ -1577,6 +1579,257 @@ def looks_like_standing_rule(text) -> bool:
     return bool(_DIRECTIVE_NUDGE_RE.search(scrubbed))
 
 
+# ── Temporal-expression detection for the recall `query_timestamp` anchor ──
+# Switchroom P2 (memory-redesign RFC §5). When the inbound prompt asks a
+# time-relative question ("what did we work on last week", "on the 12th"),
+# the recall body carries an explicit `query_timestamp` anchor so the engine
+# resolves the relative expression and anchors recency scoring against the
+# real ask-time. The anchor is ALWAYS the current wall clock — that is the
+# documented semantics ("when the query is being asked, from the user's
+# perspective", https://hindsight.vectorize.io/developer/api/recall) — the
+# regex only GATES whether the field is sent, it does not resolve the phrase
+# itself (the engine does that server-side against the anchor we supply).
+#
+# Deterministic regex only — NO model call (claude-native invariant, same as
+# the directive-capture nudge above). The match is a cheap substring scan on
+# the already-stripped prompt, so it never extends the recall hook's critical
+# path (the 12s ceiling / parallel deadline live entirely on the network I/O
+# below). Detection is deliberately conservative: it requires a preposition or
+# quantifier around ambiguous tokens (bare weekday / month / "may") so an
+# ordinary sentence does not fire the field. A false negative merely omits an
+# anchor the server would default to anyway; a false positive sends the true
+# ask-time, which is the correct anchor regardless — so both error directions
+# degrade to today's behaviour.
+_TEMPORAL_EXPRESSION_RE = re.compile(
+    r"""(?ix)
+    (?:
+      # --- absolute-relative day words ---
+        \b (?: yesterday | tonight | tomorrow ) \b
+      | \b last \s+ night \b
+      # --- this/last/next + period ---
+      | \b (?: this | last | next | past ) \s+
+            (?: week | month | year | quarter | fortnight | weekend
+              | morning | afternoon | evening | night | decade ) \b
+      # --- earlier / other-day framings ---
+      | \b the \s+ other \s+ (?: day | week | night ) \b
+      | \b earlier \s+ (?: today | this \s+ (?: week | month | year ) ) \b
+      | \b a \s+ (?: while | moment ) \s+ ago \b
+      # --- "<N> <unit> ago" (worded or digit quantifier) ---
+      | \b (?: a | an | one | two | three | four | five | six | seven | eight
+            | nine | ten | \d+ | couple \s+ of | few ) \s+
+            (?: second | minute | hour | day | week | month | year ) s? \s+ ago \b
+      # --- weekday, only with a temporal preposition/qualifier ---
+      | \b (?: this | last | next | on | since | by ) \s+
+            (?: monday | tuesday | wednesday | thursday | friday
+              | saturday | sunday ) \b
+      # --- ordinal day-of-month ("on the 12th", "by the 3rd") ---
+      | \b (?: on | by | since | before | after | around ) \s+ the \s+
+            \d{1,2} (?: st | nd | rd | th ) \b
+      # --- month name, only with a temporal preposition ---
+      | \b (?: in | on | since | during | back \s+ in | early | late ) \s+
+            (?: january | february | march | april | may | june | july
+              | august | september | october | november | december ) \b
+    )
+    """
+)
+
+
+# RFC phase4 P3 — deterministic operator-profile capture nudge.
+#
+# Ken's first stated want is "save memories about him" (RFC §0 constraint 5a).
+# Auto-retain does store transcript facts, but there is no deterministic signal
+# that a durable *profile fact* about the operator himself just went by, so
+# capture-as-profile is left to model discretion — the same per-agent lottery
+# Stage A measured for directives. This mirrors the shipped directive-capture
+# nudge (recall.py #2848): a POSITIVE regex detects a first-person durable
+# self-statement ("I prefer …", "my … is …", "I always …", "remind me that
+# I …"); a NEGATIVE regex scrubs the two shapes that would otherwise misfire —
+# questions ("do I prefer …?", "what's my …?") and third-/second-party
+# attributions ("you said I prefer …", "she claims my …") — BEFORE the positive
+# match. On a hit the hook appends a terse advisory telling the model to persist
+# the fact with an explicit mcp__hindsight__retain carrying a `profile:ken` tag
+# into THIS AGENT'S OWN bank. Pure regex — NO model callsite (the claude-native
+# invariant forbids a classifier call); the model makes the judgment in-session
+# and calls retain itself (chat-legible). The hook NEVER writes on its own.
+#
+# On by default; operators opt out per-agent via memory.profile_capture_nudge
+# =false → HINDSIGHT_PROFILE_CAPTURE_NUDGE (recall.py falls back to True).
+#
+# ROUTING CONSTRAINT (RFC §0 constraint 2, §7 Q3): the fact goes to the agent's
+# OWN bank, NOT a shared/cross-agent person bank (ken-profile/lisa-profile).
+# The tag makes the facts cheap to find and retire later if Q3 is answered
+# differently. The `profile:ken` operator identity is intentionally literal —
+# this fleet has a single named operator (Ken); a multi-operator deployment
+# would parameterise the tag, which is out of scope for P3.
+#
+# CONSERVATISM (RFC §7 Q1, unanswered): the RFC flags that this regex set is
+# derived from an ASSUMPTION that the profile facts Ken wants are
+# preference-/identity-shaped. Until Q1 is answered from a real instance, the
+# positive set is deliberately TIGHT (favouring false negatives) — only clearly
+# durable first-person framings, not bare "I like"/"I use" reactions that are
+# usually one-off. If Q1's answer is not preference-shaped, this set is wrong
+# and should be re-derived before it is relied on.
+_PROFILE_NUDGE_NEGATIVE_RE = re.compile(
+    r"""(?ix)
+    (?:
+      # --- interrogatives: a question ABOUT the operator is not a statement
+      #     OF a durable fact. Scrub the "<wh|aux> [do] I" / "<aux> my" lead so
+      #     the following "I prefer" / "my X is" can't fire. ---
+        \b (?: what | which | where | when | why | how | do | did | does
+             | should | would | could | can | are | is | was | were )
+          \s+ (?: do \s+ )? i \b
+      | \b what (?: ['’]? s | \s+ is | \s+ are ) \s+ my \b
+      | \b (?: where | when | is | are | was | were ) \s+ my \b
+      | \b remind \s+ me \s+ what \b
+      # --- attributions: a fact the operator ascribes to someone else (or to
+      #     the agent) is not the operator stating his own profile. Scrub the
+      #     attributed clause up to the next clause boundary. Stop at a comma as
+      #     well as sentence-enders so a trailing real fact in the SAME sentence
+      #     ("she said X, my timezone is Melbourne") still reaches the positive
+      #     matcher instead of being swallowed. ---
+      | \b (?: he | she | they | you | who | someone | everyone | nobody )
+          \s+ (?: said | says | say | claimed | claims | thinks? | thought
+                | told | mentions? | mentioned | asks? | asked | wants? | wanted
+                | wrote | believes? | reckons? )
+          \b [^.?!,]*
+      # --- pleasantries that embed a bare always/never after "I". ---
+      | \b i \s+ (?: always | never )
+          \s+ (?: appreciate | enjoy | 'm \s+ happy | am \s+ happy | love \s+ working ) \b
+      # --- "I'm a <hedge>" is a transient mood/quantifier, not "I'm a <noun>"
+      #     identity ("I'm a bit tired", "I'm a little confused"). Scrub the
+      #     "I'm a/an" lead so the identity arm can't fire on it. ---
+      | \b i (?: \s+ am | \s* ['’] m ) \s+ (?: a | an )
+          \s+ (?: bit | little | lot | tad | touch | bunch | couple | few
+                | fan \b | big \s+ fan ) \b
+      # --- "call me <phone-phrasing>" is a request, not a name form
+      #     ("call me back", "call me later"). Scrub so the name arm ("call me
+      #     Ken") is the only thing left that can fire. ---
+      | \b call \s+ me \s+ (?: back | later | tomorrow | tonight | soon | again
+                | when | if | once | after | before | at | on | in | asap ) \b
+    )
+    """
+)
+
+
+def detect_query_timestamp(text, now=None) -> "str | None":
+    """Return an ISO 8601 ask-time anchor when ``text`` carries a temporal
+    expression, else ``None``.
+
+    Deterministic and IO-free — a single regex scan, no model call and no
+    clock dependency the caller cannot control (``now`` is injectable so the
+    behaviour is unit-testable to the exact output string). When a temporal
+    phrase is present the anchor returned is the CURRENT time, because
+    ``query_timestamp`` is defined by the engine as *when the query is asked*,
+    not the period the phrase names — the engine resolves the phrase against
+    this anchor. ``None`` means "send no field", which keeps the recall body
+    byte-identical to a pre-P2 client.
+
+    The anchor carries the LOCAL wall-clock offset (``datetime.now()`` +
+    ``.astimezone()``), NOT UTC. This matters precisely on the dimension P2
+    serves: for a Melbourne evening query "what did we do yesterday", a
+    UTC-stamped anchor (``+00:00``) can be a calendar day ahead of the
+    operator's real day, so the engine would resolve "yesterday"/"last
+    week"/"on the 12th" against the wrong day. ``.astimezone()`` with no
+    argument attaches the process TZ (the container clock is already
+    Australia/Melbourne), which is the operator's actual day. Never
+    ``timezone.utc`` here — that would re-introduce the off-by-one this fix
+    removes.
+
+    Returns ``None`` on empty / non-string input so a caller can pass a raw
+    prompt without a guard.
+    """
+    if not isinstance(text, str) or not text.strip():
+        return None
+    if not _TEMPORAL_EXPRESSION_RE.search(text):
+        return None
+    anchor = now if now is not None else datetime.now().astimezone()
+    return anchor.isoformat()
+
+
+_PROFILE_NUDGE_RE = re.compile(
+    r"""(?ix)
+    (?:
+      # --- stated preferences / tastes ---
+        \b i \s+ prefer \b
+      | \b i['’]? d \s+ prefer \b
+      | \b my \s+ preference \s+ (?: is | are ) \b
+      | \b i \s+ (?: hate | love | despise | adore | dislike
+                  | can ['’]? t \s+ stand ) \b
+      # --- durable self-facts: "my <ATTRIBUTE> is/are/'s <value>".
+      #     ATTRIBUTE is a TIGHT allow-list of durable identity attributes.
+      #     A free noun ("my build is failing", "my container is down", "my PR
+      #     is ready") is transient dev state, not a profile fact — firing on
+      #     it inverts the RFC's favour-false-negatives constraint on this very
+      #     agent (klanker), so the free-`\w+` arm is deliberately NOT used. ---
+      | \b my \s+
+          (?: name | e-?mail | timezone | time \s+ zone | address
+            | (?: phone | mobile | cell ) (?: \s+ number )? | number
+            | birthday | birthdate | dob | anniversary | age | pronouns?
+            | handle | username | nickname | initials
+            | employer | company | partner | wife | husband | spouse
+            | girlfriend | boyfriend | kids? | children | child | son
+            | daughter | sister | brother | mother | father | mom | dad
+            | parents | dog | cat | pet | diet | allerg(?: y | ies )
+            | location | city | country | hometown )
+          (?: \s+ \w+ )?
+          (?: \s+ (?: is | are ) | \s* ['’] s ) \b
+      # --- identity / situation ---
+      | \b i \s+ live \s+ (?: in | at | near ) \b
+      | \b i \s+ work \s+ (?: at | as | for | in ) \b
+      | \b i (?: \s+ am | \s* ['’] m ) \s+ (?: allergic \s+ to
+            | based \s+ (?: in | at ) | from | located \s+ in
+            | vegetarian | vegan | pescatarian | teetotal ) \b
+      | \b i (?: \s+ am | \s* ['’] m ) \s+ (?: a | an ) \s+ \w+
+      # --- dietary / abstention identity ("I don't eat meat") ---
+      | \b i \s+ (?: do \s* n['’]? t | don['’]? t | do \s+ not )
+            \s+ (?: eat | drink | use | own | drive ) \b
+      # --- name / address form ("call me Ken") ---
+      | \b call \s+ me \b
+      # --- durable habits (first person; questions pre-scrubbed) ---
+      | \b i \s+ always \b
+      | \b i \s+ usually \b
+      | \b i \s+ normally \b
+      | \b i \s+ never \b
+      # --- explicit memory framing about the operator himself ---
+      | \b remember \s+ that \s+ i \b
+      | \b remind \s+ me \s+ that \s+ i \b
+    )
+    """
+)
+
+# Terse, advisory. The model decides IN-SESSION whether this is a durable
+# operator-profile fact and, if so, calls retain itself (chat-legible),
+# tagging it `profile:ken` and routing it to the agent's OWN bank. Kept short
+# so it costs a handful of tokens on a false positive.
+_PROFILE_CAPTURE_NUDGE = (
+    "<profile_capture_check>\n"
+    "The latest user message states a durable fact about the operator himself "
+    '(e.g. a preference "I prefer …", an identity/situation fact "my … is …", '
+    'a habit "I always …", or "remind me that I …") — NOT an instruction about '
+    "how you should behave (that is the directive path). If it is a DURABLE "
+    "fact worth remembering about him across sessions — not a one-off for this "
+    "task — persist it NOW with mcp__hindsight__retain, in his own words, "
+    'tagged ["profile:ken"], into THIS AGENT\'S OWN bank (the default bank — do '
+    "NOT route it to a shared or cross-agent person bank). If an equivalent "
+    "fact is already stored, do not duplicate it. If it is only a passing "
+    "remark, ignore this note and just answer.\n"
+    "</profile_capture_check>"
+)
+
+
+def looks_like_profile_statement(text) -> bool:
+    """Deterministic (regex-only) test for an operator durable-profile shape.
+
+    Question and attribution shapes ("do I prefer …?", "what's my …?", "you
+    said I prefer …") are scrubbed BEFORE the positive match so they can't trip
+    the first-person signals. Returns False on empty / non-string input. No
+    model call — the model does the actual judgment in-session (RFC P3)."""
+    if not isinstance(text, str) or not text.strip():
+        return False
+    scrubbed = _PROFILE_NUDGE_NEGATIVE_RE.sub(" ", text)
+    return bool(_PROFILE_NUDGE_RE.search(scrubbed))
+
+
 def _combine_context(base, nudge) -> str:
     """Join the recall/directives context with the directive-capture nudge,
     skipping empties. Either may be None/empty. The nudge is kept OUT of the
@@ -1779,6 +2032,42 @@ def main():
         nudge_block = _DIRECTIVE_CAPTURE_NUDGE
         debug_log(config, "Directive-capture nudge: inbound looks like a standing rule")
 
+    # Switchroom P2 (memory-redesign RFC §5) — anchor recall to the ask-time
+    # when the inbound prompt is time-relative, so the engine resolves "last
+    # week"/"yesterday"/"on the 12th" against the real now and scores recency
+    # from it. Deterministic regex on `_stripped` (same channel-stripped text
+    # the nudge uses); no model call, microsecond cost, off the network path.
+    # `None` when the prompt has no temporal phrase → the field is never added
+    # to the recall body (byte-identical to pre-P2). The `recallQueryTimestamp`
+    # key is an IN-CODE guard only — it has NO schema/scaffold/env surface, so
+    # a settings.json edit would be clobbered on the next `switchroom apply`
+    # (the plugin dir is re-copied from vendor/). Per RFC P2 the rollback is
+    # "stop sending the field" = revert the commit; if a runtime knob is ever
+    # wanted, wire it the full schema->scaffold->config->env way
+    # `directiveCaptureNudge` is, not by hand-editing settings.json.
+    query_timestamp = None
+    if config.get("recallQueryTimestamp", True):
+        query_timestamp = detect_query_timestamp(_stripped)
+        if query_timestamp:
+            debug_log(config, "Recall query_timestamp anchor: inbound is time-relative")
+
+    # RFC phase4 P3 — operator-profile capture nudge. Independent second nudge
+    # class: deterministic (regex) detection of a first-person durable
+    # self-statement by the operator; when it fires we append a terse advisory
+    # telling the model to persist it with an explicit retain carrying a
+    # `profile:ken` tag into the agent's OWN bank (RFC §0 constraint 2 forbids a
+    # cross-agent person bank). Computed on `_stripped` like the directive
+    # nudge, so the `<channel …>` wrapper never trips it. Both nudges may fire
+    # on one turn (e.g. "I prefer …" is both preference and profile-shaped) —
+    # they give different advice (create_directive vs profile:ken retain) and
+    # are combined independently at emit time. On by default;
+    # HINDSIGHT_PROFILE_CAPTURE_NUDGE=false (or memory.profile_capture_nudge:
+    # false) turns it off. No model callsite (claude-native invariant).
+    profile_nudge_block = None
+    if config.get("profileCaptureNudge", True) and looks_like_profile_statement(_stripped):
+        profile_nudge_block = _PROFILE_CAPTURE_NUDGE
+        debug_log(config, "Profile-capture nudge: inbound states a durable operator fact")
+
     session_id = hook_input.get("session_id") or ""
 
     # Switchroom #303 — push a "📚 recalling memories" status to the
@@ -1888,7 +2177,11 @@ def main():
             # #2848 — append the nudge to the cached context at emit time
             # (the cache stores nudge-free context; the nudge is re-derived
             # from the current prompt, so a hit can't replay a stale one).
-            _emit_cached_context(_combine_context(cached_context, nudge_block))
+            _emit_cached_context(
+                _combine_context(
+                    _combine_context(cached_context, nudge_block), profile_nudge_block
+                )
+            )
             _write_recall_log({
                 "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
                 "session_id": (session_id or "")[:32],
@@ -1900,6 +2193,10 @@ def main():
                 "directive_count": None,
                 # No directives block is built on a cache hit.
                 "directives_omitted": None,
+                # Same reason — a cache hit replays a formatted context
+                # block, not a fetched directive list, so there is nothing
+                # to derive the injected id set from this turn.
+                "directive_ids": None,
                 "demoted_count": 0,
                 # #3837 score-floor fields, present for a uniformly queryable
                 # schema. A cache hit replays a formatted context block, not a
@@ -1957,6 +2254,15 @@ def main():
                 "active_topic_alias": active_topic_alias,
                 "topic_filter_mode": _topic_filter_mode(),
                 "directive_nudge": bool(nudge_block),
+                # Switchroom P2 — the ISO ask-time anchor sent to recall, or
+                # null when the inbound had no temporal phrase. Logged on cache
+                # hits too (no bank ran, so nothing was sent this turn) for a
+                # uniformly queryable schema and to measure the firing rate
+                # from day one (precedent: `directive_nudge` above).
+                "query_timestamp": query_timestamp,
+                # RFC P3 — profile-capture nudge firing rate, measurable from
+                # day one (mirrors the directive_nudge precedent).
+                "profile_nudge": bool(profile_nudge_block),
                 # E1 / PR8 (#3369) — no banks ran on a cache hit, so the
                 # transcript fallback never fires; carry the zeroed fields for a
                 # uniformly queryable schema.
@@ -2102,6 +2408,12 @@ def main():
 
     def _make_bank_task(target_bank_id, b_tags, b_tags_match, b_tag_groups, timeout_override=None):
         def _bank_task():
+            # Switchroom P2 — include the ask-time anchor ONLY when the prompt
+            # was time-relative. Passing it conditionally (not as `=None`) keeps
+            # the client CALL — not just the wire body — byte-identical on the
+            # common non-temporal turn, so a caller/fake with a narrower recall
+            # signature is never handed a kwarg it did not have before.
+            qts_kwarg = {"query_timestamp": query_timestamp} if query_timestamp else {}
             return client.recall(
                 bank_id=target_bank_id,
                 query=search_query,
@@ -2143,6 +2455,8 @@ def main():
                     if timeout_override is None
                     else timeout_override
                 ),
+                # Switchroom P2 — present only on a time-relative turn (see above).
+                **qts_kwarg,
             )
         return _bank_task
 
@@ -2620,6 +2934,15 @@ def main():
         # never reached the agent. >0 here means the bank is over cap and the
         # doctor's directive-count check will be FAILing too.
         "directives_omitted": count_omitted_directives(directives),
+        # Switchroom memory-redesign step 1 (E-45 recommendation (b)) — WHICH
+        # directives actually reached the prompt this turn, in the same
+        # priority-descending order `format_active_directives_block` rendered
+        # them. `directive_count`/`directives_omitted` above are volume-only
+        # (how many fetched, how many the cap dropped); this is the queryable
+        # record of identity, so directive exposure — including "never once
+        # injected" — is measurable before any change to what gets injected.
+        # Purely additive: does not change `directives_block` composition.
+        "directive_ids": injected_directive_ids(directives),
         "demoted_count": demoted_count,
         # Switchroom #3837 — score-floor telemetry, deliberately alongside the
         # `injected_score_*` fields below: those are what the floor was derived
@@ -2721,6 +3044,14 @@ def main():
         "topic_filter_mode": topic_filter_mode,
         "topic_dropped": topic_dropped,
         "directive_nudge": bool(nudge_block),
+        # Switchroom P2 — the ISO ask-time anchor actually sent to recall this
+        # turn (null when the inbound had no temporal phrase), so the field's
+        # firing rate is measurable from day one against the RFC's falsification
+        # window (precedent: `directive_nudge` above).
+        "query_timestamp": query_timestamp,
+        # RFC P3 — profile-capture nudge firing rate, measurable from day one
+        # (mirrors the directive_nudge precedent).
+        "profile_nudge": bool(profile_nudge_block),
         # Switchroom hindsight-leverage E1 / PR8 (#3369) — transcript-grep
         # fallback telemetry so its firing (and its bounds) are visible per turn
         # in recall_log.jsonl. `transcript_fallback` True only on an all-zero,
@@ -2761,10 +3092,19 @@ def main():
     # is precisely the turn on which the agent must not assume it remembers.
     # #3837: so is a set the score floor withheld entirely.
     if not directives_block and not memories_block and not transcript_fallback_block:
-        if degraded_block or withheld_block or nudge_block:
+        if degraded_block or withheld_block or nudge_block or profile_nudge_block:
             _emit_cached_context(
                 "\n\n".join(
-                    [b for b in (degraded_block, withheld_block, nudge_block) if b]
+                    [
+                        b
+                        for b in (
+                            degraded_block,
+                            withheld_block,
+                            nudge_block,
+                            profile_nudge_block,
+                        )
+                        if b
+                    ]
                 )
             )
         return
@@ -2822,9 +3162,13 @@ def main():
             "hookEventName": "UserPromptSubmit",
             "additionalContext": _combine_context(
                 _combine_context(
-                    _combine_context(degraded_block, withheld_block), context_message
+                    _combine_context(
+                        _combine_context(degraded_block, withheld_block),
+                        context_message,
+                    ),
+                    nudge_block,
                 ),
-                nudge_block,
+                profile_nudge_block,
             ),
         }
     }

package/vendor/hindsight-memory/scripts/reconcile_tail.py

@@ -109,18 +109,10 @@ def _human_turns(messages: list) -> int:
     )
 
 
-def _tail_after(messages: list, last_uuid: str | None) -> list:
-    """Return the transcript entries AFTER the watermark anchor.
-
-    No watermark, or a stale anchor compaction removed → the whole transcript is
-    the gap (a safe re-upsert, never a skip).
-    """
-    if not last_uuid:
-        return list(messages)
-    for i, m in enumerate(messages):
-        if isinstance(m, dict) and m.get("uuid") == last_uuid:
-            return messages[i + 1:]
-    return list(messages)
+# The transcript tail-slice is shared with the incremental SessionEnd sweep
+# (retain.py, memory-RFC P1); the single implementation lives in lib.watermark
+# so there is only ever one copy of this reconcile/watermark-critical slice.
+_tail_after = watermark.tail_after
 
 
 def _session_id_from_path(path: str) -> str:

package/vendor/hindsight-memory/scripts/retain.py

@@ -366,6 +366,7 @@ def select_retain_window(
     overlap_turns: int,
     all_messages: list,
     force: bool = False,
+    since_uuid: str | None = None,
 ) -> tuple:
     """Decide which messages to retain and whether to send as a full window.
 
@@ -396,6 +397,17 @@ def select_retain_window(
     the whole session even if per-turn windowing had an edge. This costs a
     full sweep only ONCE per session (at end), not per turn.
 
+    ``since_uuid`` (memory-RFC P1) makes that final sweep INCREMENTAL: when
+    ``force=True`` and a committed watermark uuid is supplied, only the
+    transcript tail after it is retained, instead of re-storing the whole
+    session on top of the per-window documents that already carry it. The value
+    is resolved by the caller (``run_retain`` reads the watermark, wrapped) and
+    passed in — this function stays pure and IO-free. ``since_uuid=None``
+    reproduces the whole-session sweep exactly, which is what the vendor
+    full-session path and the ``retainEveryNTurns==1`` path pass. The slice
+    degrades to the whole transcript on a missing/compacted anchor
+    (``watermark.tail_after``), so a forced sweep never emits an empty slice.
+
     Durability invariant (jtbd-memory-survives-restart UAT): the window
     always extends to the END of the transcript (``slice_last_turns_by_user_boundary``
     returns ``messages[start:]``), so the turn that just completed — the one
@@ -412,8 +424,21 @@ def select_retain_window(
         window_turns = max(retain_every_n, 1) + overlap_turns
         messages_to_retain = slice_last_turns_by_user_boundary(all_messages, window_turns)
         return messages_to_retain, True
+    # Forced (SessionEnd) chunked sweep with a committed watermark uuid: retain
+    # only the transcript tail after it, instead of re-storing the whole session
+    # on top of the N per-window documents that already carry the same content
+    # (memory-RFC P1). ``since_uuid`` is None on the vendor full-session path and
+    # at retainEveryNTurns==1 (the caller withholds it there — see run_retain),
+    # which preserves today's whole-session behaviour byte-for-byte.
+    #
+    # tail_after is PURE and IO-free (the watermark READ happens in run_retain,
+    # wrapped) and degrades to the whole transcript on a missing/compacted
+    # anchor, so this branch can never emit an empty slice into the retain seam.
+    if force and since_uuid is not None:
+        return watermark.tail_after(all_messages, since_uuid), True
     # Full session: vendor full-session mode, OR a forced (SessionEnd) chunked
-    # sweep. Retain all messages, always as a full window.
+    # sweep with no watermark to slice against. Retain all messages, always as a
+    # full window.
     return list(all_messages), True
 
 
@@ -798,8 +823,37 @@ def run_retain(hook_input: dict, force: bool = False) -> dict:
     # select_retain_window() for the switchroom-divergence rationale
     # (Phase 6b: chunked window-slicing now works at retainEveryNTurns=1).
     overlap_turns = config.get("retainOverlapTurns", 0)
+
+    # Incremental SessionEnd sweep (memory-RFC P1): a forced chunked sweep at
+    # retainEveryNTurns>1 retains only the transcript tail after the committed
+    # watermark, not the whole session on top of the N per-window documents that
+    # already carry it. The watermark READ is filesystem IO on a JSON file that
+    # could be truncated / permission-broken / absent, so it lives HERE (not in
+    # the pure select_retain_window) inside a catch-all try/except.
+    #
+    # §4.3 HAZARD: this is the one seam where a raise DELETES a turn rather than
+    # degrading it. Every failure mode below MUST degrade to since_uuid=None,
+    # which reproduces today's whole-session sweep exactly. The except catches
+    # Exception (not a named subset) on purpose — any raise here is worse than a
+    # wrong value. The n==1 and full-session paths deliberately never compute a
+    # watermark: on those the document id is {session_id} and a tail slice would
+    # TRUNCATE the stored document (§4.2), so they keep the full-session sweep.
+    since_uuid = None
+    if force and retain_mode == "chunked" and retain_every_n > 1:
+        try:
+            _wm = watermark.load(session_id)
+            since_uuid = _wm.get("last_uuid") if _wm else None
+        except Exception as e:  # noqa: BLE001 - degrade, never raise into this seam
+            print(
+                "[Hindsight] watermark read failed for incremental sweep; "
+                f"falling back to full-session sweep: {e}",
+                file=sys.stderr,
+            )
+            since_uuid = None
+
     messages_to_retain, retain_full_window = select_retain_window(
-        retain_mode, retain_every_n, overlap_turns, all_messages, force=force
+        retain_mode, retain_every_n, overlap_turns, all_messages, force=force,
+        since_uuid=since_uuid,
     )
     if retain_mode == "chunked" and not force:
         window_turns = max(retain_every_n, 1) + overlap_turns
@@ -808,9 +862,11 @@ def run_retain(hook_input: dict, force: bool = False) -> dict:
             f"Chunked retain firing (window: {window_turns} human turns, {len(messages_to_retain)} messages)",
         )
     elif retain_mode == "chunked" and force:
+        _swept = "incremental (tail after watermark)" if since_uuid is not None else "full-session"
         debug_log(
             config,
-            f"Chunked retain, forced full-session sweep (SessionEnd): {len(all_messages)} messages",
+            f"Chunked retain, forced {_swept} sweep (SessionEnd): "
+            f"{len(messages_to_retain)}/{len(all_messages)} messages",
         )
     else:
         debug_log(config, f"Full session retain: {len(all_messages)} messages")

package/vendor/hindsight-memory/scripts/tests/test_config_retain_tool_calls_env.py

@@ -0,0 +1,98 @@
+"""Switchroom — `retainToolCalls` must have an env channel, default unchanged.
+
+RFC memory-redesign P4. `retainToolCalls` (whether retain stores tool_use
+inputs + tool_result content) had a DEFAULTS entry of True but NO entry in
+`ENV_OVERRIDES` and no scaffold stamp, so an operator could not reach it: env
+is the TOP of the plugin's config precedence chain (DEFAULTS -> settings.json
+-> ~/.hindsight/claude-code.json -> env), and without an env key
+`HINDSIGHT_RETAIN_TOOL_CALLS` could not steer the plugin at all — including a
+docker-exec'd retain/backfill that does not inherit the supervised settings.
+
+This PR ships the SETTER ONLY. The invariants under test:
+
+  1. `HINDSIGHT_RETAIN_TOOL_CALLS` is wired in `ENV_OVERRIDES` and maps to the
+     `retainToolCalls` config key.
+  2. The shipped default is unchanged — `True` — so absent any override the
+     fleet behaves byte-identically (the load resolves True with no env set).
+  3. Setting the env `false` actually LANDS as Python `False` over the True
+     default (proving `false` is reachable, which is the whole point of the
+     knob), and `true`/`1`/`yes` resolve back to True.
+
+Stdlib-only.
+"""
+
+import os
+import sys
+import unittest
+from unittest import mock
+
+SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+if SCRIPTS_DIR not in sys.path:
+    sys.path.insert(0, SCRIPTS_DIR)
+
+from lib.config import DEFAULTS, ENV_OVERRIDES, load_config  # noqa: E402
+
+ENV_NAME = "HINDSIGHT_RETAIN_TOOL_CALLS"
+CONFIG_KEY = "retainToolCalls"
+
+
+def _load_with(env):
+    """load_config() with a hermetic environment (no plugin/user settings)."""
+    with mock.patch.dict(os.environ, env, clear=True):
+        os.environ["CLAUDE_PLUGIN_ROOT"] = os.path.join(SCRIPTS_DIR, "does-not-exist")
+        os.environ["HOME"] = os.path.join(SCRIPTS_DIR, "does-not-exist")
+        return load_config()
+
+
+class RetainToolCallsHasAChannel(unittest.TestCase):
+    def test_env_name_is_wired(self):
+        self.assertIn(
+            ENV_NAME,
+            ENV_OVERRIDES,
+            f"{ENV_NAME} exported nowhere / never read — the drift P4 closes",
+        )
+
+    def test_env_name_maps_to_the_expected_config_key(self):
+        self.assertEqual(ENV_OVERRIDES[ENV_NAME][0], CONFIG_KEY)
+
+    def test_env_name_is_typed_bool(self):
+        self.assertIs(ENV_OVERRIDES[ENV_NAME][1], bool)
+
+    def test_target_key_exists_in_defaults(self):
+        self.assertIn(CONFIG_KEY, DEFAULTS)
+
+
+class DefaultIsUnchanged(unittest.TestCase):
+    """The critical P4 invariant: default stays True, byte-identical fleet."""
+
+    def test_shipped_default_is_true(self):
+        self.assertIs(DEFAULTS[CONFIG_KEY], True)
+
+    def test_no_env_reproduces_the_true_default(self):
+        cfg = _load_with({})
+        self.assertIs(cfg[CONFIG_KEY], True)
+
+
+class ValuesActuallyLand(unittest.TestCase):
+    """The setter must be reachable: env `false` lands as Python False."""
+
+    def test_false_env_overrides_the_true_default(self):
+        cfg = _load_with({ENV_NAME: "false"})
+        self.assertIs(cfg[CONFIG_KEY], False)
+        self.assertNotEqual(cfg[CONFIG_KEY], DEFAULTS[CONFIG_KEY])
+
+    def test_zero_and_no_also_resolve_false(self):
+        for falsey in ("0", "no", "False"):
+            with self.subTest(value=falsey):
+                cfg = _load_with({ENV_NAME: falsey})
+                self.assertIs(cfg[CONFIG_KEY], False)
+
+    def test_true_env_resolves_true(self):
+        for truthy in ("true", "1", "yes", "TRUE"):
+            with self.subTest(value=truthy):
+                cfg = _load_with({ENV_NAME: truthy})
+                self.assertIs(cfg[CONFIG_KEY], True)
+
+
+if __name__ == "__main__":
+    unittest.main()