switchroom 0.19.17 → 0.19.19

package/vendor/hindsight-memory/scripts/lib/client.py

@@ -5,12 +5,15 @@ Openclaw HindsightClient (client.js), adapted for Python stdlib.
 """
 
 import json
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
 from pathlib import Path
 from typing import Optional
 
+from .retain_split import part_document_id, part_metadata, split_retain_content
+
 DEFAULT_TIMEOUT = 15  # seconds
 HEALTH_CHECK_RETRIES = 3
 HEALTH_CHECK_DELAY = 2  # seconds
@@ -189,7 +192,85 @@ class HindsightClient:
         reconciliation) MUST use this so a bare async 200 can never falsely mark
         unpersisted work as committed. (Merge precondition: the daemon honours
         ``async=false`` as commit-before-ack — verified by the §1.1 probe.)
+
+        **Oversized content is split before it is posted** (``lib/retain_split``).
+        The daemon runs one sequential extraction LLM call per
+        ``retain_chunk_size`` (3000) chars, so server wall time is linear in
+        ``len(content)`` while the client has a single deadline for the whole
+        POST — past ~45,000 chars a retain cannot complete at ANY client
+        timeout and the memory is permanently unsaveable (measured: 154 of the
+        629 entries in the 2026-07-25 fleet backlog). This is the enforcement
+        point precisely because every retain POST in the plugin goes through
+        it, so the bound is code-enforced rather than left to each producer.
+
+        Parts are posted SEQUENTIALLY, each with the caller's ``timeout`` (the
+        timeout is a per-HTTP-request read deadline, and each part is its own
+        request) and each with the caller's ``async_processing`` — so a
+        durability caller still gets commit-before-ack per part. Any part
+        failing raises, exactly as an unsplit failure does, so the caller's
+        existing enqueue/retry handling is unchanged; already-committed parts
+        carry deterministic ids and are upserted, not duplicated, on retry.
+
+        ``timeout`` bounds the CALLER'S TOTAL WALL TIME, not just each HTTP
+        request. Splitting must never turn one bounded POST into N of them:
+        the Stop hook passes ``timeout=15`` because it has a hook budget to
+        respect, and ``15 × N`` would stall the session. Once the budget is
+        spent no further part is started and the call raises, exactly as an
+        unsplit failure does, so each caller's EXISTING failure path handles
+        the remainder: the hook paths (``retain.py``, ``subagent_retain.py``,
+        ``reconcile_tail.py``) enqueue, and ``pending.enqueue`` queues the
+        remainder as bounded per-part entries the drainer can finish; the
+        drain paths count an attempt and keep the entry;
+        ``backfill_transcripts.py`` logs and backs off. The parts already
+        committed are upserted on the next attempt, not duplicated.
         """
+        parts = split_retain_content(content)
+        total = len(parts)
+        response = None
+        deadline = time.monotonic() + timeout
+        part_timeout = timeout
+        for index, part in enumerate(parts):
+            if index > 0:
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    raise TimeoutError(
+                        f"retain wall budget of {timeout}s exhausted after "
+                        f"{index}/{total} parts; no further part was started. "
+                        f"The caller's own failure path decides the remainder: "
+                        f"the hook paths enqueue it (as bounded per-part "
+                        f"entries), the drain paths count an attempt and keep "
+                        f"the entry queued."
+                    )
+                # Clamp the request deadline to what is left of the budget so
+                # the final part cannot overrun it either.
+                part_timeout = max(1, int(remaining))
+            response = self._retain_one(
+                bank_id=bank_id,
+                content=part,
+                document_id=part_document_id(document_id, index, total),
+                context=context,
+                metadata=part_metadata(metadata, index, total),
+                tags=tags,
+                timeout=part_timeout,
+                async_processing=async_processing,
+            )
+        if total > 1 and isinstance(response, dict):
+            response = dict(response)
+            response["split_parts"] = total
+        return response
+
+    def _retain_one(
+        self,
+        bank_id: str,
+        content: str,
+        document_id: str,
+        context: Optional[str],
+        metadata: Optional[dict],
+        tags: Optional[list],
+        timeout: int,
+        async_processing: bool,
+    ) -> dict:
+        """POST exactly one retain item. Raises on any HTTP/transport error."""
         path = f"/v1/default/banks/{urllib.parse.quote(bank_id, safe='')}/memories"
         item = {
             "content": content,
@@ -206,6 +287,49 @@ class HindsightClient:
         }
         return self._request("POST", path, body, timeout=timeout)
 
+    def document_exists(
+        self,
+        bank_id: str,
+        document_id: str,
+        timeout: int = 30,
+    ) -> Optional[bool]:
+        """TRI-STATE presence check for one document (switchroom #3596).
+
+        ``True`` — the document is there. ``False`` — the server said 404.
+        ``None`` — **unknown** (transport error, 5xx, timeout).
+
+        The tri-state is load-bearing and must not be collapsed to a bool.
+        Two callers depend on it:
+
+        * the backlog drain's reconcile phase, which SKIPS a retain when the
+          memory already exists. Treating "unknown" as ``False`` there would
+          re-POST a document that is already durable — the duplicated-LLM-cost
+          bug this check exists to avoid (70.4% of one measured 5,751-entry
+          fleet backlog already existed as documents).
+        * commit-before-delete, which only deletes a queue entry once presence
+          is CONFIRMED. Treating "unknown" as ``True`` there would delete the
+          last on-disk copy of a turn on a flaky GET — the #3244 silent-loss
+          shape, reintroduced from the other direction.
+
+        Deliberately does NOT reuse ``_request()``: that wraps every
+        ``HTTPError`` into a ``RuntimeError``, which would make a 404
+        indistinguishable from a 503 without string-matching the message.
+        """
+        bank = urllib.parse.quote(bank_id, safe="")
+        did = urllib.parse.quote(document_id, safe="")
+        url = f"{self.api_url}/v1/default/banks/{bank}/documents/{did}"
+        req = urllib.request.Request(url, headers=self._headers(), method="GET")
+        try:
+            with urllib.request.urlopen(
+                req, timeout=self._resolve_timeout(timeout)
+            ) as resp:
+                resp.read()
+                return True
+        except urllib.error.HTTPError as e:
+            return False if e.code == 404 else None
+        except Exception:
+            return None
+
     def list_session_document_ids(
         self,
         bank_id: str,

package/vendor/hindsight-memory/scripts/lib/config.py

@@ -29,15 +29,20 @@ DEFAULTS = {
     # formatting. Set to 0 (or any non-positive value) to disable the cap
     # and inject everything Hindsight returns.
     "recallMaxMemories": 12,
-    # Switchroom-local: minimum lexical (Jaccard) overlap between the
+    # Switchroom-local: minimum lexical (containment) overlap between the
     # user's query terms and a memory's text terms. Memories below this
     # threshold are dropped before formatting. 0.0 disables the gate
     # (current behaviour: inject everything Hindsight returns up to the
     # count cap). NOTE: Hindsight's HTTP recall API DOES return per-result
     # relevance scores (`scores.final`, plus `.semantic`/`.keyword`/
     # `.reranker`) — verified at runtime — and recall.py now reads and
-    # sorts the merged set by `scores.final`. This Jaccard gate is a
-    # separate lexical-overlap quality filter layered on top — see #475.
+    # sorts the merged set by `scores.final`. This lexical gate is a
+    # separate quality filter layered on top — see #475. The metric is
+    # containment, `|Q n M| / |M|`, not Jaccard: dividing by the union made
+    # the score a function of prompt length rather than relevance — see
+    # #3541 and recall.py's design note. At the 0.10 fleet default this is
+    # close to a passthrough (a <=10-token memory clears it on one shared
+    # word); precision is the engine rerank's job, not this gate's.
     "recallMinOverlap": 0.0,
     "recallTypes": ["world", "experience"],
     # Switchroom-local: when True (default; Ken-approved ON) recall biases

package/vendor/hindsight-memory/scripts/lib/directives.py

@@ -23,10 +23,33 @@ from typing import Optional
 
 from .state import list_state_names, read_state, remove_state, write_state
 
-# Sanity cap on how many directives we ever inject into the prompt. Banks
-# with more active directives than this are pathological; truncate with a
-# footer so the agent knows there are more.
-MAX_DIRECTIVES = 15
+# Sanity cap on how many directives we ever inject into the prompt.
+#
+# This number is a COST TRADEOFF, not an arbitrary limit, and the real cost is
+# larger than "one block": `recall.py` rebuilds the <active_directives> block
+# on EVERY UserPromptSubmit (the `format_active_directives_block` call there)
+# with no per-session dedupe and no "unchanged since last turn" suppression,
+# and `additionalContext` is APPENDED into the conversation. So the block is
+# re-paid every turn and accumulates — roughly (block size) x (turn count) over
+# a session, not once.
+#
+# Measured live 2026-07-25 (fleet REST `/directives`):
+#   overlord  12 active  ~9.8 KB   ~816 chars avg  (one 2,981-char outlier)
+#   klanker   17 active  ~9.9 KB   ~585 chars avg
+#   gymbro     9 active  ~9.3 KB  ~1038 chars avg
+# At the ~700-char fleet average, a bank sitting at this cap injects ~21 KB —
+# on the order of 5,000-6,000 tokens per injection, per turn, cumulative across
+# the session. 30 is chosen to clear the observed fleet maximum with headroom;
+# it is NOT a size at which a runaway bank becomes harmless. The actual defence
+# against pile-up is the doctor's WARN/FAIL on the active directive count
+# (src/cli/doctor-memory.ts), not this cap. Raising it further is a legitimate
+# call — make it deliberately, with the per-turn-times-turns cost in mind, and
+# move the doctor thresholds with it.
+#
+# Banks with more active directives than this are pathological; we truncate
+# with an in-prompt footer, a `directives_omitted` field on the recall_log row,
+# and a stderr warning (see `format_active_directives_block`).
+MAX_DIRECTIVES = 30
 
 # Hard timeout for the list_directives call. The recall hook is on the
 # UserPromptSubmit critical path — we cannot block it for long.
@@ -208,6 +231,16 @@ def invalidate_directives_cache(bank_id: Optional[str] = None) -> None:
         remove_state(name)
 
 
+def count_omitted_directives(directives: list, max_directives: int = MAX_DIRECTIVES) -> int:
+    """How many directives `format_active_directives_block` would DROP.
+
+    Pure counterpart of the truncation branch below, so `recall.py` can put the
+    number on the recall_log row without re-deriving the cap. 0 when nothing is
+    dropped.
+    """
+    return max(0, len(directives) - max_directives)
+
+
 def format_active_directives_block(directives: list, max_directives: int = MAX_DIRECTIVES) -> Optional[str]:
     """Format directives into the <active_directives> block string.
 
@@ -252,6 +285,31 @@ def format_active_directives_block(directives: list, max_directives: int = MAX_D
     if omitted > 0:
         lines.append("")
         lines.append(f"(+{omitted} more, omitted)")
+        # The in-prompt footer above only tells the AGENT. This stderr warn is
+        # the same channel every other operational failure in this module uses
+        # (see `_fetch_directives_with_status`), and it is a LAST-RESORT
+        # breadcrumb only — do NOT rely on it reaching an operator.
+        #
+        # Measured 2026-07-25: `docker logs --tail 20000` across all 12 running
+        # agent containers returns ZERO `[Hindsight]` lines, and nothing under
+        # ~/.switchroom/logs/ contains them either, despite months of runtime
+        # and several long-standing stderr paths in recall.py. Claude Code
+        # appears to swallow hook stderr on a zero exit, so hook stderr is not
+        # an operator-visible channel.
+        #
+        # The channels that DO reach an operator:
+        #   * the `directives_omitted` field on the recall_log row
+        #     (state/recall_log.jsonl — see `count_omitted_directives`), and
+        #   * `switchroom doctor`'s WARN/FAIL on the bank's active directive
+        #     count (src/cli/doctor-memory.ts `classifyDirectiveCount`), which
+        #     reads the count from the same REST surface this module fetches.
+        print(
+            f"[Hindsight] directive truncation: {total} active directives exceeds "
+            f"MAX_DIRECTIVES={max_directives} — {omitted} lowest-priority "
+            f"directive(s) were DROPPED from this turn's prompt. Merge or retire "
+            f"directives (mental-model-curator) or raise MAX_DIRECTIVES.",
+            file=sys.stderr,
+        )
 
     lines.append("</active_directives>")
     return "\n".join(lines)