switchroom 0.20.10 → 0.20.11

package/vendor/hindsight-memory/scripts/retain.py

@@ -22,9 +22,21 @@ import json
 import os
 import sys
 import time
+from datetime import datetime, timezone
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
+# Switchroom self-improve PR4 (slice 4a) — per-turn correction tagging.
+# The self-improve Stop hook (a SEPARATE process — src/cli/self-improve-stop.ts)
+# drops SELF_IMPROVE_CORRECTION_PENDING_FILE into the shared agent state dir when
+# the deterministic gate fires on an operator-correction. This hook reads-and-
+# clears it and stamps SELF_IMPROVE_CORRECTION_TAG on that turn's retain so PR5's
+# failure-synthesis cron can recall correction turns cheaply. Both constants are
+# pinned to their TS source (src/memory/hindsight-retain-provenance.ts) by
+# tests/scaffold.retain-provenance.test.ts — they must not drift.
+SELF_IMPROVE_CORRECTION_PENDING_FILE = "self-improve-correction-pending"
+SELF_IMPROVE_CORRECTION_TAG = "self-improve:correction"
+
 from lib import watermark
 from lib.bank import derive_bank_id, ensure_bank_mission
 from lib.client import HindsightClient
@@ -38,6 +50,242 @@ from lib.pacing import inflight_lock
 from lib.state import increment_turn_count, track_retention
 
 
+def _self_improve_state_dir() -> str:
+    """State dir the self-improve Stop hook drops its correction sentinel into.
+
+    Mirrors ``resolveStateDir()`` in ``src/cli/self-improve-stop.ts``:
+    ``TELEGRAM_STATE_DIR`` when set, else ``~/.claude/channels/telegram``. Both
+    hooks run in the same agent process env, so they agree on this path.
+    """
+    env = os.environ.get("TELEGRAM_STATE_DIR")
+    if env:
+        return env
+    return os.path.join(os.path.expanduser("~"), ".claude", "channels", "telegram")
+
+
+def read_and_clear_correction_pending(state_dir: str | None = None) -> list:
+    """Read-and-clear the per-turn operator-correction sentinel (PR4 slice 4a).
+
+    Returns ``[SELF_IMPROVE_CORRECTION_TAG]`` when the sentinel is present (and
+    REMOVES it, so exactly one retain carries the tag — read-once), else ``[]``.
+    Called from ``run_retain`` only (the live Stop-hook entry), NOT from the
+    network-free ``build_retain_payload`` seam, so backfill / reconcile /
+    subagent retains never consume a live turn's marker. Best-effort and NEVER
+    raises — a triage marker must not be able to fail a retain.
+    """
+    try:
+        d = state_dir if state_dir is not None else _self_improve_state_dir()
+        path = os.path.join(d, SELF_IMPROVE_CORRECTION_PENDING_FILE)
+        if not os.path.exists(path):
+            return []
+        try:
+            os.remove(path)
+        except OSError:
+            pass
+        return [SELF_IMPROVE_CORRECTION_TAG]
+    except Exception:
+        return []
+
+
+PRIVACY_STATE_FILE = "privacy-state.json"
+
+
+def _parse_iso(value) -> "datetime | None":
+    """Best-effort parse of an ISO-8601 timestamp to an aware ``datetime``.
+
+    Accepts the ``...Z`` (UTC) suffix the gateway writes and the offset forms
+    ``datetime.fromisoformat`` understands. Returns ``None`` on anything
+    unparseable — never raises. A naive result (no tz) is coerced to UTC so all
+    comparisons in ``exclude_private_ranges`` are between aware datetimes.
+    """
+    if not isinstance(value, str) or not value:
+        return None
+    try:
+        s = value.strip()
+        if s.endswith("Z"):
+            s = s[:-1] + "+00:00"
+        dt = datetime.fromisoformat(s)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        return dt
+    except (ValueError, TypeError):
+        return None
+
+
+PRIVACY_STATE_CORRUPT_KEY = "__corrupt__"
+
+
+def read_privacy_state(state_dir: str | None = None) -> list:
+    """Read the shared /private-mode interval list (switchroom privacy PR1).
+
+    Contract with the Telegram gateway's ``/private`` / ``/public`` commands:
+    the gateway maintains ``${TELEGRAM_STATE_DIR}/privacy-state.json`` (same dir
+    fallback as ``_self_improve_state_dir``) shaped::
+
+        {"version": 1, "intervals": [
+            {"start": "<iso>", "end": "<iso>"},   # a closed private window
+            {"start": "<iso>", "end": null}       # the OPEN window = private NOW
+        ]}
+
+    Returns the ``intervals`` list (list of ``{"start", "end"}`` dicts).
+
+    ABSENT vs CORRUPT are deliberately DISTINCT (review MAJOR 1 —
+    defense-in-depth, independent of the gateway's write atomicity):
+
+    - A **missing** file, or a valid JSON object with no ``intervals`` key,
+      yields ``[]`` — the default = fully public.
+    - A file that **exists but cannot be parsed** (truncated mid-rewrite,
+      not-a-JSON-object, ``intervals`` not a list) must NOT collapse to the same
+      "public" ``[]`` — that would leak the just-completed private turn if a Stop
+      hook fired during a non-atomic rewrite. It returns a single CORRUPT
+      sentinel interval and logs loudly; downstream this fails TOWARD privacy
+      (``_has_open_interval`` → True, ``exclude_private_ranges`` → drop all).
+
+    Best-effort — NEVER raises: a privacy-state read must not be able to fail a
+    retain.
+    """
+    try:
+        d = state_dir if state_dir is not None else _self_improve_state_dir()
+        path = os.path.join(d, PRIVACY_STATE_FILE)
+        if not os.path.isfile(path):
+            return []  # absent = public (the default)
+    except Exception:
+        # Could not even resolve/stat the path — treat as absent (public).
+        return []
+    # The file EXISTS. From here a parse/read failure fails TOWARD privacy.
+    try:
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+        if not isinstance(data, dict):
+            raise ValueError("privacy-state.json is not a JSON object")
+        intervals = data.get("intervals")
+        if intervals is None:
+            return []  # valid JSON, no intervals declared = public
+        if not isinstance(intervals, list):
+            raise ValueError("privacy-state.json .intervals is not a list")
+        return [iv for iv in intervals if isinstance(iv, dict)]
+    except Exception as e:
+        print(
+            f"[Hindsight] privacy-state.json present but unreadable ({e!r}); "
+            f"failing TOWARD privacy — treating this session as private-now",
+            file=sys.stderr,
+        )
+        return [{PRIVACY_STATE_CORRUPT_KEY: True}]
+
+
+def _classify_privacy_intervals(intervals: list) -> tuple:
+    """Parse raw intervals into ``(parsed, fail_private)`` — the single source of
+    truth both ``_has_open_interval`` and ``exclude_private_ranges`` use so they
+    can NEVER disagree on what "open" means (review MINOR).
+
+    - ``parsed``: list of ``(start_dt, end_dt_or_None)`` aware-datetime tuples;
+      ``end_dt is None`` means an OPEN ``[start, ∞)`` window.
+    - ``fail_private``: True when the state must be treated as private-now,
+      unbounded — a CORRUPT sentinel, or an interval whose ``start`` is present
+      but unparseable (we cannot place the window, so we refuse to un-redact).
+
+    A malformed (present-but-unparseable) ``end`` degrades to OPEN ``[start, ∞)``
+    in BOTH consumers — and is LOGGED, not silently applied. Best-effort; never
+    raises.
+    """
+    parsed = []
+    fail_private = False
+    for iv in intervals:
+        if not isinstance(iv, dict):
+            continue
+        if iv.get(PRIVACY_STATE_CORRUPT_KEY):
+            fail_private = True
+            continue
+        start_raw = iv.get("start")
+        start_dt = _parse_iso(start_raw)
+        if start_dt is None:
+            if start_raw is not None:
+                # Present but unparseable start: cannot place the window — fail
+                # toward privacy rather than silently ignoring a private range.
+                print(
+                    f"[Hindsight] privacy interval has unparseable start "
+                    f"{start_raw!r}; failing TOWARD privacy (private-now)",
+                    file=sys.stderr,
+                )
+                fail_private = True
+            continue  # start missing entirely = an empty entry, covers nothing
+        end_raw = iv.get("end")
+        if end_raw is None:
+            parsed.append((start_dt, None))
+            continue
+        end_dt = _parse_iso(end_raw)
+        if end_dt is None:
+            # Present but unparseable end: degrade to OPEN [start, ∞) AND surface
+            # it, so the guard and the redactor agree (no silent unbounded wipe).
+            print(
+                f"[Hindsight] privacy interval has unparseable end {end_raw!r}; "
+                f"treating as OPEN [start, ∞) — check the gateway writer",
+                file=sys.stderr,
+            )
+            parsed.append((start_dt, None))
+            continue
+        parsed.append((start_dt, end_dt))
+    return parsed, fail_private
+
+
+def _has_open_interval(intervals: list) -> bool:
+    """True when the state is private-right-now: some OPEN ``[start, ∞)`` window,
+    OR a corrupt/unplaceable state that fails toward privacy. Shares the
+    classifier with ``exclude_private_ranges`` so the two never diverge."""
+    parsed, fail_private = _classify_privacy_intervals(intervals)
+    if fail_private:
+        return True
+    return any(end is None for _, end in parsed)
+
+
+def exclude_private_ranges(messages: list, intervals: list) -> list:
+    """Drop any message whose ``timestamp`` falls inside a private interval.
+
+    An interval ``{"start": s, "end": e}`` covers ``[s, e]`` (inclusive, toward
+    privacy); an OPEN interval (``end`` is null, or a malformed ``end``) covers
+    ``[s, ∞)``. A message whose ``timestamp`` is missing or unparseable is
+    dropped ONLY when some interval is open (conservative — we cannot place it,
+    and private-now is live), otherwise kept.
+
+    A CORRUPT state or an unplaceable ``start`` (``fail_private``) drops ALL
+    messages — failing toward privacy rather than leaking on a bad file.
+    Best-effort and total: the function never raises.
+    """
+    if not intervals:
+        return list(messages)
+
+    parsed, fail_private = _classify_privacy_intervals(intervals)
+    if fail_private:
+        # Corrupt / unplaceable state: refuse to retain anything this pass.
+        return []
+    if not parsed:
+        return list(messages)
+
+    any_open = any(end is None for _, end in parsed)
+
+    def _in_any_range(ts_dt) -> bool:
+        for start_dt, end_dt in parsed:
+            if ts_dt < start_dt:
+                continue
+            if end_dt is None or ts_dt <= end_dt:
+                return True
+        return False
+
+    kept = []
+    for m in messages:
+        ts_dt = _parse_iso(m.get("timestamp")) if isinstance(m, dict) else None
+        if ts_dt is None:
+            # No placeable timestamp: drop only if a private window is open now.
+            if any_open:
+                continue
+            kept.append(m)
+            continue
+        if _in_any_range(ts_dt):
+            continue
+        kept.append(m)
+    return kept
+
+
 def read_transcript(transcript_path: str, max_bytes: int | None = None) -> list:
     """Read a JSONL transcript file and return list of message dicts.
 
@@ -92,6 +340,15 @@ def read_transcript(transcript_path: str, max_bytes: int | None = None) -> list:
                             if uid is not None and "uuid" not in msg:
                                 msg = dict(msg)
                                 msg["uuid"] = uid
+                            # Surface the transcript-entry timestamp the same
+                            # way (nested format carries it on the OUTER entry).
+                            # The /private-mode redaction filters on it
+                            # (exclude_private_ranges); without surfacing it here
+                            # every message would look timestamp-less.
+                            ts = entry.get("timestamp")
+                            if ts is not None and "timestamp" not in msg:
+                                msg = dict(msg)
+                                msg["timestamp"] = ts
                             messages.append(msg)
                     # Flat format (testing / future compatibility)
                     elif "role" in entry and "content" in entry:
@@ -254,6 +511,7 @@ def build_retain_payload(
     api_token,
     retain_full_window: bool = True,
     document_id=None,
+    extra_tags=None,
 ) -> dict | None:
     """Pure transcript → retain payload + deterministic id (switchroom #3244 §1.4).
 
@@ -323,6 +581,23 @@ def build_retain_payload(
                 merged.append(lt)
         tags = merged
 
+    # Switchroom self-improve PR4 (slice 4a) — per-turn correction tag. The
+    # caller (``run_retain``) reads-and-clears the operator-correction sentinel
+    # and threads the resulting tag(s) in here; this seam stays network-free and
+    # state-free (no sentinel IO), so it composes with lesson tags and rides the
+    # SAME payload the pending-retains queue persists. ``self-improve:correction``
+    # is STABLE by contract (it must NOT match DEFAULT_VOLATILE_SCOPE_PATTERNS),
+    # so on a correction turn the retain lands in its own
+    # ``[["self-improve:correction"]]`` consolidation scope — exactly the
+    # partition PR5's failure-synthesis cron works over. Absent ⇒ tag absent ⇒
+    # scope stays byte-identical to a normal turn's ``"shared"``.
+    if extra_tags:
+        merged = list(tags) if tags else []
+        for et in extra_tags:
+            if isinstance(et, str) and et and et not in merged:
+                merged.append(et)
+        tags = merged or None
+
     metadata = {
         "retained_at": template_vars["timestamp"],
         "message_count": str(message_count),
@@ -438,6 +713,17 @@ def run_retain(hook_input: dict, force: bool = False) -> dict:
     ``lib/pending.py``). ``status="skipped"`` is the normal early-exit
     cases (auto-retain disabled, empty transcript, throttled chunk).
     """
+    # /private-mode enforcement (switchroom privacy PR1) — FIRST, before
+    # load_config(). An OPEN private interval means the operator is discussing
+    # confidential material right now, so a normal (non-forced) auto-retain must
+    # not fire at all. Placed above load_config() deliberately so a
+    # HINDSIGHT_AUTO_RETAIN=true env pin (the autoRetain gate below) cannot
+    # override the privacy guarantee. A FORCED SessionEnd sweep is EXEMPT here on
+    # purpose — it must still flush the PUBLIC portion of the session — and
+    # relies instead on exclude_private_ranges() dropping the private turns.
+    if not force and _has_open_interval(read_privacy_state()):
+        return {"status": "skipped", "reason": "private-mode"}
+
     config = load_config()
 
     if not config.get("autoRetain"):
@@ -472,6 +758,17 @@ def run_retain(hook_input: dict, force: bool = False) -> dict:
         debug_log(config, "No messages in transcript, skipping retain")
         return {"status": "skipped", "reason": "empty transcript"}
 
+    # /private-mode redaction (switchroom privacy PR1) — drop any message that
+    # falls inside a private interval BEFORE window selection. This single
+    # insertion covers BOTH the chunked sliding window AND the force/full-session
+    # slice (select_retain_window returns list(all_messages) for force=True), so
+    # a forced SessionEnd sweep flushes only the PUBLIC portion and an
+    # open-at-session-end private range is never force-swept into the bank.
+    all_messages = exclude_private_ranges(all_messages, read_privacy_state())
+    if not all_messages:
+        debug_log(config, "All messages fell inside private ranges, skipping retain")
+        return {"status": "skipped", "reason": "private-mode"}
+
     debug_log(config, f"Read {len(all_messages)} messages from transcript")
 
     # Retention mode: full session (vendor default) or chunked. Switchroom
@@ -564,6 +861,14 @@ def run_retain(hook_input: dict, force: bool = False) -> dict:
         # chunk 0 → plain session_id (backwards compatible with existing docs)
         document_id = session_id if chunk_index == 0 else f"{session_id}-c{chunk_index}"
 
+    # Switchroom self-improve PR4 (slice 4a) — read-and-clear the per-turn
+    # operator-correction sentinel here, in the live Stop path only, AFTER the
+    # throttle / re-fire / empty-transcript early-returns above so only a retain
+    # that actually FIRES consumes the marker (a throttled turn leaves it for the
+    # next firing retain, whose overlapping window still contains the correction).
+    # Read-once: the tag rides exactly one retain. Best-effort; never raises.
+    extra_tags = read_and_clear_correction_pending()
+
     # Build the payload via the shared, network-free seam (§1.4). This carries
     # the deterministic id, connection info, formatted transcript, tags and
     # metadata — sufficient to retry from a different process (drain_pending).
@@ -577,6 +882,7 @@ def run_retain(hook_input: dict, force: bool = False) -> dict:
         api_token=api_token,
         retain_full_window=retain_full_window,
         document_id=document_id,
+        extra_tags=extra_tags,
     )
     if built is None:
         debug_log(config, "Empty transcript after formatting, skipping retain")

package/vendor/hindsight-memory/scripts/subagent_retain.py

@@ -73,7 +73,13 @@ from lib.pacing import inflight_lock
 # retain.py owns the transcript reader, the deterministic-id recipe and the
 # network-free payload builder; reuse them wholesale so the sidechain path
 # stays byte-identical to the main path where it matters (dedup ids, formatting).
-from retain import build_retain_payload, read_transcript
+from retain import (
+    _has_open_interval,
+    build_retain_payload,
+    exclude_private_ranges,
+    read_privacy_state,
+    read_transcript,
+)
 
 # Retain the last N human turns of the sidechain. The window is formatted on the
 # TEXT-ONLY path (``retainToolCalls`` is forced False for the sidechain — see
@@ -333,6 +339,15 @@ def run_subagent_retain(hook_input: dict) -> dict:
          "payload": {...},   # only when status == "failed" (for enqueue)
          "error":   Exception}  # only when status == "failed"
     """
+    # /private-mode enforcement (switchroom privacy PR1) — FIRST, before
+    # load_config(). Subagents have no toggle of their own; they honor the
+    # parent session's privacy state file (same env, same path). An OPEN private
+    # interval means confidential material is being discussed right now, so the
+    # sidechain retain must not fire — placed above load_config() so a
+    # HINDSIGHT_AUTO_RETAIN=true env pin cannot override the privacy guarantee.
+    if _has_open_interval(read_privacy_state()):
+        return {"status": "skipped", "reason": "private-mode"}
+
     config = load_config()
 
     if not config.get("autoRetain"):
@@ -379,6 +394,19 @@ def run_subagent_retain(hook_input: dict) -> dict:
         debug_log(config, f"SubagentStop: empty sidechain transcript {transcript_path}")
         return {"status": "skipped", "reason": "empty transcript"}
 
+    # /private-mode redaction (review MAJOR 2) — the open-interval early-skip
+    # above only catches a range that is STILL open at SubagentStop. But a
+    # backgrounded sub-agent can be dispatched under /private, carry that
+    # confidential context, and finish AFTER /public closed the interval — no
+    # interval open here, yet its window still holds private-window material.
+    # Apply the same range redaction the parent Stop path uses so a now-CLOSED
+    # private range is excluded from the sidechain retain too. Runs BEFORE the
+    # volume gate so the gate measures the redacted content.
+    all_messages = exclude_private_ranges(all_messages, read_privacy_state())
+    if not all_messages:
+        debug_log(config, "SubagentStop: all sidechain messages fell inside private ranges")
+        return {"status": "skipped", "reason": "private-mode"}
+
     # Volume gate — skip trivial forks, log the skip for coverage auditing.
     passed, turns, chars = passes_volume_gate(all_messages, config)
     if not passed: