switchroom 0.20.10 → 0.20.12

package/vendor/hindsight-memory/scripts/tests/test_recall_latency_instrumentation.py

@@ -0,0 +1,277 @@
+"""Switchroom recall-latency instrumentation tests.
+
+recall.py is the UserPromptSubmit hook in front of every reply, and until now
+its wall time was UNMEASURED: it is a direct plugin hook (not wrapped by
+bin/run-hook.sh, so no hook-timings row) and recall_log.jsonl carried no
+duration field. These tests lock in the two things this PR added:
+
+  1. Every recall_log row (cache hit AND cache miss) carries a numeric
+     `duration_ms` — including a degraded/failed recall.
+  2. The hook emits a `hook-timings-<Ddd>.log` line matching bin/run-hook.sh's
+     schema, into the same weekday ring, honouring the same env knobs.
+  3. The instrumentation NEVER pollutes the hook's stdout contract (the
+     recall-context JSON Claude Code consumes is byte-identical).
+
+Stdlib-only (unittest + mock). Reuses the integration harness's fakes.
+"""
+
+import io
+import json
+import os
+import sys
+import tempfile
+import time
+import unittest
+from unittest.mock import patch
+
+SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+if SCRIPTS_DIR not in sys.path:
+    sys.path.insert(0, SCRIPTS_DIR)
+TESTS_DIR = os.path.abspath(os.path.dirname(__file__))
+if TESTS_DIR not in sys.path:
+    sys.path.insert(0, TESTS_DIR)
+
+import recall  # noqa: E402
+from test_recall_integration import _FakeClient, _memory, _run_main_with  # noqa: E402
+
+
+class DurationMsInRecallLogTests(unittest.TestCase):
+    """recall_log.jsonl rows must carry a numeric `duration_ms`."""
+
+    def setUp(self):
+        self._tmpdir = tempfile.mkdtemp(prefix="recall-latency-test-")
+        self._prev = os.environ.get("CLAUDE_PLUGIN_DATA")
+        os.environ["CLAUDE_PLUGIN_DATA"] = self._tmpdir
+
+    def tearDown(self):
+        import shutil
+
+        shutil.rmtree(self._tmpdir, ignore_errors=True)
+        if self._prev is None:
+            os.environ.pop("CLAUDE_PLUGIN_DATA", None)
+        else:
+            os.environ["CLAUDE_PLUGIN_DATA"] = self._prev
+
+    def _read_log(self):
+        path = os.path.join(self._tmpdir, "state", "recall_log.jsonl")
+        if not os.path.isfile(path):
+            return []
+        with open(path, encoding="utf-8") as f:
+            return [json.loads(line) for line in f if line.strip()]
+
+    def test_cache_miss_row_has_numeric_duration_ms(self):
+        client = _FakeClient(directives=[], memories=[_memory("x", mem_id="x1")])
+        _run_main_with(client)
+        e = self._read_log()[0]
+        self.assertFalse(e["cache_hit"])
+        self.assertIn("duration_ms", e)
+        self.assertIsInstance(e["duration_ms"], int)
+        self.assertGreaterEqual(e["duration_ms"], 0)
+        # duration_ms (full hook) must be >= total_elapsed_ms (recall path only):
+        # it strictly contains it, so the derived local overhead is never negative.
+        self.assertGreaterEqual(e["duration_ms"], e["total_elapsed_ms"])
+
+    def test_degraded_recall_still_records_a_duration(self):
+        # The common case right now: the own bank is unreachable. The row must
+        # still carry a numeric duration so a slow/failed recall is attributable.
+        client = _FakeClient(
+            directives=[],
+            memories=[],
+            recall_exc=RuntimeError("HTTP 503 from http://localhost:18888"),
+        )
+        _run_main_with(client)
+        e = self._read_log()[0]
+        self.assertTrue(e["bank_errored"])
+        self.assertEqual(e["result_count"], 0)
+        self.assertIsInstance(e["duration_ms"], int)
+        self.assertGreaterEqual(e["duration_ms"], 0)
+
+    def _run_with_real_state(self, client, prompt):
+        """Invoke recall.main WITHOUT stubbing write_state/read_state, so the
+        per-session recall cache actually persists to CLAUDE_PLUGIN_DATA and a
+        second identical prompt takes the cache-hit path. (The shared
+        _run_main_with harness stubs write_state, which disables the cache.)"""
+        hook_input = {
+            "prompt": prompt,
+            "session_id": "cache-session",
+            "transcript_path": "",
+            "cwd": "/tmp",
+        }
+        config = {
+            "autoRecall": True,
+            "bankId": "test-bank",
+            "recallMaxTokens": 1024,
+            "recallBudget": "mid",
+            "recallContextTurns": 1,
+            "recallMaxQueryChars": 800,
+            "recallPromptPreamble": "",
+            "directivesCacheTtlSeconds": 0,
+        }
+        with patch.object(recall, "load_config", return_value=config), patch.object(
+            recall, "get_api_url", return_value="http://localhost:18888"
+        ), patch.object(recall, "HindsightClient", return_value=client), patch.object(
+            recall, "ensure_bank_mission", return_value=None
+        ), patch("sys.stdin", new=io.StringIO(json.dumps(hook_input))), patch(
+            "sys.stdout", new=io.StringIO()
+        ), patch("sys.stderr", new=io.StringIO()):
+            recall.main()
+
+    def test_cache_hit_row_has_numeric_duration_ms(self):
+        # Populate the cache, then hit it on a second identical prompt. The
+        # cache-hit row must also carry duration_ms (total_elapsed_ms is None
+        # there, so duration_ms is the only latency number on a cache hit).
+        client = _FakeClient(directives=[], memories=[_memory("x", mem_id="x1")])
+        with patch.dict(os.environ, {"HINDSIGHT_RECALL_CACHE_TTL_SECS": "60"}):
+            self._run_with_real_state(client, "What did we decide about auth?")
+            self._run_with_real_state(client, "What did we decide about auth?")
+        rows = self._read_log()
+        hit_rows = [r for r in rows if r["cache_hit"]]
+        self.assertTrue(hit_rows, "expected at least one cache-hit row")
+        e = hit_rows[0]
+        self.assertIsNone(e["total_elapsed_ms"])
+        self.assertIsInstance(e["duration_ms"], int)
+        self.assertGreaterEqual(e["duration_ms"], 0)
+
+
+class StdoutUnchangedTests(unittest.TestCase):
+    """The instrumentation writes only to log files — the hook's stdout
+    (the recall-context JSON Claude Code consumes) must be byte-identical."""
+
+    def test_stdout_is_exact_and_carries_no_instrumentation(self):
+        client = _FakeClient(directives=[], memories=[_memory("a stored fact")])
+        # Pin the clock-derived preamble so the emitted block is deterministic,
+        # letting us assert the stdout bytes EXACTLY.
+        with patch.object(recall, "format_current_time", return_value="FIXED-TIME"):
+            ctx, raw = _run_main_with(client)
+
+        expected_context = (
+            "<hindsight_memories>\n"
+            "\n"  # empty recallPromptPreamble
+            "Current time - FIXED-TIME\n\n"
+            + recall.format_memories([_memory("a stored fact")])
+            + "\n</hindsight_memories>"
+        )
+        expected_raw = json.dumps(
+            {
+                "hookSpecificOutput": {
+                    "hookEventName": "UserPromptSubmit",
+                    "additionalContext": expected_context,
+                }
+            }
+        )
+        self.assertEqual(raw, expected_raw)
+        # Belt-and-braces: no instrumentation artifact leaked into the contract.
+        self.assertNotIn("duration_ms", raw)
+        self.assertNotIn("hook-timings", raw)
+        self.assertNotIn("total_elapsed_ms", raw)
+
+    def test_empty_recall_emits_empty_stdout(self):
+        # A no-directive, no-memory turn still emits nothing on stdout — the
+        # timing line lands in a log file, never here.
+        client = _FakeClient(directives=[], memories=[])
+        _ctx, raw = _run_main_with(client)
+        self.assertEqual(raw.strip(), "")
+
+
+class HookTimingLogEmissionTests(unittest.TestCase):
+    """_emit_hook_timing_log must reproduce bin/run-hook.sh's line schema,
+    weekday ring, and env-knob behaviour — the hook-timings row that lets this
+    UserPromptSubmit hook show up alongside every wrapped hook."""
+
+    def setUp(self):
+        self._tmpdir = tempfile.mkdtemp(prefix="hook-timings-test-")
+
+    def tearDown(self):
+        import shutil
+
+        shutil.rmtree(self._tmpdir, ignore_errors=True)
+
+    def _logfile(self):
+        dow = time.strftime("%a", time.localtime())
+        return os.path.join(self._tmpdir, f"hook-timings-{dow}.log")
+
+    def test_emits_line_with_run_hook_schema(self):
+        with patch.dict(os.environ, {"SWITCHROOM_HOOK_TIMING_DIR": self._tmpdir}):
+            recall._emit_hook_timing_log(142, 0)
+        path = self._logfile()
+        self.assertTrue(os.path.isfile(path))
+        with open(path, encoding="utf-8") as f:
+            lines = f.read().splitlines()
+        self.assertEqual(len(lines), 1)
+        row = json.loads(lines[0])
+        # Exact key set + types run-hook.sh writes.
+        self.assertEqual(
+            set(row.keys()), {"ts", "date", "source", "code", "duration_ms", "status"}
+        )
+        self.assertEqual(row["source"], "hook:hindsight-recall")
+        self.assertEqual(row["code"], "recall.py")
+        self.assertEqual(row["duration_ms"], 142)
+        self.assertEqual(row["status"], 0)
+        self.assertEqual(row["date"], time.strftime("%Y-%m-%d", time.localtime()))
+
+    def test_falls_back_to_telegram_state_dir(self):
+        with patch.dict(
+            os.environ,
+            {"TELEGRAM_STATE_DIR": self._tmpdir},
+            clear=False,
+        ), patch.dict(os.environ, {}, clear=False):
+            os.environ.pop("SWITCHROOM_HOOK_TIMING_DIR", None)
+            recall._emit_hook_timing_log(5, 0)
+        self.assertTrue(os.path.isfile(self._logfile()))
+
+    def test_disabled_by_env(self):
+        with patch.dict(
+            os.environ,
+            {
+                "SWITCHROOM_HOOK_TIMING": "0",
+                "SWITCHROOM_HOOK_TIMING_DIR": self._tmpdir,
+            },
+        ):
+            recall._emit_hook_timing_log(999, 0)
+        self.assertFalse(os.path.isfile(self._logfile()))
+
+    def test_min_ms_filters_fast_invocations(self):
+        with patch.dict(
+            os.environ,
+            {
+                "SWITCHROOM_HOOK_TIMING_DIR": self._tmpdir,
+                "SWITCHROOM_HOOK_TIMING_MIN_MS": "100",
+            },
+        ):
+            recall._emit_hook_timing_log(50, 0)  # below floor → dropped
+            recall._emit_hook_timing_log(150, 0)  # at/above → kept
+        with open(self._logfile(), encoding="utf-8") as f:
+            rows = [json.loads(x) for x in f if x.strip()]
+        self.assertEqual([r["duration_ms"] for r in rows], [150])
+
+    def test_no_dir_is_silent_noop(self):
+        # No timing dir and no TELEGRAM_STATE_DIR → nothing written, no raise.
+        with patch.dict(os.environ, {}, clear=False):
+            os.environ.pop("SWITCHROOM_HOOK_TIMING_DIR", None)
+            os.environ.pop("TELEGRAM_STATE_DIR", None)
+            recall._emit_hook_timing_log(10, 0)  # must not raise
+        self.assertFalse(os.path.isfile(self._logfile()))
+
+    def test_stale_weekday_file_is_reset(self):
+        path = self._logfile()
+        # Seed the ring file with a line dated a week ago (different date, same
+        # weekday) — the emitter must truncate it before appending today's row.
+        stale = (
+            '{"ts":"2000-01-01T00:00:00+0000","date":"2000-01-01",'
+            '"source":"hook:hindsight-recall","code":"recall.py",'
+            '"duration_ms":1,"status":0}\n'
+        )
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(stale)
+        with patch.dict(os.environ, {"SWITCHROOM_HOOK_TIMING_DIR": self._tmpdir}):
+            recall._emit_hook_timing_log(7, 0)
+        with open(path, encoding="utf-8") as f:
+            rows = [json.loads(x) for x in f if x.strip()]
+        # Stale line gone; exactly today's row remains.
+        self.assertEqual(len(rows), 1)
+        self.assertEqual(rows[0]["duration_ms"], 7)
+        self.assertEqual(rows[0]["date"], time.strftime("%Y-%m-%d", time.localtime()))
+
+
+if __name__ == "__main__":
+    unittest.main()

package/vendor/hindsight-memory/scripts/tests/test_self_improve_correction_tag.py

@@ -0,0 +1,167 @@
+"""Per-turn operator-correction tag on auto-retained turns (switchroom PR4 4a).
+
+When switchroom's self-improve gate (``src/self-improve/gate.ts``) fires on an
+``operator-correction`` signal, the Stop hook drops a per-turn sentinel into the
+shared agent state dir. This hook — a SEPARATE process — reads-and-clears that
+sentinel and stamps ``self-improve:correction`` on the turn's retain, so PR5's
+failure-synthesis cron can recall correction turns cheaply by tag filter.
+
+This module asserts the OUTCOMES at the retain seam:
+
+  1. read-and-clear is read-ONCE: a present sentinel yields the tag and is then
+     gone; a second read yields nothing.
+  2. the tag reaches the wire payload when (and ONLY when) it is threaded in —
+     a mutation that stamped it unconditionally, or never, fails here.
+  3. the tag is STABLE and therefore DOES move the consolidation scope to its
+     own ``[["self-improve:correction"]]`` partition — the opposite of the
+     forced-volatile ``source:transcript`` tag — and it matches NO volatile
+     scope pattern (in particular not ``^source:``). Absent tag ⇒ scope stays
+     the byte-identical ``"shared"``.
+
+Stdlib-only; runs under ``python3 -m unittest discover tests/``.
+"""
+
+import os
+import sys
+import tempfile
+import unittest
+
+SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+if SCRIPTS_DIR not in sys.path:
+    sys.path.insert(0, SCRIPTS_DIR)
+
+from lib.config import (  # noqa: E402
+    DEFAULTS,
+    DEFAULT_VOLATILE_SCOPE_PATTERNS,
+    compute_observation_scopes,
+)
+from retain import (  # noqa: E402
+    SELF_IMPROVE_CORRECTION_PENDING_FILE,
+    SELF_IMPROVE_CORRECTION_TAG,
+    build_retain_payload,
+    read_and_clear_correction_pending,
+)
+
+PROVENANCE_TAG = "source:transcript"
+SESSION_ID = "4c386b32-ddfd-40d1-b557-da8135b294af"
+
+
+def _cfg(**over):
+    c = {
+        "retainRoles": ["user", "assistant"],
+        "retainToolCalls": True,
+        "retainContext": "claude-code",
+        "retainMetadata": {},
+        "retainTags": ["{session_id}", PROVENANCE_TAG],
+        "lessonTagging": DEFAULTS["lessonTagging"],
+        "lessonTagMarkers": DEFAULTS["lessonTagMarkers"],
+        "observationScopeStrategy": "curated",
+    }
+    c.update(over)
+    return c
+
+
+def _build(extra_tags=None, **cfg_over):
+    msgs = [
+        {"role": "user", "content": "why did you include drafts?", "uuid": "u1"},
+        {"role": "assistant", "content": "Fixed the digest filter.", "uuid": "a1"},
+    ]
+    built = build_retain_payload(
+        _cfg(**cfg_over),
+        SESSION_ID,
+        msgs,
+        msgs,
+        bank_id="bank",
+        api_url="http://x",
+        api_token=None,
+        extra_tags=extra_tags,
+    )
+    assert built is not None
+    return built["payload"]
+
+
+class ReadAndClearIsReadOnce(unittest.TestCase):
+    def test_present_sentinel_yields_the_tag_and_is_cleared(self):
+        with tempfile.TemporaryDirectory() as d:
+            path = os.path.join(d, SELF_IMPROVE_CORRECTION_PENDING_FILE)
+            with open(path, "w") as f:
+                f.write("2026-08-06T00:00:00Z")
+            self.assertTrue(os.path.exists(path))
+
+            first = read_and_clear_correction_pending(d)
+            self.assertEqual(first, [SELF_IMPROVE_CORRECTION_TAG])
+            # Cleared as a side effect — read-once.
+            self.assertFalse(os.path.exists(path))
+
+            # A second read finds nothing: exactly one retain carries the tag.
+            second = read_and_clear_correction_pending(d)
+            self.assertEqual(second, [])
+
+    def test_absent_sentinel_yields_no_tag(self):
+        with tempfile.TemporaryDirectory() as d:
+            self.assertEqual(read_and_clear_correction_pending(d), [])
+
+    def test_never_raises_on_a_bad_state_dir(self):
+        # A nonexistent dir must degrade to "no tag", never raise.
+        self.assertEqual(
+            read_and_clear_correction_pending("/nonexistent/does/not/exist"),
+            [],
+        )
+
+
+class CorrectionTagReachesTheWire(unittest.TestCase):
+    def test_tag_on_payload_when_threaded_in(self):
+        tags = _build(extra_tags=[SELF_IMPROVE_CORRECTION_TAG])["tags"]
+        self.assertIn(SELF_IMPROVE_CORRECTION_TAG, tags)
+        # It composes — it does not clobber the session / provenance tags.
+        self.assertIn(SESSION_ID, tags)
+        self.assertIn(PROVENANCE_TAG, tags)
+
+    def test_tag_absent_when_not_threaded_in(self):
+        # Mutation guard: the tag is DATA carried by extra_tags, not unconditional
+        # behaviour. A normal turn carries no correction tag.
+        for extra in (None, []):
+            tags = _build(extra_tags=extra)["tags"]
+            self.assertNotIn(SELF_IMPROVE_CORRECTION_TAG, tags)
+
+    def test_no_duplicate_when_already_present(self):
+        tags = _build(
+            extra_tags=[SELF_IMPROVE_CORRECTION_TAG, SELF_IMPROVE_CORRECTION_TAG]
+        )["tags"]
+        self.assertEqual(tags.count(SELF_IMPROVE_CORRECTION_TAG), 1)
+
+
+class CorrectionTagScopeContract(unittest.TestCase):
+    """The tag is STABLE — it SHOULD partition scope, and matches no volatile pat."""
+
+    def test_tag_matches_no_volatile_scope_pattern(self):
+        import re
+
+        for pat in DEFAULT_VOLATILE_SCOPE_PATTERNS:
+            self.assertIsNone(
+                re.search(pat, SELF_IMPROVE_CORRECTION_TAG),
+                f"correction tag must be STABLE but matched volatile pattern {pat!r}",
+            )
+
+    def test_correction_turn_lands_in_its_own_scope(self):
+        scope = _build(extra_tags=[SELF_IMPROVE_CORRECTION_TAG])["observation_scopes"]
+        self.assertEqual(scope, [[SELF_IMPROVE_CORRECTION_TAG]])
+
+    def test_normal_turn_scope_stays_shared_and_byte_identical(self):
+        with_none = _build(extra_tags=None)["observation_scopes"]
+        with_empty = _build(extra_tags=[])["observation_scopes"]
+        self.assertEqual(with_none, "shared")
+        self.assertEqual(with_empty, "shared")
+
+    def test_compute_scope_directly_partitions_on_the_stable_tag(self):
+        # Guards the guard: prove the [[tag]] result is caused by the tag being
+        # stable, straight through compute_observation_scopes.
+        scope, err = compute_observation_scopes(
+            [SESSION_ID, PROVENANCE_TAG, SELF_IMPROVE_CORRECTION_TAG], _cfg()
+        )
+        self.assertIsNone(err)
+        self.assertEqual(scope, [[SELF_IMPROVE_CORRECTION_TAG]])
+
+
+if __name__ == "__main__":  # pragma: no cover
+    unittest.main()